Merge P2 fix: extraction-service wiring

docker-compose.dev.yml

@@ -55,6 +55,8 @@ services:
       - CC_COMPOSE_SERVICE=backend-dev
       - RUN_INIT=false
       - INIT_MODE=infra
+      # P2: route exam auto-map through the spike's full recognition pipeline (extraction service)
+      - EXAM_EXTRACT_URL=${EXAM_EXTRACT_URL:-http://192.168.0.203:8899}
     ports:
       - "18000:8000"
     depends_on:

routers/exam/templates.py

@@ -15,6 +15,7 @@ from __future__ import annotations
 import json
 import math
 import os
+import re
 import tempfile
 import time
 import uuid
@@ -30,6 +31,7 @@ from modules.database.services.exam_projection import project_template, project_
 from modules.database.supabase.utils.client import SupabaseServiceRoleClient
 from modules.database.supabase.utils.storage import StorageAdmin
 from modules.upload_validation import read_pdf_upload_bytes
+from modules.services import exam_extract
 from modules.logger_tool import initialise_logger
 from routers.exam.dependencies import ExamContext, get_exam_context, lookup_exam_code
 from routers.exam.schemas import (
@@ -455,6 +457,32 @@ def _y_to_canvas(y_value: float, page_number: int, pages: List[Dict[str, float]]
     return round(g["page_top"] + (g["page_pt_h"] - (float(y_value) - g["crop_y0"])) / g["page_pt_h"] * g["rendered_h"], 2)
 
 
+def _frac_box_to_canvas(bounds: Optional[Dict[str, Any]], page_number: int,
+                        pages: List[Dict[str, float]]) -> Optional[Dict[str, float]]:
+    """Page-fraction {x,y,w,h} (0..1 per page, from the extraction service) → 780-wide stacked canvas."""
+    if not bounds:
+        return None
+    g = _page_geom(pages, page_number)
+    try:
+        x, y, w, h = (float(bounds["x"]), float(bounds["y"]), float(bounds["w"]), float(bounds["h"]))
+    except (KeyError, TypeError, ValueError):
+        return None
+    return {
+        "x": round(x * g["rendered_w"], 2),
+        "y": round(g["page_top"] + y * g["rendered_h"], 2),
+        "w": round(w * g["rendered_w"], 2),
+        "h": round(h * g["rendered_h"], 2),
+    }
+
+
+def _frac_y_to_canvas(y_frac: Any, page_number: int, pages: List[Dict[str, float]]) -> Optional[float]:
+    g = _page_geom(pages, page_number)
+    try:
+        return round(g["page_top"] + float(y_frac) * g["rendered_h"], 2)
+    except (TypeError, ValueError):
+        return None
+
+
 def _ai_id(template_id: str, *parts: Any) -> str:
     return str(uuid.uuid5(uuid.NAMESPACE_URL, "/".join(["cc-auto-map", template_id, *[str(p) for p in parts]])))
 
@@ -660,6 +688,129 @@ def _run_auto_map_job(job_id: str, ctx: ExamContext, template_id: str, pdf_bytes
         _set_auto_map_status(job_id, {"status": "failed", "template_id": template_id, "error": str(exc)})
 
 
+_ALLOWED_RESPONSE_FORMS = {"lines", "answer-box", "working", "diagram", "tick-boxes", "table", "blanks"}
+_ALLOWED_ANSWER_TYPES = {"written", "mcq", "short", "diagram"}
+_ALLOWED_KINDS = {"response", "context", "question_number", "mark_area", "reference", "furniture"}
+_BOARD_RE = re.compile(r"^(aqa|edexcel|ocr|wjec|eduqas|ccea)-")
+
+
+def _extract_slug(ctx: ExamContext, template: Dict[str, Any]) -> str:
+    """A stable, board-prefixed slug for the extraction-service cache. Prefer the catalogue exam_code
+    (e.g. 'AQA-8463-1H-2022JUN-QP' → 'aqa-8463-1h-2022jun-qp' → board 'aqa' for the right margins);
+    fall back to a template-id slug (structure.py then defaults to AQA content-box margins)."""
+    code = None
+    exam_id = template.get("exam_id")
+    if exam_id:
+        try:
+            row = _first(ctx.supabase.table("eb_exams").select("exam_code").eq("id", exam_id).limit(1).execute())
+            code = (row or {}).get("exam_code")
+        except Exception as exc:
+            logger.info(f"extract slug: eb_exams lookup failed for {exam_id}: {exc}")
+    slug = re.sub(r"[^a-z0-9._-]+", "-", (code or "").lower()).strip("-")
+    if _BOARD_RE.match(slug):
+        return slug
+    return f"aqa-tmpl-{str(template.get('id') or '')[:12]}"
+
+
+def _map_service_contract_to_rows(template_id: str, contract: Dict[str, Any],
+                                  pdf_bytes: bytes) -> Dict[str, List[Dict[str, Any]]]:
+    """Map the extraction service's page-fraction analyse contract onto the app's canvas-space ghost rows.
+
+    Coordinates: page-fraction → the 780-wide stacked canvas. IDs: the service's deterministic uuid5s are
+    re-namespaced per template via _ai_id so two templates of the same paper don't collide and a re-map of
+    the same template re-emits stable ids (so _refresh_ai_rows preserves confirmed ghosts). FK-safe: parts
+    whose parent/owner question is absent are de-parented / dropped rather than crashing the insert.
+    """
+    pages = _pdf_page_geometry(pdf_bytes)
+    sug = contract.get("suggestions") or {}
+
+    def qid(uid: Any) -> str:
+        return _ai_id(template_id, "svc-q", uid)
+
+    questions: List[Dict[str, Any]] = []
+    q_ids: set = set()
+    for q in sug.get("questions") or []:
+        uid = q.get("uid")
+        if not uid:
+            continue
+        rid = qid(uid)
+        q_ids.add(rid)
+        at = q.get("answer_type") if q.get("answer_type") in _ALLOWED_ANSWER_TYPES else None
+        questions.append({
+            "id": rid, "template_id": template_id,
+            "parent_id": qid(q["parent_uid"]) if q.get("parent_uid") else None,
+            "label": q.get("label") or "?", "order": q.get("order", len(questions)),
+            "max_marks": _safe_marks(q.get("max_marks")), "answer_type": at,
+            "is_container": bool(q.get("is_container")),
+            "bounds": _frac_box_to_canvas(q.get("bounds"), q.get("page") or 1, pages),
+            "page": q.get("page"), "source": "ai", "confirmed": False,
+            "confidence": _safe_confidence(q.get("confidence")), "derivation": "extract-service",
+        })
+    for q in questions:                                 # FK safety: de-parent a dangling parent_id
+        if q["parent_id"] and q["parent_id"] not in q_ids:
+            q["parent_id"] = None
+
+    response_areas: List[Dict[str, Any]] = []
+    for ra in sug.get("response_areas") or []:
+        uid = ra.get("uid")
+        quid = qid(ra.get("question_uid") or "")
+        if not uid or quid not in q_ids:               # orphan region → drop (FK safety)
+            continue
+        bounds = _frac_box_to_canvas(ra.get("bounds"), ra.get("page") or 1, pages)
+        if not bounds:
+            continue
+        kind = ra.get("kind") if ra.get("kind") in _ALLOWED_KINDS else "response"
+        form = ra.get("response_form") if ra.get("response_form") in _ALLOWED_RESPONSE_FORMS else None
+        response_areas.append({
+            "id": _ai_id(template_id, "svc-ra", uid), "template_id": template_id,
+            "question_id": quid, "page": ra.get("page"), "bounds": bounds,
+            "kind": kind, "response_form": form if kind == "response" else None,
+            "context_type": ra.get("context_type"), "meta": ra.get("meta") or {},
+            "source": "ai", "confirmed": False,
+            "confidence": _safe_confidence(ra.get("confidence")), "derivation": "extract-service",
+        })
+
+    boundaries: List[Dict[str, Any]] = []
+    for i, b in enumerate(sug.get("boundaries") or []):
+        quid = qid(b.get("question_uid") or "")
+        if quid not in q_ids:
+            continue
+        page_index = b.get("page_index")
+        y = _frac_y_to_canvas(b.get("y"), (page_index or 0) + 1, pages)
+        if y is None:
+            continue
+        boundaries.append({
+            "id": _ai_id(template_id, "svc-b", b.get("question_uid") or i), "template_id": template_id,
+            "question_id": quid, "label": b.get("label") or "", "page_index": page_index,
+            "y": y, "bounds": None, "source": "ai", "confirmed": False,
+            "confidence": _safe_confidence(b.get("confidence")), "derivation": "extract-service",
+        })
+
+    return {"questions": questions, "response_areas": response_areas, "boundaries": boundaries}
+
+
+def _run_service_extract_merge(ctx: ExamContext, template_id: str, pdf_bytes: bytes, slug: str) -> Dict[str, List[Dict[str, Any]]]:
+    contract = exam_extract.extract_suggestions(slug, pdf_bytes)
+    rows = _map_service_contract_to_rows(template_id, contract, pdf_bytes)
+    _refresh_ai_rows(ctx, template_id, rows)
+    n_pages = (contract.get("meta") or {}).get("n_pages") or (contract.get("meta") or {}).get("pages")
+    if n_pages:
+        ctx.supabase.table("exam_templates").update({"page_count": n_pages}).eq("id", template_id).execute()
+    return rows
+
+
+def _run_service_extract_job(job_id: str, ctx: ExamContext, template_id: str, pdf_bytes: bytes, slug: str) -> None:
+    _set_auto_map_status(job_id, {"status": "running", "template_id": template_id, "engine": "extract-service", "slug": slug})
+    try:
+        rows = _run_service_extract_merge(ctx, template_id, pdf_bytes, slug)
+        project_template_safe(template_id)
+        _set_auto_map_status(job_id, {"status": "completed", "template_id": template_id,
+                                      "engine": "extract-service", "counts": {k: len(v) for k, v in rows.items()}})
+    except Exception as exc:
+        logger.exception(f"extract-service job failed for template {template_id}: {exc}")
+        _set_auto_map_status(job_id, {"status": "failed", "template_id": template_id, "engine": "extract-service", "error": str(exc)})
+
+
 # ─── templates ───────────────────────────────────────────────────────────────
 
 
@@ -837,6 +988,14 @@ async def auto_map_template(
         raise HTTPException(status_code=409, detail="Template has recorded marks; auto-map structural refresh is blocked.")
     bucket, path, pdf_bytes = _resolve_template_source(ctx, template)
     source_label = f"{bucket}/{path}"
+    # Extraction-service path (P2): when EXAM_EXTRACT_URL is set, route auto-map through the spike's full
+    # recognition pipeline instead of the thin first-pass. Always async — a cold paper is ~15 min.
+    if exam_extract.is_enabled():
+        slug = _extract_slug(ctx, template)
+        job_id = str(uuid.uuid4())
+        _set_auto_map_status(job_id, {"status": "queued", "template_id": template_id, "engine": "extract-service", "slug": slug})
+        background_tasks.add_task(_run_service_extract_job, job_id, ctx, template_id, pdf_bytes, slug)
+        return JSONResponse(status_code=202, content={"status": "accepted", "job_id": job_id, "engine": "extract-service"})
     try:
         fast_path = _pdf_has_text_layer(pdf_bytes)
     except Exception as exc: