diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index d8f4448..22b5749 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -55,6 +55,8 @@ services: - CC_COMPOSE_SERVICE=backend-dev - RUN_INIT=false - INIT_MODE=infra + # P2: route exam auto-map through the spike's full recognition pipeline (extraction service) + - EXAM_EXTRACT_URL=${EXAM_EXTRACT_URL:-http://192.168.0.203:8899} ports: - "18000:8000" depends_on: diff --git a/routers/exam/templates.py b/routers/exam/templates.py index 6c4fe8e..40d3394 100644 --- a/routers/exam/templates.py +++ b/routers/exam/templates.py @@ -15,6 +15,7 @@ from __future__ import annotations import json import math import os +import re import tempfile import time import uuid @@ -30,6 +31,7 @@ from modules.database.services.exam_projection import project_template, project_ from modules.database.supabase.utils.client import SupabaseServiceRoleClient from modules.database.supabase.utils.storage import StorageAdmin from modules.upload_validation import read_pdf_upload_bytes +from modules.services import exam_extract from modules.logger_tool import initialise_logger from routers.exam.dependencies import ExamContext, get_exam_context, lookup_exam_code from routers.exam.schemas import ( @@ -455,6 +457,32 @@ def _y_to_canvas(y_value: float, page_number: int, pages: List[Dict[str, float]] return round(g["page_top"] + (g["page_pt_h"] - (float(y_value) - g["crop_y0"])) / g["page_pt_h"] * g["rendered_h"], 2) +def _frac_box_to_canvas(bounds: Optional[Dict[str, Any]], page_number: int, + pages: List[Dict[str, float]]) -> Optional[Dict[str, float]]: + """Page-fraction {x,y,w,h} (0..1 per page, from the extraction service) → 780-wide stacked canvas.""" + if not bounds: + return None + g = _page_geom(pages, page_number) + try: + x, y, w, h = (float(bounds["x"]), float(bounds["y"]), float(bounds["w"]), float(bounds["h"])) + except (KeyError, TypeError, ValueError): + return None + return { + "x": round(x * g["rendered_w"], 2), + "y": round(g["page_top"] + y * g["rendered_h"], 2), + "w": round(w * g["rendered_w"], 2), + "h": round(h * g["rendered_h"], 2), + } + + +def _frac_y_to_canvas(y_frac: Any, page_number: int, pages: List[Dict[str, float]]) -> Optional[float]: + g = _page_geom(pages, page_number) + try: + return round(g["page_top"] + float(y_frac) * g["rendered_h"], 2) + except (TypeError, ValueError): + return None + + def _ai_id(template_id: str, *parts: Any) -> str: return str(uuid.uuid5(uuid.NAMESPACE_URL, "/".join(["cc-auto-map", template_id, *[str(p) for p in parts]]))) @@ -660,6 +688,129 @@ def _run_auto_map_job(job_id: str, ctx: ExamContext, template_id: str, pdf_bytes _set_auto_map_status(job_id, {"status": "failed", "template_id": template_id, "error": str(exc)}) +_ALLOWED_RESPONSE_FORMS = {"lines", "answer-box", "working", "diagram", "tick-boxes", "table", "blanks"} +_ALLOWED_ANSWER_TYPES = {"written", "mcq", "short", "diagram"} +_ALLOWED_KINDS = {"response", "context", "question_number", "mark_area", "reference", "furniture"} +_BOARD_RE = re.compile(r"^(aqa|edexcel|ocr|wjec|eduqas|ccea)-") + + +def _extract_slug(ctx: ExamContext, template: Dict[str, Any]) -> str: + """A stable, board-prefixed slug for the extraction-service cache. Prefer the catalogue exam_code + (e.g. 'AQA-8463-1H-2022JUN-QP' → 'aqa-8463-1h-2022jun-qp' → board 'aqa' for the right margins); + fall back to a template-id slug (structure.py then defaults to AQA content-box margins).""" + code = None + exam_id = template.get("exam_id") + if exam_id: + try: + row = _first(ctx.supabase.table("eb_exams").select("exam_code").eq("id", exam_id).limit(1).execute()) + code = (row or {}).get("exam_code") + except Exception as exc: + logger.info(f"extract slug: eb_exams lookup failed for {exam_id}: {exc}") + slug = re.sub(r"[^a-z0-9._-]+", "-", (code or "").lower()).strip("-") + if _BOARD_RE.match(slug): + return slug + return f"aqa-tmpl-{str(template.get('id') or '')[:12]}" + + +def _map_service_contract_to_rows(template_id: str, contract: Dict[str, Any], + pdf_bytes: bytes) -> Dict[str, List[Dict[str, Any]]]: + """Map the extraction service's page-fraction analyse contract onto the app's canvas-space ghost rows. + + Coordinates: page-fraction → the 780-wide stacked canvas. IDs: the service's deterministic uuid5s are + re-namespaced per template via _ai_id so two templates of the same paper don't collide and a re-map of + the same template re-emits stable ids (so _refresh_ai_rows preserves confirmed ghosts). FK-safe: parts + whose parent/owner question is absent are de-parented / dropped rather than crashing the insert. + """ + pages = _pdf_page_geometry(pdf_bytes) + sug = contract.get("suggestions") or {} + + def qid(uid: Any) -> str: + return _ai_id(template_id, "svc-q", uid) + + questions: List[Dict[str, Any]] = [] + q_ids: set = set() + for q in sug.get("questions") or []: + uid = q.get("uid") + if not uid: + continue + rid = qid(uid) + q_ids.add(rid) + at = q.get("answer_type") if q.get("answer_type") in _ALLOWED_ANSWER_TYPES else None + questions.append({ + "id": rid, "template_id": template_id, + "parent_id": qid(q["parent_uid"]) if q.get("parent_uid") else None, + "label": q.get("label") or "?", "order": q.get("order", len(questions)), + "max_marks": _safe_marks(q.get("max_marks")), "answer_type": at, + "is_container": bool(q.get("is_container")), + "bounds": _frac_box_to_canvas(q.get("bounds"), q.get("page") or 1, pages), + "page": q.get("page"), "source": "ai", "confirmed": False, + "confidence": _safe_confidence(q.get("confidence")), "derivation": "extract-service", + }) + for q in questions: # FK safety: de-parent a dangling parent_id + if q["parent_id"] and q["parent_id"] not in q_ids: + q["parent_id"] = None + + response_areas: List[Dict[str, Any]] = [] + for ra in sug.get("response_areas") or []: + uid = ra.get("uid") + quid = qid(ra.get("question_uid") or "") + if not uid or quid not in q_ids: # orphan region → drop (FK safety) + continue + bounds = _frac_box_to_canvas(ra.get("bounds"), ra.get("page") or 1, pages) + if not bounds: + continue + kind = ra.get("kind") if ra.get("kind") in _ALLOWED_KINDS else "response" + form = ra.get("response_form") if ra.get("response_form") in _ALLOWED_RESPONSE_FORMS else None + response_areas.append({ + "id": _ai_id(template_id, "svc-ra", uid), "template_id": template_id, + "question_id": quid, "page": ra.get("page"), "bounds": bounds, + "kind": kind, "response_form": form if kind == "response" else None, + "context_type": ra.get("context_type"), "meta": ra.get("meta") or {}, + "source": "ai", "confirmed": False, + "confidence": _safe_confidence(ra.get("confidence")), "derivation": "extract-service", + }) + + boundaries: List[Dict[str, Any]] = [] + for i, b in enumerate(sug.get("boundaries") or []): + quid = qid(b.get("question_uid") or "") + if quid not in q_ids: + continue + page_index = b.get("page_index") + y = _frac_y_to_canvas(b.get("y"), (page_index or 0) + 1, pages) + if y is None: + continue + boundaries.append({ + "id": _ai_id(template_id, "svc-b", b.get("question_uid") or i), "template_id": template_id, + "question_id": quid, "label": b.get("label") or "", "page_index": page_index, + "y": y, "bounds": None, "source": "ai", "confirmed": False, + "confidence": _safe_confidence(b.get("confidence")), "derivation": "extract-service", + }) + + return {"questions": questions, "response_areas": response_areas, "boundaries": boundaries} + + +def _run_service_extract_merge(ctx: ExamContext, template_id: str, pdf_bytes: bytes, slug: str) -> Dict[str, List[Dict[str, Any]]]: + contract = exam_extract.extract_suggestions(slug, pdf_bytes) + rows = _map_service_contract_to_rows(template_id, contract, pdf_bytes) + _refresh_ai_rows(ctx, template_id, rows) + n_pages = (contract.get("meta") or {}).get("n_pages") or (contract.get("meta") or {}).get("pages") + if n_pages: + ctx.supabase.table("exam_templates").update({"page_count": n_pages}).eq("id", template_id).execute() + return rows + + +def _run_service_extract_job(job_id: str, ctx: ExamContext, template_id: str, pdf_bytes: bytes, slug: str) -> None: + _set_auto_map_status(job_id, {"status": "running", "template_id": template_id, "engine": "extract-service", "slug": slug}) + try: + rows = _run_service_extract_merge(ctx, template_id, pdf_bytes, slug) + project_template_safe(template_id) + _set_auto_map_status(job_id, {"status": "completed", "template_id": template_id, + "engine": "extract-service", "counts": {k: len(v) for k, v in rows.items()}}) + except Exception as exc: + logger.exception(f"extract-service job failed for template {template_id}: {exc}") + _set_auto_map_status(job_id, {"status": "failed", "template_id": template_id, "engine": "extract-service", "error": str(exc)}) + + # ─── templates ─────────────────────────────────────────────────────────────── @@ -837,6 +988,14 @@ async def auto_map_template( raise HTTPException(status_code=409, detail="Template has recorded marks; auto-map structural refresh is blocked.") bucket, path, pdf_bytes = _resolve_template_source(ctx, template) source_label = f"{bucket}/{path}" + # Extraction-service path (P2): when EXAM_EXTRACT_URL is set, route auto-map through the spike's full + # recognition pipeline instead of the thin first-pass. Always async — a cold paper is ~15 min. + if exam_extract.is_enabled(): + slug = _extract_slug(ctx, template) + job_id = str(uuid.uuid4()) + _set_auto_map_status(job_id, {"status": "queued", "template_id": template_id, "engine": "extract-service", "slug": slug}) + background_tasks.add_task(_run_service_extract_job, job_id, ctx, template_id, pdf_bytes, slug) + return JSONResponse(status_code=202, content={"status": "accepted", "job_id": job_id, "engine": "extract-service"}) try: fast_path = _pdf_has_text_layer(pdf_bytes) except Exception as exc: