FX-6: seed SpecPoint catalogue for all 6 test specs (ASSESSES beyond AQA physics)

Only AQA GCSE Physics 8463 (4.1-4.8) was seeded, so any other board/spec — or
any spec_ref that didn't match those 8 — projected zero (:Part)-[:ASSESSES]->
(:SpecPoint) edges, silently. Seed the full top-level topic catalogue for the 6
current test specs (GCSE + A-level Physics/Chemistry/Biology, 44 SpecPoints) and
loop the Specification/SpecPoint MERGE over all of them (board created once).

Idempotent; deterministic uuid5 keys unchanged. Sub-point granularity remains a
later data task. Run: python3 -c "from run.initialization.init_exam_graph import
init; import json; print(json.dumps(init()))" in the ccapi container.

Co-Authored-By: Claude Opus 4.8 <[email protected]>

routers/exam/templates.py

@@ -13,6 +13,7 @@ join keys (spec §2).
 from __future__ import annotations
 
 import json
+import math
 import os
 import tempfile
 import time
@@ -342,6 +343,13 @@ def _pdf_has_text_layer(pdf_bytes: bytes) -> bool:
             pass
 
 
+# Canvas page width the frontend renders each PDF page at (app src/utils/exam-canvas/model.ts
+# PAGE_WIDTH). All auto-map canvas coords are emitted in this 780-wide, proportional-height space.
+CANVAS_PAGE_WIDTH = 780.0
+# Response/answer-region detector (api/services/docling/regions.py) renders at 144 DPI = 2 px / PDF point.
+REGIONS_PX_PER_PT = 2.0
+
+
 def _pdf_page_geometry(pdf_bytes: bytes) -> List[Dict[str, float]]:
     with tempfile.NamedTemporaryFile(prefix="cc-auto-map-geom-", suffix=".pdf", delete=False) as fh:
         fh.write(pdf_bytes)
@@ -355,14 +363,23 @@ def _pdf_page_geometry(pdf_bytes: bytes) -> List[Dict[str, float]]:
             for page in doc:
                 media = page.mediabox
                 crop = page.cropbox
-                rendered_w = float(crop.width or page.rect.width or 595.0)
+                page_pt_w = float(crop.width or page.rect.width or 1.0)
-                rendered_h = float(crop.height or page.rect.height or 842.0)
+                page_pt_h = float(crop.height or page.rect.height or 1.0)
+                # Emit canvas coords in the FRONTEND render space: the app draws each page at
+                # CANVAS_PAGE_WIDTH (app model.ts PAGE_WIDTH=780) with proportional height and stacks
+                # pages by those heights. Previously rendered_w/h were left in PDF points (~595x842),
+                # so every shape landed shrunk (~0.76x) and shifted up-left on the 780-wide canvas.
+                rendered_w = CANVAS_PAGE_WIDTH
+                # Mirror the app's canvas.height = Math.ceil(viewport.height) EXACTLY (pdfLoader.ts),
+                # so page_top accumulates identically. Using the raw float drifts ~1px/page, compounding
+                # to a visible upward shift on later pages of long papers (~36px over 40 pages).
+                rendered_h = float(math.ceil(CANVAS_PAGE_WIDTH * page_pt_h / page_pt_w))
                 pages.append({
                     "media_x0": float(media.x0),
                     "crop_x0": float(crop.x0),
                     "crop_y0": float(crop.y0),
-                    "page_pt_w": float(crop.width or page.rect.width or 1),
+                    "page_pt_w": page_pt_w,
-                    "page_pt_h": float(crop.height or page.rect.height or 1),
+                    "page_pt_h": page_pt_h,
                     "rendered_w": rendered_w,
                     "rendered_h": rendered_h,
                     "page_top": page_top,
@@ -384,11 +401,12 @@ def _pdf_page_geometry(pdf_bytes: bytes) -> List[Dict[str, float]]:
 def _page_geom(pages: List[Dict[str, float]], page_number: int) -> Dict[str, float]:
     if 1 <= page_number <= len(pages):
         return pages[page_number - 1]
+    _fallback_h = float(math.ceil(CANVAS_PAGE_WIDTH * 842.0 / 595.0))
     return {
         "media_x0": 0.0, "crop_x0": 0.0, "crop_y0": 0.0,
         "page_pt_w": 595.0, "page_pt_h": 842.0,
-        "rendered_w": 595.0, "rendered_h": 842.0,
+        "rendered_w": CANVAS_PAGE_WIDTH, "rendered_h": _fallback_h,
-        "page_top": (page_number - 1) * 842.0,
+        "page_top": (page_number - 1) * _fallback_h,
     }
 
 
@@ -397,12 +415,16 @@ def _box_to_canvas(box: Optional[Dict[str, Any]], page_number: int, pages: List[
         return None
     g = _page_geom(pages, page_number)
     if box.get("coord_origin") == "TOPLEFT" and {"x", "y", "w", "h"}.issubset(box):
-        scale = 0.5 if box.get("unit") == "px" else 1.0
+        # Scale the box into the 780-wide canvas space. px boxes (opencv/gemma regions) are in
+        # rendered-image px at REGIONS_PX_PER_PT px/point; TOPLEFT point boxes are 1 px/point.
+        px_per_pt = REGIONS_PX_PER_PT if box.get("unit") == "px" else 1.0
+        sx = g["rendered_w"] / (g["page_pt_w"] * px_per_pt)
+        sy = g["rendered_h"] / (g["page_pt_h"] * px_per_pt)
         return {
-            "x": round(float(box["x"]) * scale, 2),
+            "x": round(float(box["x"]) * sx, 2),
-            "y": round(g["page_top"] + float(box["y"]) * scale, 2),
+            "y": round(g["page_top"] + float(box["y"]) * sy, 2),
-            "w": round(float(box["w"]) * scale, 2),
+            "w": round(float(box["w"]) * sx, 2),
-            "h": round(float(box["h"]) * scale, 2),
+            "h": round(float(box["h"]) * sy, 2),
         }
     if not {"l", "t", "r", "b"}.issubset(box):
         return None
@@ -540,15 +562,47 @@ def _map_first_pass_to_rows(template_id: str, first_pass: Dict[str, Any], pdf_by
             response_form = _response_form_from_region_type(region.get("region_type"))
             if response_form:
                 response_areas.append({"id": _ai_id(template_id, "region", page_index, idx), "template_id": template_id, "question_id": first_part_by_page.get(page_index, default_qid), "page": page_index + 1, "bounds": bounds, "kind": "response", "response_form": response_form, "source": "ai", "confirmed": False, "confidence": _safe_confidence(region.get("confidence")), "derivation": region.get("detection_method") or "opencv-response-region"})
+    # Integrity guard: every response_area/boundary question_id must reference an inserted question
+    # (FK exam_response_areas/exam_boundaries -> exam_questions). On papers where band detection yields
+    # few/no questions but opencv/gemma still emit regions, those regions point at the synthetic
+    # default_qid which was never inserted. Ensure that fallback container question exists and reattach
+    # any orphan child rows to it, so persistence can't violate the FK.
+    qid_set = {q["id"] for q in questions}
+    orphans = [r for r in (response_areas + boundaries) if r.get("question_id") not in qid_set]
+    if orphans:
+        if default_qid not in qid_set:
+            questions.insert(0, {"id": default_qid, "template_id": template_id, "label": "Unassigned",
+                                 "order": 0, "max_marks": 0, "is_container": True, "source": "ai",
+                                 "confirmed": False, "confidence": 0.5,
+                                 "derivation": "auto-map-fallback-container"})
+            qid_set.add(default_qid)
+        for r in orphans:
+            r["question_id"] = default_qid
+
     return {"questions": questions, "response_areas": response_areas, "boundaries": boundaries, "layout": layout}
 
 
+def _dedupe_rows_by_id(rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Preserve first occurrence of stable AI row ids emitted by noisy OCR detectors."""
+    out: List[Dict[str, Any]] = []
+    seen: set[str] = set()
+    for row in rows:
+        row_id = row.get("id")
+        if row_id:
+            key = str(row_id)
+            if key in seen:
+                continue
+            seen.add(key)
+        out.append(row)
+    return out
+
+
 def _refresh_ai_rows(ctx: ExamContext, template_id: str, rows: Dict[str, List[Dict[str, Any]]]) -> None:
     sb = ctx.supabase
     for table in ("exam_response_areas", "exam_boundaries", "exam_template_layout", "exam_questions"):
         sb.table(table).delete().eq("template_id", template_id).eq("source", "ai").eq("confirmed", False).execute()
     for table, key in (("exam_questions", "questions"), ("exam_response_areas", "response_areas"), ("exam_boundaries", "boundaries"), ("exam_template_layout", "layout")):
-        payload = rows.get(key) or []
+        payload = _dedupe_rows_by_id(rows.get(key) or [])
         if payload:
             sb.table(table).insert(payload).execute()
 

run/initialization/init_exam_graph.py

@@ -1,17 +1,19 @@
 """
 init_exam_graph.py — Initialise the cc.public.exams Neo4j knowledge graph.
 
-Creates the shared, public exam database, its uniqueness constraints, and seeds the AQA exam
+Creates the shared, public exam database, its uniqueness constraints, and seeds the AQA exam board
-board + AQA GCSE Physics (8463) specification with its 8 top-level topic SpecPoints. Idempotent
++ the 6 current test specifications (GCSE & A-level Physics/Chemistry/Biology) with their top-level
-(CREATE DATABASE IF NOT EXISTS / CREATE CONSTRAINT IF NOT EXISTS / MERGE).
+topic SpecPoints (44 in total). Idempotent (CREATE DATABASE IF NOT EXISTS / CREATE CONSTRAINT IF NOT
+EXISTS / MERGE).
 
 Run inside the ccapi container:
     python3 -c "from run.initialization.init_exam_graph import init; import json; print(json.dumps(init()))"
 
-NOTE: the 8 SpecPoints seeded here are the real AQA GCSE Physics *top-level* topics. The full
+NOTE: only *top-level* topics are seeded (the granularity a teacher plans against). The full sub-point
-sub-point breakdown (e.g. 4.1.1.1 ...) is a later data-population task (sourceable from the AQA
+breakdown (e.g. 4.1.1.1 ...) is a later data-population task (sourceable from the AQA spec PDF via
-spec PDF via Docling). spec_code AQA-PHYS-8463 is the standalone GCSE Physics code that matches
+Docling). Seeding all 6 specs means a template's spec_ref finds a matching SpecPoint so
-"AQA Physics Paper 1H"; the eb_exams/eb_specifications seed (card S4-3) must use the same code.
+(:Part)-[:ASSESSES]->(:SpecPoint) fires beyond GCSE Physics; spec_code (e.g. AQA-PHYS-8463) must match
+the eb_exams/eb_specifications seed (card S4-3) and the app's deriveSpecCode.
 """
 import uuid
 from typing import Dict, Any
@@ -41,6 +43,37 @@ SPEC_POINTS = [
     ("4.8", "Space physics"),
 ]
 
+# Full AQA catalogue for the current test specs (top-level topics; ref = topic number). Seeding all of
+# them means a template's spec_ref finds a matching SpecPoint so (:Part)-[:ASSESSES]->(:SpecPoint) fires
+# beyond AQA GCSE Physics. Sub-point granularity (e.g. 4.1.1.1) remains a later data-population task.
+SPECIFICATIONS = [
+    {**SPEC, "topics": SPEC_POINTS},
+    {"spec_code": "AQA-CHEM-8462", "exam_board_code": "AQA", "subject_code": "CHEM", "award_code": "GCSE",
+     "title": "AQA GCSE Chemistry (8462)", "topics": [
+        ("4.1", "Atomic structure and the periodic table"), ("4.2", "Bonding, structure, and the properties of matter"),
+        ("4.3", "Quantitative chemistry"), ("4.4", "Chemical changes"), ("4.5", "Energy changes"),
+        ("4.6", "The rate and extent of chemical change"), ("4.7", "Organic chemistry"), ("4.8", "Chemical analysis"),
+        ("4.9", "Chemistry of the atmosphere"), ("4.10", "Using resources")]},
+    {"spec_code": "AQA-BIOL-8461", "exam_board_code": "AQA", "subject_code": "BIOL", "award_code": "GCSE",
+     "title": "AQA GCSE Biology (8461)", "topics": [
+        ("4.1", "Cell biology"), ("4.2", "Organisation"), ("4.3", "Infection and response"), ("4.4", "Bioenergetics"),
+        ("4.5", "Homeostasis and response"), ("4.6", "Inheritance, variation and evolution"), ("4.7", "Ecology")]},
+    {"spec_code": "AQA-PHYS-7408", "exam_board_code": "AQA", "subject_code": "PHYS", "award_code": "A-level",
+     "title": "AQA A-level Physics (7408)", "topics": [
+        ("3.1", "Measurements and their errors"), ("3.2", "Particles and radiation"), ("3.3", "Waves"),
+        ("3.4", "Mechanics and materials"), ("3.5", "Electricity"), ("3.6", "Further mechanics and thermal physics"),
+        ("3.7", "Fields and their consequences"), ("3.8", "Nuclear physics")]},
+    {"spec_code": "AQA-CHEM-7405", "exam_board_code": "AQA", "subject_code": "CHEM", "award_code": "A-level",
+     "title": "AQA A-level Chemistry (7405)", "topics": [
+        ("3.1", "Physical chemistry"), ("3.2", "Inorganic chemistry"), ("3.3", "Organic chemistry")]},
+    {"spec_code": "AQA-BIOL-7402", "exam_board_code": "AQA", "subject_code": "BIOL", "award_code": "A-level",
+     "title": "AQA A-level Biology (7402)", "topics": [
+        ("3.1", "Biological molecules"), ("3.2", "Cells"), ("3.3", "Organisms exchange substances with their environment"),
+        ("3.4", "Genetic information, variation and relationships between organisms"),
+        ("3.5", "Energy transfers in and between organisms"), ("3.6", "Organisms respond to changes"),
+        ("3.7", "Genetics, populations, evolution and ecosystems"), ("3.8", "The control of gene expression")]},
+]
+
 CONSTRAINTS = [
     "CREATE CONSTRAINT exam_board_uid    IF NOT EXISTS FOR (n:ExamBoard)     REQUIRE n.uuid_string IS UNIQUE",
     "CREATE CONSTRAINT spec_uid          IF NOT EXISTS FOR (n:Specification) REQUIRE n.uuid_string IS UNIQUE",
@@ -81,37 +114,38 @@ def init() -> Dict[str, Any]:
             s.run(c).consume()
             result["constraints"] += 1
 
-        # 3. board + spec
+        # 3. board (once)
         board_uid = _uid("ExamBoard", BOARD["code"])
-        spec_uid = _uid("Specification", SPEC["spec_code"])
         s.run(
             "MERGE (b:ExamBoard {uuid_string:$uid}) "
             "SET b.code=$code, b.name=$name, b.node_storage_path=$nsp",
             uid=board_uid, code=BOARD["code"], name=BOARD["name"],
             nsp=f"{EXAM_DB}/ExamBoard/{BOARD['code']}",
         ).consume()
-        s.run(
-            "MERGE (sp:Specification {uuid_string:$uid}) "
-            "SET sp.spec_code=$sc, sp.exam_board_code=$ebc, sp.subject_code=$subj, "
-            "    sp.award_code=$award, sp.title=$title, sp.node_storage_path=$nsp "
-            "WITH sp MATCH (b:ExamBoard {code:$ebc}) MERGE (b)-[:PUBLISHES]->(sp)",
-            uid=spec_uid, sc=SPEC["spec_code"], ebc=SPEC["exam_board_code"],
-            subj=SPEC["subject_code"], award=SPEC["award_code"], title=SPEC["title"],
-            nsp=f"{EXAM_DB}/Specification/{SPEC['spec_code']}",
-        ).consume()
 
-        # 4. spec points
+        # 4. each specification + its top-level spec points (idempotent MERGE)
-        for ref, desc in SPEC_POINTS:
+        for spec in SPECIFICATIONS:
-            sp_uid = _uid("SpecPoint", SPEC["spec_code"], ref)
+            spec_uid = _uid("Specification", spec["spec_code"])
             s.run(
-                "MERGE (p:SpecPoint {uuid_string:$uid}) "
+                "MERGE (sp:Specification {uuid_string:$uid}) "
-                "SET p.ref=$ref, p.description=$desc, p.spec_code=$sc, "
+                "SET sp.spec_code=$sc, sp.exam_board_code=$ebc, sp.subject_code=$subj, "
-                "    p.exam_board_code=$ebc, p.node_storage_path=$nsp "
+                "    sp.award_code=$award, sp.title=$title, sp.node_storage_path=$nsp "
-                "WITH p MATCH (s:Specification {spec_code:$sc}) MERGE (s)-[:HAS_SPEC_POINT]->(p)",
+                "WITH sp MATCH (b:ExamBoard {code:$ebc}) MERGE (b)-[:PUBLISHES]->(sp)",
-                uid=sp_uid, ref=ref, desc=desc, sc=SPEC["spec_code"],
+                uid=spec_uid, sc=spec["spec_code"], ebc=spec["exam_board_code"],
-                ebc=SPEC["exam_board_code"], nsp=f"{EXAM_DB}/SpecPoint/{SPEC['spec_code']}/{ref}",
+                subj=spec["subject_code"], award=spec["award_code"], title=spec["title"],
+                nsp=f"{EXAM_DB}/Specification/{spec['spec_code']}",
             ).consume()
-            result["spec_points"] += 1
+            for ref, desc in spec["topics"]:
+                sp_uid = _uid("SpecPoint", spec["spec_code"], ref)
+                s.run(
+                    "MERGE (p:SpecPoint {uuid_string:$uid}) "
+                    "SET p.ref=$ref, p.description=$desc, p.spec_code=$sc, "
+                    "    p.exam_board_code=$ebc, p.node_storage_path=$nsp "
+                    "WITH p MATCH (s:Specification {spec_code:$sc}) MERGE (s)-[:HAS_SPEC_POINT]->(p)",
+                    uid=sp_uid, ref=ref, desc=desc, sc=spec["spec_code"],
+                    ebc=spec["exam_board_code"], nsp=f"{EXAM_DB}/SpecPoint/{spec['spec_code']}/{ref}",
+                ).consume()
+                result["spec_points"] += 1
 
         counts = s.run(
             "MATCH (b:ExamBoard) WITH count(b) AS boards "

tests/test_exam_templates.py

@@ -642,6 +642,20 @@ def test_auto_map_fast_path_merges_ai_rows_and_returns_detail(monkeypatch):
     assert store["exam_boundaries"] and store["exam_boundaries"][0]["derivation"] == "docling-main-band"
 
 
+def test_auto_map_deduplicates_repeated_response_area_ids(monkeypatch):
+    store = _template_with_source()
+    client, store = make_client(store=store)
+    _patch_auto_map(monkeypatch, store, fast=True)
+    dup = {"page_index": 0, "bbox": {"l": 50, "t": 700, "r": 100, "b": 680, "coord_origin": "BOTTOMLEFT"}, "region_type": "answer_lines", "confidence": 0.9}
+    monkeypatch.setattr(templates_mod, "detect_response_regions_from_pdf", lambda *_a, **_k: [dup, dict(dup)])
+
+    resp = client.post("/api/exam/templates/t1/auto-map")
+
+    assert resp.status_code == 200
+    response_area_ids = [r["id"] for r in store["exam_response_areas"]]
+    assert len(response_area_ids) == len(set(response_area_ids))
+
+
 def test_auto_map_preserves_manual_and_confirmed_rows_on_rerun(monkeypatch):
     store = _template_with_source()
     store.update({