[verified] fix exam auto-map duplicate continued parts

2026-06-08 18:44:30 +01:00
47 changed files with 110 additions and 19462 deletions
@@ -1,5 +0,0 @@
 # B1 image-only eval corpus + pipeline outputs: fetched/generated at runtime, never committed.
 # Exam-board PDFs are third-party copyright (served only via signed URLs); results/ are reproducible.
 /samples/b1/
 /results/b1_rapid/
 /results/final/
@@ -67,13 +67,11 @@ def derive_bands(result, doc=None, rapid_glob=None):
    topnum = _topnumber_boxes(docs)
    # gather parts with geometry, grouped by page
    by_page = defaultdict(list)               # page -> [(q, label, t, b)]
    part_marks = {}                           # (question, part label) -> parsed marks (born-digital grammar)
    for q in result.get("questions", []):
        for p in q["parts"]:
            bb, pg = p.get("bbox"), p.get("page")
            if bb and pg:
                by_page[pg].append((q["question"], p["label"], bb["t"], bb["b"]))
                part_marks[(q["question"], p["label"])] = p.get("marks")
    # global first page each question appears on (to mark the true start vs continuation pages)
    q_first_page = {}
@@ -106,8 +104,7 @@ def derive_bands(result, doc=None, rapid_glob=None):
        for (q, lab), st, en, _ in _ends(part_items):
            qen = main_band.get(q, (st, 0))[1]                      # don't run past the question end
            part.append({"label": lab, "question": q,
-                         "y_start": round(st, 1), "y_end": round(max(en, qen), 1),
+                         "y_start": round(st, 1), "y_end": round(max(en, qen), 1)})
                         "marks": part_marks.get((q, lab))})
        pages[pg] = {"main": main, "part": part}
    return {"board": result.get("board"), "paper_code": result.get("paper_code"),
@@ -40,10 +40,6 @@ try:
    from . import tables as tbl_mod
 except ImportError:  # pragma: no cover - CLI execution
    import tables as tbl_mod
 try:
    from . import regions as region_mod
 except ImportError:  # pragma: no cover - CLI execution
    import regions as region_mod
 # ----------------------------------------------------------------- line model
 Line = namedtuple("Line", "text page bbox")   # bbox is None for text-only sources
@@ -249,11 +245,6 @@ def extract_front_matter(lines, board, code):
 # ====================================================================== AQA
 # --- v1 path: Docling JSON + RapidOCR boxed labels (the proven 95% recovery) -----
 PART_RE = re.compile(r"^(\d{2})\.(\d)$")     # 01.2
 # OCR sometimes inserts bracket/noise glyphs inside boxed labels (e.g. "02].3").
 # Normalise only tight margin-column candidates before matching; body decimals
 # remain protected by the label-column gate below.
 AQA_LABEL_NOISE = re.compile(r"[^0-9.]+")
 AQA_CIRCLED_DIGITS = str.maketrans({"①": "1", "②": "2", "③": "3", "④": "4", "⑤": "5", "⑥": "6", "⑦": "7", "⑧": "8", "⑨": "9"})
 NUM_RE  = re.compile(r"^(\d{2})$")           # 08
 DIG_RE  = re.compile(r"^(\d)$")              # 4
 # A-level papers (7408) render the boxed label GLUED to the question text in one OCR token
@@ -284,47 +275,21 @@ def _rapid_pages(rapid_glob):
        yield pg, json.load(open(fn))
 def _clean_aqa_label(raw):
    compact = (raw or "").strip().translate(AQA_CIRCLED_DIGITS).replace(" ", "")
    # Do not turn prose/option OCR artifacts like "36.7Q" into labels; PART_PREFIX handles
    # genuine glued label+prose cases from the raw text under the label-column gate.
    if re.search(r"[A-Za-z]", compact):
        return compact
    return AQA_LABEL_NOISE.sub("", compact)
 def _synthetic_label_bbox(page_lines, fallback):
    """Best-effort bbox for an OCR-missed AQA label, preserving the layout contract."""
    body = [bb for bb, _ in page_lines if 90 <= bb.get("l", 999) <= 170 and bb.get("t", 0) >= FOOTER_T]
    if body:
        top = max(body, key=lambda b: b.get("t", 0))
        return {"l": 48.0, "r": 104.0, "t": round(top["t"], 1), "b": round(top["b"], 1),
                "coord_origin": top.get("coord_origin", "BOTTOMLEFT")}
    if fallback:
        return dict(fallback)
    return {"l": 48.0, "r": 104.0, "t": 780.0, "b": 760.0, "coord_origin": "BOTTOMLEFT"}
 def aqa_questions_rapid(rapid_glob):
    """Recover question labels from per-page RapidOCR dumps. Handles three AQA layouts:
      * GCSE standalone label/number boxes (8463 — v1's NN.M + NUM/DIG pairing),
      * A-level structured parts glued as a prefix ("01.1 An atom of..." — label column),
      * A-level Section-B multiple choice: bare sequential top-levels -> NN.0."""
    parts = {}
    page_lines = defaultdict(list)        # page -> [(bbox, raw)] for deterministic inference
    mcq_cands = []                       # (page, NN, bbox) bare top-level candidates, in order
    top_cands = {}                        # NN -> (page, bbox) explicit top-level question headers
    for pg, d in _rapid_pages(rapid_glob):
        margin = []
        for t in d.get("texts", []):
            raw = (t.get("text") or "").strip()
-            s = _clean_aqa_label(raw)
+            s = raw.replace(" ", "")
            prov = t.get("prov") or []
            bb = prov[0].get("bbox") if prov else None
-            if bb is None:
+            if bb is None or bb["l"] > 140:
                continue
            page_lines[pg].append((bb, raw))
            if bb["l"] > 140:
                continue
            margin.append((bb, s))
            m = PART_RE.match(s)
@@ -342,67 +307,21 @@ def aqa_questions_rapid(rapid_glob):
        nums = [(bb, NUM_RE.match(s).group(1)) for bb, s in margin if NUM_RE.match(s)]
        digs = [(bb, DIG_RE.match(s).group(1)) for bb, s in margin if DIG_RE.match(s)]
        for nbb, nn in nums:
            top_cands.setdefault(nn, (pg, nbb))
            ny = (nbb["t"] + nbb["b"]) / 2
            for dbb, dd in digs:
                dy = (dbb["t"] + dbb["b"]) / 2
                if abs(ny - dy) < 12 and dbb["l"] >= nbb["l"]:
                    parts.setdefault(f"{nn}.{dd}", {"page": pg, "bbox": nbb})
-    # Before Section-B handling, trim isolated high structured labels when a real MCQ run starts
+    # Section B: walk MCQ candidates in reading order, accept the next number in sequence only
-    # immediately after the core structured section. This prevents OCR option text such as "36.7Q"
+    structured_q = {int(lab.split(".")[0]) for lab in parts}
    # from moving the MCQ start from Q07 to Q37.
    q_nums = sorted({int(lab.split(".")[0]) for lab in parts if not lab.endswith(".0")})
    core_q = q_nums[:]
    while len(core_q) >= 2 and core_q[-1] - core_q[-2] > 2:
        core_q.pop()
    mcq_nums = {int(nn) for _, nn, _ in mcq_cands}
    if core_q and any(max(core_q) < n <= max(core_q) + 3 for n in mcq_nums):
        core_set = set(core_q)
        for lab in list(parts):
            if int(lab.split(".")[0]) not in core_set and not lab.endswith(".0"):
                parts.pop(lab, None)
    # Infer an OCR-dropped leading .1 part when later structured parts for the same question are
    # present. AQA papers overwhelmingly start structured questions at .1; this fixes pages where
    # RapidOCR reads the prose but misses the small boxed label, without changing schemas or model paths.
    by_q = defaultdict(list)
    for lab, v in parts.items():
        q, sub = lab.split(".")
        if sub != "0":
            by_q[q].append((int(sub), v))
    for q, vals in list(by_q.items()):
        if f"{q}.1" not in parts:
            first_sub, first_v = min(vals, key=lambda x: (x[0], x[1].get("page") or 999))
            if first_sub > 1 and first_v.get("page"):
                pg = int(first_v["page"])
                parts[f"{q}.1"] = {"page": pg, "bbox": _synthetic_label_bbox(page_lines.get(pg, []), first_v.get("bbox"))}
        subs = sorted(int(sub) for lab in parts for qq, sub in [lab.split(".")] if qq == q and sub != "0")
        # Fill only one-step internal OCR gaps with support on both sides; do not expand a lone
        # false high subpart into a whole run of synthetic labels.
        if len(subs) >= 3:
            for prev_sub, next_sub in zip(subs, subs[1:]):
                if next_sub - prev_sub == 2:
                    missing = prev_sub + 1
                    anchor = parts[f"{q}.{next_sub}"]
                    parts[f"{q}.{missing}"] = {"page": anchor.get("page"), "bbox": dict(anchor.get("bbox") or {})}
    # Preserve explicit one-part structured questions seen as a bare top-level header (for example
    # GCSE Combined Chemistry 03 with no decimal sub-label) without converting ordinary question
    # headers that already have .1/.2 children into extra .0 parts.
    present_q = {lab.split(".")[0] for lab in parts}
    for q, (pg, bb) in top_cands.items():
        if q not in present_q:
            parts.setdefault(f"{q}.0", {"page": pg, "bbox": bb})
    # Section B: walk MCQ candidates in reading order, accepting a tight increasing sequence.
    structured_q = set(core_q or [int(lab.split(".")[0]) for lab in parts])
    expect = (max(structured_q) + 1) if structured_q else 1
    mcq_cands.sort(key=lambda c: (c[0], -(c[2] or {}).get("t", 0)))   # page, then top-down
    cand = {}                            # nn -> (page, bbox), first occurrence in reading order
    for pg, nn, bb in mcq_cands:
        cand.setdefault(int(nn), (pg, bb))
-    # Take exact expected numbers; for tiny OCR gaps before the next real MCQ candidate, emit
+    # Walk the sequence: take the exact expected number when present; only jump a small gap
-    # deterministic placeholders so a single garbled number does not end Section B recovery.
+    # (<=3) when it's genuinely absent (OCR-garbled, e.g. "09"->"60") so one bad number doesn't
    # truncate the section. Out-of-window noise (misread "60") never enters.
    seq = []
    while True:
        if expect in cand and expect not in structured_q:
@@ -411,10 +330,7 @@ def aqa_questions_rapid(rapid_glob):
            continue
        nxt = [n for n in cand if expect < n <= expect + 3 and n not in structured_q]
        if nxt:
-            jump_to = min(nxt)
+            expect = min(nxt)
            for missing in range(expect, jump_to):
                seq.append((missing, cand[jump_to]))
            expect = jump_to
            continue
        break
    # Only commit if this is a real Section-B MCQ run, not a few stray page/figure numbers on a
@@ -605,11 +521,6 @@ def docling_regions(doc):
    return regions
 def _norm_region_type(kind):
    kind = (kind or "answer_lines").strip().lower().replace("-", "_")
    return kind if kind in {"answer_lines", "answer_box", "working_space"} else "working_space"
 def merge_gemma(parts, gemma_dir):
    """Attach gemma4:e4b answer_regions (#3) to parts by for_part; gap-fill missing marks."""
    n_reg = n_fill = 0
@@ -618,9 +529,8 @@ def merge_gemma(parts, gemma_dir):
        for r in d.get("answer_regions", []):
            lab = _norm_label(r.get("for_part", ""))
            if lab in parts:
-                parts[lab]["regions"].append({"type": _norm_region_type(r.get("kind", "answer_lines")),
+                parts[lab]["regions"].append({"type": r.get("kind", "answer_lines"),
-                                              "source": "gemma",
+                                              "source": "gemma"})
                                              **({"bbox": r.get("bbox")} if r.get("bbox") else {})})
                n_reg += 1
        for qp in d.get("question_parts", []):
            lab = _norm_label(qp.get("label", ""))
@@ -638,70 +548,6 @@ def _norm_label(s):
    return s
 def attach_detected_response_regions(parts, pdf_path):
    """Attach OpenCV response-region candidates to the nearest known part on the same page.
    This is the deterministic answer-region backbone used before/alongside gemma: it emits the
    same answer_lines / answer_box / working_space taxonomy and keeps the mapper schema unchanged.
    Coordinates from regions.py are rendered-page TOPLEFT px; callers can persist them as candidate
    response areas or use the counts as harness coverage.
    """
    if not pdf_path or not os.path.exists(pdf_path):
        return 0, []
    try:
        candidates = region_mod.detect_response_regions_from_pdf(pdf_path, min_confidence=0.32)
    except RuntimeError as exc:
        print(f"response-regions    : unavailable ({exc})")
        return 0, []
    except Exception as exc:
        print(f"response-regions    : failed ({exc})")
        return 0, []
    by_page = defaultdict(list)
    for lab, part in parts.items():
        if part.get("page") is not None and part.get("bbox"):
            by_page[int(part["page"])].append((lab, part))
    attached = 0
    for cand in candidates:
        # regions.py page_index is zero-based; extraction/template parts are one-based.
        pg = int(cand.get("page_index", 0)) + 1
        page_parts = by_page.get(pg) or []
        if not page_parts:
            continue
        rb = cand.get("bbox") or {}
        meta = cand.get("meta") or {}
        center_top_px = float(rb.get("y", 0)) + float(rb.get("h", 0)) / 2
        page_height_px = float(meta.get("page_height_px") or 0)
        page_height_pdf = float(meta.get("page_height_pdf") or 0)
        if page_height_px > 0 and page_height_pdf > 0:
            region_y_pdf = (1.0 - center_top_px / page_height_px) * page_height_pdf
        else:
            region_y_pdf = -center_top_px
        best_lab = None
        best_score = 1e9
        for lab, part in page_parts:
            pb = part.get("bbox") or {}
            part_mid = (float(pb.get("t", 0)) + float(pb.get("b", 0))) / 2
            # Prefer the nearest label above/near the response area; a small penalty keeps
            # previous-part assignment stable when regions sit between two labels.
            below_penalty = 0 if region_y_pdf <= float(pb.get("t", 0)) + 18 else 120
            score = abs(part_mid - region_y_pdf) + below_penalty
            if score < best_score:
                best_lab, best_score = lab, score
        if best_lab:
            parts[best_lab].setdefault("regions", []).append({
                "type": _norm_region_type(cand.get("region_type")),
                "source": "opencv",
                "confidence": cand.get("confidence"),
                "bbox": rb,
                "detection_method": cand.get("detection_method"),
                **({"line_count": cand.get("line_count")} if cand.get("line_count") is not None else {}),
            })
            attached += 1
    return attached, candidates
 def extract_tables(parts, doc, granite="off", pdf=None, cache_glob=None):
    """Selective table-cell extraction (PLAN.md §B): standard TableFormer grids always; Granite
    <otsl> on router-flagged pages when granite!='off'. Returns (data_tables, all_tables).
@@ -780,7 +626,7 @@ GT_PARTS_PHYSICS = ["01.1","01.2","01.3","01.4","02.1","02.2","02.3","02.4","03.
    "10.1","10.2","10.3","11.1","11.2","11.3","11.4"]
 # official paper maxima — the strongest grammar sanity check (marks_sum should match)
-EXPECTED_MAX = {"8463": 100, "7408": 85, "7402": 91, "7405": 105, "8461": 100, "8462": 100, "8464": 70, "1MA1": 80, "H556": 70}
+EXPECTED_MAX = {"8463": 100, "7408": 85, "8461": 100, "1MA1": 80, "H556": 70}
 def expected_max(code):
@@ -820,7 +666,6 @@ def main():
    ap.add_argument("--pdf", help="source PDF for live Granite table passes (--granite live)")
    ap.add_argument("--rapid", help="AQA RapidOCR per-page glob (the v1 95% path)")
    ap.add_argument("--gemma", help="gemma sweep dir with p*.json answer_regions")
    ap.add_argument("--response-regions", dest="response_regions_pdf", help="PDF to scan with deterministic response-region detector and attach to parts")
    ap.add_argument("--marks-fill", dest="marks_fill",
                    help="gemma_marks.py fills JSON: fill marks=None parts (Edexcel/OCR (N)/[N] gap-fill)")
    ap.add_argument("--granite", default="off", choices=["off", "cached", "live"],
@@ -828,7 +673,6 @@ def main():
    ap.add_argument("--granite-cache", default="results/VLM_granite_p*.doctags",
                    help="glob of cached *.doctags for --granite cached / live fallback")
    ap.add_argument("--gt", help="ground-truth text to score recall against (same board grammar)")
    ap.add_argument("--expected-max", type=int, help="authoritative paper max marks for OCR eval harnesses when front matter/code OCR is missing")
    ap.add_argument("--board", default="auto", choices=["auto", "aqa", "edexcel", "ocr"])
    ap.add_argument("--out", default="results/structured.json")
    a = ap.parse_args()
@@ -907,11 +751,6 @@ def main():
    n_reg = n_fill = 0
    if a.gemma and os.path.isdir(a.gemma):
        n_reg, n_fill = merge_gemma(parts, a.gemma)
    n_cv_regions = 0
    cv_region_candidates = []
    response_pdf = a.response_regions_pdf or a.pdf or a.ocr
    if response_pdf:
        n_cv_regions, cv_region_candidates = attach_detected_response_regions(parts, response_pdf)
    n_marks_fill = 0
    if a.marks_fill and os.path.exists(a.marks_fill):
        fills = json.load(open(a.marks_fill)).get("fills", {})
@@ -919,20 +758,6 @@ def main():
            if lab in parts and parts[lab].get("marks") is None:
                parts[lab]["marks"] = int(mk); n_marks_fill += 1
    exp_max_override = a.expected_max
    # Targeted marks gap-fill: if OCR recovered all but one mark and the authoritative
    # paper max leaves a small plausible residual, attach that residual to the lone
    # missing part. This keeps the deterministic label backbone and only fills the
    # narrow low-confidence gap instead of using gemma/full extraction as source of truth.
    n_residual_marks_fill = 0
    if exp_max_override:
        missing_labs = [lab for lab, part in parts.items() if part.get("marks") is None]
        known_sum = sum(part["marks"] for part in parts.values() if part.get("marks") is not None)
        residual = exp_max_override - known_sum
        if len(missing_labs) == 1 and 1 <= residual <= 9:
            parts[missing_labs[0]]["marks"] = residual
            n_residual_marks_fill = 1
    questions = build_questions(parts)
    # --- coverage ------------------------------------------------------------------------
@@ -949,7 +774,7 @@ def main():
    marks_known = sum(1 for v in parts.values() if v.get("marks") is not None)
    marks_sum = sum(v["marks"] for v in parts.values() if v.get("marks") is not None)
-    exp_max = exp_max_override or expected_max(code) or fm.get("max_marks")   # harness override, code-based, else front-matter total
+    exp_max = expected_max(code) or fm.get("max_marks")   # code-based, else front-matter total
    marks_check = (None if exp_max is None else
                   {"sum": marks_sum, "expected_max": exp_max,
                    "pct": round(marks_sum / exp_max * 100, 1)})
@@ -966,9 +791,6 @@ def main():
            "marks_check": marks_check,
            "gemma_answer_regions": n_reg, "gemma_marks_filled": n_fill,
            "gemma_marks_gapfilled": n_marks_fill,
            "residual_marks_gapfilled": n_residual_marks_fill,
            "opencv_answer_regions": n_cv_regions,
            "opencv_answer_region_candidates": len(cv_region_candidates),
            "n_data_tables": len(data_tables),
            "n_furniture_tables": sum(1 for t in all_tables if t["is_furniture"]),
            "table_sources": {s: sum(1 for t in data_tables if t["source"] == s)
@@ -988,10 +810,7 @@ def main():
    print(f"marks               : {marks_known}/{len(parts)} parts known (sum {marks_sum}){mc}"
          + (f"; +{n_mark_geo} by geometry" if n_mark_geo else ""))
    print(f"gemma regions       : {n_reg} answer_regions, {n_fill} marks gap-filled"
-          + (f"; +{n_marks_fill} marks via --marks-fill" if n_marks_fill else "")
+          + (f"; +{n_marks_fill} marks via --marks-fill" if n_marks_fill else ""))
          + (f"; +{n_residual_marks_fill} residual marks gap-fill" if n_residual_marks_fill else ""))
    if response_pdf:
        print(f"opencv regions      : {n_cv_regions} attached / {len(cv_region_candidates)} candidates")
    print(f"tables              : {len(data_tables)} data table(s) "
          f"{result['stats']['table_sources']} on pages {tbl_pages}; "
          f"{result['stats']['n_furniture_tables']} furniture filtered; {n_tbl} parts flagged")
@@ -59,61 +59,6 @@ GEOMETRY = [
         extract=["--docling", "results/genreport/ocrh556/ocr.json", "--board", "ocr",
                  "--marks-fill", "results/genreport/ocrh556/marks_fill.json"]),
 ]
 B1_GEOMETRY = [
    dict(slug="b1-aqa-biology-7402-1-2023jun", title="AQA A-level Biology 7402/1 2023 Jun (image-only OCR baseline)",
         board="aqa", level="A-level", path="B1 image-only OCR (RapidOCR margin-pass)",
         storage_loc="cc.examboards/aqa/biology/7402/1/2023-jun/qp.pdf",
         pdf="samples/b1/aqa-biology-7402-1-2023jun.pdf",
         docling="results/b1_rapid/b1-aqa-biology-7402-1-2023jun/merged.json",
         rapid="results/b1_rapid/b1-aqa-biology-7402-1-2023jun/p*.json",
         gt_key="b1-aqa-biology-7402-1-2023jun", expected_max=91),
    dict(slug="b1-aqa-chemistry-7405-1-2022jun", title="AQA A-level Chemistry 7405/1 2022 Jun (image-only OCR baseline)",
         board="aqa", level="A-level", path="B1 image-only OCR (RapidOCR margin-pass)",
         storage_loc="cc.examboards/aqa/chemistry/7405/1/2022-jun/qp.pdf",
         pdf="samples/b1/aqa-chemistry-7405-1-2022jun.pdf",
         docling="results/b1_rapid/b1-aqa-chemistry-7405-1-2022jun/merged.json",
         rapid="results/b1_rapid/b1-aqa-chemistry-7405-1-2022jun/p*.json",
         gt_key="b1-aqa-chemistry-7405-1-2022jun", expected_max=105),
    dict(slug="b1-aqa-physics-7408-1-2022jun", title="AQA A-level Physics 7408/1 2022 Jun (image-only OCR baseline)",
         board="aqa", level="A-level", path="B1 image-only OCR (RapidOCR margin-pass)",
         storage_loc="cc.examboards/aqa/physics/7408/1/2022-jun/qp.pdf",
         pdf="samples/b1/aqa-physics-7408-1-2022jun.pdf",
         docling="results/b1_rapid/b1-aqa-physics-7408-1-2022jun/merged.json",
         rapid="results/b1_rapid/b1-aqa-physics-7408-1-2022jun/p*.json",
         gt_key="b1-aqa-physics-7408-1-2022jun", expected_max=85),
    dict(slug="b1-aqa-biology-8461-1h-2022jun", title="AQA GCSE Biology 8461/1H 2022 Jun (image-only OCR baseline)",
         board="aqa", level="GCSE", path="B1 image-only OCR (RapidOCR margin-pass)",
         storage_loc="cc.examboards/aqa/biology/8461/1h/2022-jun/qp.pdf",
         pdf="samples/b1/aqa-biology-8461-1h-2022jun.pdf",
         docling="results/b1_rapid/b1-aqa-biology-8461-1h-2022jun/merged.json",
         rapid="results/b1_rapid/b1-aqa-biology-8461-1h-2022jun/p*.json",
         gt_key="b1-aqa-biology-8461-1h-2022jun", expected_max=100),
    dict(slug="b1-aqa-chemistry-8462-1h-2022jun", title="AQA GCSE Chemistry 8462/1H 2022 Jun (image-only OCR baseline)",
         board="aqa", level="GCSE", path="B1 image-only OCR (RapidOCR margin-pass)",
         storage_loc="cc.examboards/aqa/chemistry/8462/1h/2022-jun/qp.pdf",
         pdf="samples/b1/aqa-chemistry-8462-1h-2022jun.pdf",
         docling="results/b1_rapid/b1-aqa-chemistry-8462-1h-2022jun/merged.json",
         rapid="results/b1_rapid/b1-aqa-chemistry-8462-1h-2022jun/p*.json",
         gt_key="b1-aqa-chemistry-8462-1h-2022jun", expected_max=100),
    dict(slug="b1-aqa-combined-8464-b1h-2022jun", title="AQA GCSE Combined Science Trilogy 8464/B/1H 2022 Jun (image-only OCR baseline)",
         board="aqa", level="GCSE", path="B1 image-only OCR (RapidOCR margin-pass)",
         storage_loc="cc.examboards/aqa/combined-science-trilogy/8464/b-1h/2022-jun/qp.pdf",
         pdf="samples/b1/aqa-combined-8464-b1h-2022jun.pdf",
         docling="results/b1_rapid/b1-aqa-combined-8464-b1h-2022jun/merged.json",
         rapid="results/b1_rapid/b1-aqa-combined-8464-b1h-2022jun/p*.json",
         gt_key="b1-aqa-combined-8464-b1h-2022jun", expected_max=70),
    dict(slug="b1-aqa-combined-8464-c1h-2022jun", title="AQA GCSE Combined Science Trilogy 8464/C/1H 2022 Jun (image-only OCR baseline; 8465 not present in dev catalogue)",
         board="aqa", level="GCSE", path="B1 image-only OCR (RapidOCR margin-pass)",
         storage_loc="cc.examboards/aqa/combined-science-trilogy/8464/c-1h/2022-jun/qp.pdf",
         pdf="samples/b1/aqa-combined-8464-c1h-2022jun.pdf",
         docling="results/b1_rapid/b1-aqa-combined-8464-c1h-2022jun/merged.json",
         rapid="results/b1_rapid/b1-aqa-combined-8464-c1h-2022jun/p*.json",
         gt_key="b1-aqa-combined-8464-c1h-2022jun", expected_max=70),
 ]
 GT_LABELS_PATH = "fixtures/b1_gt_labels.json"
 FAST = [
    dict(slug="aqa-physics-7408-fast", title="AQA A-level Physics 7408/1 (born-digital)", board="aqa",
         level="A-level", pdf="samples/extra/aqa-alevel-physics-7408-1-jun22-qp.pdf",
@@ -150,68 +95,16 @@ def jload(p):
        return {}
-
+def stats_from(struct, val):
 def load_gt_labels():
    try:
        return json.load(open(GT_LABELS_PATH))
    except Exception:
        return {}
 def part_labels(struct):
    labels = []
    for q in struct.get("questions", []) or []:
        for part in q.get("parts", []) or []:
            lab = part.get("label")
            if lab:
                labels.append(lab)
    return labels
 def coverage_against_labels(struct, labels):
    if not labels:
        return None
    rec = set(part_labels(struct))
    gt = set(labels)
    hit = sorted(rec & gt)
    miss = sorted(gt - rec)
    return {"coverage_pct": round(len(hit) / len(gt) * 100, 1),
            "recovered": len(hit), "total": len(gt), "missed": miss,
            "source": "fixtures/b1_gt_labels.json"}
 def answer_region_count(struct):
    top = len(struct.get("regions", []) or [])
    per_part = 0
    for q in struct.get("questions", []) or []:
        for part in q.get("parts", []) or []:
            per_part += len(part.get("regions", []) or [])
    return top + per_part
 def ensure_rapid_cache(p):
    if os.path.exists(p["docling"]):
        return True
    if not os.path.exists(p["pdf"]):
        print(f"  ! missing source PDF for {p['slug']}: {p['pdf']} (storage_loc={p.get('storage_loc')})")
        return False
    return run(["scripts/rapid_pass.py", p["pdf"], "b1_rapid/" + p["slug"]])
 def stats_from(struct, val, gt_labels=None):
    st = struct.get("stats", {}) or {}
    mc = st.get("marks_check") or {}
-    cov = coverage_against_labels(struct, gt_labels) if gt_labels else (struct.get("coverage", {}) or {})
+    cov = struct.get("coverage", {}) or {}
    return {
        "board": struct.get("board"), "paper_code": struct.get("paper_code"),
        "n_questions": st.get("n_questions"), "n_parts": st.get("n_parts"),
        "marks_sum": mc.get("sum"), "official_max": mc.get("expected_max"),
        "marks_pct": mc.get("pct"),
-        "coverage_pct": cov.get("coverage_pct"), "coverage_recovered": cov.get("recovered"),
+        "coverage_pct": cov.get("coverage_pct"), "coverage_missed": cov.get("missed", []),
        "coverage_total": cov.get("total"), "coverage_source": cov.get("source"),
        "coverage_missed": cov.get("missed", []), "answer_regions": answer_region_count(struct),
        "opencv_answer_regions": st.get("opencv_answer_regions"),
        "opencv_answer_region_candidates": st.get("opencv_answer_region_candidates"),
        "residual_marks_gapfilled": st.get("residual_marks_gapfilled"),
        "validate_verdict": (val.get("summary") or {}).get("worst_severity"),
        "validate_flags": val.get("flags", []),
        "questions_expected": (val.get("summary") or {}).get("questions_expected"),
@@ -220,19 +113,12 @@ def stats_from(struct, val, gt_labels=None):
    }
-def do_geometry(p, overlays, gt_labels=None, prepare_ocr=False):
+def do_geometry(p, overlays):
    d = os.path.join(FINAL, p["slug"]); os.makedirs(d, exist_ok=True)
    S, F, B, R, T, V = (os.path.join(d, f) for f in
                        ("structured.json", "furniture.json", "bands.json", "page_roles.json",
                         "template.json", "validate.json"))
-    if prepare_ocr and not ensure_rapid_cache(p):
+    ex = ["extract.py"] + p["extract"] + ["--out", S]
        raise RuntimeError(f"unable to prepare B1 OCR cache for {p['slug']}")
    extract_args = p.get("extract") or ["--docling", p["docling"], "--rapid", p["rapid"], "--board", p.get("board", "aqa")]
    ex = ["extract.py"] + extract_args + ["--out", S]
    if p.get("pdf"):
        ex += ["--response-regions", p["pdf"]]
    if p.get("expected_max"):
        ex += ["--expected-max", str(p["expected_max"])]
    if p.get("gt"):
        ex += ["--gt", p["gt"]]
    run(ex)
@@ -252,7 +138,7 @@ def do_geometry(p, overlays, gt_labels=None, prepare_ocr=False):
        odbg = os.path.join(d, "overlays", "debug")
        run(["scripts/overlay.py", S, p["pdf"], "--docling", p["docling"], "--bands", B,
             "--furniture", F, "--pages", "1,2,3,4,5", "--dpi", "120", "--out", odbg])
-    return stats_from(jload(S), jload(V), gt_labels), d
+    return stats_from(jload(S), jload(V)), d
 def do_fast(p):
@@ -278,9 +164,6 @@ def per_paper_report(p, s, d, kind):
             + (f"  (missed {s['coverage_missed'][:8]})" if s.get('coverage_missed') else "")
             if s['coverage_pct'] is not None else "- **coverage vs GT:** n/a",
             f"- **G6 verdict:** {s['validate_verdict']}",
             f"- **answer-region count:** {s.get('answer_regions')}",
             f"- **opencv response regions:** {s.get('opencv_answer_regions')} attached / "
             f"{s.get('opencv_answer_region_candidates')} candidates",
             ]
    if s["validate_flags"]:
        lines += ["", "**Flags (human-review hints):**"] + [f"- {f}" for f in s["validate_flags"]]
@@ -295,28 +178,21 @@ def per_paper_report(p, s, d, kind):
 def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--no-overlays", action="store_true")
    ap.add_argument("--b1-only", action="store_true", help="run only the Sprint B1 image-only OCR eval corpus")
    ap.add_argument("--prepare-ocr", action="store_true", help="populate missing B1 RapidOCR caches via dsync before running")
    a = ap.parse_args()
    os.makedirs(FINAL, exist_ok=True)
    catalog = {"generated_at": datetime.datetime.now().isoformat(timespec="seconds"),
               "papers": []}
    total_imgs = 0
-    gt_fixtures = load_gt_labels()
+    for p in GEOMETRY:
    geometry = B1_GEOMETRY if a.b1_only else GEOMETRY
    fast = [] if a.b1_only else FAST
    for p in geometry:
        print(f"[geometry] {p['slug']}")
-        gt_labels = (gt_fixtures.get(p.get("gt_key") or p["slug"], {}) or {}).get("labels")
+        s, d = do_geometry(p, not a.no_overlays)
        s, d = do_geometry(p, not a.no_overlays, gt_labels=gt_labels, prepare_ocr=a.prepare_ocr)
        n = per_paper_report(p, s, d, p["path"])
        total_imgs += n
        catalog["papers"].append({**{k: p[k] for k in ("slug", "title", "board", "level")},
                                  "kind": "geometry", "path": p["path"], "dir": d,
                                  "overlay_images": n, **s})
-    for p in fast:
+    for p in FAST:
        print(f"[fast] {p['slug']}")
        s, d = do_fast(p)
        per_paper_report(p, s, d, "born-digital fast-path")
@@ -338,13 +214,13 @@ def write_index(catalog, total_imgs):
         "`overlays/template/` (human-review view, all pages) and `overlays/debug/` (raw-detection view).",
         "Machine catalog: `catalog.json`.", "",
         "## Image-only / OCR-path (with geometry + overlays)", "",
-         "| Paper | Board / level | Q/parts | Marks/max | Coverage | Answer regions | G6 | Images |",
+         "| Paper | Board / level | Q/parts | Marks/max | Coverage | G6 | Images |",
-         "|---|---|---|---|---|---|---|---|"]
+         "|---|---|---|---|---|---|---|"]
    for p in g:
        cov = f"{p['coverage_pct']}%" if p['coverage_pct'] is not None else "n/a"
        L.append(f"| [{p['title']}]({p['slug']}/report.md) | {p['board']} {p['level']} | "
                 f"{p['n_questions']}/{p['n_parts']} | {p['marks_sum']}/{p['official_max']} "
-                 f"({p['marks_pct']}%) | {cov} | {p.get('answer_regions')} | {p['validate_verdict']} | "
+                 f"({p['marks_pct']}%) | {cov} | {p['validate_verdict']} | "
                 f"{p['overlay_images']} |")
    L += ["", "## Born-digital fast-path (CPU, no geometry)", "",
          "| Paper | Board / level | Q/parts | Marks/max | Coverage | G6 |",
@@ -1,356 +0,0 @@
 {
  "b1-aqa-biology-7402-1-2023jun": {
    "source_pdf": "cc.examboards/aqa/biology/7402/1/2023-jun/qp.pdf",
    "source_method": "AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.",
    "board_detected": "aqa",
    "paper_code_detected": "7402/1",
    "labels": [
      "01.1",
      "01.2",
      "01.3",
      "02.1",
      "02.2",
      "02.3",
      "03.1",
      "03.2",
      "03.3",
      "03.4",
      "03.5",
      "04.1",
      "04.2",
      "04.3",
      "05.1",
      "05.2",
      "05.3",
      "05.4",
      "05.5",
      "06.1",
      "06.2",
      "06.3",
      "06.4",
      "07.1",
      "07.2",
      "89.6",
      "08.1",
      "08.2",
      "08.3",
      "08.4",
      "09.1",
      "09.2",
      "09.3",
      "09.4",
      "09.5",
      "09.6",
      "10.1",
      "10.2",
      "10.3"
    ]
  },
  "b1-aqa-chemistry-7405-1-2022jun": {
    "source_pdf": "cc.examboards/aqa/chemistry/7405/1/2022-jun/qp.pdf",
    "source_method": "AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.",
    "board_detected": "aqa",
    "paper_code_detected": "7405/1",
    "labels": [
      "01.1",
      "01.2",
      "01.3",
      "01.4",
      "01.5",
      "01.6",
      "02.1",
      "02.2",
      "02.3",
      "02.4",
      "02.5",
      "03.1",
      "03.2",
      "03.3",
      "03.4",
      "03.5",
      "04.1",
      "04.2",
      "04.3",
      "04.4",
      "04.5",
      "05.1",
      "05.2",
      "05.3",
      "05.4",
      "05.5",
      "05.6",
      "05.7",
      "06.1",
      "06.2",
      "06.3",
      "06.4",
      "06.5",
      "06.6",
      "06.7",
      "07.1",
      "07.2",
      "07.3",
      "07.4",
      "07.5",
      "07.6",
      "07.7",
      "08.1",
      "08.2",
      "08.3",
      "08.4",
      "08.5"
    ]
  },
  "b1-aqa-physics-7408-1-2022jun": {
    "source_pdf": "cc.examboards/aqa/physics/7408/1/2022-jun/qp.pdf",
    "source_method": "AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.",
    "board_detected": "aqa",
    "paper_code_detected": "7408/1",
    "labels": [
      "01.1",
      "01.2",
      "01.3",
      "01.4",
      "01.5",
      "02.1",
      "02.2",
      "02.3",
      "02.4",
      "03.1",
      "03.2",
      "03.3",
      "03.4",
      "03.5",
      "04.1",
      "04.2",
      "04.3",
      "04.4",
      "04.5",
      "05.1",
      "05.2",
      "05.3",
      "05.4",
      "05.5",
      "05.6",
      "06.1",
      "06.2",
      "06.3",
      "07.0",
      "08.0",
      "09.0",
      "10.0",
      "11.0",
      "12.0",
      "13.0",
      "14.0",
      "15.0",
      "16.0",
      "17.0",
      "18.0",
      "19.0",
      "20.0",
      "21.0",
      "22.0",
      "23.0",
      "24.0",
      "25.0",
      "26.0",
      "27.0",
      "28.0",
      "29.0",
      "30.0",
      "31.0"
    ]
  },
  "b1-aqa-biology-8461-1h-2022jun": {
    "source_pdf": "cc.examboards/aqa/biology/8461/1h/2022-jun/qp.pdf",
    "source_method": "AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.",
    "board_detected": "aqa",
    "paper_code_detected": "8461/1",
    "labels": [
      "01.1",
      "01.2",
      "01.3",
      "01.4",
      "01.5",
      "01.6",
      "01.7",
      "01.8",
      "01.9",
      "02.1",
      "02.2",
      "02.3",
      "02.4",
      "02.5",
      "02.6",
      "03.1",
      "03.2",
      "03.3",
      "03.4",
      "03.5",
      "04.1",
      "04.2",
      "04.3",
      "04.4",
      "04.5",
      "05.1",
      "05.2",
      "05.3",
      "05.4",
      "05.5",
      "06.1",
      "06.2",
      "06.3",
      "06.4",
      "06.5",
      "07.1",
      "07.2",
      "07.3",
      "07.4",
      "07.5",
      "07.6",
      "07.7",
      "07.8"
    ]
  },
  "b1-aqa-chemistry-8462-1h-2022jun": {
    "source_pdf": "cc.examboards/aqa/chemistry/8462/1h/2022-jun/qp.pdf",
    "source_method": "AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.",
    "board_detected": "aqa",
    "paper_code_detected": "8462/1",
    "labels": [
      "01.1",
      "01.2",
      "01.3",
      "01.4",
      "01.5",
      "01.6",
      "01.7",
      "02.1",
      "02.2",
      "02.3",
      "02.4",
      "02.5",
      "02.6",
      "03.1",
      "03.2",
      "03.3",
      "03.4",
      "03.5",
      "04.1",
      "04.2",
      "04.3",
      "04.4",
      "04.5",
      "04.6",
      "04.7",
      "05.1",
      "05.2",
      "05.3",
      "05.4",
      "05.5",
      "06.1",
      "06.2",
      "06.3",
      "06.4",
      "06.5",
      "06.6",
      "07.1",
      "07.2",
      "07.3",
      "07.4",
      "07.5",
      "07.6",
      "08.1",
      "08.2",
      "08.3",
      "08.4",
      "08.5"
    ]
  },
  "b1-aqa-combined-8464-b1h-2022jun": {
    "source_pdf": "cc.examboards/aqa/combined-science-trilogy/8464/b-1h/2022-jun/qp.pdf",
    "source_method": "AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.",
    "board_detected": "aqa",
    "paper_code_detected": null,
    "labels": [
      "01.1",
      "01.2",
      "01.3",
      "01.4",
      "01.5",
      "01.6",
      "01.7",
      "01.8",
      "02.1",
      "02.2",
      "02.3",
      "02.4",
      "02.5",
      "02.6",
      "02.7",
      "03.1",
      "03.2",
      "03.3",
      "03.4",
      "03.5",
      "03.6",
      "03.7",
      "04.1",
      "04.2",
      "04.3",
      "04.4",
      "04.5",
      "05.1",
      "05.2",
      "05.3",
      "05.4",
      "05.5",
      "05.6",
      "06.1",
      "06.2",
      "06.3"
    ]
  },
  "b1-aqa-combined-8464-c1h-2022jun": {
    "source_pdf": "cc.examboards/aqa/combined-science-trilogy/8464/c-1h/2022-jun/qp.pdf",
    "source_method": "AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.",
    "board_detected": "aqa",
    "paper_code_detected": null,
    "labels": [
      "01.1",
      "01.2",
      "01.3",
      "01.4",
      "01.5",
      "02.1",
      "02.2",
      "02.3",
      "02.4",
      "02.5",
      "03.0",
      "04.1",
      "04.2",
      "04.3",
      "04.4",
      "04.5",
      "04.6",
      "04.7",
      "05.1",
      "05.2",
      "05.3",
      "05.4",
      "06.1",
      "06.2",
      "06.3",
      "06.4",
      "06.5",
      "07.1",
      "07.2",
      "07.3",
      "07.4",
      "07.5",
      "07.6"
    ]
  }
 }
@@ -162,16 +162,7 @@ def detect_response_regions_from_pdf(
                page_index=page_index,
                min_confidence=min_confidence,
            )
-            for candidate in page_candidates:
+            candidates.extend(candidate.to_mapper_dict() for candidate in page_candidates)
                item = candidate.to_mapper_dict()
                item.setdefault("meta", {}).update({
                    "page_width_px": pix.width,
                    "page_height_px": pix.height,
                    "page_width_pdf": float(doc[page_index].rect.width),
                    "page_height_pdf": float(doc[page_index].rect.height),
                    "render_dpi": dpi,
                })
                candidates.append(item)
        return candidates
    finally:
        doc.close()
@@ -289,7 +280,7 @@ def _detect_answer_lines(binary: np.ndarray, *, page_index: int, width: int, hei
        span_ratio = box_w / max(width, 1)
        count_bonus = min(0.2, max(0, line_count - 1) * 0.05)
        confidence = min(0.92, 0.42 + span_ratio * 0.35 + count_bonus)
-        region_type = "answer_lines"
+        region_type = "answer_lines" if line_count > 1 else "working_space"
        candidates.append(
            RegionCandidate(
                page_index=page_index,
@@ -360,7 +351,6 @@ def _detect_answer_boxes(binary: np.ndarray, *, page_index: int, width: int, hei
        if rectangularity < 0.03:
            continue
        confidence = min(0.88, 0.46 + min(0.24, w / width * 0.24) + min(0.18, h / height * 0.5))
        region_type = "working_space" if (h > height * 0.12 and rectangularity < 0.18) else "answer_box"
        padded_x = max(0, x - 2)
        padded_y = max(0, y - 2)
        padded_right = min(width, x + w + 2)
@@ -372,7 +362,7 @@ def _detect_answer_boxes(binary: np.ndarray, *, page_index: int, width: int, hei
                y=padded_y,
                w=padded_right - padded_x,
                h=padded_bottom - padded_y,
-                region_type=region_type,
+                region_type="answer_box",
                confidence=confidence,
                detection_method="opencv_contour_box",
                meta={"rectangularity": round(float(rectangularity), 3)},
@@ -1,87 +0,0 @@
 #!/usr/bin/env python3
 """Populate the gitignored B1 image-only eval corpus from the .94 exam-board store.
 The B1 eval papers are NOT committed (third-party copyright; served only via signed URLs).
 This script downloads each B1_GEOMETRY paper's `storage_loc` object from cc.examboards via the
 Storage API into its local `pdf` path (under samples/b1/), so finalize.py --b1-only and the
 B1-2/B1-3 generalization work can run against a real corpus.
 Run from api/services/docling/ inside the cc-api-dev container (SUPABASE_URL/SERVICE_ROLE_KEY in env):
    python3 scripts/fetch_b1_corpus.py            # fetch all B1 papers (skip existing)
    python3 scripts/fetch_b1_corpus.py --force    # re-download
    python3 scripts/fetch_b1_corpus.py --only b1-aqa-physics-7408-1-2022jun
    python3 scripts/fetch_b1_corpus.py --list     # show what would be fetched, no download
 """
 from __future__ import annotations
 import argparse
 import os
 import sys
 # Import the canonical B1 corpus definition (slug, storage_loc, local pdf path) from finalize.
 _HERE = os.path.dirname(os.path.abspath(__file__))
 _DOCLING_DIR = os.path.dirname(_HERE)
 sys.path.insert(0, _DOCLING_DIR)
 from finalize import B1_GEOMETRY  # noqa: E402
 def _split_storage_loc(storage_loc: str) -> tuple[str, str]:
    """'cc.examboards/aqa/biology/7402/1/2023-jun/qp.pdf' -> ('cc.examboards', 'aqa/.../qp.pdf')."""
    bucket, _, path = storage_loc.partition("/")
    if not bucket or not path:
        raise ValueError(f"malformed storage_loc: {storage_loc!r}")
    return bucket, path
 def _entries(only: str | None):
    for p in B1_GEOMETRY:
        loc = p.get("storage_loc")
        pdf = p.get("pdf")
        if not loc or not pdf:
            continue
        if only and p.get("slug") != only:
            continue
        yield p["slug"], loc, pdf
 def main() -> int:
    ap = argparse.ArgumentParser(description="Fetch the B1 image-only eval corpus from .94 cc.examboards")
    ap.add_argument("--force", action="store_true", help="re-download even if the local file exists")
    ap.add_argument("--only", help="fetch a single paper by slug")
    ap.add_argument("--list", action="store_true", help="list what would be fetched and exit")
    args = ap.parse_args()
    todo = list(_entries(args.only))
    if not todo:
        print("no matching B1 papers", file=sys.stderr)
        return 1
    if args.list:
        for slug, loc, pdf in todo:
            print(f"{slug}\t{loc}\t-> {pdf}")
        return 0
    from modules.database.supabase.utils.storage import StorageAdmin
    storage = StorageAdmin()
    ok = skipped = 0
    for slug, loc, pdf in todo:
        dest = os.path.join(_DOCLING_DIR, pdf) if not os.path.isabs(pdf) else pdf
        if os.path.exists(dest) and not args.force:
            print(f"[skip] {slug} (exists)")
            skipped += 1
            continue
        bucket, path = _split_storage_loc(loc)
        data = storage.download_file(bucket, path)
        os.makedirs(os.path.dirname(dest), exist_ok=True)
        with open(dest, "wb") as fh:
            fh.write(data)
        print(f"[ok]   {slug} <- {bucket}/{path} ({len(data)} bytes)")
        ok += 1
    print(f"fetched {ok}, skipped {skipped}, of {len(todo)}")
    return 0
 if __name__ == "__main__":
    raise SystemExit(main())
@@ -1,32 +0,0 @@
 import json, sys
 from pathlib import Path
 base=Path('/app/api/services/docling')
 sys.path.insert(0, str(base))
 import extract
 papers=[
 ('b1-aqa-biology-7402-1-2023jun','samples/b1/aqa-biology-7402-1-2023jun.pdf','cc.examboards/aqa/biology/7402/1/2023-jun/qp.pdf'),
 ('b1-aqa-chemistry-7405-1-2022jun','samples/b1/aqa-chemistry-7405-1-2022jun.pdf','cc.examboards/aqa/chemistry/7405/1/2022-jun/qp.pdf'),
 ('b1-aqa-physics-7408-1-2022jun','samples/b1/aqa-physics-7408-1-2022jun.pdf','cc.examboards/aqa/physics/7408/1/2022-jun/qp.pdf'),
 ('b1-aqa-biology-8461-1h-2022jun','samples/b1/aqa-biology-8461-1h-2022jun.pdf','cc.examboards/aqa/biology/8461/1h/2022-jun/qp.pdf'),
 ('b1-aqa-chemistry-8462-1h-2022jun','samples/b1/aqa-chemistry-8462-1h-2022jun.pdf','cc.examboards/aqa/chemistry/8462/1h/2022-jun/qp.pdf'),
 ('b1-aqa-combined-8464-b1h-2022jun','samples/b1/aqa-combined-8464-b1h-2022jun.pdf','cc.examboards/aqa/combined-science-trilogy/8464/b-1h/2022-jun/qp.pdf'),
 ('b1-aqa-combined-8464-c1h-2022jun','samples/b1/aqa-combined-8464-c1h-2022jun.pdf','cc.examboards/aqa/combined-science-trilogy/8464/c-1h/2022-jun/qp.pdf'),
 ]
 out={}
 for slug, rel, storage in papers:
    lines=extract.lines_from_pdftext(str(base/rel))
    board, code=extract.detect_board(lines)
    if board != 'aqa':
        raise RuntimeError(f'{slug}: expected AQA board, detected {board!r} ({code!r})')
    parts=extract.parse_text_by_board(lines, board)
    labels=list(parts)
    out[slug]={
        'source_pdf': storage,
        'source_method': 'AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.',
        'board_detected': board,
        'paper_code_detected': code,
        'labels': labels,
    }
    print(slug, board, code, len(labels), labels[:5], labels[-5:])
 Path(base/'fixtures').mkdir(exist_ok=True)
 Path(base/'fixtures/b1_gt_labels.json').write_text(json.dumps(out, indent=2)+"\n")
@@ -1,69 +0,0 @@
 #!/usr/bin/env python3
 """
 rapid_pass.py — generalise the proven AQA "RapidOCR margin-pass" (95.2% on the image-only
 8463 paper) to any AQA paper. Born-digital AQA QPs ship a text layer, so we force RapidOCR
 over the *rendered* page (`force_ocr:true`) to simulate the image-only redistribution case
 and recover the boxed `NN.M` question numbers Tesseract shatters.
 For each page it writes results/<outdir>/p{N}.json (a full per-page DoclingDocument, the
 shape extract.py's aqa_questions_rapid expects) and a merged.json (for board / front-matter
 detection). All GPU work is serialised + OOM-resilient through dsync.
 Usage:
  python scripts/rapid_pass.py samples/extra/aqa-alevel-physics-7408-1-jun22-qp.pdf rapid_7408
  python scripts/rapid_pass.py <pdf> <outdir-slug> [first_page] [last_page]
 """
 import os, sys, json, subprocess, re
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 import dsync
 OPTS = {"ocr_engine": "rapidocr", "force_ocr": True}
 def npages(pdf):
    out = subprocess.check_output(["pdfinfo", pdf]).decode()
    return int(out.split("Pages:")[1].split()[0])
 def main():
    pdf = sys.argv[1]
    slug = sys.argv[2]
    if os.path.isabs(slug) or ".." in slug.split(os.sep) or not re.fullmatch(r"[A-Za-z0-9._/-]+", slug):
        raise SystemExit(f"unsafe output slug: {slug!r}")
    n = npages(pdf)
    first = int(sys.argv[3]) if len(sys.argv) > 3 else 1
    last = min(int(sys.argv[4]), n) if len(sys.argv) > 4 else n
    if first > n or first > last:
        print(f"requested page range {first}-{last} is outside PDF ({n} pages); nothing to do")
        return
    outdir = os.path.join("results", slug)
    os.makedirs(outdir, exist_ok=True)
    r = dsync._redis()
    print(f"redis: {'connected' if r else 'NO CACHE'}  pdf={pdf}  pages {first}-{last}/{n}")
    merged = {"texts": [], "tables": [], "pictures": [], "pages": {}, "_failed_pages": []}
    for pg in range(first, last + 1):
        page_path = os.path.join(outdir, f"p{pg}.json")
        if os.path.exists(page_path):
            doc = json.load(open(page_path))
            print(f"  p{pg}: file cache HIT ({len(doc.get(texts, []))} texts)")
        else:
            doc = dsync.convert_page(pdf, pg, OPTS, r=r)
        if not doc:
            merged["_failed_pages"].append(pg)
            print(f"  p{pg}: FAILED")
            continue
        json.dump(doc, open(page_path, "w"))
        for k in ("texts", "tables", "pictures"):
            merged[k].extend(doc.get(k, []))
        merged["pages"].update(doc.get("pages", {}))
        nmarg = sum(1 for t in doc.get("texts", [])
                    if (t.get("prov") or [{}])[0].get("bbox", {}).get("l", 999) <= 140)
        print(f"  p{pg}: {len(doc.get('texts', []))} texts ({nmarg} left-margin)")
    json.dump(merged, open(os.path.join(outdir, "merged.json"), "w"))
    print(f"-> {outdir}/  ({last-first+1-len(merged['_failed_pages'])} pages, "
          f"failed={merged['_failed_pages']})")
 if __name__ == "__main__":
    main()
@@ -159,7 +159,6 @@ def build(structured, bands, furniture, pdf=None, page_roles=None):
                "y_start": p["y_start"], "y_end": p["y_end"],
                "label_box": part_bbox.get(p["label"]),  # anchor, not the part extent
                "box": synthesize_part_box(p, xband),
                "marks": p.get("marks"),                 # parsed per-part marks (born-digital)
                "source": "auto", "confirmed": False,
            })
        pr = page_roles.get(pgs) or page_roles.get(pg) or {}
@@ -55,8 +55,6 @@ services:
      - CC_COMPOSE_SERVICE=backend-dev
      - RUN_INIT=false
      - INIT_MODE=infra
      # P2: route exam auto-map through the spike's full recognition pipeline (extraction service)
      - EXAM_EXTRACT_URL=${EXAM_EXTRACT_URL:-http://192.168.0.203:8899}
    ports:
      - "18000:8000"
    depends_on:
@@ -75,14 +75,6 @@ if [ "$RUN_INIT" = "true" ]; then
                }
                print_success "GAIS data import completed"
                ;;
            "exam-corpus")
                print_status "Seeding exam-paper corpus (manifest-gated; skips if none configured)..."
                python3 main.py --mode exam-corpus || {
                    print_error "Exam corpus seed failed!"
                    exit 1
                }
                print_success "Exam corpus seed completed"
                ;;
            "full")
                print_status "Running full initialization..."
                python3 main.py --mode infra || exit 1
@@ -323,52 +323,6 @@ def run_gais_data_mode():
 # Old clear_dev_redis_queue function removed - now handled by Redis Manager
 def run_exam_corpus_mode():
    """Seed the public exam-paper corpus from a manifest (optional, gated).
    Env controls:
      EXAM_CORPUS_MANIFEST    - path to the corpus manifest (required to do anything)
      EXAM_CORPUS_DRY_RUN     - 'true' to validate + report only
      EXAM_CORPUS_FORCE       - 'true' to re-upload/overwrite existing objects
      EXAM_CORPUS_BOARD/_SPEC - filter to one exam_board_code / spec_code
      EXAM_CORPUS_USER_SUBSET - 'true' to also seed a user-side test subset
      EXAM_CORPUS_FIRST_SWEEP - 'true' to run the docling/auto-map first pass
    Skips gracefully (success) when no manifest is configured/present, so it is safe
    in a comma-mode list (e.g. INIT_MODE=infra,seed,exam-corpus) before papers exist.
    Buckets are NOT created here — infra mode (buckets.py) owns provisioning.
    """
    logger.info("Running in exam-corpus seed mode")
    manifest = os.getenv("EXAM_CORPUS_MANIFEST")
    if not manifest or not os.path.exists(manifest):
        logger.warning(
            f"exam-corpus: no manifest at EXAM_CORPUS_MANIFEST={manifest!r}; skipping (nothing to seed yet)"
        )
        return True
    try:
        from run.initialization.seed_exam_corpus import load
        rep = load(
            manifest,
            dry_run=_truthy_env("EXAM_CORPUS_DRY_RUN"),
            force=_truthy_env("EXAM_CORPUS_FORCE"),
            board_filter=os.getenv("EXAM_CORPUS_BOARD") or None,
            spec_filter=os.getenv("EXAM_CORPUS_SPEC") or None,
            user_subset=_truthy_env("EXAM_CORPUS_USER_SUBSET"),
            do_first_sweep=_truthy_env("EXAM_CORPUS_FIRST_SWEEP"),
        )
        if rep.errors:
            logger.error(f"exam-corpus seed completed with {len(rep.errors)} error(s)")
            return False
        logger.info(
            f"exam-corpus seed ok: specs={rep.specs_upserted} papers={rep.papers_upserted} "
            f"uploaded={rep.files_uploaded}"
        )
        return True
    except Exception as e:
        logger.error(f"exam-corpus seed failed: {e}")
        return False
 def run_development_mode():
    """Run the server in development mode with auto-reload"""
    logger.info("Running in development mode")
@@ -457,7 +411,7 @@ Startup modes:
    parser.add_argument(
        '--mode', '-m',
-        choices=['infra', 'seed', 'seed-test', 'gais-data', 'exam-corpus', 'dev', 'prod'],
+        choices=['infra', 'seed', 'seed-test', 'gais-data', 'dev', 'prod'],
        default='dev',
        help='Startup mode (default: dev)'
    )
@@ -493,11 +447,6 @@ if __name__ == "__main__":
        success = run_gais_data_mode()
        sys.exit(0 if success else 1)
    elif args.mode == 'exam-corpus':
        # Seed the public exam-paper corpus from a manifest (gated; skips if none configured)
        success = run_exam_corpus_mode()
        sys.exit(0 if success else 1)
    elif args.mode == 'dev':
        # Run development server
        run_development_mode()
@@ -1,86 +0,0 @@
 """Client for the exam extraction SERVICE (docling-exam-spike, P1).
 The service runs the spike's FULL recognition pipeline (textlayer → question tree → OMR/figure/table
 sidecars → structure fusion → analyse) and returns the ghost-region contract the app already consumes.
 This module is a thin HTTP client: POST the paper, poll, return the `analyse` suggestions. The app's
 auto-map merges them (coordinate-adapted, id-remapped) in routers/exam/templates.py.
 Config: EXAM_EXTRACT_URL (e.g. http://192.168.0.203:8899). If unset, the app keeps its thin first-pass.
 """
 from __future__ import annotations
 import base64
 import os
 import time
 from typing import Any, Dict, Optional
 import requests
 from modules.logger_tool import initialise_logger
 logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), "default", True)
 class ExtractError(RuntimeError):
    pass
 def service_url() -> Optional[str]:
    url = os.getenv("EXAM_EXTRACT_URL")
    return url.rstrip("/") if url else None
 def is_enabled() -> bool:
    return bool(service_url())
 def _get(base: str, slug: str, timeout: int = 30) -> Dict[str, Any]:
    r = requests.get(f"{base}/api/extract/{slug}", timeout=timeout)
    r.raise_for_status()
    return r.json()
 def get_replica(slug: str, timeout: int = 30) -> Dict[str, Any]:
    """The digital-replica markdown for a paper (P4): {slug, title, n_questions, total_marks, markdown,
    questions:[{label, marks, markdown}]}. Raises ExtractError if the paper has no replica yet."""
    base = service_url()
    if not base:
        raise ExtractError("EXAM_EXTRACT_URL not configured")
    r = requests.get(f"{base}/api/replica/{slug}", timeout=timeout)
    if r.status_code == 404:
        raise ExtractError(f"no digital replica for {slug}")
    r.raise_for_status()
    return r.json()
 def extract_suggestions(slug: str, pdf_bytes: bytes, *, force: bool = False,
                        poll_timeout: int = 1500, poll_interval: int = 5) -> Dict[str, Any]:
    """POST the paper to the service and poll until the analyse contract is ready.
    Returns the full analyse payload: {status, coordinate_space:'page_fraction', margins,
    suggestions:{questions[], response_areas[], boundaries[]}, meta}. Raises ExtractError on
    failure/timeout. A cold paper is ~15 min (Docling per masked page); cached papers return instantly.
    """
    base = service_url()
    if not base:
        raise ExtractError("EXAM_EXTRACT_URL not configured")
    payload = {"slug": slug, "pdf_b64": base64.b64encode(pdf_bytes).decode(), "force": force}
    r = requests.post(f"{base}/api/extract", json=payload, timeout=120)
    r.raise_for_status()
    started = r.json()
    if not started.get("ok", True):
        raise ExtractError(started.get("error") or "service rejected the request")
    # cached → fetch the contract straight away; otherwise poll the running job
    deadline = time.time() + poll_timeout
    while True:
        d = _get(base, slug)
        status = d.get("status")
        if status == "complete":
            if not (d.get("suggestions") or {}):
                raise ExtractError("service returned complete with no suggestions")
            return d
        if status == "error":
            raise ExtractError(f"extraction failed: {d.get('error')}")
        if time.time() >= deadline:
            raise ExtractError(f"extraction timed out after {poll_timeout}s (slug={slug})")
        time.sleep(poll_interval)
@@ -1,99 +0,0 @@
 """Upload boundary validation shared by file-upload endpoints.
 E3 hardening: keep user-facing upload routes from buffering arbitrary data and
 from accepting arbitrary MIME/types into Supabase storage.
 """
 from __future__ import annotations
 import os
 from typing import Iterable, Optional
 from fastapi import HTTPException, UploadFile
 # Conservative defaults: Classroom Copilot uploads are user documents/images.
 # Exam scan uploads already have their own 50 MB PDF-only guard in routers.exam.batches.
 MAX_UPLOAD_BYTES = int(os.getenv("CC_UPLOAD_MAX_BYTES", str(25 * 1024 * 1024)))
 UPLOAD_CHUNK_BYTES = 1024 * 1024
 ALLOWED_UPLOAD_MIME_TYPES = frozenset(
    mt.strip().lower()
    for mt in os.getenv(
        "CC_UPLOAD_ALLOWED_MIME_TYPES",
        ",".join(
            [
                "application/pdf",
                "image/png",
                "image/jpeg",
                "image/webp",
                "image/gif",
                "text/plain",
                "text/csv",
                "text/markdown",
                "application/msword",
                "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
                "application/vnd.ms-powerpoint",
                "application/vnd.openxmlformats-officedocument.presentationml.presentation",
                "application/vnd.ms-excel",
                "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
            ]
        ),
    ).split(",")
    if mt.strip()
 )
 _PDF_MIME_TYPES = {"application/pdf", "application/x-pdf"}
 def allowed_upload_mime_types_csv() -> str:
    """Stable display string for evidence/errors without leaking config internals."""
    return ", ".join(sorted(ALLOWED_UPLOAD_MIME_TYPES))
 def _declared_mime(upload: UploadFile) -> str:
    return (upload.content_type or "application/octet-stream").split(";", 1)[0].strip().lower()
 def validate_upload_mime(upload: UploadFile, *, allowed_mime_types: Optional[Iterable[str]] = None) -> str:
    """Validate client-declared upload MIME/type and return its normalised value."""
    declared = _declared_mime(upload)
    allowed = {mt.lower() for mt in (allowed_mime_types or ALLOWED_UPLOAD_MIME_TYPES)}
    if declared not in allowed:
        raise HTTPException(
            status_code=415,
            detail=(
                f"Unsupported upload type '{declared}'. Allowed MIME types: "
                f"{', '.join(sorted(allowed))}"
            ),
        )
    return declared
 async def read_upload_bytes(
    upload: UploadFile,
    *,
    max_bytes: int = MAX_UPLOAD_BYTES,
    allowed_mime_types: Optional[Iterable[str]] = None,
 ) -> tuple[bytes, str]:
    """Validate MIME and read an UploadFile with a hard size ceiling."""
    mime_type = validate_upload_mime(upload, allowed_mime_types=allowed_mime_types)
    chunks: list[bytes] = []
    total = 0
    while True:
        chunk = await upload.read(UPLOAD_CHUNK_BYTES)
        if not chunk:
            break
        total += len(chunk)
        if total > max_bytes:
            raise HTTPException(status_code=413, detail=f"Upload exceeds max size ({max_bytes} bytes)")
        chunks.append(chunk)
    return b"".join(chunks), mime_type
 async def read_pdf_upload_bytes(upload: UploadFile, *, max_bytes: int = MAX_UPLOAD_BYTES) -> bytes:
    """Read a PDF-only upload with size and lightweight magic-header validation."""
    data, _mime_type = await read_upload_bytes(upload, max_bytes=max_bytes, allowed_mime_types=_PDF_MIME_TYPES)
    if not data:
        raise HTTPException(status_code=400, detail="Uploaded PDF is empty")
    if not data.startswith(b"%PDF-"):
        raise HTTPException(status_code=415, detail="Uploaded file is not a valid PDF")
    return data
@@ -12,7 +12,6 @@ from modules.auth.supabase_bearer import SupabaseBearer, verify_supabase_jwt_str
 from modules.logger_tool import initialise_logger
 from modules.database.supabase.utils.client import SupabaseServiceRoleClient
 from modules.database.supabase.utils.storage import StorageAdmin
 from modules.upload_validation import read_upload_bytes
 from modules.document_processor import DocumentProcessor
 from modules.queue_system import (
    enqueue_tika_task, enqueue_docling_task, enqueue_split_map_task,
@@ -37,24 +36,6 @@ DOCLING_NOOCR_TIMEOUT = int(os.getenv('DOCLING_NOOCR_TIMEOUT', '3600'))  # 1 hou
 logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
 def _user_id_from_payload(payload: Dict[str, Any]) -> str:
    user_id = payload.get('sub') or payload.get('user_id')
    if not user_id:
        raise HTTPException(status_code=401, detail="Invalid token payload")
    return user_id
 def _cabinet_visible_to_user(client: SupabaseServiceRoleClient, cabinet_id: str, user_id: str) -> bool:
    """Require cabinet ownership before service-role reads file metadata."""
    owned = (
        client.supabase.table('file_cabinets')
        .select('id')
        .eq('id', cabinet_id)
        .eq('user_id', user_id)
        .limit(1)
        .execute()
    )
    return bool(owned.data)
 def _safe_filename(name: str) -> str:
    base = os.path.basename(name or 'file')
    return re.sub(r"[^A-Za-z0-9._-]+", "_", base)
@@ -89,13 +70,13 @@ async def upload_file(
    # Stage DB row to get file_id
    staged_path = f"{cabinet_id}/staging/{uuid.uuid4()}"
    name = _safe_filename(path or file.filename)
-    file_bytes, mime_type = await read_upload_bytes(file)
+    file_bytes = await file.read()
    insert_res = client.supabase.table('files').insert({
        'cabinet_id': cabinet_id,
        'name': name,
        'path': staged_path,
        'bucket': bucket,
-        'mime_type': mime_type,
+        'mime_type': file.content_type,
        'uploaded_by': user_id,
        'size_bytes': len(file_bytes),
        'source': 'classroomcopilot-web'
@@ -108,7 +89,7 @@ async def upload_file(
    # Final storage path: bucket/cabinet_id/file_id/file
    final_storage_path = f"{cabinet_id}/{file_id}/{name}"
    try:
-        storage.upload_file(bucket, final_storage_path, file_bytes, mime_type, upsert=True)
+        storage.upload_file(bucket, final_storage_path, file_bytes, file.content_type or 'application/octet-stream', upsert=True)
    except Exception as e:
        # cleanup staged row
        client.supabase.table('files').delete().eq('id', file_id).execute()
@@ -136,10 +117,7 @@ async def upload_file(
@router.get("/files")
 def list_files(cabinet_id: str, payload: Dict[str, Any] = Depends(auth)):
    user_id = _user_id_from_payload(payload)
    client = SupabaseServiceRoleClient()
    if not _cabinet_visible_to_user(client, cabinet_id, user_id):
        return []
    res = client.supabase.table('files').select('*').eq('cabinet_id', cabinet_id).execute()
    return res.data
@@ -19,7 +19,6 @@ from fastapi.responses import JSONResponse
 from modules.auth.supabase_bearer import SupabaseBearer
 from modules.database.supabase.utils.client import SupabaseServiceRoleClient
 from modules.database.supabase.utils.storage import StorageAdmin
 from modules.upload_validation import read_upload_bytes
 from modules.logger_tool import initialise_logger
 router = APIRouter()
@@ -27,24 +26,6 @@ auth = SupabaseBearer()
 logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
 def _user_id_from_payload(payload: Dict[str, Any]) -> str:
    user_id = payload.get('sub') or payload.get('user_id')
    if not user_id:
        raise HTTPException(status_code=401, detail="Invalid token payload")
    return user_id
 def _cabinet_visible_to_user(client: SupabaseServiceRoleClient, cabinet_id: str, user_id: str) -> bool:
    """Require cabinet ownership before service-role reads file metadata."""
    owned = (
        client.supabase.table('file_cabinets')
        .select('id')
        .eq('id', cabinet_id)
        .eq('user_id', user_id)
        .limit(1)
        .execute()
    )
    return bool(owned.data)
 def _choose_bucket(scope: str, user_id: str, school_id: Optional[str]) -> str:
    """Choose appropriate bucket based on scope - matches old system logic."""
    scope = (scope or 'teacher').lower()
@@ -73,9 +54,10 @@ async def upload_file(
        if not user_id:
            raise HTTPException(status_code=401, detail="User ID required")
-        # Validate MIME/type and read file content with a hard size limit.
+        # Read file content
-        file_bytes, mime_type = await read_upload_bytes(file)
+        file_bytes = await file.read()
        file_size = len(file_bytes)
        mime_type = file.content_type or 'application/octet-stream'
        filename = file.filename or path
        logger.info(f"📤 Simplified upload: {filename} ({file_size} bytes) for user {user_id}")
@@ -152,10 +134,7 @@ async def upload_file(
@router.get("/files")
 def list_files(cabinet_id: str, payload: Dict[str, Any] = Depends(auth)):
    """List files in a cabinet."""
    user_id = _user_id_from_payload(payload)
    client = SupabaseServiceRoleClient()
    if not _cabinet_visible_to_user(client, cabinet_id, user_id):
        return []
    res = client.supabase.table('files').select('*').eq('cabinet_id', cabinet_id).execute()
    return res.data
@@ -126,28 +126,13 @@ async def platform_stats(
@router.post("/reset")
 async def reset_environment(
    scope: str = "all",
    _: dict = Depends(require_platform_admin),
 ) -> Dict[str, Any]:
-    """DESTRUCTIVE: wipe test data. Platform admin only.
+    """DESTRUCTIVE: wipe all test data. Neo4j + Supabase. Platform admin only."""
    scope (query param):
      - all          : full wipe (Neo4j + Supabase data + auth users) AND the entire
                       exam-marker subsystem below.
      - exam-corpus  : ONLY the entire exam-marker subsystem, not just public papers:
                       public corpus/eb_* data, cc.examboards storage objects, exam
                       templates, template layouts, questions, boundaries, response
                       areas, marking batches, student submissions, and mark entries
                       (without touching schools/users).
      - timetable    : ONLY timetable/calendar materialization tables.
    """
    if scope not in ("all", "exam-corpus", "timetable"):
        raise HTTPException(status_code=400, detail="scope must be one of: all, exam-corpus, timetable")
    import asyncio
    import functools
    from run.initialization.reset_environment import reset as _reset
    loop = asyncio.get_event_loop()
-    result = await loop.run_in_executor(None, functools.partial(_reset, scope))
+    result = await loop.run_in_executor(None, _reset)
    return {"status": "ok", **result}
@@ -8,13 +8,9 @@ from fastapi import APIRouter
 from routers.exam.templates import router as templates_router
 from routers.exam.batches import router as batches_router
 from routers.exam.bank import router as bank_router
 from routers.exam.corpus import router as corpus_router
 router = APIRouter()
 router.include_router(templates_router)
 router.include_router(batches_router)
 router.include_router(bank_router)
 router.include_router(corpus_router)
 __all__ = ["router"]
@@ -1,152 +0,0 @@
 """Question bank + custom-paper assembly (/api/exam/bank, /api/exam/custom-papers) — mode 3.
 Build-your-own-from-the-spec. The BANK is a browsable index over the leaf questions across the templates
 the caller can see (RLS-scoped, institute-wide); a CUSTOM PAPER is a normal exam_template whose questions
 are COPIES of the selected bank questions, so it flows through the existing setup / marking / projection
 pipeline unchanged. Copy-on-assemble (v1) avoids decoupling question identity from a template; a shared-
 question model is Phase-2. Design: ~/cc/ideas/2026-07-02-mode3-question-bank-design.md.
 All access is as-the-user (E1): the bank query returns only questions in templates RLS lets the caller see,
 and a custom paper is created owned by the caller + institute-scoped (writes pass the same RLS with-check).
 """
 from __future__ import annotations
 import os
 import uuid
 from typing import Any, Dict, List, Optional
 from fastapi import APIRouter, Depends, HTTPException
 from pydantic import BaseModel
 from modules.database.services.exam_projection import project_template_safe
 from modules.logger_tool import initialise_logger
 from routers.exam.dependencies import ExamContext, get_exam_context
 logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), "default", True)
 router = APIRouter()
 def _rows(result: Any) -> List[Dict[str, Any]]:
    data = getattr(result, "data", None)
    if not data:
        return []
    return data if isinstance(data, list) else [data]
 def _first(result: Any) -> Optional[Dict[str, Any]]:
    rows = _rows(result)
    return rows[0] if rows else None
@router.get("/bank")
 async def list_bank(
    spec_ref: Optional[str] = None,
    subject: Optional[str] = None,
    ctx: ExamContext = Depends(get_exam_context),
 ) -> Dict[str, Any]:
    """Leaf questions across the caller's institute templates, with their source-paper context.
    Filter by `spec_ref` (spec point) and/or `subject`. Facets over the full visible set drive the UI
    filters (so filtering by one axis doesn't hide the others' options). This is the spec-planning surface:
    a teacher picks a spec point → gets every question tagged it across their papers → assembles a paper.
    """
    sel = (
        "id, template_id, label, max_marks, answer_type, spec_ref, bounds, page, "
        "exam_templates(id, title, subject, exam_code)"
    )
    query = ctx.supabase.table("exam_questions").select(sel).eq("is_container", False)
    if spec_ref:
        query = query.eq("spec_ref", spec_ref)
    rows = _rows(query.execute())
    facets_spec: Dict[str, int] = {}
    facets_subject: Dict[str, int] = {}
    items: List[Dict[str, Any]] = []
    for r in rows:
        tmpl = r.get("exam_templates") or {}
        subj = tmpl.get("subject")
        if r.get("spec_ref"):
            facets_spec[r["spec_ref"]] = facets_spec.get(r["spec_ref"], 0) + 1
        if subj:
            facets_subject[subj] = facets_subject.get(subj, 0) + 1
        if subject and subj != subject:
            continue
        items.append({
            "id": r["id"], "template_id": r["template_id"], "label": r.get("label"),
            "max_marks": r.get("max_marks"), "answer_type": r.get("answer_type"),
            "spec_ref": r.get("spec_ref"), "bounds": r.get("bounds"), "page": r.get("page"),
            "paper": {"id": tmpl.get("id"), "title": tmpl.get("title"),
                      "subject": subj, "exam_code": tmpl.get("exam_code")},
        })
    return {"questions": items, "n": len(items),
            "facets": {"spec_ref": facets_spec, "subject": facets_subject}}
 class CustomPaperRequest(BaseModel):
    title: str
    subject: Optional[str] = None
    institute_id: Optional[str] = None
    question_ids: List[str]
@router.post("/custom-papers")
 async def create_custom_paper(
    body: CustomPaperRequest,
    ctx: ExamContext = Depends(get_exam_context),
 ) -> Dict[str, Any]:
    """Assemble the selected bank questions into a new template (copy-on-assemble), then project it."""
    if not body.question_ids:
        raise HTTPException(status_code=400, detail="question_ids is required")
    institute_id = ctx.resolve_institute(body.institute_id)
    # RLS makes this return only leaf questions the caller may see; unknown/forbidden ids are silently dropped.
    srcs = _rows(
        ctx.supabase.table("exam_questions").select("*")
        .in_("id", body.question_ids).eq("is_container", False).execute()
    )
    by_id = {s["id"]: s for s in srcs}
    ordered = [by_id[qid] for qid in body.question_ids if qid in by_id]  # preserve the caller's order
    if not ordered:
        raise HTTPException(status_code=404, detail="No accessible questions for the given ids")
    template_id = str(uuid.uuid4())
    ctx.supabase.table("exam_templates").insert({
        "id": template_id, "title": body.title, "subject": body.subject,
        "institute_id": institute_id, "teacher_id": ctx.user_id, "status": "draft",
    }).execute()
    id_map: Dict[str, str] = {}
    q_rows: List[Dict[str, Any]] = []
    for order, s in enumerate(ordered):
        new_id = str(uuid.uuid4())
        id_map[s["id"]] = new_id
        q_rows.append({
            "id": new_id, "template_id": template_id, "parent_id": None,
            "label": s.get("label"), "order": order, "max_marks": s.get("max_marks") or 0,
            "answer_type": s.get("answer_type"), "mcq_options": s.get("mcq_options"),
            "mark_scheme": s.get("mark_scheme") or {}, "is_container": False,
            "spec_ref": s.get("spec_ref"), "bounds": s.get("bounds"), "page": s.get("page"),
            "source": "manual", "confirmed": True,
        })
    ctx.supabase.table("exam_questions").insert(q_rows).execute()
    # Copy each selected question's response areas onto its new copy (geometry travels with the question).
    ra_rows: List[Dict[str, Any]] = []
    if id_map:
        for ra in _rows(
            ctx.supabase.table("exam_response_areas").select("*")
            .in_("question_id", list(id_map.keys())).execute()
        ):
            ra_rows.append({
                "id": str(uuid.uuid4()), "question_id": id_map[ra["question_id"]], "template_id": template_id,
                "page": ra.get("page"), "bounds": ra.get("bounds"), "kind": ra.get("kind"),
                "response_form": ra.get("response_form"), "context_type": ra.get("context_type"),
                "source": "manual", "confirmed": True,
            })
        if ra_rows:
            ctx.supabase.table("exam_response_areas").insert(ra_rows).execute()
    project_template_safe(template_id)  # a custom paper is a normal paper in the graph
    return {"id": template_id, "title": body.title, "subject": body.subject,
            "n_questions": len(q_rows), "n_response_areas": len(ra_rows)}
@@ -245,36 +245,6 @@ async def batch_csv(
 # ─── marks ───────────────────────────────────────────────────────────────────
 def _advance_completion(ctx: ExamContext, batch_id: str, submission_id: str) -> None:
    """After a mark upsert, advance statuses: a submission with a mark for every markable (leaf)
    question → complete; a batch whose every non-absent submission is complete → complete. Nothing
    here regresses a status (only promotes to complete), so it is safe to run on every upsert."""
    batch = _first(
        ctx.supabase.table("marking_batches").select("id, template_id, status").eq("id", batch_id).limit(1).execute()
    )
    if not batch:
        return
    markable = {
        q["id"] for q in _rows(
            ctx.supabase.table("exam_questions").select("id, is_container").eq("template_id", batch["template_id"]).execute()
        ) if not q.get("is_container")
    }
    if not markable:
        return
    marked = {
        m["question_id"] for m in _rows(
            ctx.supabase.table("mark_entries").select("question_id").eq("submission_id", submission_id).execute()
        )
    }
    if not markable.issubset(marked):
        return
    ctx.supabase.table("student_submissions").update({"status": "complete"}).eq("id", submission_id).execute()
    subs = _rows(ctx.supabase.table("student_submissions").select("status").eq("batch_id", batch_id).execute())
    active = [s for s in subs if s.get("status") != "absent"]
    if active and all(s.get("status") == "complete" for s in active) and batch.get("status") != "complete":
        ctx.supabase.table("marking_batches").update({"status": "complete"}).eq("id", batch_id).execute()
@router.put("/marks/{mark_id}")
 async def upsert_mark(
    mark_id: str,
@@ -289,15 +259,6 @@ async def upsert_mark(
    if not submission:
        raise HTTPException(status_code=404, detail="Submission not found")
    # Reject an award that exceeds the question's max (only when a max is actually set; 0/None means
    # "not scored yet" for AI/unmapped questions, so we can't validate those).
    question = _first(
        ctx.supabase.table("exam_questions").select("id, max_marks").eq("id", body.question_id).limit(1).execute()
    )
    max_marks = (question or {}).get("max_marks")
    if isinstance(max_marks, (int, float)) and max_marks > 0 and body.awarded_marks is not None and body.awarded_marks > max_marks:
        raise HTTPException(status_code=422, detail=f"awarded_marks {body.awarded_marks} exceeds max_marks {max_marks} for this question")
    row = {
        "id": mark_id,
        "submission_id": body.submission_id,
@@ -324,9 +285,6 @@ async def upsert_mark(
    if submission.get("status") in ("absent", "unmatched"):
        ctx.supabase.table("student_submissions").update({"status": "marking"}).eq("id", body.submission_id).execute()
    # Promote the submission/batch to complete once every markable question has a mark.
    _advance_completion(ctx, submission["batch_id"], body.submission_id)
    return upserted
@@ -1,98 +0,0 @@
 """Exam-bank corpus coverage (/api/exam/corpus) — the state of the collected exam bank.
 Read-only view over the seeded exam-board catalogue (eb_specifications + eb_exams): board → subject →
 specification → papers, with per-session QP/MS/ER coverage and rollup counts. Shows what the app has
 COLLECTED (question papers, mark schemes, examiner reports) so a teacher/admin can see the bank's state
 and where science coverage is complete vs thin. Catalogue data is public reference — read as-the-user.
 """
 from __future__ import annotations
 import os
 import re
 from typing import Any, Dict, List, Optional
 from fastapi import APIRouter, Depends
 from modules.logger_tool import initialise_logger
 from routers.exam.dependencies import ExamContext, get_exam_context
 logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), "default", True)
 router = APIRouter()
 DOC_TYPES = ("QP", "MS", "ER")
 def _rows(result: Any) -> List[Dict[str, Any]]:
    data = getattr(result, "data", None)
    if not data:
        return []
    return data if isinstance(data, list) else [data]
 def _award_level(spec: Dict[str, Any]) -> str:
    """Best-effort GCSE / AS / A-level from the spec code (AQA GCSE = 8xxx, A-level/AS = 7xxx)."""
    code = re.sub(r"\D", "", spec.get("award_code") or spec.get("spec_code") or "")
    if code.startswith("8"):
        return "GCSE"
    if code.startswith("7"):
        return "A-level"
    return spec.get("award_level") or "Other"
@router.get("/corpus")
 async def corpus_coverage(ctx: ExamContext = Depends(get_exam_context)) -> Dict[str, Any]:
    specs = _rows(
        ctx.supabase.table("eb_specifications")
        .select("spec_code, exam_board_code, subject_code, award_code, first_teach").execute()
    )
    exams = _rows(
        ctx.supabase.table("eb_exams")
        .select("exam_code, spec_code, paper_code, tier, session, type_code, storage_loc").execute()
    )
    # group exam docs → per spec → per paper (paper_code + session) → which doc types are present
    by_spec: Dict[str, Dict[str, Dict[str, Any]]] = {}
    for e in exams:
        sc = e.get("spec_code")
        if not sc:
            continue
        key = f"{e.get('paper_code') or '?'}|{e.get('session') or '?'}"
        paper = by_spec.setdefault(sc, {}).setdefault(key, {
            "paper_code": e.get("paper_code"), "session": e.get("session"),
            "tier": e.get("tier"), "docs": {}, "exam_codes": {},
        })
        dt = (e.get("type_code") or "").upper()
        if dt in DOC_TYPES:
            paper["docs"][dt] = bool(e.get("storage_loc"))
            paper["exam_codes"][dt] = e.get("exam_code")
    totals = {"specs": 0, "papers": 0, "sessions": 0, **{d: 0 for d in DOC_TYPES}}
    boards: Dict[str, Dict[str, Any]] = {}
    for s in specs:
        sc = s["spec_code"]
        papers_map = by_spec.get(sc, {})
        if not papers_map:
            continue
        totals["specs"] += 1
        board = s.get("exam_board_code") or "?"
        level = _award_level(s)
        papers = sorted(papers_map.values(), key=lambda p: (str(p["session"]), str(p["paper_code"])))
        counts = {d: sum(1 for p in papers if p["docs"].get(d)) for d in DOC_TYPES}
        for d in DOC_TYPES:
            totals[d] += counts[d]
        totals["papers"] += len(papers)
        totals["sessions"] += len({p["session"] for p in papers})
        spec_entry = {
            "spec_code": sc, "subject": (s.get("subject_code") or "").title(), "level": level,
            "board": board, "first_teach": s.get("first_teach"),
            "n_papers": len(papers), "counts": counts, "papers": papers,
        }
        boards.setdefault(board, {"board": board, "specs": []})["specs"].append(spec_entry)
    board_list = []
    for board in sorted(boards):
        specs_sorted = sorted(boards[board]["specs"], key=lambda x: (x["level"], x["subject"], x["spec_code"]))
        board_list.append({"board": board, "n_specs": len(specs_sorted), "specs": specs_sorted})
    return {"totals": totals, "boards": board_list}
@@ -80,9 +80,6 @@ class ResponseAreaPayload(BaseModel):
    ] = None
    # Optional Context differentiation (v1 generic; future graph/chart/data_table/diagram/code_block/passage).
    context_type: Optional[str] = None
    # Rich recognition payload (75-exam-marker-region-meta.sql): figure name/description, OMR geometry, unit…
    # Carried on canvas save so a named context figure survives a round-trip.
    meta: Optional[Dict[str, Any]] = None
    source: Literal["manual", "ai"] = "manual"
    confirmed: bool = True
    confidence: Optional[float] = Field(default=None, ge=0, le=1)
@@ -13,9 +13,7 @@ join keys (spec §2).
 from __future__ import annotations
 import json
 import math
 import os
 import re
 import tempfile
 import time
 import uuid
@@ -30,8 +28,6 @@ from api.services.docling.regions import detect_response_regions_from_pdf
 from modules.database.services.exam_projection import project_template, project_template_safe
 from modules.database.supabase.utils.client import SupabaseServiceRoleClient
 from modules.database.supabase.utils.storage import StorageAdmin
 from modules.upload_validation import read_pdf_upload_bytes
 from modules.services import exam_extract
 from modules.logger_tool import initialise_logger
 from routers.exam.dependencies import ExamContext, get_exam_context, lookup_exam_code
 from routers.exam.schemas import (
@@ -140,22 +136,6 @@ def _lookup_exam_storage_loc(exam_id: str) -> Optional[str]:
        return None
 def _signed_url_value(result: Any) -> str:
    """Normalise supabase-py signed URL responses across v1/v2 shapes."""
    if isinstance(result, str):
        return result
    if isinstance(result, dict):
        value = result.get("signedURL") or result.get("signedUrl") or result.get("signed_url")
        if value:
            return str(value)
    data = getattr(result, "data", None)
    if isinstance(data, dict):
        value = data.get("signedURL") or data.get("signedUrl") or data.get("signed_url")
        if value:
            return str(value)
    raise ValueError("Storage service did not return a signed URL")
 async def _parse_create_template_request(request: Request) -> tuple[CreateTemplateRequest, Optional[UploadFile]]:
    content_type = request.headers.get("content-type", "")
    if "multipart/form-data" in content_type:
@@ -184,7 +164,11 @@ async def _upload_template_source_file(
    institute_id: str,
    upload: UploadFile,
 ) -> str:
-    file_bytes = await read_pdf_upload_bytes(upload)
+    file_bytes = await upload.read()
    if not file_bytes:
        raise HTTPException(status_code=400, detail="Uploaded PDF is empty")
    if upload.content_type and upload.content_type != "application/pdf":
        raise HTTPException(status_code=400, detail="Uploaded file must be a PDF")
    service = SupabaseServiceRoleClient()
    storage = StorageAdmin()
@@ -345,13 +329,6 @@ def _pdf_has_text_layer(pdf_bytes: bytes) -> bool:
            pass
 # Canvas page width the frontend renders each PDF page at (app src/utils/exam-canvas/model.ts
 # PAGE_WIDTH). All auto-map canvas coords are emitted in this 780-wide, proportional-height space.
 CANVAS_PAGE_WIDTH = 780.0
 # Response/answer-region detector (api/services/docling/regions.py) renders at 144 DPI = 2 px / PDF point.
 REGIONS_PX_PER_PT = 2.0
 def _pdf_page_geometry(pdf_bytes: bytes) -> List[Dict[str, float]]:
    with tempfile.NamedTemporaryFile(prefix="cc-auto-map-geom-", suffix=".pdf", delete=False) as fh:
        fh.write(pdf_bytes)
@@ -365,23 +342,14 @@ def _pdf_page_geometry(pdf_bytes: bytes) -> List[Dict[str, float]]:
            for page in doc:
                media = page.mediabox
                crop = page.cropbox
-                page_pt_w = float(crop.width or page.rect.width or 1.0)
+                rendered_w = float(crop.width or page.rect.width or 595.0)
-                page_pt_h = float(crop.height or page.rect.height or 1.0)
+                rendered_h = float(crop.height or page.rect.height or 842.0)
                # Emit canvas coords in the FRONTEND render space: the app draws each page at
                # CANVAS_PAGE_WIDTH (app model.ts PAGE_WIDTH=780) with proportional height and stacks
                # pages by those heights. Previously rendered_w/h were left in PDF points (~595x842),
                # so every shape landed shrunk (~0.76x) and shifted up-left on the 780-wide canvas.
                rendered_w = CANVAS_PAGE_WIDTH
                # Mirror the app's canvas.height = Math.ceil(viewport.height) EXACTLY (pdfLoader.ts),
                # so page_top accumulates identically. Using the raw float drifts ~1px/page, compounding
                # to a visible upward shift on later pages of long papers (~36px over 40 pages).
                rendered_h = float(math.ceil(CANVAS_PAGE_WIDTH * page_pt_h / page_pt_w))
                pages.append({
                    "media_x0": float(media.x0),
                    "crop_x0": float(crop.x0),
                    "crop_y0": float(crop.y0),
-                    "page_pt_w": page_pt_w,
+                    "page_pt_w": float(crop.width or page.rect.width or 1),
-                    "page_pt_h": page_pt_h,
+                    "page_pt_h": float(crop.height or page.rect.height or 1),
                    "rendered_w": rendered_w,
                    "rendered_h": rendered_h,
                    "page_top": page_top,
@@ -403,12 +371,11 @@ def _pdf_page_geometry(pdf_bytes: bytes) -> List[Dict[str, float]]:
 def _page_geom(pages: List[Dict[str, float]], page_number: int) -> Dict[str, float]:
    if 1 <= page_number <= len(pages):
        return pages[page_number - 1]
    _fallback_h = float(math.ceil(CANVAS_PAGE_WIDTH * 842.0 / 595.0))
    return {
        "media_x0": 0.0, "crop_x0": 0.0, "crop_y0": 0.0,
        "page_pt_w": 595.0, "page_pt_h": 842.0,
-        "rendered_w": CANVAS_PAGE_WIDTH, "rendered_h": _fallback_h,
+        "rendered_w": 595.0, "rendered_h": 842.0,
-        "page_top": (page_number - 1) * _fallback_h,
+        "page_top": (page_number - 1) * 842.0,
    }
@@ -417,16 +384,12 @@ def _box_to_canvas(box: Optional[Dict[str, Any]], page_number: int, pages: List[
        return None
    g = _page_geom(pages, page_number)
    if box.get("coord_origin") == "TOPLEFT" and {"x", "y", "w", "h"}.issubset(box):
-        # Scale the box into the 780-wide canvas space. px boxes (opencv/gemma regions) are in
+        scale = 0.5 if box.get("unit") == "px" else 1.0
        # rendered-image px at REGIONS_PX_PER_PT px/point; TOPLEFT point boxes are 1 px/point.
        px_per_pt = REGIONS_PX_PER_PT if box.get("unit") == "px" else 1.0
        sx = g["rendered_w"] / (g["page_pt_w"] * px_per_pt)
        sy = g["rendered_h"] / (g["page_pt_h"] * px_per_pt)
        return {
-            "x": round(float(box["x"]) * sx, 2),
+            "x": round(float(box["x"]) * scale, 2),
-            "y": round(g["page_top"] + float(box["y"]) * sy, 2),
+            "y": round(g["page_top"] + float(box["y"]) * scale, 2),
-            "w": round(float(box["w"]) * sx, 2),
+            "w": round(float(box["w"]) * scale, 2),
-            "h": round(float(box["h"]) * sy, 2),
+            "h": round(float(box["h"]) * scale, 2),
        }
    if not {"l", "t", "r", "b"}.issubset(box):
        return None
@@ -457,32 +420,6 @@ def _y_to_canvas(y_value: float, page_number: int, pages: List[Dict[str, float]]
    return round(g["page_top"] + (g["page_pt_h"] - (float(y_value) - g["crop_y0"])) / g["page_pt_h"] * g["rendered_h"], 2)
 def _frac_box_to_canvas(bounds: Optional[Dict[str, Any]], page_number: int,
                        pages: List[Dict[str, float]]) -> Optional[Dict[str, float]]:
    """Page-fraction {x,y,w,h} (0..1 per page, from the extraction service) → 780-wide stacked canvas."""
    if not bounds:
        return None
    g = _page_geom(pages, page_number)
    try:
        x, y, w, h = (float(bounds["x"]), float(bounds["y"]), float(bounds["w"]), float(bounds["h"]))
    except (KeyError, TypeError, ValueError):
        return None
    return {
        "x": round(x * g["rendered_w"], 2),
        "y": round(g["page_top"] + y * g["rendered_h"], 2),
        "w": round(w * g["rendered_w"], 2),
        "h": round(h * g["rendered_h"], 2),
    }
 def _frac_y_to_canvas(y_frac: Any, page_number: int, pages: List[Dict[str, float]]) -> Optional[float]:
    g = _page_geom(pages, page_number)
    try:
        return round(g["page_top"] + float(y_frac) * g["rendered_h"], 2)
    except (TypeError, ValueError):
        return None
 def _ai_id(template_id: str, *parts: Any) -> str:
    return str(uuid.uuid5(uuid.NAMESPACE_URL, "/".join(["cc-auto-map", template_id, *[str(p) for p in parts]])))
@@ -493,17 +430,6 @@ def _safe_confidence(value: Any = None) -> float:
    return 0.75
 def _safe_marks(value: Any = None) -> int:
    """Parsed per-part marks → a non-negative int; unknown/None → 0 (image-only OCR has no marks yet)."""
    if isinstance(value, bool):
        return 0
    if isinstance(value, (int, float)):
        return max(0, int(value))
    if isinstance(value, str) and value.strip().isdigit():
        return int(value.strip())
    return 0
 def _margin_values(first_pass: Dict[str, Any], page_number: int) -> Dict[str, Optional[float]]:
    vals: Dict[str, Optional[float]] = {"left": None, "right": None, "top": None, "bottom": None}
    for m in first_pass.get("margins") or []:
@@ -580,7 +506,7 @@ def _map_first_pass_to_rows(template_id: str, first_pass: Dict[str, Any], pdf_by
                top = max(float(y1), float(y2)); bottom = min(float(y1), float(y2))
                bounds = _box_to_canvas({"l": margins["left"], "r": margins["right"], "t": top, "b": bottom, "coord_origin": "BOTTOMLEFT"}, page_number, pages_geom)
            bounds = bounds or _box_to_canvas(band.get("label_box"), page_number, pages_geom)
-            questions.append({"id": pid, "template_id": template_id, "parent_id": parent_id, "label": label, "order": len(questions), "max_marks": _safe_marks(band.get("marks")), "is_container": False, "bounds": bounds, "page": page_number, "source": "ai", "confirmed": False, "confidence": _safe_confidence(band.get("confidence")), "derivation": "docling-part-band-x-margins"})
+            questions.append({"id": pid, "template_id": template_id, "parent_id": parent_id, "label": label, "order": len(questions), "max_marks": 0, "is_container": False, "bounds": bounds, "page": page_number, "source": "ai", "confirmed": False, "confidence": _safe_confidence(band.get("confidence")), "derivation": "docling-part-band-x-margins"})
    default_qid = questions[0]["id"] if questions else _ai_id(template_id, "question", "auto")
    for page_key in sorted(pages_obj, key=lambda k: int(k)):
@@ -601,53 +527,15 @@ def _map_first_pass_to_rows(template_id: str, first_pass: Dict[str, Any], pdf_by
            response_form = _response_form_from_region_type(region.get("region_type"))
            if response_form:
                response_areas.append({"id": _ai_id(template_id, "region", page_index, idx), "template_id": template_id, "question_id": first_part_by_page.get(page_index, default_qid), "page": page_index + 1, "bounds": bounds, "kind": "response", "response_form": response_form, "source": "ai", "confirmed": False, "confidence": _safe_confidence(region.get("confidence")), "derivation": region.get("detection_method") or "opencv-response-region"})
    # Integrity guard: every response_area/boundary question_id must reference an inserted question
    # (FK exam_response_areas/exam_boundaries -> exam_questions). On papers where band detection yields
    # few/no questions but opencv/gemma still emit regions, those regions point at the synthetic
    # default_qid which was never inserted. Ensure that fallback container question exists and reattach
    # any orphan child rows to it, so persistence can't violate the FK.
    qid_set = {q["id"] for q in questions}
    orphans = [r for r in (response_areas + boundaries) if r.get("question_id") not in qid_set]
    if orphans:
        if default_qid not in qid_set:
            questions.insert(0, {"id": default_qid, "template_id": template_id, "label": "Unassigned",
                                 "order": 0, "max_marks": 0, "is_container": True, "source": "ai",
                                 "confirmed": False, "confidence": 0.5,
                                 "derivation": "auto-map-fallback-container"})
            qid_set.add(default_qid)
        for r in orphans:
            r["question_id"] = default_qid
    return {"questions": questions, "response_areas": response_areas, "boundaries": boundaries, "layout": layout}
 def _dedupe_rows_by_id(rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
    """Preserve first occurrence of stable AI row ids emitted by noisy OCR detectors."""
    out: List[Dict[str, Any]] = []
    seen: set[str] = set()
    for row in rows:
        row_id = row.get("id")
        if row_id:
            key = str(row_id)
            if key in seen:
                continue
            seen.add(key)
        out.append(row)
    return out
 def _refresh_ai_rows(ctx: ExamContext, template_id: str, rows: Dict[str, List[Dict[str, Any]]]) -> None:
    sb = ctx.supabase
    for table in ("exam_response_areas", "exam_boundaries", "exam_template_layout", "exam_questions"):
        sb.table(table).delete().eq("template_id", template_id).eq("source", "ai").eq("confirmed", False).execute()
    for table, key in (("exam_questions", "questions"), ("exam_response_areas", "response_areas"), ("exam_boundaries", "boundaries"), ("exam_template_layout", "layout")):
-        # A row that survived the delete above is confirmed-AI or manual — the teacher's curated version.
+        payload = rows.get(key) or []
        # AI ids are a deterministic uuid5 of (template_id, semantic key), so a re-run re-emits the SAME id
        # for a confirmed ghost; re-inserting it would PK-collide and fail the whole batch. Skip those ids so
        # confirmed work is preserved and the insert is safe (fixes the re-map-after-confirm collision).
        kept = sb.table(table).select("id").eq("template_id", template_id).execute()
        kept_ids = {str(r["id"]) for r in (getattr(kept, "data", None) or []) if isinstance(r, dict) and r.get("id")}
        payload = [r for r in _dedupe_rows_by_id(rows.get(key) or []) if str(r.get("id")) not in kept_ids]
        if payload:
            sb.table(table).insert(payload).execute()
@@ -679,153 +567,12 @@ def _run_auto_map_job(job_id: str, ctx: ExamContext, template_id: str, pdf_bytes
    _set_auto_map_status(job_id, {"status": "running", "template_id": template_id})
    try:
        rows = _run_auto_map_merge(ctx, template_id, pdf_bytes, source_label)
        # Project to Neo4j like the born-digital fast path does — otherwise image-only papers (R3's
        # primary target, routed here because they need OCR) never reach the graph after auto-map.
        project_template_safe(template_id)
        _set_auto_map_status(job_id, {"status": "completed", "template_id": template_id, "counts": {k: len(v) for k, v in rows.items()}})
    except Exception as exc:
        logger.exception(f"auto-map job failed for template {template_id}: {exc}")
        _set_auto_map_status(job_id, {"status": "failed", "template_id": template_id, "error": str(exc)})
 _ALLOWED_RESPONSE_FORMS = {"lines", "answer-box", "working", "diagram", "tick-boxes", "table", "blanks"}
 _ALLOWED_ANSWER_TYPES = {"written", "mcq", "short", "diagram"}
 _ALLOWED_KINDS = {"response", "context", "question_number", "mark_area", "reference", "furniture"}
 _BOARD_RE = re.compile(r"^(aqa|edexcel|ocr|wjec|eduqas|ccea)-")
 def _extract_slug(ctx: ExamContext, template: Dict[str, Any]) -> str:
    """A stable, board-prefixed slug for the extraction-service cache. Prefer the catalogue exam_code
    (e.g. 'AQA-8463-1H-2022JUN-QP' → 'aqa-8463-1h-2022jun-qp' → board 'aqa' for the right margins);
    fall back to a template-id slug (structure.py then defaults to AQA content-box margins)."""
    code = None
    exam_id = template.get("exam_id")
    if exam_id:
        try:
            row = _first(ctx.supabase.table("eb_exams").select("exam_code").eq("id", exam_id).limit(1).execute())
            code = (row or {}).get("exam_code")
        except Exception as exc:
            logger.info(f"extract slug: eb_exams lookup failed for {exam_id}: {exc}")
    slug = re.sub(r"[^a-z0-9._-]+", "-", (code or "").lower()).strip("-")
    if _BOARD_RE.match(slug):
        return slug
    return f"aqa-tmpl-{str(template.get('id') or '')[:12]}"
 def _map_service_contract_to_rows(template_id: str, contract: Dict[str, Any],
                                  pdf_bytes: bytes) -> Dict[str, List[Dict[str, Any]]]:
    """Map the extraction service's page-fraction analyse contract onto the app's canvas-space ghost rows.
    Coordinates: page-fraction → the 780-wide stacked canvas. IDs: the service's deterministic uuid5s are
    re-namespaced per template via _ai_id so two templates of the same paper don't collide and a re-map of
    the same template re-emits stable ids (so _refresh_ai_rows preserves confirmed ghosts). FK-safe: parts
    whose parent/owner question is absent are de-parented / dropped rather than crashing the insert.
    """
    pages = _pdf_page_geometry(pdf_bytes)
    sug = contract.get("suggestions") or {}
    def qid(uid: Any) -> str:
        return _ai_id(template_id, "svc-q", uid)
    questions: List[Dict[str, Any]] = []
    q_ids: set = set()
    for q in sug.get("questions") or []:
        uid = q.get("uid")
        if not uid:
            continue
        rid = qid(uid)
        q_ids.add(rid)
        at = q.get("answer_type") if q.get("answer_type") in _ALLOWED_ANSWER_TYPES else None
        questions.append({
            "id": rid, "template_id": template_id,
            "parent_id": qid(q["parent_uid"]) if q.get("parent_uid") else None,
            "label": q.get("label") or "?", "order": q.get("order", len(questions)),
            "max_marks": _safe_marks(q.get("max_marks")), "answer_type": at,
            "is_container": bool(q.get("is_container")),
            "bounds": _frac_box_to_canvas(q.get("bounds"), q.get("page") or 1, pages),
            "page": q.get("page"), "source": "ai", "confirmed": False,
            "confidence": _safe_confidence(q.get("confidence")), "derivation": "extract-service",
            # analyse contract v2 (migration 78): command verb + stem prose per part
            "command_word": (q.get("command_word") or None) if not q.get("is_container") else None,
            "preamble": q.get("preamble") or None,
        })
    for q in questions:                                 # FK safety: de-parent a dangling parent_id
        if q["parent_id"] and q["parent_id"] not in q_ids:
            q["parent_id"] = None
    response_areas: List[Dict[str, Any]] = []
    for ra in sug.get("response_areas") or []:
        uid = ra.get("uid")
        quid = qid(ra.get("question_uid") or "")
        if not uid or quid not in q_ids:               # orphan region → drop (FK safety)
            continue
        bounds = _frac_box_to_canvas(ra.get("bounds"), ra.get("page") or 1, pages)
        if not bounds:
            continue
        kind = ra.get("kind") if ra.get("kind") in _ALLOWED_KINDS else "response"
        form = ra.get("response_form") if ra.get("response_form") in _ALLOWED_RESPONSE_FORMS else None
        response_areas.append({
            "id": _ai_id(template_id, "svc-ra", uid), "template_id": template_id,
            "question_id": quid, "page": ra.get("page"), "bounds": bounds,
            "kind": kind, "response_form": form if kind == "response" else None,
            "context_type": ra.get("context_type"), "meta": ra.get("meta") or {},
            "source": "ai", "confirmed": False,
            "confidence": _safe_confidence(ra.get("confidence")), "derivation": "extract-service",
        })
    boundaries: List[Dict[str, Any]] = []
    for i, b in enumerate(sug.get("boundaries") or []):
        quid = qid(b.get("question_uid") or "")
        if quid not in q_ids:
            continue
        page_index = b.get("page_index")
        y = _frac_y_to_canvas(b.get("y"), (page_index or 0) + 1, pages)
        if y is None:
            continue
        boundaries.append({
            "id": _ai_id(template_id, "svc-b", b.get("question_uid") or i), "template_id": template_id,
            "question_id": quid, "label": b.get("label") or "", "page_index": page_index,
            "y": y, "bounds": None, "source": "ai", "confirmed": False,
            "confidence": _safe_confidence(b.get("confidence")), "derivation": "extract-service",
        })
    return {"questions": questions, "response_areas": response_areas, "boundaries": boundaries}
 def _run_service_extract_merge(ctx: ExamContext, template_id: str, pdf_bytes: bytes, slug: str) -> Dict[str, List[Dict[str, Any]]]:
    contract = exam_extract.extract_suggestions(slug, pdf_bytes)
    rows = _map_service_contract_to_rows(template_id, contract, pdf_bytes)
    _refresh_ai_rows(ctx, template_id, rows)
    meta = contract.get("meta") or {}
    # P3: record provenance + the audit cover-reconciliation gate so the setup UI can flag under-reads,
    # and store the slug so the digital-text view can resolve this paper's replica.
    updates: Dict[str, Any] = {"extraction_meta": {
        "engine": "extract-service", "slug": slug, "audit": meta.get("audit") or {},
        "counts": {k: len(v) for k, v in rows.items()},
        # question-layout signals for the setup UI: paper sections (e.g. A-level Section A/B) + EITHER/OR choices
        "sections": meta.get("sections") or [],
        "choice_groups": meta.get("choice_groups") or [],
        "marks_confidence": meta.get("marks_confidence"),
    }}
    n_pages = meta.get("n_pages") or meta.get("pages")
    if n_pages:
        updates["page_count"] = n_pages
    ctx.supabase.table("exam_templates").update(updates).eq("id", template_id).execute()
    return rows
 def _run_service_extract_job(job_id: str, ctx: ExamContext, template_id: str, pdf_bytes: bytes, slug: str) -> None:
    _set_auto_map_status(job_id, {"status": "running", "template_id": template_id, "engine": "extract-service", "slug": slug})
    try:
        rows = _run_service_extract_merge(ctx, template_id, pdf_bytes, slug)
        project_template_safe(template_id)
        _set_auto_map_status(job_id, {"status": "completed", "template_id": template_id,
                                      "engine": "extract-service", "counts": {k: len(v) for k, v in rows.items()}})
    except Exception as exc:
        logger.exception(f"extract-service job failed for template {template_id}: {exc}")
        _set_auto_map_status(job_id, {"status": "failed", "template_id": template_id, "engine": "extract-service", "error": str(exc)})
 # ─── templates ───────────────────────────────────────────────────────────────
@@ -870,13 +617,12 @@ async def create_template(
@router.get("/catalogue")
-async def list_catalogue_papers(
+async def list_catalogue_papers() -> Dict[str, Any]:
-    ctx: ExamContext = Depends(get_exam_context),
+    """Lightweight exam-board paper catalogue for the create dialog."""
 ) -> Dict[str, Any]:
    """Lightweight authenticated exam-board metadata catalogue for the create dialog."""
    try:
        sb = SupabaseServiceRoleClient().supabase
        res = (
-            ctx.supabase.table("eb_exams")
+            sb.table("eb_exams")
            .select("id, exam_code, spec_code, paper_code, tier, session, type_code, storage_loc")
            .eq("type_code", "QP")
            .order("exam_code")
@@ -887,50 +633,6 @@ async def list_catalogue_papers(
        raise HTTPException(status_code=502, detail=f"Could not load catalogue papers: {exc}")
@router.get("/catalogue/{exam_id}/signed-url")
 async def get_catalogue_paper_signed_url(
    exam_id: str,
    expires_in: int = 300,
    ctx: ExamContext = Depends(get_exam_context),
 ) -> Dict[str, Any]:
    """Return a short-lived signed URL for an authenticated user's catalogue PDF access.
    The storage operation uses service role as a scoped backend exception for signing only;
    raw cc.examboards object reads remain denied by storage.objects RLS.
    """
    expires_in = max(60, min(int(expires_in or 300), 3600))
    try:
        row = _first(
            ctx.supabase.table("eb_exams")
            .select("id, exam_code, storage_loc")
            .eq("id", exam_id)
            .eq("type_code", "QP")
            .limit(1)
            .execute()
        )
        if not row or not row.get("storage_loc"):
            raise HTTPException(status_code=404, detail="Catalogue paper not found")
        try:
            bucket, path = _parse_storage_loc(row["storage_loc"])
        except ValueError:
            raise HTTPException(status_code=404, detail="Catalogue paper not found")
        if bucket != "cc.examboards":
            raise HTTPException(status_code=404, detail="Catalogue paper not found")
        signed_url = _signed_url_value(StorageAdmin().create_signed_url(bucket, path, expires_in))
        return {
            "exam_id": row["id"],
            "exam_code": row.get("exam_code"),
            "bucket": bucket,
            "path": path,
            "expires_in": expires_in,
            "signed_url": signed_url,
        }
    except HTTPException:
        raise
    except Exception as exc:
        raise HTTPException(status_code=502, detail=f"Could not sign catalogue paper URL: {exc}")
@router.get("/templates")
 async def list_templates(
    include_archived: bool = False,
@@ -1003,14 +705,6 @@ async def auto_map_template(
        raise HTTPException(status_code=409, detail="Template has recorded marks; auto-map structural refresh is blocked.")
    bucket, path, pdf_bytes = _resolve_template_source(ctx, template)
    source_label = f"{bucket}/{path}"
    # Extraction-service path (P2): when EXAM_EXTRACT_URL is set, route auto-map through the spike's full
    # recognition pipeline instead of the thin first-pass. Always async — a cold paper is ~15 min.
    if exam_extract.is_enabled():
        slug = _extract_slug(ctx, template)
        job_id = str(uuid.uuid4())
        _set_auto_map_status(job_id, {"status": "queued", "template_id": template_id, "engine": "extract-service", "slug": slug})
        background_tasks.add_task(_run_service_extract_job, job_id, ctx, template_id, pdf_bytes, slug)
        return JSONResponse(status_code=202, content={"status": "accepted", "job_id": job_id, "engine": "extract-service"})
    try:
        fast_path = _pdf_has_text_layer(pdf_bytes)
    except Exception as exc:
@@ -1049,24 +743,6 @@ async def auto_map_status(
    return body
@router.get("/templates/{template_id}/digital-text")
 async def template_digital_text(
    template_id: str,
    ctx: ExamContext = Depends(get_exam_context),
 ) -> Dict[str, Any]:
    """P4: the digital-replica markdown for this template's paper (stem text, parts, marks, answer-space
    placeholders, spec refs). Resolves the paper via the slug the extraction used."""
    template = _fetch_template_or_404(ctx, template_id)
    _require_source_visibility_or_404(ctx, template)
    if not exam_extract.is_enabled():
        raise HTTPException(status_code=503, detail="Extraction service not configured")
    slug = (template.get("extraction_meta") or {}).get("slug") or _extract_slug(ctx, template)
    try:
        return exam_extract.get_replica(slug)
    except exam_extract.ExtractError as exc:
        raise HTTPException(status_code=404, detail=f"No digital text yet — run auto-map first ({exc})")
@router.put("/templates/{template_id}")
 async def replace_template(
    template_id: str,
@@ -1147,7 +823,6 @@ async def replace_template(
                "kind": ra.kind,
                "response_form": ra.response_form,
                "context_type": ra.context_type,   # 73: optional Context differentiation
                "meta": ra.meta,                   # 75: rich recognition payload (name/description/OMR/…)
                "source": ra.source,
                "confirmed": ra.confirmed,
                "confidence": ra.confidence,
@@ -1,256 +0,0 @@
 """Running class markbook / gradebook API (/api/markbook).
 A markbook is a term-long teacher-editable ledger: roster rows × arbitrary assessment
 columns. It deliberately does not depend on exam-marker batches. All user-facing reads and
 writes use the as-user Supabase client so class/markbook RLS gates are enforced.
 """
 from __future__ import annotations
 import csv
 import io
 import os
 from datetime import date as Date
 from typing import Any, Dict, List, Optional
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import Response
 from pydantic import BaseModel, Field
 from modules.logger_tool import initialise_logger
 from routers.exam.dependencies import ExamContext, get_exam_context, resolve_student_names
 logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), "default", True)
 router = APIRouter()
 class CreateAssessmentRequest(BaseModel):
    title: str = Field(..., min_length=1, max_length=160)
    date: Optional[Date] = None
    max_marks: float = Field(default=100, gt=0)
 class MarkUpsertRequest(BaseModel):
    mark: Optional[float] = Field(default=None, ge=0)
 def _rows(result: Any) -> List[Dict[str, Any]]:
    data = getattr(result, "data", None)
    if not data:
        return []
    return data if isinstance(data, list) else [data]
 def _first(result: Any) -> Optional[Dict[str, Any]]:
    rows = _rows(result)
    return rows[0] if rows else None
 def _round(value: Optional[float]) -> Optional[float]:
    return None if value is None else round(float(value), 1)
 def _fetch_class_or_404(ctx: ExamContext, class_id: str) -> Dict[str, Any]:
    row = _first(ctx.supabase.table("classes").select("id, name, institute_id").eq("id", class_id).limit(1).execute())
    if not row:
        raise HTTPException(status_code=404, detail="Class not found")
    return row
 def _active_roster(ctx: ExamContext, class_id: str) -> List[Dict[str, Any]]:
    roster = _rows(
        ctx.supabase.table("class_students")
        .select("student_id, status, enrolled_at")
        .eq("class_id", class_id)
        .eq("status", "active")
        .execute()
    )
    names = resolve_student_names([r["student_id"] for r in roster if r.get("student_id")])
    return [
        {
            "student_id": r["student_id"],
            "student_name": names.get(r["student_id"]) or r["student_id"],
            "status": r.get("status"),
            "enrolled_at": r.get("enrolled_at"),
        }
        for r in roster
        if r.get("student_id")
    ]
 def _assessments(ctx: ExamContext, class_id: str) -> List[Dict[str, Any]]:
    return _rows(
        ctx.supabase.table("class_assessments")
        .select("id, class_id, tenant_id, title, date, max_marks, created_at, updated_at")
        .eq("class_id", class_id)
        .order("date")
        .order("created_at")
        .execute()
    )
 def _marks(ctx: ExamContext, assessment_ids: List[str]) -> List[Dict[str, Any]]:
    if not assessment_ids:
        return []
    return _rows(
        ctx.supabase.table("assessment_marks")
        .select("assessment_id, student_id, mark, updated_at, updated_by")
        .in_("assessment_id", assessment_ids)
        .execute()
    )
 def _assemble_grid(ctx: ExamContext, class_id: str) -> Dict[str, Any]:
    cls = _fetch_class_or_404(ctx, class_id)
    roster = _active_roster(ctx, class_id)
    assessments = _assessments(ctx, class_id)
    assessment_ids = [a["id"] for a in assessments]
    marks = _marks(ctx, assessment_ids)
    marks_by_student: Dict[str, Dict[str, Optional[float]]] = {r["student_id"]: {} for r in roster}
    for m in marks:
        sid = m.get("student_id")
        aid = m.get("assessment_id")
        if isinstance(sid, str) and isinstance(aid, str) and sid in marks_by_student and aid in assessment_ids:
            marks_by_student[sid][aid] = m.get("mark")
    max_total = sum(float(a.get("max_marks") or 0) for a in assessments)
    students = []
    all_entered: List[float] = []
    for idx, student in enumerate(roster, start=1):
        entered = [
            float(v)
            for v in (marks_by_student.get(student["student_id"], {}).get(aid) for aid in assessment_ids)
            if v is not None
        ]
        total = sum(entered) if entered else None
        students.append(
            {
                **student,
                "row_number": idx,
                "marks": {aid: marks_by_student.get(student["student_id"], {}).get(aid) for aid in assessment_ids},
                "total": total,
                "percentage": _round((total / max_total) * 100) if total is not None and max_total > 0 else None,
            }
        )
        all_entered.extend(entered)
    assessment_summaries = []
    for a in assessments:
        vals = []
        for s in roster:
            maybe_mark = marks_by_student.get(s["student_id"], {}).get(a["id"])
            if maybe_mark is not None:
                vals.append(float(maybe_mark))
        max_marks = float(a.get("max_marks") or 0)
        assessment_summaries.append(
            {
                "assessment_id": a["id"],
                "entered_count": len(vals),
                "average_mark": _round(sum(vals) / len(vals)) if vals else None,
                "average_percentage": _round((sum(vals) / len(vals) / max_marks) * 100) if vals and max_marks > 0 else None,
            }
        )
    return {
        "class": cls,
        "students": students,
        "assessments": assessments,
        "assessment_summaries": assessment_summaries,
        "summary": {
            "student_count": len(roster),
            "assessment_count": len(assessments),
            "entered_mark_count": len(all_entered),
            "class_average_mark": _round(sum(all_entered) / len(all_entered)) if all_entered else None,
        },
    }
@router.get("/classes/{class_id}/assessments")
 async def list_assessments(class_id: str, ctx: ExamContext = Depends(get_exam_context)) -> Dict[str, Any]:
    _fetch_class_or_404(ctx, class_id)
    return {"assessments": _assessments(ctx, class_id)}
@router.post("/classes/{class_id}/assessments")
 async def create_assessment(class_id: str, body: CreateAssessmentRequest, ctx: ExamContext = Depends(get_exam_context)) -> Dict[str, Any]:
    cls = _fetch_class_or_404(ctx, class_id)
    row = {
        "class_id": class_id,
        "tenant_id": cls["institute_id"],
        "title": body.title.strip(),
        "date": body.date.isoformat() if body.date else None,
        "max_marks": body.max_marks,
    }
    created = _first(ctx.supabase.table("class_assessments").insert(row).execute())
    if not created:
        raise HTTPException(status_code=500, detail="Failed to create assessment")
    logger.info(f"Markbook assessment {created.get('id')} created for class {class_id} by {ctx.user_id}")
    return created
@router.get("/classes/{class_id}/grid")
 async def get_grid(class_id: str, ctx: ExamContext = Depends(get_exam_context)) -> Dict[str, Any]:
    return _assemble_grid(ctx, class_id)
@router.put("/classes/{class_id}/assessments/{assessment_id}/marks/{student_id}")
 async def upsert_mark(
    class_id: str,
    assessment_id: str,
    student_id: str,
    body: MarkUpsertRequest,
    ctx: ExamContext = Depends(get_exam_context),
 ) -> Dict[str, Any]:
    cls = _fetch_class_or_404(ctx, class_id)
    assessment = _first(
        ctx.supabase.table("class_assessments")
        .select("id, class_id, tenant_id, max_marks")
        .eq("id", assessment_id)
        .eq("class_id", class_id)
        .limit(1)
        .execute()
    )
    if not assessment:
        raise HTTPException(status_code=404, detail="Assessment not found")
    roster_row = _first(
        ctx.supabase.table("class_students")
        .select("student_id")
        .eq("class_id", class_id)
        .eq("student_id", student_id)
        .eq("status", "active")
        .limit(1)
        .execute()
    )
    if not roster_row:
        raise HTTPException(status_code=404, detail="Student is not active in this class")
    if body.mark is not None and body.mark > float(assessment.get("max_marks") or 0):
        raise HTTPException(status_code=422, detail="mark exceeds assessment max_marks")
    row = {
        "assessment_id": assessment_id,
        "student_id": student_id,
        "tenant_id": assessment.get("tenant_id") or cls["institute_id"],
        "mark": body.mark,
        "updated_by": ctx.user_id,
    }
    upserted = _first(ctx.supabase.table("assessment_marks").upsert(row, on_conflict="assessment_id,student_id").execute())
    if not upserted:
        raise HTTPException(status_code=500, detail="Failed to upsert mark")
    return {"status": "ok", "mark": upserted}
@router.get("/classes/{class_id}/csv")
 async def export_csv(class_id: str, ctx: ExamContext = Depends(get_exam_context)) -> Response:
    data = _assemble_grid(ctx, class_id)
    assessments = data["assessments"]
    buf = io.StringIO()
    writer = csv.writer(buf)
    writer.writerow(["row", "student_name", "student_id"] + [a["title"] for a in assessments] + ["total", "percentage"])
    for student in data["students"]:
        writer.writerow(
            [student["row_number"], student.get("student_name") or "", student.get("student_id") or ""]
            + ["" if student["marks"].get(a["id"]) is None else student["marks"].get(a["id"]) for a in assessments]
            + ["" if student["total"] is None else student["total"], "" if student["percentage"] is None else student["percentage"]]
        )
    filename = f"markbook-{class_id}.csv"
    return Response(content=buf.getvalue(), media_type="text/csv", headers={"Content-Disposition": f'attachment; filename="{filename}"'})
@@ -26,7 +26,6 @@ from fastapi.responses import JSONResponse
 from modules.auth.supabase_bearer import SupabaseBearer
 from modules.database.supabase.utils.client import SupabaseServiceRoleClient
 from modules.database.supabase.utils.storage import StorageAdmin
 from modules.upload_validation import read_upload_bytes
 from modules.logger_tool import initialise_logger
 router = APIRouter()
@@ -60,9 +59,10 @@ async def upload_single_file(
        if not user_id:
            raise HTTPException(status_code=401, detail="User ID required")
-        # Validate MIME/type and read file content with a hard size limit.
+        # Read file content
-        file_bytes, mime_type = await read_upload_bytes(file)
+        file_bytes = await file.read()
        file_size = len(file_bytes)
        mime_type = file.content_type or 'application/octet-stream'
        filename = file.filename or path
        logger.info(f"📤 Simple upload: {filename} ({file_size} bytes) for user {user_id}")
@@ -234,9 +234,10 @@ async def upload_directory(
            # Process each file
            for i, (file, relative_path) in enumerate(zip(files, relative_paths)):
                try:
-                    # Validate MIME/type and read file content with a hard size limit.
+                    # Read file content
-                    file_bytes, mime_type = await read_upload_bytes(file)
+                    file_bytes = await file.read()
                    file_size = len(file_bytes)
                    mime_type = file.content_type or 'application/octet-stream'
                    filename = file.filename or f"file_{i}"
                    total_size += file_size
@@ -290,8 +291,6 @@ async def upload_directory(
                    logger.info(f"📄 Uploaded file {i+1}/{len(files)}: {relative_path}")
                except HTTPException:
                    raise
                except Exception as e:
                    logger.error(f"Failed to upload file {relative_path}: {e}")
                    # Continue with other files, don't fail entire upload
@@ -46,7 +46,7 @@ def initialize_buckets() -> dict:
                    file_size_limit=1000 * 1024 * 1024,  # 1GB
                )
            },
-            # Exam Board files (admin-curated public exam corpus: QP/MS/insert/ER + specs)
+            # Exam Board files
            {
                "id": "cc.examboards",
                "options": CreateBucketOptions(
@@ -55,34 +55,6 @@ def initialize_buckets() -> dict:
                    file_size_limit=1000 * 1024 * 1024,  # 1GB
                )
            },
            # ── Storage taxonomy bins (access scoped by RLS on bucket + leading path segment; RLS = D1) ──
            # Platform-managed public/shared assets (readable by all authenticated users).
            {
                "id": "cc.public",
                "options": CreateBucketOptions(
                    name="Classroom Copilot Public",
                    public=False,
                    file_size_limit=1000 * 1024 * 1024,  # 1GB
                )
            },
            # Institute-scoped operational assets: cc.institutes/{institute_id}/...
            {
                "id": "cc.institutes",
                "options": CreateBucketOptions(
                    name="Classroom Copilot Institutes",
                    public=False,
                    file_size_limit=1000 * 1024 * 1024,  # 1GB
                )
            },
            # Platform-admin-only assets, seeds, intake/staging for unidentified papers.
            {
                "id": "cc.admin",
                "options": CreateBucketOptions(
                    name="Classroom Copilot Admin",
                    public=False,
                    file_size_limit=1000 * 1024 * 1024,  # 1GB
                )
            },
        ]
        results = {}
@@ -109,12 +81,6 @@ def initialize_buckets() -> dict:
                    logger.error(f"Failed to create bucket: {bucket['id']}")
            except Exception as e:
                # Idempotent: an already-existing bucket is not a failure on re-run.
                if any(s in str(e).lower() for s in ("already exists", "duplicate", "resource already")):
                    results[bucket["id"]] = {"status": "exists", "result": str(e)}
                    success_count += 1
                    logger.info(f"Bucket already exists (ok): {bucket['id']}")
                else:
                results[bucket["id"]] = {
                    "status": "error",
                    "error": str(e)
@@ -1,19 +1,17 @@
 """
 init_exam_graph.py — Initialise the cc.public.exams Neo4j knowledge graph.
-Creates the shared, public exam database, its uniqueness constraints, and seeds the AQA exam board
+Creates the shared, public exam database, its uniqueness constraints, and seeds the AQA exam
-+ the 6 current test specifications (GCSE & A-level Physics/Chemistry/Biology) with their top-level
+board + AQA GCSE Physics (8463) specification with its 8 top-level topic SpecPoints. Idempotent
-topic SpecPoints (44 in total). Idempotent (CREATE DATABASE IF NOT EXISTS / CREATE CONSTRAINT IF NOT
+(CREATE DATABASE IF NOT EXISTS / CREATE CONSTRAINT IF NOT EXISTS / MERGE).
 EXISTS / MERGE).
 Run inside the ccapi container:
    python3 -c "from run.initialization.init_exam_graph import init; import json; print(json.dumps(init()))"
-NOTE: only *top-level* topics are seeded (the granularity a teacher plans against). The full sub-point
+NOTE: the 8 SpecPoints seeded here are the real AQA GCSE Physics *top-level* topics. The full
-breakdown (e.g. 4.1.1.1 ...) is a later data-population task (sourceable from the AQA spec PDF via
+sub-point breakdown (e.g. 4.1.1.1 ...) is a later data-population task (sourceable from the AQA
-Docling). Seeding all 6 specs means a template's spec_ref finds a matching SpecPoint so
+spec PDF via Docling). spec_code AQA-PHYS-8463 is the standalone GCSE Physics code that matches
-(:Part)-[:ASSESSES]->(:SpecPoint) fires beyond GCSE Physics; spec_code (e.g. AQA-PHYS-8463) must match
+"AQA Physics Paper 1H"; the eb_exams/eb_specifications seed (card S4-3) must use the same code.
 the eb_exams/eb_specifications seed (card S4-3) and the app's deriveSpecCode.
 """
 import uuid
 from typing import Dict, Any
@@ -43,37 +41,6 @@ SPEC_POINTS = [
    ("4.8", "Space physics"),
 ]
 # Full AQA catalogue for the current test specs (top-level topics; ref = topic number). Seeding all of
 # them means a template's spec_ref finds a matching SpecPoint so (:Part)-[:ASSESSES]->(:SpecPoint) fires
 # beyond AQA GCSE Physics. Sub-point granularity (e.g. 4.1.1.1) remains a later data-population task.
 SPECIFICATIONS = [
    {**SPEC, "topics": SPEC_POINTS},
    {"spec_code": "AQA-CHEM-8462", "exam_board_code": "AQA", "subject_code": "CHEM", "award_code": "GCSE",
     "title": "AQA GCSE Chemistry (8462)", "topics": [
        ("4.1", "Atomic structure and the periodic table"), ("4.2", "Bonding, structure, and the properties of matter"),
        ("4.3", "Quantitative chemistry"), ("4.4", "Chemical changes"), ("4.5", "Energy changes"),
        ("4.6", "The rate and extent of chemical change"), ("4.7", "Organic chemistry"), ("4.8", "Chemical analysis"),
        ("4.9", "Chemistry of the atmosphere"), ("4.10", "Using resources")]},
    {"spec_code": "AQA-BIOL-8461", "exam_board_code": "AQA", "subject_code": "BIOL", "award_code": "GCSE",
     "title": "AQA GCSE Biology (8461)", "topics": [
        ("4.1", "Cell biology"), ("4.2", "Organisation"), ("4.3", "Infection and response"), ("4.4", "Bioenergetics"),
        ("4.5", "Homeostasis and response"), ("4.6", "Inheritance, variation and evolution"), ("4.7", "Ecology")]},
    {"spec_code": "AQA-PHYS-7408", "exam_board_code": "AQA", "subject_code": "PHYS", "award_code": "A-level",
     "title": "AQA A-level Physics (7408)", "topics": [
        ("3.1", "Measurements and their errors"), ("3.2", "Particles and radiation"), ("3.3", "Waves"),
        ("3.4", "Mechanics and materials"), ("3.5", "Electricity"), ("3.6", "Further mechanics and thermal physics"),
        ("3.7", "Fields and their consequences"), ("3.8", "Nuclear physics")]},
    {"spec_code": "AQA-CHEM-7405", "exam_board_code": "AQA", "subject_code": "CHEM", "award_code": "A-level",
     "title": "AQA A-level Chemistry (7405)", "topics": [
        ("3.1", "Physical chemistry"), ("3.2", "Inorganic chemistry"), ("3.3", "Organic chemistry")]},
    {"spec_code": "AQA-BIOL-7402", "exam_board_code": "AQA", "subject_code": "BIOL", "award_code": "A-level",
     "title": "AQA A-level Biology (7402)", "topics": [
        ("3.1", "Biological molecules"), ("3.2", "Cells"), ("3.3", "Organisms exchange substances with their environment"),
        ("3.4", "Genetic information, variation and relationships between organisms"),
        ("3.5", "Energy transfers in and between organisms"), ("3.6", "Organisms respond to changes"),
        ("3.7", "Genetics, populations, evolution and ecosystems"), ("3.8", "The control of gene expression")]},
 ]
 CONSTRAINTS = [
    "CREATE CONSTRAINT exam_board_uid    IF NOT EXISTS FOR (n:ExamBoard)     REQUIRE n.uuid_string IS UNIQUE",
    "CREATE CONSTRAINT spec_uid          IF NOT EXISTS FOR (n:Specification) REQUIRE n.uuid_string IS UNIQUE",
@@ -114,36 +81,35 @@ def init() -> Dict[str, Any]:
            s.run(c).consume()
            result["constraints"] += 1
-        # 3. board (once)
+        # 3. board + spec
        board_uid = _uid("ExamBoard", BOARD["code"])
        spec_uid = _uid("Specification", SPEC["spec_code"])
        s.run(
            "MERGE (b:ExamBoard {uuid_string:$uid}) "
            "SET b.code=$code, b.name=$name, b.node_storage_path=$nsp",
            uid=board_uid, code=BOARD["code"], name=BOARD["name"],
            nsp=f"{EXAM_DB}/ExamBoard/{BOARD['code']}",
        ).consume()
        # 4. each specification + its top-level spec points (idempotent MERGE)
        for spec in SPECIFICATIONS:
            spec_uid = _uid("Specification", spec["spec_code"])
        s.run(
            "MERGE (sp:Specification {uuid_string:$uid}) "
            "SET sp.spec_code=$sc, sp.exam_board_code=$ebc, sp.subject_code=$subj, "
            "    sp.award_code=$award, sp.title=$title, sp.node_storage_path=$nsp "
            "WITH sp MATCH (b:ExamBoard {code:$ebc}) MERGE (b)-[:PUBLISHES]->(sp)",
-                uid=spec_uid, sc=spec["spec_code"], ebc=spec["exam_board_code"],
+            uid=spec_uid, sc=SPEC["spec_code"], ebc=SPEC["exam_board_code"],
-                subj=spec["subject_code"], award=spec["award_code"], title=spec["title"],
+            subj=SPEC["subject_code"], award=SPEC["award_code"], title=SPEC["title"],
-                nsp=f"{EXAM_DB}/Specification/{spec['spec_code']}",
+            nsp=f"{EXAM_DB}/Specification/{SPEC['spec_code']}",
        ).consume()
-            for ref, desc in spec["topics"]:
+
-                sp_uid = _uid("SpecPoint", spec["spec_code"], ref)
+        # 4. spec points
        for ref, desc in SPEC_POINTS:
            sp_uid = _uid("SpecPoint", SPEC["spec_code"], ref)
            s.run(
                "MERGE (p:SpecPoint {uuid_string:$uid}) "
                "SET p.ref=$ref, p.description=$desc, p.spec_code=$sc, "
                "    p.exam_board_code=$ebc, p.node_storage_path=$nsp "
                "WITH p MATCH (s:Specification {spec_code:$sc}) MERGE (s)-[:HAS_SPEC_POINT]->(p)",
-                    uid=sp_uid, ref=ref, desc=desc, sc=spec["spec_code"],
+                uid=sp_uid, ref=ref, desc=desc, sc=SPEC["spec_code"],
-                    ebc=spec["exam_board_code"], nsp=f"{EXAM_DB}/SpecPoint/{spec['spec_code']}/{ref}",
+                ebc=SPEC["exam_board_code"], nsp=f"{EXAM_DB}/SpecPoint/{SPEC['spec_code']}/{ref}",
            ).consume()
            result["spec_points"] += 1
@@ -1,3 +0,0 @@
 # Persistent local corpus store — PDFs are NOT committed (re-downloadable from manifest).
 *
 !.gitignore
@@ -1,501 +0,0 @@
 #!/usr/bin/env python3
 """
 generate_corpus_manifest.py — build the public exam-corpus manifest from OFFICIAL sources,
 verifying every source URL is live before it is written.
 Output: exam-corpus.yaml (consumed by run/initialization/seed_exam_corpus.py).
 Sources (all official exam-board hosts; public past-paper PDFs):
  AQA      filestore.aqa.org.uk  — fully templatable; enumerated + HEAD-verified here.
  Edexcel  qualifications.pearson.com — date suffix non-derivable; confirmed URLs embedded.
  OCR      www.ocr.org.uk/Images — opaque doc-id; confirmed URLs embedded.
 Every URL is HEAD/GET-checked (200 + application/pdf) before inclusion, so the committed
 manifest never carries a dead or wrong-cased link. Re-run to refresh as more sessions go public.
 Conventions (locked — see ~/cc/ideas/2026-06-07-exam-paper-ingestion.md):
  session   = "YYYY-Mon"        e.g. 2022-Jun
  exam_code = BOARD-award-PAPER-SESSIONCOMPACT-ROLE   e.g. AQA-8463-1H-2022JUN-QP
 """
 from __future__ import annotations
 import concurrent.futures as cf
 import os
 import sys
 import urllib.error
 import urllib.request
 from typing import Any, Dict, List, Optional, Tuple
 import yaml
 AQA_BASE = "https://filestore.aqa.org.uk/sample-papers-and-mark-schemes"
 ROLE_TOKEN = {"QP": "QP", "MS": "MS", "ER": "WRE"}       # AQA filestore role tokens
 MONTHS = {"JUN": ("june", "Jun"), "NOV": ("november", "Nov")}
 FETCHED = "2026-06-07"
 def head_ok(url: str, timeout: int = 20) -> bool:
    """True iff the URL resolves to a real PDF (200 + application/pdf), following redirects.
    AQA soft-404s redirect to www.aqa.org.uk/req_path=... (text/html), so we check content-type.
    Uses a tiny Range GET (stdlib urllib) so we never pull the whole PDF just to verify it."""
    req = urllib.request.Request(url, headers={"Range": "bytes=0-3", "User-Agent": "cc-corpus/1.0"})
    try:
        with urllib.request.urlopen(req, timeout=timeout) as r:
            ctype = (r.headers.get("content-type") or "").lower()
            return r.status in (200, 206) and "pdf" in ctype
    except urllib.error.HTTPError as e:
        # A 206/200 PDF never lands here; 404/redirect-to-html will.
        ctype = (e.headers.get("content-type") or "").lower() if e.headers else ""
        return e.code in (200, 206) and "pdf" in ctype
    except Exception:
        return False
 # ─────────────────────────── AQA catalogue ───────────────────────────
 # spec_code, subject, award, award_level, first_teach, [(filestore_papercode, paper_code, tier), ...]
 def _gcse_single(award: str) -> List[Tuple[str, str, Optional[str]]]:
    out = []
    for paper in ("1", "2"):
        for tier in ("F", "H"):
            out.append((f"{award}{paper}{tier}", f"{award}/{paper}{tier}", tier))
    return out
 def _trilogy(award: str) -> List[Tuple[str, str, Optional[str]]]:
    out = []
    for subj in ("B", "C", "P"):
        for paper in ("1", "2"):
            for tier in ("F", "H"):
                out.append((f"{award}{subj}{paper}{tier}", f"{award}/{subj}/{paper}{tier}", tier))
    return out
 def _alevel(award: str, papers=("1", "2", "3")) -> List[Tuple[str, str, Optional[str]]]:
    return [(f"{award}{p}", f"{award}/{p}", None) for p in papers]
 def _subj(award: str, papers, tiers=(None,)) -> List[Tuple[str, str, Optional[str]]]:
    """Generic GCSE/A-level builder. tiers=('F','H') for tiered subjects (Maths/Science),
    tiers=(None,) for untiered (English/Geography/CS/Business/Psychology)."""
    out = []
    for p in papers:
        for t in tiers:
            tl = t or ""
            out.append((f"{award}{p}{tl}", f"{award}/{p}{tl}", t))
    return out
 def _mfl(award: str) -> List[Tuple[str, str, Optional[str]]]:
    """AQA MFL: Listening/Reading/Writing papers, each Foundation/Higher (Speaking is teacher-conducted,
    no public QP). Filestore code encodes skill+tier, e.g. 8658LH = French Listening Higher."""
    out = []
    for skill in ("L", "R", "W"):
        for t in ("F", "H"):
            out.append((f"{award}{skill}{t}", f"{award}/{skill}{t}", t))
    return out
 AQA_SPECS = [
    # ── Sciences (round 1 — kept at full depth) ──────────────────────────────────────
    ("AQA-BIOL-8461", "BIOLOGY", "8461", "GCSE", "2016", _gcse_single("8461")),
    ("AQA-CHEM-8462", "CHEMISTRY", "8462", "GCSE", "2016", _gcse_single("8462")),
    ("AQA-PHYS-8463", "PHYSICS", "8463", "GCSE", "2016", _gcse_single("8463")),
    ("AQA-COMB-8464", "COMBINED SCIENCE TRILOGY", "8464", "GCSE", "2016", _trilogy("8464")),
    ("AQA-BIOL-7401", "BIOLOGY", "7401", "AS", "2015", _alevel("7401", ("1", "2"))),
    ("AQA-BIOL-7402", "BIOLOGY", "7402", "A-level", "2015", _alevel("7402")),
    ("AQA-CHEM-7404", "CHEMISTRY", "7404", "AS", "2015", _alevel("7404", ("1", "2"))),
    ("AQA-CHEM-7405", "CHEMISTRY", "7405", "A-level", "2015", _alevel("7405")),
    ("AQA-PHYS-7407", "PHYSICS", "7407", "AS", "2015", _alevel("7407", ("1", "2"))),
    ("AQA-PHYS-7408", "PHYSICS", "7408", "A-level", "2015", _alevel("7408")),
    # ── Round 2 breadth — high-volume core (Maths, English) ───────────────────────────
    ("AQA-MATH-8300", "MATHEMATICS", "8300", "GCSE", "2015", _subj("8300", ("1", "2", "3"), ("F", "H"))),
    ("AQA-MATH-7357", "MATHEMATICS", "7357", "A-level", "2017", _alevel("7357", ("1", "2", "3"))),
    ("AQA-MATH-7356", "MATHEMATICS", "7356", "AS", "2017", _alevel("7356", ("1", "2"))),
    ("AQA-ENGL-8700", "ENGLISH LANGUAGE", "8700", "GCSE", "2015", _subj("8700", ("1", "2"))),
    ("AQA-ENGLIT-8702", "ENGLISH LITERATURE", "8702", "GCSE", "2015", _subj("8702", ("1", "2"))),
    ("AQA-ENGL-7702", "ENGLISH LANGUAGE", "7702", "A-level", "2015", _alevel("7702", ("1", "2"))),
    ("AQA-ENGLIT-7712", "ENGLISH LITERATURE A", "7712", "A-level", "2015", _alevel("7712", ("1", "2"))),
    # ── Round 2 breadth — humanities / others ─────────────────────────────────────────
    ("AQA-GEOG-8035", "GEOGRAPHY", "8035", "GCSE", "2016", _subj("8035", ("1", "2", "3"))),
    ("AQA-GEOG-7037", "GEOGRAPHY", "7037", "A-level", "2016", _alevel("7037", ("1", "2"))),
    ("AQA-COMP-8525", "COMPUTER SCIENCE", "8525", "GCSE", "2020", _subj("8525", ("1", "2"))),
    ("AQA-COMP-7517", "COMPUTER SCIENCE", "7517", "A-level", "2015", _alevel("7517", ("1", "2"))),
    ("AQA-BUS-8132", "BUSINESS", "8132", "GCSE", "2017", _subj("8132", ("1", "2"))),
    ("AQA-BUS-7132", "BUSINESS", "7132", "A-level", "2015", _alevel("7132", ("1", "2", "3"))),
    ("AQA-PSYC-8182", "PSYCHOLOGY", "8182", "GCSE", "2017", _subj("8182", ("1", "2"))),
    ("AQA-PSYC-7182", "PSYCHOLOGY", "7182", "A-level", "2015", _alevel("7182", ("1", "2", "3"))),
    # ── Round 2 breadth — modern foreign languages (Listening/Reading/Writing, F+H) ───
    ("AQA-FREN-8658", "FRENCH", "8658", "GCSE", "2016", _mfl("8658")),
    ("AQA-SPAN-8698", "SPANISH", "8698", "GCSE", "2016", _mfl("8698")),
    ("AQA-GERM-8668", "GERMAN", "8668", "GCSE", "2016", _mfl("8668")),
    ("AQA-FREN-7652", "FRENCH", "7652", "A-level", "2016", _alevel("7652", ("1", "2"))),
    ("AQA-SPAN-7692", "SPANISH", "7692", "A-level", "2016", _alevel("7692", ("1", "2"))),
    ("AQA-GERM-7662", "GERMAN", "7662", "A-level", "2016", _alevel("7662", ("1", "2"))),
 ]
 AQA_SESSIONS = ["JUN18", "JUN19", "NOV20", "NOV21", "JUN22", "JUN23", "JUN24"]
 AQA_ROLES = ["QP", "MS", "ER"]
 def aqa_url(papercode: str, role: str, session: str) -> Tuple[str, str]:
    mon = session[:3]
    yy = session[3:]
    folder, _ = MONTHS[mon]
    year = "20" + yy
    fname = f"AQA-{papercode}-{ROLE_TOKEN[role]}-{session}.PDF"
    return f"{AQA_BASE}/{year}/{folder}/{fname}", fname
 def session_pretty(session: str) -> Tuple[str, str]:
    mon = session[:3]       # "JUN" | "NOV"
    yy = session[3:]        # "22"
    _, pretty = MONTHS[mon]
    # ("2022-Jun" display session, "2022JUN" compact for exam_code — year-first, matches the
    # locked exam_code convention and the Edexcel/OCR entries).
    return f"20{yy}-{pretty}", f"20{yy}{mon}"
 def build_aqa() -> Dict[str, Any]:
    candidates: List[Tuple[str, str, str, str, str, str, Optional[str], str, str, str]] = []
    # (spec_code, subject, award, paper_fc, paper_code, tier, role, session, url, fname)
    spec_meta = {}
    for spec_code, subject, award, level, first_teach, papers in AQA_SPECS:
        spec_meta[spec_code] = (subject, award, level, first_teach)
        for paper_fc, paper_code, tier in papers:
            for session in AQA_SESSIONS:
                for role in AQA_ROLES:
                    url, fname = aqa_url(paper_fc, role, session)
                    candidates.append((spec_code, subject, award, paper_fc, paper_code, tier,
                                       role, session, url, fname))
    print(f"[AQA] HEAD-verifying {len(candidates)} candidate URLs...", file=sys.stderr)
    live: Dict[int, bool] = {}
    with cf.ThreadPoolExecutor(max_workers=24) as ex:
        futs = {ex.submit(head_ok, c[8]): i for i, c in enumerate(candidates)}
        done = 0
        for fut in cf.as_completed(futs):
            i = futs[fut]
            live[i] = fut.result()
            done += 1
            if done % 60 == 0:
                print(f"  ...{done}/{len(candidates)} ({sum(live.values())} live)", file=sys.stderr)
    specs: Dict[str, Dict[str, Any]] = {}
    for i, c in enumerate(candidates):
        if not live.get(i):
            continue
        spec_code, subject, award, paper_fc, paper_code, tier, role, session, url, fname = c
        sess_pretty, sess_compact = session_pretty(session)
        token = paper_fc[len(award):]  # "1H" / "P1H" / "1"
        exam_code = f"AQA-{award}-{token}-{sess_compact}-{role}"
        spec = specs.setdefault(spec_code, {"papers": []})
        spec["papers"].append({
            "exam_code": exam_code,
            "paper_code": paper_code,
            "tier": tier,
            "session": sess_pretty,
            "doc_type": role,
            "file": {
                "source": f"url:{url}",
                "original_name": fname,
                "provenance": {"source_url": url, "fetched": FETCHED,
                               "license": "AQA public past paper"},
            },
        })
    spec_list = []
    for spec_code, subject, award, level, first_teach, _papers in AQA_SPECS:
        if spec_code not in specs:
            continue
        papers = sorted(specs[spec_code]["papers"], key=lambda p: p["exam_code"])
        spec_list.append({
            "spec_code": spec_code, "exam_board_code": "AQA", "subject_code": subject,
            "award_code": award, "award_level": level, "first_teach": first_teach,
            "papers": papers,
        })
        print(f"[AQA] {spec_code}: {len(papers)} live papers", file=sys.stderr)
    return {"exam_board_code": "AQA", "specifications": spec_list}
 # ─────────────── Edexcel / OCR — confirmed direct URLs (re-verified at build) ───────────────
 # These boards aren't templatable (Edexcel has a non-derivable date suffix; OCR uses opaque
 # doc-ids), so confirmed URLs are listed as 6-tuples: (spec_code, paper_code, tier, session, role,
 # url). exam_code is DERIVED (see _mk_exam_code) so it always matches the locked convention.
 EXAM_CODE_PREFIX = {"EDEXCEL": "EDX", "OCR": "OCR"}
 def _ec_token(paper_code: str) -> str:
    t = paper_code.split("/")[-1]
    return str(int(t)) if t.isdigit() else t   # "01"->"1", "1H"->"1H", "1CH"->"1CH", "11"->"11"
 def _mk_exam_code(prefix: str, award: str, paper_code: str, session: str, role: str) -> str:
    y, m = session.split("-")
    return f"{prefix}-{award}-{_ec_token(paper_code)}-{y}{m.upper()}-{role}"
 _PE = "https://qualifications.pearson.com/content/dam/pdf"
 _EDX = f"{_PE}/GCSE/Science/2016"
 _OCR = "https://www.ocr.org.uk/Images"
 EDEXCEL_SPECS = {
    "EDX-BIOL-1BI0": ("BIOLOGY", "1BI0", "GCSE", "2016"),
    "EDX-CHEM-1CH0": ("CHEMISTRY", "1CH0", "GCSE", "2016"),
    "EDX-PHYS-1PH0": ("PHYSICS", "1PH0", "GCSE", "2016"),
    "EDX-COMB-1SC0": ("COMBINED SCIENCE", "1SC0", "GCSE", "2016"),
    "EDX-MATH-1MA1": ("MATHEMATICS", "1MA1", "GCSE", "2015"),
    "EDX-ENGL-1EN0": ("ENGLISH LANGUAGE", "1EN0", "GCSE", "2015"),
    "EDX-ENGLIT-1ET0": ("ENGLISH LITERATURE", "1ET0", "GCSE", "2015"),
    "EDX-GEOG-1GA0": ("GEOGRAPHY A", "1GA0", "GCSE", "2016"),
    "EDX-HIST-1HI0": ("HISTORY", "1HI0", "GCSE", "2016"),
    "EDX-BUS-1BS0": ("BUSINESS", "1BS0", "GCSE", "2017"),
    "EDX-COMP-1CP2": ("COMPUTER SCIENCE", "1CP2", "GCSE", "2020"),
    "EDX-MATH-9MA0": ("MATHEMATICS", "9MA0", "A-level", "2017"),
    "EDX-ENGL-9EN0": ("ENGLISH LANGUAGE", "9EN0", "A-level", "2015"),
    "EDX-ENGLIT-9ET0": ("ENGLISH LITERATURE", "9ET0", "A-level", "2015"),
    "EDX-GEOG-9GE0": ("GEOGRAPHY", "9GE0", "A-level", "2016"),
 }
 EDEXCEL_PAPERS = [
    # ── Sciences (round 1) ──
    ("EDX-BIOL-1BI0", "1BI0/1H", "H", "2024-Jun", "QP", f"{_EDX}/Exam-materials/1bi0-1h-que-20240511.pdf"),
    ("EDX-BIOL-1BI0", "1BI0/2F", "F", "2023-Jun", "QP", f"{_EDX}/Exam-materials/1bi0-2f-que-20230610.pdf"),
    ("EDX-BIOL-1BI0", "1BI0/2H", "H", "2023-Jun", "QP", f"{_EDX}/Exam-materials/1bi0-2h-que-20230610.pdf"),
    ("EDX-BIOL-1BI0", "1BI0/1F", "F", "2023-Jun", "MS", f"{_EDX}/Exam-materials/1bi0-1f-rms-20230824.pdf"),
    ("EDX-BIOL-1BI0", "1BI0/1H", "H", "2024-Jun", "MS", f"{_EDX}/Exam-materials/1bi0-1h-rms-20240822.pdf"),
    ("EDX-BIOL-1BI0", "1BI0/1H", "H", "2022-Jun", "MS", f"{_EDX}/exam-materials/1bi0-1h-rms-20220825.pdf"),
    ("EDX-CHEM-1CH0", "1CH0/1F", "F", "2023-Jun", "QP", f"{_EDX}/Exam-materials/1ch0-1f-que-20230523.pdf"),
    ("EDX-CHEM-1CH0", "1CH0/1H", "H", "2024-Jun", "QP", f"{_EDX}/Exam-materials/1ch0-1h-que-20240518.pdf"),
    ("EDX-CHEM-1CH0", "1CH0/2H", "H", "2024-Jun", "MS", f"{_EDX}/Exam-materials/1ch0-2h-rms-20240822.pdf"),
    ("EDX-PHYS-1PH0", "1PH0/1H", "H", "2023-Jun", "QP", f"{_EDX}/Exam-materials/1ph0-1h-que-20230526.pdf"),
    ("EDX-PHYS-1PH0", "1PH0/2F", "F", "2023-Jun", "QP", f"{_EDX}/Exam-materials/1ph0-2f-que-20230617.pdf"),
    ("EDX-PHYS-1PH0", "1PH0/1H", "H", "2024-Jun", "QP", f"{_EDX}/Exam-materials/1ph0-1h-que-20240523.pdf"),
    ("EDX-PHYS-1PH0", "1PH0/2H", "H", "2023-Jun", "MS", f"{_EDX}/Exam-materials/1ph0-2h-rms-20230824.pdf"),
    ("EDX-PHYS-1PH0", "1PH0/2H", "H", "2022-Jun", "MS", f"{_EDX}/exam-materials/1ph0-2h-rms-20220825.pdf"),
    ("EDX-COMB-1SC0", "1SC0/1CH", None, "2023-Jun", "MS", f"{_EDX}/Exam-materials/1sc0-1ch-rms-20230824.pdf"),
    # ── Maths 1MA1 (round 2) ──
    ("EDX-MATH-1MA1", "1MA1/1H", "H", "2023-Jun", "QP", f"{_PE}/GCSE/Mathematics/2015/Exam-materials/1ma1-1h-que-20230520.pdf"),
    ("EDX-MATH-1MA1", "1MA1/1H", "H", "2023-Jun", "MS", f"{_PE}/GCSE/Mathematics/2015/Exam-materials/1ma1-1h-rms-20230824.pdf"),
    ("EDX-MATH-1MA1", "1MA1/1F", "F", "2023-Jun", "MS", f"{_PE}/GCSE/Mathematics/2015/Exam-materials/1ma1-1f-rms-20230824.pdf"),
    ("EDX-MATH-1MA1", "1MA1/1F", "F", "2024-Jun", "QP", f"{_PE}/GCSE/Mathematics/2015/Exam-materials/1ma1-1f-que-20240517.pdf"),
    ("EDX-MATH-1MA1", "1MA1/1H", "H", "2024-Jun", "QP", f"{_PE}/GCSE/Mathematics/2015/Exam-materials/1ma1-1h-que-20240517.pdf"),
    ("EDX-MATH-1MA1", "1MA1/1F", "F", "2024-Jun", "MS", f"{_PE}/GCSE/Mathematics/2015/Exam-materials/1ma1-1f-rms-20240822.pdf"),
    ("EDX-MATH-1MA1", "1MA1/1H", "H", "2023-Nov", "MS", f"{_PE}/GCSE/Mathematics/2015/Exam-materials/1ma1-1h-rms-20240111.pdf"),
    ("EDX-MATH-1MA1", "1MA1/1H", "H", "2022-Jun", "MS", f"{_PE}/GCSE/mathematics/2015/exam-materials/1ma1-1h-rms-20220825.pdf"),
    ("EDX-MATH-1MA1", "1MA1/3H", "H", "2022-Jun", "MS", f"{_PE}/GCSE/mathematics/2015/exam-materials/1ma1-3h-rms-20220825.pdf"),
    # ── English Language 1EN0 / Literature 1ET0 (round 2) ──
    ("EDX-ENGL-1EN0", "1EN0/01", None, "2024-Jun", "QP", f"{_PE}/GCSE/English-Language/2015/Exam-materials/1en0-01-que-20240524.pdf"),
    ("EDX-ENGL-1EN0", "1EN0/01", None, "2023-Nov", "QP", f"{_PE}/GCSE/English-Language/2015/Exam-materials/1en0-01-que-20231108.pdf"),
    ("EDX-ENGL-1EN0", "1EN0/01", None, "2024-Jun", "MS", f"{_PE}/GCSE/English-Language/2015/Exam-materials/1en0-01-rms-20240822.pdf"),
    ("EDX-ENGL-1EN0", "1EN0/02", None, "2024-Jun", "MS", f"{_PE}/GCSE/English-Language/2015/Exam-materials/1en0-02-rms-20240822.pdf"),
    ("EDX-ENGL-1EN0", "1EN0/01", None, "2023-Jun", "MS", f"{_PE}/GCSE/English-Language/2015/Exam-materials/1en0-01-rms-20230824.pdf"),
    ("EDX-ENGL-1EN0", "1EN0/02", None, "2023-Jun", "MS", f"{_PE}/GCSE/English-Language/2015/Exam-materials/1en0-02-rms-20230824.pdf"),
    ("EDX-ENGLIT-1ET0", "1ET0/01", None, "2023-Jun", "QP", f"{_PE}/GCSE/English-Literature/2015/Exam-materials/1et0-01-que-20230518.pdf"),
    ("EDX-ENGLIT-1ET0", "1ET0/02", None, "2023-Jun", "QP", f"{_PE}/GCSE/English-Literature/2015/Exam-materials/1et0-02-que-20230525.pdf"),
    ("EDX-ENGLIT-1ET0", "1ET0/02", None, "2024-Jun", "QP", f"{_PE}/GCSE/English-Literature/2015/Exam-materials/1et0-02-que-20240521.pdf"),
    ("EDX-ENGLIT-1ET0", "1ET0/01", None, "2023-Jun", "MS", f"{_PE}/GCSE/English-Literature/2015/Exam-materials/1et0-01-rms-20230824.pdf"),
    ("EDX-ENGLIT-1ET0", "1ET0/01", None, "2024-Jun", "MS", f"{_PE}/GCSE/English-Literature/2015/Exam-materials/1et0-01-rms-20240822.pdf"),
    # ── A-level Maths 9MA0 / English 9EN0 / 9ET0 (round 2) ──
    ("EDX-MATH-9MA0", "9MA0/01", None, "2023-Jun", "QP", f"{_PE}/A-Level/Mathematics/2017/Exam-materials/9ma0-01-que-20230607.pdf"),
    ("EDX-MATH-9MA0", "9MA0/31", None, "2023-Jun", "QP", f"{_PE}/A-Level/Mathematics/2017/Exam-materials/9ma0-31-que-20230621.pdf"),
    ("EDX-MATH-9MA0", "9MA0/02", None, "2024-Jun", "QP", f"{_PE}/A-Level/Mathematics/2017/Exam-materials/9ma0-02-que-20240612.pdf"),
    ("EDX-MATH-9MA0", "9MA0/31", None, "2023-Jun", "MS", f"{_PE}/A-Level/Mathematics/2017/Exam-materials/9ma0-31-rms-20230817.pdf"),
    ("EDX-MATH-9MA0", "9MA0/01", None, "2024-Jun", "MS", f"{_PE}/A-Level/Mathematics/2017/Exam-materials/9ma0-01-rms-20240815.pdf"),
    ("EDX-ENGL-9EN0", "9EN0/01", None, "2024-Jun", "MS", f"{_PE}/A-Level/English-Language/2015/Exam-materials/9en0-01-rms-20240815.pdf"),
    ("EDX-ENGL-9EN0", "9EN0/02", None, "2024-Jun", "MS", f"{_PE}/A-Level/English-Language/2015/Exam-materials/9en0-02-rms-20240815.pdf"),
    ("EDX-ENGLIT-9ET0", "9ET0/01", None, "2024-Jun", "QP", f"{_PE}/A-Level/English-Literature/2015/Exam-materials/9et0-01-que-20240525.pdf"),
    ("EDX-ENGLIT-9ET0", "9ET0/01", None, "2023-Jun", "MS", f"{_PE}/A-Level/English-Literature/2015/Exam-materials/9et0-01-rms-20230817.pdf"),
    ("EDX-ENGLIT-9ET0", "9ET0/03", None, "2023-Jun", "MS", f"{_PE}/A-Level/English-Literature/2015/Exam-materials/9et0-03-rms-20230817.pdf"),
    # ── Humanities (round 2) ──
    ("EDX-GEOG-1GA0", "1GA0/01", None, "2023-Jun", "QP", f"{_PE}/GCSE/Geography-A/2016/Exam-materials/1ga0-01-que-20230523.pdf"),
    ("EDX-GEOG-1GA0", "1GA0/01", None, "2023-Jun", "MS", f"{_PE}/GCSE/Geography-A/2016/Exam-materials/1ga0-01-rms-20230824.pdf"),
    ("EDX-GEOG-1GA0", "1GA0/02", None, "2023-Jun", "QP", f"{_PE}/GCSE/Geography-A/2016/Exam-materials/1ga0-02-que-20230610.pdf"),
    ("EDX-GEOG-1GA0", "1GA0/02", None, "2023-Jun", "MS", f"{_PE}/GCSE/Geography-A/2016/Exam-materials/1ga0-02-rms-20230824.pdf"),
    ("EDX-GEOG-1GA0", "1GA0/01", None, "2024-Jun", "MS", f"{_PE}/GCSE/Geography-A/2016/Exam-materials/1ga0-01-rms-20240822.pdf"),
    ("EDX-GEOG-1GA0", "1GA0/03", None, "2024-Jun", "QP", f"{_PE}/GCSE/Geography-A/2016/Exam-materials/1ga0-03-que-20240615.pdf"),
    ("EDX-HIST-1HI0", "1HI0/10", None, "2023-Jun", "QP", f"{_PE}/GCSE/History/2016/Exam-materials/1hi0-10-que-20230519.pdf"),
    ("EDX-HIST-1HI0", "1HI0/10", None, "2023-Jun", "MS", f"{_PE}/GCSE/History/2016/Exam-materials/1hi0-10-rms-20230824.pdf"),
    ("EDX-HIST-1HI0", "1HI0/12", None, "2023-Jun", "MS", f"{_PE}/GCSE/History/2016/Exam-materials/1hi0-12-rms-20230824.pdf"),
    ("EDX-HIST-1HI0", "1HI0/13", None, "2024-Jun", "MS", f"{_PE}/GCSE/History/2016/Exam-materials/1hi0-13-rms-20240822.pdf"),
    ("EDX-HIST-1HI0", "1HI0/33", None, "2023-Jun", "MS", f"{_PE}/GCSE/History/2016/Exam-materials/1hi0-33-rms-20230824.pdf"),
    ("EDX-BUS-1BS0", "1BS0/01", None, "2023-Jun", "QP", f"{_PE}/GCSE/Business/2017/Exam-materials/1bs0-01-que-20230519.pdf"),
    ("EDX-BUS-1BS0", "1BS0/02", None, "2023-Jun", "QP", f"{_PE}/GCSE/Business/2017/Exam-materials/1bs0-02-que-20230613.pdf"),
    ("EDX-BUS-1BS0", "1BS0/02", None, "2023-Jun", "MS", f"{_PE}/GCSE/Business/2017/Exam-materials/1bs0-02-rms-20230824.pdf"),
    ("EDX-BUS-1BS0", "1BS0/02", None, "2024-Jun", "QP", f"{_PE}/GCSE/Business/2017/Exam-materials/1bs0-02-que-20240606.pdf"),
    ("EDX-BUS-1BS0", "1BS0/01", None, "2024-Jun", "MS", f"{_PE}/GCSE/Business/2017/Exam-materials/1bs0-01-rms-20240822.pdf"),
    ("EDX-COMP-1CP2", "1CP2/01", None, "2023-Jun", "QP", f"{_PE}/GCSE/Computer-science/2020/Exam-materials/1cp2-01-que-20230520.pdf"),
    ("EDX-COMP-1CP2", "1CP2/01", None, "2023-Jun", "MS", f"{_PE}/GCSE/Computer-science/2020/Exam-materials/1cp2-01-rms-20230824.pdf"),
    ("EDX-COMP-1CP2", "1CP2/02", None, "2023-Jun", "QP", f"{_PE}/GCSE/Computer-science/2020/Exam-materials/1cp2-02-que-20230526.pdf"),
    ("EDX-COMP-1CP2", "1CP2/01", None, "2024-Jun", "QP", f"{_PE}/GCSE/Computer-Science/2020/Exam-materials/1cp2-01-que-20240702.pdf"),
    ("EDX-COMP-1CP2", "1CP2/01", None, "2024-Jun", "MS", f"{_PE}/GCSE/Computer-science/2020/Exam-materials/1cp2-01-rms-20240822.pdf"),
    ("EDX-GEOG-9GE0", "9GE0/01", None, "2023-Jun", "QP", f"{_PE}/A-Level/Geography/2016/Exam-materials/9ge0-01-que-20230518.pdf"),
 ]
 OCR_SPECS = {
    "OCR-BIOL-J247": ("BIOLOGY", "J247", "GCSE", "2016"),
    "OCR-CHEM-J248": ("CHEMISTRY", "J248", "GCSE", "2016"),
    "OCR-PHYS-J249": ("PHYSICS", "J249", "GCSE", "2016"),
    "OCR-COMB-J250": ("COMBINED SCIENCE", "J250", "GCSE", "2016"),
    "OCR-MATH-J560": ("MATHEMATICS", "J560", "GCSE", "2015"),
    "OCR-ENGL-J351": ("ENGLISH LANGUAGE", "J351", "GCSE", "2015"),
    "OCR-ENGLIT-J352": ("ENGLISH LITERATURE", "J352", "GCSE", "2015"),
    "OCR-COMP-J277": ("COMPUTER SCIENCE", "J277", "GCSE", "2020"),
    "OCR-GEOG-J383": ("GEOGRAPHY A", "J383", "GCSE", "2016"),
    "OCR-BUS-J204": ("BUSINESS", "J204", "GCSE", "2017"),
    "OCR-HIST-J411": ("HISTORY B (SHP)", "J411", "GCSE", "2016"),
    "OCR-MATH-H240": ("MATHEMATICS A", "H240", "A-level", "2017"),
    "OCR-ENGLIT-H472": ("ENGLISH LITERATURE", "H472", "A-level", "2015"),
    "OCR-ENGL-H470": ("ENGLISH LANGUAGE", "H470", "A-level", "2015"),
 }
 OCR_PAPERS = [
    # ── Sciences (round 1) ──
    ("OCR-BIOL-J247", "J247/01", "F", "2024-Jun", "QP", f"{_OCR}/727713-question-paper-paper-1.pdf"),
    ("OCR-BIOL-J247", "J247/01", "F", "2024-Jun", "MS", f"{_OCR}/727745-mark-scheme-paper-1.pdf"),
    ("OCR-BIOL-J247", "J247/03", "H", "2024-Jun", "QP", f"{_OCR}/727715-question-paper-paper-3.pdf"),
    ("OCR-BIOL-J247", "J247/03", "H", "2024-Jun", "MS", f"{_OCR}/727747-mark-scheme-paper-3.pdf"),
    ("OCR-BIOL-J247", "J247/01", "F", "2023-Jun", "QP", f"{_OCR}/704945-question-paper-paper-1.pdf"),
    ("OCR-BIOL-J247", "J247/03", "H", "2023-Jun", "MS", f"{_OCR}/704979-mark-scheme-paper-3.pdf"),
    ("OCR-BIOL-J247", "J247/03", "H", "2022-Jun", "QP", f"{_OCR}/678031-question-paper-paper-3.pdf"),
    ("OCR-BIOL-J247", "J247/01", "F", "2022-Jun", "MS", f"{_OCR}/678076-mark-scheme-paper-1.pdf"),
    ("OCR-CHEM-J248", "J248/01", "F", "2024-Jun", "QP", f"{_OCR}/727718-question-paper-paper-1.pdf"),
    ("OCR-CHEM-J248", "J248/03", "H", "2024-Jun", "MS", f"{_OCR}/727751-mark-scheme-paper-3.pdf"),
    ("OCR-CHEM-J248", "J248/01", "F", "2023-Jun", "QP", f"{_OCR}/704950-question-paper-paper-1.pdf"),
    ("OCR-CHEM-J248", "J248/03", "H", "2022-Jun", "QP", f"{_OCR}/678036-question-paper-paper-3.pdf"),
    ("OCR-PHYS-J249", "J249/01", "F", "2024-Jun", "QP", f"{_OCR}/727724-question-paper-paper-1.pdf"),
    ("OCR-PHYS-J249", "J249/03", "H", "2024-Jun", "MS", f"{_OCR}/727755-mark-scheme-paper-3.pdf"),
    ("OCR-PHYS-J249", "J249/01", "F", "2023-Jun", "QP", f"{_OCR}/704956-question-paper-paper-1.pdf"),
    ("OCR-PHYS-J249", "J249/03", "H", "2022-Jun", "MS", f"{_OCR}/678086-mark-scheme-paper-3.pdf"),
    ("OCR-COMB-J250", "J250/01", "F", "2024-Jun", "QP", f"{_OCR}/727730-question-paper-paper-1.pdf"),
    ("OCR-COMB-J250", "J250/07", "H", "2024-Jun", "MS", f"{_OCR}/727763-mark-scheme-paper-7.pdf"),
    # ── Maths J560 (round 2) ──
    ("OCR-MATH-J560", "J560/01", "F", "2024-Jun", "QP", f"{_OCR}/727817-question-paper-paper-1.pdf"),
    ("OCR-MATH-J560", "J560/01", "F", "2024-Jun", "MS", f"{_OCR}/727824-mark-scheme-paper-1.pdf"),
    ("OCR-MATH-J560", "J560/04", "H", "2024-Jun", "QP", f"{_OCR}/727820-question-paper-paper-4.pdf"),
    ("OCR-MATH-J560", "J560/04", "H", "2024-Jun", "MS", f"{_OCR}/727827-mark-scheme-paper-4.pdf"),
    ("OCR-MATH-J560", "J560/01", "F", "2023-Jun", "QP", f"{_OCR}/705050-question-paper-paper-1.pdf"),
    ("OCR-MATH-J560", "J560/01", "F", "2023-Jun", "MS", f"{_OCR}/705057-mark-scheme-paper-1.pdf"),
    ("OCR-MATH-J560", "J560/04", "H", "2023-Jun", "QP", f"{_OCR}/705053-question-paper-paper-4.pdf"),
    ("OCR-MATH-J560", "J560/04", "H", "2023-Jun", "MS", f"{_OCR}/705060-mark-scheme-paper-4.pdf"),
    ("OCR-MATH-J560", "J560/01", "F", "2022-Jun", "QP", f"{_OCR}/678149-question-paper-paper-1.pdf"),
    ("OCR-MATH-J560", "J560/01", "F", "2022-Jun", "MS", f"{_OCR}/678156-mark-scheme-paper-1.pdf"),
    ("OCR-MATH-J560", "J560/04", "H", "2022-Jun", "QP", f"{_OCR}/678152-question-paper-paper-4.pdf"),
    ("OCR-MATH-J560", "J560/04", "H", "2022-Jun", "MS", f"{_OCR}/678159-mark-scheme-paper-4.pdf"),
    # ── English Language J351 / Literature J352 (round 2) ──
    ("OCR-ENGL-J351", "J351/01", None, "2024-Jun", "QP", f"{_OCR}/727556-question-paper-communicating-information-and-ideas.pdf"),
    ("OCR-ENGL-J351", "J351/01", None, "2024-Jun", "MS", f"{_OCR}/727658-mark-scheme-communication-information-and-ideas.pdf"),
    ("OCR-ENGL-J351", "J351/02", None, "2024-Jun", "QP", f"{_OCR}/727558-question-paper-exploring-effects-and-impact.pdf"),
    ("OCR-ENGL-J351", "J351/02", None, "2024-Jun", "MS", f"{_OCR}/727659-mark-scheme-exploring-effects-and-impact.pdf"),
    ("OCR-ENGL-J351", "J351/01", None, "2023-Jun", "QP", f"{_OCR}/704782-question-paper-communicating-information-and-ideas.pdf"),
    ("OCR-ENGL-J351", "J351/01", None, "2023-Jun", "MS", f"{_OCR}/704888-mark-scheme-communication-information-and-ideas.pdf"),
    ("OCR-ENGL-J351", "J351/01", None, "2022-Jun", "QP", f"{_OCR}/677852-question-paper-communicating-information-and-ideas.pdf"),
    ("OCR-ENGL-J351", "J351/01", None, "2022-Jun", "MS", f"{_OCR}/677967-mark-scheme-communication-information-and-ideas.pdf"),
    ("OCR-ENGLIT-J352", "J352/01", None, "2024-Jun", "QP", f"{_OCR}/727830-question-paper-exploring-modern-and-literary-heritage-texts.pdf"),
    ("OCR-ENGLIT-J352", "J352/01", None, "2024-Jun", "MS", f"{_OCR}/727832-mark-scheme-exploring-modern-and-literary-heritage-texts.pdf"),
    ("OCR-ENGLIT-J352", "J352/02", None, "2024-Jun", "QP", f"{_OCR}/727831-question-paper-exploring-poetry-and-shakespeare.pdf"),
    ("OCR-ENGLIT-J352", "J352/02", None, "2024-Jun", "MS", f"{_OCR}/727833-mark-scheme-exploring-poetry-and-shakespeare.pdf"),
    ("OCR-ENGLIT-J352", "J352/01", None, "2023-Jun", "QP", f"{_OCR}/705069-question-paper-exploring-modern-and-literary-heritage-texts.pdf"),
    ("OCR-ENGLIT-J352", "J352/01", None, "2023-Jun", "MS", f"{_OCR}/705075-mark-scheme-exploring-modern-and-literary-heritage-texts.pdf"),
    # ── A-level Maths H240 / English Lit H472 / Lang H470 (round 2) ──
    ("OCR-MATH-H240", "H240/01", None, "2024-Jun", "QP", f"{_OCR}/726654-question-paper-pure-mathematics.pdf"),
    ("OCR-MATH-H240", "H240/01", None, "2024-Jun", "MS", f"{_OCR}/726795-mark-scheme-pure-mathematics.pdf"),
    ("OCR-MATH-H240", "H240/02", None, "2024-Jun", "QP", f"{_OCR}/726656-question-paper-pure-mathematics-and-statistics.pdf"),
    ("OCR-MATH-H240", "H240/02", None, "2024-Jun", "MS", f"{_OCR}/726796-mark-scheme-pure-mathematics-and-statistics.pdf"),
    ("OCR-MATH-H240", "H240/01", None, "2023-Jun", "QP", f"{_OCR}/703866-question-paper-pure-mathematics.pdf"),
    ("OCR-MATH-H240", "H240/01", None, "2023-Jun", "MS", f"{_OCR}/704008-mark-scheme-pure-mathematics.pdf"),
    ("OCR-MATH-H240", "H240/01", None, "2022-Jun", "QP", f"{_OCR}/676845-question-paper-pure-mathematics.pdf"),
    ("OCR-MATH-H240", "H240/01", None, "2022-Jun", "MS", f"{_OCR}/677005-mark-scheme-pure-mathematics.pdf"),
    ("OCR-ENGLIT-H472", "H472/01", None, "2024-Jun", "QP", f"{_OCR}/726602-question-paper-drama-and-poetry-pre-1900.pdf"),
    ("OCR-ENGLIT-H472", "H472/01", None, "2024-Jun", "MS", f"{_OCR}/726762-mark-scheme-drama-and-poetry-pre-1900.pdf"),
    ("OCR-ENGLIT-H472", "H472/01", None, "2023-Jun", "QP", f"{_OCR}/703813-question-paper-drama-and-poetry-pre-1900.pdf"),
    ("OCR-ENGLIT-H472", "H472/01", None, "2023-Jun", "MS", f"{_OCR}/703974-mark-scheme-drama-and-poetry-pre-1900.pdf"),
    ("OCR-ENGLIT-H472", "H472/01", None, "2022-Jun", "QP", f"{_OCR}/676783-question-paper-drama-and-poetry-pre-1900.pdf"),
    ("OCR-ENGLIT-H472", "H472/01", None, "2022-Jun", "MS", f"{_OCR}/676965-mark-scheme-drama-and-poetry-pre-1900.pdf"),
    ("OCR-ENGL-H470", "H470/01", None, "2024-Jun", "QP", f"{_OCR}/726595-question-paper-exploring-language.pdf"),
    ("OCR-ENGL-H470", "H470/01", None, "2024-Jun", "MS", f"{_OCR}/726764-mark-scheme-exploring-language.pdf"),
    ("OCR-ENGL-H470", "H470/01", None, "2023-Jun", "QP", f"{_OCR}/703806-question-paper-exploring-language.pdf"),
    ("OCR-ENGL-H470", "H470/01", None, "2023-Jun", "MS", f"{_OCR}/703976-mark-scheme-exploring-language.pdf"),
    ("OCR-ENGL-H470", "H470/01", None, "2022-Jun", "QP", f"{_OCR}/676772-question-paper-exploring-language.pdf"),
    ("OCR-ENGL-H470", "H470/01", None, "2022-Jun", "MS", f"{_OCR}/676967-mark-scheme-exploring-language.pdf"),
    # ── Humanities (round 2) ──
    ("OCR-COMP-J277", "J277/01", None, "2024-Jun", "QP", f"{_OCR}/727534-question-paper-computer-systems.pdf"),
    ("OCR-COMP-J277", "J277/01", None, "2024-Jun", "MS", f"{_OCR}/727652-mark-scheme-computer-systems.pdf"),
    ("OCR-COMP-J277", "J277/02", None, "2024-Jun", "QP", f"{_OCR}/727535-question-paper-computational-thinking-algorithms-and-programming.pdf"),
    ("OCR-COMP-J277", "J277/02", None, "2024-Jun", "MS", f"{_OCR}/727653-mark-scheme-computational-thinking-algorithms-and-programming.pdf"),
    ("OCR-GEOG-J383", "J383/01", None, "2024-Jun", "QP", f"{_OCR}/727564-question-paper-living-in-the-uk-today.pdf"),
    ("OCR-GEOG-J383", "J383/01", None, "2024-Jun", "MS", f"{_OCR}/727661-mark-scheme-living-in-the-uk-today.pdf"),
    ("OCR-GEOG-J383", "J383/02", None, "2024-Jun", "QP", f"{_OCR}/727566-question-paper-the-world-around-us.pdf"),
    ("OCR-GEOG-J383", "J383/02", None, "2024-Jun", "MS", f"{_OCR}/727662-mark-scheme-the-world-around-us.pdf"),
    ("OCR-BUS-J204", "J204/01", None, "2024-Jun", "QP", f"{_OCR}/727519-question-paper-business-1-business-activity-marketing-and-people.pdf"),
    ("OCR-BUS-J204", "J204/01", None, "2024-Jun", "MS", f"{_OCR}/727634-mark-scheme-business-1-business-activity-marketing-and-people.pdf"),
    ("OCR-BUS-J204", "J204/02", None, "2024-Jun", "QP", f"{_OCR}/727520-question-paper-business-2-operations-finance-and-influences-on-business.pdf"),
    ("OCR-BUS-J204", "J204/02", None, "2024-Jun", "MS", f"{_OCR}/727635-mark-scheme-business-2-operations-finance-and-influences-on-business.pdf"),
    ("OCR-BUS-J204", "J204/01", None, "2023-Jun", "QP", f"{_OCR}/704745-question-paper-business-1-business-activity-marketing-and-people.pdf"),
    ("OCR-BUS-J204", "J204/01", None, "2023-Jun", "MS", f"{_OCR}/704864-mark-scheme-business-1-business-activity-marketing-and-people.pdf"),
    ("OCR-HIST-J411", "J411/11", None, "2024-Jun", "QP", f"{_OCR}/727590-question-paper-the-people-s-health-c.1250-to-present-with-the-norman-conquest-1065-1087.pdf"),
    ("OCR-HIST-J411", "J411/11", None, "2024-Jun", "MS", f"{_OCR}/727678-mark-scheme-the-people-s-health-c.1250-to-present-with-the-norman-conquest-1065-1087.pdf"),
 ]
 def build_board(board_code: str, specs_meta: Dict, papers: List) -> Dict[str, Any]:
    prefix = EXAM_CODE_PREFIX[board_code]
    print(f"[{board_code}] re-verifying {len(papers)} confirmed URLs...", file=sys.stderr)
    live: Dict[int, bool] = {}
    with cf.ThreadPoolExecutor(max_workers=24) as ex:
        futs = {ex.submit(head_ok, p[5]): i for i, p in enumerate(papers)}
        for fut in cf.as_completed(futs):
            live[futs[fut]] = fut.result()
    by_spec: Dict[str, List[Dict[str, Any]]] = {}
    for i, (spec_code, paper_code, tier, session, role, url) in enumerate(papers):
        if not live.get(i):
            print(f"  DROP (not live): {url}", file=sys.stderr)
            continue
        award = specs_meta[spec_code][1]
        by_spec.setdefault(spec_code, []).append({
            "exam_code": _mk_exam_code(prefix, award, paper_code, session, role),
            "paper_code": paper_code, "tier": tier,
            "session": session, "doc_type": role,
            "file": {"source": f"url:{url}", "original_name": os.path.basename(url),
                     "provenance": {"source_url": url, "fetched": FETCHED,
                                    "license": f"{board_code} public past paper"}},
        })
    spec_list = []
    for spec_code, (subject, award, level, first_teach) in specs_meta.items():
        if spec_code not in by_spec:
            continue
        spec_list.append({
            "spec_code": spec_code, "exam_board_code": board_code, "subject_code": subject,
            "award_code": award, "award_level": level, "first_teach": first_teach,
            "papers": sorted(by_spec[spec_code], key=lambda p: p["exam_code"]),
        })
        print(f"[{board_code}] {spec_code}: {len(by_spec[spec_code])} live papers", file=sys.stderr)
    return {"exam_board_code": board_code, "specifications": spec_list}
 def main() -> None:
    out_path = os.path.join(os.path.dirname(__file__), "exam-corpus.yaml")
    boards = [
        build_aqa(),
        build_board("EDEXCEL", EDEXCEL_SPECS, EDEXCEL_PAPERS),
        build_board("OCR", OCR_SPECS, OCR_PAPERS),
    ]
    n_specs = sum(len(b["specifications"]) for b in boards)
    n_papers = sum(len(s["papers"]) for b in boards for s in b["specifications"])
    manifest = {
        "version": 1,
        "defaults": {"bucket": "cc.examboards"},
        "provenance": {
            "collected_by": "kcar",
            "collected_at": FETCHED,
            "license_posture": ("Public exam-board past papers downloaded from each board's own "
                                "official site (AQA filestore, Pearson DAM, OCR Images). Stored in "
                                "the private dev cc.examboards bucket for internal exam-marker dev/test. "
                                "Each item records its source_url. Review redistribution rights before "
                                "any public exposure."),
            "sources": {
                "AQA": "https://filestore.aqa.org.uk/sample-papers-and-mark-schemes/",
                "EDEXCEL": "https://qualifications.pearson.com/en/support/support-topics/exams/past-papers.html",
                "OCR": "https://www.ocr.org.uk/qualifications/past-paper-finder/",
            },
        },
        # Optional: uncomment + set on dev .94 to exercise user-side flows / first-sweep.
        # "test_subset": {"user_email": "[email protected]", "papers": 2},
        # "system_identity": {"user_email": "[email protected]"},
        "boards": boards,
    }
    with open(out_path, "w") as fh:
        yaml.safe_dump(manifest, fh, sort_keys=False, default_flow_style=False, width=120)
    print(f"\nWROTE {out_path}: {n_specs} specs, {n_papers} papers across {len(boards)} boards",
          file=sys.stderr)
 if __name__ == "__main__":
    main()
@@ -5,7 +5,6 @@ Clears:
  - Neo4j: drops ALL databases except system, neo4j (including gaisdata, cc.users.*, cc.institutes.*)
  - Supabase: deletes ALL data tables except gais_local_authorities and gais_schools
  - Supabase: deletes all auth users except kcar, then re-seeds kcar profile state
  - Granular scopes can clear exam corpus, timetable data, or --user-subset seed copies
 Safe invariants (never touched):
  - kcar auth account
@@ -83,45 +82,6 @@ SUPABASE_TABLES_TO_CLEAR = [
    "admin_profiles",
 ]
 # Exam-marker subsystem tables, FK child-first. scope="exam-corpus" is deliberately
 # broader than "public papers": it wipes public corpus eb_* rows, templates, layouts,
 # questions, boundaries, response areas, marking batches, student submissions, and mark
 # entries. NOT in the list above — the previous full reset() never cleared exam data
 # or storage at all; the granular scopes below fold it in.
 EXAM_CORPUS_TABLES = [
    "mark_entries",
    "student_submissions",
    "marking_batches",
    "exam_response_areas",
    "exam_boundaries",
    "exam_template_layout",
    "exam_questions",
    "exam_templates",
    "eb_exams",
    "eb_specifications",
 ]
 # Timetable / calendar materialization subset (for scope='timetable').
 TIMETABLE_TABLES = [
    "lesson_deliveries",
    "lesson_collaborators",
    "taught_lessons",
    "academic_periods",
    "academic_days",
    "academic_weeks",
    "academic_term_breaks",
    "academic_terms",
    "academic_years",
    "teacher_timetable_slots",
    "teacher_timetables",
    "school_timetables",
    "planned_lessons",
 ]
 # Bucket whose objects scope="exam-corpus" clears for the whole exam-marker subsystem
 # (Storage API — protect_delete blocks raw SQL).
 EXAM_STORAGE_BUCKET = "cc.examboards"
 def _sb_headers():
    url = os.environ["SUPABASE_URL"]
@@ -134,28 +94,6 @@ def _sb_headers():
    }
 # Markers that identify a production Supabase target. Destructive reset against any of these is
 # refused by default (project rule: ".94 only; .156 human-gated") — set RESET_ALLOW_PROD=1 to override.
 PROD_TARGET_MARKERS = ("192.168.0.156", "supabase.classroomcopilot")
 def _assert_reset_allowed(url: str, scope: str) -> None:
    """Default-deny destructive reset against a production-looking Supabase target.
    The /admin/reset route and this module both act on os.environ['SUPABASE_URL']; without this guard
    a platform-admin call on a prod-deployed API would wipe prod data + exam corpus + storage. We refuse
    when the target matches a known prod marker unless an explicit RESET_ALLOW_PROD opt-in is set.
    """
    target = (url or "").lower()
    looks_prod = any(m in target for m in PROD_TARGET_MARKERS)
    override = os.environ.get("RESET_ALLOW_PROD", "").strip().lower() in ("1", "true", "yes")
    if looks_prod and not override:
        raise RuntimeError(
            f"refusing destructive reset (scope={scope}) against production-looking target {target!r}; "
            f"this is human-gated — set RESET_ALLOW_PROD=1 to override."
        )
 # ─── Neo4j helpers ────────────────────────────────────────────────────────────
 def _neo4j_drop_all_non_system() -> Dict[str, List[str]]:
@@ -208,133 +146,13 @@ def _supabase_delete_auth_user(url: str, headers: dict, uid: str):
        logger.warning(f"  Delete auth user {uid}: {r.status_code} {r.text[:80]}")
 # ─── Granular helpers ───────────────────────────────────────────────────────────
 def _clear_tables(url: str, headers: dict, tables: List[str]) -> "tuple[List[str], List[str]]":
    cleared, failed = [], []
    for table in tables:
        if _sb_clear_table(url, headers, table) in (200, 204):
            cleared.append(table)
            logger.info(f"  ✓ {table}")
        else:
            failed.append(table)
    return cleared, failed
 def _clear_exam_storage() -> Dict[str, Any]:
    """Remove cc.examboards objects for the exam-marker subsystem.
    scope="exam-corpus" is not limited to public-paper metadata: it also removes the
    storage objects that back exam board corpus files and any downstream exam-marker
    artifacts referenced from eb_exams/eb_specifications. Gathers storage_loc from
    eb_exams/eb_specifications BEFORE the rows are cleared.
    """
    try:
        from modules.database.supabase.utils.client import SupabaseServiceRoleClient
        from modules.database.supabase.utils.storage import StorageAdmin
    except Exception as exc:
        logger.warning(f"  exam storage clear skipped (import): {exc}")
        return {"removed": 0, "error": str(exc)}
    sb = SupabaseServiceRoleClient().supabase
    storage = StorageAdmin()
    locs: List[str] = []
    for table in ("eb_exams", "eb_specifications"):
        try:
            rows = sb.table(table).select("storage_loc").execute().data or []
            locs += [r["storage_loc"] for r in rows if r.get("storage_loc")]
        except Exception as exc:
            logger.warning(f"  storage_loc gather {table}: {exc}")
    by_bucket: Dict[str, List[str]] = {}
    for loc in locs:
        if "/" in loc:
            b, _, p = loc.partition("/")
            by_bucket.setdefault(b, []).append(p)
    removed = 0
    for b, paths in by_bucket.items():
        for i in range(0, len(paths), 100):
            chunk = paths[i:i + 100]
            try:
                storage.client.supabase.storage.from_(b).remove(chunk)
                removed += len(chunk)
            except Exception as exc:
                logger.warning(f"  storage remove {b}: {exc}")
    logger.info(f"  exam storage removed {removed} objects from {list(by_bucket)}")
    return {"removed": removed, "buckets": list(by_bucket)}
 def _clear_user_subset_files() -> Dict[str, Any]:
    """Remove files rows and cc.users storage objects created by --user-subset seeding.
    Reuses the seed/unseed implementation so reset(scope="user-subset") has the
    same storage-before-row deletion order and idempotency guarantees as
    seed_exam_corpus.py --unseed. The helper only targets rows marked by the seeder:
    bucket='cc.users', source='exam-corpus-seed', path LIKE 'exam-marker/%'.
    """
    try:
        from modules.database.supabase.utils.client import SupabaseServiceRoleClient
        from modules.database.supabase.utils.storage import StorageAdmin
        from run.initialization.seed_exam_corpus import LoadReport, _delete_user_subset_files
    except Exception as exc:
        logger.warning(f"  user-subset clear skipped (import): {exc}")
        return {"files_rows_deleted": 0, "storage_objects_removed": 0, "errors": [str(exc)]}
    rep = LoadReport()
    _delete_user_subset_files(
        SupabaseServiceRoleClient(),
        StorageAdmin(),
        exam_codes=None,
        rep=rep,
    )
    return {
        "files_rows_deleted": rep.unseed_user_files,
        "storage_objects_removed": rep.unseed_objects,
        "errors": rep.errors,
    }
 # ─── Main reset ───────────────────────────────────────────────────────────────
-def reset(scope: str = "all") -> Dict[str, Any]:
+def reset() -> Dict[str, Any]:
    """Destructive reset. scope ∈ {all, exam-corpus, timetable, user-subset}.
    - all          : full wipe (Neo4j + Supabase data + auth users) AND the entire
                     exam-marker subsystem listed below, including --user-subset copies.
    - exam-corpus  : ONLY the entire exam-marker subsystem, not just public papers:
                     public corpus/eb_* data, cc.examboards storage objects, exam
                     templates, template layouts, questions, boundaries, response
                     areas, marking batches, student submissions, mark entries, and
                     --user-subset cc.users copies.
    - timetable    : ONLY timetable/calendar materialization tables.
    - user-subset  : ONLY files rows and cc.users storage objects created by
                     seed_exam_corpus.py --user-subset.
    """
    scope = (scope or "all").lower()
    if scope not in ("all", "exam-corpus", "timetable", "user-subset"):
        raise ValueError(f"invalid scope {scope!r} (want all|exam-corpus|timetable|user-subset)")
    url, headers = _sb_headers()
    _assert_reset_allowed(url, scope)
    if scope == "exam-corpus":
        logger.info("RESET (scope=exam-corpus) — entire exam-marker subsystem: public corpus/eb_* data, cc.examboards storage, templates/layout/questions/boundaries/response areas, marking batches, submissions, mark entries, and --user-subset copies")
        user_subset = _clear_user_subset_files()
        storage = _clear_exam_storage()
        cleared, failed = _clear_tables(url, headers, EXAM_CORPUS_TABLES)
        return {"scope": scope, "user_subset": user_subset, "exam_storage": storage, "tables_cleared": cleared, "tables_failed": failed}
    if scope == "timetable":
        logger.info("RESET (scope=timetable) — timetable/calendar tables")
        cleared, failed = _clear_tables(url, headers, TIMETABLE_TABLES)
        return {"scope": scope, "tables_cleared": cleared, "tables_failed": failed}
    if scope == "user-subset":
        logger.info("RESET (scope=user-subset) — --user-subset cc.users storage objects and files rows")
        user_subset = _clear_user_subset_files()
        return {"scope": scope, "user_subset": user_subset}
    logger.info("=" * 60)
    logger.info("RESET ENVIRONMENT — full destructive wipe starting")
    logger.info("=" * 60)
-    results: Dict[str, Any] = {"scope": scope}
+    results: Dict[str, Any] = {}
    # ── 1. Neo4j: drop everything except system + neo4j ──────────────────────
    logger.info("\n[Neo4j] Dropping all non-system databases...")
@@ -343,9 +161,6 @@ def reset(scope: str = "all") -> Dict[str, Any]:
    results["neo4j"] = {"dropped": dropped}
    # ── 2. Supabase: clear all data tables (GAIS preserved) ──────────────────
    # First remove --user-subset cc.users storage objects (+ their files rows) via the
    # Storage API, so the generic files-table clear below doesn't strand orphaned objects.
    results["user_subset"] = _clear_user_subset_files()
    logger.info("\n[Supabase] Clearing data tables (preserving gais_*)...")
    url, headers = _sb_headers()
    cleared, failed = [], []
@@ -398,25 +213,11 @@ def reset(scope: str = "all") -> Dict[str, Any]:
    )
    logger.info("  kcar → admin_profiles restored ✓")
    # ── 5. Exam-marker subsystem: storage objects (Storage API) + all exam tables ──
    # This is the same destructive surface as scope="exam-corpus": public corpus/eb_*
    # rows, cc.examboards storage, templates/layout/questions/boundaries/response
    # areas, marking batches, submissions, and mark entries. (The legacy full reset
    # cleared neither exam tables nor storage — folded in here.)
    logger.info("\n[Supabase] Clearing entire exam-marker subsystem (public corpus, storage, templates/layout/questions/boundaries/response areas, marking batches, submissions, mark entries)...")
    exam_storage = _clear_exam_storage()
    exam_cleared, exam_failed = _clear_tables(url, headers, EXAM_CORPUS_TABLES)
    results["supabase"] = {
        "tables_cleared": cleared,
        "tables_failed": failed,
        "deleted_users": deleted_emails,
    }
    results["exam"] = {
        "storage": exam_storage,
        "tables_cleared": exam_cleared,
        "tables_failed": exam_failed,
    }
    logger.info("\n" + "=" * 60)
    logger.info("RESET COMPLETE")
@@ -1,20 +1,15 @@
 """
-seed_curriculum.py — DEPRECATED hardcoded curriculum/exam seeder.
+seed_curriculum.py — Create curriculum data: exam board specifications and exams.
-⚠️ SUPERSEDED (2026-06-07) by the manifest-driven corpus loader:
+Seeds eb_specifications and eb_exams tables with realistic UK exam board data
-    run/initialization/seed_exam_corpus.py  (+ manifests/exam-corpus.yaml)
+(AQA, Edexcel, OCR) for Physics, Maths, and Computer Science across both schools.
-The exam-board parts of this file (eb_specifications / eb_exams) are now seeded from a
+Also seeds curriculum_topics in Neo4j for the school databases.
 verified, provenance-bearing manifest with real uploaded PDFs — not the hardcoded rows
 below. This module also had a storage_loc inconsistency the overhaul standardises away:
 exam-board files belong in the `cc.examboards` bucket at the canonical path
 `cc.examboards/{board}/{subject}/{award}/{paper}/{session}/{role}.pdf`, NOT under
 `cc.public.snapshots/curriculum/...` (the placeholder rows below still show the old path).
-KEEP ONLY for the Neo4j `curriculum_topics` seed (step [3]) which has no replacement yet.
+Tables: eb_specifications, eb_exams
-Do NOT use the eb_specifications/eb_exams blocks for new work — use seed_exam_corpus.py.
+Neo4j: curriculum topic nodes in school databases
-Run (Neo4j curriculum topics only is the supported remaining use):
+Run inside ccapi container:
    python3 -c "from run.initialization.seed_curriculum import seed; seed()"
 """
 import os
@@ -1,867 +0,0 @@
 """
 seed_exam_corpus.py — manifest-driven loader for the public exam-paper corpus.
 SCOPE (separate from infra): assumes storage buckets already exist (provisioned by
 run/initialization/buckets.py during infra init). This loader UPLOADS papers and
 SEEDS the catalogue; it does NOT create buckets.
 Pipeline per manifest item:
  validate -> resolve source bytes (local path | url:, cached) -> upload file to
  cc.examboards (canonical path, skip-if-exists unless --force) -> upsert
  eb_specifications / eb_exams (catalogue) -> (optional, --user-subset) copy a subset
  into a test user's exam space so user-side flows are testable -> (optional,
  --first-sweep) run the docling/auto-map first pass to gather structure.
 Manifest template: ~/cc/specs/exam-corpus-manifest.example.yaml
 Catalogue columns (real — verified against volumes/db/cc/61-core-schema.sql):
  eb_specifications(spec_code UNIQUE, exam_board_code, award_code, subject_code,
                    first_teach, spec_ver, storage_loc, doc_type CHECK(pdf|json|...),
                    doc_details jsonb, docling_docs jsonb)
  eb_exams(exam_code UNIQUE, spec_code FK, paper_code, tier, session, type_code,
           storage_loc, doc_type CHECK(pdf|json|...), doc_details jsonb, docling_docs jsonb)
 IMPORTANT schema note: the QP/MS/INSERT/ER *document role* is stored in `type_code`
 (the `/catalogue` endpoint filters `type_code == 'QP'`). The `doc_type` column is the
 *file format* and is CHECK-constrained to {pdf,json,md,html,txt,doctags} — so it is
 always 'pdf' here. (The manifest field is named `doc_type` for the role; the loader
 maps manifest.doc_type -> DB.type_code and sets DB.doc_type = 'pdf'.)
 Locked conventions (see ~/cc/ideas/2026-06-07-exam-paper-ingestion.md):
  session    = "YYYY-Mon"  e.g. "2022-Jun", "2021-Nov"
  exam_code  = "{BOARD}-{award}-{paper_safe}-{SESSIONCOMPACT}-{ROLE}"  e.g. AQA-8463-1H-2022JUN-QP
  spec path  = cc.examboards/{board}/{subject}/{award}/spec/{spec_ver}.pdf
  paper path = cc.examboards/{board}/{subject}/{award}/{paper_safe}/{session}/{role}.pdf
 Run inside the api container (env: SUPABASE_URL + SERVICE_ROLE_KEY for dev .94), e.g.:
  python3 -m run.initialization.seed_exam_corpus --manifest /path/exam-corpus.yaml --dry-run
  python3 -m run.initialization.seed_exam_corpus --manifest ... --board AQA
  python3 -m run.initialization.seed_exam_corpus --manifest ... --first-sweep
 """
 from __future__ import annotations
 import argparse
 import hashlib
 import os
 from dataclasses import dataclass, field
 from typing import Any, Dict, List, Optional, Tuple
 import requests
 import yaml  # PyYAML
 from modules.logger_tool import initialise_logger
 from modules.database.supabase.utils.client import SupabaseServiceRoleClient
 from modules.database.supabase.utils.storage import StorageAdmin, StorageError
 logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), "default", True)
 EXAM_BUCKET = "cc.examboards"
 # Manifest `doc_type` carries the document ROLE (stored in eb_exams.type_code).
 DOC_ROLES = {"QP", "MS", "INSERT", "ER", "SPECIMEN", "GRADE_BOUNDARIES", "DATA_SHEET"}
 TIERS = {"H", "F", None}
 # Default working dir for cached url: downloads (override with --cache-dir / EXAM_CORPUS_CACHE).
 DEFAULT_CACHE_DIR = os.getenv("EXAM_CORPUS_CACHE", "/tmp/exam-corpus-cache")
 # Persistent, mountable local store laid out exactly like the bucket (download once, seed many,
 # offline-repeatable). Override with --store-dir / EXAM_CORPUS_STORE. Distinct from --cache-dir,
 # which is a throwaway url hash-cache.
 DEFAULT_STORE_DIR = os.getenv(
    "EXAM_CORPUS_STORE",
    os.path.join(os.path.dirname(os.path.abspath(__file__)), "manifests", "_corpus_store"),
 )
 # ─────────────────────────────── canonical storage paths ───────────────────────────────
 def _lc(s: str) -> str:
    return (s or "").strip().lower().replace(" ", "-")
 def _paper_safe(paper_code: str) -> str:
    # Drop the award prefix, keep all remaining segments so combined-science sub-papers
    # don't collide on the storage path:
    #   "8463/1H"    -> "1h"
    #   "8464/B/1H"  -> "b-1h"   (Trilogy: subject letter + paper + tier)
    #   "7408/1"     -> "1"
    parts = _lc(paper_code).split("/")
    return "-".join(parts[1:]) if len(parts) > 1 else parts[0]
 def spec_storage_loc(board: str, subject: str, award: str, spec_ver: str) -> str:
    # e.g. cc.examboards/aqa/physics/8463/spec/1.1.pdf
    return f"{EXAM_BUCKET}/{_lc(board)}/{_lc(subject)}/{_lc(award)}/spec/{_lc(spec_ver or 'spec')}.pdf"
 def paper_storage_loc(board: str, subject: str, award: str, paper_code: str, session: str, doc_role: str) -> str:
    # e.g. cc.examboards/aqa/physics/8463/1h/2022-jun/qp.pdf
    return f"{EXAM_BUCKET}/{_lc(board)}/{_lc(subject)}/{_lc(award)}/{_paper_safe(paper_code)}/{_lc(session)}/{_lc(doc_role)}.pdf"
 # ─────────────────────────────── report ───────────────────────────────
@dataclass
 class LoadReport:
    specs_upserted: int = 0
    papers_upserted: int = 0
    files_uploaded: int = 0
    files_skipped: int = 0
    files_failed: int = 0
    user_copies: int = 0
    swept: int = 0
    sweep_failed: int = 0
    downloaded: int = 0
    download_cached: int = 0
    unseed_objects: int = 0
    unseed_user_files: int = 0
    unseed_exams: int = 0
    unseed_specs: int = 0
    unseed_templates: int = 0
    errors: List[str] = field(default_factory=list)
    def as_dict(self) -> Dict[str, Any]:
        return {
            "specs_upserted": self.specs_upserted,
            "papers_upserted": self.papers_upserted,
            "downloaded": self.downloaded,
            "download_cached": self.download_cached,
            "unseed_objects": self.unseed_objects,
            "unseed_user_files": self.unseed_user_files,
            "unseed_exams": self.unseed_exams,
            "unseed_specs": self.unseed_specs,
            "unseed_templates": self.unseed_templates,
            "files_uploaded": self.files_uploaded,
            "files_skipped": self.files_skipped,
            "files_failed": self.files_failed,
            "user_copies": self.user_copies,
            "swept": self.swept,
            "sweep_failed": self.sweep_failed,
            "errors": self.errors,
        }
 # ─────────────────────────────── validation ───────────────────────────────
 def validate_manifest(m: Dict[str, Any]) -> List[str]:
    errs: List[str] = []
    seen_specs, seen_exams = set(), set()
    for board in m.get("boards", []):
        bcode = board.get("exam_board_code")
        if not bcode:
            errs.append("board missing exam_board_code")
        for spec in board.get("specifications", []):
            sc = spec.get("spec_code")
            if not sc or sc in seen_specs:
                errs.append(f"spec_code missing/duplicate: {sc!r}")
            seen_specs.add(sc)
            for field_name in ("award_code", "subject_code"):
                if not spec.get(field_name):
                    errs.append(f"{sc}: missing {field_name}")
            for p in spec.get("papers", []):
                ec = p.get("exam_code")
                if not ec or ec in seen_exams:
                    errs.append(f"exam_code missing/duplicate: {ec!r}")
                seen_exams.add(ec)
                if p.get("doc_type") not in DOC_ROLES:
                    errs.append(f"{ec}: bad doc_type/role {p.get('doc_type')!r} (want one of {sorted(DOC_ROLES)})")
                if p.get("tier") not in TIERS:
                    errs.append(f"{ec}: bad tier {p.get('tier')!r} (want H|F|null)")
                if not p.get("paper_code"):
                    errs.append(f"{ec}: missing paper_code")
                if not p.get("session"):
                    errs.append(f"{ec}: missing session")
                src = (p.get("file") or {}).get("source")
                if not src:
                    errs.append(f"{ec}: missing file.source")
                elif not src.startswith("url:") and not os.path.exists(src):
                    errs.append(f"{ec}: local source not found: {src}")
    return errs
 # ─────────────────────────────── source resolution (local | url:, cached) ───────────────────────────────
 def _resolve_source_bytes(source: str, *, cache_dir: str) -> bytes:
    """Resolve a manifest file source to bytes.
    'url:https://...'  -> fetch (cached to cache_dir by url hash) ; verifies non-empty.
    '<local path>'     -> read from disk.
    """
    if source.startswith("url:"):
        url = source[len("url:"):]
        os.makedirs(cache_dir, exist_ok=True)
        cache_key = hashlib.sha1(url.encode("utf-8")).hexdigest()
        cache_path = os.path.join(cache_dir, f"{cache_key}.pdf")
        if os.path.exists(cache_path) and os.path.getsize(cache_path) > 0:
            with open(cache_path, "rb") as fh:
                return fh.read()
        logger.info(f"[fetch] {url}")
        resp = requests.get(url, timeout=60, allow_redirects=True)
        resp.raise_for_status()
        data = resp.content
        ctype = resp.headers.get("content-type", "")
        if not data:
            raise ValueError(f"empty download: {url}")
        if "pdf" not in ctype.lower() and not data[:5].startswith(b"%PDF"):
            raise ValueError(f"not a PDF (content-type={ctype!r}): {url}")
        tmp = cache_path + ".part"
        with open(tmp, "wb") as fh:
            fh.write(data)
        os.replace(tmp, cache_path)
        return data
    with open(source, "rb") as fh:
        return fh.read()
 # ─────────────────────── persistent local store (download-once, seed-many) ───────────────────────
 def _store_path(store_dir: str, storage_loc: str) -> str:
    """Local path mirroring the bucket layout (so the store is directly mountable as the corpus):
    storage_loc 'cc.examboards/aqa/physics/8463/1h/2022-jun/qp.pdf'
      -> {store_dir}/aqa/physics/8463/1h/2022-jun/qp.pdf
    """
    _, _, path = storage_loc.partition("/")
    return os.path.join(store_dir, path)
 def _item_bytes(source: str, storage_loc: str, *, store_dir: Optional[str], cache_dir: str,
                populate: bool = True, rep: Optional[LoadReport] = None) -> bytes:
    """Resolve bytes for an item, preferring the persistent local store when present.
    If store_dir holds the file → read it (offline). Otherwise resolve the source (local|url:) and,
    when populate=True, write it into the store at its canonical path for future offline runs.
    """
    if store_dir:
        sp = _store_path(store_dir, storage_loc)
        if os.path.exists(sp) and os.path.getsize(sp) > 0:
            if rep is not None:
                rep.download_cached += 1
            with open(sp, "rb") as fh:
                return fh.read()
    data = _resolve_source_bytes(source, cache_dir=cache_dir)
    if store_dir and populate:
        sp = _store_path(store_dir, storage_loc)
        os.makedirs(os.path.dirname(sp), exist_ok=True)
        tmp = sp + ".part"
        with open(tmp, "wb") as fh:
            fh.write(data)
        os.replace(tmp, sp)
        if rep is not None:
            rep.downloaded += 1
    return data
 def download_corpus(m: Dict[str, Any], *, store_dir: str, board_filter: Optional[str],
                    spec_filter: Optional[str], cache_dir: str, rep: LoadReport) -> None:
    """--download-only: populate the persistent local store from the manifest. No DB/bucket writes.
    A later run with the same --store-dir (e.g. mounted into the container) seeds offline from it."""
    for board in m.get("boards", []):
        if board_filter and board.get("exam_board_code") != board_filter:
            continue
        for spec in board.get("specifications", []):
            if spec_filter and spec.get("spec_code") != spec_filter:
                continue
            sf = spec.get("spec_file")
            if sf and sf.get("source"):
                sloc = spec_storage_loc(board["exam_board_code"], spec.get("subject_code", ""),
                                        spec.get("award_code", ""), spec.get("spec_ver", ""))
                try:
                    _item_bytes(sf["source"], sloc, store_dir=store_dir, cache_dir=cache_dir, rep=rep)
                except Exception as exc:
                    rep.errors.append(f"download spec {spec.get('spec_code')}: {exc}")
            for p in spec.get("papers", []):
                ploc = paper_storage_loc(board["exam_board_code"], spec.get("subject_code", ""),
                                         spec.get("award_code", ""), p["paper_code"], p["session"], p["doc_type"])
                try:
                    _item_bytes(p["file"]["source"], ploc, store_dir=store_dir, cache_dir=cache_dir, rep=rep)
                except Exception as exc:
                    rep.errors.append(f"download {p.get('exam_code')}: {exc}")
    logger.info(f"download-only done: downloaded={rep.downloaded} already_in_store={rep.download_cached} "
                f"errors={len(rep.errors)} store={store_dir}")
 # ─────────────────────────────── storage upload (skip-if-exists + sha256) ───────────────────────────────
 def _split_loc(storage_loc: str) -> Tuple[str, str]:
    bucket, _, path = storage_loc.partition("/")
    return bucket, path
 def _object_exists(storage: StorageAdmin, bucket: str, path: str) -> bool:
    """Existence check by listing the object's parent folder (Supabase storage has no stat)."""
    parent, _, name = path.rpartition("/")
    try:
        listing = storage.client.supabase.storage.from_(bucket).list(parent)
    except Exception as exc:
        logger.warning(f"[exists?] list failed for {bucket}/{parent}: {exc}")
        return False
    return any((item.get("name") == name) for item in (listing or []))
 def upload_file(storage: StorageAdmin, storage_loc: str, data: bytes, *, force: bool, rep: LoadReport) -> str:
    """Upload PDF bytes to storage at storage_loc. Returns the sha256 of the bytes.
    Idempotent: if the object already exists and --force was not given, skips the upload
    (the catalogue upsert still runs and records the checksum). With --force, overwrites.
    """
    sha = hashlib.sha256(data).hexdigest()
    bucket, path = _split_loc(storage_loc)
    if not force and _object_exists(storage, bucket, path):
        logger.info(f"[upload] skip-exists {storage_loc} (sha256={sha[:12]})")
        rep.files_skipped += 1
        return sha
    try:
        storage.upload_file(bucket, path, data, "application/pdf", upsert=True)
        logger.info(f"[upload] {storage_loc} ({len(data)} bytes, sha256={sha[:12]}) force={force}")
        rep.files_uploaded += 1
    except StorageError as exc:
        logger.error(f"[upload] FAILED {storage_loc}: {exc}")
        rep.files_failed += 1
        rep.errors.append(f"upload {storage_loc}: {exc}")
    return sha
 # ─────────────────────────────── catalogue upserts ───────────────────────────────
 def upsert_specification(client: SupabaseServiceRoleClient, spec: Dict[str, Any],
                         storage_loc: Optional[str], sha: Optional[str], rep: LoadReport) -> None:
    sf = spec.get("spec_file") or {}
    doc_details = {
        "award_level": spec.get("award_level"),
        "provenance": sf.get("provenance"),
        "original_name": sf.get("original_name"),
        "sha256": sha,
    }
    row = {
        "spec_code": spec["spec_code"],
        "exam_board_code": spec["exam_board_code"],
        "award_code": spec.get("award_code"),
        "subject_code": spec.get("subject_code"),
        "first_teach": spec.get("first_teach"),
        "spec_ver": spec.get("spec_ver"),
        "storage_loc": storage_loc,
        "doc_type": "pdf",  # file format (CHECK-constrained); the role lives on eb_exams.type_code
        "doc_details": {k: v for k, v in doc_details.items() if v is not None},
    }
    try:
        client.supabase.table("eb_specifications").upsert(row, on_conflict="spec_code").execute()
        logger.info(f"[spec] upsert {row['spec_code']}")
        rep.specs_upserted += 1
    except Exception as exc:
        logger.error(f"[spec] FAILED {row['spec_code']}: {exc}")
        rep.errors.append(f"spec {row['spec_code']}: {exc}")
 def upsert_paper(client: SupabaseServiceRoleClient, spec_code: str, p: Dict[str, Any],
                 storage_loc: str, sha: Optional[str], rep: LoadReport) -> None:
    f = p.get("file") or {}
    doc_role = p["doc_type"]  # manifest role: QP|MS|INSERT|ER...
    doc_details = {
        "doc_role": doc_role,                # mirror of type_code for clarity
        "original_name": f.get("original_name"),
        "provenance": f.get("provenance"),
        "sha256": sha,
    }
    row = {
        "exam_code": p["exam_code"],
        "spec_code": spec_code,
        "paper_code": p.get("paper_code"),
        "tier": p.get("tier"),
        "session": p.get("session"),
        "type_code": doc_role,               # ROLE goes here (QP/MS/INSERT/ER)
        "doc_type": "pdf",                   # file format (CHECK-constrained)
        "storage_loc": storage_loc,
        "doc_details": {k: v for k, v in doc_details.items() if v is not None},
    }
    try:
        client.supabase.table("eb_exams").upsert(row, on_conflict="exam_code").execute()
        logger.info(f"[paper] upsert {row['exam_code']} type_code={doc_role}")
        rep.papers_upserted += 1
    except Exception as exc:
        logger.error(f"[paper] FAILED {row['exam_code']}: {exc}")
        rep.errors.append(f"paper {row['exam_code']}: {exc}")
 # ─────────────────────────────── user-side test subset ───────────────────────────────
 def _resolve_test_user(client: SupabaseServiceRoleClient, cfg: Dict[str, Any]) -> Optional[Tuple[str, str]]:
    """Resolve (user_id, institute_id) for the user-side subset from config, with discovery fallback."""
    user_id = cfg.get("user_id")
    if not user_id and cfg.get("user_email"):
        res = client.supabase.table("profiles").select("id").eq("email", cfg["user_email"]).limit(1).execute()
        rows = getattr(res, "data", None) or []
        user_id = rows[0]["id"] if rows else None
    if not user_id:
        logger.warning("[user-subset] no test user resolvable (set test_subset.user_id or user_email); skipping")
        return None
    institute_id = cfg.get("institute_id")
    if not institute_id:
        res = client.supabase.table("institute_memberships").select("institute_id").eq("profile_id", user_id).limit(1).execute()
        rows = getattr(res, "data", None) or []
        institute_id = rows[0]["institute_id"] if rows else None
    if not institute_id:
        logger.warning(f"[user-subset] no institute for user {user_id}; skipping")
        return None
    return user_id, institute_id
 def copy_user_test_subset(client: SupabaseServiceRoleClient, storage: StorageAdmin,
                          m: Dict[str, Any], rep: LoadReport) -> None:
    """Copy a small subset of admin papers into a test user's exam space so user-side flows
    (upload-as-exam / promote-from-cabinet / mark) are testable.
    Driven by an optional manifest `test_subset:` block:
        test_subset:
          user_id: <uuid>            # or user_email: <email>
          institute_id: <uuid>       # optional; discovered from membership if omitted
          papers: 2                  # how many QP papers to copy (default 2)
    Degrades gracefully (logs + skips) if no test user is resolvable on this env.
    """
    cfg = m.get("test_subset") or {}
    resolved = _resolve_test_user(client, cfg)
    if not resolved:
        return
    user_id, institute_id = resolved
    limit = int(cfg.get("papers", 2))
    # Gather candidate QP papers (admin corpus already uploaded to cc.examboards).
    candidates: List[Tuple[str, Dict[str, Any]]] = []
    for board in m.get("boards", []):
        for spec in board.get("specifications", []):
            for p in spec.get("papers", []):
                if p.get("doc_type") == "QP":
                    candidates.append((board["exam_board_code"], spec, p))
    candidates = candidates[:limit]
    if not candidates:
        logger.info("[user-subset] no QP papers to copy")
        return
    # Ensure a cabinet for the user.
    cab_name = "Exam Marker Template Sources"
    res = client.supabase.table("file_cabinets").select("id").eq("user_id", user_id).eq("name", cab_name).limit(1).execute()
    rows = getattr(res, "data", None) or []
    if rows:
        cabinet_id = rows[0]["id"]
    else:
        ins = client.supabase.table("file_cabinets").insert({"user_id": user_id, "name": cab_name}).execute()
        cabinet_id = (getattr(ins, "data", None) or [{}])[0].get("id")
    if not cabinet_id:
        logger.warning("[user-subset] could not ensure cabinet; skipping")
        return
    import uuid as _uuid
    for board_code, spec, p in candidates:
        src_loc = paper_storage_loc(board_code, spec.get("subject_code", ""), spec.get("award_code", ""),
                                    p["paper_code"], p["session"], p["doc_type"])
        sbucket, spath = _split_loc(src_loc)
        try:
            data = storage.download_file(sbucket, spath)
        except Exception as exc:
            logger.warning(f"[user-subset] source missing {src_loc}: {exc}; skipping {p['exam_code']}")
            continue
        file_id = str(_uuid.uuid4())
        safe_name = f"{p['exam_code']}.pdf"
        dst_bucket = "cc.users"
        dst_path = f"exam-marker/{institute_id}/{cabinet_id}/{file_id}/{safe_name}"
        try:
            storage.upload_file(dst_bucket, dst_path, data, "application/pdf", upsert=True)
        except Exception as exc:
            logger.warning(f"[user-subset] copy upload failed {dst_path}: {exc}")
            continue
        client.supabase.table("files").upsert({
            "id": file_id, "cabinet_id": cabinet_id, "name": safe_name, "path": dst_path,
            "bucket": dst_bucket, "mime_type": "application/pdf", "uploaded_by": user_id,
            "size_bytes": len(data), "source": "exam-corpus-seed", "is_directory": False,
            "relative_path": safe_name, "processing_status": "uploaded",
        }).execute()
        logger.info(f"[user-subset] copied {p['exam_code']} -> {dst_bucket}/{dst_path}")
        rep.user_copies += 1
 # ─────────────────────────────── first sweep (docling auto-map) ───────────────────────────────
 def _resolve_system_identity(client: SupabaseServiceRoleClient, m: Dict[str, Any]) -> Optional[Tuple[str, str]]:
    cfg = m.get("system_identity") or m.get("test_subset") or {}
    user_id = cfg.get("teacher_id") or cfg.get("user_id")
    if not user_id and cfg.get("user_email"):
        res = client.supabase.table("profiles").select("id").eq("email", cfg["user_email"]).limit(1).execute()
        rows = getattr(res, "data", None) or []
        user_id = rows[0]["id"] if rows else None
    institute_id = cfg.get("institute_id")
    if user_id and not institute_id:
        res = client.supabase.table("institute_memberships").select("institute_id").eq("profile_id", user_id).limit(1).execute()
        rows = getattr(res, "data", None) or []
        institute_id = rows[0]["institute_id"] if rows else None
    if not user_id or not institute_id:
        logger.warning("[first-sweep] no system identity (set system_identity.teacher_id+institute_id); skipping sweep")
        return None
    return user_id, institute_id
 def first_sweep(client: SupabaseServiceRoleClient, storage: StorageAdmin,
                m: Dict[str, Any], board_filter: Optional[str], spec_filter: Optional[str],
                cache_dir: str, rep: LoadReport) -> None:
    """Run the docling/auto_map first pass over seeded QP papers and persist the resulting
    template structure (questions/response areas/boundaries/layout) via the same mapping the
    /auto-map endpoint uses. System-owned exam_templates are created per QP paper.
    Requires a resolvable `system_identity` (teacher_id/user_email + institute_id) on this env.
    """
    identity = _resolve_system_identity(client, m)
    if not identity:
        return
    teacher_id, institute_id = identity
    # Import the auto-map mapping helpers lazily (pulls fastapi/router only when sweeping).
    try:
        from api.services.docling import auto_map, AutoMapError
        from routers.exam.templates import _map_first_pass_to_rows
    except Exception as exc:
        logger.error(f"[first-sweep] could not import auto-map pipeline: {exc}")
        rep.errors.append(f"first-sweep import: {exc}")
        return
    sb = client.supabase
    for board in m.get("boards", []):
        if board_filter and board.get("exam_board_code") != board_filter:
            continue
        for spec in board.get("specifications", []):
            if spec_filter and spec.get("spec_code") != spec_filter:
                continue
            for p in spec.get("papers", []):
                if p.get("doc_type") != "QP":
                    continue
                # Resolve the seeded eb_exams row (id) for the template join.
                ex = sb.table("eb_exams").select("id, exam_code").eq("exam_code", p["exam_code"]).limit(1).execute()
                ex_rows = getattr(ex, "data", None) or []
                exam_id = ex_rows[0]["id"] if ex_rows else None
                loc = paper_storage_loc(board["exam_board_code"], spec.get("subject_code", ""),
                                        spec.get("award_code", ""), p["paper_code"], p["session"], p["doc_type"])
                bkt, path = _split_loc(loc)
                try:
                    pdf_bytes = storage.download_file(bkt, path)
                except Exception as exc:
                    logger.warning(f"[first-sweep] source missing {loc}: {exc}; skipping {p['exam_code']}")
                    continue
                # Ensure a system-owned template for this paper (idempotent on exam_code+teacher).
                tpl = sb.table("exam_templates").select("id").eq("exam_code", p["exam_code"]).eq("teacher_id", teacher_id).limit(1).execute()
                tpl_rows = getattr(tpl, "data", None) or []
                if tpl_rows:
                    template_id = tpl_rows[0]["id"]
                else:
                    new_tpl = sb.table("exam_templates").insert({
                        "exam_id": exam_id, "exam_code": p["exam_code"], "institute_id": institute_id,
                        "teacher_id": teacher_id, "title": f"{p['exam_code']} (auto-map seed)",
                        "subject": spec.get("subject_code"), "status": "draft",
                    }).execute()
                    template_id = (getattr(new_tpl, "data", None) or [{}])[0].get("id")
                if not template_id:
                    logger.warning(f"[first-sweep] could not ensure template for {p['exam_code']}; skipping")
                    continue
                try:
                    first_pass = auto_map(pdf_bytes, source_pdf=loc)
                    rows = _map_first_pass_to_rows(template_id, first_pass, pdf_bytes)
                except (AutoMapError, ValueError) as exc:
                    logger.warning(f"[first-sweep] auto-map failed for {p['exam_code']}: {exc}")
                    rep.sweep_failed += 1
                    continue
                except Exception as exc:
                    logger.exception(f"[first-sweep] unexpected error for {p['exam_code']}: {exc}")
                    rep.sweep_failed += 1
                    continue
                # Refresh derived rows. Seed templates are system-owned with no human edits to
                # preserve, so we clear ALL child rows for the template (not just ai/unconfirmed)
                # and re-insert id-deduped payloads — idempotent across re-runs and robust to the
                # deterministic uuid5 ids the mapper can repeat within a batch.
                for table in ("exam_response_areas", "exam_boundaries", "exam_template_layout", "exam_questions"):
                    sb.table(table).delete().eq("template_id", template_id).execute()
                for table, key in (("exam_questions", "questions"), ("exam_response_areas", "response_areas"),
                                   ("exam_boundaries", "boundaries"), ("exam_template_layout", "layout")):
                    seen_ids: set = set()
                    payload = []
                    for r in (rows.get(key) or []):
                        rid = r.get("id")
                        if rid is not None and rid in seen_ids:
                            continue
                        if rid is not None:
                            seen_ids.add(rid)
                        payload.append(r)
                    if payload:
                        sb.table(table).insert(payload).execute()
                updates = {"page_count": first_pass.get("meta", {}).get("n_pages")}
                sb.table("exam_templates").update({k: v for k, v in updates.items() if v is not None}).eq("id", template_id).execute()
                logger.info(f"[first-sweep] swept {p['exam_code']} -> template {template_id} "
                            f"(q={len(rows.get('questions', []))} ra={len(rows.get('response_areas', []))})")
                rep.swept += 1
 # ─────────────────────────────── unseed (inverse of the loader) ───────────────────────────────
 def _chunks(seq: List[Any], n: int = 100):
    for i in range(0, len(seq), n):
        yield seq[i:i + n]
 def _storage_remove(storage: StorageAdmin, bucket: str, paths: List[str]) -> None:
    """Remove object paths from a bucket through the Supabase Storage API.
    The python client treats missing objects as a successful no-op, which is useful for
    unseed idempotency. Any API/permission failure is raised so callers can avoid
    deleting the matching DB rows while storage may still exist.
    """
    result = storage.client.supabase.storage.from_(bucket).remove(paths)
    error = getattr(result, "error", None)
    if error:
        raise StorageError(str(error))
    if isinstance(result, dict) and result.get("error"):
        raise StorageError(str(result["error"]))
 def _delete_user_subset_files(client: SupabaseServiceRoleClient, storage: StorageAdmin, *,
                              exam_codes: Optional[List[str]], rep: LoadReport) -> None:
    """Delete --user-subset files from cc.users storage, then their files rows.
    User-subset seeding writes rows with source='exam-corpus-seed', bucket='cc.users',
    and paths under exam-marker/. Storage must be removed before the files rows: the
    files GC trigger also tries to delete storage when rows are deleted, so removing
    objects first avoids trigger failures and keeps this operation idempotent.
    exam_codes=None means remove all user-subset seed rows (used by unscoped unseed
    even if the eb_* rows were already removed by a prior partial run).
    """
    sb = client.supabase
    seeded_files: List[Dict[str, Any]] = []
    def _base_query():
        return sb.table("files").select("id, bucket, path, name, source") \
            .eq("bucket", "cc.users").eq("source", "exam-corpus-seed") \
            .like("path", "exam-marker/%")
    if exam_codes is None:
        seeded_files.extend(getattr(_base_query().execute(), "data", None) or [])
    elif exam_codes:
        for chunk in _chunks([f"{code}.pdf" for code in exam_codes if code], 100):
            seeded_files.extend(getattr(_base_query().in_("name", chunk).execute(), "data", None) or [])
    rows_by_id: Dict[str, Dict[str, Any]] = {}
    paths_by_bucket: Dict[str, List[str]] = {}
    seen_paths: set = set()
    for row in seeded_files:
        row_id = row.get("id")
        bucket = row.get("bucket")
        path = row.get("path")
        if row_id:
            rows_by_id[str(row_id)] = row
        if bucket == "cc.users" and isinstance(path, str) and path.startswith("exam-marker/"):
            key = (bucket, path)
            if key not in seen_paths:
                seen_paths.add(key)
                paths_by_bucket.setdefault(bucket, []).append(path)
    removable_ids = list(rows_by_id)
    if not removable_ids and not paths_by_bucket:
        logger.info("[unseed] no user-subset cc.users files to remove")
        return
    for bkt, paths in paths_by_bucket.items():
        for chunk in _chunks(paths, 100):
            try:
                _storage_remove(storage, bkt, chunk)
                rep.unseed_objects += len(chunk)
            except Exception as exc:
                logger.warning(f"[unseed] user-subset storage remove failed ({bkt}, {len(chunk)} objs): {exc}")
                rep.errors.append(f"user-subset storage remove {bkt}: {exc}")
                return
    for chunk in _chunks(removable_ids, 100):
        try:
            sb.table("files").delete().in_("id", chunk).execute()
            rep.unseed_user_files += len(chunk)
        except Exception as exc:
            logger.warning(f"[unseed] user-subset files delete failed: {exc}")
            rep.errors.append(f"user-subset files delete: {exc}")
 def unseed(client: SupabaseServiceRoleClient, storage: StorageAdmin, *,
           board_filter: Optional[str], spec_filter: Optional[str],
           drop_specs: bool = True, drop_seed_templates: bool = True, rep: LoadReport) -> None:
    """Inverse of the loader: remove the seeded public corpus, scoped by --board/--spec (or all).
    Deletes (in FK-safe order): cc.examboards storage objects (via the Storage API, since the
    protect_delete trigger blocks direct SQL deletes), first-sweep exam_templates created by the
    seed (title '... (auto-map seed)', cascades children), eb_exams rows, then eb_specifications.
    """
    sb = client.supabase
    q = sb.table("eb_specifications").select("spec_code, storage_loc, exam_board_code")
    if board_filter:
        q = q.eq("exam_board_code", board_filter)
    if spec_filter:
        q = q.eq("spec_code", spec_filter)
    specs = getattr(q.execute(), "data", None) or []
    spec_codes = [s["spec_code"] for s in specs]
    if not spec_codes:
        if not board_filter and not spec_filter:
            _delete_user_subset_files(client, storage, exam_codes=None, rep=rep)
        logger.info("[unseed] no matching specifications; nothing to do")
        return
    exams: List[Dict[str, Any]] = []
    for chunk in _chunks(spec_codes):
        res = sb.table("eb_exams").select("id, exam_code, storage_loc").in_("spec_code", chunk).execute()
        exams.extend(getattr(res, "data", None) or [])
    # 1) User-subset storage/rows. Storage is removed before files rows so trg_files_gc has
    # nothing left to collect when rows are deleted.
    user_subset_exam_codes = None if not board_filter and not spec_filter else [
        e.get("exam_code") for e in exams if e.get("exam_code")
    ]
    _delete_user_subset_files(client, storage, exam_codes=user_subset_exam_codes, rep=rep)
    # 2) Storage objects (Storage API; batch-remove per bucket). Specs may carry a spec PDF too.
    by_bucket: Dict[str, List[str]] = {}
    for row in exams + specs:
        loc = row.get("storage_loc")
        if not loc or "/" not in loc:
            continue
        bkt, _, path = loc.partition("/")
        by_bucket.setdefault(bkt, []).append(path)
    for bkt, paths in by_bucket.items():
        for chunk in _chunks(paths, 100):
            try:
                storage.client.supabase.storage.from_(bkt).remove(chunk)
                rep.unseed_objects += len(chunk)
            except Exception as exc:
                logger.warning(f"[unseed] storage remove failed ({bkt}, {len(chunk)} objs): {exc}")
    # 3) First-sweep templates created by the seed (cascades questions/regions/boundaries/layout).
    if drop_seed_templates and exams:
        exam_codes = [e["exam_code"] for e in exams if e.get("exam_code")]
        for chunk in _chunks(exam_codes, 100):
            try:
                res = sb.table("exam_templates").delete(count="exact") \
                    .in_("exam_code", chunk).like("title", "%(auto-map seed)%").execute()
                rep.unseed_templates += getattr(res, "count", None) or len(getattr(res, "data", []) or [])
            except Exception as exc:
                logger.warning(f"[unseed] template delete failed: {exc}")
    # 4) Catalogue rows: eb_exams (by id), then eb_specifications (by spec_code).
    exam_ids = [e["id"] for e in exams]
    for chunk in _chunks(exam_ids, 100):
        try:
            sb.table("eb_exams").delete().in_("id", chunk).execute()
            rep.unseed_exams += len(chunk)
        except Exception as exc:
            logger.warning(f"[unseed] eb_exams delete failed: {exc}")
    if drop_specs:
        for chunk in _chunks(spec_codes, 100):
            try:
                sb.table("eb_specifications").delete().in_("spec_code", chunk).execute()
                rep.unseed_specs += len(chunk)
            except Exception as exc:
                logger.warning(f"[unseed] eb_specifications delete failed: {exc}")
    logger.info(f"unseed done: storage_objects={rep.unseed_objects} user_files={rep.unseed_user_files} "
                f"templates={rep.unseed_templates} exams={rep.unseed_exams} specs={rep.unseed_specs}")
 # ─────────────────────────────── orchestration ───────────────────────────────
 def load(manifest_path: str, *, dry_run: bool, force: bool, board_filter: Optional[str],
         spec_filter: Optional[str], user_subset: bool, do_first_sweep: bool,
         cache_dir: str = DEFAULT_CACHE_DIR, store_dir: Optional[str] = None) -> LoadReport:
    with open(manifest_path) as f:
        m = yaml.safe_load(f)
    rep = LoadReport()
    errs = validate_manifest(m)
    if errs:
        rep.errors = list(errs)
        logger.error(f"manifest validation failed: {len(errs)} error(s)")
        for e in errs[:40]:
            logger.error(f"  - {e}")
        if not dry_run:
            return rep
    client = None if dry_run else SupabaseServiceRoleClient()
    storage = None if dry_run else StorageAdmin()
    for board in m.get("boards", []):
        if board_filter and board.get("exam_board_code") != board_filter:
            continue
        for spec in board.get("specifications", []):
            if spec_filter and spec.get("spec_code") != spec_filter:
                continue
            # Specification document (optional).
            sloc = None
            spec_sha = None
            sf = spec.get("spec_file")
            if sf and sf.get("source"):
                sloc = spec_storage_loc(board["exam_board_code"], spec.get("subject_code", ""),
                                        spec.get("award_code", ""), spec.get("spec_ver", ""))
                if not dry_run:
                    try:
                        spec_sha = upload_file(storage, sloc,
                                               _item_bytes(sf["source"], sloc, store_dir=store_dir,
                                                           cache_dir=cache_dir, rep=rep),
                                               force=force, rep=rep)
                    except Exception as exc:
                        logger.error(f"[spec-file] {spec.get('spec_code')}: {exc}")
                        rep.files_failed += 1
                        rep.errors.append(f"spec-file {spec.get('spec_code')}: {exc}")
            if not dry_run:
                upsert_specification(client, spec, sloc, spec_sha, rep)
            # Papers.
            for p in spec.get("papers", []):
                ploc = paper_storage_loc(board["exam_board_code"], spec.get("subject_code", ""),
                                         spec.get("award_code", ""), p["paper_code"], p["session"], p["doc_type"])
                if dry_run:
                    continue
                psha = None
                try:
                    psha = upload_file(storage, ploc,
                                       _item_bytes(p["file"]["source"], ploc, store_dir=store_dir,
                                                   cache_dir=cache_dir, rep=rep),
                                       force=force, rep=rep)
                except Exception as exc:
                    logger.error(f"[paper-file] {p.get('exam_code')}: {exc}")
                    rep.files_failed += 1
                    rep.errors.append(f"paper-file {p.get('exam_code')}: {exc}")
                upsert_paper(client, spec["spec_code"], p, ploc, psha, rep)
    if user_subset and not dry_run:
        copy_user_test_subset(client, storage, m, rep)
    if do_first_sweep and not dry_run:
        first_sweep(client, storage, m, board_filter, spec_filter, cache_dir, rep)
    logger.info(f"corpus load done: specs={rep.specs_upserted} papers={rep.papers_upserted} "
                f"uploaded={rep.files_uploaded} skipped={rep.files_skipped} failed={rep.files_failed} "
                f"user_copies={rep.user_copies} swept={rep.swept} errors={len(rep.errors)}")
    return rep
 def main() -> None:
    ap = argparse.ArgumentParser(description="Seed (or unseed) the public exam-paper corpus from a manifest.")
    ap.add_argument("--manifest", help="corpus manifest (required except for --unseed)")
    ap.add_argument("--dry-run", action="store_true", help="validate + report, no writes")
    ap.add_argument("--force", action="store_true", help="re-upload/overwrite existing storage objects")
    ap.add_argument("--board", default=None, help="only this exam_board_code")
    ap.add_argument("--spec", default=None, help="only this spec_code")
    ap.add_argument("--user-subset", action="store_true", help="also seed a user-side test subset")
    ap.add_argument("--first-sweep", action="store_true", help="run docling/auto-map first pass on seeded papers")
    ap.add_argument("--cache-dir", default=DEFAULT_CACHE_DIR, help="throwaway url-hash cache dir")
    ap.add_argument("--store-dir", default=DEFAULT_STORE_DIR,
                    help="persistent, bucket-shaped local store (download-once, seed-many)")
    ap.add_argument("--no-store", action="store_true",
                    help="ignore the local store; always fetch from source (don't read/populate the store)")
    ap.add_argument("--download-only", action="store_true",
                    help="populate the local store from the manifest; no DB/bucket writes")
    ap.add_argument("--unseed", action="store_true",
                    help="INVERSE: remove seeded eb_*/storage/first-sweep templates (scoped by --board/--spec)")
    a = ap.parse_args()
    store_dir = None if a.no_store else a.store_dir
    import json
    if a.unseed:
        rep = LoadReport()
        unseed(SupabaseServiceRoleClient(), StorageAdmin(),
               board_filter=a.board, spec_filter=a.spec, rep=rep)
        print(json.dumps(rep.as_dict(), indent=2))
        return
    if not a.manifest:
        ap.error("--manifest is required unless --unseed is given")
    if a.download_only:
        with open(a.manifest) as f:
            m = yaml.safe_load(f)
        rep = LoadReport()
        download_corpus(m, store_dir=(a.store_dir), board_filter=a.board, spec_filter=a.spec,
                        cache_dir=a.cache_dir, rep=rep)
        print(json.dumps(rep.as_dict(), indent=2))
        return
    rep = load(a.manifest, dry_run=a.dry_run, force=a.force, board_filter=a.board, spec_filter=a.spec,
               user_subset=a.user_subset, do_first_sweep=a.first_sweep, cache_dir=a.cache_dir,
               store_dir=store_dir)
    print(json.dumps(rep.as_dict(), indent=2))
 if __name__ == "__main__":
    main()
@@ -41,7 +41,6 @@ from routers.transcribe.keywords import router as keywords_router
 from routers.me.bootstrap_router import router as me_bootstrap_router
 from routers import tlsync_token as tlsync_token_router
 from routers.exam import router as exam_router
 from routers.markbook import router as markbook_router
 def register_routes(app: FastAPI):
    logger.info("Starting to register routes...")
@@ -139,9 +138,6 @@ def register_routes(app: FastAPI):
    # Exam-marker Routes (as-user Supabase, RLS-enforced; spec §4)
    app.include_router(exam_router, prefix="/api/exam", tags=["Exam"])
    # Running markbook / gradebook Routes (as-user Supabase, RLS-enforced)
    app.include_router(markbook_router, prefix="/api/markbook", tags=["Markbook"])
    # Transcription Routes (CIS Phase 1)
    app.include_router(sessions_router, prefix="/transcribe", tags=["Transcription Sessions"])
    app.include_router(canvas_events_router, prefix="/transcribe", tags=["Transcription Canvas Events"])
@@ -1,81 +0,0 @@
 from api.services.docling.extract import aqa_questions_rapid
 def _text(raw, page, l, t, r=120, b=None):
    return {
        "text": raw,
        "prov": [{"page_no": page, "bbox": {"l": l, "t": t, "r": r, "b": b if b is not None else t - 14, "coord_origin": "BOTTOMLEFT"}}],
    }
 def test_aqa_rapid_cleans_noisy_margin_label(tmp_path):
    (tmp_path / "p1.json").write_text(
        '{"texts":[{"text":"02].3","prov":[{"page_no":1,"bbox":{"l":49,"t":515,"r":102,"b":500,"coord_origin":"BOTTOMLEFT"}}]}]}'
    )
    parts = aqa_questions_rapid(str(tmp_path / "p*.json"))
    assert "02.3" in parts
 def test_aqa_rapid_infers_missing_leading_part_from_next_part(tmp_path):
    (tmp_path / "p1.json").write_text(
        '{"texts":[{"text":"Question prose starts here","prov":[{"page_no":1,"bbox":{"l":114,"t":774,"r":500,"b":760,"coord_origin":"BOTTOMLEFT"}}]}]}'
    )
    (tmp_path / "p2.json").write_text(
        '{"texts":[{"text":"07.2","prov":[{"page_no":2,"bbox":{"l":49,"t":776,"r":104,"b":761,"coord_origin":"BOTTOMLEFT"}}]}]}'
    )
    parts = aqa_questions_rapid(str(tmp_path / "p*.json"))
    assert parts["07.1"]["page"] == 2
    assert parts["07.1"]["bbox"]["l"] == 49
    assert "07.2" in parts
 def test_aqa_rapid_fills_small_mcq_gaps(tmp_path):
    texts = [_text("06.1 Structured question before Section B", 1, 49, 820)]
    for idx, n in enumerate(["07", "08", "11", "12", "13"]):
        texts.append(_text(n + " Which statement is correct?", 1, 49, 780 - idx * 40))
    import json
    (tmp_path / "p1.json").write_text(json.dumps({"texts": texts}))
    parts = aqa_questions_rapid(str(tmp_path / "p*.json"))
    for label in ["07.0", "08.0", "09.0", "10.0", "11.0", "12.0", "13.0"]:
        assert label in parts
 def test_aqa_rapid_maps_circled_digit_labels(tmp_path):
    import json
    (tmp_path / "p1.json").write_text(json.dumps({"texts": [_text("01.③", 1, 49, 515, r=102, b=500)]}))
    parts = aqa_questions_rapid(str(tmp_path / "p*.json"))
    assert "01.3" in parts
 def test_aqa_rapid_infers_internal_structured_gap(tmp_path):
    import json
    texts = [
        _text("05.2 Some question text", 1, 49, 700),
        _text("05.3 Middle question text", 1, 49, 620),
        _text("05.5 Later question text", 2, 49, 740),
    ]
    (tmp_path / "p1.json").write_text(json.dumps({"texts": texts[:2]}))
    (tmp_path / "p2.json").write_text(json.dumps({"texts": texts[2:]}))
    parts = aqa_questions_rapid(str(tmp_path / "p*.json"))
    assert "05.1" in parts
    assert "05.4" in parts
    assert "05.5" in parts
 def test_aqa_rapid_keeps_bare_single_part_question(tmp_path):
    import json
    (tmp_path / "p1.json").write_text(json.dumps({"texts": [_text("03", 1, 49, 775)]}))
    parts = aqa_questions_rapid(str(tmp_path / "p*.json"))
    assert "03.0" in parts
@@ -2,7 +2,6 @@ from __future__ import annotations
 from PIL import Image, ImageDraw
 from api.services.docling import extract
 from api.services.docling.regions import detect_response_regions_from_image
@@ -38,46 +37,3 @@ def test_detects_answer_box() -> None:
    assert boxes
    assert boxes[0]["bbox"]["w"] > 600
    assert boxes[0]["bbox"]["h"] > 200
 def test_detect_response_region_taxonomy_for_lines_and_boxes():
    img = Image.new("RGB", (800, 1000), "white")
    draw = ImageDraw.Draw(img)
    for y in (220, 260, 300):
        draw.line((120, y, 680, y), fill="black", width=2)
    draw.rectangle((140, 520, 660, 640), outline="black", width=3)
    regions = detect_response_regions_from_image(img, min_confidence=0.1)
    types = {r.region_type for r in regions}
    assert "answer_lines" in types
    assert "answer_box" in types
 def test_attach_detected_response_regions_normalizes_taxonomy_and_uses_pdf_y(monkeypatch, tmp_path):
    pdf = tmp_path / "paper.pdf"
    pdf.write_bytes(b"%PDF test placeholder")
    parts = {
        "01.1": {"q": "01", "page": 1, "bbox": {"l": 50, "r": 80, "t": 700, "b": 680}, "regions": []},
        "01.2": {"q": "01", "page": 1, "bbox": {"l": 50, "r": 80, "t": 500, "b": 480}, "regions": []},
    }
    def fake_detect(path, min_confidence=0.32):
        return [{
            "page_index": 0,
            "region_type": "answer-box",
            "confidence": 0.77,
            "bbox": {"x": 100, "y": 335, "w": 500, "h": 40},
            "detection_method": "test",
            "meta": {"page_height_px": 1000, "page_height_pdf": 800},
        }]
    monkeypatch.setattr(extract.region_mod, "detect_response_regions_from_pdf", fake_detect)
    attached, candidates = extract.attach_detected_response_regions(parts, str(pdf))
    assert attached == 1
    assert len(candidates) == 1
    assert parts["01.1"]["regions"] == []
    assert parts["01.2"]["regions"][0]["type"] == "answer_box"
    assert parts["01.2"]["regions"][0]["source"] == "opencv"
@@ -1,134 +0,0 @@
 """Tests for the question-bank + custom-paper assembly router (/api/exam/bank, /custom-papers) — mode 3.
 Same FakeSupabase + dependency-override pattern as test_exam_templates.py; the as-user RLS itself is
 verified against the live dev DB (.94), not here.
 """
 import pytest
 from fastapi import FastAPI
 from fastapi.testclient import TestClient
 import routers.exam.bank as bank_mod
 from routers.exam.bank import router
 from routers.exam.dependencies import ExamContext, get_exam_context
 TEACHER = "00000000-0000-0000-0000-000000000001"
 INST_A = "10000000-0000-0000-0000-000000000001"
@pytest.fixture(autouse=True)
 def _stub_projection(monkeypatch):
    monkeypatch.setattr(bank_mod, "project_template_safe", lambda tid: None)
 class FakeResult:
    def __init__(self, data):
        self.data = data
 class FakeQuery:
    def __init__(self, store, table):
        self.store = store
        self.table = table
        self.rows = list(store.get(table, []))
        self._op = None
        self._payload = None
    def select(self, *_a, **_k):
        self._op = "select"
        return self
    def insert(self, payload):
        self._op = "insert"
        self._payload = payload
        return self
    def eq(self, key, value):
        self.rows = [r for r in self.rows if r.get(key) == value]
        return self
    def in_(self, key, values):
        values = set(values)
        self.rows = [r for r in self.rows if r.get(key) in values]
        return self
    def execute(self):
        backing = self.store.setdefault(self.table, [])
        if self._op == "insert":
            payloads = self._payload if isinstance(self._payload, list) else [self._payload]
            inserted = []
            for p in payloads:
                row = dict(p)
                row.setdefault("id", f"gen-{self.table}-{len(backing)}")
                backing.append(row)
                inserted.append(row)
            return FakeResult(inserted)
        return FakeResult(self.rows)
 class FakeSupabase:
    def __init__(self, store):
        self.store = store
    def table(self, name):
        return FakeQuery(self.store, name)
 def make_client(store, user_id=TEACHER, institute_ids=(INST_A,)):
    app = FastAPI()
    app.include_router(router, prefix="/api/exam")
    app.dependency_overrides[get_exam_context] = lambda: ExamContext(user_id, "tok", FakeSupabase(store), list(institute_ids))
    return TestClient(app), store
 def test_bank_lists_leaf_questions_with_facets_excluding_containers():
    store = {"exam_questions": [
        {"id": "c1", "template_id": "t1", "label": "Q1", "is_container": True, "spec_ref": None,
         "exam_templates": {"id": "t1", "title": "Physics 1H", "subject": "physics", "exam_code": "AQA-PHYS-8463"}},
        {"id": "q1", "template_id": "t1", "label": "01.1", "is_container": False, "max_marks": 3, "spec_ref": "4.2",
         "exam_templates": {"id": "t1", "title": "Physics 1H", "subject": "physics", "exam_code": "AQA-PHYS-8463"}},
        {"id": "q2", "template_id": "t1", "label": "01.2", "is_container": False, "max_marks": 2, "spec_ref": "4.2",
         "exam_templates": {"id": "t1", "title": "Physics 1H", "subject": "physics", "exam_code": "AQA-PHYS-8463"}},
    ]}
    c, _ = make_client(store)
    body = c.get("/api/exam/bank").json()
    labels = {q["label"] for q in body["questions"]}
    assert labels == {"01.1", "01.2"}  # container excluded
    assert body["facets"]["spec_ref"] == {"4.2": 2}
    assert body["questions"][0]["paper"]["exam_code"] == "AQA-PHYS-8463"
 def test_bank_filters_by_spec_ref():
    store = {"exam_questions": [
        {"id": "q1", "template_id": "t1", "label": "a", "is_container": False, "spec_ref": "4.2", "exam_templates": {"subject": "physics"}},
        {"id": "q2", "template_id": "t1", "label": "b", "is_container": False, "spec_ref": "4.5", "exam_templates": {"subject": "physics"}},
    ]}
    c, _ = make_client(store)
    body = c.get("/api/exam/bank?spec_ref=4.2").json()
    assert [q["id"] for q in body["questions"]] == ["q1"]
 def test_custom_paper_copies_selected_questions_in_order():
    store = {"exam_questions": [
        {"id": "q1", "template_id": "t1", "label": "01.1", "is_container": False, "max_marks": 3, "spec_ref": "4.2", "answer_type": "short"},
        {"id": "q2", "template_id": "t2", "label": "05.1", "is_container": False, "max_marks": 6, "spec_ref": "4.5", "answer_type": "written"},
    ]}
    c, store = make_client(store)
    r = c.post("/api/exam/custom-papers", json={"title": "Electricity mini-test", "subject": "physics", "question_ids": ["q2", "q1"]})
    assert r.status_code == 200
    body = r.json()
    assert body["n_questions"] == 2
    # a new template row was created, owned + institute-scoped
    new_tpls = [t for t in store["exam_templates"] if t["id"] == body["id"]]
    assert new_tpls and new_tpls[0]["teacher_id"] == TEACHER and new_tpls[0]["institute_id"] == INST_A
    # two COPIES exist under the new template, order preserving the requested [q2, q1], with fresh ids
    copies = sorted([q for q in store["exam_questions"] if q["template_id"] == body["id"]], key=lambda q: q["order"])
    assert [q["label"] for q in copies] == ["05.1", "01.1"]
    assert all(q["id"] not in ("q1", "q2") for q in copies)  # not the originals
    assert copies[0]["max_marks"] == 6 and copies[0]["spec_ref"] == "4.5"
 def test_custom_paper_requires_accessible_questions():
    c, _ = make_client({"exam_questions": []})
    assert c.post("/api/exam/custom-papers", json={"title": "x", "question_ids": ["nope"]}).status_code == 404
    assert c.post("/api/exam/custom-papers", json={"title": "x", "question_ids": []}).status_code == 400
@@ -249,30 +249,6 @@ def test_upsert_mark_submission_404():
    assert c.put("/api/exam/marks/mk-x", json={"submission_id": "nope", "question_id": "q1", "awarded_marks": 1}).status_code == 404
 def test_upsert_mark_rejects_over_max():
    c = make_client(_batch_with_cohort())  # q1 max_marks = 3
    r = c.put("/api/exam/marks/mk-over", json={"submission_id": "sub2", "question_id": "q1", "awarded_marks": 4})
    assert r.status_code == 422
 def test_upsert_mark_completes_submission_and_batch():
    store = base_store(
        marking_batches=[{"id": "b1", "template_id": TPL, "institute_id": INST_A, "teacher_id": TEACHER, "status": "marking"}],
        exam_questions=[
            {"id": "q0", "template_id": TPL, "label": "Q1", "max_marks": 0, "order": 0, "is_container": True},
            {"id": "q1", "template_id": TPL, "label": "01", "max_marks": 3, "order": 1, "is_container": False},
            {"id": "q2", "template_id": TPL, "label": "02", "max_marks": 5, "order": 2, "is_container": False},
        ],
        student_submissions=[{"id": "sub1", "batch_id": "b1", "student_id": "s1", "status": "marking"}],
        mark_entries=[{"id": "m1", "batch_id": "b1", "submission_id": "sub1", "question_id": "q1", "awarded_marks": 2}],
    )
    c = make_client(store)
    # marking the last leaf question completes the submission (container q0 doesn't block) and the batch
    assert c.put("/api/exam/marks/m2", json={"submission_id": "sub1", "question_id": "q2", "awarded_marks": 4}).status_code == 200
    assert next(s for s in store["student_submissions"] if s["id"] == "sub1")["status"] == "complete"
    assert next(b for b in store["marking_batches"] if b["id"] == "b1")["status"] == "complete"
 # ─── scans (E3 guards) ───────────────────────────────────────────────────────
 def _batch_store():
@@ -1,48 +0,0 @@
 """Test the exam-bank corpus coverage grouping (/api/exam/corpus)."""
 from fastapi import FastAPI
 from fastapi.testclient import TestClient
 from routers.exam.corpus import router
 from routers.exam.dependencies import ExamContext, get_exam_context
 TEACHER = "00000000-0000-0000-0000-000000000001"
 INST_A = "10000000-0000-0000-0000-000000000001"
 class FakeResult:
    def __init__(self, data): self.data = data
 class FakeQuery:
    def __init__(self, store, table): self.store, self.table = store, table
    def select(self, *_a, **_k): return self
    def execute(self): return FakeResult(list(self.store.get(self.table, [])))
 class FakeSupabase:
    def __init__(self, store): self.store = store
    def table(self, name): return FakeQuery(self.store, name)
 def _client(store):
    app = FastAPI(); app.include_router(router, prefix="/api/exam")
    app.dependency_overrides[get_exam_context] = lambda: ExamContext(TEACHER, "tok", FakeSupabase(store), [INST_A])
    return TestClient(app)
 def test_corpus_groups_papers_by_session_with_qp_ms_er_coverage():
    store = {
        "eb_specifications": [{"spec_code": "AQA-PHYS-8463", "exam_board_code": "AQA", "subject_code": "PHYSICS", "award_code": "8463", "first_teach": "2016"}],
        "eb_exams": [
            {"exam_code": "a", "spec_code": "AQA-PHYS-8463", "paper_code": "8463/1H", "session": "2022-Jun", "type_code": "QP", "storage_loc": "cc.examboards/a.pdf"},
            {"exam_code": "b", "spec_code": "AQA-PHYS-8463", "paper_code": "8463/1H", "session": "2022-Jun", "type_code": "MS", "storage_loc": "cc.examboards/b.pdf"},
            {"exam_code": "c", "spec_code": "AQA-PHYS-8463", "paper_code": "8463/1H", "session": "2022-Jun", "type_code": "ER", "storage_loc": None},
            {"exam_code": "d", "spec_code": "AQA-PHYS-8463", "paper_code": "8463/2H", "session": "2023-Jun", "type_code": "QP", "storage_loc": "cc.examboards/d.pdf"},
        ],
    }
    body = _client(store).get("/api/exam/corpus").json()
    assert body["totals"]["specs"] == 1 and body["totals"]["papers"] == 2
    spec = body["boards"][0]["specs"][0]
    assert spec["level"] == "GCSE" and spec["subject"] == "Physics"
    assert spec["counts"] == {"QP": 2, "MS": 1, "ER": 0}  # ER present but not stored → not counted
    p = next(p for p in spec["papers"] if p["paper_code"] == "8463/1H")
    assert p["docs"] == {"QP": True, "MS": True, "ER": False}
@@ -1,75 +0,0 @@
 """Test the extraction-service contract → app ghost-row mapping (P2).
 Mocks the PDF geometry so the mapper is tested in isolation: page-fraction → 780-canvas conversion,
 per-template uuid remap, FK-safety (orphan regions / dangling parents), and rich meta passthrough.
 """
 import routers.exam.templates as T
 TID = "11111111-1111-1111-1111-111111111111"
 # two pages, each rendered 780 wide × 1000 tall; page 2 stacked below page 1
 _GEOM = [
    {"rendered_w": 780.0, "rendered_h": 1000.0, "page_top": 0.0, "page_pt_w": 595.0, "page_pt_h": 842.0, "crop_x0": 0.0, "crop_y0": 0.0},
    {"rendered_w": 780.0, "rendered_h": 1000.0, "page_top": 1000.0, "page_pt_w": 595.0, "page_pt_h": 842.0, "crop_x0": 0.0, "crop_y0": 0.0},
 ]
 _CONTRACT = {
    "coordinate_space": "page_fraction",
    "suggestions": {
        "questions": [
            {"uid": "Q1", "label": "1", "order": 0, "max_marks": 5, "is_container": True, "page": 1},
            {"uid": "Q1a", "parent_uid": "Q1", "label": "1(a)", "order": 1, "max_marks": 3,
             "answer_type": "short", "is_container": False, "page": 1},
            {"uid": "Q1b", "parent_uid": "Q1", "label": "1(b)", "order": 2, "max_marks": 2,
             "answer_type": "mcq", "is_container": False, "page": 2},
        ],
        "response_areas": [
            {"uid": "RA1", "question_uid": "Q1a", "page": 1, "kind": "response", "response_form": "answer-box",
             "confidence": 0.9, "bounds": {"x": 0.1, "y": 0.2, "w": 0.5, "h": 0.05},
             "meta": {"unit": "m/s", "quantity": "v", "n_lines": 1}},
            {"uid": "RA2", "question_uid": "Q1b", "page": 2, "kind": "response", "response_form": "tick-boxes",
             "bounds": {"x": 0.2, "y": 0.3, "w": 0.4, "h": 0.2},
             "meta": {"select_n": 2, "n_options": 5, "boxes": [{"x0": 0.2, "y0": 0.3, "fill": 0.8}]}},
            {"uid": "CTX1", "question_uid": "Q1b", "page": 2, "kind": "context", "response_form": None,
             "bounds": {"x": 0.1, "y": 0.1, "w": 0.6, "h": 0.15}},
            {"uid": "ORPH", "question_uid": "GHOST", "page": 1, "kind": "response", "response_form": "lines",
             "bounds": {"x": 0.1, "y": 0.5, "w": 0.5, "h": 0.1}},   # orphan owner → must be dropped
        ],
        "boundaries": [
            {"question_uid": "Q1", "label": "1", "page_index": 0, "y": 0.18, "confidence": 0.85},
        ],
    },
    "meta": {"n_pages": 2},
 }
 def test_contract_maps_to_canvas_rows(monkeypatch):
    monkeypatch.setattr(T, "_pdf_page_geometry", lambda _b: _GEOM)
    rows = T._map_service_contract_to_rows(TID, _CONTRACT, b"%PDF-1.4")
    q = rows["questions"]
    assert len(q) == 3
    container = next(x for x in q if x["label"] == "1")
    a = next(x for x in q if x["label"] == "1(a)")
    assert container["is_container"] is True and container["parent_id"] is None
    assert a["parent_id"] == container["id"]          # parent remapped to the container's per-template id
    assert a["answer_type"] == "short" and a["max_marks"] == 3
    assert all(x["source"] == "ai" and x["confirmed"] is False for x in q)
    ra = rows["response_areas"]
    assert len(ra) == 3                                # orphan (owner GHOST) dropped
    ids = {r["id"] for r in ra}
    assert len(ids) == 3                               # ids unique
    box = next(r for r in ra if r["response_form"] == "answer-box")
    # page-fraction → 780-canvas: x=0.1*780=78, y=0.2*1000=200, w=0.5*780=390, h=0.05*1000=50
    assert box["bounds"] == {"x": 78.0, "y": 200.0, "w": 390.0, "h": 50.0}
    assert box["meta"]["unit"] == "m/s"               # rich meta preserved
    tick = next(r for r in ra if r["response_form"] == "tick-boxes")
    # page 2 y stacked: 0.3*1000 + page_top(1000) = 1300
    assert tick["bounds"]["y"] == 1300.0 and tick["meta"]["select_n"] == 2
    ctx = next(r for r in ra if r["kind"] == "context")
    assert ctx["response_form"] is None               # context carries no answer form
    assert all(r["question_id"] in {x["id"] for x in q} for r in ra)   # FK-safe
    b = rows["boundaries"]
    assert len(b) == 1 and b[0]["y"] == 180.0 and b[0]["question_id"] == container["id"]
@@ -143,9 +143,6 @@ class _FakeStorageAdmin:
    def download_file(self, bucket_id, file_path):
        return b"%PDF-1.7 fake"
    def create_signed_url(self, bucket_id, file_path, expires_in=3600):
        return {"signedURL": f"https://storage.test/{bucket_id}/{file_path}?token=fake&expires_in={expires_in}"}
 class _FakeServiceRoleClient:
    def __init__(self, store):
@@ -174,65 +171,6 @@ def test_requires_auth_when_not_overridden():
    assert resp.status_code in (401, 403)  # unauthenticated, not processed
 def test_catalogue_requires_auth_when_not_overridden():
    app = FastAPI()
    app.include_router(router, prefix="/api/exam")
    resp = TestClient(app).get("/api/exam/catalogue")
    assert resp.status_code in (401, 403)
 def test_list_catalogue_papers_uses_as_user_metadata():
    store = {
        "eb_exams": [
            {"id": "e1", "exam_code": "AQA-1", "type_code": "QP", "storage_loc": "cc.examboards/aqa/p.pdf"},
            {"id": "e2", "exam_code": "AQA-MS", "type_code": "MS", "storage_loc": "cc.examboards/aqa/ms.pdf"},
        ]
    }
    client, _ = make_client(store=store)
    resp = client.get("/api/exam/catalogue")
    assert resp.status_code == 200
    assert [p["id"] for p in resp.json()["papers"]] == ["e1"]
 def test_catalogue_signed_url_requires_auth_and_signs_examboard_pdf(monkeypatch):
    store = {
        "eb_exams": [
            {"id": "e1", "exam_code": "AQA-1", "type_code": "QP", "storage_loc": "cc.examboards/aqa/physics/qp.pdf"},
        ]
    }
    client, _ = make_client(store=store)
    monkeypatch.setattr(templates_mod, "StorageAdmin", _FakeStorageAdmin)
    resp = client.get("/api/exam/catalogue/e1/signed-url?expires_in=120")
    assert resp.status_code == 200
    body = resp.json()
    assert body["bucket"] == "cc.examboards"
    assert body["path"] == "aqa/physics/qp.pdf"
    assert body["expires_in"] == 120
    assert "token=fake" in body["signed_url"]
 def test_catalogue_signed_url_rejects_non_examboard_storage(monkeypatch):
    store = {
        "eb_exams": [
            {"id": "e1", "exam_code": "AQA-1", "type_code": "QP", "storage_loc": "cc.public/aqa/physics/qp.pdf"},
        ]
    }
    client, _ = make_client(store=store)
    monkeypatch.setattr(templates_mod, "StorageAdmin", _FakeStorageAdmin)
    assert client.get("/api/exam/catalogue/e1/signed-url").status_code == 404
 def test_catalogue_signed_url_rejects_non_catalogue_doc_type(monkeypatch):
    store = {
        "eb_exams": [
            {"id": "e1", "exam_code": "AQA-MS", "type_code": "MS", "storage_loc": "cc.examboards/aqa/physics/ms.pdf"},
        ]
    }
    client, _ = make_client(store=store)
    monkeypatch.setattr(templates_mod, "StorageAdmin", _FakeStorageAdmin)
    assert client.get("/api/exam/catalogue/e1/signed-url").status_code == 404
 def test_create_template_sets_owner_and_institute():
    client, store = make_client()
    resp = client.post("/api/exam/templates", json={"title": "AQA Physics 1H", "subject": "Physics"})
@@ -642,35 +580,6 @@ def test_auto_map_fast_path_merges_ai_rows_and_returns_detail(monkeypatch):
    assert store["exam_boundaries"] and store["exam_boundaries"][0]["derivation"] == "docling-main-band"
 def test_auto_map_surfaces_born_digital_part_marks(monkeypatch):
    # Regression: the born-digital grammar parses per-part marks, but the row mapper hardcoded
    # max_marks=0. A part band carrying `marks` must flow through to the question row's max_marks.
    store = _template_with_source()
    store.update({"exam_questions": [], "exam_response_areas": [], "exam_boundaries": [], "exam_template_layout": []})
    client, store = make_client(store=store)
    fp = _first_pass_template()
    fp["pages"]["1"]["part_bands"][0]["marks"] = 4
    _patch_auto_map(monkeypatch, store, fast=True)
    monkeypatch.setattr(templates_mod, "auto_map", lambda *_a, **_k: fp)  # override with the marked part band
    assert client.post("/api/exam/templates/t1/auto-map").status_code == 200
    part = next(q for q in store["exam_questions"] if q.get("label") == "01.1")
    assert part["max_marks"] == 4
 def test_auto_map_deduplicates_repeated_response_area_ids(monkeypatch):
    store = _template_with_source()
    client, store = make_client(store=store)
    _patch_auto_map(monkeypatch, store, fast=True)
    dup = {"page_index": 0, "bbox": {"l": 50, "t": 700, "r": 100, "b": 680, "coord_origin": "BOTTOMLEFT"}, "region_type": "answer_lines", "confidence": 0.9}
    monkeypatch.setattr(templates_mod, "detect_response_regions_from_pdf", lambda *_a, **_k: [dup, dict(dup)])
    resp = client.post("/api/exam/templates/t1/auto-map")
    assert resp.status_code == 200
    response_area_ids = [r["id"] for r in store["exam_response_areas"]]
    assert len(response_area_ids) == len(set(response_area_ids))
 def test_auto_map_preserves_manual_and_confirmed_rows_on_rerun(monkeypatch):
    store = _template_with_source()
    store.update({
@@ -689,27 +598,6 @@ def test_auto_map_preserves_manual_and_confirmed_rows_on_rerun(monkeypatch):
    assert "old-ai" not in ids
 def test_auto_map_rerun_after_confirm_does_not_pk_collide(monkeypatch):
    # Regression: a confirmed ghost keeps its DETERMINISTIC uuid5 id, which auto-map re-emits on a
    # re-run — the old code re-inserted that id and PK-collided. Use the real generated id (not an
    # arbitrary one, unlike the test above) so the collision path is actually exercised.
    store = _template_with_source()
    store.update({"exam_questions": [], "exam_response_areas": [], "exam_boundaries": [], "exam_template_layout": []})
    client, store = make_client(store=store)
    _patch_auto_map(monkeypatch, store, fast=True)
    assert client.post("/api/exam/templates/t1/auto-map").status_code == 200
    ghost_id = next(q["id"] for q in store["exam_questions"] if q["source"] == "ai")
    # teacher confirms that ghost (row kept, deterministic id unchanged)
    for q in store["exam_questions"]:
        if q["id"] == ghost_id:
            q["confirmed"] = True
    # re-run auto-map: must not re-insert the same id, and must preserve the teacher's confirmation
    assert client.post("/api/exam/templates/t1/auto-map").status_code == 200
    same = [r for r in store["exam_questions"] if r["id"] == ghost_id]
    assert len(same) == 1, "confirmed ghost must not be duplicated (PK collision) on re-run"
    assert same[0]["confirmed"] is True, "teacher's confirmation must survive a re-run"
 def test_auto_map_non_owner_is_403_before_download(monkeypatch):
    store = _template_with_source(owner=OTHER_TEACHER)
    client, store = make_client(user_id=TEACHER, institute_ids=(INST_A,), store=store)
@@ -754,16 +642,4 @@ def test_auto_map_ocr_returns_job_id_and_status_completes(monkeypatch):
    body = status.json()
    assert body["status"] == "completed"
    assert body["counts"]["questions"] >= 2
 def test_auto_map_ocr_path_projects_to_neo4j(monkeypatch, _stub_projection):
    # Image-only papers route through the async OCR job; regression: that path must project to Neo4j
    # like the born-digital fast path, or the graph is never built for the primary target.
    store = _template_with_source()
    client, store = make_client(store=store)
    _patch_auto_map(monkeypatch, store, fast=False)
    resp = client.post("/api/exam/templates/t1/auto-map")
    assert resp.status_code == 202
    # the BackgroundTask runs after the response under TestClient
    assert "t1" in _stub_projection
    assert body["template"]["layout"]
@@ -1,103 +0,0 @@
 from types import SimpleNamespace
 import pytest
 import routers.database.files.files as files_router
 import routers.database.files.files_simplified as files_simplified_router
 ROUTERS = [files_router, files_simplified_router]
 USER_A = "00000000-0000-0000-0000-000000000001"
 USER_B = "00000000-0000-0000-0000-000000000002"
 CAB_A = "10000000-0000-0000-0000-000000000001"
 CAB_B = "10000000-0000-0000-0000-000000000002"
 class FakeQuery:
    def __init__(self, rows):
        self.rows = list(rows)
    def select(self, *_args, **_kwargs):
        return self
    def eq(self, key, value):
        self.rows = [row for row in self.rows if row.get(key) == value]
        return self
    def limit(self, _n):
        return self
    def execute(self):
        return SimpleNamespace(data=self.rows)
 class FakeSupabase:
    def __init__(self, store):
        self.store = store
    def table(self, name):
        return FakeQuery(self.store.get(name, []))
 class FakeServiceRoleClient:
    def __init__(self, store):
        self.supabase = FakeSupabase(store)
@pytest.mark.parametrize("router_module", ROUTERS)
 def test_list_files_hides_unowned_unshared_cabinet(monkeypatch, router_module):
    store = {
        "file_cabinets": [
            {"id": CAB_A, "user_id": USER_A},
            {"id": CAB_B, "user_id": USER_B},
        ],
        "cabinet_memberships": [],
        "files": [
            {"id": "file-a", "cabinet_id": CAB_A, "uploaded_by": USER_A},
            {"id": "file-b", "cabinet_id": CAB_B, "uploaded_by": USER_B},
        ],
    }
    monkeypatch.setattr(
        router_module,
        "SupabaseServiceRoleClient",
        lambda: FakeServiceRoleClient(store),
    )
    assert router_module.list_files(CAB_B, {"sub": USER_A}) == []
@pytest.mark.parametrize("router_module", ROUTERS)
 def test_list_files_allows_own_cabinet(monkeypatch, router_module):
    store = {
        "file_cabinets": [{"id": CAB_A, "user_id": USER_A}],
        "cabinet_memberships": [],
        "files": [{"id": "file-a", "cabinet_id": CAB_A, "uploaded_by": USER_A}],
    }
    monkeypatch.setattr(
        router_module,
        "SupabaseServiceRoleClient",
        lambda: FakeServiceRoleClient(store),
    )
    assert router_module.list_files(CAB_A, {"sub": USER_A}) == [
        {"id": "file-a", "cabinet_id": CAB_A, "uploaded_by": USER_A}
    ]
@pytest.mark.parametrize("router_module", ROUTERS)
 def test_list_files_denies_non_owner_even_with_cabinet_membership(monkeypatch, router_module):
    store = {
        "file_cabinets": [{"id": CAB_B, "user_id": USER_B}],
        "cabinet_memberships": [
            {"cabinet_id": CAB_B, "profile_id": USER_A, "role": "viewer"}
        ],
        "files": [{"id": "file-b", "cabinet_id": CAB_B, "uploaded_by": USER_B}],
    }
    monkeypatch.setattr(
        router_module,
        "SupabaseServiceRoleClient",
        lambda: FakeServiceRoleClient(store),
    )
    assert router_module.list_files(CAB_B, {"sub": USER_A}) == []
@@ -1,167 +0,0 @@
 import pytest
 from fastapi import FastAPI
 from fastapi.testclient import TestClient
 import routers.markbook as markbook_mod
 from routers.exam.dependencies import ExamContext
 from routers.markbook import router
 TEACHER = "00000000-0000-0000-0000-000000000001"
 INST = "10000000-0000-0000-0000-000000000001"
 CLASS = "c-1"
 class FakeResult:
    def __init__(self, data):
        self.data = data
 class FakeQuery:
    def __init__(self, store, table):
        self.store = store
        self.table = table
        self.rows = list(store.get(table, []))
        self._filters = []
        self._op = None
        self._payload = None
        self._limit = None
    def select(self, *_a, **_k):
        self._op = "select"; return self
    def insert(self, payload):
        self._op = "insert"; self._payload = payload; return self
    def upsert(self, payload, **_kwargs):
        self._op = "upsert"; self._payload = payload; return self
    def eq(self, k, v):
        self._filters.append(("eq", k, v)); self.rows = [r for r in self.rows if r.get(k) == v]; return self
    def in_(self, k, vals):
        vals = set(vals); self._filters.append(("in", k, vals)); self.rows = [r for r in self.rows if r.get(k) in vals]; return self
    def order(self, *_a, **_k):
        return self
    def limit(self, n):
        self._limit = n; return self
    def _match(self, row):
        for op, k, v in self._filters:
            if op == "eq" and row.get(k) != v:
                return False
            if op == "in" and row.get(k) not in v:
                return False
        return True
    def execute(self):
        backing = self.store.setdefault(self.table, [])
        if self._op in ("insert", "upsert"):
            payloads = self._payload if isinstance(self._payload, list) else [self._payload]
            out = []
            for p in payloads:
                row = dict(p)
                if self._op == "upsert" and self.table == "assessment_marks":
                    existing = next((r for r in backing if r.get("assessment_id") == row.get("assessment_id") and r.get("student_id") == row.get("student_id")), None)
                    if existing:
                        existing.update(row); out.append(existing); continue
                if self._op == "upsert" and row.get("id") is not None:
                    existing = next((r for r in backing if r.get("id") == row["id"]), None)
                    if existing:
                        existing.update(row); out.append(existing); continue
                row.setdefault("id", f"gen-{self.table}-{len(backing)}")
                backing.append(row); out.append(row)
            return FakeResult(out)
        rows = self.rows[: self._limit] if self._limit is not None else self.rows
        return FakeResult(rows)
 class FakeSupabase:
    def __init__(self, store):
        self.store = store
    def table(self, name):
        return FakeQuery(self.store, name)
 def make_client(store):
    app = FastAPI()
    app.include_router(router, prefix="/api/markbook")
    from routers.exam.dependencies import get_exam_context
    app.dependency_overrides[get_exam_context] = lambda: ExamContext(TEACHER, "tok", FakeSupabase(store), [INST])
    return TestClient(app)
 def base_store(**extra):
    store = {
        "classes": [{"id": CLASS, "name": "10A Maths", "institute_id": INST}],
        "class_students": [
            {"class_id": CLASS, "student_id": "s1", "status": "active"},
            {"class_id": CLASS, "student_id": "s2", "status": "active"},
            {"class_id": CLASS, "student_id": "s3", "status": "inactive"},
        ],
    }
    store.update(extra)
    return store
@pytest.fixture(autouse=True)
 def names(monkeypatch):
    monkeypatch.setattr(markbook_mod, "resolve_student_names", lambda ids: {sid: {"s1": "Alice", "s2": "Bob"}.get(sid, sid) for sid in ids})
 def test_create_assessment_sets_class_and_tenant():
    store = base_store()
    c = make_client(store)
    r = c.post(f"/api/markbook/classes/{CLASS}/assessments", json={"title": "Homework 1", "date": "2026-09-10", "max_marks": 20})
    assert r.status_code == 200
    body = r.json()
    assert body["class_id"] == CLASS
    assert body["tenant_id"] == INST
    assert body["title"] == "Homework 1"
    assert body["max_marks"] == 20
 def test_grid_reuses_active_roster_and_computes_summaries():
    store = base_store(
        class_assessments=[
            {"id": "a1", "class_id": CLASS, "tenant_id": INST, "title": "HW", "date": "2026-09-01", "max_marks": 10},
            {"id": "a2", "class_id": CLASS, "tenant_id": INST, "title": "Quiz", "date": "2026-09-08", "max_marks": 20},
        ],
        assessment_marks=[
            {"assessment_id": "a1", "student_id": "s1", "mark": 8},
            {"assessment_id": "a2", "student_id": "s1", "mark": 18},
            {"assessment_id": "a1", "student_id": "s2", "mark": 6},
        ],
    )
    body = make_client(store).get(f"/api/markbook/classes/{CLASS}/grid").json()
    assert [s["student_name"] for s in body["students"]] == ["Alice", "Bob"]
    alice = body["students"][0]
    assert alice["marks"] == {"a1": 8, "a2": 18}
    assert alice["total"] == 26
    assert alice["percentage"] == 86.7
    assert body["assessment_summaries"][0]["average_mark"] == 7.0
    assert body["summary"]["entered_mark_count"] == 3
 def test_mark_upsert_rejects_over_max_and_inactive_students():
    store = base_store(class_assessments=[{"id": "a1", "class_id": CLASS, "tenant_id": INST, "title": "HW", "max_marks": 10}])
    c = make_client(store)
    assert c.put(f"/api/markbook/classes/{CLASS}/assessments/a1/marks/s1", json={"mark": 11}).status_code == 422
    assert c.put(f"/api/markbook/classes/{CLASS}/assessments/a1/marks/s3", json={"mark": 5}).status_code == 404
    ok = c.put(f"/api/markbook/classes/{CLASS}/assessments/a1/marks/s1", json={"mark": 9})
    assert ok.status_code == 200
    assert ok.json()["mark"]["mark"] == 9
 def test_csv_export_has_roster_rows_and_assessment_columns():
    store = base_store(
        class_assessments=[{"id": "a1", "class_id": CLASS, "tenant_id": INST, "title": "HW", "max_marks": 10}],
        assessment_marks=[{"assessment_id": "a1", "student_id": "s1", "mark": 8}],
    )
    text = make_client(store).get(f"/api/markbook/classes/{CLASS}/csv").text
    lines = text.strip().splitlines()
    assert lines[0] == "row,student_name,student_id,HW,total,percentage"
    assert lines[1].startswith("1,Alice,s1,8,8")
    assert lines[2].startswith("2,Bob,s2,,,")
@@ -1,51 +0,0 @@
 from run.initialization import reset_environment
 def test_reset_user_subset_scope_only_runs_user_subset_cleanup(monkeypatch):
    calls = []
    monkeypatch.setattr(
        reset_environment,
        "_sb_headers",
        lambda: ("http://192.168.0.94:8000", {"Authorization": "Bearer redacted"}),
    )
    monkeypatch.setattr(
        reset_environment,
        "_assert_reset_allowed",
        lambda url, scope: calls.append(("guard", url, scope)),
    )
    monkeypatch.setattr(
        reset_environment,
        "_clear_user_subset_files",
        lambda: {"files_rows_deleted": 2, "storage_objects_removed": 2, "errors": []},
    )
    def fail_if_called(*_args, **_kwargs):
        raise AssertionError("reset(scope='user-subset') must not clear unrelated tables or databases")
    monkeypatch.setattr(reset_environment, "_clear_tables", fail_if_called)
    monkeypatch.setattr(reset_environment, "_neo4j_drop_all_non_system", fail_if_called)
    monkeypatch.setattr(reset_environment, "_clear_exam_storage", fail_if_called)
    result = reset_environment.reset(scope="user-subset")
    assert calls == [("guard", "http://192.168.0.94:8000", "user-subset")]
    assert result == {
        "scope": "user-subset",
        "user_subset": {"files_rows_deleted": 2, "storage_objects_removed": 2, "errors": []},
    }
 def test_reset_accepts_case_insensitive_user_subset_scope(monkeypatch):
    monkeypatch.setattr(reset_environment, "_sb_headers", lambda: ("http://192.168.0.94:8000", {}))
    monkeypatch.setattr(reset_environment, "_assert_reset_allowed", lambda *_args, **_kwargs: None)
    monkeypatch.setattr(
        reset_environment,
        "_clear_user_subset_files",
        lambda: {"files_rows_deleted": 0, "storage_objects_removed": 0, "errors": []},
    )
    assert reset_environment.reset(scope="USER-SUBSET") == {
        "scope": "user-subset",
        "user_subset": {"files_rows_deleted": 0, "storage_objects_removed": 0, "errors": []},
    }
@@ -1,171 +0,0 @@
 from run.initialization.seed_exam_corpus import LoadReport, _delete_user_subset_files
 class _Result:
    def __init__(self, data=None):
        self.data = data or []
 class _FilesQuery:
    def __init__(self, db, op="select"):
        self.db = db
        self.op = op
        self.filters = []
        self.in_filters = []
    def select(self, *_args, **_kwargs):
        return self
    def delete(self, *_args, **_kwargs):
        self.op = "delete"
        return self
    def eq(self, key, value):
        self.filters.append(("eq", key, value))
        return self
    def like(self, key, pattern):
        self.filters.append(("like", key, pattern))
        return self
    def in_(self, key, values):
        self.in_filters.append((key, set(values)))
        return self
    def _matches(self, row):
        for kind, key, value in self.filters:
            actual = row.get(key)
            if kind == "eq" and actual != value:
                return False
            if kind == "like":
                assert value.endswith("%")
                if not isinstance(actual, str) or not actual.startswith(value[:-1]):
                    return False
        for key, values in self.in_filters:
            if row.get(key) not in values:
                return False
        return True
    def execute(self):
        matched = [row for row in self.db.rows if self._matches(row)]
        if self.op == "delete":
            self.db.ops.append(("delete", [row["id"] for row in matched]))
            self.db.rows = [row for row in self.db.rows if not self._matches(row)]
            return _Result(matched)
        return _Result(matched)
 class _FakeDb:
    def __init__(self, rows):
        self.rows = list(rows)
        self.ops = []
    def table(self, name):
        assert name == "files"
        return _FilesQuery(self)
 class _FakeStorageBucket:
    def __init__(self, storage, bucket):
        self.storage = storage
        self.bucket = bucket
    def remove(self, paths):
        self.storage.ops.append(("remove", self.bucket, list(paths)))
        if self.storage.fail:
            raise RuntimeError("storage unavailable")
        if self.storage.result_error:
            return {"error": self.storage.result_error}
        return []
 class _FakeStorageRoot:
    def __init__(self, storage):
        self.storage = storage
    def from_(self, bucket):
        return _FakeStorageBucket(self.storage, bucket)
 class _FakeStorage:
    def __init__(self, fail=False, result_error=None):
        self.fail = fail
        self.result_error = result_error
        self.ops = []
        self.client = type("Client", (), {"supabase": type("SB", (), {"storage": _FakeStorageRoot(self)})()})()
 class _FakeClient:
    def __init__(self, db):
        self.supabase = db
 def test_delete_user_subset_storage_before_files_rows_for_scoped_exams():
    db = _FakeDb([
        {"id": "f1", "bucket": "cc.users", "path": "exam-marker/i/c/f1/A.pdf", "name": "A.pdf", "source": "exam-corpus-seed"},
        {"id": "f2", "bucket": "cc.users", "path": "exam-marker/i/c/f2/B.pdf", "name": "B.pdf", "source": "exam-corpus-seed"},
        {"id": "f3", "bucket": "cc.users", "path": "exam-marker/i/c/f3/A.pdf", "name": "A.pdf", "source": "manual"},
        {"id": "f4", "bucket": "cc.users", "path": "other/f4/A.pdf", "name": "A.pdf", "source": "exam-corpus-seed"},
    ])
    storage = _FakeStorage()
    rep = LoadReport()
    _delete_user_subset_files(_FakeClient(db), storage, exam_codes=["A"], rep=rep)
    assert storage.ops == [("remove", "cc.users", ["exam-marker/i/c/f1/A.pdf"])]
    assert db.ops == [("delete", ["f1"])]
    assert [row["id"] for row in db.rows] == ["f2", "f3", "f4"]
    assert rep.unseed_objects == 1
    assert rep.unseed_user_files == 1
    assert rep.errors == []
 def test_delete_user_subset_keeps_files_rows_when_storage_remove_fails():
    db = _FakeDb([
        {"id": "f1", "bucket": "cc.users", "path": "exam-marker/i/c/f1/A.pdf", "name": "A.pdf", "source": "exam-corpus-seed"},
    ])
    storage = _FakeStorage(fail=True)
    rep = LoadReport()
    _delete_user_subset_files(_FakeClient(db), storage, exam_codes=["A"], rep=rep)
    assert storage.ops == [("remove", "cc.users", ["exam-marker/i/c/f1/A.pdf"])]
    assert db.ops == []
    assert [row["id"] for row in db.rows] == ["f1"]
    assert rep.unseed_objects == 0
    assert rep.unseed_user_files == 0
    assert rep.errors
 def test_delete_user_subset_keeps_files_rows_when_storage_remove_returns_error():
    db = _FakeDb([
        {"id": "f1", "bucket": "cc.users", "path": "exam-marker/i/c/f1/A.pdf", "name": "A.pdf", "source": "exam-corpus-seed"},
    ])
    storage = _FakeStorage(result_error="permission denied")
    rep = LoadReport()
    _delete_user_subset_files(_FakeClient(db), storage, exam_codes=["A"], rep=rep)
    assert storage.ops == [("remove", "cc.users", ["exam-marker/i/c/f1/A.pdf"])]
    assert db.ops == []
    assert [row["id"] for row in db.rows] == ["f1"]
    assert rep.unseed_objects == 0
    assert rep.unseed_user_files == 0
    assert rep.errors
 def test_delete_user_subset_unscoped_cleans_all_seeded_exam_marker_rows():
    db = _FakeDb([
        {"id": "f1", "bucket": "cc.users", "path": "exam-marker/i/c/f1/A.pdf", "name": "A.pdf", "source": "exam-corpus-seed"},
        {"id": "f2", "bucket": "cc.users", "path": "exam-marker/i/c/f2/B.pdf", "name": "B.pdf", "source": "exam-corpus-seed"},
    ])
    storage = _FakeStorage()
    rep = LoadReport()
    _delete_user_subset_files(_FakeClient(db), storage, exam_codes=None, rep=rep)
    assert storage.ops == [("remove", "cc.users", ["exam-marker/i/c/f1/A.pdf", "exam-marker/i/c/f2/B.pdf"])]
    assert db.ops == [("delete", ["f1", "f2"])]
    assert db.rows == []
    assert rep.unseed_objects == 2
    assert rep.unseed_user_files == 2
@@ -1,54 +0,0 @@
 import asyncio
 import pytest
 from fastapi import HTTPException
 from modules.upload_validation import MAX_UPLOAD_BYTES, read_pdf_upload_bytes, read_upload_bytes
 class FakeUpload:
    def __init__(self, data: bytes, content_type: str, filename: str = "file.bin"):
        self._data = data
        self._pos = 0
        self.content_type = content_type
        self.filename = filename
    async def read(self, size: int = -1) -> bytes:
        if self._pos >= len(self._data):
            return b""
        if size is None or size < 0:
            size = len(self._data) - self._pos
        chunk = self._data[self._pos : self._pos + size]
        self._pos += len(chunk)
        return chunk
 def run(coro):
    return asyncio.run(coro)
 def test_valid_pdf_upload_passes_and_returns_mime():
    data, mime = run(read_upload_bytes(FakeUpload(b"%PDF-1.7\n", "application/pdf")))
    assert data.startswith(b"%PDF-")
    assert mime == "application/pdf"
 def test_disallowed_mime_rejected_with_415():
    with pytest.raises(HTTPException) as exc:
        run(read_upload_bytes(FakeUpload(b"print(1)", "application/x-python")))
    assert exc.value.status_code == 415
    assert "Unsupported upload type" in exc.value.detail
 def test_oversize_upload_rejected_with_413():
    with pytest.raises(HTTPException) as exc:
        run(read_upload_bytes(FakeUpload(b"x" * (MAX_UPLOAD_BYTES + 1), "text/plain")))
    assert exc.value.status_code == 413
    assert "exceeds max size" in exc.value.detail
 def test_pdf_helper_rejects_spoofed_pdf_mime():
    with pytest.raises(HTTPException) as exc:
        run(read_pdf_upload_bytes(FakeUpload(b"not a pdf", "application/pdf")))
    assert exc.value.status_code == 415
    assert "not a valid PDF" in exc.value.detail