api/services/docling/.gitignore

@@ -1,5 +0,0 @@
-# B1 image-only eval corpus + pipeline outputs: fetched/generated at runtime, never committed.
-# Exam-board PDFs are third-party copyright (served only via signed URLs); results/ are reproducible.
-/samples/b1/
-/results/b1_rapid/
-/results/final/

api/services/docling/extract.py

@@ -40,10 +40,6 @@ try:
     from . import tables as tbl_mod
 except ImportError:  # pragma: no cover - CLI execution
     import tables as tbl_mod
-try:
-    from . import regions as region_mod
-except ImportError:  # pragma: no cover - CLI execution
-    import regions as region_mod
 
 # ----------------------------------------------------------------- line model
 Line = namedtuple("Line", "text page bbox")   # bbox is None for text-only sources
@@ -249,11 +245,6 @@ def extract_front_matter(lines, board, code):
 # ====================================================================== AQA
 # --- v1 path: Docling JSON + RapidOCR boxed labels (the proven 95% recovery) -----
 PART_RE = re.compile(r"^(\d{2})\.(\d)$")     # 01.2
-# OCR sometimes inserts bracket/noise glyphs inside boxed labels (e.g. "02].3").
-# Normalise only tight margin-column candidates before matching; body decimals
-# remain protected by the label-column gate below.
-AQA_LABEL_NOISE = re.compile(r"[^0-9.]+")
-AQA_CIRCLED_DIGITS = str.maketrans({"①": "1", "②": "2", "③": "3", "④": "4", "⑤": "5", "⑥": "6", "⑦": "7", "⑧": "8", "⑨": "9"})
 NUM_RE  = re.compile(r"^(\d{2})$")           # 08
 DIG_RE  = re.compile(r"^(\d)$")              # 4
 # A-level papers (7408) render the boxed label GLUED to the question text in one OCR token
@@ -284,47 +275,21 @@ def _rapid_pages(rapid_glob):
         yield pg, json.load(open(fn))
 
 
-def _clean_aqa_label(raw):
-    compact = (raw or "").strip().translate(AQA_CIRCLED_DIGITS).replace(" ", "")
-    # Do not turn prose/option OCR artifacts like "36.7Q" into labels; PART_PREFIX handles
-    # genuine glued label+prose cases from the raw text under the label-column gate.
-    if re.search(r"[A-Za-z]", compact):
-        return compact
-    return AQA_LABEL_NOISE.sub("", compact)
-
-
-def _synthetic_label_bbox(page_lines, fallback):
-    """Best-effort bbox for an OCR-missed AQA label, preserving the layout contract."""
-    body = [bb for bb, _ in page_lines if 90 <= bb.get("l", 999) <= 170 and bb.get("t", 0) >= FOOTER_T]
-    if body:
-        top = max(body, key=lambda b: b.get("t", 0))
-        return {"l": 48.0, "r": 104.0, "t": round(top["t"], 1), "b": round(top["b"], 1),
-                "coord_origin": top.get("coord_origin", "BOTTOMLEFT")}
-    if fallback:
-        return dict(fallback)
-    return {"l": 48.0, "r": 104.0, "t": 780.0, "b": 760.0, "coord_origin": "BOTTOMLEFT"}
-
-
 def aqa_questions_rapid(rapid_glob):
     """Recover question labels from per-page RapidOCR dumps. Handles three AQA layouts:
       * GCSE standalone label/number boxes (8463 — v1's NN.M + NUM/DIG pairing),
       * A-level structured parts glued as a prefix ("01.1 An atom of..." — label column),
       * A-level Section-B multiple choice: bare sequential top-levels -> NN.0."""
     parts = {}
-    page_lines = defaultdict(list)        # page -> [(bbox, raw)] for deterministic inference
     mcq_cands = []                       # (page, NN, bbox) bare top-level candidates, in order
-    top_cands = {}                        # NN -> (page, bbox) explicit top-level question headers
     for pg, d in _rapid_pages(rapid_glob):
         margin = []
         for t in d.get("texts", []):
             raw = (t.get("text") or "").strip()
-            s = _clean_aqa_label(raw)
+            s = raw.replace(" ", "")
             prov = t.get("prov") or []
             bb = prov[0].get("bbox") if prov else None
-            if bb is None:
+            if bb is None or bb["l"] > 140:
-                continue
-            page_lines[pg].append((bb, raw))
-            if bb["l"] > 140:
                 continue
             margin.append((bb, s))
             m = PART_RE.match(s)
@@ -342,67 +307,21 @@ def aqa_questions_rapid(rapid_glob):
         nums = [(bb, NUM_RE.match(s).group(1)) for bb, s in margin if NUM_RE.match(s)]
         digs = [(bb, DIG_RE.match(s).group(1)) for bb, s in margin if DIG_RE.match(s)]
         for nbb, nn in nums:
-            top_cands.setdefault(nn, (pg, nbb))
             ny = (nbb["t"] + nbb["b"]) / 2
             for dbb, dd in digs:
                 dy = (dbb["t"] + dbb["b"]) / 2
                 if abs(ny - dy) < 12 and dbb["l"] >= nbb["l"]:
                     parts.setdefault(f"{nn}.{dd}", {"page": pg, "bbox": nbb})
-    # Before Section-B handling, trim isolated high structured labels when a real MCQ run starts
+    # Section B: walk MCQ candidates in reading order, accept the next number in sequence only
-    # immediately after the core structured section. This prevents OCR option text such as "36.7Q"
+    structured_q = {int(lab.split(".")[0]) for lab in parts}
-    # from moving the MCQ start from Q07 to Q37.
-    q_nums = sorted({int(lab.split(".")[0]) for lab in parts if not lab.endswith(".0")})
-    core_q = q_nums[:]
-    while len(core_q) >= 2 and core_q[-1] - core_q[-2] > 2:
-        core_q.pop()
-    mcq_nums = {int(nn) for _, nn, _ in mcq_cands}
-    if core_q and any(max(core_q) < n <= max(core_q) + 3 for n in mcq_nums):
-        core_set = set(core_q)
-        for lab in list(parts):
-            if int(lab.split(".")[0]) not in core_set and not lab.endswith(".0"):
-                parts.pop(lab, None)
-
-    # Infer an OCR-dropped leading .1 part when later structured parts for the same question are
-    # present. AQA papers overwhelmingly start structured questions at .1; this fixes pages where
-    # RapidOCR reads the prose but misses the small boxed label, without changing schemas or model paths.
-    by_q = defaultdict(list)
-    for lab, v in parts.items():
-        q, sub = lab.split(".")
-        if sub != "0":
-            by_q[q].append((int(sub), v))
-    for q, vals in list(by_q.items()):
-        if f"{q}.1" not in parts:
-            first_sub, first_v = min(vals, key=lambda x: (x[0], x[1].get("page") or 999))
-            if first_sub > 1 and first_v.get("page"):
-                pg = int(first_v["page"])
-                parts[f"{q}.1"] = {"page": pg, "bbox": _synthetic_label_bbox(page_lines.get(pg, []), first_v.get("bbox"))}
-        subs = sorted(int(sub) for lab in parts for qq, sub in [lab.split(".")] if qq == q and sub != "0")
-        # Fill only one-step internal OCR gaps with support on both sides; do not expand a lone
-        # false high subpart into a whole run of synthetic labels.
-        if len(subs) >= 3:
-            for prev_sub, next_sub in zip(subs, subs[1:]):
-                if next_sub - prev_sub == 2:
-                    missing = prev_sub + 1
-                    anchor = parts[f"{q}.{next_sub}"]
-                    parts[f"{q}.{missing}"] = {"page": anchor.get("page"), "bbox": dict(anchor.get("bbox") or {})}
-
-    # Preserve explicit one-part structured questions seen as a bare top-level header (for example
-    # GCSE Combined Chemistry 03 with no decimal sub-label) without converting ordinary question
-    # headers that already have .1/.2 children into extra .0 parts.
-    present_q = {lab.split(".")[0] for lab in parts}
-    for q, (pg, bb) in top_cands.items():
-        if q not in present_q:
-            parts.setdefault(f"{q}.0", {"page": pg, "bbox": bb})
-
-    # Section B: walk MCQ candidates in reading order, accepting a tight increasing sequence.
-    structured_q = set(core_q or [int(lab.split(".")[0]) for lab in parts])
     expect = (max(structured_q) + 1) if structured_q else 1
     mcq_cands.sort(key=lambda c: (c[0], -(c[2] or {}).get("t", 0)))   # page, then top-down
     cand = {}                            # nn -> (page, bbox), first occurrence in reading order
     for pg, nn, bb in mcq_cands:
         cand.setdefault(int(nn), (pg, bb))
-    # Take exact expected numbers; for tiny OCR gaps before the next real MCQ candidate, emit
+    # Walk the sequence: take the exact expected number when present; only jump a small gap
-    # deterministic placeholders so a single garbled number does not end Section B recovery.
+    # (<=3) when it's genuinely absent (OCR-garbled, e.g. "09"->"60") so one bad number doesn't
+    # truncate the section. Out-of-window noise (misread "60") never enters.
     seq = []
     while True:
         if expect in cand and expect not in structured_q:
@@ -411,10 +330,7 @@ def aqa_questions_rapid(rapid_glob):
             continue
         nxt = [n for n in cand if expect < n <= expect + 3 and n not in structured_q]
         if nxt:
-            jump_to = min(nxt)
+            expect = min(nxt)
-            for missing in range(expect, jump_to):
-                seq.append((missing, cand[jump_to]))
-            expect = jump_to
             continue
         break
     # Only commit if this is a real Section-B MCQ run, not a few stray page/figure numbers on a
@@ -605,11 +521,6 @@ def docling_regions(doc):
     return regions
 
 
-def _norm_region_type(kind):
-    kind = (kind or "answer_lines").strip().lower().replace("-", "_")
-    return kind if kind in {"answer_lines", "answer_box", "working_space"} else "working_space"
-
-
 def merge_gemma(parts, gemma_dir):
     """Attach gemma4:e4b answer_regions (#3) to parts by for_part; gap-fill missing marks."""
     n_reg = n_fill = 0
@@ -618,9 +529,8 @@ def merge_gemma(parts, gemma_dir):
         for r in d.get("answer_regions", []):
             lab = _norm_label(r.get("for_part", ""))
             if lab in parts:
-                parts[lab]["regions"].append({"type": _norm_region_type(r.get("kind", "answer_lines")),
+                parts[lab]["regions"].append({"type": r.get("kind", "answer_lines"),
-                                              "source": "gemma",
+                                              "source": "gemma"})
-                                              **({"bbox": r.get("bbox")} if r.get("bbox") else {})})
                 n_reg += 1
         for qp in d.get("question_parts", []):
             lab = _norm_label(qp.get("label", ""))
@@ -638,70 +548,6 @@ def _norm_label(s):
     return s
 
 
-
-def attach_detected_response_regions(parts, pdf_path):
-    """Attach OpenCV response-region candidates to the nearest known part on the same page.
-
-    This is the deterministic answer-region backbone used before/alongside gemma: it emits the
-    same answer_lines / answer_box / working_space taxonomy and keeps the mapper schema unchanged.
-    Coordinates from regions.py are rendered-page TOPLEFT px; callers can persist them as candidate
-    response areas or use the counts as harness coverage.
-    """
-    if not pdf_path or not os.path.exists(pdf_path):
-        return 0, []
-    try:
-        candidates = region_mod.detect_response_regions_from_pdf(pdf_path, min_confidence=0.32)
-    except RuntimeError as exc:
-        print(f"response-regions    : unavailable ({exc})")
-        return 0, []
-    except Exception as exc:
-        print(f"response-regions    : failed ({exc})")
-        return 0, []
-
-    by_page = defaultdict(list)
-    for lab, part in parts.items():
-        if part.get("page") is not None and part.get("bbox"):
-            by_page[int(part["page"])].append((lab, part))
-
-    attached = 0
-    for cand in candidates:
-        # regions.py page_index is zero-based; extraction/template parts are one-based.
-        pg = int(cand.get("page_index", 0)) + 1
-        page_parts = by_page.get(pg) or []
-        if not page_parts:
-            continue
-        rb = cand.get("bbox") or {}
-        meta = cand.get("meta") or {}
-        center_top_px = float(rb.get("y", 0)) + float(rb.get("h", 0)) / 2
-        page_height_px = float(meta.get("page_height_px") or 0)
-        page_height_pdf = float(meta.get("page_height_pdf") or 0)
-        if page_height_px > 0 and page_height_pdf > 0:
-            region_y_pdf = (1.0 - center_top_px / page_height_px) * page_height_pdf
-        else:
-            region_y_pdf = -center_top_px
-        best_lab = None
-        best_score = 1e9
-        for lab, part in page_parts:
-            pb = part.get("bbox") or {}
-            part_mid = (float(pb.get("t", 0)) + float(pb.get("b", 0))) / 2
-            # Prefer the nearest label above/near the response area; a small penalty keeps
-            # previous-part assignment stable when regions sit between two labels.
-            below_penalty = 0 if region_y_pdf <= float(pb.get("t", 0)) + 18 else 120
-            score = abs(part_mid - region_y_pdf) + below_penalty
-            if score < best_score:
-                best_lab, best_score = lab, score
-        if best_lab:
-            parts[best_lab].setdefault("regions", []).append({
-                "type": _norm_region_type(cand.get("region_type")),
-                "source": "opencv",
-                "confidence": cand.get("confidence"),
-                "bbox": rb,
-                "detection_method": cand.get("detection_method"),
-                **({"line_count": cand.get("line_count")} if cand.get("line_count") is not None else {}),
-            })
-            attached += 1
-    return attached, candidates
-
 def extract_tables(parts, doc, granite="off", pdf=None, cache_glob=None):
     """Selective table-cell extraction (PLAN.md §B): standard TableFormer grids always; Granite
     <otsl> on router-flagged pages when granite!='off'. Returns (data_tables, all_tables).
@@ -780,7 +626,7 @@ GT_PARTS_PHYSICS = ["01.1","01.2","01.3","01.4","02.1","02.2","02.3","02.4","03.
     "10.1","10.2","10.3","11.1","11.2","11.3","11.4"]
 
 # official paper maxima — the strongest grammar sanity check (marks_sum should match)
-EXPECTED_MAX = {"8463": 100, "7408": 85, "7402": 91, "7405": 105, "8461": 100, "8462": 100, "8464": 70, "1MA1": 80, "H556": 70}
+EXPECTED_MAX = {"8463": 100, "7408": 85, "8461": 100, "1MA1": 80, "H556": 70}
 
 
 def expected_max(code):
@@ -820,7 +666,6 @@ def main():
     ap.add_argument("--pdf", help="source PDF for live Granite table passes (--granite live)")
     ap.add_argument("--rapid", help="AQA RapidOCR per-page glob (the v1 95% path)")
     ap.add_argument("--gemma", help="gemma sweep dir with p*.json answer_regions")
-    ap.add_argument("--response-regions", dest="response_regions_pdf", help="PDF to scan with deterministic response-region detector and attach to parts")
     ap.add_argument("--marks-fill", dest="marks_fill",
                     help="gemma_marks.py fills JSON: fill marks=None parts (Edexcel/OCR (N)/[N] gap-fill)")
     ap.add_argument("--granite", default="off", choices=["off", "cached", "live"],
@@ -828,7 +673,6 @@ def main():
     ap.add_argument("--granite-cache", default="results/VLM_granite_p*.doctags",
                     help="glob of cached *.doctags for --granite cached / live fallback")
     ap.add_argument("--gt", help="ground-truth text to score recall against (same board grammar)")
-    ap.add_argument("--expected-max", type=int, help="authoritative paper max marks for OCR eval harnesses when front matter/code OCR is missing")
     ap.add_argument("--board", default="auto", choices=["auto", "aqa", "edexcel", "ocr"])
     ap.add_argument("--out", default="results/structured.json")
     a = ap.parse_args()
@@ -907,11 +751,6 @@ def main():
     n_reg = n_fill = 0
     if a.gemma and os.path.isdir(a.gemma):
         n_reg, n_fill = merge_gemma(parts, a.gemma)
-    n_cv_regions = 0
-    cv_region_candidates = []
-    response_pdf = a.response_regions_pdf or a.pdf or a.ocr
-    if response_pdf:
-        n_cv_regions, cv_region_candidates = attach_detected_response_regions(parts, response_pdf)
     n_marks_fill = 0
     if a.marks_fill and os.path.exists(a.marks_fill):
         fills = json.load(open(a.marks_fill)).get("fills", {})
@@ -919,20 +758,6 @@ def main():
             if lab in parts and parts[lab].get("marks") is None:
                 parts[lab]["marks"] = int(mk); n_marks_fill += 1
 
-    exp_max_override = a.expected_max
-    # Targeted marks gap-fill: if OCR recovered all but one mark and the authoritative
-    # paper max leaves a small plausible residual, attach that residual to the lone
-    # missing part. This keeps the deterministic label backbone and only fills the
-    # narrow low-confidence gap instead of using gemma/full extraction as source of truth.
-    n_residual_marks_fill = 0
-    if exp_max_override:
-        missing_labs = [lab for lab, part in parts.items() if part.get("marks") is None]
-        known_sum = sum(part["marks"] for part in parts.values() if part.get("marks") is not None)
-        residual = exp_max_override - known_sum
-        if len(missing_labs) == 1 and 1 <= residual <= 9:
-            parts[missing_labs[0]]["marks"] = residual
-            n_residual_marks_fill = 1
-
     questions = build_questions(parts)
 
     # --- coverage ------------------------------------------------------------------------
@@ -949,7 +774,7 @@ def main():
 
     marks_known = sum(1 for v in parts.values() if v.get("marks") is not None)
     marks_sum = sum(v["marks"] for v in parts.values() if v.get("marks") is not None)
-    exp_max = exp_max_override or expected_max(code) or fm.get("max_marks")   # harness override, code-based, else front-matter total
+    exp_max = expected_max(code) or fm.get("max_marks")   # code-based, else front-matter total
     marks_check = (None if exp_max is None else
                    {"sum": marks_sum, "expected_max": exp_max,
                     "pct": round(marks_sum / exp_max * 100, 1)})
@@ -966,9 +791,6 @@ def main():
             "marks_check": marks_check,
             "gemma_answer_regions": n_reg, "gemma_marks_filled": n_fill,
             "gemma_marks_gapfilled": n_marks_fill,
-            "residual_marks_gapfilled": n_residual_marks_fill,
-            "opencv_answer_regions": n_cv_regions,
-            "opencv_answer_region_candidates": len(cv_region_candidates),
             "n_data_tables": len(data_tables),
             "n_furniture_tables": sum(1 for t in all_tables if t["is_furniture"]),
             "table_sources": {s: sum(1 for t in data_tables if t["source"] == s)
@@ -988,10 +810,7 @@ def main():
     print(f"marks               : {marks_known}/{len(parts)} parts known (sum {marks_sum}){mc}"
           + (f"; +{n_mark_geo} by geometry" if n_mark_geo else ""))
     print(f"gemma regions       : {n_reg} answer_regions, {n_fill} marks gap-filled"
-          + (f"; +{n_marks_fill} marks via --marks-fill" if n_marks_fill else "")
+          + (f"; +{n_marks_fill} marks via --marks-fill" if n_marks_fill else ""))
-          + (f"; +{n_residual_marks_fill} residual marks gap-fill" if n_residual_marks_fill else ""))
-    if response_pdf:
-        print(f"opencv regions      : {n_cv_regions} attached / {len(cv_region_candidates)} candidates")
     print(f"tables              : {len(data_tables)} data table(s) "
           f"{result['stats']['table_sources']} on pages {tbl_pages}; "
           f"{result['stats']['n_furniture_tables']} furniture filtered; {n_tbl} parts flagged")

api/services/docling/finalize.py

@@ -59,61 +59,6 @@ GEOMETRY = [
          extract=["--docling", "results/genreport/ocrh556/ocr.json", "--board", "ocr",
                   "--marks-fill", "results/genreport/ocrh556/marks_fill.json"]),
 ]
-
-B1_GEOMETRY = [
-    dict(slug="b1-aqa-biology-7402-1-2023jun", title="AQA A-level Biology 7402/1 2023 Jun (image-only OCR baseline)",
-         board="aqa", level="A-level", path="B1 image-only OCR (RapidOCR margin-pass)",
-         storage_loc="cc.examboards/aqa/biology/7402/1/2023-jun/qp.pdf",
-         pdf="samples/b1/aqa-biology-7402-1-2023jun.pdf",
-         docling="results/b1_rapid/b1-aqa-biology-7402-1-2023jun/merged.json",
-         rapid="results/b1_rapid/b1-aqa-biology-7402-1-2023jun/p*.json",
-         gt_key="b1-aqa-biology-7402-1-2023jun", expected_max=91),
-    dict(slug="b1-aqa-chemistry-7405-1-2022jun", title="AQA A-level Chemistry 7405/1 2022 Jun (image-only OCR baseline)",
-         board="aqa", level="A-level", path="B1 image-only OCR (RapidOCR margin-pass)",
-         storage_loc="cc.examboards/aqa/chemistry/7405/1/2022-jun/qp.pdf",
-         pdf="samples/b1/aqa-chemistry-7405-1-2022jun.pdf",
-         docling="results/b1_rapid/b1-aqa-chemistry-7405-1-2022jun/merged.json",
-         rapid="results/b1_rapid/b1-aqa-chemistry-7405-1-2022jun/p*.json",
-         gt_key="b1-aqa-chemistry-7405-1-2022jun", expected_max=105),
-    dict(slug="b1-aqa-physics-7408-1-2022jun", title="AQA A-level Physics 7408/1 2022 Jun (image-only OCR baseline)",
-         board="aqa", level="A-level", path="B1 image-only OCR (RapidOCR margin-pass)",
-         storage_loc="cc.examboards/aqa/physics/7408/1/2022-jun/qp.pdf",
-         pdf="samples/b1/aqa-physics-7408-1-2022jun.pdf",
-         docling="results/b1_rapid/b1-aqa-physics-7408-1-2022jun/merged.json",
-         rapid="results/b1_rapid/b1-aqa-physics-7408-1-2022jun/p*.json",
-         gt_key="b1-aqa-physics-7408-1-2022jun", expected_max=85),
-    dict(slug="b1-aqa-biology-8461-1h-2022jun", title="AQA GCSE Biology 8461/1H 2022 Jun (image-only OCR baseline)",
-         board="aqa", level="GCSE", path="B1 image-only OCR (RapidOCR margin-pass)",
-         storage_loc="cc.examboards/aqa/biology/8461/1h/2022-jun/qp.pdf",
-         pdf="samples/b1/aqa-biology-8461-1h-2022jun.pdf",
-         docling="results/b1_rapid/b1-aqa-biology-8461-1h-2022jun/merged.json",
-         rapid="results/b1_rapid/b1-aqa-biology-8461-1h-2022jun/p*.json",
-         gt_key="b1-aqa-biology-8461-1h-2022jun", expected_max=100),
-    dict(slug="b1-aqa-chemistry-8462-1h-2022jun", title="AQA GCSE Chemistry 8462/1H 2022 Jun (image-only OCR baseline)",
-         board="aqa", level="GCSE", path="B1 image-only OCR (RapidOCR margin-pass)",
-         storage_loc="cc.examboards/aqa/chemistry/8462/1h/2022-jun/qp.pdf",
-         pdf="samples/b1/aqa-chemistry-8462-1h-2022jun.pdf",
-         docling="results/b1_rapid/b1-aqa-chemistry-8462-1h-2022jun/merged.json",
-         rapid="results/b1_rapid/b1-aqa-chemistry-8462-1h-2022jun/p*.json",
-         gt_key="b1-aqa-chemistry-8462-1h-2022jun", expected_max=100),
-    dict(slug="b1-aqa-combined-8464-b1h-2022jun", title="AQA GCSE Combined Science Trilogy 8464/B/1H 2022 Jun (image-only OCR baseline)",
-         board="aqa", level="GCSE", path="B1 image-only OCR (RapidOCR margin-pass)",
-         storage_loc="cc.examboards/aqa/combined-science-trilogy/8464/b-1h/2022-jun/qp.pdf",
-         pdf="samples/b1/aqa-combined-8464-b1h-2022jun.pdf",
-         docling="results/b1_rapid/b1-aqa-combined-8464-b1h-2022jun/merged.json",
-         rapid="results/b1_rapid/b1-aqa-combined-8464-b1h-2022jun/p*.json",
-         gt_key="b1-aqa-combined-8464-b1h-2022jun", expected_max=70),
-    dict(slug="b1-aqa-combined-8464-c1h-2022jun", title="AQA GCSE Combined Science Trilogy 8464/C/1H 2022 Jun (image-only OCR baseline; 8465 not present in dev catalogue)",
-         board="aqa", level="GCSE", path="B1 image-only OCR (RapidOCR margin-pass)",
-         storage_loc="cc.examboards/aqa/combined-science-trilogy/8464/c-1h/2022-jun/qp.pdf",
-         pdf="samples/b1/aqa-combined-8464-c1h-2022jun.pdf",
-         docling="results/b1_rapid/b1-aqa-combined-8464-c1h-2022jun/merged.json",
-         rapid="results/b1_rapid/b1-aqa-combined-8464-c1h-2022jun/p*.json",
-         gt_key="b1-aqa-combined-8464-c1h-2022jun", expected_max=70),
-]
-
-GT_LABELS_PATH = "fixtures/b1_gt_labels.json"
-
 FAST = [
     dict(slug="aqa-physics-7408-fast", title="AQA A-level Physics 7408/1 (born-digital)", board="aqa",
          level="A-level", pdf="samples/extra/aqa-alevel-physics-7408-1-jun22-qp.pdf",
@@ -150,68 +95,16 @@ def jload(p):
         return {}
 
 
-
+def stats_from(struct, val):
-def load_gt_labels():
-    try:
-        return json.load(open(GT_LABELS_PATH))
-    except Exception:
-        return {}
-
-
-def part_labels(struct):
-    labels = []
-    for q in struct.get("questions", []) or []:
-        for part in q.get("parts", []) or []:
-            lab = part.get("label")
-            if lab:
-                labels.append(lab)
-    return labels
-
-
-def coverage_against_labels(struct, labels):
-    if not labels:
-        return None
-    rec = set(part_labels(struct))
-    gt = set(labels)
-    hit = sorted(rec & gt)
-    miss = sorted(gt - rec)
-    return {"coverage_pct": round(len(hit) / len(gt) * 100, 1),
-            "recovered": len(hit), "total": len(gt), "missed": miss,
-            "source": "fixtures/b1_gt_labels.json"}
-
-
-def answer_region_count(struct):
-    top = len(struct.get("regions", []) or [])
-    per_part = 0
-    for q in struct.get("questions", []) or []:
-        for part in q.get("parts", []) or []:
-            per_part += len(part.get("regions", []) or [])
-    return top + per_part
-
-
-def ensure_rapid_cache(p):
-    if os.path.exists(p["docling"]):
-        return True
-    if not os.path.exists(p["pdf"]):
-        print(f"  ! missing source PDF for {p['slug']}: {p['pdf']} (storage_loc={p.get('storage_loc')})")
-        return False
-    return run(["scripts/rapid_pass.py", p["pdf"], "b1_rapid/" + p["slug"]])
-
-def stats_from(struct, val, gt_labels=None):
     st = struct.get("stats", {}) or {}
     mc = st.get("marks_check") or {}
-    cov = coverage_against_labels(struct, gt_labels) if gt_labels else (struct.get("coverage", {}) or {})
+    cov = struct.get("coverage", {}) or {}
     return {
         "board": struct.get("board"), "paper_code": struct.get("paper_code"),
         "n_questions": st.get("n_questions"), "n_parts": st.get("n_parts"),
         "marks_sum": mc.get("sum"), "official_max": mc.get("expected_max"),
         "marks_pct": mc.get("pct"),
-        "coverage_pct": cov.get("coverage_pct"), "coverage_recovered": cov.get("recovered"),
+        "coverage_pct": cov.get("coverage_pct"), "coverage_missed": cov.get("missed", []),
-        "coverage_total": cov.get("total"), "coverage_source": cov.get("source"),
-        "coverage_missed": cov.get("missed", []), "answer_regions": answer_region_count(struct),
-        "opencv_answer_regions": st.get("opencv_answer_regions"),
-        "opencv_answer_region_candidates": st.get("opencv_answer_region_candidates"),
-        "residual_marks_gapfilled": st.get("residual_marks_gapfilled"),
         "validate_verdict": (val.get("summary") or {}).get("worst_severity"),
         "validate_flags": val.get("flags", []),
         "questions_expected": (val.get("summary") or {}).get("questions_expected"),
@@ -220,19 +113,12 @@ def stats_from(struct, val, gt_labels=None):
     }
 
 
-def do_geometry(p, overlays, gt_labels=None, prepare_ocr=False):
+def do_geometry(p, overlays):
     d = os.path.join(FINAL, p["slug"]); os.makedirs(d, exist_ok=True)
     S, F, B, R, T, V = (os.path.join(d, f) for f in
                         ("structured.json", "furniture.json", "bands.json", "page_roles.json",
                          "template.json", "validate.json"))
-    if prepare_ocr and not ensure_rapid_cache(p):
+    ex = ["extract.py"] + p["extract"] + ["--out", S]
-        raise RuntimeError(f"unable to prepare B1 OCR cache for {p['slug']}")
-    extract_args = p.get("extract") or ["--docling", p["docling"], "--rapid", p["rapid"], "--board", p.get("board", "aqa")]
-    ex = ["extract.py"] + extract_args + ["--out", S]
-    if p.get("pdf"):
-        ex += ["--response-regions", p["pdf"]]
-    if p.get("expected_max"):
-        ex += ["--expected-max", str(p["expected_max"])]
     if p.get("gt"):
         ex += ["--gt", p["gt"]]
     run(ex)
@@ -252,7 +138,7 @@ def do_geometry(p, overlays, gt_labels=None, prepare_ocr=False):
         odbg = os.path.join(d, "overlays", "debug")
         run(["scripts/overlay.py", S, p["pdf"], "--docling", p["docling"], "--bands", B,
              "--furniture", F, "--pages", "1,2,3,4,5", "--dpi", "120", "--out", odbg])
-    return stats_from(jload(S), jload(V), gt_labels), d
+    return stats_from(jload(S), jload(V)), d
 
 
 def do_fast(p):
@@ -278,9 +164,6 @@ def per_paper_report(p, s, d, kind):
              + (f"  (missed {s['coverage_missed'][:8]})" if s.get('coverage_missed') else "")
              if s['coverage_pct'] is not None else "- **coverage vs GT:** n/a",
              f"- **G6 verdict:** {s['validate_verdict']}",
-             f"- **answer-region count:** {s.get('answer_regions')}",
-             f"- **opencv response regions:** {s.get('opencv_answer_regions')} attached / "
-             f"{s.get('opencv_answer_region_candidates')} candidates",
              ]
     if s["validate_flags"]:
         lines += ["", "**Flags (human-review hints):**"] + [f"- {f}" for f in s["validate_flags"]]
@@ -295,28 +178,21 @@ def per_paper_report(p, s, d, kind):
 def main():
     ap = argparse.ArgumentParser()
     ap.add_argument("--no-overlays", action="store_true")
-    ap.add_argument("--b1-only", action="store_true", help="run only the Sprint B1 image-only OCR eval corpus")
-    ap.add_argument("--prepare-ocr", action="store_true", help="populate missing B1 RapidOCR caches via dsync before running")
     a = ap.parse_args()
     os.makedirs(FINAL, exist_ok=True)
     catalog = {"generated_at": datetime.datetime.now().isoformat(timespec="seconds"),
                "papers": []}
     total_imgs = 0
 
-    gt_fixtures = load_gt_labels()
+    for p in GEOMETRY:
-    geometry = B1_GEOMETRY if a.b1_only else GEOMETRY
-    fast = [] if a.b1_only else FAST
-
-    for p in geometry:
         print(f"[geometry] {p['slug']}")
-        gt_labels = (gt_fixtures.get(p.get("gt_key") or p["slug"], {}) or {}).get("labels")
+        s, d = do_geometry(p, not a.no_overlays)
-        s, d = do_geometry(p, not a.no_overlays, gt_labels=gt_labels, prepare_ocr=a.prepare_ocr)
         n = per_paper_report(p, s, d, p["path"])
         total_imgs += n
         catalog["papers"].append({**{k: p[k] for k in ("slug", "title", "board", "level")},
                                   "kind": "geometry", "path": p["path"], "dir": d,
                                   "overlay_images": n, **s})
-    for p in fast:
+    for p in FAST:
         print(f"[fast] {p['slug']}")
         s, d = do_fast(p)
         per_paper_report(p, s, d, "born-digital fast-path")
@@ -338,13 +214,13 @@ def write_index(catalog, total_imgs):
          "`overlays/template/` (human-review view, all pages) and `overlays/debug/` (raw-detection view).",
          "Machine catalog: `catalog.json`.", "",
          "## Image-only / OCR-path (with geometry + overlays)", "",
-         "| Paper | Board / level | Q/parts | Marks/max | Coverage | Answer regions | G6 | Images |",
+         "| Paper | Board / level | Q/parts | Marks/max | Coverage | G6 | Images |",
-         "|---|---|---|---|---|---|---|---|"]
+         "|---|---|---|---|---|---|---|"]
     for p in g:
         cov = f"{p['coverage_pct']}%" if p['coverage_pct'] is not None else "n/a"
         L.append(f"| [{p['title']}]({p['slug']}/report.md) | {p['board']} {p['level']} | "
                  f"{p['n_questions']}/{p['n_parts']} | {p['marks_sum']}/{p['official_max']} "
-                 f"({p['marks_pct']}%) | {cov} | {p.get('answer_regions')} | {p['validate_verdict']} | "
+                 f"({p['marks_pct']}%) | {cov} | {p['validate_verdict']} | "
                  f"{p['overlay_images']} |")
     L += ["", "## Born-digital fast-path (CPU, no geometry)", "",
           "| Paper | Board / level | Q/parts | Marks/max | Coverage | G6 |",

api/services/docling/fixtures/b1_gt_labels.json

@@ -1,356 +0,0 @@
-{
-  "b1-aqa-biology-7402-1-2023jun": {
-    "source_pdf": "cc.examboards/aqa/biology/7402/1/2023-jun/qp.pdf",
-    "source_method": "AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.",
-    "board_detected": "aqa",
-    "paper_code_detected": "7402/1",
-    "labels": [
-      "01.1",
-      "01.2",
-      "01.3",
-      "02.1",
-      "02.2",
-      "02.3",
-      "03.1",
-      "03.2",
-      "03.3",
-      "03.4",
-      "03.5",
-      "04.1",
-      "04.2",
-      "04.3",
-      "05.1",
-      "05.2",
-      "05.3",
-      "05.4",
-      "05.5",
-      "06.1",
-      "06.2",
-      "06.3",
-      "06.4",
-      "07.1",
-      "07.2",
-      "89.6",
-      "08.1",
-      "08.2",
-      "08.3",
-      "08.4",
-      "09.1",
-      "09.2",
-      "09.3",
-      "09.4",
-      "09.5",
-      "09.6",
-      "10.1",
-      "10.2",
-      "10.3"
-    ]
-  },
-  "b1-aqa-chemistry-7405-1-2022jun": {
-    "source_pdf": "cc.examboards/aqa/chemistry/7405/1/2022-jun/qp.pdf",
-    "source_method": "AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.",
-    "board_detected": "aqa",
-    "paper_code_detected": "7405/1",
-    "labels": [
-      "01.1",
-      "01.2",
-      "01.3",
-      "01.4",
-      "01.5",
-      "01.6",
-      "02.1",
-      "02.2",
-      "02.3",
-      "02.4",
-      "02.5",
-      "03.1",
-      "03.2",
-      "03.3",
-      "03.4",
-      "03.5",
-      "04.1",
-      "04.2",
-      "04.3",
-      "04.4",
-      "04.5",
-      "05.1",
-      "05.2",
-      "05.3",
-      "05.4",
-      "05.5",
-      "05.6",
-      "05.7",
-      "06.1",
-      "06.2",
-      "06.3",
-      "06.4",
-      "06.5",
-      "06.6",
-      "06.7",
-      "07.1",
-      "07.2",
-      "07.3",
-      "07.4",
-      "07.5",
-      "07.6",
-      "07.7",
-      "08.1",
-      "08.2",
-      "08.3",
-      "08.4",
-      "08.5"
-    ]
-  },
-  "b1-aqa-physics-7408-1-2022jun": {
-    "source_pdf": "cc.examboards/aqa/physics/7408/1/2022-jun/qp.pdf",
-    "source_method": "AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.",
-    "board_detected": "aqa",
-    "paper_code_detected": "7408/1",
-    "labels": [
-      "01.1",
-      "01.2",
-      "01.3",
-      "01.4",
-      "01.5",
-      "02.1",
-      "02.2",
-      "02.3",
-      "02.4",
-      "03.1",
-      "03.2",
-      "03.3",
-      "03.4",
-      "03.5",
-      "04.1",
-      "04.2",
-      "04.3",
-      "04.4",
-      "04.5",
-      "05.1",
-      "05.2",
-      "05.3",
-      "05.4",
-      "05.5",
-      "05.6",
-      "06.1",
-      "06.2",
-      "06.3",
-      "07.0",
-      "08.0",
-      "09.0",
-      "10.0",
-      "11.0",
-      "12.0",
-      "13.0",
-      "14.0",
-      "15.0",
-      "16.0",
-      "17.0",
-      "18.0",
-      "19.0",
-      "20.0",
-      "21.0",
-      "22.0",
-      "23.0",
-      "24.0",
-      "25.0",
-      "26.0",
-      "27.0",
-      "28.0",
-      "29.0",
-      "30.0",
-      "31.0"
-    ]
-  },
-  "b1-aqa-biology-8461-1h-2022jun": {
-    "source_pdf": "cc.examboards/aqa/biology/8461/1h/2022-jun/qp.pdf",
-    "source_method": "AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.",
-    "board_detected": "aqa",
-    "paper_code_detected": "8461/1",
-    "labels": [
-      "01.1",
-      "01.2",
-      "01.3",
-      "01.4",
-      "01.5",
-      "01.6",
-      "01.7",
-      "01.8",
-      "01.9",
-      "02.1",
-      "02.2",
-      "02.3",
-      "02.4",
-      "02.5",
-      "02.6",
-      "03.1",
-      "03.2",
-      "03.3",
-      "03.4",
-      "03.5",
-      "04.1",
-      "04.2",
-      "04.3",
-      "04.4",
-      "04.5",
-      "05.1",
-      "05.2",
-      "05.3",
-      "05.4",
-      "05.5",
-      "06.1",
-      "06.2",
-      "06.3",
-      "06.4",
-      "06.5",
-      "07.1",
-      "07.2",
-      "07.3",
-      "07.4",
-      "07.5",
-      "07.6",
-      "07.7",
-      "07.8"
-    ]
-  },
-  "b1-aqa-chemistry-8462-1h-2022jun": {
-    "source_pdf": "cc.examboards/aqa/chemistry/8462/1h/2022-jun/qp.pdf",
-    "source_method": "AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.",
-    "board_detected": "aqa",
-    "paper_code_detected": "8462/1",
-    "labels": [
-      "01.1",
-      "01.2",
-      "01.3",
-      "01.4",
-      "01.5",
-      "01.6",
-      "01.7",
-      "02.1",
-      "02.2",
-      "02.3",
-      "02.4",
-      "02.5",
-      "02.6",
-      "03.1",
-      "03.2",
-      "03.3",
-      "03.4",
-      "03.5",
-      "04.1",
-      "04.2",
-      "04.3",
-      "04.4",
-      "04.5",
-      "04.6",
-      "04.7",
-      "05.1",
-      "05.2",
-      "05.3",
-      "05.4",
-      "05.5",
-      "06.1",
-      "06.2",
-      "06.3",
-      "06.4",
-      "06.5",
-      "06.6",
-      "07.1",
-      "07.2",
-      "07.3",
-      "07.4",
-      "07.5",
-      "07.6",
-      "08.1",
-      "08.2",
-      "08.3",
-      "08.4",
-      "08.5"
-    ]
-  },
-  "b1-aqa-combined-8464-b1h-2022jun": {
-    "source_pdf": "cc.examboards/aqa/combined-science-trilogy/8464/b-1h/2022-jun/qp.pdf",
-    "source_method": "AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.",
-    "board_detected": "aqa",
-    "paper_code_detected": null,
-    "labels": [
-      "01.1",
-      "01.2",
-      "01.3",
-      "01.4",
-      "01.5",
-      "01.6",
-      "01.7",
-      "01.8",
-      "02.1",
-      "02.2",
-      "02.3",
-      "02.4",
-      "02.5",
-      "02.6",
-      "02.7",
-      "03.1",
-      "03.2",
-      "03.3",
-      "03.4",
-      "03.5",
-      "03.6",
-      "03.7",
-      "04.1",
-      "04.2",
-      "04.3",
-      "04.4",
-      "04.5",
-      "05.1",
-      "05.2",
-      "05.3",
-      "05.4",
-      "05.5",
-      "05.6",
-      "06.1",
-      "06.2",
-      "06.3"
-    ]
-  },
-  "b1-aqa-combined-8464-c1h-2022jun": {
-    "source_pdf": "cc.examboards/aqa/combined-science-trilogy/8464/c-1h/2022-jun/qp.pdf",
-    "source_method": "AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.",
-    "board_detected": "aqa",
-    "paper_code_detected": null,
-    "labels": [
-      "01.1",
-      "01.2",
-      "01.3",
-      "01.4",
-      "01.5",
-      "02.1",
-      "02.2",
-      "02.3",
-      "02.4",
-      "02.5",
-      "03.0",
-      "04.1",
-      "04.2",
-      "04.3",
-      "04.4",
-      "04.5",
-      "04.6",
-      "04.7",
-      "05.1",
-      "05.2",
-      "05.3",
-      "05.4",
-      "06.1",
-      "06.2",
-      "06.3",
-      "06.4",
-      "06.5",
-      "07.1",
-      "07.2",
-      "07.3",
-      "07.4",
-      "07.5",
-      "07.6"
-    ]
-  }
-}

api/services/docling/regions.py

@@ -162,16 +162,7 @@ def detect_response_regions_from_pdf(
                 page_index=page_index,
                 min_confidence=min_confidence,
             )
-            for candidate in page_candidates:
+            candidates.extend(candidate.to_mapper_dict() for candidate in page_candidates)
-                item = candidate.to_mapper_dict()
-                item.setdefault("meta", {}).update({
-                    "page_width_px": pix.width,
-                    "page_height_px": pix.height,
-                    "page_width_pdf": float(doc[page_index].rect.width),
-                    "page_height_pdf": float(doc[page_index].rect.height),
-                    "render_dpi": dpi,
-                })
-                candidates.append(item)
         return candidates
     finally:
         doc.close()
@@ -289,7 +280,7 @@ def _detect_answer_lines(binary: np.ndarray, *, page_index: int, width: int, hei
         span_ratio = box_w / max(width, 1)
         count_bonus = min(0.2, max(0, line_count - 1) * 0.05)
         confidence = min(0.92, 0.42 + span_ratio * 0.35 + count_bonus)
-        region_type = "answer_lines"
+        region_type = "answer_lines" if line_count > 1 else "working_space"
         candidates.append(
             RegionCandidate(
                 page_index=page_index,
@@ -360,7 +351,6 @@ def _detect_answer_boxes(binary: np.ndarray, *, page_index: int, width: int, hei
         if rectangularity < 0.03:
             continue
         confidence = min(0.88, 0.46 + min(0.24, w / width * 0.24) + min(0.18, h / height * 0.5))
-        region_type = "working_space" if (h > height * 0.12 and rectangularity < 0.18) else "answer_box"
         padded_x = max(0, x - 2)
         padded_y = max(0, y - 2)
         padded_right = min(width, x + w + 2)
@@ -372,7 +362,7 @@ def _detect_answer_boxes(binary: np.ndarray, *, page_index: int, width: int, hei
                 y=padded_y,
                 w=padded_right - padded_x,
                 h=padded_bottom - padded_y,
-                region_type=region_type,
+                region_type="answer_box",
                 confidence=confidence,
                 detection_method="opencv_contour_box",
                 meta={"rectangularity": round(float(rectangularity), 3)},

api/services/docling/scripts/fetch_b1_corpus.py

@@ -1,87 +0,0 @@
-#!/usr/bin/env python3
-"""Populate the gitignored B1 image-only eval corpus from the .94 exam-board store.
-
-The B1 eval papers are NOT committed (third-party copyright; served only via signed URLs).
-This script downloads each B1_GEOMETRY paper's `storage_loc` object from cc.examboards via the
-Storage API into its local `pdf` path (under samples/b1/), so finalize.py --b1-only and the
-B1-2/B1-3 generalization work can run against a real corpus.
-
-Run from api/services/docling/ inside the cc-api-dev container (SUPABASE_URL/SERVICE_ROLE_KEY in env):
-    python3 scripts/fetch_b1_corpus.py            # fetch all B1 papers (skip existing)
-    python3 scripts/fetch_b1_corpus.py --force    # re-download
-    python3 scripts/fetch_b1_corpus.py --only b1-aqa-physics-7408-1-2022jun
-    python3 scripts/fetch_b1_corpus.py --list     # show what would be fetched, no download
-"""
-from __future__ import annotations
-
-import argparse
-import os
-import sys
-
-# Import the canonical B1 corpus definition (slug, storage_loc, local pdf path) from finalize.
-_HERE = os.path.dirname(os.path.abspath(__file__))
-_DOCLING_DIR = os.path.dirname(_HERE)
-sys.path.insert(0, _DOCLING_DIR)
-from finalize import B1_GEOMETRY  # noqa: E402
-
-
-def _split_storage_loc(storage_loc: str) -> tuple[str, str]:
-    """'cc.examboards/aqa/biology/7402/1/2023-jun/qp.pdf' -> ('cc.examboards', 'aqa/.../qp.pdf')."""
-    bucket, _, path = storage_loc.partition("/")
-    if not bucket or not path:
-        raise ValueError(f"malformed storage_loc: {storage_loc!r}")
-    return bucket, path
-
-
-def _entries(only: str | None):
-    for p in B1_GEOMETRY:
-        loc = p.get("storage_loc")
-        pdf = p.get("pdf")
-        if not loc or not pdf:
-            continue
-        if only and p.get("slug") != only:
-            continue
-        yield p["slug"], loc, pdf
-
-
-def main() -> int:
-    ap = argparse.ArgumentParser(description="Fetch the B1 image-only eval corpus from .94 cc.examboards")
-    ap.add_argument("--force", action="store_true", help="re-download even if the local file exists")
-    ap.add_argument("--only", help="fetch a single paper by slug")
-    ap.add_argument("--list", action="store_true", help="list what would be fetched and exit")
-    args = ap.parse_args()
-
-    todo = list(_entries(args.only))
-    if not todo:
-        print("no matching B1 papers", file=sys.stderr)
-        return 1
-
-    if args.list:
-        for slug, loc, pdf in todo:
-            print(f"{slug}\t{loc}\t-> {pdf}")
-        return 0
-
-    from modules.database.supabase.utils.storage import StorageAdmin
-    storage = StorageAdmin()
-
-    ok = skipped = 0
-    for slug, loc, pdf in todo:
-        dest = os.path.join(_DOCLING_DIR, pdf) if not os.path.isabs(pdf) else pdf
-        if os.path.exists(dest) and not args.force:
-            print(f"[skip] {slug} (exists)")
-            skipped += 1
-            continue
-        bucket, path = _split_storage_loc(loc)
-        data = storage.download_file(bucket, path)
-        os.makedirs(os.path.dirname(dest), exist_ok=True)
-        with open(dest, "wb") as fh:
-            fh.write(data)
-        print(f"[ok]   {slug} <- {bucket}/{path} ({len(data)} bytes)")
-        ok += 1
-
-    print(f"fetched {ok}, skipped {skipped}, of {len(todo)}")
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

api/services/docling/scripts/make_b1_gt.py

@@ -1,32 +0,0 @@
-import json, sys
-from pathlib import Path
-base=Path('/app/api/services/docling')
-sys.path.insert(0, str(base))
-import extract
-papers=[
-('b1-aqa-biology-7402-1-2023jun','samples/b1/aqa-biology-7402-1-2023jun.pdf','cc.examboards/aqa/biology/7402/1/2023-jun/qp.pdf'),
-('b1-aqa-chemistry-7405-1-2022jun','samples/b1/aqa-chemistry-7405-1-2022jun.pdf','cc.examboards/aqa/chemistry/7405/1/2022-jun/qp.pdf'),
-('b1-aqa-physics-7408-1-2022jun','samples/b1/aqa-physics-7408-1-2022jun.pdf','cc.examboards/aqa/physics/7408/1/2022-jun/qp.pdf'),
-('b1-aqa-biology-8461-1h-2022jun','samples/b1/aqa-biology-8461-1h-2022jun.pdf','cc.examboards/aqa/biology/8461/1h/2022-jun/qp.pdf'),
-('b1-aqa-chemistry-8462-1h-2022jun','samples/b1/aqa-chemistry-8462-1h-2022jun.pdf','cc.examboards/aqa/chemistry/8462/1h/2022-jun/qp.pdf'),
-('b1-aqa-combined-8464-b1h-2022jun','samples/b1/aqa-combined-8464-b1h-2022jun.pdf','cc.examboards/aqa/combined-science-trilogy/8464/b-1h/2022-jun/qp.pdf'),
-('b1-aqa-combined-8464-c1h-2022jun','samples/b1/aqa-combined-8464-c1h-2022jun.pdf','cc.examboards/aqa/combined-science-trilogy/8464/c-1h/2022-jun/qp.pdf'),
-]
-out={}
-for slug, rel, storage in papers:
-    lines=extract.lines_from_pdftext(str(base/rel))
-    board, code=extract.detect_board(lines)
-    if board != 'aqa':
-        raise RuntimeError(f'{slug}: expected AQA board, detected {board!r} ({code!r})')
-    parts=extract.parse_text_by_board(lines, board)
-    labels=list(parts)
-    out[slug]={
-        'source_pdf': storage,
-        'source_method': 'AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.',
-        'board_detected': board,
-        'paper_code_detected': code,
-        'labels': labels,
-    }
-    print(slug, board, code, len(labels), labels[:5], labels[-5:])
-Path(base/'fixtures').mkdir(exist_ok=True)
-Path(base/'fixtures/b1_gt_labels.json').write_text(json.dumps(out, indent=2)+"\n")

api/services/docling/scripts/rapid_pass.py

@@ -1,69 +0,0 @@
-#!/usr/bin/env python3
-"""
-rapid_pass.py — generalise the proven AQA "RapidOCR margin-pass" (95.2% on the image-only
-8463 paper) to any AQA paper. Born-digital AQA QPs ship a text layer, so we force RapidOCR
-over the *rendered* page (`force_ocr:true`) to simulate the image-only redistribution case
-and recover the boxed `NN.M` question numbers Tesseract shatters.
-
-For each page it writes results/<outdir>/p{N}.json (a full per-page DoclingDocument, the
-shape extract.py's aqa_questions_rapid expects) and a merged.json (for board / front-matter
-detection). All GPU work is serialised + OOM-resilient through dsync.
-
-Usage:
-  python scripts/rapid_pass.py samples/extra/aqa-alevel-physics-7408-1-jun22-qp.pdf rapid_7408
-  python scripts/rapid_pass.py <pdf> <outdir-slug> [first_page] [last_page]
-"""
-import os, sys, json, subprocess, re
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-import dsync
-
-OPTS = {"ocr_engine": "rapidocr", "force_ocr": True}
-
-
-def npages(pdf):
-    out = subprocess.check_output(["pdfinfo", pdf]).decode()
-    return int(out.split("Pages:")[1].split()[0])
-
-
-def main():
-    pdf = sys.argv[1]
-    slug = sys.argv[2]
-    if os.path.isabs(slug) or ".." in slug.split(os.sep) or not re.fullmatch(r"[A-Za-z0-9._/-]+", slug):
-        raise SystemExit(f"unsafe output slug: {slug!r}")
-    n = npages(pdf)
-    first = int(sys.argv[3]) if len(sys.argv) > 3 else 1
-    last = min(int(sys.argv[4]), n) if len(sys.argv) > 4 else n
-    if first > n or first > last:
-        print(f"requested page range {first}-{last} is outside PDF ({n} pages); nothing to do")
-        return
-    outdir = os.path.join("results", slug)
-    os.makedirs(outdir, exist_ok=True)
-
-    r = dsync._redis()
-    print(f"redis: {'connected' if r else 'NO CACHE'}  pdf={pdf}  pages {first}-{last}/{n}")
-    merged = {"texts": [], "tables": [], "pictures": [], "pages": {}, "_failed_pages": []}
-    for pg in range(first, last + 1):
-        page_path = os.path.join(outdir, f"p{pg}.json")
-        if os.path.exists(page_path):
-            doc = json.load(open(page_path))
-            print(f"  p{pg}: file cache HIT ({len(doc.get(texts, []))} texts)")
-        else:
-            doc = dsync.convert_page(pdf, pg, OPTS, r=r)
-        if not doc:
-            merged["_failed_pages"].append(pg)
-            print(f"  p{pg}: FAILED")
-            continue
-        json.dump(doc, open(page_path, "w"))
-        for k in ("texts", "tables", "pictures"):
-            merged[k].extend(doc.get(k, []))
-        merged["pages"].update(doc.get("pages", {}))
-        nmarg = sum(1 for t in doc.get("texts", [])
-                    if (t.get("prov") or [{}])[0].get("bbox", {}).get("l", 999) <= 140)
-        print(f"  p{pg}: {len(doc.get('texts', []))} texts ({nmarg} left-margin)")
-    json.dump(merged, open(os.path.join(outdir, "merged.json"), "w"))
-    print(f"-> {outdir}/  ({last-first+1-len(merged['_failed_pages'])} pages, "
-          f"failed={merged['_failed_pages']})")
-
-
-if __name__ == "__main__":
-    main()

docker-entrypoint.sh

@@ -75,14 +75,6 @@ if [ "$RUN_INIT" = "true" ]; then
                 }
                 print_success "GAIS data import completed"
                 ;;
-            "exam-corpus")
-                print_status "Seeding exam-paper corpus (manifest-gated; skips if none configured)..."
-                python3 main.py --mode exam-corpus || {
-                    print_error "Exam corpus seed failed!"
-                    exit 1
-                }
-                print_success "Exam corpus seed completed"
-                ;;
             "full")
                 print_status "Running full initialization..."
                 python3 main.py --mode infra || exit 1

main.py

@@ -323,52 +323,6 @@ def run_gais_data_mode():
 
 # Old clear_dev_redis_queue function removed - now handled by Redis Manager
 
-def run_exam_corpus_mode():
-    """Seed the public exam-paper corpus from a manifest (optional, gated).
-
-    Env controls:
-      EXAM_CORPUS_MANIFEST    - path to the corpus manifest (required to do anything)
-      EXAM_CORPUS_DRY_RUN     - 'true' to validate + report only
-      EXAM_CORPUS_FORCE       - 'true' to re-upload/overwrite existing objects
-      EXAM_CORPUS_BOARD/_SPEC - filter to one exam_board_code / spec_code
-      EXAM_CORPUS_USER_SUBSET - 'true' to also seed a user-side test subset
-      EXAM_CORPUS_FIRST_SWEEP - 'true' to run the docling/auto-map first pass
-
-    Skips gracefully (success) when no manifest is configured/present, so it is safe
-    in a comma-mode list (e.g. INIT_MODE=infra,seed,exam-corpus) before papers exist.
-    Buckets are NOT created here — infra mode (buckets.py) owns provisioning.
-    """
-    logger.info("Running in exam-corpus seed mode")
-    manifest = os.getenv("EXAM_CORPUS_MANIFEST")
-    if not manifest or not os.path.exists(manifest):
-        logger.warning(
-            f"exam-corpus: no manifest at EXAM_CORPUS_MANIFEST={manifest!r}; skipping (nothing to seed yet)"
-        )
-        return True
-    try:
-        from run.initialization.seed_exam_corpus import load
-        rep = load(
-            manifest,
-            dry_run=_truthy_env("EXAM_CORPUS_DRY_RUN"),
-            force=_truthy_env("EXAM_CORPUS_FORCE"),
-            board_filter=os.getenv("EXAM_CORPUS_BOARD") or None,
-            spec_filter=os.getenv("EXAM_CORPUS_SPEC") or None,
-            user_subset=_truthy_env("EXAM_CORPUS_USER_SUBSET"),
-            do_first_sweep=_truthy_env("EXAM_CORPUS_FIRST_SWEEP"),
-        )
-        if rep.errors:
-            logger.error(f"exam-corpus seed completed with {len(rep.errors)} error(s)")
-            return False
-        logger.info(
-            f"exam-corpus seed ok: specs={rep.specs_upserted} papers={rep.papers_upserted} "
-            f"uploaded={rep.files_uploaded}"
-        )
-        return True
-    except Exception as e:
-        logger.error(f"exam-corpus seed failed: {e}")
-        return False
-
-
 def run_development_mode():
     """Run the server in development mode with auto-reload"""
     logger.info("Running in development mode")
@@ -457,7 +411,7 @@ Startup modes:
     
     parser.add_argument(
         '--mode', '-m',
-        choices=['infra', 'seed', 'seed-test', 'gais-data', 'exam-corpus', 'dev', 'prod'],
+        choices=['infra', 'seed', 'seed-test', 'gais-data', 'dev', 'prod'],
         default='dev',
         help='Startup mode (default: dev)'
     )
@@ -493,11 +447,6 @@ if __name__ == "__main__":
         success = run_gais_data_mode()
         sys.exit(0 if success else 1)
     
-    elif args.mode == 'exam-corpus':
-        # Seed the public exam-paper corpus from a manifest (gated; skips if none configured)
-        success = run_exam_corpus_mode()
-        sys.exit(0 if success else 1)
-
     elif args.mode == 'dev':
         # Run development server
         run_development_mode()

modules/upload_validation.py

@@ -1,99 +0,0 @@
-"""Upload boundary validation shared by file-upload endpoints.
-
-E3 hardening: keep user-facing upload routes from buffering arbitrary data and
-from accepting arbitrary MIME/types into Supabase storage.
-"""
-from __future__ import annotations
-
-import os
-from typing import Iterable, Optional
-
-from fastapi import HTTPException, UploadFile
-
-# Conservative defaults: Classroom Copilot uploads are user documents/images.
-# Exam scan uploads already have their own 50 MB PDF-only guard in routers.exam.batches.
-MAX_UPLOAD_BYTES = int(os.getenv("CC_UPLOAD_MAX_BYTES", str(25 * 1024 * 1024)))
-UPLOAD_CHUNK_BYTES = 1024 * 1024
-
-ALLOWED_UPLOAD_MIME_TYPES = frozenset(
-    mt.strip().lower()
-    for mt in os.getenv(
-        "CC_UPLOAD_ALLOWED_MIME_TYPES",
-        ",".join(
-            [
-                "application/pdf",
-                "image/png",
-                "image/jpeg",
-                "image/webp",
-                "image/gif",
-                "text/plain",
-                "text/csv",
-                "text/markdown",
-                "application/msword",
-                "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-                "application/vnd.ms-powerpoint",
-                "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-                "application/vnd.ms-excel",
-                "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-            ]
-        ),
-    ).split(",")
-    if mt.strip()
-)
-
-_PDF_MIME_TYPES = {"application/pdf", "application/x-pdf"}
-
-
-def allowed_upload_mime_types_csv() -> str:
-    """Stable display string for evidence/errors without leaking config internals."""
-    return ", ".join(sorted(ALLOWED_UPLOAD_MIME_TYPES))
-
-
-def _declared_mime(upload: UploadFile) -> str:
-    return (upload.content_type or "application/octet-stream").split(";", 1)[0].strip().lower()
-
-
-def validate_upload_mime(upload: UploadFile, *, allowed_mime_types: Optional[Iterable[str]] = None) -> str:
-    """Validate client-declared upload MIME/type and return its normalised value."""
-    declared = _declared_mime(upload)
-    allowed = {mt.lower() for mt in (allowed_mime_types or ALLOWED_UPLOAD_MIME_TYPES)}
-    if declared not in allowed:
-        raise HTTPException(
-            status_code=415,
-            detail=(
-                f"Unsupported upload type '{declared}'. Allowed MIME types: "
-                f"{', '.join(sorted(allowed))}"
-            ),
-        )
-    return declared
-
-
-async def read_upload_bytes(
-    upload: UploadFile,
-    *,
-    max_bytes: int = MAX_UPLOAD_BYTES,
-    allowed_mime_types: Optional[Iterable[str]] = None,
-) -> tuple[bytes, str]:
-    """Validate MIME and read an UploadFile with a hard size ceiling."""
-    mime_type = validate_upload_mime(upload, allowed_mime_types=allowed_mime_types)
-    chunks: list[bytes] = []
-    total = 0
-    while True:
-        chunk = await upload.read(UPLOAD_CHUNK_BYTES)
-        if not chunk:
-            break
-        total += len(chunk)
-        if total > max_bytes:
-            raise HTTPException(status_code=413, detail=f"Upload exceeds max size ({max_bytes} bytes)")
-        chunks.append(chunk)
-    return b"".join(chunks), mime_type
-
-
-async def read_pdf_upload_bytes(upload: UploadFile, *, max_bytes: int = MAX_UPLOAD_BYTES) -> bytes:
-    """Read a PDF-only upload with size and lightweight magic-header validation."""
-    data, _mime_type = await read_upload_bytes(upload, max_bytes=max_bytes, allowed_mime_types=_PDF_MIME_TYPES)
-    if not data:
-        raise HTTPException(status_code=400, detail="Uploaded PDF is empty")
-    if not data.startswith(b"%PDF-"):
-        raise HTTPException(status_code=415, detail="Uploaded file is not a valid PDF")
-    return data

routers/database/files/files.py

@@ -12,7 +12,6 @@ from modules.auth.supabase_bearer import SupabaseBearer, verify_supabase_jwt_str
 from modules.logger_tool import initialise_logger
 from modules.database.supabase.utils.client import SupabaseServiceRoleClient
 from modules.database.supabase.utils.storage import StorageAdmin
-from modules.upload_validation import read_upload_bytes
 from modules.document_processor import DocumentProcessor
 from modules.queue_system import (
     enqueue_tika_task, enqueue_docling_task, enqueue_split_map_task,
@@ -37,24 +36,6 @@ DOCLING_NOOCR_TIMEOUT = int(os.getenv('DOCLING_NOOCR_TIMEOUT', '3600'))  # 1 hou
 
 logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
 
-def _user_id_from_payload(payload: Dict[str, Any]) -> str:
-    user_id = payload.get('sub') or payload.get('user_id')
-    if not user_id:
-        raise HTTPException(status_code=401, detail="Invalid token payload")
-    return user_id
-
-def _cabinet_visible_to_user(client: SupabaseServiceRoleClient, cabinet_id: str, user_id: str) -> bool:
-    """Require cabinet ownership before service-role reads file metadata."""
-    owned = (
-        client.supabase.table('file_cabinets')
-        .select('id')
-        .eq('id', cabinet_id)
-        .eq('user_id', user_id)
-        .limit(1)
-        .execute()
-    )
-    return bool(owned.data)
-
 def _safe_filename(name: str) -> str:
     base = os.path.basename(name or 'file')
     return re.sub(r"[^A-Za-z0-9._-]+", "_", base)
@@ -89,13 +70,13 @@ async def upload_file(
     # Stage DB row to get file_id
     staged_path = f"{cabinet_id}/staging/{uuid.uuid4()}"
     name = _safe_filename(path or file.filename)
-    file_bytes, mime_type = await read_upload_bytes(file)
+    file_bytes = await file.read()
     insert_res = client.supabase.table('files').insert({
         'cabinet_id': cabinet_id,
         'name': name,
         'path': staged_path,
         'bucket': bucket,
-        'mime_type': mime_type,
+        'mime_type': file.content_type,
         'uploaded_by': user_id,
         'size_bytes': len(file_bytes),
         'source': 'classroomcopilot-web'
@@ -108,7 +89,7 @@ async def upload_file(
     # Final storage path: bucket/cabinet_id/file_id/file
     final_storage_path = f"{cabinet_id}/{file_id}/{name}"
     try:
-        storage.upload_file(bucket, final_storage_path, file_bytes, mime_type, upsert=True)
+        storage.upload_file(bucket, final_storage_path, file_bytes, file.content_type or 'application/octet-stream', upsert=True)
     except Exception as e:
         # cleanup staged row
         client.supabase.table('files').delete().eq('id', file_id).execute()
@@ -136,10 +117,7 @@ async def upload_file(
 
 @router.get("/files")
 def list_files(cabinet_id: str, payload: Dict[str, Any] = Depends(auth)):
-    user_id = _user_id_from_payload(payload)
     client = SupabaseServiceRoleClient()
-    if not _cabinet_visible_to_user(client, cabinet_id, user_id):
-        return []
     res = client.supabase.table('files').select('*').eq('cabinet_id', cabinet_id).execute()
     return res.data
 

routers/database/files/files_simplified.py

@@ -19,7 +19,6 @@ from fastapi.responses import JSONResponse
 from modules.auth.supabase_bearer import SupabaseBearer
 from modules.database.supabase.utils.client import SupabaseServiceRoleClient
 from modules.database.supabase.utils.storage import StorageAdmin
-from modules.upload_validation import read_upload_bytes
 from modules.logger_tool import initialise_logger
 
 router = APIRouter()
@@ -27,24 +26,6 @@ auth = SupabaseBearer()
 
 logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
 
-def _user_id_from_payload(payload: Dict[str, Any]) -> str:
-    user_id = payload.get('sub') or payload.get('user_id')
-    if not user_id:
-        raise HTTPException(status_code=401, detail="Invalid token payload")
-    return user_id
-
-def _cabinet_visible_to_user(client: SupabaseServiceRoleClient, cabinet_id: str, user_id: str) -> bool:
-    """Require cabinet ownership before service-role reads file metadata."""
-    owned = (
-        client.supabase.table('file_cabinets')
-        .select('id')
-        .eq('id', cabinet_id)
-        .eq('user_id', user_id)
-        .limit(1)
-        .execute()
-    )
-    return bool(owned.data)
-
 def _choose_bucket(scope: str, user_id: str, school_id: Optional[str]) -> str:
     """Choose appropriate bucket based on scope - matches old system logic."""
     scope = (scope or 'teacher').lower()
@@ -73,9 +54,10 @@ async def upload_file(
         if not user_id:
             raise HTTPException(status_code=401, detail="User ID required")
         
-        # Validate MIME/type and read file content with a hard size limit.
+        # Read file content
-        file_bytes, mime_type = await read_upload_bytes(file)
+        file_bytes = await file.read()
         file_size = len(file_bytes)
+        mime_type = file.content_type or 'application/octet-stream'
         filename = file.filename or path
         
         logger.info(f"📤 Simplified upload: {filename} ({file_size} bytes) for user {user_id}")
@@ -152,10 +134,7 @@ async def upload_file(
 @router.get("/files")
 def list_files(cabinet_id: str, payload: Dict[str, Any] = Depends(auth)):
     """List files in a cabinet."""
-    user_id = _user_id_from_payload(payload)
     client = SupabaseServiceRoleClient()
-    if not _cabinet_visible_to_user(client, cabinet_id, user_id):
-        return []
     res = client.supabase.table('files').select('*').eq('cabinet_id', cabinet_id).execute()
     return res.data
 

routers/database/tools/platform_admin_router.py

@@ -126,28 +126,13 @@ async def platform_stats(
 
 @router.post("/reset")
 async def reset_environment(
-    scope: str = "all",
     _: dict = Depends(require_platform_admin),
 ) -> Dict[str, Any]:
-    """DESTRUCTIVE: wipe test data. Platform admin only.
+    """DESTRUCTIVE: wipe all test data. Neo4j + Supabase. Platform admin only."""
-
-    scope (query param):
-      - all          : full wipe (Neo4j + Supabase data + auth users) AND the entire
-                       exam-marker subsystem below.
-      - exam-corpus  : ONLY the entire exam-marker subsystem, not just public papers:
-                       public corpus/eb_* data, cc.examboards storage objects, exam
-                       templates, template layouts, questions, boundaries, response
-                       areas, marking batches, student submissions, and mark entries
-                       (without touching schools/users).
-      - timetable    : ONLY timetable/calendar materialization tables.
-    """
-    if scope not in ("all", "exam-corpus", "timetable"):
-        raise HTTPException(status_code=400, detail="scope must be one of: all, exam-corpus, timetable")
     import asyncio
-    import functools
     from run.initialization.reset_environment import reset as _reset
     loop = asyncio.get_event_loop()
-    result = await loop.run_in_executor(None, functools.partial(_reset, scope))
+    result = await loop.run_in_executor(None, _reset)
     return {"status": "ok", **result}
 
 

routers/exam/templates.py

@@ -28,7 +28,6 @@ from api.services.docling.regions import detect_response_regions_from_pdf
 from modules.database.services.exam_projection import project_template, project_template_safe
 from modules.database.supabase.utils.client import SupabaseServiceRoleClient
 from modules.database.supabase.utils.storage import StorageAdmin
-from modules.upload_validation import read_pdf_upload_bytes
 from modules.logger_tool import initialise_logger
 from routers.exam.dependencies import ExamContext, get_exam_context, lookup_exam_code
 from routers.exam.schemas import (
@@ -137,22 +136,6 @@ def _lookup_exam_storage_loc(exam_id: str) -> Optional[str]:
         return None
 
 
-def _signed_url_value(result: Any) -> str:
-    """Normalise supabase-py signed URL responses across v1/v2 shapes."""
-    if isinstance(result, str):
-        return result
-    if isinstance(result, dict):
-        value = result.get("signedURL") or result.get("signedUrl") or result.get("signed_url")
-        if value:
-            return str(value)
-    data = getattr(result, "data", None)
-    if isinstance(data, dict):
-        value = data.get("signedURL") or data.get("signedUrl") or data.get("signed_url")
-        if value:
-            return str(value)
-    raise ValueError("Storage service did not return a signed URL")
-
-
 async def _parse_create_template_request(request: Request) -> tuple[CreateTemplateRequest, Optional[UploadFile]]:
     content_type = request.headers.get("content-type", "")
     if "multipart/form-data" in content_type:
@@ -181,7 +164,11 @@ async def _upload_template_source_file(
     institute_id: str,
     upload: UploadFile,
 ) -> str:
-    file_bytes = await read_pdf_upload_bytes(upload)
+    file_bytes = await upload.read()
+    if not file_bytes:
+        raise HTTPException(status_code=400, detail="Uploaded PDF is empty")
+    if upload.content_type and upload.content_type != "application/pdf":
+        raise HTTPException(status_code=400, detail="Uploaded file must be a PDF")
 
     service = SupabaseServiceRoleClient()
     storage = StorageAdmin()
@@ -543,27 +530,12 @@ def _map_first_pass_to_rows(template_id: str, first_pass: Dict[str, Any], pdf_by
     return {"questions": questions, "response_areas": response_areas, "boundaries": boundaries, "layout": layout}
 
 
-def _dedupe_rows_by_id(rows: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
-    """Preserve first occurrence of stable AI row ids emitted by noisy OCR detectors."""
-    out: List[Dict[str, Any]] = []
-    seen: set[str] = set()
-    for row in rows:
-        row_id = row.get("id")
-        if row_id:
-            key = str(row_id)
-            if key in seen:
-                continue
-            seen.add(key)
-        out.append(row)
-    return out
-
-
 def _refresh_ai_rows(ctx: ExamContext, template_id: str, rows: Dict[str, List[Dict[str, Any]]]) -> None:
     sb = ctx.supabase
     for table in ("exam_response_areas", "exam_boundaries", "exam_template_layout", "exam_questions"):
         sb.table(table).delete().eq("template_id", template_id).eq("source", "ai").eq("confirmed", False).execute()
     for table, key in (("exam_questions", "questions"), ("exam_response_areas", "response_areas"), ("exam_boundaries", "boundaries"), ("exam_template_layout", "layout")):
-        payload = _dedupe_rows_by_id(rows.get(key) or [])
+        payload = rows.get(key) or []
         if payload:
             sb.table(table).insert(payload).execute()
 
@@ -645,13 +617,12 @@ async def create_template(
 
 
 @router.get("/catalogue")
-async def list_catalogue_papers(
+async def list_catalogue_papers() -> Dict[str, Any]:
-    ctx: ExamContext = Depends(get_exam_context),
+    """Lightweight exam-board paper catalogue for the create dialog."""
-) -> Dict[str, Any]:
-    """Lightweight authenticated exam-board metadata catalogue for the create dialog."""
     try:
+        sb = SupabaseServiceRoleClient().supabase
         res = (
-            ctx.supabase.table("eb_exams")
+            sb.table("eb_exams")
             .select("id, exam_code, spec_code, paper_code, tier, session, type_code, storage_loc")
             .eq("type_code", "QP")
             .order("exam_code")
@@ -662,50 +633,6 @@ async def list_catalogue_papers(
         raise HTTPException(status_code=502, detail=f"Could not load catalogue papers: {exc}")
 
 
-@router.get("/catalogue/{exam_id}/signed-url")
-async def get_catalogue_paper_signed_url(
-    exam_id: str,
-    expires_in: int = 300,
-    ctx: ExamContext = Depends(get_exam_context),
-) -> Dict[str, Any]:
-    """Return a short-lived signed URL for an authenticated user's catalogue PDF access.
-
-    The storage operation uses service role as a scoped backend exception for signing only;
-    raw cc.examboards object reads remain denied by storage.objects RLS.
-    """
-    expires_in = max(60, min(int(expires_in or 300), 3600))
-    try:
-        row = _first(
-            ctx.supabase.table("eb_exams")
-            .select("id, exam_code, storage_loc")
-            .eq("id", exam_id)
-            .eq("type_code", "QP")
-            .limit(1)
-            .execute()
-        )
-        if not row or not row.get("storage_loc"):
-            raise HTTPException(status_code=404, detail="Catalogue paper not found")
-        try:
-            bucket, path = _parse_storage_loc(row["storage_loc"])
-        except ValueError:
-            raise HTTPException(status_code=404, detail="Catalogue paper not found")
-        if bucket != "cc.examboards":
-            raise HTTPException(status_code=404, detail="Catalogue paper not found")
-        signed_url = _signed_url_value(StorageAdmin().create_signed_url(bucket, path, expires_in))
-        return {
-            "exam_id": row["id"],
-            "exam_code": row.get("exam_code"),
-            "bucket": bucket,
-            "path": path,
-            "expires_in": expires_in,
-            "signed_url": signed_url,
-        }
-    except HTTPException:
-        raise
-    except Exception as exc:
-        raise HTTPException(status_code=502, detail=f"Could not sign catalogue paper URL: {exc}")
-
-
 @router.get("/templates")
 async def list_templates(
     include_archived: bool = False,

routers/simple_upload.py

@@ -26,7 +26,6 @@ from fastapi.responses import JSONResponse
 from modules.auth.supabase_bearer import SupabaseBearer
 from modules.database.supabase.utils.client import SupabaseServiceRoleClient
 from modules.database.supabase.utils.storage import StorageAdmin
-from modules.upload_validation import read_upload_bytes
 from modules.logger_tool import initialise_logger
 
 router = APIRouter()
@@ -60,9 +59,10 @@ async def upload_single_file(
         if not user_id:
             raise HTTPException(status_code=401, detail="User ID required")
         
-        # Validate MIME/type and read file content with a hard size limit.
+        # Read file content
-        file_bytes, mime_type = await read_upload_bytes(file)
+        file_bytes = await file.read()
         file_size = len(file_bytes)
+        mime_type = file.content_type or 'application/octet-stream'
         filename = file.filename or path
         
         logger.info(f"📤 Simple upload: {filename} ({file_size} bytes) for user {user_id}")
@@ -234,9 +234,10 @@ async def upload_directory(
             # Process each file
             for i, (file, relative_path) in enumerate(zip(files, relative_paths)):
                 try:
-                    # Validate MIME/type and read file content with a hard size limit.
+                    # Read file content
-                    file_bytes, mime_type = await read_upload_bytes(file)
+                    file_bytes = await file.read()
                     file_size = len(file_bytes)
+                    mime_type = file.content_type or 'application/octet-stream'
                     filename = file.filename or f"file_{i}"
                     
                     total_size += file_size
@@ -290,8 +291,6 @@ async def upload_directory(
                     
                     logger.info(f"📄 Uploaded file {i+1}/{len(files)}: {relative_path}")
                     
-                except HTTPException:
-                    raise
                 except Exception as e:
                     logger.error(f"Failed to upload file {relative_path}: {e}")
                     # Continue with other files, don't fail entire upload

run/initialization/buckets.py

@@ -46,7 +46,7 @@ def initialize_buckets() -> dict:
                     file_size_limit=1000 * 1024 * 1024,  # 1GB
                 )
             },
-            # Exam Board files (admin-curated public exam corpus: QP/MS/insert/ER + specs)
+            # Exam Board files
             {
                 "id": "cc.examboards",
                 "options": CreateBucketOptions(
@@ -55,34 +55,6 @@ def initialize_buckets() -> dict:
                     file_size_limit=1000 * 1024 * 1024,  # 1GB
                 )
             },
-            # ── Storage taxonomy bins (access scoped by RLS on bucket + leading path segment; RLS = D1) ──
-            # Platform-managed public/shared assets (readable by all authenticated users).
-            {
-                "id": "cc.public",
-                "options": CreateBucketOptions(
-                    name="Classroom Copilot Public",
-                    public=False,
-                    file_size_limit=1000 * 1024 * 1024,  # 1GB
-                )
-            },
-            # Institute-scoped operational assets: cc.institutes/{institute_id}/...
-            {
-                "id": "cc.institutes",
-                "options": CreateBucketOptions(
-                    name="Classroom Copilot Institutes",
-                    public=False,
-                    file_size_limit=1000 * 1024 * 1024,  # 1GB
-                )
-            },
-            # Platform-admin-only assets, seeds, intake/staging for unidentified papers.
-            {
-                "id": "cc.admin",
-                "options": CreateBucketOptions(
-                    name="Classroom Copilot Admin",
-                    public=False,
-                    file_size_limit=1000 * 1024 * 1024,  # 1GB
-                )
-            },
         ]
         
         results = {}
@@ -109,17 +81,11 @@ def initialize_buckets() -> dict:
                     logger.error(f"Failed to create bucket: {bucket['id']}")
                     
             except Exception as e:
-                # Idempotent: an already-existing bucket is not a failure on re-run.
+                results[bucket["id"]] = {
-                if any(s in str(e).lower() for s in ("already exists", "duplicate", "resource already")):
+                    "status": "error",
-                    results[bucket["id"]] = {"status": "exists", "result": str(e)}
+                    "error": str(e)
-                    success_count += 1
+                }
-                    logger.info(f"Bucket already exists (ok): {bucket['id']}")
+                logger.error(f"Error creating bucket {bucket['id']}: {str(e)}")
-                else:
-                    results[bucket["id"]] = {
-                        "status": "error",
-                        "error": str(e)
-                    }
-                    logger.error(f"Error creating bucket {bucket['id']}: {str(e)}")
         
         # Determine overall success
         if success_count == total_count:

run/initialization/manifests/_corpus_store/.gitignore

@@ -1,3 +0,0 @@
-# Persistent local corpus store — PDFs are NOT committed (re-downloadable from manifest).
-*
-!.gitignore

run/initialization/manifests/generate_corpus_manifest.py

@@ -1,501 +0,0 @@
-#!/usr/bin/env python3
-"""
-generate_corpus_manifest.py — build the public exam-corpus manifest from OFFICIAL sources,
-verifying every source URL is live before it is written.
-
-Output: exam-corpus.yaml (consumed by run/initialization/seed_exam_corpus.py).
-
-Sources (all official exam-board hosts; public past-paper PDFs):
-  AQA      filestore.aqa.org.uk  — fully templatable; enumerated + HEAD-verified here.
-  Edexcel  qualifications.pearson.com — date suffix non-derivable; confirmed URLs embedded.
-  OCR      www.ocr.org.uk/Images — opaque doc-id; confirmed URLs embedded.
-
-Every URL is HEAD/GET-checked (200 + application/pdf) before inclusion, so the committed
-manifest never carries a dead or wrong-cased link. Re-run to refresh as more sessions go public.
-
-Conventions (locked — see ~/cc/ideas/2026-06-07-exam-paper-ingestion.md):
-  session   = "YYYY-Mon"        e.g. 2022-Jun
-  exam_code = BOARD-award-PAPER-SESSIONCOMPACT-ROLE   e.g. AQA-8463-1H-2022JUN-QP
-"""
-from __future__ import annotations
-import concurrent.futures as cf
-import os
-import sys
-import urllib.error
-import urllib.request
-from typing import Any, Dict, List, Optional, Tuple
-
-import yaml
-
-AQA_BASE = "https://filestore.aqa.org.uk/sample-papers-and-mark-schemes"
-ROLE_TOKEN = {"QP": "QP", "MS": "MS", "ER": "WRE"}       # AQA filestore role tokens
-MONTHS = {"JUN": ("june", "Jun"), "NOV": ("november", "Nov")}
-FETCHED = "2026-06-07"
-
-
-def head_ok(url: str, timeout: int = 20) -> bool:
-    """True iff the URL resolves to a real PDF (200 + application/pdf), following redirects.
-    AQA soft-404s redirect to www.aqa.org.uk/req_path=... (text/html), so we check content-type.
-    Uses a tiny Range GET (stdlib urllib) so we never pull the whole PDF just to verify it."""
-    req = urllib.request.Request(url, headers={"Range": "bytes=0-3", "User-Agent": "cc-corpus/1.0"})
-    try:
-        with urllib.request.urlopen(req, timeout=timeout) as r:
-            ctype = (r.headers.get("content-type") or "").lower()
-            return r.status in (200, 206) and "pdf" in ctype
-    except urllib.error.HTTPError as e:
-        # A 206/200 PDF never lands here; 404/redirect-to-html will.
-        ctype = (e.headers.get("content-type") or "").lower() if e.headers else ""
-        return e.code in (200, 206) and "pdf" in ctype
-    except Exception:
-        return False
-
-
-# ─────────────────────────── AQA catalogue ───────────────────────────
-# spec_code, subject, award, award_level, first_teach, [(filestore_papercode, paper_code, tier), ...]
-def _gcse_single(award: str) -> List[Tuple[str, str, Optional[str]]]:
-    out = []
-    for paper in ("1", "2"):
-        for tier in ("F", "H"):
-            out.append((f"{award}{paper}{tier}", f"{award}/{paper}{tier}", tier))
-    return out
-
-def _trilogy(award: str) -> List[Tuple[str, str, Optional[str]]]:
-    out = []
-    for subj in ("B", "C", "P"):
-        for paper in ("1", "2"):
-            for tier in ("F", "H"):
-                out.append((f"{award}{subj}{paper}{tier}", f"{award}/{subj}/{paper}{tier}", tier))
-    return out
-
-def _alevel(award: str, papers=("1", "2", "3")) -> List[Tuple[str, str, Optional[str]]]:
-    return [(f"{award}{p}", f"{award}/{p}", None) for p in papers]
-
-def _subj(award: str, papers, tiers=(None,)) -> List[Tuple[str, str, Optional[str]]]:
-    """Generic GCSE/A-level builder. tiers=('F','H') for tiered subjects (Maths/Science),
-    tiers=(None,) for untiered (English/Geography/CS/Business/Psychology)."""
-    out = []
-    for p in papers:
-        for t in tiers:
-            tl = t or ""
-            out.append((f"{award}{p}{tl}", f"{award}/{p}{tl}", t))
-    return out
-
-def _mfl(award: str) -> List[Tuple[str, str, Optional[str]]]:
-    """AQA MFL: Listening/Reading/Writing papers, each Foundation/Higher (Speaking is teacher-conducted,
-    no public QP). Filestore code encodes skill+tier, e.g. 8658LH = French Listening Higher."""
-    out = []
-    for skill in ("L", "R", "W"):
-        for t in ("F", "H"):
-            out.append((f"{award}{skill}{t}", f"{award}/{skill}{t}", t))
-    return out
-
-AQA_SPECS = [
-    # ── Sciences (round 1 — kept at full depth) ──────────────────────────────────────
-    ("AQA-BIOL-8461", "BIOLOGY", "8461", "GCSE", "2016", _gcse_single("8461")),
-    ("AQA-CHEM-8462", "CHEMISTRY", "8462", "GCSE", "2016", _gcse_single("8462")),
-    ("AQA-PHYS-8463", "PHYSICS", "8463", "GCSE", "2016", _gcse_single("8463")),
-    ("AQA-COMB-8464", "COMBINED SCIENCE TRILOGY", "8464", "GCSE", "2016", _trilogy("8464")),
-    ("AQA-BIOL-7401", "BIOLOGY", "7401", "AS", "2015", _alevel("7401", ("1", "2"))),
-    ("AQA-BIOL-7402", "BIOLOGY", "7402", "A-level", "2015", _alevel("7402")),
-    ("AQA-CHEM-7404", "CHEMISTRY", "7404", "AS", "2015", _alevel("7404", ("1", "2"))),
-    ("AQA-CHEM-7405", "CHEMISTRY", "7405", "A-level", "2015", _alevel("7405")),
-    ("AQA-PHYS-7407", "PHYSICS", "7407", "AS", "2015", _alevel("7407", ("1", "2"))),
-    ("AQA-PHYS-7408", "PHYSICS", "7408", "A-level", "2015", _alevel("7408")),
-    # ── Round 2 breadth — high-volume core (Maths, English) ───────────────────────────
-    ("AQA-MATH-8300", "MATHEMATICS", "8300", "GCSE", "2015", _subj("8300", ("1", "2", "3"), ("F", "H"))),
-    ("AQA-MATH-7357", "MATHEMATICS", "7357", "A-level", "2017", _alevel("7357", ("1", "2", "3"))),
-    ("AQA-MATH-7356", "MATHEMATICS", "7356", "AS", "2017", _alevel("7356", ("1", "2"))),
-    ("AQA-ENGL-8700", "ENGLISH LANGUAGE", "8700", "GCSE", "2015", _subj("8700", ("1", "2"))),
-    ("AQA-ENGLIT-8702", "ENGLISH LITERATURE", "8702", "GCSE", "2015", _subj("8702", ("1", "2"))),
-    ("AQA-ENGL-7702", "ENGLISH LANGUAGE", "7702", "A-level", "2015", _alevel("7702", ("1", "2"))),
-    ("AQA-ENGLIT-7712", "ENGLISH LITERATURE A", "7712", "A-level", "2015", _alevel("7712", ("1", "2"))),
-    # ── Round 2 breadth — humanities / others ─────────────────────────────────────────
-    ("AQA-GEOG-8035", "GEOGRAPHY", "8035", "GCSE", "2016", _subj("8035", ("1", "2", "3"))),
-    ("AQA-GEOG-7037", "GEOGRAPHY", "7037", "A-level", "2016", _alevel("7037", ("1", "2"))),
-    ("AQA-COMP-8525", "COMPUTER SCIENCE", "8525", "GCSE", "2020", _subj("8525", ("1", "2"))),
-    ("AQA-COMP-7517", "COMPUTER SCIENCE", "7517", "A-level", "2015", _alevel("7517", ("1", "2"))),
-    ("AQA-BUS-8132", "BUSINESS", "8132", "GCSE", "2017", _subj("8132", ("1", "2"))),
-    ("AQA-BUS-7132", "BUSINESS", "7132", "A-level", "2015", _alevel("7132", ("1", "2", "3"))),
-    ("AQA-PSYC-8182", "PSYCHOLOGY", "8182", "GCSE", "2017", _subj("8182", ("1", "2"))),
-    ("AQA-PSYC-7182", "PSYCHOLOGY", "7182", "A-level", "2015", _alevel("7182", ("1", "2", "3"))),
-    # ── Round 2 breadth — modern foreign languages (Listening/Reading/Writing, F+H) ───
-    ("AQA-FREN-8658", "FRENCH", "8658", "GCSE", "2016", _mfl("8658")),
-    ("AQA-SPAN-8698", "SPANISH", "8698", "GCSE", "2016", _mfl("8698")),
-    ("AQA-GERM-8668", "GERMAN", "8668", "GCSE", "2016", _mfl("8668")),
-    ("AQA-FREN-7652", "FRENCH", "7652", "A-level", "2016", _alevel("7652", ("1", "2"))),
-    ("AQA-SPAN-7692", "SPANISH", "7692", "A-level", "2016", _alevel("7692", ("1", "2"))),
-    ("AQA-GERM-7662", "GERMAN", "7662", "A-level", "2016", _alevel("7662", ("1", "2"))),
-]
-AQA_SESSIONS = ["JUN18", "JUN19", "NOV20", "NOV21", "JUN22", "JUN23", "JUN24"]
-AQA_ROLES = ["QP", "MS", "ER"]
-
-
-def aqa_url(papercode: str, role: str, session: str) -> Tuple[str, str]:
-    mon = session[:3]
-    yy = session[3:]
-    folder, _ = MONTHS[mon]
-    year = "20" + yy
-    fname = f"AQA-{papercode}-{ROLE_TOKEN[role]}-{session}.PDF"
-    return f"{AQA_BASE}/{year}/{folder}/{fname}", fname
-
-
-def session_pretty(session: str) -> Tuple[str, str]:
-    mon = session[:3]       # "JUN" | "NOV"
-    yy = session[3:]        # "22"
-    _, pretty = MONTHS[mon]
-    # ("2022-Jun" display session, "2022JUN" compact for exam_code — year-first, matches the
-    # locked exam_code convention and the Edexcel/OCR entries).
-    return f"20{yy}-{pretty}", f"20{yy}{mon}"
-
-
-def build_aqa() -> Dict[str, Any]:
-    candidates: List[Tuple[str, str, str, str, str, str, Optional[str], str, str, str]] = []
-    # (spec_code, subject, award, paper_fc, paper_code, tier, role, session, url, fname)
-    spec_meta = {}
-    for spec_code, subject, award, level, first_teach, papers in AQA_SPECS:
-        spec_meta[spec_code] = (subject, award, level, first_teach)
-        for paper_fc, paper_code, tier in papers:
-            for session in AQA_SESSIONS:
-                for role in AQA_ROLES:
-                    url, fname = aqa_url(paper_fc, role, session)
-                    candidates.append((spec_code, subject, award, paper_fc, paper_code, tier,
-                                       role, session, url, fname))
-
-    print(f"[AQA] HEAD-verifying {len(candidates)} candidate URLs...", file=sys.stderr)
-    live: Dict[int, bool] = {}
-    with cf.ThreadPoolExecutor(max_workers=24) as ex:
-        futs = {ex.submit(head_ok, c[8]): i for i, c in enumerate(candidates)}
-        done = 0
-        for fut in cf.as_completed(futs):
-            i = futs[fut]
-            live[i] = fut.result()
-            done += 1
-            if done % 60 == 0:
-                print(f"  ...{done}/{len(candidates)} ({sum(live.values())} live)", file=sys.stderr)
-
-    specs: Dict[str, Dict[str, Any]] = {}
-    for i, c in enumerate(candidates):
-        if not live.get(i):
-            continue
-        spec_code, subject, award, paper_fc, paper_code, tier, role, session, url, fname = c
-        sess_pretty, sess_compact = session_pretty(session)
-        token = paper_fc[len(award):]  # "1H" / "P1H" / "1"
-        exam_code = f"AQA-{award}-{token}-{sess_compact}-{role}"
-        spec = specs.setdefault(spec_code, {"papers": []})
-        spec["papers"].append({
-            "exam_code": exam_code,
-            "paper_code": paper_code,
-            "tier": tier,
-            "session": sess_pretty,
-            "doc_type": role,
-            "file": {
-                "source": f"url:{url}",
-                "original_name": fname,
-                "provenance": {"source_url": url, "fetched": FETCHED,
-                               "license": "AQA public past paper"},
-            },
-        })
-
-    spec_list = []
-    for spec_code, subject, award, level, first_teach, _papers in AQA_SPECS:
-        if spec_code not in specs:
-            continue
-        papers = sorted(specs[spec_code]["papers"], key=lambda p: p["exam_code"])
-        spec_list.append({
-            "spec_code": spec_code, "exam_board_code": "AQA", "subject_code": subject,
-            "award_code": award, "award_level": level, "first_teach": first_teach,
-            "papers": papers,
-        })
-        print(f"[AQA] {spec_code}: {len(papers)} live papers", file=sys.stderr)
-    return {"exam_board_code": "AQA", "specifications": spec_list}
-
-
-# ─────────────── Edexcel / OCR — confirmed direct URLs (re-verified at build) ───────────────
-# These boards aren't templatable (Edexcel has a non-derivable date suffix; OCR uses opaque
-# doc-ids), so confirmed URLs are listed as 6-tuples: (spec_code, paper_code, tier, session, role,
-# url). exam_code is DERIVED (see _mk_exam_code) so it always matches the locked convention.
-EXAM_CODE_PREFIX = {"EDEXCEL": "EDX", "OCR": "OCR"}
-
-def _ec_token(paper_code: str) -> str:
-    t = paper_code.split("/")[-1]
-    return str(int(t)) if t.isdigit() else t   # "01"->"1", "1H"->"1H", "1CH"->"1CH", "11"->"11"
-
-def _mk_exam_code(prefix: str, award: str, paper_code: str, session: str, role: str) -> str:
-    y, m = session.split("-")
-    return f"{prefix}-{award}-{_ec_token(paper_code)}-{y}{m.upper()}-{role}"
-
-_PE = "https://qualifications.pearson.com/content/dam/pdf"
-_EDX = f"{_PE}/GCSE/Science/2016"
-_OCR = "https://www.ocr.org.uk/Images"
-
-EDEXCEL_SPECS = {
-    "EDX-BIOL-1BI0": ("BIOLOGY", "1BI0", "GCSE", "2016"),
-    "EDX-CHEM-1CH0": ("CHEMISTRY", "1CH0", "GCSE", "2016"),
-    "EDX-PHYS-1PH0": ("PHYSICS", "1PH0", "GCSE", "2016"),
-    "EDX-COMB-1SC0": ("COMBINED SCIENCE", "1SC0", "GCSE", "2016"),
-    "EDX-MATH-1MA1": ("MATHEMATICS", "1MA1", "GCSE", "2015"),
-    "EDX-ENGL-1EN0": ("ENGLISH LANGUAGE", "1EN0", "GCSE", "2015"),
-    "EDX-ENGLIT-1ET0": ("ENGLISH LITERATURE", "1ET0", "GCSE", "2015"),
-    "EDX-GEOG-1GA0": ("GEOGRAPHY A", "1GA0", "GCSE", "2016"),
-    "EDX-HIST-1HI0": ("HISTORY", "1HI0", "GCSE", "2016"),
-    "EDX-BUS-1BS0": ("BUSINESS", "1BS0", "GCSE", "2017"),
-    "EDX-COMP-1CP2": ("COMPUTER SCIENCE", "1CP2", "GCSE", "2020"),
-    "EDX-MATH-9MA0": ("MATHEMATICS", "9MA0", "A-level", "2017"),
-    "EDX-ENGL-9EN0": ("ENGLISH LANGUAGE", "9EN0", "A-level", "2015"),
-    "EDX-ENGLIT-9ET0": ("ENGLISH LITERATURE", "9ET0", "A-level", "2015"),
-    "EDX-GEOG-9GE0": ("GEOGRAPHY", "9GE0", "A-level", "2016"),
-}
-EDEXCEL_PAPERS = [
-    # ── Sciences (round 1) ──
-    ("EDX-BIOL-1BI0", "1BI0/1H", "H", "2024-Jun", "QP", f"{_EDX}/Exam-materials/1bi0-1h-que-20240511.pdf"),
-    ("EDX-BIOL-1BI0", "1BI0/2F", "F", "2023-Jun", "QP", f"{_EDX}/Exam-materials/1bi0-2f-que-20230610.pdf"),
-    ("EDX-BIOL-1BI0", "1BI0/2H", "H", "2023-Jun", "QP", f"{_EDX}/Exam-materials/1bi0-2h-que-20230610.pdf"),
-    ("EDX-BIOL-1BI0", "1BI0/1F", "F", "2023-Jun", "MS", f"{_EDX}/Exam-materials/1bi0-1f-rms-20230824.pdf"),
-    ("EDX-BIOL-1BI0", "1BI0/1H", "H", "2024-Jun", "MS", f"{_EDX}/Exam-materials/1bi0-1h-rms-20240822.pdf"),
-    ("EDX-BIOL-1BI0", "1BI0/1H", "H", "2022-Jun", "MS", f"{_EDX}/exam-materials/1bi0-1h-rms-20220825.pdf"),
-    ("EDX-CHEM-1CH0", "1CH0/1F", "F", "2023-Jun", "QP", f"{_EDX}/Exam-materials/1ch0-1f-que-20230523.pdf"),
-    ("EDX-CHEM-1CH0", "1CH0/1H", "H", "2024-Jun", "QP", f"{_EDX}/Exam-materials/1ch0-1h-que-20240518.pdf"),
-    ("EDX-CHEM-1CH0", "1CH0/2H", "H", "2024-Jun", "MS", f"{_EDX}/Exam-materials/1ch0-2h-rms-20240822.pdf"),
-    ("EDX-PHYS-1PH0", "1PH0/1H", "H", "2023-Jun", "QP", f"{_EDX}/Exam-materials/1ph0-1h-que-20230526.pdf"),
-    ("EDX-PHYS-1PH0", "1PH0/2F", "F", "2023-Jun", "QP", f"{_EDX}/Exam-materials/1ph0-2f-que-20230617.pdf"),
-    ("EDX-PHYS-1PH0", "1PH0/1H", "H", "2024-Jun", "QP", f"{_EDX}/Exam-materials/1ph0-1h-que-20240523.pdf"),
-    ("EDX-PHYS-1PH0", "1PH0/2H", "H", "2023-Jun", "MS", f"{_EDX}/Exam-materials/1ph0-2h-rms-20230824.pdf"),
-    ("EDX-PHYS-1PH0", "1PH0/2H", "H", "2022-Jun", "MS", f"{_EDX}/exam-materials/1ph0-2h-rms-20220825.pdf"),
-    ("EDX-COMB-1SC0", "1SC0/1CH", None, "2023-Jun", "MS", f"{_EDX}/Exam-materials/1sc0-1ch-rms-20230824.pdf"),
-    # ── Maths 1MA1 (round 2) ──
-    ("EDX-MATH-1MA1", "1MA1/1H", "H", "2023-Jun", "QP", f"{_PE}/GCSE/Mathematics/2015/Exam-materials/1ma1-1h-que-20230520.pdf"),
-    ("EDX-MATH-1MA1", "1MA1/1H", "H", "2023-Jun", "MS", f"{_PE}/GCSE/Mathematics/2015/Exam-materials/1ma1-1h-rms-20230824.pdf"),
-    ("EDX-MATH-1MA1", "1MA1/1F", "F", "2023-Jun", "MS", f"{_PE}/GCSE/Mathematics/2015/Exam-materials/1ma1-1f-rms-20230824.pdf"),
-    ("EDX-MATH-1MA1", "1MA1/1F", "F", "2024-Jun", "QP", f"{_PE}/GCSE/Mathematics/2015/Exam-materials/1ma1-1f-que-20240517.pdf"),
-    ("EDX-MATH-1MA1", "1MA1/1H", "H", "2024-Jun", "QP", f"{_PE}/GCSE/Mathematics/2015/Exam-materials/1ma1-1h-que-20240517.pdf"),
-    ("EDX-MATH-1MA1", "1MA1/1F", "F", "2024-Jun", "MS", f"{_PE}/GCSE/Mathematics/2015/Exam-materials/1ma1-1f-rms-20240822.pdf"),
-    ("EDX-MATH-1MA1", "1MA1/1H", "H", "2023-Nov", "MS", f"{_PE}/GCSE/Mathematics/2015/Exam-materials/1ma1-1h-rms-20240111.pdf"),
-    ("EDX-MATH-1MA1", "1MA1/1H", "H", "2022-Jun", "MS", f"{_PE}/GCSE/mathematics/2015/exam-materials/1ma1-1h-rms-20220825.pdf"),
-    ("EDX-MATH-1MA1", "1MA1/3H", "H", "2022-Jun", "MS", f"{_PE}/GCSE/mathematics/2015/exam-materials/1ma1-3h-rms-20220825.pdf"),
-    # ── English Language 1EN0 / Literature 1ET0 (round 2) ──
-    ("EDX-ENGL-1EN0", "1EN0/01", None, "2024-Jun", "QP", f"{_PE}/GCSE/English-Language/2015/Exam-materials/1en0-01-que-20240524.pdf"),
-    ("EDX-ENGL-1EN0", "1EN0/01", None, "2023-Nov", "QP", f"{_PE}/GCSE/English-Language/2015/Exam-materials/1en0-01-que-20231108.pdf"),
-    ("EDX-ENGL-1EN0", "1EN0/01", None, "2024-Jun", "MS", f"{_PE}/GCSE/English-Language/2015/Exam-materials/1en0-01-rms-20240822.pdf"),
-    ("EDX-ENGL-1EN0", "1EN0/02", None, "2024-Jun", "MS", f"{_PE}/GCSE/English-Language/2015/Exam-materials/1en0-02-rms-20240822.pdf"),
-    ("EDX-ENGL-1EN0", "1EN0/01", None, "2023-Jun", "MS", f"{_PE}/GCSE/English-Language/2015/Exam-materials/1en0-01-rms-20230824.pdf"),
-    ("EDX-ENGL-1EN0", "1EN0/02", None, "2023-Jun", "MS", f"{_PE}/GCSE/English-Language/2015/Exam-materials/1en0-02-rms-20230824.pdf"),
-    ("EDX-ENGLIT-1ET0", "1ET0/01", None, "2023-Jun", "QP", f"{_PE}/GCSE/English-Literature/2015/Exam-materials/1et0-01-que-20230518.pdf"),
-    ("EDX-ENGLIT-1ET0", "1ET0/02", None, "2023-Jun", "QP", f"{_PE}/GCSE/English-Literature/2015/Exam-materials/1et0-02-que-20230525.pdf"),
-    ("EDX-ENGLIT-1ET0", "1ET0/02", None, "2024-Jun", "QP", f"{_PE}/GCSE/English-Literature/2015/Exam-materials/1et0-02-que-20240521.pdf"),
-    ("EDX-ENGLIT-1ET0", "1ET0/01", None, "2023-Jun", "MS", f"{_PE}/GCSE/English-Literature/2015/Exam-materials/1et0-01-rms-20230824.pdf"),
-    ("EDX-ENGLIT-1ET0", "1ET0/01", None, "2024-Jun", "MS", f"{_PE}/GCSE/English-Literature/2015/Exam-materials/1et0-01-rms-20240822.pdf"),
-    # ── A-level Maths 9MA0 / English 9EN0 / 9ET0 (round 2) ──
-    ("EDX-MATH-9MA0", "9MA0/01", None, "2023-Jun", "QP", f"{_PE}/A-Level/Mathematics/2017/Exam-materials/9ma0-01-que-20230607.pdf"),
-    ("EDX-MATH-9MA0", "9MA0/31", None, "2023-Jun", "QP", f"{_PE}/A-Level/Mathematics/2017/Exam-materials/9ma0-31-que-20230621.pdf"),
-    ("EDX-MATH-9MA0", "9MA0/02", None, "2024-Jun", "QP", f"{_PE}/A-Level/Mathematics/2017/Exam-materials/9ma0-02-que-20240612.pdf"),
-    ("EDX-MATH-9MA0", "9MA0/31", None, "2023-Jun", "MS", f"{_PE}/A-Level/Mathematics/2017/Exam-materials/9ma0-31-rms-20230817.pdf"),
-    ("EDX-MATH-9MA0", "9MA0/01", None, "2024-Jun", "MS", f"{_PE}/A-Level/Mathematics/2017/Exam-materials/9ma0-01-rms-20240815.pdf"),
-    ("EDX-ENGL-9EN0", "9EN0/01", None, "2024-Jun", "MS", f"{_PE}/A-Level/English-Language/2015/Exam-materials/9en0-01-rms-20240815.pdf"),
-    ("EDX-ENGL-9EN0", "9EN0/02", None, "2024-Jun", "MS", f"{_PE}/A-Level/English-Language/2015/Exam-materials/9en0-02-rms-20240815.pdf"),
-    ("EDX-ENGLIT-9ET0", "9ET0/01", None, "2024-Jun", "QP", f"{_PE}/A-Level/English-Literature/2015/Exam-materials/9et0-01-que-20240525.pdf"),
-    ("EDX-ENGLIT-9ET0", "9ET0/01", None, "2023-Jun", "MS", f"{_PE}/A-Level/English-Literature/2015/Exam-materials/9et0-01-rms-20230817.pdf"),
-    ("EDX-ENGLIT-9ET0", "9ET0/03", None, "2023-Jun", "MS", f"{_PE}/A-Level/English-Literature/2015/Exam-materials/9et0-03-rms-20230817.pdf"),
-    # ── Humanities (round 2) ──
-    ("EDX-GEOG-1GA0", "1GA0/01", None, "2023-Jun", "QP", f"{_PE}/GCSE/Geography-A/2016/Exam-materials/1ga0-01-que-20230523.pdf"),
-    ("EDX-GEOG-1GA0", "1GA0/01", None, "2023-Jun", "MS", f"{_PE}/GCSE/Geography-A/2016/Exam-materials/1ga0-01-rms-20230824.pdf"),
-    ("EDX-GEOG-1GA0", "1GA0/02", None, "2023-Jun", "QP", f"{_PE}/GCSE/Geography-A/2016/Exam-materials/1ga0-02-que-20230610.pdf"),
-    ("EDX-GEOG-1GA0", "1GA0/02", None, "2023-Jun", "MS", f"{_PE}/GCSE/Geography-A/2016/Exam-materials/1ga0-02-rms-20230824.pdf"),
-    ("EDX-GEOG-1GA0", "1GA0/01", None, "2024-Jun", "MS", f"{_PE}/GCSE/Geography-A/2016/Exam-materials/1ga0-01-rms-20240822.pdf"),
-    ("EDX-GEOG-1GA0", "1GA0/03", None, "2024-Jun", "QP", f"{_PE}/GCSE/Geography-A/2016/Exam-materials/1ga0-03-que-20240615.pdf"),
-    ("EDX-HIST-1HI0", "1HI0/10", None, "2023-Jun", "QP", f"{_PE}/GCSE/History/2016/Exam-materials/1hi0-10-que-20230519.pdf"),
-    ("EDX-HIST-1HI0", "1HI0/10", None, "2023-Jun", "MS", f"{_PE}/GCSE/History/2016/Exam-materials/1hi0-10-rms-20230824.pdf"),
-    ("EDX-HIST-1HI0", "1HI0/12", None, "2023-Jun", "MS", f"{_PE}/GCSE/History/2016/Exam-materials/1hi0-12-rms-20230824.pdf"),
-    ("EDX-HIST-1HI0", "1HI0/13", None, "2024-Jun", "MS", f"{_PE}/GCSE/History/2016/Exam-materials/1hi0-13-rms-20240822.pdf"),
-    ("EDX-HIST-1HI0", "1HI0/33", None, "2023-Jun", "MS", f"{_PE}/GCSE/History/2016/Exam-materials/1hi0-33-rms-20230824.pdf"),
-    ("EDX-BUS-1BS0", "1BS0/01", None, "2023-Jun", "QP", f"{_PE}/GCSE/Business/2017/Exam-materials/1bs0-01-que-20230519.pdf"),
-    ("EDX-BUS-1BS0", "1BS0/02", None, "2023-Jun", "QP", f"{_PE}/GCSE/Business/2017/Exam-materials/1bs0-02-que-20230613.pdf"),
-    ("EDX-BUS-1BS0", "1BS0/02", None, "2023-Jun", "MS", f"{_PE}/GCSE/Business/2017/Exam-materials/1bs0-02-rms-20230824.pdf"),
-    ("EDX-BUS-1BS0", "1BS0/02", None, "2024-Jun", "QP", f"{_PE}/GCSE/Business/2017/Exam-materials/1bs0-02-que-20240606.pdf"),
-    ("EDX-BUS-1BS0", "1BS0/01", None, "2024-Jun", "MS", f"{_PE}/GCSE/Business/2017/Exam-materials/1bs0-01-rms-20240822.pdf"),
-    ("EDX-COMP-1CP2", "1CP2/01", None, "2023-Jun", "QP", f"{_PE}/GCSE/Computer-science/2020/Exam-materials/1cp2-01-que-20230520.pdf"),
-    ("EDX-COMP-1CP2", "1CP2/01", None, "2023-Jun", "MS", f"{_PE}/GCSE/Computer-science/2020/Exam-materials/1cp2-01-rms-20230824.pdf"),
-    ("EDX-COMP-1CP2", "1CP2/02", None, "2023-Jun", "QP", f"{_PE}/GCSE/Computer-science/2020/Exam-materials/1cp2-02-que-20230526.pdf"),
-    ("EDX-COMP-1CP2", "1CP2/01", None, "2024-Jun", "QP", f"{_PE}/GCSE/Computer-Science/2020/Exam-materials/1cp2-01-que-20240702.pdf"),
-    ("EDX-COMP-1CP2", "1CP2/01", None, "2024-Jun", "MS", f"{_PE}/GCSE/Computer-science/2020/Exam-materials/1cp2-01-rms-20240822.pdf"),
-    ("EDX-GEOG-9GE0", "9GE0/01", None, "2023-Jun", "QP", f"{_PE}/A-Level/Geography/2016/Exam-materials/9ge0-01-que-20230518.pdf"),
-]
-
-OCR_SPECS = {
-    "OCR-BIOL-J247": ("BIOLOGY", "J247", "GCSE", "2016"),
-    "OCR-CHEM-J248": ("CHEMISTRY", "J248", "GCSE", "2016"),
-    "OCR-PHYS-J249": ("PHYSICS", "J249", "GCSE", "2016"),
-    "OCR-COMB-J250": ("COMBINED SCIENCE", "J250", "GCSE", "2016"),
-    "OCR-MATH-J560": ("MATHEMATICS", "J560", "GCSE", "2015"),
-    "OCR-ENGL-J351": ("ENGLISH LANGUAGE", "J351", "GCSE", "2015"),
-    "OCR-ENGLIT-J352": ("ENGLISH LITERATURE", "J352", "GCSE", "2015"),
-    "OCR-COMP-J277": ("COMPUTER SCIENCE", "J277", "GCSE", "2020"),
-    "OCR-GEOG-J383": ("GEOGRAPHY A", "J383", "GCSE", "2016"),
-    "OCR-BUS-J204": ("BUSINESS", "J204", "GCSE", "2017"),
-    "OCR-HIST-J411": ("HISTORY B (SHP)", "J411", "GCSE", "2016"),
-    "OCR-MATH-H240": ("MATHEMATICS A", "H240", "A-level", "2017"),
-    "OCR-ENGLIT-H472": ("ENGLISH LITERATURE", "H472", "A-level", "2015"),
-    "OCR-ENGL-H470": ("ENGLISH LANGUAGE", "H470", "A-level", "2015"),
-}
-OCR_PAPERS = [
-    # ── Sciences (round 1) ──
-    ("OCR-BIOL-J247", "J247/01", "F", "2024-Jun", "QP", f"{_OCR}/727713-question-paper-paper-1.pdf"),
-    ("OCR-BIOL-J247", "J247/01", "F", "2024-Jun", "MS", f"{_OCR}/727745-mark-scheme-paper-1.pdf"),
-    ("OCR-BIOL-J247", "J247/03", "H", "2024-Jun", "QP", f"{_OCR}/727715-question-paper-paper-3.pdf"),
-    ("OCR-BIOL-J247", "J247/03", "H", "2024-Jun", "MS", f"{_OCR}/727747-mark-scheme-paper-3.pdf"),
-    ("OCR-BIOL-J247", "J247/01", "F", "2023-Jun", "QP", f"{_OCR}/704945-question-paper-paper-1.pdf"),
-    ("OCR-BIOL-J247", "J247/03", "H", "2023-Jun", "MS", f"{_OCR}/704979-mark-scheme-paper-3.pdf"),
-    ("OCR-BIOL-J247", "J247/03", "H", "2022-Jun", "QP", f"{_OCR}/678031-question-paper-paper-3.pdf"),
-    ("OCR-BIOL-J247", "J247/01", "F", "2022-Jun", "MS", f"{_OCR}/678076-mark-scheme-paper-1.pdf"),
-    ("OCR-CHEM-J248", "J248/01", "F", "2024-Jun", "QP", f"{_OCR}/727718-question-paper-paper-1.pdf"),
-    ("OCR-CHEM-J248", "J248/03", "H", "2024-Jun", "MS", f"{_OCR}/727751-mark-scheme-paper-3.pdf"),
-    ("OCR-CHEM-J248", "J248/01", "F", "2023-Jun", "QP", f"{_OCR}/704950-question-paper-paper-1.pdf"),
-    ("OCR-CHEM-J248", "J248/03", "H", "2022-Jun", "QP", f"{_OCR}/678036-question-paper-paper-3.pdf"),
-    ("OCR-PHYS-J249", "J249/01", "F", "2024-Jun", "QP", f"{_OCR}/727724-question-paper-paper-1.pdf"),
-    ("OCR-PHYS-J249", "J249/03", "H", "2024-Jun", "MS", f"{_OCR}/727755-mark-scheme-paper-3.pdf"),
-    ("OCR-PHYS-J249", "J249/01", "F", "2023-Jun", "QP", f"{_OCR}/704956-question-paper-paper-1.pdf"),
-    ("OCR-PHYS-J249", "J249/03", "H", "2022-Jun", "MS", f"{_OCR}/678086-mark-scheme-paper-3.pdf"),
-    ("OCR-COMB-J250", "J250/01", "F", "2024-Jun", "QP", f"{_OCR}/727730-question-paper-paper-1.pdf"),
-    ("OCR-COMB-J250", "J250/07", "H", "2024-Jun", "MS", f"{_OCR}/727763-mark-scheme-paper-7.pdf"),
-    # ── Maths J560 (round 2) ──
-    ("OCR-MATH-J560", "J560/01", "F", "2024-Jun", "QP", f"{_OCR}/727817-question-paper-paper-1.pdf"),
-    ("OCR-MATH-J560", "J560/01", "F", "2024-Jun", "MS", f"{_OCR}/727824-mark-scheme-paper-1.pdf"),
-    ("OCR-MATH-J560", "J560/04", "H", "2024-Jun", "QP", f"{_OCR}/727820-question-paper-paper-4.pdf"),
-    ("OCR-MATH-J560", "J560/04", "H", "2024-Jun", "MS", f"{_OCR}/727827-mark-scheme-paper-4.pdf"),
-    ("OCR-MATH-J560", "J560/01", "F", "2023-Jun", "QP", f"{_OCR}/705050-question-paper-paper-1.pdf"),
-    ("OCR-MATH-J560", "J560/01", "F", "2023-Jun", "MS", f"{_OCR}/705057-mark-scheme-paper-1.pdf"),
-    ("OCR-MATH-J560", "J560/04", "H", "2023-Jun", "QP", f"{_OCR}/705053-question-paper-paper-4.pdf"),
-    ("OCR-MATH-J560", "J560/04", "H", "2023-Jun", "MS", f"{_OCR}/705060-mark-scheme-paper-4.pdf"),
-    ("OCR-MATH-J560", "J560/01", "F", "2022-Jun", "QP", f"{_OCR}/678149-question-paper-paper-1.pdf"),
-    ("OCR-MATH-J560", "J560/01", "F", "2022-Jun", "MS", f"{_OCR}/678156-mark-scheme-paper-1.pdf"),
-    ("OCR-MATH-J560", "J560/04", "H", "2022-Jun", "QP", f"{_OCR}/678152-question-paper-paper-4.pdf"),
-    ("OCR-MATH-J560", "J560/04", "H", "2022-Jun", "MS", f"{_OCR}/678159-mark-scheme-paper-4.pdf"),
-    # ── English Language J351 / Literature J352 (round 2) ──
-    ("OCR-ENGL-J351", "J351/01", None, "2024-Jun", "QP", f"{_OCR}/727556-question-paper-communicating-information-and-ideas.pdf"),
-    ("OCR-ENGL-J351", "J351/01", None, "2024-Jun", "MS", f"{_OCR}/727658-mark-scheme-communication-information-and-ideas.pdf"),
-    ("OCR-ENGL-J351", "J351/02", None, "2024-Jun", "QP", f"{_OCR}/727558-question-paper-exploring-effects-and-impact.pdf"),
-    ("OCR-ENGL-J351", "J351/02", None, "2024-Jun", "MS", f"{_OCR}/727659-mark-scheme-exploring-effects-and-impact.pdf"),
-    ("OCR-ENGL-J351", "J351/01", None, "2023-Jun", "QP", f"{_OCR}/704782-question-paper-communicating-information-and-ideas.pdf"),
-    ("OCR-ENGL-J351", "J351/01", None, "2023-Jun", "MS", f"{_OCR}/704888-mark-scheme-communication-information-and-ideas.pdf"),
-    ("OCR-ENGL-J351", "J351/01", None, "2022-Jun", "QP", f"{_OCR}/677852-question-paper-communicating-information-and-ideas.pdf"),
-    ("OCR-ENGL-J351", "J351/01", None, "2022-Jun", "MS", f"{_OCR}/677967-mark-scheme-communication-information-and-ideas.pdf"),
-    ("OCR-ENGLIT-J352", "J352/01", None, "2024-Jun", "QP", f"{_OCR}/727830-question-paper-exploring-modern-and-literary-heritage-texts.pdf"),
-    ("OCR-ENGLIT-J352", "J352/01", None, "2024-Jun", "MS", f"{_OCR}/727832-mark-scheme-exploring-modern-and-literary-heritage-texts.pdf"),
-    ("OCR-ENGLIT-J352", "J352/02", None, "2024-Jun", "QP", f"{_OCR}/727831-question-paper-exploring-poetry-and-shakespeare.pdf"),
-    ("OCR-ENGLIT-J352", "J352/02", None, "2024-Jun", "MS", f"{_OCR}/727833-mark-scheme-exploring-poetry-and-shakespeare.pdf"),
-    ("OCR-ENGLIT-J352", "J352/01", None, "2023-Jun", "QP", f"{_OCR}/705069-question-paper-exploring-modern-and-literary-heritage-texts.pdf"),
-    ("OCR-ENGLIT-J352", "J352/01", None, "2023-Jun", "MS", f"{_OCR}/705075-mark-scheme-exploring-modern-and-literary-heritage-texts.pdf"),
-    # ── A-level Maths H240 / English Lit H472 / Lang H470 (round 2) ──
-    ("OCR-MATH-H240", "H240/01", None, "2024-Jun", "QP", f"{_OCR}/726654-question-paper-pure-mathematics.pdf"),
-    ("OCR-MATH-H240", "H240/01", None, "2024-Jun", "MS", f"{_OCR}/726795-mark-scheme-pure-mathematics.pdf"),
-    ("OCR-MATH-H240", "H240/02", None, "2024-Jun", "QP", f"{_OCR}/726656-question-paper-pure-mathematics-and-statistics.pdf"),
-    ("OCR-MATH-H240", "H240/02", None, "2024-Jun", "MS", f"{_OCR}/726796-mark-scheme-pure-mathematics-and-statistics.pdf"),
-    ("OCR-MATH-H240", "H240/01", None, "2023-Jun", "QP", f"{_OCR}/703866-question-paper-pure-mathematics.pdf"),
-    ("OCR-MATH-H240", "H240/01", None, "2023-Jun", "MS", f"{_OCR}/704008-mark-scheme-pure-mathematics.pdf"),
-    ("OCR-MATH-H240", "H240/01", None, "2022-Jun", "QP", f"{_OCR}/676845-question-paper-pure-mathematics.pdf"),
-    ("OCR-MATH-H240", "H240/01", None, "2022-Jun", "MS", f"{_OCR}/677005-mark-scheme-pure-mathematics.pdf"),
-    ("OCR-ENGLIT-H472", "H472/01", None, "2024-Jun", "QP", f"{_OCR}/726602-question-paper-drama-and-poetry-pre-1900.pdf"),
-    ("OCR-ENGLIT-H472", "H472/01", None, "2024-Jun", "MS", f"{_OCR}/726762-mark-scheme-drama-and-poetry-pre-1900.pdf"),
-    ("OCR-ENGLIT-H472", "H472/01", None, "2023-Jun", "QP", f"{_OCR}/703813-question-paper-drama-and-poetry-pre-1900.pdf"),
-    ("OCR-ENGLIT-H472", "H472/01", None, "2023-Jun", "MS", f"{_OCR}/703974-mark-scheme-drama-and-poetry-pre-1900.pdf"),
-    ("OCR-ENGLIT-H472", "H472/01", None, "2022-Jun", "QP", f"{_OCR}/676783-question-paper-drama-and-poetry-pre-1900.pdf"),
-    ("OCR-ENGLIT-H472", "H472/01", None, "2022-Jun", "MS", f"{_OCR}/676965-mark-scheme-drama-and-poetry-pre-1900.pdf"),
-    ("OCR-ENGL-H470", "H470/01", None, "2024-Jun", "QP", f"{_OCR}/726595-question-paper-exploring-language.pdf"),
-    ("OCR-ENGL-H470", "H470/01", None, "2024-Jun", "MS", f"{_OCR}/726764-mark-scheme-exploring-language.pdf"),
-    ("OCR-ENGL-H470", "H470/01", None, "2023-Jun", "QP", f"{_OCR}/703806-question-paper-exploring-language.pdf"),
-    ("OCR-ENGL-H470", "H470/01", None, "2023-Jun", "MS", f"{_OCR}/703976-mark-scheme-exploring-language.pdf"),
-    ("OCR-ENGL-H470", "H470/01", None, "2022-Jun", "QP", f"{_OCR}/676772-question-paper-exploring-language.pdf"),
-    ("OCR-ENGL-H470", "H470/01", None, "2022-Jun", "MS", f"{_OCR}/676967-mark-scheme-exploring-language.pdf"),
-    # ── Humanities (round 2) ──
-    ("OCR-COMP-J277", "J277/01", None, "2024-Jun", "QP", f"{_OCR}/727534-question-paper-computer-systems.pdf"),
-    ("OCR-COMP-J277", "J277/01", None, "2024-Jun", "MS", f"{_OCR}/727652-mark-scheme-computer-systems.pdf"),
-    ("OCR-COMP-J277", "J277/02", None, "2024-Jun", "QP", f"{_OCR}/727535-question-paper-computational-thinking-algorithms-and-programming.pdf"),
-    ("OCR-COMP-J277", "J277/02", None, "2024-Jun", "MS", f"{_OCR}/727653-mark-scheme-computational-thinking-algorithms-and-programming.pdf"),
-    ("OCR-GEOG-J383", "J383/01", None, "2024-Jun", "QP", f"{_OCR}/727564-question-paper-living-in-the-uk-today.pdf"),
-    ("OCR-GEOG-J383", "J383/01", None, "2024-Jun", "MS", f"{_OCR}/727661-mark-scheme-living-in-the-uk-today.pdf"),
-    ("OCR-GEOG-J383", "J383/02", None, "2024-Jun", "QP", f"{_OCR}/727566-question-paper-the-world-around-us.pdf"),
-    ("OCR-GEOG-J383", "J383/02", None, "2024-Jun", "MS", f"{_OCR}/727662-mark-scheme-the-world-around-us.pdf"),
-    ("OCR-BUS-J204", "J204/01", None, "2024-Jun", "QP", f"{_OCR}/727519-question-paper-business-1-business-activity-marketing-and-people.pdf"),
-    ("OCR-BUS-J204", "J204/01", None, "2024-Jun", "MS", f"{_OCR}/727634-mark-scheme-business-1-business-activity-marketing-and-people.pdf"),
-    ("OCR-BUS-J204", "J204/02", None, "2024-Jun", "QP", f"{_OCR}/727520-question-paper-business-2-operations-finance-and-influences-on-business.pdf"),
-    ("OCR-BUS-J204", "J204/02", None, "2024-Jun", "MS", f"{_OCR}/727635-mark-scheme-business-2-operations-finance-and-influences-on-business.pdf"),
-    ("OCR-BUS-J204", "J204/01", None, "2023-Jun", "QP", f"{_OCR}/704745-question-paper-business-1-business-activity-marketing-and-people.pdf"),
-    ("OCR-BUS-J204", "J204/01", None, "2023-Jun", "MS", f"{_OCR}/704864-mark-scheme-business-1-business-activity-marketing-and-people.pdf"),
-    ("OCR-HIST-J411", "J411/11", None, "2024-Jun", "QP", f"{_OCR}/727590-question-paper-the-people-s-health-c.1250-to-present-with-the-norman-conquest-1065-1087.pdf"),
-    ("OCR-HIST-J411", "J411/11", None, "2024-Jun", "MS", f"{_OCR}/727678-mark-scheme-the-people-s-health-c.1250-to-present-with-the-norman-conquest-1065-1087.pdf"),
-]
-
-
-def build_board(board_code: str, specs_meta: Dict, papers: List) -> Dict[str, Any]:
-    prefix = EXAM_CODE_PREFIX[board_code]
-    print(f"[{board_code}] re-verifying {len(papers)} confirmed URLs...", file=sys.stderr)
-    live: Dict[int, bool] = {}
-    with cf.ThreadPoolExecutor(max_workers=24) as ex:
-        futs = {ex.submit(head_ok, p[5]): i for i, p in enumerate(papers)}
-        for fut in cf.as_completed(futs):
-            live[futs[fut]] = fut.result()
-    by_spec: Dict[str, List[Dict[str, Any]]] = {}
-    for i, (spec_code, paper_code, tier, session, role, url) in enumerate(papers):
-        if not live.get(i):
-            print(f"  DROP (not live): {url}", file=sys.stderr)
-            continue
-        award = specs_meta[spec_code][1]
-        by_spec.setdefault(spec_code, []).append({
-            "exam_code": _mk_exam_code(prefix, award, paper_code, session, role),
-            "paper_code": paper_code, "tier": tier,
-            "session": session, "doc_type": role,
-            "file": {"source": f"url:{url}", "original_name": os.path.basename(url),
-                     "provenance": {"source_url": url, "fetched": FETCHED,
-                                    "license": f"{board_code} public past paper"}},
-        })
-    spec_list = []
-    for spec_code, (subject, award, level, first_teach) in specs_meta.items():
-        if spec_code not in by_spec:
-            continue
-        spec_list.append({
-            "spec_code": spec_code, "exam_board_code": board_code, "subject_code": subject,
-            "award_code": award, "award_level": level, "first_teach": first_teach,
-            "papers": sorted(by_spec[spec_code], key=lambda p: p["exam_code"]),
-        })
-        print(f"[{board_code}] {spec_code}: {len(by_spec[spec_code])} live papers", file=sys.stderr)
-    return {"exam_board_code": board_code, "specifications": spec_list}
-
-
-def main() -> None:
-    out_path = os.path.join(os.path.dirname(__file__), "exam-corpus.yaml")
-    boards = [
-        build_aqa(),
-        build_board("EDEXCEL", EDEXCEL_SPECS, EDEXCEL_PAPERS),
-        build_board("OCR", OCR_SPECS, OCR_PAPERS),
-    ]
-    n_specs = sum(len(b["specifications"]) for b in boards)
-    n_papers = sum(len(s["papers"]) for b in boards for s in b["specifications"])
-    manifest = {
-        "version": 1,
-        "defaults": {"bucket": "cc.examboards"},
-        "provenance": {
-            "collected_by": "kcar",
-            "collected_at": FETCHED,
-            "license_posture": ("Public exam-board past papers downloaded from each board's own "
-                                "official site (AQA filestore, Pearson DAM, OCR Images). Stored in "
-                                "the private dev cc.examboards bucket for internal exam-marker dev/test. "
-                                "Each item records its source_url. Review redistribution rights before "
-                                "any public exposure."),
-            "sources": {
-                "AQA": "https://filestore.aqa.org.uk/sample-papers-and-mark-schemes/",
-                "EDEXCEL": "https://qualifications.pearson.com/en/support/support-topics/exams/past-papers.html",
-                "OCR": "https://www.ocr.org.uk/qualifications/past-paper-finder/",
-            },
-        },
-        # Optional: uncomment + set on dev .94 to exercise user-side flows / first-sweep.
-        # "test_subset": {"user_email": "teacher@kevlarai.test", "papers": 2},
-        # "system_identity": {"user_email": "teacher@kevlarai.test"},
-        "boards": boards,
-    }
-    with open(out_path, "w") as fh:
-        yaml.safe_dump(manifest, fh, sort_keys=False, default_flow_style=False, width=120)
-    print(f"\nWROTE {out_path}: {n_specs} specs, {n_papers} papers across {len(boards)} boards",
-          file=sys.stderr)
-
-
-if __name__ == "__main__":
-    main()

run/initialization/reset_environment.py

@@ -5,7 +5,6 @@ Clears:
   - Neo4j: drops ALL databases except system, neo4j (including gaisdata, cc.users.*, cc.institutes.*)
   - Supabase: deletes ALL data tables except gais_local_authorities and gais_schools
   - Supabase: deletes all auth users except kcar, then re-seeds kcar profile state
-  - Granular scopes can clear exam corpus, timetable data, or --user-subset seed copies
 
 Safe invariants (never touched):
   - kcar auth account
@@ -83,45 +82,6 @@ SUPABASE_TABLES_TO_CLEAR = [
     "admin_profiles",
 ]
 
-# Exam-marker subsystem tables, FK child-first. scope="exam-corpus" is deliberately
-# broader than "public papers": it wipes public corpus eb_* rows, templates, layouts,
-# questions, boundaries, response areas, marking batches, student submissions, and mark
-# entries. NOT in the list above — the previous full reset() never cleared exam data
-# or storage at all; the granular scopes below fold it in.
-EXAM_CORPUS_TABLES = [
-    "mark_entries",
-    "student_submissions",
-    "marking_batches",
-    "exam_response_areas",
-    "exam_boundaries",
-    "exam_template_layout",
-    "exam_questions",
-    "exam_templates",
-    "eb_exams",
-    "eb_specifications",
-]
-
-# Timetable / calendar materialization subset (for scope='timetable').
-TIMETABLE_TABLES = [
-    "lesson_deliveries",
-    "lesson_collaborators",
-    "taught_lessons",
-    "academic_periods",
-    "academic_days",
-    "academic_weeks",
-    "academic_term_breaks",
-    "academic_terms",
-    "academic_years",
-    "teacher_timetable_slots",
-    "teacher_timetables",
-    "school_timetables",
-    "planned_lessons",
-]
-
-# Bucket whose objects scope="exam-corpus" clears for the whole exam-marker subsystem
-# (Storage API — protect_delete blocks raw SQL).
-EXAM_STORAGE_BUCKET = "cc.examboards"
-
 
 def _sb_headers():
     url = os.environ["SUPABASE_URL"]
@@ -134,28 +94,6 @@ def _sb_headers():
     }
 
 
-# Markers that identify a production Supabase target. Destructive reset against any of these is
-# refused by default (project rule: ".94 only; .156 human-gated") — set RESET_ALLOW_PROD=1 to override.
-PROD_TARGET_MARKERS = ("192.168.0.156", "supabase.classroomcopilot")
-
-
-def _assert_reset_allowed(url: str, scope: str) -> None:
-    """Default-deny destructive reset against a production-looking Supabase target.
-
-    The /admin/reset route and this module both act on os.environ['SUPABASE_URL']; without this guard
-    a platform-admin call on a prod-deployed API would wipe prod data + exam corpus + storage. We refuse
-    when the target matches a known prod marker unless an explicit RESET_ALLOW_PROD opt-in is set.
-    """
-    target = (url or "").lower()
-    looks_prod = any(m in target for m in PROD_TARGET_MARKERS)
-    override = os.environ.get("RESET_ALLOW_PROD", "").strip().lower() in ("1", "true", "yes")
-    if looks_prod and not override:
-        raise RuntimeError(
-            f"refusing destructive reset (scope={scope}) against production-looking target {target!r}; "
-            f"this is human-gated — set RESET_ALLOW_PROD=1 to override."
-        )
-
-
 # ─── Neo4j helpers ────────────────────────────────────────────────────────────
 
 def _neo4j_drop_all_non_system() -> Dict[str, List[str]]:
@@ -208,133 +146,13 @@ def _supabase_delete_auth_user(url: str, headers: dict, uid: str):
         logger.warning(f"  Delete auth user {uid}: {r.status_code} {r.text[:80]}")
 
 
-# ─── Granular helpers ───────────────────────────────────────────────────────────
-
-def _clear_tables(url: str, headers: dict, tables: List[str]) -> "tuple[List[str], List[str]]":
-    cleared, failed = [], []
-    for table in tables:
-        if _sb_clear_table(url, headers, table) in (200, 204):
-            cleared.append(table)
-            logger.info(f"  ✓ {table}")
-        else:
-            failed.append(table)
-    return cleared, failed
-
-
-def _clear_exam_storage() -> Dict[str, Any]:
-    """Remove cc.examboards objects for the exam-marker subsystem.
-
-    scope="exam-corpus" is not limited to public-paper metadata: it also removes the
-    storage objects that back exam board corpus files and any downstream exam-marker
-    artifacts referenced from eb_exams/eb_specifications. Gathers storage_loc from
-    eb_exams/eb_specifications BEFORE the rows are cleared.
-    """
-    try:
-        from modules.database.supabase.utils.client import SupabaseServiceRoleClient
-        from modules.database.supabase.utils.storage import StorageAdmin
-    except Exception as exc:
-        logger.warning(f"  exam storage clear skipped (import): {exc}")
-        return {"removed": 0, "error": str(exc)}
-    sb = SupabaseServiceRoleClient().supabase
-    storage = StorageAdmin()
-    locs: List[str] = []
-    for table in ("eb_exams", "eb_specifications"):
-        try:
-            rows = sb.table(table).select("storage_loc").execute().data or []
-            locs += [r["storage_loc"] for r in rows if r.get("storage_loc")]
-        except Exception as exc:
-            logger.warning(f"  storage_loc gather {table}: {exc}")
-    by_bucket: Dict[str, List[str]] = {}
-    for loc in locs:
-        if "/" in loc:
-            b, _, p = loc.partition("/")
-            by_bucket.setdefault(b, []).append(p)
-    removed = 0
-    for b, paths in by_bucket.items():
-        for i in range(0, len(paths), 100):
-            chunk = paths[i:i + 100]
-            try:
-                storage.client.supabase.storage.from_(b).remove(chunk)
-                removed += len(chunk)
-            except Exception as exc:
-                logger.warning(f"  storage remove {b}: {exc}")
-    logger.info(f"  exam storage removed {removed} objects from {list(by_bucket)}")
-    return {"removed": removed, "buckets": list(by_bucket)}
-
-
-def _clear_user_subset_files() -> Dict[str, Any]:
-    """Remove files rows and cc.users storage objects created by --user-subset seeding.
-
-    Reuses the seed/unseed implementation so reset(scope="user-subset") has the
-    same storage-before-row deletion order and idempotency guarantees as
-    seed_exam_corpus.py --unseed. The helper only targets rows marked by the seeder:
-    bucket='cc.users', source='exam-corpus-seed', path LIKE 'exam-marker/%'.
-    """
-    try:
-        from modules.database.supabase.utils.client import SupabaseServiceRoleClient
-        from modules.database.supabase.utils.storage import StorageAdmin
-        from run.initialization.seed_exam_corpus import LoadReport, _delete_user_subset_files
-    except Exception as exc:
-        logger.warning(f"  user-subset clear skipped (import): {exc}")
-        return {"files_rows_deleted": 0, "storage_objects_removed": 0, "errors": [str(exc)]}
-
-    rep = LoadReport()
-    _delete_user_subset_files(
-        SupabaseServiceRoleClient(),
-        StorageAdmin(),
-        exam_codes=None,
-        rep=rep,
-    )
-    return {
-        "files_rows_deleted": rep.unseed_user_files,
-        "storage_objects_removed": rep.unseed_objects,
-        "errors": rep.errors,
-    }
-
-
 # ─── Main reset ───────────────────────────────────────────────────────────────
 
-def reset(scope: str = "all") -> Dict[str, Any]:
+def reset() -> Dict[str, Any]:
-    """Destructive reset. scope ∈ {all, exam-corpus, timetable, user-subset}.
-
-    - all          : full wipe (Neo4j + Supabase data + auth users) AND the entire
-                     exam-marker subsystem listed below, including --user-subset copies.
-    - exam-corpus  : ONLY the entire exam-marker subsystem, not just public papers:
-                     public corpus/eb_* data, cc.examboards storage objects, exam
-                     templates, template layouts, questions, boundaries, response
-                     areas, marking batches, student submissions, mark entries, and
-                     --user-subset cc.users copies.
-    - timetable    : ONLY timetable/calendar materialization tables.
-    - user-subset  : ONLY files rows and cc.users storage objects created by
-                     seed_exam_corpus.py --user-subset.
-    """
-    scope = (scope or "all").lower()
-    if scope not in ("all", "exam-corpus", "timetable", "user-subset"):
-        raise ValueError(f"invalid scope {scope!r} (want all|exam-corpus|timetable|user-subset)")
-    url, headers = _sb_headers()
-    _assert_reset_allowed(url, scope)
-
-    if scope == "exam-corpus":
-        logger.info("RESET (scope=exam-corpus) — entire exam-marker subsystem: public corpus/eb_* data, cc.examboards storage, templates/layout/questions/boundaries/response areas, marking batches, submissions, mark entries, and --user-subset copies")
-        user_subset = _clear_user_subset_files()
-        storage = _clear_exam_storage()
-        cleared, failed = _clear_tables(url, headers, EXAM_CORPUS_TABLES)
-        return {"scope": scope, "user_subset": user_subset, "exam_storage": storage, "tables_cleared": cleared, "tables_failed": failed}
-
-    if scope == "timetable":
-        logger.info("RESET (scope=timetable) — timetable/calendar tables")
-        cleared, failed = _clear_tables(url, headers, TIMETABLE_TABLES)
-        return {"scope": scope, "tables_cleared": cleared, "tables_failed": failed}
-
-    if scope == "user-subset":
-        logger.info("RESET (scope=user-subset) — --user-subset cc.users storage objects and files rows")
-        user_subset = _clear_user_subset_files()
-        return {"scope": scope, "user_subset": user_subset}
-
     logger.info("=" * 60)
     logger.info("RESET ENVIRONMENT — full destructive wipe starting")
     logger.info("=" * 60)
-    results: Dict[str, Any] = {"scope": scope}
+    results: Dict[str, Any] = {}
 
     # ── 1. Neo4j: drop everything except system + neo4j ──────────────────────
     logger.info("\n[Neo4j] Dropping all non-system databases...")
@@ -343,9 +161,6 @@ def reset(scope: str = "all") -> Dict[str, Any]:
     results["neo4j"] = {"dropped": dropped}
 
     # ── 2. Supabase: clear all data tables (GAIS preserved) ──────────────────
-    # First remove --user-subset cc.users storage objects (+ their files rows) via the
-    # Storage API, so the generic files-table clear below doesn't strand orphaned objects.
-    results["user_subset"] = _clear_user_subset_files()
     logger.info("\n[Supabase] Clearing data tables (preserving gais_*)...")
     url, headers = _sb_headers()
     cleared, failed = [], []
@@ -398,25 +213,11 @@ def reset(scope: str = "all") -> Dict[str, Any]:
     )
     logger.info("  kcar → admin_profiles restored ✓")
 
-    # ── 5. Exam-marker subsystem: storage objects (Storage API) + all exam tables ──
-    # This is the same destructive surface as scope="exam-corpus": public corpus/eb_*
-    # rows, cc.examboards storage, templates/layout/questions/boundaries/response
-    # areas, marking batches, submissions, and mark entries. (The legacy full reset
-    # cleared neither exam tables nor storage — folded in here.)
-    logger.info("\n[Supabase] Clearing entire exam-marker subsystem (public corpus, storage, templates/layout/questions/boundaries/response areas, marking batches, submissions, mark entries)...")
-    exam_storage = _clear_exam_storage()
-    exam_cleared, exam_failed = _clear_tables(url, headers, EXAM_CORPUS_TABLES)
-
     results["supabase"] = {
         "tables_cleared": cleared,
         "tables_failed": failed,
         "deleted_users": deleted_emails,
     }
-    results["exam"] = {
-        "storage": exam_storage,
-        "tables_cleared": exam_cleared,
-        "tables_failed": exam_failed,
-    }
 
     logger.info("\n" + "=" * 60)
     logger.info("RESET COMPLETE")

run/initialization/seed_curriculum.py

@@ -1,20 +1,15 @@
 """
-seed_curriculum.py — DEPRECATED hardcoded curriculum/exam seeder.
+seed_curriculum.py — Create curriculum data: exam board specifications and exams.
 
-⚠️ SUPERSEDED (2026-06-07) by the manifest-driven corpus loader:
+Seeds eb_specifications and eb_exams tables with realistic UK exam board data
-    run/initialization/seed_exam_corpus.py  (+ manifests/exam-corpus.yaml)
+(AQA, Edexcel, OCR) for Physics, Maths, and Computer Science across both schools.
 
-The exam-board parts of this file (eb_specifications / eb_exams) are now seeded from a
+Also seeds curriculum_topics in Neo4j for the school databases.
-verified, provenance-bearing manifest with real uploaded PDFs — not the hardcoded rows
-below. This module also had a storage_loc inconsistency the overhaul standardises away:
-exam-board files belong in the `cc.examboards` bucket at the canonical path
-`cc.examboards/{board}/{subject}/{award}/{paper}/{session}/{role}.pdf`, NOT under
-`cc.public.snapshots/curriculum/...` (the placeholder rows below still show the old path).
 
-KEEP ONLY for the Neo4j `curriculum_topics` seed (step [3]) which has no replacement yet.
+Tables: eb_specifications, eb_exams
-Do NOT use the eb_specifications/eb_exams blocks for new work — use seed_exam_corpus.py.
+Neo4j: curriculum topic nodes in school databases
 
-Run (Neo4j curriculum topics only is the supported remaining use):
+Run inside ccapi container:
     python3 -c "from run.initialization.seed_curriculum import seed; seed()"
 """
 import os

run/initialization/seed_exam_corpus.py

@@ -1,867 +0,0 @@
-"""
-seed_exam_corpus.py — manifest-driven loader for the public exam-paper corpus.
-
-SCOPE (separate from infra): assumes storage buckets already exist (provisioned by
-run/initialization/buckets.py during infra init). This loader UPLOADS papers and
-SEEDS the catalogue; it does NOT create buckets.
-
-Pipeline per manifest item:
-  validate -> resolve source bytes (local path | url:, cached) -> upload file to
-  cc.examboards (canonical path, skip-if-exists unless --force) -> upsert
-  eb_specifications / eb_exams (catalogue) -> (optional, --user-subset) copy a subset
-  into a test user's exam space so user-side flows are testable -> (optional,
-  --first-sweep) run the docling/auto-map first pass to gather structure.
-
-Manifest template: ~/cc/specs/exam-corpus-manifest.example.yaml
-
-Catalogue columns (real — verified against volumes/db/cc/61-core-schema.sql):
-  eb_specifications(spec_code UNIQUE, exam_board_code, award_code, subject_code,
-                    first_teach, spec_ver, storage_loc, doc_type CHECK(pdf|json|...),
-                    doc_details jsonb, docling_docs jsonb)
-  eb_exams(exam_code UNIQUE, spec_code FK, paper_code, tier, session, type_code,
-           storage_loc, doc_type CHECK(pdf|json|...), doc_details jsonb, docling_docs jsonb)
-
-IMPORTANT schema note: the QP/MS/INSERT/ER *document role* is stored in `type_code`
-(the `/catalogue` endpoint filters `type_code == 'QP'`). The `doc_type` column is the
-*file format* and is CHECK-constrained to {pdf,json,md,html,txt,doctags} — so it is
-always 'pdf' here. (The manifest field is named `doc_type` for the role; the loader
-maps manifest.doc_type -> DB.type_code and sets DB.doc_type = 'pdf'.)
-
-Locked conventions (see ~/cc/ideas/2026-06-07-exam-paper-ingestion.md):
-  session    = "YYYY-Mon"  e.g. "2022-Jun", "2021-Nov"
-  exam_code  = "{BOARD}-{award}-{paper_safe}-{SESSIONCOMPACT}-{ROLE}"  e.g. AQA-8463-1H-2022JUN-QP
-  spec path  = cc.examboards/{board}/{subject}/{award}/spec/{spec_ver}.pdf
-  paper path = cc.examboards/{board}/{subject}/{award}/{paper_safe}/{session}/{role}.pdf
-
-Run inside the api container (env: SUPABASE_URL + SERVICE_ROLE_KEY for dev .94), e.g.:
-  python3 -m run.initialization.seed_exam_corpus --manifest /path/exam-corpus.yaml --dry-run
-  python3 -m run.initialization.seed_exam_corpus --manifest ... --board AQA
-  python3 -m run.initialization.seed_exam_corpus --manifest ... --first-sweep
-"""
-from __future__ import annotations
-import argparse
-import hashlib
-import os
-from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional, Tuple
-
-import requests
-import yaml  # PyYAML
-
-from modules.logger_tool import initialise_logger
-from modules.database.supabase.utils.client import SupabaseServiceRoleClient
-from modules.database.supabase.utils.storage import StorageAdmin, StorageError
-
-logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), "default", True)
-
-EXAM_BUCKET = "cc.examboards"
-# Manifest `doc_type` carries the document ROLE (stored in eb_exams.type_code).
-DOC_ROLES = {"QP", "MS", "INSERT", "ER", "SPECIMEN", "GRADE_BOUNDARIES", "DATA_SHEET"}
-TIERS = {"H", "F", None}
-# Default working dir for cached url: downloads (override with --cache-dir / EXAM_CORPUS_CACHE).
-DEFAULT_CACHE_DIR = os.getenv("EXAM_CORPUS_CACHE", "/tmp/exam-corpus-cache")
-# Persistent, mountable local store laid out exactly like the bucket (download once, seed many,
-# offline-repeatable). Override with --store-dir / EXAM_CORPUS_STORE. Distinct from --cache-dir,
-# which is a throwaway url hash-cache.
-DEFAULT_STORE_DIR = os.getenv(
-    "EXAM_CORPUS_STORE",
-    os.path.join(os.path.dirname(os.path.abspath(__file__)), "manifests", "_corpus_store"),
-)
-
-
-# ─────────────────────────────── canonical storage paths ───────────────────────────────
-def _lc(s: str) -> str:
-    return (s or "").strip().lower().replace(" ", "-")
-
-def _paper_safe(paper_code: str) -> str:
-    # Drop the award prefix, keep all remaining segments so combined-science sub-papers
-    # don't collide on the storage path:
-    #   "8463/1H"    -> "1h"
-    #   "8464/B/1H"  -> "b-1h"   (Trilogy: subject letter + paper + tier)
-    #   "7408/1"     -> "1"
-    parts = _lc(paper_code).split("/")
-    return "-".join(parts[1:]) if len(parts) > 1 else parts[0]
-
-def spec_storage_loc(board: str, subject: str, award: str, spec_ver: str) -> str:
-    # e.g. cc.examboards/aqa/physics/8463/spec/1.1.pdf
-    return f"{EXAM_BUCKET}/{_lc(board)}/{_lc(subject)}/{_lc(award)}/spec/{_lc(spec_ver or 'spec')}.pdf"
-
-def paper_storage_loc(board: str, subject: str, award: str, paper_code: str, session: str, doc_role: str) -> str:
-    # e.g. cc.examboards/aqa/physics/8463/1h/2022-jun/qp.pdf
-    return f"{EXAM_BUCKET}/{_lc(board)}/{_lc(subject)}/{_lc(award)}/{_paper_safe(paper_code)}/{_lc(session)}/{_lc(doc_role)}.pdf"
-
-
-# ─────────────────────────────── report ───────────────────────────────
-@dataclass
-class LoadReport:
-    specs_upserted: int = 0
-    papers_upserted: int = 0
-    files_uploaded: int = 0
-    files_skipped: int = 0
-    files_failed: int = 0
-    user_copies: int = 0
-    swept: int = 0
-    sweep_failed: int = 0
-    downloaded: int = 0
-    download_cached: int = 0
-    unseed_objects: int = 0
-    unseed_user_files: int = 0
-    unseed_exams: int = 0
-    unseed_specs: int = 0
-    unseed_templates: int = 0
-    errors: List[str] = field(default_factory=list)
-
-    def as_dict(self) -> Dict[str, Any]:
-        return {
-            "specs_upserted": self.specs_upserted,
-            "papers_upserted": self.papers_upserted,
-            "downloaded": self.downloaded,
-            "download_cached": self.download_cached,
-            "unseed_objects": self.unseed_objects,
-            "unseed_user_files": self.unseed_user_files,
-            "unseed_exams": self.unseed_exams,
-            "unseed_specs": self.unseed_specs,
-            "unseed_templates": self.unseed_templates,
-            "files_uploaded": self.files_uploaded,
-            "files_skipped": self.files_skipped,
-            "files_failed": self.files_failed,
-            "user_copies": self.user_copies,
-            "swept": self.swept,
-            "sweep_failed": self.sweep_failed,
-            "errors": self.errors,
-        }
-
-
-# ─────────────────────────────── validation ───────────────────────────────
-def validate_manifest(m: Dict[str, Any]) -> List[str]:
-    errs: List[str] = []
-    seen_specs, seen_exams = set(), set()
-    for board in m.get("boards", []):
-        bcode = board.get("exam_board_code")
-        if not bcode:
-            errs.append("board missing exam_board_code")
-        for spec in board.get("specifications", []):
-            sc = spec.get("spec_code")
-            if not sc or sc in seen_specs:
-                errs.append(f"spec_code missing/duplicate: {sc!r}")
-            seen_specs.add(sc)
-            for field_name in ("award_code", "subject_code"):
-                if not spec.get(field_name):
-                    errs.append(f"{sc}: missing {field_name}")
-            for p in spec.get("papers", []):
-                ec = p.get("exam_code")
-                if not ec or ec in seen_exams:
-                    errs.append(f"exam_code missing/duplicate: {ec!r}")
-                seen_exams.add(ec)
-                if p.get("doc_type") not in DOC_ROLES:
-                    errs.append(f"{ec}: bad doc_type/role {p.get('doc_type')!r} (want one of {sorted(DOC_ROLES)})")
-                if p.get("tier") not in TIERS:
-                    errs.append(f"{ec}: bad tier {p.get('tier')!r} (want H|F|null)")
-                if not p.get("paper_code"):
-                    errs.append(f"{ec}: missing paper_code")
-                if not p.get("session"):
-                    errs.append(f"{ec}: missing session")
-                src = (p.get("file") or {}).get("source")
-                if not src:
-                    errs.append(f"{ec}: missing file.source")
-                elif not src.startswith("url:") and not os.path.exists(src):
-                    errs.append(f"{ec}: local source not found: {src}")
-    return errs
-
-
-# ─────────────────────────────── source resolution (local | url:, cached) ───────────────────────────────
-def _resolve_source_bytes(source: str, *, cache_dir: str) -> bytes:
-    """Resolve a manifest file source to bytes.
-
-    'url:https://...'  -> fetch (cached to cache_dir by url hash) ; verifies non-empty.
-    '<local path>'     -> read from disk.
-    """
-    if source.startswith("url:"):
-        url = source[len("url:"):]
-        os.makedirs(cache_dir, exist_ok=True)
-        cache_key = hashlib.sha1(url.encode("utf-8")).hexdigest()
-        cache_path = os.path.join(cache_dir, f"{cache_key}.pdf")
-        if os.path.exists(cache_path) and os.path.getsize(cache_path) > 0:
-            with open(cache_path, "rb") as fh:
-                return fh.read()
-        logger.info(f"[fetch] {url}")
-        resp = requests.get(url, timeout=60, allow_redirects=True)
-        resp.raise_for_status()
-        data = resp.content
-        ctype = resp.headers.get("content-type", "")
-        if not data:
-            raise ValueError(f"empty download: {url}")
-        if "pdf" not in ctype.lower() and not data[:5].startswith(b"%PDF"):
-            raise ValueError(f"not a PDF (content-type={ctype!r}): {url}")
-        tmp = cache_path + ".part"
-        with open(tmp, "wb") as fh:
-            fh.write(data)
-        os.replace(tmp, cache_path)
-        return data
-    with open(source, "rb") as fh:
-        return fh.read()
-
-
-# ─────────────────────── persistent local store (download-once, seed-many) ───────────────────────
-def _store_path(store_dir: str, storage_loc: str) -> str:
-    """Local path mirroring the bucket layout (so the store is directly mountable as the corpus):
-    storage_loc 'cc.examboards/aqa/physics/8463/1h/2022-jun/qp.pdf'
-      -> {store_dir}/aqa/physics/8463/1h/2022-jun/qp.pdf
-    """
-    _, _, path = storage_loc.partition("/")
-    return os.path.join(store_dir, path)
-
-def _item_bytes(source: str, storage_loc: str, *, store_dir: Optional[str], cache_dir: str,
-                populate: bool = True, rep: Optional[LoadReport] = None) -> bytes:
-    """Resolve bytes for an item, preferring the persistent local store when present.
-
-    If store_dir holds the file → read it (offline). Otherwise resolve the source (local|url:) and,
-    when populate=True, write it into the store at its canonical path for future offline runs.
-    """
-    if store_dir:
-        sp = _store_path(store_dir, storage_loc)
-        if os.path.exists(sp) and os.path.getsize(sp) > 0:
-            if rep is not None:
-                rep.download_cached += 1
-            with open(sp, "rb") as fh:
-                return fh.read()
-    data = _resolve_source_bytes(source, cache_dir=cache_dir)
-    if store_dir and populate:
-        sp = _store_path(store_dir, storage_loc)
-        os.makedirs(os.path.dirname(sp), exist_ok=True)
-        tmp = sp + ".part"
-        with open(tmp, "wb") as fh:
-            fh.write(data)
-        os.replace(tmp, sp)
-        if rep is not None:
-            rep.downloaded += 1
-    return data
-
-def download_corpus(m: Dict[str, Any], *, store_dir: str, board_filter: Optional[str],
-                    spec_filter: Optional[str], cache_dir: str, rep: LoadReport) -> None:
-    """--download-only: populate the persistent local store from the manifest. No DB/bucket writes.
-    A later run with the same --store-dir (e.g. mounted into the container) seeds offline from it."""
-    for board in m.get("boards", []):
-        if board_filter and board.get("exam_board_code") != board_filter:
-            continue
-        for spec in board.get("specifications", []):
-            if spec_filter and spec.get("spec_code") != spec_filter:
-                continue
-            sf = spec.get("spec_file")
-            if sf and sf.get("source"):
-                sloc = spec_storage_loc(board["exam_board_code"], spec.get("subject_code", ""),
-                                        spec.get("award_code", ""), spec.get("spec_ver", ""))
-                try:
-                    _item_bytes(sf["source"], sloc, store_dir=store_dir, cache_dir=cache_dir, rep=rep)
-                except Exception as exc:
-                    rep.errors.append(f"download spec {spec.get('spec_code')}: {exc}")
-            for p in spec.get("papers", []):
-                ploc = paper_storage_loc(board["exam_board_code"], spec.get("subject_code", ""),
-                                         spec.get("award_code", ""), p["paper_code"], p["session"], p["doc_type"])
-                try:
-                    _item_bytes(p["file"]["source"], ploc, store_dir=store_dir, cache_dir=cache_dir, rep=rep)
-                except Exception as exc:
-                    rep.errors.append(f"download {p.get('exam_code')}: {exc}")
-    logger.info(f"download-only done: downloaded={rep.downloaded} already_in_store={rep.download_cached} "
-                f"errors={len(rep.errors)} store={store_dir}")
-
-
-# ─────────────────────────────── storage upload (skip-if-exists + sha256) ───────────────────────────────
-def _split_loc(storage_loc: str) -> Tuple[str, str]:
-    bucket, _, path = storage_loc.partition("/")
-    return bucket, path
-
-def _object_exists(storage: StorageAdmin, bucket: str, path: str) -> bool:
-    """Existence check by listing the object's parent folder (Supabase storage has no stat)."""
-    parent, _, name = path.rpartition("/")
-    try:
-        listing = storage.client.supabase.storage.from_(bucket).list(parent)
-    except Exception as exc:
-        logger.warning(f"[exists?] list failed for {bucket}/{parent}: {exc}")
-        return False
-    return any((item.get("name") == name) for item in (listing or []))
-
-def upload_file(storage: StorageAdmin, storage_loc: str, data: bytes, *, force: bool, rep: LoadReport) -> str:
-    """Upload PDF bytes to storage at storage_loc. Returns the sha256 of the bytes.
-
-    Idempotent: if the object already exists and --force was not given, skips the upload
-    (the catalogue upsert still runs and records the checksum). With --force, overwrites.
-    """
-    sha = hashlib.sha256(data).hexdigest()
-    bucket, path = _split_loc(storage_loc)
-    if not force and _object_exists(storage, bucket, path):
-        logger.info(f"[upload] skip-exists {storage_loc} (sha256={sha[:12]})")
-        rep.files_skipped += 1
-        return sha
-    try:
-        storage.upload_file(bucket, path, data, "application/pdf", upsert=True)
-        logger.info(f"[upload] {storage_loc} ({len(data)} bytes, sha256={sha[:12]}) force={force}")
-        rep.files_uploaded += 1
-    except StorageError as exc:
-        logger.error(f"[upload] FAILED {storage_loc}: {exc}")
-        rep.files_failed += 1
-        rep.errors.append(f"upload {storage_loc}: {exc}")
-    return sha
-
-
-# ─────────────────────────────── catalogue upserts ───────────────────────────────
-def upsert_specification(client: SupabaseServiceRoleClient, spec: Dict[str, Any],
-                         storage_loc: Optional[str], sha: Optional[str], rep: LoadReport) -> None:
-    sf = spec.get("spec_file") or {}
-    doc_details = {
-        "award_level": spec.get("award_level"),
-        "provenance": sf.get("provenance"),
-        "original_name": sf.get("original_name"),
-        "sha256": sha,
-    }
-    row = {
-        "spec_code": spec["spec_code"],
-        "exam_board_code": spec["exam_board_code"],
-        "award_code": spec.get("award_code"),
-        "subject_code": spec.get("subject_code"),
-        "first_teach": spec.get("first_teach"),
-        "spec_ver": spec.get("spec_ver"),
-        "storage_loc": storage_loc,
-        "doc_type": "pdf",  # file format (CHECK-constrained); the role lives on eb_exams.type_code
-        "doc_details": {k: v for k, v in doc_details.items() if v is not None},
-    }
-    try:
-        client.supabase.table("eb_specifications").upsert(row, on_conflict="spec_code").execute()
-        logger.info(f"[spec] upsert {row['spec_code']}")
-        rep.specs_upserted += 1
-    except Exception as exc:
-        logger.error(f"[spec] FAILED {row['spec_code']}: {exc}")
-        rep.errors.append(f"spec {row['spec_code']}: {exc}")
-
-def upsert_paper(client: SupabaseServiceRoleClient, spec_code: str, p: Dict[str, Any],
-                 storage_loc: str, sha: Optional[str], rep: LoadReport) -> None:
-    f = p.get("file") or {}
-    doc_role = p["doc_type"]  # manifest role: QP|MS|INSERT|ER...
-    doc_details = {
-        "doc_role": doc_role,                # mirror of type_code for clarity
-        "original_name": f.get("original_name"),
-        "provenance": f.get("provenance"),
-        "sha256": sha,
-    }
-    row = {
-        "exam_code": p["exam_code"],
-        "spec_code": spec_code,
-        "paper_code": p.get("paper_code"),
-        "tier": p.get("tier"),
-        "session": p.get("session"),
-        "type_code": doc_role,               # ROLE goes here (QP/MS/INSERT/ER)
-        "doc_type": "pdf",                   # file format (CHECK-constrained)
-        "storage_loc": storage_loc,
-        "doc_details": {k: v for k, v in doc_details.items() if v is not None},
-    }
-    try:
-        client.supabase.table("eb_exams").upsert(row, on_conflict="exam_code").execute()
-        logger.info(f"[paper] upsert {row['exam_code']} type_code={doc_role}")
-        rep.papers_upserted += 1
-    except Exception as exc:
-        logger.error(f"[paper] FAILED {row['exam_code']}: {exc}")
-        rep.errors.append(f"paper {row['exam_code']}: {exc}")
-
-
-# ─────────────────────────────── user-side test subset ───────────────────────────────
-def _resolve_test_user(client: SupabaseServiceRoleClient, cfg: Dict[str, Any]) -> Optional[Tuple[str, str]]:
-    """Resolve (user_id, institute_id) for the user-side subset from config, with discovery fallback."""
-    user_id = cfg.get("user_id")
-    if not user_id and cfg.get("user_email"):
-        res = client.supabase.table("profiles").select("id").eq("email", cfg["user_email"]).limit(1).execute()
-        rows = getattr(res, "data", None) or []
-        user_id = rows[0]["id"] if rows else None
-    if not user_id:
-        logger.warning("[user-subset] no test user resolvable (set test_subset.user_id or user_email); skipping")
-        return None
-    institute_id = cfg.get("institute_id")
-    if not institute_id:
-        res = client.supabase.table("institute_memberships").select("institute_id").eq("profile_id", user_id).limit(1).execute()
-        rows = getattr(res, "data", None) or []
-        institute_id = rows[0]["institute_id"] if rows else None
-    if not institute_id:
-        logger.warning(f"[user-subset] no institute for user {user_id}; skipping")
-        return None
-    return user_id, institute_id
-
-def copy_user_test_subset(client: SupabaseServiceRoleClient, storage: StorageAdmin,
-                          m: Dict[str, Any], rep: LoadReport) -> None:
-    """Copy a small subset of admin papers into a test user's exam space so user-side flows
-    (upload-as-exam / promote-from-cabinet / mark) are testable.
-
-    Driven by an optional manifest `test_subset:` block:
-        test_subset:
-          user_id: <uuid>            # or user_email: <email>
-          institute_id: <uuid>       # optional; discovered from membership if omitted
-          papers: 2                  # how many QP papers to copy (default 2)
-    Degrades gracefully (logs + skips) if no test user is resolvable on this env.
-    """
-    cfg = m.get("test_subset") or {}
-    resolved = _resolve_test_user(client, cfg)
-    if not resolved:
-        return
-    user_id, institute_id = resolved
-    limit = int(cfg.get("papers", 2))
-
-    # Gather candidate QP papers (admin corpus already uploaded to cc.examboards).
-    candidates: List[Tuple[str, Dict[str, Any]]] = []
-    for board in m.get("boards", []):
-        for spec in board.get("specifications", []):
-            for p in spec.get("papers", []):
-                if p.get("doc_type") == "QP":
-                    candidates.append((board["exam_board_code"], spec, p))
-    candidates = candidates[:limit]
-    if not candidates:
-        logger.info("[user-subset] no QP papers to copy")
-        return
-
-    # Ensure a cabinet for the user.
-    cab_name = "Exam Marker Template Sources"
-    res = client.supabase.table("file_cabinets").select("id").eq("user_id", user_id).eq("name", cab_name).limit(1).execute()
-    rows = getattr(res, "data", None) or []
-    if rows:
-        cabinet_id = rows[0]["id"]
-    else:
-        ins = client.supabase.table("file_cabinets").insert({"user_id": user_id, "name": cab_name}).execute()
-        cabinet_id = (getattr(ins, "data", None) or [{}])[0].get("id")
-    if not cabinet_id:
-        logger.warning("[user-subset] could not ensure cabinet; skipping")
-        return
-
-    import uuid as _uuid
-    for board_code, spec, p in candidates:
-        src_loc = paper_storage_loc(board_code, spec.get("subject_code", ""), spec.get("award_code", ""),
-                                    p["paper_code"], p["session"], p["doc_type"])
-        sbucket, spath = _split_loc(src_loc)
-        try:
-            data = storage.download_file(sbucket, spath)
-        except Exception as exc:
-            logger.warning(f"[user-subset] source missing {src_loc}: {exc}; skipping {p['exam_code']}")
-            continue
-        file_id = str(_uuid.uuid4())
-        safe_name = f"{p['exam_code']}.pdf"
-        dst_bucket = "cc.users"
-        dst_path = f"exam-marker/{institute_id}/{cabinet_id}/{file_id}/{safe_name}"
-        try:
-            storage.upload_file(dst_bucket, dst_path, data, "application/pdf", upsert=True)
-        except Exception as exc:
-            logger.warning(f"[user-subset] copy upload failed {dst_path}: {exc}")
-            continue
-        client.supabase.table("files").upsert({
-            "id": file_id, "cabinet_id": cabinet_id, "name": safe_name, "path": dst_path,
-            "bucket": dst_bucket, "mime_type": "application/pdf", "uploaded_by": user_id,
-            "size_bytes": len(data), "source": "exam-corpus-seed", "is_directory": False,
-            "relative_path": safe_name, "processing_status": "uploaded",
-        }).execute()
-        logger.info(f"[user-subset] copied {p['exam_code']} -> {dst_bucket}/{dst_path}")
-        rep.user_copies += 1
-
-
-# ─────────────────────────────── first sweep (docling auto-map) ───────────────────────────────
-def _resolve_system_identity(client: SupabaseServiceRoleClient, m: Dict[str, Any]) -> Optional[Tuple[str, str]]:
-    cfg = m.get("system_identity") or m.get("test_subset") or {}
-    user_id = cfg.get("teacher_id") or cfg.get("user_id")
-    if not user_id and cfg.get("user_email"):
-        res = client.supabase.table("profiles").select("id").eq("email", cfg["user_email"]).limit(1).execute()
-        rows = getattr(res, "data", None) or []
-        user_id = rows[0]["id"] if rows else None
-    institute_id = cfg.get("institute_id")
-    if user_id and not institute_id:
-        res = client.supabase.table("institute_memberships").select("institute_id").eq("profile_id", user_id).limit(1).execute()
-        rows = getattr(res, "data", None) or []
-        institute_id = rows[0]["institute_id"] if rows else None
-    if not user_id or not institute_id:
-        logger.warning("[first-sweep] no system identity (set system_identity.teacher_id+institute_id); skipping sweep")
-        return None
-    return user_id, institute_id
-
-def first_sweep(client: SupabaseServiceRoleClient, storage: StorageAdmin,
-                m: Dict[str, Any], board_filter: Optional[str], spec_filter: Optional[str],
-                cache_dir: str, rep: LoadReport) -> None:
-    """Run the docling/auto_map first pass over seeded QP papers and persist the resulting
-    template structure (questions/response areas/boundaries/layout) via the same mapping the
-    /auto-map endpoint uses. System-owned exam_templates are created per QP paper.
-
-    Requires a resolvable `system_identity` (teacher_id/user_email + institute_id) on this env.
-    """
-    identity = _resolve_system_identity(client, m)
-    if not identity:
-        return
-    teacher_id, institute_id = identity
-
-    # Import the auto-map mapping helpers lazily (pulls fastapi/router only when sweeping).
-    try:
-        from api.services.docling import auto_map, AutoMapError
-        from routers.exam.templates import _map_first_pass_to_rows
-    except Exception as exc:
-        logger.error(f"[first-sweep] could not import auto-map pipeline: {exc}")
-        rep.errors.append(f"first-sweep import: {exc}")
-        return
-
-    sb = client.supabase
-    for board in m.get("boards", []):
-        if board_filter and board.get("exam_board_code") != board_filter:
-            continue
-        for spec in board.get("specifications", []):
-            if spec_filter and spec.get("spec_code") != spec_filter:
-                continue
-            for p in spec.get("papers", []):
-                if p.get("doc_type") != "QP":
-                    continue
-                # Resolve the seeded eb_exams row (id) for the template join.
-                ex = sb.table("eb_exams").select("id, exam_code").eq("exam_code", p["exam_code"]).limit(1).execute()
-                ex_rows = getattr(ex, "data", None) or []
-                exam_id = ex_rows[0]["id"] if ex_rows else None
-
-                loc = paper_storage_loc(board["exam_board_code"], spec.get("subject_code", ""),
-                                        spec.get("award_code", ""), p["paper_code"], p["session"], p["doc_type"])
-                bkt, path = _split_loc(loc)
-                try:
-                    pdf_bytes = storage.download_file(bkt, path)
-                except Exception as exc:
-                    logger.warning(f"[first-sweep] source missing {loc}: {exc}; skipping {p['exam_code']}")
-                    continue
-
-                # Ensure a system-owned template for this paper (idempotent on exam_code+teacher).
-                tpl = sb.table("exam_templates").select("id").eq("exam_code", p["exam_code"]).eq("teacher_id", teacher_id).limit(1).execute()
-                tpl_rows = getattr(tpl, "data", None) or []
-                if tpl_rows:
-                    template_id = tpl_rows[0]["id"]
-                else:
-                    new_tpl = sb.table("exam_templates").insert({
-                        "exam_id": exam_id, "exam_code": p["exam_code"], "institute_id": institute_id,
-                        "teacher_id": teacher_id, "title": f"{p['exam_code']} (auto-map seed)",
-                        "subject": spec.get("subject_code"), "status": "draft",
-                    }).execute()
-                    template_id = (getattr(new_tpl, "data", None) or [{}])[0].get("id")
-                if not template_id:
-                    logger.warning(f"[first-sweep] could not ensure template for {p['exam_code']}; skipping")
-                    continue
-
-                try:
-                    first_pass = auto_map(pdf_bytes, source_pdf=loc)
-                    rows = _map_first_pass_to_rows(template_id, first_pass, pdf_bytes)
-                except (AutoMapError, ValueError) as exc:
-                    logger.warning(f"[first-sweep] auto-map failed for {p['exam_code']}: {exc}")
-                    rep.sweep_failed += 1
-                    continue
-                except Exception as exc:
-                    logger.exception(f"[first-sweep] unexpected error for {p['exam_code']}: {exc}")
-                    rep.sweep_failed += 1
-                    continue
-
-                # Refresh derived rows. Seed templates are system-owned with no human edits to
-                # preserve, so we clear ALL child rows for the template (not just ai/unconfirmed)
-                # and re-insert id-deduped payloads — idempotent across re-runs and robust to the
-                # deterministic uuid5 ids the mapper can repeat within a batch.
-                for table in ("exam_response_areas", "exam_boundaries", "exam_template_layout", "exam_questions"):
-                    sb.table(table).delete().eq("template_id", template_id).execute()
-                for table, key in (("exam_questions", "questions"), ("exam_response_areas", "response_areas"),
-                                   ("exam_boundaries", "boundaries"), ("exam_template_layout", "layout")):
-                    seen_ids: set = set()
-                    payload = []
-                    for r in (rows.get(key) or []):
-                        rid = r.get("id")
-                        if rid is not None and rid in seen_ids:
-                            continue
-                        if rid is not None:
-                            seen_ids.add(rid)
-                        payload.append(r)
-                    if payload:
-                        sb.table(table).insert(payload).execute()
-                updates = {"page_count": first_pass.get("meta", {}).get("n_pages")}
-                sb.table("exam_templates").update({k: v for k, v in updates.items() if v is not None}).eq("id", template_id).execute()
-                logger.info(f"[first-sweep] swept {p['exam_code']} -> template {template_id} "
-                            f"(q={len(rows.get('questions', []))} ra={len(rows.get('response_areas', []))})")
-                rep.swept += 1
-
-
-# ─────────────────────────────── unseed (inverse of the loader) ───────────────────────────────
-def _chunks(seq: List[Any], n: int = 100):
-    for i in range(0, len(seq), n):
-        yield seq[i:i + n]
-
-def _storage_remove(storage: StorageAdmin, bucket: str, paths: List[str]) -> None:
-    """Remove object paths from a bucket through the Supabase Storage API.
-
-    The python client treats missing objects as a successful no-op, which is useful for
-    unseed idempotency. Any API/permission failure is raised so callers can avoid
-    deleting the matching DB rows while storage may still exist.
-    """
-    result = storage.client.supabase.storage.from_(bucket).remove(paths)
-    error = getattr(result, "error", None)
-    if error:
-        raise StorageError(str(error))
-    if isinstance(result, dict) and result.get("error"):
-        raise StorageError(str(result["error"]))
-
-def _delete_user_subset_files(client: SupabaseServiceRoleClient, storage: StorageAdmin, *,
-                              exam_codes: Optional[List[str]], rep: LoadReport) -> None:
-    """Delete --user-subset files from cc.users storage, then their files rows.
-
-    User-subset seeding writes rows with source='exam-corpus-seed', bucket='cc.users',
-    and paths under exam-marker/. Storage must be removed before the files rows: the
-    files GC trigger also tries to delete storage when rows are deleted, so removing
-    objects first avoids trigger failures and keeps this operation idempotent.
-
-    exam_codes=None means remove all user-subset seed rows (used by unscoped unseed
-    even if the eb_* rows were already removed by a prior partial run).
-    """
-    sb = client.supabase
-    seeded_files: List[Dict[str, Any]] = []
-
-    def _base_query():
-        return sb.table("files").select("id, bucket, path, name, source") \
-            .eq("bucket", "cc.users").eq("source", "exam-corpus-seed") \
-            .like("path", "exam-marker/%")
-
-    if exam_codes is None:
-        seeded_files.extend(getattr(_base_query().execute(), "data", None) or [])
-    elif exam_codes:
-        for chunk in _chunks([f"{code}.pdf" for code in exam_codes if code], 100):
-            seeded_files.extend(getattr(_base_query().in_("name", chunk).execute(), "data", None) or [])
-
-    rows_by_id: Dict[str, Dict[str, Any]] = {}
-    paths_by_bucket: Dict[str, List[str]] = {}
-    seen_paths: set = set()
-    for row in seeded_files:
-        row_id = row.get("id")
-        bucket = row.get("bucket")
-        path = row.get("path")
-        if row_id:
-            rows_by_id[str(row_id)] = row
-        if bucket == "cc.users" and isinstance(path, str) and path.startswith("exam-marker/"):
-            key = (bucket, path)
-            if key not in seen_paths:
-                seen_paths.add(key)
-                paths_by_bucket.setdefault(bucket, []).append(path)
-
-    removable_ids = list(rows_by_id)
-    if not removable_ids and not paths_by_bucket:
-        logger.info("[unseed] no user-subset cc.users files to remove")
-        return
-
-    for bkt, paths in paths_by_bucket.items():
-        for chunk in _chunks(paths, 100):
-            try:
-                _storage_remove(storage, bkt, chunk)
-                rep.unseed_objects += len(chunk)
-            except Exception as exc:
-                logger.warning(f"[unseed] user-subset storage remove failed ({bkt}, {len(chunk)} objs): {exc}")
-                rep.errors.append(f"user-subset storage remove {bkt}: {exc}")
-                return
-
-    for chunk in _chunks(removable_ids, 100):
-        try:
-            sb.table("files").delete().in_("id", chunk).execute()
-            rep.unseed_user_files += len(chunk)
-        except Exception as exc:
-            logger.warning(f"[unseed] user-subset files delete failed: {exc}")
-            rep.errors.append(f"user-subset files delete: {exc}")
-
-def unseed(client: SupabaseServiceRoleClient, storage: StorageAdmin, *,
-           board_filter: Optional[str], spec_filter: Optional[str],
-           drop_specs: bool = True, drop_seed_templates: bool = True, rep: LoadReport) -> None:
-    """Inverse of the loader: remove the seeded public corpus, scoped by --board/--spec (or all).
-
-    Deletes (in FK-safe order): cc.examboards storage objects (via the Storage API, since the
-    protect_delete trigger blocks direct SQL deletes), first-sweep exam_templates created by the
-    seed (title '... (auto-map seed)', cascades children), eb_exams rows, then eb_specifications.
-    """
-    sb = client.supabase
-    q = sb.table("eb_specifications").select("spec_code, storage_loc, exam_board_code")
-    if board_filter:
-        q = q.eq("exam_board_code", board_filter)
-    if spec_filter:
-        q = q.eq("spec_code", spec_filter)
-    specs = getattr(q.execute(), "data", None) or []
-    spec_codes = [s["spec_code"] for s in specs]
-    if not spec_codes:
-        if not board_filter and not spec_filter:
-            _delete_user_subset_files(client, storage, exam_codes=None, rep=rep)
-        logger.info("[unseed] no matching specifications; nothing to do")
-        return
-
-    exams: List[Dict[str, Any]] = []
-    for chunk in _chunks(spec_codes):
-        res = sb.table("eb_exams").select("id, exam_code, storage_loc").in_("spec_code", chunk).execute()
-        exams.extend(getattr(res, "data", None) or [])
-
-    # 1) User-subset storage/rows. Storage is removed before files rows so trg_files_gc has
-    # nothing left to collect when rows are deleted.
-    user_subset_exam_codes = None if not board_filter and not spec_filter else [
-        e.get("exam_code") for e in exams if e.get("exam_code")
-    ]
-    _delete_user_subset_files(client, storage, exam_codes=user_subset_exam_codes, rep=rep)
-
-    # 2) Storage objects (Storage API; batch-remove per bucket). Specs may carry a spec PDF too.
-    by_bucket: Dict[str, List[str]] = {}
-    for row in exams + specs:
-        loc = row.get("storage_loc")
-        if not loc or "/" not in loc:
-            continue
-        bkt, _, path = loc.partition("/")
-        by_bucket.setdefault(bkt, []).append(path)
-    for bkt, paths in by_bucket.items():
-        for chunk in _chunks(paths, 100):
-            try:
-                storage.client.supabase.storage.from_(bkt).remove(chunk)
-                rep.unseed_objects += len(chunk)
-            except Exception as exc:
-                logger.warning(f"[unseed] storage remove failed ({bkt}, {len(chunk)} objs): {exc}")
-
-    # 3) First-sweep templates created by the seed (cascades questions/regions/boundaries/layout).
-    if drop_seed_templates and exams:
-        exam_codes = [e["exam_code"] for e in exams if e.get("exam_code")]
-        for chunk in _chunks(exam_codes, 100):
-            try:
-                res = sb.table("exam_templates").delete(count="exact") \
-                    .in_("exam_code", chunk).like("title", "%(auto-map seed)%").execute()
-                rep.unseed_templates += getattr(res, "count", None) or len(getattr(res, "data", []) or [])
-            except Exception as exc:
-                logger.warning(f"[unseed] template delete failed: {exc}")
-
-    # 4) Catalogue rows: eb_exams (by id), then eb_specifications (by spec_code).
-    exam_ids = [e["id"] for e in exams]
-    for chunk in _chunks(exam_ids, 100):
-        try:
-            sb.table("eb_exams").delete().in_("id", chunk).execute()
-            rep.unseed_exams += len(chunk)
-        except Exception as exc:
-            logger.warning(f"[unseed] eb_exams delete failed: {exc}")
-    if drop_specs:
-        for chunk in _chunks(spec_codes, 100):
-            try:
-                sb.table("eb_specifications").delete().in_("spec_code", chunk).execute()
-                rep.unseed_specs += len(chunk)
-            except Exception as exc:
-                logger.warning(f"[unseed] eb_specifications delete failed: {exc}")
-
-    logger.info(f"unseed done: storage_objects={rep.unseed_objects} user_files={rep.unseed_user_files} "
-                f"templates={rep.unseed_templates} exams={rep.unseed_exams} specs={rep.unseed_specs}")
-
-
-# ─────────────────────────────── orchestration ───────────────────────────────
-def load(manifest_path: str, *, dry_run: bool, force: bool, board_filter: Optional[str],
-         spec_filter: Optional[str], user_subset: bool, do_first_sweep: bool,
-         cache_dir: str = DEFAULT_CACHE_DIR, store_dir: Optional[str] = None) -> LoadReport:
-    with open(manifest_path) as f:
-        m = yaml.safe_load(f)
-    rep = LoadReport()
-
-    errs = validate_manifest(m)
-    if errs:
-        rep.errors = list(errs)
-        logger.error(f"manifest validation failed: {len(errs)} error(s)")
-        for e in errs[:40]:
-            logger.error(f"  - {e}")
-        if not dry_run:
-            return rep
-
-    client = None if dry_run else SupabaseServiceRoleClient()
-    storage = None if dry_run else StorageAdmin()
-
-    for board in m.get("boards", []):
-        if board_filter and board.get("exam_board_code") != board_filter:
-            continue
-        for spec in board.get("specifications", []):
-            if spec_filter and spec.get("spec_code") != spec_filter:
-                continue
-            # Specification document (optional).
-            sloc = None
-            spec_sha = None
-            sf = spec.get("spec_file")
-            if sf and sf.get("source"):
-                sloc = spec_storage_loc(board["exam_board_code"], spec.get("subject_code", ""),
-                                        spec.get("award_code", ""), spec.get("spec_ver", ""))
-                if not dry_run:
-                    try:
-                        spec_sha = upload_file(storage, sloc,
-                                               _item_bytes(sf["source"], sloc, store_dir=store_dir,
-                                                           cache_dir=cache_dir, rep=rep),
-                                               force=force, rep=rep)
-                    except Exception as exc:
-                        logger.error(f"[spec-file] {spec.get('spec_code')}: {exc}")
-                        rep.files_failed += 1
-                        rep.errors.append(f"spec-file {spec.get('spec_code')}: {exc}")
-            if not dry_run:
-                upsert_specification(client, spec, sloc, spec_sha, rep)
-
-            # Papers.
-            for p in spec.get("papers", []):
-                ploc = paper_storage_loc(board["exam_board_code"], spec.get("subject_code", ""),
-                                         spec.get("award_code", ""), p["paper_code"], p["session"], p["doc_type"])
-                if dry_run:
-                    continue
-                psha = None
-                try:
-                    psha = upload_file(storage, ploc,
-                                       _item_bytes(p["file"]["source"], ploc, store_dir=store_dir,
-                                                   cache_dir=cache_dir, rep=rep),
-                                       force=force, rep=rep)
-                except Exception as exc:
-                    logger.error(f"[paper-file] {p.get('exam_code')}: {exc}")
-                    rep.files_failed += 1
-                    rep.errors.append(f"paper-file {p.get('exam_code')}: {exc}")
-                upsert_paper(client, spec["spec_code"], p, ploc, psha, rep)
-
-    if user_subset and not dry_run:
-        copy_user_test_subset(client, storage, m, rep)
-    if do_first_sweep and not dry_run:
-        first_sweep(client, storage, m, board_filter, spec_filter, cache_dir, rep)
-
-    logger.info(f"corpus load done: specs={rep.specs_upserted} papers={rep.papers_upserted} "
-                f"uploaded={rep.files_uploaded} skipped={rep.files_skipped} failed={rep.files_failed} "
-                f"user_copies={rep.user_copies} swept={rep.swept} errors={len(rep.errors)}")
-    return rep
-
-
-def main() -> None:
-    ap = argparse.ArgumentParser(description="Seed (or unseed) the public exam-paper corpus from a manifest.")
-    ap.add_argument("--manifest", help="corpus manifest (required except for --unseed)")
-    ap.add_argument("--dry-run", action="store_true", help="validate + report, no writes")
-    ap.add_argument("--force", action="store_true", help="re-upload/overwrite existing storage objects")
-    ap.add_argument("--board", default=None, help="only this exam_board_code")
-    ap.add_argument("--spec", default=None, help="only this spec_code")
-    ap.add_argument("--user-subset", action="store_true", help="also seed a user-side test subset")
-    ap.add_argument("--first-sweep", action="store_true", help="run docling/auto-map first pass on seeded papers")
-    ap.add_argument("--cache-dir", default=DEFAULT_CACHE_DIR, help="throwaway url-hash cache dir")
-    ap.add_argument("--store-dir", default=DEFAULT_STORE_DIR,
-                    help="persistent, bucket-shaped local store (download-once, seed-many)")
-    ap.add_argument("--no-store", action="store_true",
-                    help="ignore the local store; always fetch from source (don't read/populate the store)")
-    ap.add_argument("--download-only", action="store_true",
-                    help="populate the local store from the manifest; no DB/bucket writes")
-    ap.add_argument("--unseed", action="store_true",
-                    help="INVERSE: remove seeded eb_*/storage/first-sweep templates (scoped by --board/--spec)")
-    a = ap.parse_args()
-    store_dir = None if a.no_store else a.store_dir
-    import json
-
-    if a.unseed:
-        rep = LoadReport()
-        unseed(SupabaseServiceRoleClient(), StorageAdmin(),
-               board_filter=a.board, spec_filter=a.spec, rep=rep)
-        print(json.dumps(rep.as_dict(), indent=2))
-        return
-
-    if not a.manifest:
-        ap.error("--manifest is required unless --unseed is given")
-
-    if a.download_only:
-        with open(a.manifest) as f:
-            m = yaml.safe_load(f)
-        rep = LoadReport()
-        download_corpus(m, store_dir=(a.store_dir), board_filter=a.board, spec_filter=a.spec,
-                        cache_dir=a.cache_dir, rep=rep)
-        print(json.dumps(rep.as_dict(), indent=2))
-        return
-
-    rep = load(a.manifest, dry_run=a.dry_run, force=a.force, board_filter=a.board, spec_filter=a.spec,
-               user_subset=a.user_subset, do_first_sweep=a.first_sweep, cache_dir=a.cache_dir,
-               store_dir=store_dir)
-    print(json.dumps(rep.as_dict(), indent=2))
-
-
-if __name__ == "__main__":
-    main()

tests/test_docling_extract.py

@@ -1,81 +0,0 @@
-from api.services.docling.extract import aqa_questions_rapid
-
-
-def _text(raw, page, l, t, r=120, b=None):
-    return {
-        "text": raw,
-        "prov": [{"page_no": page, "bbox": {"l": l, "t": t, "r": r, "b": b if b is not None else t - 14, "coord_origin": "BOTTOMLEFT"}}],
-    }
-
-
-def test_aqa_rapid_cleans_noisy_margin_label(tmp_path):
-    (tmp_path / "p1.json").write_text(
-        '{"texts":[{"text":"02].3","prov":[{"page_no":1,"bbox":{"l":49,"t":515,"r":102,"b":500,"coord_origin":"BOTTOMLEFT"}}]}]}'
-    )
-
-    parts = aqa_questions_rapid(str(tmp_path / "p*.json"))
-
-    assert "02.3" in parts
-
-
-def test_aqa_rapid_infers_missing_leading_part_from_next_part(tmp_path):
-    (tmp_path / "p1.json").write_text(
-        '{"texts":[{"text":"Question prose starts here","prov":[{"page_no":1,"bbox":{"l":114,"t":774,"r":500,"b":760,"coord_origin":"BOTTOMLEFT"}}]}]}'
-    )
-    (tmp_path / "p2.json").write_text(
-        '{"texts":[{"text":"07.2","prov":[{"page_no":2,"bbox":{"l":49,"t":776,"r":104,"b":761,"coord_origin":"BOTTOMLEFT"}}]}]}'
-    )
-
-    parts = aqa_questions_rapid(str(tmp_path / "p*.json"))
-
-    assert parts["07.1"]["page"] == 2
-    assert parts["07.1"]["bbox"]["l"] == 49
-    assert "07.2" in parts
-
-
-def test_aqa_rapid_fills_small_mcq_gaps(tmp_path):
-    texts = [_text("06.1 Structured question before Section B", 1, 49, 820)]
-    for idx, n in enumerate(["07", "08", "11", "12", "13"]):
-        texts.append(_text(n + " Which statement is correct?", 1, 49, 780 - idx * 40))
-    import json
-    (tmp_path / "p1.json").write_text(json.dumps({"texts": texts}))
-
-    parts = aqa_questions_rapid(str(tmp_path / "p*.json"))
-
-    for label in ["07.0", "08.0", "09.0", "10.0", "11.0", "12.0", "13.0"]:
-        assert label in parts
-
-
-def test_aqa_rapid_maps_circled_digit_labels(tmp_path):
-    import json
-    (tmp_path / "p1.json").write_text(json.dumps({"texts": [_text("01.③", 1, 49, 515, r=102, b=500)]}))
-
-    parts = aqa_questions_rapid(str(tmp_path / "p*.json"))
-
-    assert "01.3" in parts
-
-
-def test_aqa_rapid_infers_internal_structured_gap(tmp_path):
-    import json
-    texts = [
-        _text("05.2 Some question text", 1, 49, 700),
-        _text("05.3 Middle question text", 1, 49, 620),
-        _text("05.5 Later question text", 2, 49, 740),
-    ]
-    (tmp_path / "p1.json").write_text(json.dumps({"texts": texts[:2]}))
-    (tmp_path / "p2.json").write_text(json.dumps({"texts": texts[2:]}))
-
-    parts = aqa_questions_rapid(str(tmp_path / "p*.json"))
-
-    assert "05.1" in parts
-    assert "05.4" in parts
-    assert "05.5" in parts
-
-
-def test_aqa_rapid_keeps_bare_single_part_question(tmp_path):
-    import json
-    (tmp_path / "p1.json").write_text(json.dumps({"texts": [_text("03", 1, 49, 775)]}))
-
-    parts = aqa_questions_rapid(str(tmp_path / "p*.json"))
-
-    assert "03.0" in parts

tests/test_docling_regions.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 from PIL import Image, ImageDraw
 
-from api.services.docling import extract
 from api.services.docling.regions import detect_response_regions_from_image
 
 
@@ -38,46 +37,3 @@ def test_detects_answer_box() -> None:
     assert boxes
     assert boxes[0]["bbox"]["w"] > 600
     assert boxes[0]["bbox"]["h"] > 200
-
-
-def test_detect_response_region_taxonomy_for_lines_and_boxes():
-    img = Image.new("RGB", (800, 1000), "white")
-    draw = ImageDraw.Draw(img)
-    for y in (220, 260, 300):
-        draw.line((120, y, 680, y), fill="black", width=2)
-    draw.rectangle((140, 520, 660, 640), outline="black", width=3)
-
-    regions = detect_response_regions_from_image(img, min_confidence=0.1)
-    types = {r.region_type for r in regions}
-
-    assert "answer_lines" in types
-    assert "answer_box" in types
-
-
-def test_attach_detected_response_regions_normalizes_taxonomy_and_uses_pdf_y(monkeypatch, tmp_path):
-    pdf = tmp_path / "paper.pdf"
-    pdf.write_bytes(b"%PDF test placeholder")
-    parts = {
-        "01.1": {"q": "01", "page": 1, "bbox": {"l": 50, "r": 80, "t": 700, "b": 680}, "regions": []},
-        "01.2": {"q": "01", "page": 1, "bbox": {"l": 50, "r": 80, "t": 500, "b": 480}, "regions": []},
-    }
-
-    def fake_detect(path, min_confidence=0.32):
-        return [{
-            "page_index": 0,
-            "region_type": "answer-box",
-            "confidence": 0.77,
-            "bbox": {"x": 100, "y": 335, "w": 500, "h": 40},
-            "detection_method": "test",
-            "meta": {"page_height_px": 1000, "page_height_pdf": 800},
-        }]
-
-    monkeypatch.setattr(extract.region_mod, "detect_response_regions_from_pdf", fake_detect)
-
-    attached, candidates = extract.attach_detected_response_regions(parts, str(pdf))
-
-    assert attached == 1
-    assert len(candidates) == 1
-    assert parts["01.1"]["regions"] == []
-    assert parts["01.2"]["regions"][0]["type"] == "answer_box"
-    assert parts["01.2"]["regions"][0]["source"] == "opencv"

tests/test_exam_templates.py

@@ -143,9 +143,6 @@ class _FakeStorageAdmin:
     def download_file(self, bucket_id, file_path):
         return b"%PDF-1.7 fake"
 
-    def create_signed_url(self, bucket_id, file_path, expires_in=3600):
-        return {"signedURL": f"https://storage.test/{bucket_id}/{file_path}?token=fake&expires_in={expires_in}"}
-
 
 class _FakeServiceRoleClient:
     def __init__(self, store):
@@ -174,65 +171,6 @@ def test_requires_auth_when_not_overridden():
     assert resp.status_code in (401, 403)  # unauthenticated, not processed
 
 
-def test_catalogue_requires_auth_when_not_overridden():
-    app = FastAPI()
-    app.include_router(router, prefix="/api/exam")
-    resp = TestClient(app).get("/api/exam/catalogue")
-    assert resp.status_code in (401, 403)
-
-
-def test_list_catalogue_papers_uses_as_user_metadata():
-    store = {
-        "eb_exams": [
-            {"id": "e1", "exam_code": "AQA-1", "type_code": "QP", "storage_loc": "cc.examboards/aqa/p.pdf"},
-            {"id": "e2", "exam_code": "AQA-MS", "type_code": "MS", "storage_loc": "cc.examboards/aqa/ms.pdf"},
-        ]
-    }
-    client, _ = make_client(store=store)
-    resp = client.get("/api/exam/catalogue")
-    assert resp.status_code == 200
-    assert [p["id"] for p in resp.json()["papers"]] == ["e1"]
-
-
-def test_catalogue_signed_url_requires_auth_and_signs_examboard_pdf(monkeypatch):
-    store = {
-        "eb_exams": [
-            {"id": "e1", "exam_code": "AQA-1", "type_code": "QP", "storage_loc": "cc.examboards/aqa/physics/qp.pdf"},
-        ]
-    }
-    client, _ = make_client(store=store)
-    monkeypatch.setattr(templates_mod, "StorageAdmin", _FakeStorageAdmin)
-    resp = client.get("/api/exam/catalogue/e1/signed-url?expires_in=120")
-    assert resp.status_code == 200
-    body = resp.json()
-    assert body["bucket"] == "cc.examboards"
-    assert body["path"] == "aqa/physics/qp.pdf"
-    assert body["expires_in"] == 120
-    assert "token=fake" in body["signed_url"]
-
-
-def test_catalogue_signed_url_rejects_non_examboard_storage(monkeypatch):
-    store = {
-        "eb_exams": [
-            {"id": "e1", "exam_code": "AQA-1", "type_code": "QP", "storage_loc": "cc.public/aqa/physics/qp.pdf"},
-        ]
-    }
-    client, _ = make_client(store=store)
-    monkeypatch.setattr(templates_mod, "StorageAdmin", _FakeStorageAdmin)
-    assert client.get("/api/exam/catalogue/e1/signed-url").status_code == 404
-
-
-def test_catalogue_signed_url_rejects_non_catalogue_doc_type(monkeypatch):
-    store = {
-        "eb_exams": [
-            {"id": "e1", "exam_code": "AQA-MS", "type_code": "MS", "storage_loc": "cc.examboards/aqa/physics/ms.pdf"},
-        ]
-    }
-    client, _ = make_client(store=store)
-    monkeypatch.setattr(templates_mod, "StorageAdmin", _FakeStorageAdmin)
-    assert client.get("/api/exam/catalogue/e1/signed-url").status_code == 404
-
-
 def test_create_template_sets_owner_and_institute():
     client, store = make_client()
     resp = client.post("/api/exam/templates", json={"title": "AQA Physics 1H", "subject": "Physics"})
@@ -642,20 +580,6 @@ def test_auto_map_fast_path_merges_ai_rows_and_returns_detail(monkeypatch):
     assert store["exam_boundaries"] and store["exam_boundaries"][0]["derivation"] == "docling-main-band"
 
 
-def test_auto_map_deduplicates_repeated_response_area_ids(monkeypatch):
-    store = _template_with_source()
-    client, store = make_client(store=store)
-    _patch_auto_map(monkeypatch, store, fast=True)
-    dup = {"page_index": 0, "bbox": {"l": 50, "t": 700, "r": 100, "b": 680, "coord_origin": "BOTTOMLEFT"}, "region_type": "answer_lines", "confidence": 0.9}
-    monkeypatch.setattr(templates_mod, "detect_response_regions_from_pdf", lambda *_a, **_k: [dup, dict(dup)])
-
-    resp = client.post("/api/exam/templates/t1/auto-map")
-
-    assert resp.status_code == 200
-    response_area_ids = [r["id"] for r in store["exam_response_areas"]]
-    assert len(response_area_ids) == len(set(response_area_ids))
-
-
 def test_auto_map_preserves_manual_and_confirmed_rows_on_rerun(monkeypatch):
     store = _template_with_source()
     store.update({

tests/test_files_idor.py

@@ -1,103 +0,0 @@
-from types import SimpleNamespace
-
-import pytest
-
-import routers.database.files.files as files_router
-import routers.database.files.files_simplified as files_simplified_router
-
-
-ROUTERS = [files_router, files_simplified_router]
-
-USER_A = "00000000-0000-0000-0000-000000000001"
-USER_B = "00000000-0000-0000-0000-000000000002"
-CAB_A = "10000000-0000-0000-0000-000000000001"
-CAB_B = "10000000-0000-0000-0000-000000000002"
-
-
-class FakeQuery:
-    def __init__(self, rows):
-        self.rows = list(rows)
-
-    def select(self, *_args, **_kwargs):
-        return self
-
-    def eq(self, key, value):
-        self.rows = [row for row in self.rows if row.get(key) == value]
-        return self
-
-    def limit(self, _n):
-        return self
-
-    def execute(self):
-        return SimpleNamespace(data=self.rows)
-
-
-class FakeSupabase:
-    def __init__(self, store):
-        self.store = store
-
-    def table(self, name):
-        return FakeQuery(self.store.get(name, []))
-
-
-class FakeServiceRoleClient:
-    def __init__(self, store):
-        self.supabase = FakeSupabase(store)
-
-
-@pytest.mark.parametrize("router_module", ROUTERS)
-def test_list_files_hides_unowned_unshared_cabinet(monkeypatch, router_module):
-    store = {
-        "file_cabinets": [
-            {"id": CAB_A, "user_id": USER_A},
-            {"id": CAB_B, "user_id": USER_B},
-        ],
-        "cabinet_memberships": [],
-        "files": [
-            {"id": "file-a", "cabinet_id": CAB_A, "uploaded_by": USER_A},
-            {"id": "file-b", "cabinet_id": CAB_B, "uploaded_by": USER_B},
-        ],
-    }
-    monkeypatch.setattr(
-        router_module,
-        "SupabaseServiceRoleClient",
-        lambda: FakeServiceRoleClient(store),
-    )
-
-    assert router_module.list_files(CAB_B, {"sub": USER_A}) == []
-
-
-@pytest.mark.parametrize("router_module", ROUTERS)
-def test_list_files_allows_own_cabinet(monkeypatch, router_module):
-    store = {
-        "file_cabinets": [{"id": CAB_A, "user_id": USER_A}],
-        "cabinet_memberships": [],
-        "files": [{"id": "file-a", "cabinet_id": CAB_A, "uploaded_by": USER_A}],
-    }
-    monkeypatch.setattr(
-        router_module,
-        "SupabaseServiceRoleClient",
-        lambda: FakeServiceRoleClient(store),
-    )
-
-    assert router_module.list_files(CAB_A, {"sub": USER_A}) == [
-        {"id": "file-a", "cabinet_id": CAB_A, "uploaded_by": USER_A}
-    ]
-
-
-@pytest.mark.parametrize("router_module", ROUTERS)
-def test_list_files_denies_non_owner_even_with_cabinet_membership(monkeypatch, router_module):
-    store = {
-        "file_cabinets": [{"id": CAB_B, "user_id": USER_B}],
-        "cabinet_memberships": [
-            {"cabinet_id": CAB_B, "profile_id": USER_A, "role": "viewer"}
-        ],
-        "files": [{"id": "file-b", "cabinet_id": CAB_B, "uploaded_by": USER_B}],
-    }
-    monkeypatch.setattr(
-        router_module,
-        "SupabaseServiceRoleClient",
-        lambda: FakeServiceRoleClient(store),
-    )
-
-    assert router_module.list_files(CAB_B, {"sub": USER_A}) == []

tests/test_reset_environment_user_subset.py

@@ -1,51 +0,0 @@
-from run.initialization import reset_environment
-
-
-def test_reset_user_subset_scope_only_runs_user_subset_cleanup(monkeypatch):
-    calls = []
-
-    monkeypatch.setattr(
-        reset_environment,
-        "_sb_headers",
-        lambda: ("http://192.168.0.94:8000", {"Authorization": "Bearer redacted"}),
-    )
-    monkeypatch.setattr(
-        reset_environment,
-        "_assert_reset_allowed",
-        lambda url, scope: calls.append(("guard", url, scope)),
-    )
-    monkeypatch.setattr(
-        reset_environment,
-        "_clear_user_subset_files",
-        lambda: {"files_rows_deleted": 2, "storage_objects_removed": 2, "errors": []},
-    )
-
-    def fail_if_called(*_args, **_kwargs):
-        raise AssertionError("reset(scope='user-subset') must not clear unrelated tables or databases")
-
-    monkeypatch.setattr(reset_environment, "_clear_tables", fail_if_called)
-    monkeypatch.setattr(reset_environment, "_neo4j_drop_all_non_system", fail_if_called)
-    monkeypatch.setattr(reset_environment, "_clear_exam_storage", fail_if_called)
-
-    result = reset_environment.reset(scope="user-subset")
-
-    assert calls == [("guard", "http://192.168.0.94:8000", "user-subset")]
-    assert result == {
-        "scope": "user-subset",
-        "user_subset": {"files_rows_deleted": 2, "storage_objects_removed": 2, "errors": []},
-    }
-
-
-def test_reset_accepts_case_insensitive_user_subset_scope(monkeypatch):
-    monkeypatch.setattr(reset_environment, "_sb_headers", lambda: ("http://192.168.0.94:8000", {}))
-    monkeypatch.setattr(reset_environment, "_assert_reset_allowed", lambda *_args, **_kwargs: None)
-    monkeypatch.setattr(
-        reset_environment,
-        "_clear_user_subset_files",
-        lambda: {"files_rows_deleted": 0, "storage_objects_removed": 0, "errors": []},
-    )
-
-    assert reset_environment.reset(scope="USER-SUBSET") == {
-        "scope": "user-subset",
-        "user_subset": {"files_rows_deleted": 0, "storage_objects_removed": 0, "errors": []},
-    }

tests/test_seed_exam_corpus_unseed.py

@@ -1,171 +0,0 @@
-from run.initialization.seed_exam_corpus import LoadReport, _delete_user_subset_files
-
-
-class _Result:
-    def __init__(self, data=None):
-        self.data = data or []
-
-
-class _FilesQuery:
-    def __init__(self, db, op="select"):
-        self.db = db
-        self.op = op
-        self.filters = []
-        self.in_filters = []
-
-    def select(self, *_args, **_kwargs):
-        return self
-
-    def delete(self, *_args, **_kwargs):
-        self.op = "delete"
-        return self
-
-    def eq(self, key, value):
-        self.filters.append(("eq", key, value))
-        return self
-
-    def like(self, key, pattern):
-        self.filters.append(("like", key, pattern))
-        return self
-
-    def in_(self, key, values):
-        self.in_filters.append((key, set(values)))
-        return self
-
-    def _matches(self, row):
-        for kind, key, value in self.filters:
-            actual = row.get(key)
-            if kind == "eq" and actual != value:
-                return False
-            if kind == "like":
-                assert value.endswith("%")
-                if not isinstance(actual, str) or not actual.startswith(value[:-1]):
-                    return False
-        for key, values in self.in_filters:
-            if row.get(key) not in values:
-                return False
-        return True
-
-    def execute(self):
-        matched = [row for row in self.db.rows if self._matches(row)]
-        if self.op == "delete":
-            self.db.ops.append(("delete", [row["id"] for row in matched]))
-            self.db.rows = [row for row in self.db.rows if not self._matches(row)]
-            return _Result(matched)
-        return _Result(matched)
-
-
-class _FakeDb:
-    def __init__(self, rows):
-        self.rows = list(rows)
-        self.ops = []
-
-    def table(self, name):
-        assert name == "files"
-        return _FilesQuery(self)
-
-
-class _FakeStorageBucket:
-    def __init__(self, storage, bucket):
-        self.storage = storage
-        self.bucket = bucket
-
-    def remove(self, paths):
-        self.storage.ops.append(("remove", self.bucket, list(paths)))
-        if self.storage.fail:
-            raise RuntimeError("storage unavailable")
-        if self.storage.result_error:
-            return {"error": self.storage.result_error}
-        return []
-
-
-class _FakeStorageRoot:
-    def __init__(self, storage):
-        self.storage = storage
-
-    def from_(self, bucket):
-        return _FakeStorageBucket(self.storage, bucket)
-
-
-class _FakeStorage:
-    def __init__(self, fail=False, result_error=None):
-        self.fail = fail
-        self.result_error = result_error
-        self.ops = []
-        self.client = type("Client", (), {"supabase": type("SB", (), {"storage": _FakeStorageRoot(self)})()})()
-
-
-class _FakeClient:
-    def __init__(self, db):
-        self.supabase = db
-
-
-def test_delete_user_subset_storage_before_files_rows_for_scoped_exams():
-    db = _FakeDb([
-        {"id": "f1", "bucket": "cc.users", "path": "exam-marker/i/c/f1/A.pdf", "name": "A.pdf", "source": "exam-corpus-seed"},
-        {"id": "f2", "bucket": "cc.users", "path": "exam-marker/i/c/f2/B.pdf", "name": "B.pdf", "source": "exam-corpus-seed"},
-        {"id": "f3", "bucket": "cc.users", "path": "exam-marker/i/c/f3/A.pdf", "name": "A.pdf", "source": "manual"},
-        {"id": "f4", "bucket": "cc.users", "path": "other/f4/A.pdf", "name": "A.pdf", "source": "exam-corpus-seed"},
-    ])
-    storage = _FakeStorage()
-    rep = LoadReport()
-
-    _delete_user_subset_files(_FakeClient(db), storage, exam_codes=["A"], rep=rep)
-
-    assert storage.ops == [("remove", "cc.users", ["exam-marker/i/c/f1/A.pdf"])]
-    assert db.ops == [("delete", ["f1"])]
-    assert [row["id"] for row in db.rows] == ["f2", "f3", "f4"]
-    assert rep.unseed_objects == 1
-    assert rep.unseed_user_files == 1
-    assert rep.errors == []
-
-
-def test_delete_user_subset_keeps_files_rows_when_storage_remove_fails():
-    db = _FakeDb([
-        {"id": "f1", "bucket": "cc.users", "path": "exam-marker/i/c/f1/A.pdf", "name": "A.pdf", "source": "exam-corpus-seed"},
-    ])
-    storage = _FakeStorage(fail=True)
-    rep = LoadReport()
-
-    _delete_user_subset_files(_FakeClient(db), storage, exam_codes=["A"], rep=rep)
-
-    assert storage.ops == [("remove", "cc.users", ["exam-marker/i/c/f1/A.pdf"])]
-    assert db.ops == []
-    assert [row["id"] for row in db.rows] == ["f1"]
-    assert rep.unseed_objects == 0
-    assert rep.unseed_user_files == 0
-    assert rep.errors
-
-
-def test_delete_user_subset_keeps_files_rows_when_storage_remove_returns_error():
-    db = _FakeDb([
-        {"id": "f1", "bucket": "cc.users", "path": "exam-marker/i/c/f1/A.pdf", "name": "A.pdf", "source": "exam-corpus-seed"},
-    ])
-    storage = _FakeStorage(result_error="permission denied")
-    rep = LoadReport()
-
-    _delete_user_subset_files(_FakeClient(db), storage, exam_codes=["A"], rep=rep)
-
-    assert storage.ops == [("remove", "cc.users", ["exam-marker/i/c/f1/A.pdf"])]
-    assert db.ops == []
-    assert [row["id"] for row in db.rows] == ["f1"]
-    assert rep.unseed_objects == 0
-    assert rep.unseed_user_files == 0
-    assert rep.errors
-
-
-def test_delete_user_subset_unscoped_cleans_all_seeded_exam_marker_rows():
-    db = _FakeDb([
-        {"id": "f1", "bucket": "cc.users", "path": "exam-marker/i/c/f1/A.pdf", "name": "A.pdf", "source": "exam-corpus-seed"},
-        {"id": "f2", "bucket": "cc.users", "path": "exam-marker/i/c/f2/B.pdf", "name": "B.pdf", "source": "exam-corpus-seed"},
-    ])
-    storage = _FakeStorage()
-    rep = LoadReport()
-
-    _delete_user_subset_files(_FakeClient(db), storage, exam_codes=None, rep=rep)
-
-    assert storage.ops == [("remove", "cc.users", ["exam-marker/i/c/f1/A.pdf", "exam-marker/i/c/f2/B.pdf"])]
-    assert db.ops == [("delete", ["f1", "f2"])]
-    assert db.rows == []
-    assert rep.unseed_objects == 2
-    assert rep.unseed_user_files == 2

tests/test_upload_validation.py

@@ -1,54 +0,0 @@
-import asyncio
-
-import pytest
-from fastapi import HTTPException
-
-from modules.upload_validation import MAX_UPLOAD_BYTES, read_pdf_upload_bytes, read_upload_bytes
-
-
-class FakeUpload:
-    def __init__(self, data: bytes, content_type: str, filename: str = "file.bin"):
-        self._data = data
-        self._pos = 0
-        self.content_type = content_type
-        self.filename = filename
-
-    async def read(self, size: int = -1) -> bytes:
-        if self._pos >= len(self._data):
-            return b""
-        if size is None or size < 0:
-            size = len(self._data) - self._pos
-        chunk = self._data[self._pos : self._pos + size]
-        self._pos += len(chunk)
-        return chunk
-
-
-def run(coro):
-    return asyncio.run(coro)
-
-
-def test_valid_pdf_upload_passes_and_returns_mime():
-    data, mime = run(read_upload_bytes(FakeUpload(b"%PDF-1.7\n", "application/pdf")))
-    assert data.startswith(b"%PDF-")
-    assert mime == "application/pdf"
-
-
-def test_disallowed_mime_rejected_with_415():
-    with pytest.raises(HTTPException) as exc:
-        run(read_upload_bytes(FakeUpload(b"print(1)", "application/x-python")))
-    assert exc.value.status_code == 415
-    assert "Unsupported upload type" in exc.value.detail
-
-
-def test_oversize_upload_rejected_with_413():
-    with pytest.raises(HTTPException) as exc:
-        run(read_upload_bytes(FakeUpload(b"x" * (MAX_UPLOAD_BYTES + 1), "text/plain")))
-    assert exc.value.status_code == 413
-    assert "exceeds max size" in exc.value.detail
-
-
-def test_pdf_helper_rejects_spoofed_pdf_mime():
-    with pytest.raises(HTTPException) as exc:
-        run(read_pdf_upload_bytes(FakeUpload(b"not a pdf", "application/pdf")))
-    assert exc.value.status_code == 415
-    assert "not a valid PDF" in exc.value.detail