[verified] add docling auto-map package wrapper

api/services/docling/extract.py

@@ -0,0 +1,824 @@
+#!/usr/bin/env python3
+"""
+extract.py v2 — board-aware structured extraction of UK exam papers.
+
+v1 (see extract_v1_backup.py) proved a thin custom layer over Docling output recovers the
+exam-marker taxonomy at 95% on the image-only AQA Physics paper, but was AQA-only and read
+question labels from a RapidOCR per-page pass. v2 generalises across exam boards while
+*preserving* that proven AQA path:
+
+  * BOARD DETECTION   <- paper code in the front matter (8463/7408/8461 -> AQA, 1MA1 -> Edexcel,
+      H556 -> OCR). Each board uses a different numbering + marks grammar (PLAN.md D1).
+  * AQA              <- when Docling JSON + RapidOCR dumps are available, use v1's boxed-label
+      recovery (the 95% path). Otherwise fall back to the AQA text grammar.
+  * EDEXCEL          <- top-level integers anchored on "Total for Question N is M marks" (the
+      precise signal that kills false-positive content numbers like 0/24/62), parts (a)(b)(c),
+      per-part marks (N).
+  * OCR              <- sequential top-level integers followed by question text, parts (a)/(i),
+      marks [N]; `(b)*` flags an extended-response part.
+  * REGIONS          <- Docling layout labels mapped to taxonomy + gemma4:e4b `answer_regions`
+      (taxonomy #3 — the one structure no deterministic pass emits) merged by part.
+  * TABLES           <- Docling `tables` carried through; parts on a table page flagged has_table.
+  * COVERAGE         <- recall vs a ground-truth label set: built-in physics GT (regression guard)
+      or the born-digital GT text parsed with the same board grammar.
+
+The extractor works off a unified line stream so the same grammars serve both the OCR path
+(Docling JSON, has geometry) and the born-digital fast-path (pdftotext, no GPU).
+
+Usage:
+  python extract.py                                  # AQA physics, v1 path -> 95% (regression guard)
+  python extract.py --text results/gt_extra/edexcel-gcse-maths-1ma1-1h-jun22-qp.txt
+  python extract.py --text PAPER.pdf --gemma results/gemma_sweep_physics_200 --out out.json
+  python extract.py --ocr samples/extra/ocr-...-qp.pdf   # live OCR via dsync (uses shared GPU)
+  python extract.py --auto PAPER.pdf                     # detect text layer -> fast-path, else
+                                                         #   report the OCR path is required
+"""
+import json, re, glob, argparse, subprocess, os
+from collections import defaultdict, namedtuple
+import xml.etree.ElementTree as ET
+try:
+    from . import tables as tbl_mod
+except ImportError:  # pragma: no cover - CLI execution
+    import tables as tbl_mod
+
+# ----------------------------------------------------------------- line model
+Line = namedtuple("Line", "text page bbox")   # bbox is None for text-only sources
+
+
+def _union_bbox(boxes):
+    return {"l": min(b["l"] for b in boxes), "r": max(b["r"] for b in boxes),
+            "t": max(b["t"] for b in boxes), "b": min(b["b"] for b in boxes)}
+
+
+def _bbox_lines_from_pdftotext(path):
+    """Return (lines, pages) from `pdftotext -bbox`.
+
+    Poppler emits XHTML word boxes with a TOP-LEFT y-axis. The spike/app contract uses Docling-style
+    PDF points with a BOTTOM-LEFT origin, so convert every word/line box via page height:
+      l=xMin, r=xMax, t=page_height-yMin, b=page_height-yMax.
+    The text grammar still consumes line strings; grouping words on the same y band preserves enough
+    spacing for board grammars while adding geometry to the born-digital fast path.
+    """
+    raw = subprocess.check_output(["pdftotext", "-bbox", path, "-"]).decode("utf-8", "replace")
+    root = ET.fromstring(raw)
+    ns = {"x": "http://www.w3.org/1999/xhtml"}
+    out, pages = [], []
+    for pg, page in enumerate(root.findall(".//x:page", ns), 1):
+        width = float(page.get("width") or 0)
+        height = float(page.get("height") or 0)
+        pages.append({"page": pg, "width": width, "height": height,
+                      "bbox": {"l": 0.0, "r": width, "t": height, "b": 0.0}})
+        words = []
+        for w in page.findall("x:word", ns):
+            txt = (w.text or "").strip()
+            if not txt:
+                continue
+            x0, y0 = float(w.get("xMin") or 0), float(w.get("yMin") or 0)
+            x1, y1 = float(w.get("xMax") or 0), float(w.get("yMax") or 0)
+            bb = {"l": x0, "r": x1, "t": height - y0, "b": height - y1}
+            words.append((y0, x0, txt, bb))
+        words.sort()
+        groups = []
+        for y0, x0, txt, bb in words:
+            # Same baseline/text row. 3pt handles minor glyph-height jitter without merging rows.
+            if not groups or abs(groups[-1]["y0"] - y0) > 3.0:
+                groups.append({"y0": y0, "words": []})
+            groups[-1]["words"].append((x0, txt, bb))
+        for g in groups:
+            g["words"].sort(key=lambda x: x[0])
+            text = " ".join(txt for _, txt, _ in g["words"])
+            out.append(Line(text, pg, _union_bbox([bb for _, _, bb in g["words"]])))
+    return out, pages
+
+
+def lines_from_pdftext(path):
+    """Born-digital fast-path / GT source: pdftotext, with word/line bbox for PDFs."""
+    if path.endswith(".pdf"):
+        return _bbox_lines_from_pdftotext(path)[0]
+    raw = open(path, encoding="utf-8", errors="replace").read()
+    out = []
+    for pg, page in enumerate(raw.split("\f"), 1):
+        for ln in page.splitlines():
+            if ln.strip():
+                out.append(Line(ln, pg, None))
+    return out
+
+
+def pages_from_pdftext(path):
+    if path and path.endswith(".pdf"):
+        return _bbox_lines_from_pdftotext(path)[1]
+    return []
+
+
+def _prefix_bbox(line, width=52):
+    """Approximate the leading label box within a pdftotext-bbox line.
+
+    The fast-path line bbox spans the full text row (label + question prose). For template/overlay use,
+    part geometry should mark the label at the row start, not the whole row. Poppler word geometry is
+    currently collapsed to line boxes, so keep the row's vertical extent and cap the horizontal extent
+    to the left prefix where exam-board labels live.
+    """
+    if not line.bbox:
+        return None
+    return {"l": line.bbox["l"], "r": min(line.bbox["r"], line.bbox["l"] + width),
+            "t": line.bbox["t"], "b": line.bbox["b"]}
+
+
+# ----------------------------------------------------------------- text-layer auto-detect
+# Every UK exam-board QP PDF ships a text layer (born-digital), making pdftotext the common-case
+# production path: no GPU, no OCR. The only exception in practice is a *redistribution* that has
+# been re-rendered to images (e.g. samples/AQA-Physics-Paper-1H-2022-with-qr.pdf), which carries
+# NO text layer and must go through the OCR pipeline. We pick the path automatically by measuring
+# how much real text pdftotext recovers, normalised per page.
+#
+# Calibration (measured on the corpus, non-whitespace chars / page from `pdftotext -layout`):
+#   image-only AQA-Physics-...-with-qr.pdf .....   0  -> OCR path
+#   edexcel 1MA1/1H (sparsest born-digital) ....  ~326
+#   every other born-digital QP ................  400-1200
+# A born-digital QP yields hundreds of chars/page; an image-only PDF yields ~0 (a stray QR/footer
+# might leak a handful). 40 chars/page sits an order of magnitude below the sparsest real paper
+# and well above any image-only leakage, so it cleanly separates the two with wide margin.
+TEXT_LAYER_MIN_CHARS_PER_PAGE = 40
+
+
+def text_layer_chars_per_page(path):
+    """Return (total_non_space_chars, n_pages, chars_per_page) for a PDF's pdftotext output.
+
+    chars_per_page is the auto-detect signal: it normalises out paper length so a long sparse
+    paper isn't mistaken for image-only and a short dense one isn't over-counted."""
+    raw = subprocess.check_output(["pdftotext", "-layout", path, "-"]).decode("utf-8", "replace")
+    chars = sum(1 for c in raw if not c.isspace())
+    n_pages = raw.count("\f") + 1                # pdftotext emits a form-feed after each page
+    return chars, n_pages, (chars / n_pages if n_pages else 0)
+
+
+def has_text_layer(path):
+    """True if `path` is a born-digital PDF (substantive text layer) suitable for the fast-path.
+
+    A PDF re-rendered to images (the scanned/image-only redistribution case) returns False and
+    must be routed to the OCR pipeline (--ocr / the AQA RapidOCR margin-pass)."""
+    _, _, cpp = text_layer_chars_per_page(path)
+    return cpp >= TEXT_LAYER_MIN_CHARS_PER_PAGE
+
+
+def lines_from_docling(doc):
+    """OCR path: one line per Docling text item, in reading order, carrying page + bbox."""
+    items = []
+    for t in doc.get("texts", []):
+        prov = t.get("prov") or []
+        if not prov:
+            items.append(Line(t.get("text") or "", None, None)); continue
+        page, bb = prov[0].get("page_no"), prov[0].get("bbox")
+        items.append(Line(t.get("text") or "", page, bb))
+    # reading order: page, then top-down (Docling bbox origin is bottom-left -> larger t = higher)
+    items.sort(key=lambda l: (l.page or 0, -((l.bbox or {}).get("t", 0)), (l.bbox or {}).get("l", 0)))
+    return items
+
+
+# ----------------------------------------------------------------- board detection
+PAPER_CODE_RES = [
+    ("aqa",     re.compile(r"\b(7408|8463|8461|8464|8462|8702|8700|7405|7402)/\d")),
+    ("edexcel", re.compile(r"\b1MA1/\d", re.I)),
+    ("ocr",     re.compile(r"\bH\d{3}/?\d?\b")),
+]
+WORDMARK_RES = [
+    ("edexcel", re.compile(r"Pearson|Edexcel", re.I)),
+    ("ocr",     re.compile(r"Oxford Cambridge and RSA|\bOCR\b")),
+    ("aqa",     re.compile(r"\bAQA\b")),
+]
+# structural grammar signals — the board-specific tokens themselves. These survive OCR far better
+# than cover-page branding / paper codes (which mangle: "1MA1/1F" -> "IMA1 1", "Pearson Edexcel"
+# split across lines), so they're the robust fallback before wordmarks.
+EDX_SIG = re.compile(r"Total for Question\s+\d+\s+is\s+\d+\s+marks?", re.I)
+OCR_SIG = re.compile(r"Oxford Cambridge and RSA", re.I)
+AQA_SIG = re.compile(r"\[\s*\d+\s*marks?\s*\]")              # [N marks] — AQA, not OCR's bare [N]
+
+
+def detect_board(lines):
+    """Return (board, paper_code|None). Order: paper code (authoritative) -> structural grammar
+    signal (OCR-robust) -> wordmark -> default."""
+    blob = "\n".join(l.text for l in lines[:1500])           # whole front + body, not just cover
+    for board, rx in PAPER_CODE_RES:
+        m = rx.search(blob)
+        if m:
+            return board, m.group(0)
+    if EDX_SIG.search(blob):
+        return "edexcel", None
+    if OCR_SIG.search(blob):
+        return "ocr", None
+    if len(AQA_SIG.findall(blob)) >= 3:
+        return "aqa", None
+    for board, rx in WORDMARK_RES:
+        if rx.search(blob):
+            return board, None
+    return "aqa", None   # safe default
+
+
+# ----------------------------------------------------------------- front matter
+def extract_front_matter(lines, board, code):
+    blob = "\n".join(l.text for l in lines[:400])
+    fm = {"exam_board": {"aqa": "AQA", "edexcel": "Pearson Edexcel", "ocr": "OCR"}[board]}
+    if code:
+        fm["paper_code"] = code
+    m = re.search(r"\b(GCSE|GCE|A-?level|AS)\b\s+([A-Z][A-Za-z ]+)", blob)
+    if m:
+        fm["qualification"] = m.group(1).upper().replace("-", "")
+        fm["subject"] = m.group(2).split("\n")[0].strip().title()
+    m = re.search(r"(Higher|Foundation)\s+Tier", blob, re.I)
+    if m:
+        fm["tier"] = m.group(1).title()
+    m = re.search(r"Time\s+allowed[:\s]+([0-9].*?(?:hour|minute)s?[^\n]*)", blob, re.I)
+    if m:
+        fm["time_allowed"] = m.group(1).strip()
+    # authoritative paper-total phrasings first, then the generic fallback
+    m = (re.search(r"TOTAL FOR PAPER IS\s+(\d{2,3})\s+MARKS", blob, re.I)
+         or re.search(r"total mark for this paper is\s+(\d{2,3})", blob, re.I)
+         or re.search(r"(?:maximum mark|maximum mark for this paper)\D{0,8}(\d{2,3})", blob, re.I))
+    if m:
+        fm["max_marks"] = int(m.group(1))
+    m = re.search(r"(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\w*\s*\.?\s*(\d{2,4})", blob)
+    if m:
+        fm["session"] = f"{m.group(1)} 20{m.group(2)[-2:]}"
+    return fm
+
+
+# ====================================================================== AQA
+# --- v1 path: Docling JSON + RapidOCR boxed labels (the proven 95% recovery) -----
+PART_RE = re.compile(r"^(\d{2})\.(\d)$")     # 01.2
+NUM_RE  = re.compile(r"^(\d{2})$")           # 08
+DIG_RE  = re.compile(r"^(\d)$")              # 4
+# A-level papers (7408) render the boxed label GLUED to the question text in one OCR token
+# ("01.1 An atom of ...") rather than as a standalone box (GCSE 8463). Recover it as a prefix,
+# but only inside the tight left label column — body text (incl. decimals like "10.5 N") sits
+# at l>=92, so this column gate is the precision filter that keeps false positives out.
+# real part decimals run .1-.9 (never .0), so [1-9] rejects decimal content like "10.0" that
+# happens to land in the label column; ".0" is reserved for our MCQ top-level placeholders.
+PART_PREFIX = re.compile(r"^\s*(\d)\s*(\d)\s*\.\s*([1-9])(?=\s|$)")   # "01.1 ..." / "0 1 . 1 ..."
+LABEL_COL_MAX = 75                                                  # left edge of the label box
+MIN_MCQ_RUN = 5         # a real Section-B MCQ run is long+contiguous; fewer = stray page numbers
+FOOTER_T = 60           # bbox bottom-left origin: t<~30 is the page-number footer, not content
+
+
+# A-level Section B is multiple-choice: bare sequential top-level numbers ("07 Which two...",
+# or a lone "07") with no decimal part. They render glued in the label column. The sequence
+# gate (each accepted number == previous + 1) is the precision filter that rejects OCR noise
+# (misread "60", "10 6") — exactly as the OCR-board grammar gates bare integers.
+MCQ_TOP = re.compile(r"^(\d{2})(?:\s+[A-Z(].*|\s*)$")
+
+
+def _rapid_pages(rapid_glob):
+    """Yield (page_no, doc) in NUMERIC page order (glob sorts lexically: p1,p10,p2...)."""
+    files = sorted(glob.glob(rapid_glob),
+                   key=lambda f: int(re.search(r"p(\d+)\.json", f).group(1)))
+    for fn in files:
+        pg = int(re.search(r"p(\d+)\.json", fn).group(1))
+        yield pg, json.load(open(fn))
+
+
+def aqa_questions_rapid(rapid_glob):
+    """Recover question labels from per-page RapidOCR dumps. Handles three AQA layouts:
+      * GCSE standalone label/number boxes (8463 — v1's NN.M + NUM/DIG pairing),
+      * A-level structured parts glued as a prefix ("01.1 An atom of..." — label column),
+      * A-level Section-B multiple choice: bare sequential top-levels -> NN.0."""
+    parts = {}
+    mcq_cands = []                       # (page, NN, bbox) bare top-level candidates, in order
+    for pg, d in _rapid_pages(rapid_glob):
+        margin = []
+        for t in d.get("texts", []):
+            raw = (t.get("text") or "").strip()
+            s = raw.replace(" ", "")
+            prov = t.get("prov") or []
+            bb = prov[0].get("bbox") if prov else None
+            if bb is None or bb["l"] > 140:
+                continue
+            margin.append((bb, s))
+            m = PART_RE.match(s)
+            if m and m.group(2) != "0":
+                parts.setdefault(f"{m.group(1)}.{m.group(2)}", {"page": pg, "bbox": bb})
+            elif bb["l"] <= LABEL_COL_MAX:
+                mp = PART_PREFIX.match(raw)
+                if mp:
+                    parts.setdefault(f"{mp.group(1)}{mp.group(2)}.{mp.group(3)}",
+                                     {"page": pg, "bbox": bb})
+                elif bb["t"] >= FOOTER_T:          # skip page-number footers (page N -> "N")
+                    mc = MCQ_TOP.match(raw)
+                    if mc:
+                        mcq_cands.append((pg, mc.group(1), bb))
+        nums = [(bb, NUM_RE.match(s).group(1)) for bb, s in margin if NUM_RE.match(s)]
+        digs = [(bb, DIG_RE.match(s).group(1)) for bb, s in margin if DIG_RE.match(s)]
+        for nbb, nn in nums:
+            ny = (nbb["t"] + nbb["b"]) / 2
+            for dbb, dd in digs:
+                dy = (dbb["t"] + dbb["b"]) / 2
+                if abs(ny - dy) < 12 and dbb["l"] >= nbb["l"]:
+                    parts.setdefault(f"{nn}.{dd}", {"page": pg, "bbox": nbb})
+    # Section B: walk MCQ candidates in reading order, accept the next number in sequence only
+    structured_q = {int(lab.split(".")[0]) for lab in parts}
+    expect = (max(structured_q) + 1) if structured_q else 1
+    mcq_cands.sort(key=lambda c: (c[0], -(c[2] or {}).get("t", 0)))   # page, then top-down
+    cand = {}                            # nn -> (page, bbox), first occurrence in reading order
+    for pg, nn, bb in mcq_cands:
+        cand.setdefault(int(nn), (pg, bb))
+    # Walk the sequence: take the exact expected number when present; only jump a small gap
+    # (<=3) when it's genuinely absent (OCR-garbled, e.g. "09"->"60") so one bad number doesn't
+    # truncate the section. Out-of-window noise (misread "60") never enters.
+    seq = []
+    while True:
+        if expect in cand and expect not in structured_q:
+            seq.append((expect, cand[expect]))
+            expect += 1
+            continue
+        nxt = [n for n in cand if expect < n <= expect + 3 and n not in structured_q]
+        if nxt:
+            expect = min(nxt)
+            continue
+        break
+    # Only commit if this is a real Section-B MCQ run, not a few stray page/figure numbers on a
+    # paper with no MCQ section (a GCSE structured paper yields a short, gappy set; a real MCQ
+    # section is a long contiguous run).
+    if len(seq) >= MIN_MCQ_RUN:
+        for n, (pg, bb) in seq:
+            parts.setdefault(f"{n:02d}.0", {"page": pg, "bbox": bb})
+    # In the rapid path every ".0" label is a Section-B multiple-choice question, worth 1 mark
+    # each (they carry no "[N marks]" token for geometry to bind). Structured parts stay None
+    # until attach_marks_by_geometry fills them from the marks list.
+    return {lab: {"q": lab.split(".")[0], "page": v["page"], "bbox": v["bbox"],
+                  "marks": (1 if lab.endswith(".0") else None), "regions": []}
+            for lab, v in parts.items()}
+
+
+# --- AQA text grammar (born-digital AQA papers, e.g. 7408 with a text layer) ------
+AQA_MARK = re.compile(r"\[\s*(\d+)\s*marks?\s*\]", re.I)
+
+
+# AQA boxed labels render SPACE-SEPARATED in pdftotext ("0 1 . 1"); decimal content
+# ("10.5 N") does not. `pdftotext -bbox` normalises gaps to single spaces, while `-layout`
+# preserved wider runs, so the top-box grammar tolerates either one-or-more spaces.
+AQA_PART_BOX = re.compile(r"^\s*(\d)\s+(\d)\s*\.\s*(\d)(?=\s|$)")        # 0 1 . 1
+AQA_TOP_BOX  = re.compile(r"^\s*(\d)\s+(\d)\s+(?=[A-Z(])")              # 0 2 Carbon...
+
+
+def aqa_questions_text(lines):
+    parts = {}
+    cur = None
+    for l in lines:
+        mp = AQA_PART_BOX.match(l.text)
+        if mp:
+            q = f"{mp.group(1)}{mp.group(2)}"
+            lab = f"{q}.{mp.group(3)}"
+            cur = parts.setdefault(lab, {"q": q, "page": l.page, "bbox": _prefix_bbox(l),
+                                         "marks": None, "regions": []})
+        else:
+            mt = AQA_TOP_BOX.match(l.text)
+            if mt:
+                q = f"{mt.group(1)}{mt.group(2)}"
+                cur = parts.setdefault(f"{q}.0", {"q": q, "page": l.page, "bbox": _prefix_bbox(l),
+                                                  "marks": None, "regions": []})
+        mm = AQA_MARK.search(l.text)
+        if mm and cur is not None and cur.get("marks") is None:
+            cur["marks"] = int(mm.group(1))
+    # drop a placeholder ".0" part if the same question also has real numbered parts
+    for q in {v["q"] for v in parts.values()}:
+        if f"{q}.0" in parts and any(parts[k]["q"] == q and k != f"{q}.0" for k in parts):
+            parts.pop(f"{q}.0")
+    return parts
+
+
+# ====================================================================== Edexcel
+EDX_TOTAL = re.compile(r"Total for Question\s+(\d+)\s+is\s+(\d+)\s+marks?", re.I)
+EDX_LEAD  = re.compile(r"^\s*(\d{1,2})\s+(.*)$")        # number, gap, then the rest of the line
+EDX_PART  = re.compile(r"\(([a-h])\)")                  # may appear inline after the number
+EDX_SUB   = re.compile(r"^\s*\(([ivx]{1,4})\)")
+EDX_MARK  = re.compile(r"^\s*\((\d+)\)\s*$")
+
+
+def edexcel_questions(lines):
+    # anchor top-level numbers on the robust "Total for Question N is M" signal (precision)
+    anchors = {}            # qnum -> (total marks, anchor line)
+    for l in lines:
+        m = EDX_TOTAL.search(l.text)
+        if m:
+            anchors[int(m.group(1))] = (int(m.group(2)), l)
+    parts = {}
+    haspart = set()         # questions that own lettered parts
+    curq = curlet = lastlab = None
+
+    def add(lab, q, l):
+        nonlocal lastlab
+        parts.setdefault(lab, {"q": q, "page": l.page, "bbox": _prefix_bbox(l, 40), "marks": None, "regions": []})
+        lastlab = lab
+
+    for l in lines:
+        if EDX_TOTAL.search(l.text):
+            curq = curlet = None
+            continue
+        ml = EDX_LEAD.match(l.text)
+        if ml and int(ml.group(1)) in anchors and (ml.group(2)[:1].isupper()
+                                                    or ml.group(2).lstrip().startswith("(")):
+            curq, rest = ml.group(1), ml.group(2)
+            curlet = None
+            inline = EDX_PART.search(rest)             # capture "(a)" sharing the lead line
+            if inline:
+                curlet = inline.group(1); haspart.add(curq); add(f"{curq}{curlet}", curq, l)
+            continue
+        if curq is None:
+            continue
+        mp = EDX_PART.match(l.text.lstrip())
+        if mp:
+            curlet = mp.group(1); haspart.add(curq); add(f"{curq}{curlet}", curq, l)
+        ms = EDX_SUB.match(l.text)
+        if ms and curlet:
+            add(f"{curq}{curlet}{ms.group(1)}", curq, l)
+        mm = EDX_MARK.match(l.text)
+        if mm and lastlab:
+            parts[lastlab]["marks"] = int(mm.group(1))
+    # part-less questions: one part carrying the authoritative Total-for-Question mark
+    for q, (total, anchor_line) in anchors.items():
+        if str(q) not in haspart:
+            parts.setdefault(str(q), {"q": str(q), "page": anchor_line.page,
+                                      "bbox": _prefix_bbox(anchor_line, 40),
+                                      "marks": total, "regions": []})
+    return parts, {}, anchors
+
+
+# ====================================================================== OCR
+OCR_PART = re.compile(r"^\s*\(([a-h])\)")
+OCR_SUB  = re.compile(r"^\s*\(([ivx]{1,4})\)")
+OCR_MARK = re.compile(r"\[(\d+)\]")
+OCR_EXT  = re.compile(r"^\s*\(([a-h])\)\s*\*|^\s*(\d{1,2})\s*\*")
+
+
+def ocr_questions(lines):
+    parts = {}
+    curq = curlet = None
+    expect = 1
+    inferred = 0          # OCR may drop the margin question number; infer from part structure
+    for l in lines:
+        # top-level = the NEXT integer in sequence, gap, then question text OR a part opener "(a)"
+        # (Q3 opens straight into (a)). Sequence gate = the precision filter.
+        ml = re.match(r"^\s*(\d{1,2})\s+(\(|[A-Z])", l.text)
+        if ml and int(ml.group(1)) == expect:
+            curq = ml.group(1); curlet = None; expect += 1
+            parts.setdefault(curq, {"q": curq, "page": l.page, "bbox": _prefix_bbox(l, 36),
+                                    "marks": None, "regions": [], "_lead": True})
+        if curq is None:
+            # number was OCR-dropped: start an inferred question on its first part "(a)"
+            m0 = OCR_PART.match(l.text.lstrip())
+            if m0 and m0.group(1) == "a":
+                inferred += 1; curq = f"~{inferred}"; curlet = None
+            else:
+                continue
+        ext = bool(re.search(r"\(\s*[a-h]\s*\)\s*\*", l.text))
+        mp = OCR_PART.match(l.text)
+        if mp:
+            # a repeat "(a)" while this question already owns one => next question, number dropped
+            if mp.group(1) == "a" and f"{curq}a" in parts:
+                inferred += 1; curq = f"~{inferred}"
+            curlet = mp.group(1)
+            parts.pop(curq, None)
+            lab = f"{curq}{curlet}"
+            parts.setdefault(lab, {"q": curq, "page": l.page, "bbox": _prefix_bbox(l, 36),
+                                   "marks": None, "regions": [], "extended": ext})
+        ms = OCR_SUB.match(l.text)
+        if ms and curlet:
+            lab = f"{curq}{curlet}{ms.group(1)}"
+            parts.setdefault(lab, {"q": curq, "page": l.page, "bbox": _prefix_bbox(l, 36),
+                                   "marks": None, "regions": [], "extended": ext})
+        mm = OCR_MARK.search(l.text)
+        if mm:
+            sib = [k for k in parts if parts[k]["q"] == curq and not parts[k].get("_lead")]
+            if sib:
+                parts[sib[-1]]["marks"] = int(mm.group(1))
+    for v in parts.values():
+        v.pop("_lead", None)
+    return parts
+
+
+# ====================================================================== shared layers
+LABEL_TO_TAXONOMY = {
+    "checkbox_unselected": "mcq_option", "checkbox_selected": "mcq_option",
+    "picture": "context_figure", "table": "context_data", "caption": "context_caption",
+    "page_header": "furniture", "page_footer": "furniture",
+    "section_header": "heading", "list_item": "instruction",
+}
+
+
+def docling_regions(doc):
+    regions = []
+    for key in ("texts", "pictures", "tables"):
+        for it in doc.get(key, []):
+            lab = it.get("label", key[:-1])
+            tax = LABEL_TO_TAXONOMY.get(lab)
+            if not tax:
+                continue
+            prov = it.get("prov") or []
+            bb = prov[0].get("bbox") if prov else None
+            pg = prov[0].get("page_no") if prov else None
+            if bb is None:
+                continue
+            regions.append({"type": tax, "docling_label": lab, "page": pg, "bbox": bb,
+                            "text": (it.get("text") or "")[:80]})
+    return regions
+
+
+def merge_gemma(parts, gemma_dir):
+    """Attach gemma4:e4b answer_regions (#3) to parts by for_part; gap-fill missing marks."""
+    n_reg = n_fill = 0
+    for fn in sorted(glob.glob(os.path.join(gemma_dir, "p*.json"))):
+        d = json.load(open(fn))
+        for r in d.get("answer_regions", []):
+            lab = _norm_label(r.get("for_part", ""))
+            if lab in parts:
+                parts[lab]["regions"].append({"type": r.get("kind", "answer_lines"),
+                                              "source": "gemma"})
+                n_reg += 1
+        for qp in d.get("question_parts", []):
+            lab = _norm_label(qp.get("label", ""))
+            if lab in parts and parts[lab].get("marks") is None and qp.get("marks") is not None:
+                parts[lab]["marks"] = qp["marks"]; n_fill += 1
+    return n_reg, n_fill
+
+
+def _norm_label(s):
+    """gemma sometimes emits '0_4' or '01_2' for AQA -> '01.4'/'01.2'."""
+    s = (s or "").strip().replace("_", ".")
+    m = re.match(r"^(\d)\.(\d)$", s)
+    if m:                                  # '0.4' -> drop, ambiguous; keep as-is otherwise
+        return s
+    return s
+
+
+def extract_tables(parts, doc, granite="off", pdf=None, cache_glob=None):
+    """Selective table-cell extraction (PLAN.md §B): standard TableFormer grids always; Granite
+    <otsl> on router-flagged pages when granite!='off'. Returns (data_tables, all_tables).
+    Granite tables win over the standard pipeline on pages it covers (cleaner grids)."""
+    std = tbl_mod.tables_from_standard(doc)
+    gran = []
+    if granite != "off":
+        pages = tbl_mod.candidate_pages(doc)
+        if granite == "cached":
+            cache = tbl_mod._load_cached_doctags(cache_glob or "")
+            for pg in pages:
+                for t in tbl_mod.parse_otsl(cache.get(pg, "")):
+                    t["page"] = pg; gran.append(t)
+        elif granite == "live" and pdf:
+            gran = tbl_mod.granite_tables(pdf, pages, cached_glob=cache_glob)
+    gran_pages = {t["page"] for t in gran}
+    combined = gran + [t for t in std if t["page"] not in gran_pages]
+    data = tbl_mod.attach_to_questions(combined, parts)
+    for v in parts.values():
+        if v.get("tables"):
+            v["has_table"] = True
+    return data, combined
+
+
+def attach_marks_by_geometry(parts, doc):
+    """AQA Docling path: marks live in a flat [N marks] list; bind each to the nearest
+    preceding part on the same page by vertical position."""
+    marks = []
+    for t in doc.get("texts", []):
+        prov = t.get("prov") or []
+        bb = prov[0].get("bbox") if prov else None
+        pg = prov[0].get("page_no") if prov else None
+        for m in AQA_MARK.finditer(t.get("text") or ""):
+            marks.append((pg, bb, int(m.group(1))))
+    by_page = defaultdict(list)
+    for lab, v in parts.items():
+        if v.get("page") is not None:
+            by_page[v["page"]].append((lab, v))
+    n = 0
+    for pg, bb, val in marks:
+        cands = by_page.get(pg, [])
+        if not cands or bb is None:
+            continue
+        my = (bb["t"] + bb["b"]) / 2
+        best = min(cands, key=lambda kv: abs(((kv[1]["bbox"] or {}).get("t", 0)
+                                              + (kv[1]["bbox"] or {}).get("b", 0)) / 2 - my)
+                   if kv[1].get("bbox") else 1e9)
+        if best[1].get("marks") is None:
+            best[1]["marks"] = val; n += 1
+    return n, marks
+
+
+# ----------------------------------------------------------------- assembly + coverage
+def build_questions(parts):
+    qs = defaultdict(list)
+    for lab in parts:
+        qs[parts[lab]["q"]].append(lab)
+    out = []
+    for q in sorted(qs, key=lambda x: (len(x), x)):
+        plist = sorted(qs[q])
+        out.append({
+            "question": q,
+            "parts": [{"label": lab, "page": parts[lab].get("page"),
+                       "bbox": parts[lab].get("bbox"),   # label geometry (None for born-digital text)
+                       "marks": parts[lab].get("marks"),
+                       "regions": parts[lab].get("regions", []),
+                       "has_table": parts[lab].get("has_table", False),
+                       "extended": parts[lab].get("extended", False)} for lab in plist],
+        })
+    return out
+
+
+GT_PARTS_PHYSICS = ["01.1","01.2","01.3","01.4","02.1","02.2","02.3","02.4","03.1","03.2","03.3","03.4",
+    "04.1","04.2","04.3","04.4","04.5","05.1","05.2","05.3","05.4","06.1","06.2","06.3",
+    "07.1","07.2","07.3","08.1","08.2","08.3","08.4","08.5","09.1","09.2","09.3",
+    "10.1","10.2","10.3","11.1","11.2","11.3","11.4"]
+
+# official paper maxima — the strongest grammar sanity check (marks_sum should match)
+EXPECTED_MAX = {"8463": 100, "7408": 85, "8461": 100, "1MA1": 80, "H556": 70}
+
+
+def expected_max(code):
+    if not code:
+        return None
+    for k, v in EXPECTED_MAX.items():
+        if code.startswith(k):
+            return v
+    return None
+
+
+def parse_text_by_board(lines, board):
+    """Run the board grammar over a line stream -> parts dict (used for GT + born-digital)."""
+    if board == "edexcel":
+        parts, _, _ = edexcel_questions(lines); return parts
+    if board == "ocr":
+        return ocr_questions(lines)
+    return aqa_questions_text(lines)
+
+
+def coverage(parts, gt_labels):
+    rec = set(parts)
+    hit = sorted(rec & set(gt_labels))
+    miss = sorted(set(gt_labels) - rec)
+    return {"coverage_pct": round(len(hit) / len(gt_labels) * 100, 1) if gt_labels else None,
+            "recovered": len(hit), "total": len(gt_labels), "missed": miss}
+
+
+# ----------------------------------------------------------------- main
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--text", help="pdftotext source or .txt (born-digital / GT path)")
+    ap.add_argument("--auto", help="PDF: auto-detect text layer -> fast-path (--text), else "
+                                   "report the OCR path is required (no GPU work attempted here)")
+    ap.add_argument("--ocr", help="PDF to OCR live via dsync (uses the shared GPU)")
+    ap.add_argument("--docling", help="cached Docling JSON (OCR path without re-running dsync)")
+    ap.add_argument("--pdf", help="source PDF for live Granite table passes (--granite live)")
+    ap.add_argument("--rapid", help="AQA RapidOCR per-page glob (the v1 95% path)")
+    ap.add_argument("--gemma", help="gemma sweep dir with p*.json answer_regions")
+    ap.add_argument("--marks-fill", dest="marks_fill",
+                    help="gemma_marks.py fills JSON: fill marks=None parts (Edexcel/OCR (N)/[N] gap-fill)")
+    ap.add_argument("--granite", default="off", choices=["off", "cached", "live"],
+                    help="selective Granite-Docling <otsl> tables: cached doctags or live via dsync")
+    ap.add_argument("--granite-cache", default="results/VLM_granite_p*.doctags",
+                    help="glob of cached *.doctags for --granite cached / live fallback")
+    ap.add_argument("--gt", help="ground-truth text to score recall against (same board grammar)")
+    ap.add_argument("--board", default="auto", choices=["auto", "aqa", "edexcel", "ocr"])
+    ap.add_argument("--out", default="results/structured.json")
+    a = ap.parse_args()
+
+    # --- auto path selection -------------------------------------------------------------
+    # Caller need not know in advance whether the PDF is born-digital or image-only: detect the
+    # text layer and either fold --auto into the fast-path (--text) or report that the OCR path
+    # is required. This is ADDITIVE — it only resolves --auto; every other mode is untouched.
+    if a.auto:
+        chars, n_pages, cpp = text_layer_chars_per_page(a.auto)
+        if cpp >= TEXT_LAYER_MIN_CHARS_PER_PAGE:
+            print(f"auto-detect         : born-digital text layer "
+                  f"({chars} chars / {n_pages} pages = {cpp:.0f} chars/page "
+                  f">= {TEXT_LAYER_MIN_CHARS_PER_PAGE}) -> fast-path (pdftotext, no GPU)")
+            a.text = a.auto
+        else:
+            print(f"auto-detect         : NO usable text layer "
+                  f"({chars} chars / {n_pages} pages = {cpp:.1f} chars/page "
+                  f"< {TEXT_LAYER_MIN_CHARS_PER_PAGE}) -> OCR path required")
+            print("route               : run the OCR pipeline, e.g.")
+            print(f"                      python extract.py --ocr {a.auto}")
+            print("                      (AQA image-only papers use the RapidOCR margin-pass; "
+                  "see scripts/rapid_pass.py)")
+            return
+
+    # default invocation == v1 AQA physics regression guard
+    if not (a.text or a.ocr or a.docling):
+        a.docling = "results/E_tess_full.json"
+        a.rapid = a.rapid or "results/rapid_pages/p*.json"
+        a.gemma = a.gemma or "results/gemma_sweep_physics_200"
+        a.pdf = a.pdf or "samples/AQA-Physics-Paper-1H-2022-with-qr.pdf"
+        a.out = "results/physics_structured.json" if a.out == "results/structured.json" else a.out
+
+    doc = None
+    pages = []
+    if a.ocr:
+        try:
+            from . import dsync
+        except ImportError:  # pragma: no cover - CLI execution
+            import dsync
+        doc = dsync.convert_document(a.ocr, {"ocr_engine": "tesseract", "force_ocr": True})
+        lines = lines_from_docling(doc)
+    elif a.docling:
+        doc = json.load(open(a.docling))
+        lines = lines_from_docling(doc)
+    else:
+        if a.text and a.text.endswith(".pdf"):
+            lines, pages = _bbox_lines_from_pdftotext(a.text)
+        else:
+            lines = lines_from_pdftext(a.text)
+
+    board, code = detect_board(lines)
+    if a.board != "auto":
+        board = a.board
+    fm = extract_front_matter(lines, board, code)
+
+    # --- questions: AQA uses the proven Docling+RapidOCR path when inputs exist ----------
+    if board == "aqa" and a.rapid and glob.glob(a.rapid):
+        parts = aqa_questions_rapid(a.rapid)
+        path_used = "aqa-docling+rapidocr (v1)"
+    else:
+        parts = parse_text_by_board(lines, board)
+        path_used = f"{board}-text-grammar"
+
+    # --- shared enrichment ---------------------------------------------------------------
+    regions = docling_regions(doc) if doc else []
+    n_mark_geo = 0
+    if doc and board == "aqa":
+        n_mark_geo, _ = attach_marks_by_geometry(parts, doc)
+    data_tables, all_tables = ([], [])
+    if doc:
+        data_tables, all_tables = extract_tables(parts, doc, granite=a.granite,
+                                                 pdf=(a.pdf or a.ocr), cache_glob=a.granite_cache)
+    n_tbl = sum(1 for v in parts.values() if v.get("has_table"))
+    tbl_pages = sorted({t["page"] for t in data_tables if t["page"]})
+    n_reg = n_fill = 0
+    if a.gemma and os.path.isdir(a.gemma):
+        n_reg, n_fill = merge_gemma(parts, a.gemma)
+    n_marks_fill = 0
+    if a.marks_fill and os.path.exists(a.marks_fill):
+        fills = json.load(open(a.marks_fill)).get("fills", {})
+        for lab, mk in fills.items():
+            if lab in parts and parts[lab].get("marks") is None:
+                parts[lab]["marks"] = int(mk); n_marks_fill += 1
+
+    questions = build_questions(parts)
+
+    # --- coverage ------------------------------------------------------------------------
+    if a.gt:
+        gt_lines = lines_from_pdftext(a.gt)
+        gt_parts = parse_text_by_board(gt_lines, board)
+        cov = coverage(parts, list(gt_parts))
+        cov["source"] = "gt-text-same-grammar"
+    elif board == "aqa" and "rapidocr" in path_used:
+        cov = coverage(parts, GT_PARTS_PHYSICS)
+        cov["source"] = "builtin-physics-gt"
+    else:
+        cov = {"coverage_pct": None, "note": "no GT provided"}
+
+    marks_known = sum(1 for v in parts.values() if v.get("marks") is not None)
+    marks_sum = sum(v["marks"] for v in parts.values() if v.get("marks") is not None)
+    exp_max = expected_max(code) or fm.get("max_marks")   # code-based, else front-matter total
+    marks_check = (None if exp_max is None else
+                   {"sum": marks_sum, "expected_max": exp_max,
+                    "pct": round(marks_sum / exp_max * 100, 1)})
+    result = {
+        "board": board, "paper_code": code, "front_matter": fm, "path": path_used,
+        "pages": pages,
+        "questions": questions,
+        "regions": regions,
+        "tables": data_tables,
+        "stats": {
+            "n_questions": len({v["q"] for v in parts.values()}),
+            "n_parts": len(parts),
+            "marks_parts_known": marks_known, "marks_sum": marks_sum,
+            "marks_check": marks_check,
+            "gemma_answer_regions": n_reg, "gemma_marks_filled": n_fill,
+            "gemma_marks_gapfilled": n_marks_fill,
+            "n_data_tables": len(data_tables),
+            "n_furniture_tables": sum(1 for t in all_tables if t["is_furniture"]),
+            "table_sources": {s: sum(1 for t in data_tables if t["source"] == s)
+                              for s in sorted({t["source"] for t in data_tables})},
+            "table_pages": tbl_pages,
+            "region_type_counts": {t: sum(1 for r in regions if r["type"] == t)
+                                   for t in sorted({r["type"] for r in regions})},
+        },
+        "coverage": cov,
+    }
+    json.dump(result, open(a.out, "w"), indent=2)
+
+    print(f"board               : {board}  ({code or 'wordmark'})  [{path_used}]")
+    print(f"front-matter        : {', '.join(f'{k}={v}' for k,v in fm.items() if not isinstance(v,(list,dict)))}")
+    print(f"questions           : {result['stats']['n_questions']} top-level, {len(parts)} parts")
+    mc = f" | {marks_sum}/{exp_max} of max ({marks_check['pct']}%)" if marks_check else ""
+    print(f"marks               : {marks_known}/{len(parts)} parts known (sum {marks_sum}){mc}"
+          + (f"; +{n_mark_geo} by geometry" if n_mark_geo else ""))
+    print(f"gemma regions       : {n_reg} answer_regions, {n_fill} marks gap-filled"
+          + (f"; +{n_marks_fill} marks via --marks-fill" if n_marks_fill else ""))
+    print(f"tables              : {len(data_tables)} data table(s) "
+          f"{result['stats']['table_sources']} on pages {tbl_pages}; "
+          f"{result['stats']['n_furniture_tables']} furniture filtered; {n_tbl} parts flagged")
+    if cov.get("coverage_pct") is not None:
+        print(f"COVERAGE            : {cov['coverage_pct']}%  ({cov['recovered']}/{cov['total']})"
+              f"  missed: {cov['missed'][:8]}{'…' if len(cov['missed'])>8 else ''}  [{cov['source']}]")
+    print(f"-> wrote {a.out}")
+
+
+if __name__ == "__main__":
+    main()