feat(docling): B1 image-only OCR eval harness (overwatch-cleaned)
api-ci-deploy / test-build-deploy (push) Has been cancelled
api-ci-deploy / test-build-deploy (push) Has been cancelled
Eval harness for AQA A-level + GCSE-science image-only papers: finalize.py --b1-only, RapidOCR runner (rapid_pass.py via dsync), GT fixtures (make_b1_gt.py + b1_gt_labels.json), and fetch_b1_corpus.py to pull the eval corpus from .94 cc.examboards at runtime. Salvaged from t_15be12ed (which timed out on iteration budget re-running OCR): exam PDFs and generated OCR caches/reports are NOT committed (third-party copyright + reproducible) — gitignored and fetched/generated at runtime. Baseline coverage recorded in the task evidence file. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
CC Worker
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
34fc7edd68
commit
69d9c46abe
@@ -59,6 +59,61 @@ GEOMETRY = [
|
||||
extract=["--docling", "results/genreport/ocrh556/ocr.json", "--board", "ocr",
|
||||
"--marks-fill", "results/genreport/ocrh556/marks_fill.json"]),
|
||||
]
|
||||
|
||||
B1_GEOMETRY = [
|
||||
dict(slug="b1-aqa-biology-7402-1-2023jun", title="AQA A-level Biology 7402/1 2023 Jun (image-only OCR baseline)",
|
||||
board="aqa", level="A-level", path="B1 image-only OCR (RapidOCR margin-pass)",
|
||||
storage_loc="cc.examboards/aqa/biology/7402/1/2023-jun/qp.pdf",
|
||||
pdf="samples/b1/aqa-biology-7402-1-2023jun.pdf",
|
||||
docling="results/b1_rapid/b1-aqa-biology-7402-1-2023jun/merged.json",
|
||||
rapid="results/b1_rapid/b1-aqa-biology-7402-1-2023jun/p*.json",
|
||||
gt_key="b1-aqa-biology-7402-1-2023jun"),
|
||||
dict(slug="b1-aqa-chemistry-7405-1-2022jun", title="AQA A-level Chemistry 7405/1 2022 Jun (image-only OCR baseline)",
|
||||
board="aqa", level="A-level", path="B1 image-only OCR (RapidOCR margin-pass)",
|
||||
storage_loc="cc.examboards/aqa/chemistry/7405/1/2022-jun/qp.pdf",
|
||||
pdf="samples/b1/aqa-chemistry-7405-1-2022jun.pdf",
|
||||
docling="results/b1_rapid/b1-aqa-chemistry-7405-1-2022jun/merged.json",
|
||||
rapid="results/b1_rapid/b1-aqa-chemistry-7405-1-2022jun/p*.json",
|
||||
gt_key="b1-aqa-chemistry-7405-1-2022jun"),
|
||||
dict(slug="b1-aqa-physics-7408-1-2022jun", title="AQA A-level Physics 7408/1 2022 Jun (image-only OCR baseline)",
|
||||
board="aqa", level="A-level", path="B1 image-only OCR (RapidOCR margin-pass)",
|
||||
storage_loc="cc.examboards/aqa/physics/7408/1/2022-jun/qp.pdf",
|
||||
pdf="samples/b1/aqa-physics-7408-1-2022jun.pdf",
|
||||
docling="results/b1_rapid/b1-aqa-physics-7408-1-2022jun/merged.json",
|
||||
rapid="results/b1_rapid/b1-aqa-physics-7408-1-2022jun/p*.json",
|
||||
gt_key="b1-aqa-physics-7408-1-2022jun"),
|
||||
dict(slug="b1-aqa-biology-8461-1h-2022jun", title="AQA GCSE Biology 8461/1H 2022 Jun (image-only OCR baseline)",
|
||||
board="aqa", level="GCSE", path="B1 image-only OCR (RapidOCR margin-pass)",
|
||||
storage_loc="cc.examboards/aqa/biology/8461/1h/2022-jun/qp.pdf",
|
||||
pdf="samples/b1/aqa-biology-8461-1h-2022jun.pdf",
|
||||
docling="results/b1_rapid/b1-aqa-biology-8461-1h-2022jun/merged.json",
|
||||
rapid="results/b1_rapid/b1-aqa-biology-8461-1h-2022jun/p*.json",
|
||||
gt_key="b1-aqa-biology-8461-1h-2022jun"),
|
||||
dict(slug="b1-aqa-chemistry-8462-1h-2022jun", title="AQA GCSE Chemistry 8462/1H 2022 Jun (image-only OCR baseline)",
|
||||
board="aqa", level="GCSE", path="B1 image-only OCR (RapidOCR margin-pass)",
|
||||
storage_loc="cc.examboards/aqa/chemistry/8462/1h/2022-jun/qp.pdf",
|
||||
pdf="samples/b1/aqa-chemistry-8462-1h-2022jun.pdf",
|
||||
docling="results/b1_rapid/b1-aqa-chemistry-8462-1h-2022jun/merged.json",
|
||||
rapid="results/b1_rapid/b1-aqa-chemistry-8462-1h-2022jun/p*.json",
|
||||
gt_key="b1-aqa-chemistry-8462-1h-2022jun"),
|
||||
dict(slug="b1-aqa-combined-8464-b1h-2022jun", title="AQA GCSE Combined Science Trilogy 8464/B/1H 2022 Jun (image-only OCR baseline)",
|
||||
board="aqa", level="GCSE", path="B1 image-only OCR (RapidOCR margin-pass)",
|
||||
storage_loc="cc.examboards/aqa/combined-science-trilogy/8464/b-1h/2022-jun/qp.pdf",
|
||||
pdf="samples/b1/aqa-combined-8464-b1h-2022jun.pdf",
|
||||
docling="results/b1_rapid/b1-aqa-combined-8464-b1h-2022jun/merged.json",
|
||||
rapid="results/b1_rapid/b1-aqa-combined-8464-b1h-2022jun/p*.json",
|
||||
gt_key="b1-aqa-combined-8464-b1h-2022jun"),
|
||||
dict(slug="b1-aqa-combined-8464-c1h-2022jun", title="AQA GCSE Combined Science Trilogy 8464/C/1H 2022 Jun (image-only OCR baseline; 8465 not present in dev catalogue)",
|
||||
board="aqa", level="GCSE", path="B1 image-only OCR (RapidOCR margin-pass)",
|
||||
storage_loc="cc.examboards/aqa/combined-science-trilogy/8464/c-1h/2022-jun/qp.pdf",
|
||||
pdf="samples/b1/aqa-combined-8464-c1h-2022jun.pdf",
|
||||
docling="results/b1_rapid/b1-aqa-combined-8464-c1h-2022jun/merged.json",
|
||||
rapid="results/b1_rapid/b1-aqa-combined-8464-c1h-2022jun/p*.json",
|
||||
gt_key="b1-aqa-combined-8464-c1h-2022jun"),
|
||||
]
|
||||
|
||||
GT_LABELS_PATH = "fixtures/b1_gt_labels.json"
|
||||
|
||||
FAST = [
|
||||
dict(slug="aqa-physics-7408-fast", title="AQA A-level Physics 7408/1 (born-digital)", board="aqa",
|
||||
level="A-level", pdf="samples/extra/aqa-alevel-physics-7408-1-jun22-qp.pdf",
|
||||
@@ -95,16 +150,65 @@ def jload(p):
|
||||
return {}
|
||||
|
||||
|
||||
def stats_from(struct, val):
|
||||
|
||||
def load_gt_labels():
|
||||
try:
|
||||
return json.load(open(GT_LABELS_PATH))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def part_labels(struct):
|
||||
labels = []
|
||||
for q in struct.get("questions", []) or []:
|
||||
for part in q.get("parts", []) or []:
|
||||
lab = part.get("label")
|
||||
if lab:
|
||||
labels.append(lab)
|
||||
return labels
|
||||
|
||||
|
||||
def coverage_against_labels(struct, labels):
|
||||
if not labels:
|
||||
return None
|
||||
rec = set(part_labels(struct))
|
||||
gt = set(labels)
|
||||
hit = sorted(rec & gt)
|
||||
miss = sorted(gt - rec)
|
||||
return {"coverage_pct": round(len(hit) / len(gt) * 100, 1),
|
||||
"recovered": len(hit), "total": len(gt), "missed": miss,
|
||||
"source": "fixtures/b1_gt_labels.json"}
|
||||
|
||||
|
||||
def answer_region_count(struct):
|
||||
top = len(struct.get("regions", []) or [])
|
||||
per_part = 0
|
||||
for q in struct.get("questions", []) or []:
|
||||
for part in q.get("parts", []) or []:
|
||||
per_part += len(part.get("regions", []) or [])
|
||||
return top + per_part
|
||||
|
||||
|
||||
def ensure_rapid_cache(p):
|
||||
if os.path.exists(p["docling"]):
|
||||
return True
|
||||
if not os.path.exists(p["pdf"]):
|
||||
print(f" ! missing source PDF for {p['slug']}: {p['pdf']} (storage_loc={p.get('storage_loc')})")
|
||||
return False
|
||||
return run(["scripts/rapid_pass.py", p["pdf"], "b1_rapid/" + p["slug"]])
|
||||
|
||||
def stats_from(struct, val, gt_labels=None):
|
||||
st = struct.get("stats", {}) or {}
|
||||
mc = st.get("marks_check") or {}
|
||||
cov = struct.get("coverage", {}) or {}
|
||||
cov = coverage_against_labels(struct, gt_labels) if gt_labels else (struct.get("coverage", {}) or {})
|
||||
return {
|
||||
"board": struct.get("board"), "paper_code": struct.get("paper_code"),
|
||||
"n_questions": st.get("n_questions"), "n_parts": st.get("n_parts"),
|
||||
"marks_sum": mc.get("sum"), "official_max": mc.get("expected_max"),
|
||||
"marks_pct": mc.get("pct"),
|
||||
"coverage_pct": cov.get("coverage_pct"), "coverage_missed": cov.get("missed", []),
|
||||
"coverage_pct": cov.get("coverage_pct"), "coverage_recovered": cov.get("recovered"),
|
||||
"coverage_total": cov.get("total"), "coverage_source": cov.get("source"),
|
||||
"coverage_missed": cov.get("missed", []), "answer_regions": answer_region_count(struct),
|
||||
"validate_verdict": (val.get("summary") or {}).get("worst_severity"),
|
||||
"validate_flags": val.get("flags", []),
|
||||
"questions_expected": (val.get("summary") or {}).get("questions_expected"),
|
||||
@@ -113,12 +217,15 @@ def stats_from(struct, val):
|
||||
}
|
||||
|
||||
|
||||
def do_geometry(p, overlays):
|
||||
def do_geometry(p, overlays, gt_labels=None, prepare_ocr=False):
|
||||
d = os.path.join(FINAL, p["slug"]); os.makedirs(d, exist_ok=True)
|
||||
S, F, B, R, T, V = (os.path.join(d, f) for f in
|
||||
("structured.json", "furniture.json", "bands.json", "page_roles.json",
|
||||
"template.json", "validate.json"))
|
||||
ex = ["extract.py"] + p["extract"] + ["--out", S]
|
||||
if prepare_ocr and not ensure_rapid_cache(p):
|
||||
raise RuntimeError(f"unable to prepare B1 OCR cache for {p['slug']}")
|
||||
extract_args = p.get("extract") or ["--docling", p["docling"], "--rapid", p["rapid"], "--board", p.get("board", "aqa")]
|
||||
ex = ["extract.py"] + extract_args + ["--out", S]
|
||||
if p.get("gt"):
|
||||
ex += ["--gt", p["gt"]]
|
||||
run(ex)
|
||||
@@ -138,7 +245,7 @@ def do_geometry(p, overlays):
|
||||
odbg = os.path.join(d, "overlays", "debug")
|
||||
run(["scripts/overlay.py", S, p["pdf"], "--docling", p["docling"], "--bands", B,
|
||||
"--furniture", F, "--pages", "1,2,3,4,5", "--dpi", "120", "--out", odbg])
|
||||
return stats_from(jload(S), jload(V)), d
|
||||
return stats_from(jload(S), jload(V), gt_labels), d
|
||||
|
||||
|
||||
def do_fast(p):
|
||||
@@ -164,6 +271,7 @@ def per_paper_report(p, s, d, kind):
|
||||
+ (f" (missed {s['coverage_missed'][:8]})" if s.get('coverage_missed') else "")
|
||||
if s['coverage_pct'] is not None else "- **coverage vs GT:** n/a",
|
||||
f"- **G6 verdict:** {s['validate_verdict']}",
|
||||
f"- **answer-region count:** {s.get('answer_regions')}",
|
||||
]
|
||||
if s["validate_flags"]:
|
||||
lines += ["", "**Flags (human-review hints):**"] + [f"- {f}" for f in s["validate_flags"]]
|
||||
@@ -178,21 +286,28 @@ def per_paper_report(p, s, d, kind):
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--no-overlays", action="store_true")
|
||||
ap.add_argument("--b1-only", action="store_true", help="run only the Sprint B1 image-only OCR eval corpus")
|
||||
ap.add_argument("--prepare-ocr", action="store_true", help="populate missing B1 RapidOCR caches via dsync before running")
|
||||
a = ap.parse_args()
|
||||
os.makedirs(FINAL, exist_ok=True)
|
||||
catalog = {"generated_at": datetime.datetime.now().isoformat(timespec="seconds"),
|
||||
"papers": []}
|
||||
total_imgs = 0
|
||||
|
||||
for p in GEOMETRY:
|
||||
gt_fixtures = load_gt_labels()
|
||||
geometry = B1_GEOMETRY if a.b1_only else GEOMETRY
|
||||
fast = [] if a.b1_only else FAST
|
||||
|
||||
for p in geometry:
|
||||
print(f"[geometry] {p['slug']}")
|
||||
s, d = do_geometry(p, not a.no_overlays)
|
||||
gt_labels = (gt_fixtures.get(p.get("gt_key") or p["slug"], {}) or {}).get("labels")
|
||||
s, d = do_geometry(p, not a.no_overlays, gt_labels=gt_labels, prepare_ocr=a.prepare_ocr)
|
||||
n = per_paper_report(p, s, d, p["path"])
|
||||
total_imgs += n
|
||||
catalog["papers"].append({**{k: p[k] for k in ("slug", "title", "board", "level")},
|
||||
"kind": "geometry", "path": p["path"], "dir": d,
|
||||
"overlay_images": n, **s})
|
||||
for p in FAST:
|
||||
for p in fast:
|
||||
print(f"[fast] {p['slug']}")
|
||||
s, d = do_fast(p)
|
||||
per_paper_report(p, s, d, "born-digital fast-path")
|
||||
@@ -214,13 +329,13 @@ def write_index(catalog, total_imgs):
|
||||
"`overlays/template/` (human-review view, all pages) and `overlays/debug/` (raw-detection view).",
|
||||
"Machine catalog: `catalog.json`.", "",
|
||||
"## Image-only / OCR-path (with geometry + overlays)", "",
|
||||
"| Paper | Board / level | Q/parts | Marks/max | Coverage | G6 | Images |",
|
||||
"|---|---|---|---|---|---|---|"]
|
||||
"| Paper | Board / level | Q/parts | Marks/max | Coverage | Answer regions | G6 | Images |",
|
||||
"|---|---|---|---|---|---|---|---|"]
|
||||
for p in g:
|
||||
cov = f"{p['coverage_pct']}%" if p['coverage_pct'] is not None else "n/a"
|
||||
L.append(f"| [{p['title']}]({p['slug']}/report.md) | {p['board']} {p['level']} | "
|
||||
f"{p['n_questions']}/{p['n_parts']} | {p['marks_sum']}/{p['official_max']} "
|
||||
f"({p['marks_pct']}%) | {cov} | {p['validate_verdict']} | "
|
||||
f"({p['marks_pct']}%) | {cov} | {p.get('answer_regions')} | {p['validate_verdict']} | "
|
||||
f"{p['overlay_images']} |")
|
||||
L += ["", "## Born-digital fast-path (CPU, no geometry)", "",
|
||||
"| Paper | Board / level | Q/parts | Marks/max | Coverage | G6 |",
|
||||
|
||||
Reference in New Issue
Block a user