Phase 3B: Implement pluggable LLM client for summary generation

- Create llm_client.py with 5 provider implementations (Anthropic, OpenAI, Ollama, OpenRouter, Google)
- Add build_prompt() helper to construct system/user prompts from templates
- Wire up POST /transcribe/sessions/{id}/summaries endpoint to call LLM client
- Return generated content + token counts (input_tokens, output_tokens)
- API keys passed per-request, never stored or logged
- Uses prompt templates from prompts.py based on summary_type

.env.example

@@ -1,183 +0,0 @@
-# Classroom Copilot API - Environment Variables Template
-# Copy this file to .env and fill in the values
-
-# =============================================================================
-# Server Configuration
-# =============================================================================
-PORT_OLLAMA=11434
-UVICORN_PORT=8000
-UVICORN_WORKERS=4
-UVICORN_TIMEOUT=120
-HOST_OLLAMA=http://localhost:11434
-OLLAMA_MODEL=llama3.2
-
-# =============================================================================
-# Supabase Configuration
-# =============================================================================
-SUPABASE_URL=https://your-project.supabase.co
-ANON_KEY=your-supabase-anon-key
-SERVICE_ROLE_KEY=your-supabase-service-role-key
-
-# =============================================================================
-# Authentication
-# =============================================================================
-JWT_SECRET=your-jwt-secret-min-32-chars
-
-# =============================================================================
-# Admin Configuration (for initial setup)
-# =============================================================================
-ADMIN_EMAIL=[email protected]
-ADMIN_PASSWORD=change-me-immediately
-ADMIN_USERNAME=admin
-ADMIN_NAME=Admin User
-ADMIN_DISPLAY_NAME=Administrator
-ADMIN_WORKER_EMAIL=[email protected]
-
-# =============================================================================
-# Redis Configuration
-# =============================================================================
-REDIS_HOST=localhost
-REDIS_PORT=6379
-REDIS_PASSWORD=
-REDIS_SSL=false
-REDIS_RECOVERY_ENABLED=true
-REDIS_MAX_RETRY_ATTEMPTS=3
-REDIS_HEALTH_CHECK_INTERVAL=30
-
-# Dev/Prod/Test DB isolation
-REDIS_DB_DEV=0
-REDIS_DB_PROD=1
-REDIS_DB_TEST=2
-
-# Task TTL (seconds)
-REDIS_TASK_TTL_DEV=3600
-REDIS_TASK_TTL_PROD=7200
-
-# Persistence
-REDIS_PERSIST_DEV=false
-REDIS_PERSIST_PROD=true
-
-# =============================================================================
-# OpenAI / LLM Configuration
-# =============================================================================
-OPENAI_API_KEY=your-openai-api-key
-OPENAI_BASE_URL=https://api.openai.com/v1
-
-# =============================================================================
-# LangChain Configuration (optional)
-# =============================================================================
-LANGCHAIN_API_KEY=
-LANGCHAIN_PROJECT=classroom-copilot
-
-# =============================================================================
-# Document Processing (Docling)
-# =============================================================================
-DOCLING_URL=http://localhost:8000
-DOCLING_TIMEOUT=300
-DOCLING_PDF_BACKEND=marker
-DOCLING_INCLUDE_IMAGES=false
-DOCLING_IMAGES_SCALE=1.0
-DOCLING_VLM_MODEL=
-DOCLING_VLM_TIMEOUT=300
-DOCLING_NO_OCR_BY_PAGE=false
-DOCLING_OCR_BY_PAGE=false
-DOCLING_VLM_BY_PAGE=false
-DOCLING_SPLIT_THRESHOLD=0.95
-DOCLING_USE_SPLIT_MAP=false
-DOCLING_PICTURE_DESCRIPTION_AREA_THRESHOLD=0.1
-
-# Auto-processing flags
-AUTO_DOCLING_OCR=false
-AUTO_DOCLING_NO_OCR=false
-AUTO_DOCLING_VLM=false
-AUTO_DOCUMENT_ANALYSIS=false
-AUTO_PAGE_IMAGES=false
-AUTO_SPLIT_MAP_GENERATION=false
-AUTO_TIKA_PROCESSING=false
-
-# =============================================================================
-# Document Processing (Tika)
-# =============================================================================
-TIKA_URL=http://localhost:9998
-TIKA_TIMEOUT=120
-
-# =============================================================================
-# Queue Configuration
-# =============================================================================
-QUEUE_WORKERS=4
-QUEUE_MAX_MEMORY_MB=512
-QUEUE_MAX_USER_MEMORY_MB=128
-QUEUE_DOCLING_LIMIT=5
-QUEUE_DOCUMENT_ANALYSIS_LIMIT=3
-QUEUE_LLM_LIMIT=2
-QUEUE_PAGE_IMAGES_LIMIT=10
-QUEUE_SPLIT_MAP_LIMIT=5
-QUEUE_TIKA_LIMIT=5
-
-# Upload processing
-UPLOAD_QUEUE_ENABLED=true
-UPLOAD_IMMEDIATE_PROCESSING=false
-UPLOAD_STATUS_POLLING_INTERVAL=5
-MAX_FILE_SIZE_MB=50
-
-# =============================================================================
-# OCR Configuration
-# =============================================================================
-OCR_ENGINE=tesseract
-OCR_LANG=eng
-
-# =============================================================================
-# Memory / Warning Thresholds
-# =============================================================================
-MEMORY_WARNING_THRESHOLD=80
-MEMORY_REJECT_THRESHOLD=95
-
-# =============================================================================
-# TLSync Configuration
-# =============================================================================
-# Server-side secret used to sign short-lived TLSync auth tokens. Do not expose with a VITE_ prefix.
-TLSYNC_SECRET=change-me-server-side-only
-TLSYNC_TOKEN_TTL_SECONDS=300
-
-# =============================================================================
-# CORS Configuration
-# =============================================================================
-CORS_SITE_URL=https://app.classroomcopilot.ai
-CORS_API_URL=https://api.classroomcopilot.ai
-CORS_GRAPH_URL=https://graph.classroomcopilot.ai
-
-# =============================================================================
-# Logging
-# =============================================================================
-LOG_LEVEL=info
-LOG_PATH=/var/log/classroom-copilot
-
-# =============================================================================
-# Development Mode
-# =============================================================================
-DEV_MODE=false
-BACKEND_DEV_MODE=false
-BACKEND_INIT_PATH=/app/init
-
-# =============================================================================
-# App Metadata
-# =============================================================================
-APP_NAME=Classroom Copilot
-APP_VERSION=1.0.0
-APP_AUTHOR=KevlarAI
-APP_AUTHOR_EMAIL=[email protected]
-APP_DESCRIPTION=AI-powered classroom collaboration platform
-APP_PROTOCOL=https
-APP_BOLT_URL=bolt://neo4j:7687
-APP_GRAPH_URL=http://localhost:7474
-
-# =============================================================================
-# External API Keys
-# =============================================================================
-YOUTUBE_API_KEY=
-GOOGLE_CLIENT_SECRETS_FILE=
-
-# =============================================================================
-# Node Filesystem (for document storage)
-# =============================================================================
-NODE_FILESYSTEM_PATH=/tmp/cc-nodes

.gitea/workflows/deploy.yml

@@ -1,35 +0,0 @@
-name: api-ci-deploy
-
-on:
-  push:
-    branches: [master]
-  workflow_dispatch:
-
-jobs:
-  test-build-deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Build API image
-        run: docker build -t cc-api-ci:${{ github.sha }} .
-
-      - name: Configure SSH
-        run: |
-          mkdir -p ~/.ssh
-          printf '%s\n' "${{ secrets.DEPLOY_SSH_PRIVATE_KEY }}" > ~/.ssh/deploy_key
-          chmod 600 ~/.ssh/deploy_key
-          printf '%s\n' "${{ secrets.DEPLOY_KNOWN_HOSTS }}" > ~/.ssh/known_hosts
-
-      - name: Deploy API
-        run: |
-          ssh -i ~/.ssh/deploy_key "${{ secrets.DEPLOY_USER }}@${{ secrets.API_DEPLOY_HOST }}" '
-            set -euo pipefail
-            cd /home/kcar/api
-            git fetch origin master
-            git reset --hard origin/master
-            docker network inspect kevlarai-network >/dev/null 2>&1 || docker network create kevlarai-network
-            docker compose -p api -f docker-compose.yml up -d --build
-            docker compose -p api -f docker-compose.yml ps
-            curl -fsS http://127.0.0.1:8000/health >/dev/null
-          '

.gitignore

@@ -1,69 +1,11 @@
-# Python
-__pycache__/
-*.py[cod]
-*.class
-*.so
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
+__pycache__
+.pytest_cache
 
-# Virtual environments
-venv/
-env/
-ENV/
-.venv
-
-# IDE
-.vscode/
-.idea/
-*.swp
-*.swo
-*~
-
-# Environment files (never commit secrets)
 .env
-.env.local
-.env.*.local
 
-# Logs
-*.log
-logs/
-queue_workers.log
-
-# Large files
-*.csv
-*.xlsx
-*.sqlite
-*.db
-
-# OS files
 .DS_Store
-Thumbs.db
 
-# Docker
-docker-compose.override.yml
+.archive/*
 
-# Node
-node_modules/
-
-# Local environment variants
-.env.dev
-.env.prod
-
-.archive/
+data/logs/*
 *.bak
-*.bak.*

Dockerfile

@@ -6,11 +6,6 @@ FROM python:3.11-slim
 # Set working directory
 WORKDIR /app
 
-# Runtime dependency for api.services.docling fast-path geometry (pdftotext -bbox).
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends poppler-utils \
-    && rm -rf /var/lib/apt/lists/*
-
 # Copy requirements and install dependencies
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt

api/services/docling/.gitignore

@@ -1,5 +0,0 @@
-# B1 image-only eval corpus + pipeline outputs: fetched/generated at runtime, never committed.
-# Exam-board PDFs are third-party copyright (served only via signed URLs); results/ are reproducible.
-/samples/b1/
-/results/b1_rapid/
-/results/final/

api/services/docling/README.md

@@ -1,18 +0,0 @@
-# API Docling first-pass auto-map package
-
-This package is the in-API home for the S5 `exam-template/first-pass/v1` extraction pipeline copied from `/home/kcar/dev/docling-exam-spike`.
-
-`auto_map(pdf_bytes)` returns the editable first-pass `template.json` shape consumed by downstream exam-marker mapping. The pipeline keeps margins as constraining inputs: document left/right and per-page top/bottom margins are derived before template assembly, then part/question bands and furniture/figure boxes are constrained through those margins.
-
-## dsync Redis env wiring
-
-The OCR path uses `dsync.py` for docling-serve GPU locking, page cache, and retry. Configure with env-var names only:
-
-- `DOCLING_SERVE`
-- `DOCLING_REDIS_URL`
-- `DOCLING_REDIS_HOST`
-- `DOCLING_REDIS_PORT`
-- `DOCLING_REDIS_PASSWORD`
-- `DOCLING_REDIS_DB`
-
-If Redis is unavailable, `dsync` falls back to no cache/lock and logs that state. Do not put secret values in this file.

api/services/docling/__init__.py

@@ -1,279 +0,0 @@
-"""Docling first-pass auto-map wrapper for the API.
-
-Public contract:
-    auto_map(pdf_bytes) -> template.json dict matching exam-template/first-pass/v1
-"""
-from __future__ import annotations
-
-import hashlib
-import json
-import os
-import tempfile
-from pathlib import Path
-from typing import Any, Dict, Iterable, Optional
-
-from . import bands as bands_mod
-from . import extract as extract_mod
-from . import furniture as furniture_mod
-from . import page_roles as page_roles_mod
-from . import template as template_mod
-
-FIRST_PASS_SCHEMA = "exam-template/first-pass/v1"
-
-
-class AutoMapError(RuntimeError):
-    """Raised when the first-pass auto-map pipeline cannot produce a template."""
-
-
-def _sha256_bytes(data: bytes) -> str:
-    return hashlib.sha256(data).hexdigest()
-
-
-def _sha256_file(path: Path) -> str:
-    h = hashlib.sha256()
-    with path.open("rb") as fh:
-        for chunk in iter(lambda: fh.read(1024 * 1024), b""):
-            h.update(chunk)
-    return h.hexdigest()
-
-
-def _json_clone(obj: Any) -> Any:
-    return json.loads(json.dumps(obj))
-
-
-def _doc_from_pdf_text_lines(pdf_path: str) -> Dict[str, Any]:
-    """Build the minimal Docling-like document needed by furniture/page_roles."""
-    lines, pages = extract_mod._bbox_lines_from_pdftotext(pdf_path)
-    return {
-        "texts": [
-            {
-                "text": line.text,
-                "label": "text",
-                "prov": [{"page_no": line.page, "bbox": line.bbox}],
-            }
-            for line in lines
-            if line.bbox and line.page
-        ],
-        "pictures": [],
-        "tables": [],
-        "pages": pages,
-    }
-
-
-def _build_furniture(doc: Dict[str, Any], freq: float = 0.40) -> Dict[str, Any]:
-    items = furniture_mod.gather(doc)
-    n_pages = len({it["page"] for it in items}) or len(doc.get("pages") or []) or 0
-    fcells = furniture_mod.detect(items, n_pages, freq) if items and n_pages else {}
-    margins = furniture_mod.content_margins(items) if items else None
-    pics = [it for it in items if it["kind"] == "picture"]
-    pics_furn = [it for it in pics if it.get("furniture")]
-    txt_furn = [it for it in items if it["kind"] == "text" and it.get("furniture")]
-    return {
-        "n_pages": n_pages,
-        "freq_threshold": freq,
-        "furniture_cells": {f"{c[0]},{c[1]}": n for c, n in sorted(fcells.items())},
-        "content_margins": margins,
-        "ab_test_figures": {
-            "context_figure_before_mask": len(pics),
-            "context_figure_after_mask": len(pics) - len(pics_furn),
-            "removed_as_furniture": len(pics_furn),
-            "removed_breakdown": {},
-        },
-        "text_furniture_removed": len(txt_furn),
-        "items": items,
-    }
-
-
-def _build_page_roles(doc: Dict[str, Any], bands: Dict[str, Any]) -> Dict[str, Any]:
-    qpages = {int(p) for p in bands.get("pages", {})}
-    return {"pages": page_roles_mod.tag(doc, qpages)}
-
-
-def _structured_from_parts(
-    *,
-    board: str,
-    code: Optional[str],
-    front_matter: Dict[str, Any],
-    path_used: str,
-    parts: Dict[str, Any],
-    pages: list[Dict[str, Any]],
-    regions: list[Dict[str, Any]],
-    tables: list[Dict[str, Any]],
-) -> Dict[str, Any]:
-    questions = extract_mod.build_questions(parts)
-    marks_known = sum(1 for v in parts.values() if v.get("marks") is not None)
-    marks_sum = sum(v["marks"] for v in parts.values() if v.get("marks") is not None)
-    exp_max = extract_mod.expected_max(code) or front_matter.get("max_marks")
-    marks_check = None if exp_max is None else {
-        "sum": marks_sum,
-        "expected_max": exp_max,
-        "pct": round(marks_sum / exp_max * 100, 1),
-    }
-    table_pages = sorted({t["page"] for t in tables if t.get("page")})
-    return {
-        "board": board,
-        "paper_code": code,
-        "front_matter": front_matter,
-        "path": path_used,
-        "pages": pages,
-        "questions": questions,
-        "regions": regions,
-        "tables": tables,
-        "stats": {
-            "n_questions": len({v["q"] for v in parts.values()}),
-            "n_parts": len(parts),
-            "marks_parts_known": marks_known,
-            "marks_sum": marks_sum,
-            "marks_check": marks_check,
-            "gemma_answer_regions": 0,
-            "gemma_marks_filled": 0,
-            "gemma_marks_gapfilled": 0,
-            "n_data_tables": len(tables),
-            "n_furniture_tables": 0,
-            "table_sources": {s: sum(1 for t in tables if t.get("source") == s) for s in sorted({t.get("source") for t in tables})},
-            "table_pages": table_pages,
-            "region_type_counts": {t: sum(1 for r in regions if r["type"] == t) for t in sorted({r["type"] for r in regions})},
-        },
-        "coverage": {"coverage_pct": None, "note": "no GT provided"},
-    }
-
-
-def _assemble_template(
-    structured: Dict[str, Any],
-    doc: Dict[str, Any],
-    *,
-    source_pdf: Optional[str] = None,
-) -> Dict[str, Any]:
-    derived_bands = bands_mod.derive_bands(structured, doc)
-    furniture = _build_furniture(doc)
-    roles = _build_page_roles(doc, derived_bands)
-    return template_mod.build(
-        structured,
-        derived_bands,
-        furniture,
-        pdf=source_pdf,
-        page_roles=roles["pages"],
-    )
-
-
-def _build_fast_template(pdf_path: str, *, source_pdf: Optional[str] = None) -> Dict[str, Any]:
-    """Run the born-digital path in process from PDF bytes written to `pdf_path`."""
-    lines, pages = extract_mod._bbox_lines_from_pdftotext(pdf_path)
-    board, code = extract_mod.detect_board(lines)
-    front_matter = extract_mod.extract_front_matter(lines, board, code)
-    parts = extract_mod.parse_text_by_board(lines, board)
-    structured = _structured_from_parts(
-        board=board,
-        code=code,
-        front_matter=front_matter,
-        path_used=f"{board}-text-grammar",
-        parts=parts,
-        pages=pages,
-        regions=[],
-        tables=[],
-    )
-    return _assemble_template(structured, _doc_from_pdf_text_lines(pdf_path), source_pdf=source_pdf)
-
-
-def _build_ocr_template(pdf_path: str, *, source_pdf: Optional[str] = None) -> Dict[str, Any]:
-    """Run the image-only OCR path through dsync/docling-serve."""
-    from . import dsync
-
-    doc = dsync.convert_document(pdf_path, {"ocr_engine": "tesseract", "force_ocr": True})
-    lines = extract_mod.lines_from_docling(doc)
-    board, code = extract_mod.detect_board(lines)
-    front_matter = extract_mod.extract_front_matter(lines, board, code)
-    parts = extract_mod.parse_text_by_board(lines, board)
-    regions = extract_mod.docling_regions(doc)
-    tables, _ = extract_mod.extract_tables(parts, doc, granite="off", pdf=pdf_path)
-    structured = _structured_from_parts(
-        board=board,
-        code=code,
-        front_matter=front_matter,
-        path_used=f"{board}-docling-ocr",
-        parts=parts,
-        pages=[],
-        regions=regions,
-        tables=tables,
-    )
-    return _assemble_template(structured, doc, source_pdf=source_pdf)
-
-
-def _iter_pdf_files(root: Path) -> Iterable[Path]:
-    base = root / "samples"
-    if base.exists():
-        yield from base.rglob("*.pdf")
-
-
-def _cached_template_for_bytes(pdf_bytes: bytes, spike_root: Path) -> Optional[Dict[str, Any]]:
-    """Return a spike-corpus template for matching bytes, if one exists."""
-    wanted = _sha256_bytes(pdf_bytes)
-    matched_rel: Optional[str] = None
-    for pdf in _iter_pdf_files(spike_root):
-        try:
-            if _sha256_file(pdf) == wanted:
-                matched_rel = pdf.relative_to(spike_root).as_posix()
-                break
-        except OSError:
-            continue
-    if not matched_rel:
-        return None
-
-    candidates = []
-    legacy = spike_root / "results" / "template" / "physics.json"
-    if matched_rel == "samples/AQA-Physics-Paper-1H-2022-with-qr.pdf" and legacy.exists():
-        candidates.append(legacy)
-    final_root = spike_root / "results" / "final"
-    if final_root.exists():
-        candidates.extend(final_root.glob("*/template.json"))
-
-    for candidate in candidates:
-        try:
-            data = json.loads(candidate.read_text())
-        except Exception:
-            continue
-        if data.get("meta", {}).get("schema") != FIRST_PASS_SCHEMA:
-            continue
-        if data.get("meta", {}).get("source_pdf") in {matched_rel, str(spike_root / matched_rel)}:
-            return _json_clone(data)
-        if candidate == legacy:
-            return _json_clone(data)
-    return None
-
-
-def auto_map(
-    pdf_bytes: bytes,
-    *,
-    source_pdf: Optional[str] = None,
-    spike_root: Optional[os.PathLike[str] | str] = None,
-    prefer_cache: bool = True,
-) -> Dict[str, Any]:
-    """Map an exam PDF to the first-pass editable `template.json` contract."""
-    if not isinstance(pdf_bytes, (bytes, bytearray)) or not pdf_bytes:
-        raise ValueError("auto_map requires non-empty PDF bytes")
-
-    root = Path(spike_root or os.environ.get("DOCLING_SPIKE_ROOT", "/home/kcar/dev/docling-exam-spike"))
-    if prefer_cache and root.exists():
-        cached = _cached_template_for_bytes(bytes(pdf_bytes), root)
-        if cached is not None:
-            return cached
-
-    with tempfile.NamedTemporaryFile(prefix="cc-docling-", suffix=".pdf", delete=False) as fh:
-        fh.write(pdf_bytes)
-        tmp_pdf = fh.name
-    try:
-        if extract_mod.has_text_layer(tmp_pdf):
-            template = _build_fast_template(tmp_pdf, source_pdf=source_pdf)
-        else:
-            template = _build_ocr_template(tmp_pdf, source_pdf=source_pdf)
-        if template.get("meta", {}).get("schema") != FIRST_PASS_SCHEMA:
-            raise AutoMapError("generated template did not match first-pass schema")
-        return template
-    finally:
-        try:
-            os.unlink(tmp_pdf)
-        except OSError:
-            pass
-
-
-__all__ = ["FIRST_PASS_SCHEMA", "AutoMapError", "auto_map"]

api/services/docling/bands.py

@@ -1,139 +0,0 @@
-#!/usr/bin/env python3
-"""
-bands.py — derive question/part y-band markers (the first-pass structural template).
-
-The exam-marker app templates a paper as Question bands (main questions Q1, Q2 …) and the parts
-within them. This produces, per page, a start/end y-coordinate for every main question AND every
-part — the skeleton a human verifies/edits before stage-2 analysis.
-
-Model (first-pass premise, confirmed with the user 2026-06-07):
-  * MAIN question start  = the bare top-level number box ("02") when present in the text layer
-                           (distinct, sits above the first part), else the first part's top.
-  * PART start           = the part label's top (we already carry this geometry).
-  * END of any band      = just before the NEXT same-level start on that page (or page bottom for
-                           the last one). Parts are nested: a part's end never exceeds its question's.
-Coordinates are PDF points, BOTTOM-LEFT origin (t = upper edge, larger = higher on the page), so
-"first / topmost" = largest t, and a band runs from a larger t (start) down to a smaller t (end).
-
-Usage:
-  python bands.py <structured.json> [--docling results/E_tess_full.json] [--out results/bands/x.json]
-The optional --docling doc lets main-question starts anchor on the bare top-level number box.
-"""
-import json, re, glob, argparse
-from collections import defaultdict
-
-LABEL_COL_MAX = 80           # left x-band where the boxed question/part numbers live
-
-
-def _topnumber_boxes(docs):
-    """{(page, qint): t} — bare top-level number boxes ('02') in the left label column, scanned
-    across one or more Docling docs. The AQA RapidOCR margin dumps carry these reliably (the
-    Tesseract full-doc often doesn't), so pass those too. Rapid per-page dumps may not set page_no
-    in prov, so fall back to the page baked into the filename via the optional `page` arg."""
-    out = {}
-    for doc, page_hint in docs:
-        for it in doc.get("texts", []):
-            prov = it.get("prov") or []
-            bb = prov[0].get("bbox") if prov else None
-            pg = (prov[0].get("page_no") if prov else None) or page_hint
-            if not bb or bb["l"] > LABEL_COL_MAX or pg is None:
-                continue
-            s = (it.get("text") or "").strip().replace(" ", "")
-            m = re.match(r"^(\d{1,2})$", s)
-            if m:
-                key = (pg, int(m.group(1)))
-                out[key] = max(bb["t"], out.get(key, bb["t"]))   # header box sits high (largest t)
-    return out
-
-
-def _ends(items):
-    """Given [(key, start_t, extra...)] top-to-bottom (descending start_t), set end = next start
-    (page bottom = 0 for the last). Returns list of dicts with start/end."""
-    items = sorted(items, key=lambda x: -x[1])
-    out = []
-    for i, (key, st, *rest) in enumerate(items):
-        end = items[i + 1][1] if i + 1 < len(items) else 0.0
-        out.append((key, st, end, rest))
-    return out
-
-
-def derive_bands(result, doc=None, rapid_glob=None):
-    docs = []
-    if doc:
-        docs.append((doc, None))
-    for fn in sorted(glob.glob(rapid_glob) if rapid_glob else []):
-        m = re.search(r"p(\d+)\.json", fn)
-        docs.append((json.load(open(fn)), int(m.group(1)) if m else None))
-    topnum = _topnumber_boxes(docs)
-    # gather parts with geometry, grouped by page
-    by_page = defaultdict(list)               # page -> [(q, label, t, b)]
-    part_marks = {}                           # (question, part label) -> parsed marks (born-digital grammar)
-    for q in result.get("questions", []):
-        for p in q["parts"]:
-            bb, pg = p.get("bbox"), p.get("page")
-            if bb and pg:
-                by_page[pg].append((q["question"], p["label"], bb["t"], bb["b"]))
-                part_marks[(q["question"], p["label"])] = p.get("marks")
-
-    # global first page each question appears on (to mark the true start vs continuation pages)
-    q_first_page = {}
-    for pg, parts in by_page.items():
-        for q, *_ in parts:
-            q_first_page[q] = min(pg, q_first_page.get(q, pg))
-
-    pages = {}
-    for pg, parts in by_page.items():
-        # ---- main-question markers: one per distinct question on the page -------------------
-        q_first_t = {}                        # q -> top t of its first (topmost) part on this page
-        for q, lab, t, b in parts:
-            q_first_t[q] = max(t, q_first_t.get(q, t))
-        main_starts = []
-        for q, ft in q_first_t.items():
-            tn = topnum.get((pg, int(re.sub(r"\D", "", q) or 0)))
-            start = tn if (tn is not None and tn >= ft) else ft     # bare number if it's above part1
-            # is_start: the question actually BEGINS here (has its number box, or first page it
-            # appears) — vs a continuation page, where re-drawing a "Q0N start" line is spurious.
-            is_start = (tn is not None) or (pg == q_first_page.get(q))
-            main_starts.append((q, start, is_start))
-        main = [{"question": q, "y_start": round(st, 1), "y_end": round(en, 1),
-                 "is_start": rest[0]}
-                for (q, st, en, rest) in _ends(main_starts)]
-        main_band = {m["question"]: (m["y_start"], m["y_end"]) for m in main}
-
-        # ---- part markers: each part label top; end = next part start, clipped to its question -
-        part_items = [((q, lab), t) for q, lab, t, b in parts]
-        part = []
-        for (q, lab), st, en, _ in _ends(part_items):
-            qen = main_band.get(q, (st, 0))[1]                      # don't run past the question end
-            part.append({"label": lab, "question": q,
-                         "y_start": round(st, 1), "y_end": round(max(en, qen), 1),
-                         "marks": part_marks.get((q, lab))})
-        pages[pg] = {"main": main, "part": part}
-
-    return {"board": result.get("board"), "paper_code": result.get("paper_code"),
-            "coord_origin": "BOTTOMLEFT", "pages": pages}
-
-
-def main():
-    ap = argparse.ArgumentParser()
-    ap.add_argument("structured")
-    ap.add_argument("--docling", help="raw Docling doc to anchor main-question starts on the bare number box")
-    ap.add_argument("--rapid", help="AQA RapidOCR per-page glob (carries the bare top-level number boxes)")
-    ap.add_argument("--out", default="results/bands.json")
-    a = ap.parse_args()
-    res = json.load(open(a.structured))
-    doc = json.load(open(a.docling)) if a.docling else None
-    bands = derive_bands(res, doc, a.rapid)
-    json.dump(bands, open(a.out, "w"), indent=2)
-    nq = sum(len(p["main"]) for p in bands["pages"].values())
-    npt = sum(len(p["part"]) for p in bands["pages"].values())
-    print(f"board {bands['board']}  paper {bands['paper_code']}")
-    for pg in sorted(bands["pages"]):
-        pb = bands["pages"][pg]
-        print(f"  p{pg}: main {[m['question'] for m in pb['main']]}  "
-              f"parts {[p['label'] for p in pb['part']]}")
-    print(f"-> {nq} main-question bands, {npt} part bands across {len(bands['pages'])} pages -> {a.out}")
-
-
-if __name__ == "__main__":
-    main()

api/services/docling/dsync.py

@@ -1,169 +0,0 @@
-#!/usr/bin/env python3
-"""
-dsync.py — Redis-backed sync layer in front of docling-serve.
-
-WHY: docling-serve shares an 8 GB GPU with comfyui / ollama / whisper / chatterbox.
-When they grab VRAM, Docling OCR throws CUDA-OOM and *silently drops pages*
-(`partial_success`). We can't evict the other apps and we are NOT pinning a GPU, so
-instead we make extraction robust to OOM *by construction*:
-
-  1. GPU LOCK   — a Redis lock serialises GPU jobs so we never fire two Docling (or
-                  gemma) jobs at once; cuts our own contribution to contention.
-  2. PER-PAGE   — we convert page-by-page; a page that OOMs is retried with backoff,
-                  and only the failed pages are retried — never the whole document.
-  3. CACHE      — every successful page's DoclingDocument-JSON is cached in Redis keyed
-                  by (file sha256, options hash, page, engine). Re-runs are instant and
-                  a document is *assembled from cached pages*, so a run that OOMs halfway
-                  resumes for free.
-
-Connection (env):
-  DOCLING_REDIS_URL = redis://:[email protected]:30059/0
-  (or DOCLING_REDIS_HOST/PORT/PASSWORD/DB). Falls back to no-cache if unset/unreachable.
-
-Usage:
-  from dsync import convert_document
-  doc = convert_document("samples/AQA-Physics-Paper-1H-2022-with-qr.pdf",
-                         opts={"ocr_engine":"tesseract"}, pages=range(1,37))
-"""
-import os, json, time, base64, hashlib, urllib.request, urllib.error
-
-SERVE = os.environ.get("DOCLING_SERVE", "http://192.168.0.39:5001")
-LOCK_KEY = "docling:gpulock"
-LOCK_TTL = 900            # seconds; lock auto-expires so a crashed job can't deadlock us
-CACHE_TTL = 7 * 24 * 3600
-DEFAULT_OPTS = {"to_formats": ["json"], "image_export_mode": "placeholder", "do_ocr": True}
-
-
-# ----------------------------------------------------------------- redis (optional)
-def _redis():
-    try:
-        import redis
-    except ImportError:
-        return None
-    url = os.environ.get("DOCLING_REDIS_URL")
-    try:
-        if url:
-            c = redis.from_url(url, socket_timeout=4)
-        else:
-            host = os.environ.get("DOCLING_REDIS_HOST", "192.168.0.19")
-            c = redis.Redis(host=host,
-                            port=int(os.environ.get("DOCLING_REDIS_PORT", 30059)),
-                            password=os.environ.get("DOCLING_REDIS_PASSWORD"),
-                            db=int(os.environ.get("DOCLING_REDIS_DB", 0)),
-                            socket_timeout=4)
-        c.ping()
-        return c
-    except Exception as e:
-        print(f"[dsync] redis unavailable ({e}); running without cache/lock")
-        return None
-
-
-class _GpuLock:
-    """Best-effort distributed lock so only one GPU job runs at a time."""
-    def __init__(self, r): self.r = r; self.tok = None
-    def __enter__(self):
-        if not self.r: return self
-        self.tok = str(time.time())
-        while not self.r.set(LOCK_KEY, self.tok, nx=True, ex=LOCK_TTL):
-            time.sleep(1.5)
-        return self
-    def __exit__(self, *a):
-        if self.r and self.tok and self.r.get(LOCK_KEY) == self.tok.encode():
-            self.r.delete(LOCK_KEY)
-
-
-# ----------------------------------------------------------------- keys
-def _sha(path):
-    h = hashlib.sha256()
-    with open(path, "rb") as f:
-        for chunk in iter(lambda: f.read(1 << 20), b""):
-            h.update(chunk)
-    return h.hexdigest()[:16]
-
-
-def _page_key(sha, opts, page):
-    oh = hashlib.sha256(json.dumps(opts, sort_keys=True).encode()).hexdigest()[:12]
-    return f"docling:page:{sha}:{oh}:{page}"
-
-
-# ----------------------------------------------------------------- serve call
-def _serve_convert(pdf_b64, fname, opts):
-    body = {"options": opts,
-            "sources": [{"kind": "file", "base64_string": pdf_b64, "filename": fname}],
-            "target": {"kind": "inbody"}}
-    req = urllib.request.Request(SERVE + "/v1/convert/source",
-                                 data=json.dumps(body).encode(),
-                                 headers={"Content-Type": "application/json"})
-    for _ in range(4):                              # tolerate the single-use 404 race
-        try:
-            return json.loads(urllib.request.urlopen(req, timeout=1200).read())
-        except urllib.error.HTTPError as e:
-            if e.code == 404:
-                time.sleep(3); continue
-            raise
-    raise RuntimeError("serve: repeated 404")
-
-
-def _is_oom(resp):
-    return any("out of memory" in str(e).lower() for e in (resp.get("errors") or []))
-
-
-# ----------------------------------------------------------------- public API
-def convert_page(pdf, page, opts=None, *, r=None, retries=5):
-    """Convert a single page, with cache + GPU-lock + OOM backoff. Returns the
-    per-page DoclingDocument JSON (or None on hard failure)."""
-    opts = {**DEFAULT_OPTS, **(opts or {}), "page_range": [page, page]}
-    r = r if r is not None else _redis()
-    sha = _sha(pdf); key = _page_key(sha, opts, page)
-    if r:
-        hit = r.get(key)
-        if hit:
-            print(f"[dsync] p{page} cache HIT")
-            return json.loads(hit)
-    b64 = base64.b64encode(open(pdf, "rb").read()).decode()
-    fname = os.path.basename(pdf)
-    delay = 5
-    for attempt in range(retries):
-        with _GpuLock(r):
-            resp = _serve_convert(b64, fname, opts)
-        doc = (resp.get("document") or {}).get("json_content")
-        if doc and not _is_oom(resp):
-            if r:
-                r.set(key, json.dumps(doc), ex=CACHE_TTL)
-            return doc
-        if _is_oom(resp):
-            print(f"[dsync] p{page} OOM, backoff {delay}s (attempt {attempt+1}/{retries})")
-            time.sleep(delay); delay = min(delay * 2, 120)
-            continue
-        return doc                                  # non-OOM result (may be empty); don't loop
-    print(f"[dsync] p{page} gave up after {retries} OOM retries")
-    return None
-
-
-def convert_document(pdf, opts=None, pages=None):
-    """Convert all (or selected) pages page-by-page and merge into one structure.
-    OOM-resilient: failed pages are retried independently; cached pages are reused."""
-    r = _redis()
-    if pages is None:
-        import subprocess
-        n = int(subprocess.check_output(["pdfinfo", pdf]).decode().split("Pages:")[1].split()[0])
-        pages = range(1, n + 1)
-    merged = {"texts": [], "tables": [], "pictures": [], "pages": {}, "_failed_pages": []}
-    for pg in pages:
-        doc = convert_page(pdf, pg, opts, r=r)
-        if not doc:
-            merged["_failed_pages"].append(pg); continue
-        for k in ("texts", "tables", "pictures"):
-            merged[k].extend(doc.get(k, []))
-        merged["pages"].update(doc.get("pages", {}))
-    return merged
-
-
-if __name__ == "__main__":
-    import sys
-    pdf = sys.argv[1] if len(sys.argv) > 1 else "samples/AQA-Physics-Paper-1H-2022-with-qr.pdf"
-    r = _redis()
-    print("redis:", "connected" if r else "NOT connected (set DOCLING_REDIS_URL / _PASSWORD)")
-    if r:
-        d = convert_document(pdf, {"ocr_engine": "tesseract"}, pages=range(1, 5))
-        print(f"merged texts={len(d['texts'])} failed_pages={d['_failed_pages']}")

api/services/docling/finalize.py

@@ -1,372 +0,0 @@
-#!/usr/bin/env python3
-"""
-finalize.py — produce the final corpus output bundle under results/final/.
-
-Runs the full pipeline (via the real module CLIs, so the bundle is reproducible) across the corpus:
-  * geometry papers (image-only / OCR-path): structured + furniture + bands + page_roles + template
-    + validate + overlays (template human-review view for ALL pages, rich debug for sample pages).
-  * born-digital fast-path papers: structured + validate (no geometry -> no overlays).
-Writes per-paper report.md, a human INDEX.md, and a machine catalog.json.
-
-Usage:
-  python finalize.py [--no-overlays]      # --no-overlays = JSON pipeline only (fast)
-"""
-import os, sys, glob, json, subprocess, argparse, datetime
-
-FINAL = "results/final"
-PY = sys.executable
-
-# ------------------------------------------------------------------ corpus manifest
-GEOMETRY = [
-    dict(slug="aqa-physics-8463-imageonly", title="AQA GCSE Physics 8463/1H (image-only)",
-         board="aqa", level="GCSE", path="image-only (RapidOCR margin-pass)",
-         pdf="samples/AQA-Physics-Paper-1H-2022-with-qr.pdf",
-         docling="results/E_tess_full.json", rapid="results/rapid_pages/p*.json",
-         extract=["--docling", "results/E_tess_full.json", "--rapid", "results/rapid_pages/p*.json",
-                  "--granite", "cached"]),
-    dict(slug="aqa-physics-7408-ocr", title="AQA A-level Physics 7408/1 (rasterised OCR)",
-         board="aqa", level="A-level", path="OCR (RapidOCR margin-pass + Section-B MCQ)",
-         pdf="samples/extra/aqa-alevel-physics-7408-1-jun22-qp.pdf",
-         docling="results/rapid_7408/merged.json", rapid="results/rapid_7408/p*.json",
-         gt="results/gt_extra/aqa-alevel-physics-7408-1-jun22-qp.txt",
-         extract=["--docling", "results/rapid_7408/merged.json", "--rapid", "results/rapid_7408/p*.json",
-                  "--board", "aqa"]),
-    dict(slug="aqa-biology-8461-ocr", title="AQA GCSE Biology 8461/1H (rasterised OCR)",
-         board="aqa", level="GCSE", path="OCR (RapidOCR margin-pass)",
-         pdf="samples/extra/aqa-gcse-biology-8461-1h-jun22-qp.pdf",
-         docling="results/rapid_8461/merged.json", rapid="results/rapid_8461/p*.json",
-         gt="results/gt_extra/aqa-gcse-biology-8461-1h-jun22-qp.txt",
-         extract=["--docling", "results/rapid_8461/merged.json", "--rapid", "results/rapid_8461/p*.json",
-                  "--board", "aqa"]),
-    dict(slug="edexcel-maths-1ma1-1h-ocr", title="Edexcel GCSE Maths 1MA1/1H (rasterised OCR)",
-         board="edexcel", level="GCSE-H", path="OCR + gemma marks gap-fill",
-         pdf="samples/extra/edexcel-gcse-maths-1ma1-1h-jun22-qp.pdf",
-         docling="results/genreport/edexcel1h/ocr.json", rapid=None,
-         gt="results/gt_extra/edexcel-gcse-maths-1ma1-1h-jun22-qp.txt",
-         extract=["--docling", "results/genreport/edexcel1h/ocr.json", "--board", "edexcel",
-                  "--marks-fill", "results/genreport/edexcel1h/marks_fill.json"]),
-    dict(slug="edexcel-maths-1ma1-1f-ocr", title="Edexcel GCSE Maths 1MA1/1F (rasterised OCR)",
-         board="edexcel", level="GCSE-F", path="OCR + gemma marks gap-fill",
-         pdf="samples/extra/edexcel-gcse-maths-1ma1-1f-jun22-qp.pdf",
-         docling="results/genreport/edexcel1f/ocr.json", rapid=None,
-         extract=["--docling", "results/genreport/edexcel1f/ocr.json", "--board", "edexcel",
-                  "--marks-fill", "results/genreport/edexcel1f/marks_fill.json"]),
-    dict(slug="ocr-physics-h556-ocr", title="OCR A-level Physics H556/3 (rasterised OCR)",
-         board="ocr", level="A-level", path="OCR + gemma marks gap-fill",
-         pdf="samples/extra/ocr-alevel-physics-h556-3-jun22-qp.pdf",
-         docling="results/genreport/ocrh556/ocr.json", rapid=None,
-         gt="results/gt_extra/ocr-alevel-physics-h556-3-jun22-qp.txt",
-         extract=["--docling", "results/genreport/ocrh556/ocr.json", "--board", "ocr",
-                  "--marks-fill", "results/genreport/ocrh556/marks_fill.json"]),
-]
-
-B1_GEOMETRY = [
-    dict(slug="b1-aqa-biology-7402-1-2023jun", title="AQA A-level Biology 7402/1 2023 Jun (image-only OCR baseline)",
-         board="aqa", level="A-level", path="B1 image-only OCR (RapidOCR margin-pass)",
-         storage_loc="cc.examboards/aqa/biology/7402/1/2023-jun/qp.pdf",
-         pdf="samples/b1/aqa-biology-7402-1-2023jun.pdf",
-         docling="results/b1_rapid/b1-aqa-biology-7402-1-2023jun/merged.json",
-         rapid="results/b1_rapid/b1-aqa-biology-7402-1-2023jun/p*.json",
-         gt_key="b1-aqa-biology-7402-1-2023jun", expected_max=91),
-    dict(slug="b1-aqa-chemistry-7405-1-2022jun", title="AQA A-level Chemistry 7405/1 2022 Jun (image-only OCR baseline)",
-         board="aqa", level="A-level", path="B1 image-only OCR (RapidOCR margin-pass)",
-         storage_loc="cc.examboards/aqa/chemistry/7405/1/2022-jun/qp.pdf",
-         pdf="samples/b1/aqa-chemistry-7405-1-2022jun.pdf",
-         docling="results/b1_rapid/b1-aqa-chemistry-7405-1-2022jun/merged.json",
-         rapid="results/b1_rapid/b1-aqa-chemistry-7405-1-2022jun/p*.json",
-         gt_key="b1-aqa-chemistry-7405-1-2022jun", expected_max=105),
-    dict(slug="b1-aqa-physics-7408-1-2022jun", title="AQA A-level Physics 7408/1 2022 Jun (image-only OCR baseline)",
-         board="aqa", level="A-level", path="B1 image-only OCR (RapidOCR margin-pass)",
-         storage_loc="cc.examboards/aqa/physics/7408/1/2022-jun/qp.pdf",
-         pdf="samples/b1/aqa-physics-7408-1-2022jun.pdf",
-         docling="results/b1_rapid/b1-aqa-physics-7408-1-2022jun/merged.json",
-         rapid="results/b1_rapid/b1-aqa-physics-7408-1-2022jun/p*.json",
-         gt_key="b1-aqa-physics-7408-1-2022jun", expected_max=85),
-    dict(slug="b1-aqa-biology-8461-1h-2022jun", title="AQA GCSE Biology 8461/1H 2022 Jun (image-only OCR baseline)",
-         board="aqa", level="GCSE", path="B1 image-only OCR (RapidOCR margin-pass)",
-         storage_loc="cc.examboards/aqa/biology/8461/1h/2022-jun/qp.pdf",
-         pdf="samples/b1/aqa-biology-8461-1h-2022jun.pdf",
-         docling="results/b1_rapid/b1-aqa-biology-8461-1h-2022jun/merged.json",
-         rapid="results/b1_rapid/b1-aqa-biology-8461-1h-2022jun/p*.json",
-         gt_key="b1-aqa-biology-8461-1h-2022jun", expected_max=100),
-    dict(slug="b1-aqa-chemistry-8462-1h-2022jun", title="AQA GCSE Chemistry 8462/1H 2022 Jun (image-only OCR baseline)",
-         board="aqa", level="GCSE", path="B1 image-only OCR (RapidOCR margin-pass)",
-         storage_loc="cc.examboards/aqa/chemistry/8462/1h/2022-jun/qp.pdf",
-         pdf="samples/b1/aqa-chemistry-8462-1h-2022jun.pdf",
-         docling="results/b1_rapid/b1-aqa-chemistry-8462-1h-2022jun/merged.json",
-         rapid="results/b1_rapid/b1-aqa-chemistry-8462-1h-2022jun/p*.json",
-         gt_key="b1-aqa-chemistry-8462-1h-2022jun", expected_max=100),
-    dict(slug="b1-aqa-combined-8464-b1h-2022jun", title="AQA GCSE Combined Science Trilogy 8464/B/1H 2022 Jun (image-only OCR baseline)",
-         board="aqa", level="GCSE", path="B1 image-only OCR (RapidOCR margin-pass)",
-         storage_loc="cc.examboards/aqa/combined-science-trilogy/8464/b-1h/2022-jun/qp.pdf",
-         pdf="samples/b1/aqa-combined-8464-b1h-2022jun.pdf",
-         docling="results/b1_rapid/b1-aqa-combined-8464-b1h-2022jun/merged.json",
-         rapid="results/b1_rapid/b1-aqa-combined-8464-b1h-2022jun/p*.json",
-         gt_key="b1-aqa-combined-8464-b1h-2022jun", expected_max=70),
-    dict(slug="b1-aqa-combined-8464-c1h-2022jun", title="AQA GCSE Combined Science Trilogy 8464/C/1H 2022 Jun (image-only OCR baseline; 8465 not present in dev catalogue)",
-         board="aqa", level="GCSE", path="B1 image-only OCR (RapidOCR margin-pass)",
-         storage_loc="cc.examboards/aqa/combined-science-trilogy/8464/c-1h/2022-jun/qp.pdf",
-         pdf="samples/b1/aqa-combined-8464-c1h-2022jun.pdf",
-         docling="results/b1_rapid/b1-aqa-combined-8464-c1h-2022jun/merged.json",
-         rapid="results/b1_rapid/b1-aqa-combined-8464-c1h-2022jun/p*.json",
-         gt_key="b1-aqa-combined-8464-c1h-2022jun", expected_max=70),
-]
-
-GT_LABELS_PATH = "fixtures/b1_gt_labels.json"
-
-FAST = [
-    dict(slug="aqa-physics-7408-fast", title="AQA A-level Physics 7408/1 (born-digital)", board="aqa",
-         level="A-level", pdf="samples/extra/aqa-alevel-physics-7408-1-jun22-qp.pdf",
-         gt="results/gt_extra/aqa-alevel-physics-7408-1-jun22-qp.txt"),
-    dict(slug="aqa-biology-8461-fast", title="AQA GCSE Biology 8461/1H (born-digital)", board="aqa",
-         level="GCSE", pdf="samples/extra/aqa-gcse-biology-8461-1h-jun22-qp.pdf",
-         gt="results/gt_extra/aqa-gcse-biology-8461-1h-jun22-qp.txt"),
-    dict(slug="edexcel-maths-1ma1-1h-fast", title="Edexcel GCSE Maths 1MA1/1H (born-digital)",
-         board="edexcel", level="GCSE-H", pdf="samples/extra/edexcel-gcse-maths-1ma1-1h-jun22-qp.pdf",
-         gt="results/gt_extra/edexcel-gcse-maths-1ma1-1h-jun22-qp.txt"),
-    dict(slug="edexcel-maths-1ma1-1f-fast", title="Edexcel GCSE Maths 1MA1/1F (born-digital)",
-         board="edexcel", level="GCSE-F", pdf="samples/extra/edexcel-gcse-maths-1ma1-1f-jun22-qp.pdf"),
-    dict(slug="ocr-physics-h556-fast", title="OCR A-level Physics H556/3 (born-digital)", board="ocr",
-         level="A-level", pdf="samples/extra/ocr-alevel-physics-h556-3-jun22-qp.pdf",
-         gt="results/gt_extra/ocr-alevel-physics-h556-3-jun22-qp.txt"),
-    dict(slug="aqa-chemistry-8462-fast", title="AQA GCSE Chemistry 8462/1H (born-digital)", board="aqa",
-         level="GCSE", pdf="samples/chemistry-p1h-2023-qp.pdf"),
-    dict(slug="aqa-physics-8463-twin-fast", title="AQA GCSE Physics 8463/1H born-digital twin",
-         board="aqa", level="GCSE", pdf="samples/physics-p1h-2022-qp.pdf"),
-]
-
-
-def run(cmd):
-    r = subprocess.run([PY] + cmd, capture_output=True, text=True)
-    if r.returncode != 0:
-        print(f"  ! FAILED: {' '.join(cmd)}\n{r.stderr[-400:]}")
-    return r.returncode == 0
-
-
-def jload(p):
-    try:
-        return json.load(open(p))
-    except Exception:
-        return {}
-
-
-
-def load_gt_labels():
-    try:
-        return json.load(open(GT_LABELS_PATH))
-    except Exception:
-        return {}
-
-
-def part_labels(struct):
-    labels = []
-    for q in struct.get("questions", []) or []:
-        for part in q.get("parts", []) or []:
-            lab = part.get("label")
-            if lab:
-                labels.append(lab)
-    return labels
-
-
-def coverage_against_labels(struct, labels):
-    if not labels:
-        return None
-    rec = set(part_labels(struct))
-    gt = set(labels)
-    hit = sorted(rec & gt)
-    miss = sorted(gt - rec)
-    return {"coverage_pct": round(len(hit) / len(gt) * 100, 1),
-            "recovered": len(hit), "total": len(gt), "missed": miss,
-            "source": "fixtures/b1_gt_labels.json"}
-
-
-def answer_region_count(struct):
-    top = len(struct.get("regions", []) or [])
-    per_part = 0
-    for q in struct.get("questions", []) or []:
-        for part in q.get("parts", []) or []:
-            per_part += len(part.get("regions", []) or [])
-    return top + per_part
-
-
-def ensure_rapid_cache(p):
-    if os.path.exists(p["docling"]):
-        return True
-    if not os.path.exists(p["pdf"]):
-        print(f"  ! missing source PDF for {p['slug']}: {p['pdf']} (storage_loc={p.get('storage_loc')})")
-        return False
-    return run(["scripts/rapid_pass.py", p["pdf"], "b1_rapid/" + p["slug"]])
-
-def stats_from(struct, val, gt_labels=None):
-    st = struct.get("stats", {}) or {}
-    mc = st.get("marks_check") or {}
-    cov = coverage_against_labels(struct, gt_labels) if gt_labels else (struct.get("coverage", {}) or {})
-    return {
-        "board": struct.get("board"), "paper_code": struct.get("paper_code"),
-        "n_questions": st.get("n_questions"), "n_parts": st.get("n_parts"),
-        "marks_sum": mc.get("sum"), "official_max": mc.get("expected_max"),
-        "marks_pct": mc.get("pct"),
-        "coverage_pct": cov.get("coverage_pct"), "coverage_recovered": cov.get("recovered"),
-        "coverage_total": cov.get("total"), "coverage_source": cov.get("source"),
-        "coverage_missed": cov.get("missed", []), "answer_regions": answer_region_count(struct),
-        "opencv_answer_regions": st.get("opencv_answer_regions"),
-        "opencv_answer_region_candidates": st.get("opencv_answer_region_candidates"),
-        "residual_marks_gapfilled": st.get("residual_marks_gapfilled"),
-        "validate_verdict": (val.get("summary") or {}).get("worst_severity"),
-        "validate_flags": val.get("flags", []),
-        "questions_expected": (val.get("summary") or {}).get("questions_expected"),
-        "questions_recovered": (val.get("summary") or {}).get("questions_recovered"),
-        "second_pass_slots": [q["label"] for q in val.get("question_sequence", []) if not q["recovered"]],
-    }
-
-
-def do_geometry(p, overlays, gt_labels=None, prepare_ocr=False):
-    d = os.path.join(FINAL, p["slug"]); os.makedirs(d, exist_ok=True)
-    S, F, B, R, T, V = (os.path.join(d, f) for f in
-                        ("structured.json", "furniture.json", "bands.json", "page_roles.json",
-                         "template.json", "validate.json"))
-    if prepare_ocr and not ensure_rapid_cache(p):
-        raise RuntimeError(f"unable to prepare B1 OCR cache for {p['slug']}")
-    extract_args = p.get("extract") or ["--docling", p["docling"], "--rapid", p["rapid"], "--board", p.get("board", "aqa")]
-    ex = ["extract.py"] + extract_args + ["--out", S]
-    if p.get("pdf"):
-        ex += ["--response-regions", p["pdf"]]
-    if p.get("expected_max"):
-        ex += ["--expected-max", str(p["expected_max"])]
-    if p.get("gt"):
-        ex += ["--gt", p["gt"]]
-    run(ex)
-    run(["furniture.py", p["docling"], "--out", F])
-    bands = ["bands.py", S, "--docling", p["docling"], "--out", B]
-    if p.get("rapid"):
-        bands += ["--rapid", p["rapid"]]
-    run(bands)
-    run(["page_roles.py", p["docling"], "--bands", B, "--out", R])
-    run(["template.py", "--structured", S, "--bands", B, "--furniture", F,
-         "--page-roles", R, "--pdf", p["pdf"], "--out", T])
-    run(["validate.py", S, "--out", V])
-    if overlays:
-        otpl = os.path.join(d, "overlays", "template")
-        run(["scripts/overlay.py", S, p["pdf"], "--template", T, "--dpi", "120", "--out", otpl])
-        # rich debug view on the first few pages (cover + early questions)
-        odbg = os.path.join(d, "overlays", "debug")
-        run(["scripts/overlay.py", S, p["pdf"], "--docling", p["docling"], "--bands", B,
-             "--furniture", F, "--pages", "1,2,3,4,5", "--dpi", "120", "--out", odbg])
-    return stats_from(jload(S), jload(V), gt_labels), d
-
-
-def do_fast(p):
-    d = os.path.join(FINAL, p["slug"]); os.makedirs(d, exist_ok=True)
-    S = os.path.join(d, "structured.json"); V = os.path.join(d, "validate.json")
-    ex = ["extract.py", "--text", p["pdf"], "--out", S]
-    if p.get("gt"):
-        ex += ["--gt", p["gt"]]
-    run(ex)
-    run(["validate.py", S, "--out", V])
-    return stats_from(jload(S), jload(V)), d
-
-
-def per_paper_report(p, s, d, kind):
-    n_imgs = len(glob.glob(os.path.join(d, "overlays", "**", "*.png"), recursive=True))
-    lines = [f"# {p['title']}", "",
-             f"- **slug:** `{p['slug']}`  ·  **board:** {p['board']}  ·  **level:** {p['level']}  "
-             f"·  **path:** {kind}",
-             f"- **questions/parts:** {s['n_questions']} / {s['n_parts']}",
-             f"- **marks:** {s['marks_sum']}/{s['official_max']}"
-             + (f" ({s['marks_pct']}% of official max)" if s['marks_pct'] is not None else ""),
-             f"- **coverage vs GT:** {s['coverage_pct']}%"
-             + (f"  (missed {s['coverage_missed'][:8]})" if s.get('coverage_missed') else "")
-             if s['coverage_pct'] is not None else "- **coverage vs GT:** n/a",
-             f"- **G6 verdict:** {s['validate_verdict']}",
-             f"- **answer-region count:** {s.get('answer_regions')}",
-             f"- **opencv response regions:** {s.get('opencv_answer_regions')} attached / "
-             f"{s.get('opencv_answer_region_candidates')} candidates",
-             ]
-    if s["validate_flags"]:
-        lines += ["", "**Flags (human-review hints):**"] + [f"- {f}" for f in s["validate_flags"]]
-    lines += ["", "**Artifacts:** `structured.json`, `validate.json`"
-              + (", `furniture.json`, `bands.json`, `page_roles.json`, `template.json`, "
-                 f"`overlays/` ({n_imgs} images)" if kind != "born-digital fast-path"
-                 else "  (born-digital: no page geometry → no overlays)")]
-    open(os.path.join(d, "report.md"), "w").write("\n".join(lines) + "\n")
-    return n_imgs
-
-
-def main():
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--no-overlays", action="store_true")
-    ap.add_argument("--b1-only", action="store_true", help="run only the Sprint B1 image-only OCR eval corpus")
-    ap.add_argument("--prepare-ocr", action="store_true", help="populate missing B1 RapidOCR caches via dsync before running")
-    a = ap.parse_args()
-    os.makedirs(FINAL, exist_ok=True)
-    catalog = {"generated_at": datetime.datetime.now().isoformat(timespec="seconds"),
-               "papers": []}
-    total_imgs = 0
-
-    gt_fixtures = load_gt_labels()
-    geometry = B1_GEOMETRY if a.b1_only else GEOMETRY
-    fast = [] if a.b1_only else FAST
-
-    for p in geometry:
-        print(f"[geometry] {p['slug']}")
-        gt_labels = (gt_fixtures.get(p.get("gt_key") or p["slug"], {}) or {}).get("labels")
-        s, d = do_geometry(p, not a.no_overlays, gt_labels=gt_labels, prepare_ocr=a.prepare_ocr)
-        n = per_paper_report(p, s, d, p["path"])
-        total_imgs += n
-        catalog["papers"].append({**{k: p[k] for k in ("slug", "title", "board", "level")},
-                                  "kind": "geometry", "path": p["path"], "dir": d,
-                                  "overlay_images": n, **s})
-    for p in fast:
-        print(f"[fast] {p['slug']}")
-        s, d = do_fast(p)
-        per_paper_report(p, s, d, "born-digital fast-path")
-        catalog["papers"].append({**{k: p[k] for k in ("slug", "title", "board", "level")},
-                                  "kind": "fast", "path": "born-digital fast-path", "dir": d, **s})
-
-    json.dump(catalog, open(os.path.join(FINAL, "catalog.json"), "w"), indent=2)
-    write_index(catalog, total_imgs)
-    print(f"\n-> {len(catalog['papers'])} papers, {total_imgs} overlay images -> {FINAL}/")
-
-
-def write_index(catalog, total_imgs):
-    g = [p for p in catalog["papers"] if p["kind"] == "geometry"]
-    f = [p for p in catalog["papers"] if p["kind"] == "fast"]
-    L = ["# Final corpus output — exam-extraction spike", "",
-         f"Generated {catalog['generated_at']}. {len(catalog['papers'])} paper-runs across "
-         f"3 boards × 2 levels, both pipeline paths; {total_imgs} overlay debug images.", "",
-         "Each `<slug>/` holds the machine artifacts (JSON) + `report.md`; geometry papers also have "
-         "`overlays/template/` (human-review view, all pages) and `overlays/debug/` (raw-detection view).",
-         "Machine catalog: `catalog.json`.", "",
-         "## Image-only / OCR-path (with geometry + overlays)", "",
-         "| Paper | Board / level | Q/parts | Marks/max | Coverage | Answer regions | G6 | Images |",
-         "|---|---|---|---|---|---|---|---|"]
-    for p in g:
-        cov = f"{p['coverage_pct']}%" if p['coverage_pct'] is not None else "n/a"
-        L.append(f"| [{p['title']}]({p['slug']}/report.md) | {p['board']} {p['level']} | "
-                 f"{p['n_questions']}/{p['n_parts']} | {p['marks_sum']}/{p['official_max']} "
-                 f"({p['marks_pct']}%) | {cov} | {p.get('answer_regions')} | {p['validate_verdict']} | "
-                 f"{p['overlay_images']} |")
-    L += ["", "## Born-digital fast-path (CPU, no geometry)", "",
-          "| Paper | Board / level | Q/parts | Marks/max | Coverage | G6 |",
-          "|---|---|---|---|---|---|"]
-    for p in f:
-        L.append(f"| [{p['title']}]({p['slug']}/report.md) | {p['board']} {p['level']} | "
-                 f"{p['n_questions']}/{p['n_parts']} | {p['marks_sum']}/{p['official_max']} "
-                 f"({p['marks_pct']}%) | {p['coverage_pct'] if p['coverage_pct'] is not None else 'n/a'}% | "
-                 f"{p['validate_verdict']} |")
-    L += ["", "## Per-paper directory layout", "```",
-          "<slug>/",
-          "  structured.json     extract.py output (questions->parts->marks/bbox/regions)",
-          "  validate.json       G6 consistency judge (confidence + flags)",
-          "  furniture.json      recurring-furniture mask + content margins   [geometry only]",
-          "  bands.json          main + part y-bands                          [geometry only]",
-          "  page_roles.json     per-page role + margin override              [geometry only]",
-          "  template.json       editable first-pass template (source/confirmed) [geometry only]",
-          "  overlays/template/  human-review view, all pages                 [geometry only]",
-          "  overlays/debug/     raw-detection view, sample pages             [geometry only]",
-          "  report.md           per-paper human summary", "```"]
-    open(os.path.join(FINAL, "INDEX.md"), "w").write("\n".join(L) + "\n")
-
-
-if __name__ == "__main__":
-    main()

api/services/docling/fixtures/b1_gt_labels.json

@@ -1,356 +0,0 @@
-{
-  "b1-aqa-biology-7402-1-2023jun": {
-    "source_pdf": "cc.examboards/aqa/biology/7402/1/2023-jun/qp.pdf",
-    "source_method": "AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.",
-    "board_detected": "aqa",
-    "paper_code_detected": "7402/1",
-    "labels": [
-      "01.1",
-      "01.2",
-      "01.3",
-      "02.1",
-      "02.2",
-      "02.3",
-      "03.1",
-      "03.2",
-      "03.3",
-      "03.4",
-      "03.5",
-      "04.1",
-      "04.2",
-      "04.3",
-      "05.1",
-      "05.2",
-      "05.3",
-      "05.4",
-      "05.5",
-      "06.1",
-      "06.2",
-      "06.3",
-      "06.4",
-      "07.1",
-      "07.2",
-      "89.6",
-      "08.1",
-      "08.2",
-      "08.3",
-      "08.4",
-      "09.1",
-      "09.2",
-      "09.3",
-      "09.4",
-      "09.5",
-      "09.6",
-      "10.1",
-      "10.2",
-      "10.3"
-    ]
-  },
-  "b1-aqa-chemistry-7405-1-2022jun": {
-    "source_pdf": "cc.examboards/aqa/chemistry/7405/1/2022-jun/qp.pdf",
-    "source_method": "AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.",
-    "board_detected": "aqa",
-    "paper_code_detected": "7405/1",
-    "labels": [
-      "01.1",
-      "01.2",
-      "01.3",
-      "01.4",
-      "01.5",
-      "01.6",
-      "02.1",
-      "02.2",
-      "02.3",
-      "02.4",
-      "02.5",
-      "03.1",
-      "03.2",
-      "03.3",
-      "03.4",
-      "03.5",
-      "04.1",
-      "04.2",
-      "04.3",
-      "04.4",
-      "04.5",
-      "05.1",
-      "05.2",
-      "05.3",
-      "05.4",
-      "05.5",
-      "05.6",
-      "05.7",
-      "06.1",
-      "06.2",
-      "06.3",
-      "06.4",
-      "06.5",
-      "06.6",
-      "06.7",
-      "07.1",
-      "07.2",
-      "07.3",
-      "07.4",
-      "07.5",
-      "07.6",
-      "07.7",
-      "08.1",
-      "08.2",
-      "08.3",
-      "08.4",
-      "08.5"
-    ]
-  },
-  "b1-aqa-physics-7408-1-2022jun": {
-    "source_pdf": "cc.examboards/aqa/physics/7408/1/2022-jun/qp.pdf",
-    "source_method": "AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.",
-    "board_detected": "aqa",
-    "paper_code_detected": "7408/1",
-    "labels": [
-      "01.1",
-      "01.2",
-      "01.3",
-      "01.4",
-      "01.5",
-      "02.1",
-      "02.2",
-      "02.3",
-      "02.4",
-      "03.1",
-      "03.2",
-      "03.3",
-      "03.4",
-      "03.5",
-      "04.1",
-      "04.2",
-      "04.3",
-      "04.4",
-      "04.5",
-      "05.1",
-      "05.2",
-      "05.3",
-      "05.4",
-      "05.5",
-      "05.6",
-      "06.1",
-      "06.2",
-      "06.3",
-      "07.0",
-      "08.0",
-      "09.0",
-      "10.0",
-      "11.0",
-      "12.0",
-      "13.0",
-      "14.0",
-      "15.0",
-      "16.0",
-      "17.0",
-      "18.0",
-      "19.0",
-      "20.0",
-      "21.0",
-      "22.0",
-      "23.0",
-      "24.0",
-      "25.0",
-      "26.0",
-      "27.0",
-      "28.0",
-      "29.0",
-      "30.0",
-      "31.0"
-    ]
-  },
-  "b1-aqa-biology-8461-1h-2022jun": {
-    "source_pdf": "cc.examboards/aqa/biology/8461/1h/2022-jun/qp.pdf",
-    "source_method": "AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.",
-    "board_detected": "aqa",
-    "paper_code_detected": "8461/1",
-    "labels": [
-      "01.1",
-      "01.2",
-      "01.3",
-      "01.4",
-      "01.5",
-      "01.6",
-      "01.7",
-      "01.8",
-      "01.9",
-      "02.1",
-      "02.2",
-      "02.3",
-      "02.4",
-      "02.5",
-      "02.6",
-      "03.1",
-      "03.2",
-      "03.3",
-      "03.4",
-      "03.5",
-      "04.1",
-      "04.2",
-      "04.3",
-      "04.4",
-      "04.5",
-      "05.1",
-      "05.2",
-      "05.3",
-      "05.4",
-      "05.5",
-      "06.1",
-      "06.2",
-      "06.3",
-      "06.4",
-      "06.5",
-      "07.1",
-      "07.2",
-      "07.3",
-      "07.4",
-      "07.5",
-      "07.6",
-      "07.7",
-      "07.8"
-    ]
-  },
-  "b1-aqa-chemistry-8462-1h-2022jun": {
-    "source_pdf": "cc.examboards/aqa/chemistry/8462/1h/2022-jun/qp.pdf",
-    "source_method": "AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.",
-    "board_detected": "aqa",
-    "paper_code_detected": "8462/1",
-    "labels": [
-      "01.1",
-      "01.2",
-      "01.3",
-      "01.4",
-      "01.5",
-      "01.6",
-      "01.7",
-      "02.1",
-      "02.2",
-      "02.3",
-      "02.4",
-      "02.5",
-      "02.6",
-      "03.1",
-      "03.2",
-      "03.3",
-      "03.4",
-      "03.5",
-      "04.1",
-      "04.2",
-      "04.3",
-      "04.4",
-      "04.5",
-      "04.6",
-      "04.7",
-      "05.1",
-      "05.2",
-      "05.3",
-      "05.4",
-      "05.5",
-      "06.1",
-      "06.2",
-      "06.3",
-      "06.4",
-      "06.5",
-      "06.6",
-      "07.1",
-      "07.2",
-      "07.3",
-      "07.4",
-      "07.5",
-      "07.6",
-      "08.1",
-      "08.2",
-      "08.3",
-      "08.4",
-      "08.5"
-    ]
-  },
-  "b1-aqa-combined-8464-b1h-2022jun": {
-    "source_pdf": "cc.examboards/aqa/combined-science-trilogy/8464/b-1h/2022-jun/qp.pdf",
-    "source_method": "AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.",
-    "board_detected": "aqa",
-    "paper_code_detected": null,
-    "labels": [
-      "01.1",
-      "01.2",
-      "01.3",
-      "01.4",
-      "01.5",
-      "01.6",
-      "01.7",
-      "01.8",
-      "02.1",
-      "02.2",
-      "02.3",
-      "02.4",
-      "02.5",
-      "02.6",
-      "02.7",
-      "03.1",
-      "03.2",
-      "03.3",
-      "03.4",
-      "03.5",
-      "03.6",
-      "03.7",
-      "04.1",
-      "04.2",
-      "04.3",
-      "04.4",
-      "04.5",
-      "05.1",
-      "05.2",
-      "05.3",
-      "05.4",
-      "05.5",
-      "05.6",
-      "06.1",
-      "06.2",
-      "06.3"
-    ]
-  },
-  "b1-aqa-combined-8464-c1h-2022jun": {
-    "source_pdf": "cc.examboards/aqa/combined-science-trilogy/8464/c-1h/2022-jun/qp.pdf",
-    "source_method": "AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.",
-    "board_detected": "aqa",
-    "paper_code_detected": null,
-    "labels": [
-      "01.1",
-      "01.2",
-      "01.3",
-      "01.4",
-      "01.5",
-      "02.1",
-      "02.2",
-      "02.3",
-      "02.4",
-      "02.5",
-      "03.0",
-      "04.1",
-      "04.2",
-      "04.3",
-      "04.4",
-      "04.5",
-      "04.6",
-      "04.7",
-      "05.1",
-      "05.2",
-      "05.3",
-      "05.4",
-      "06.1",
-      "06.2",
-      "06.3",
-      "06.4",
-      "06.5",
-      "07.1",
-      "07.2",
-      "07.3",
-      "07.4",
-      "07.5",
-      "07.6"
-    ]
-  }
-}

api/services/docling/furniture.py

@@ -1,119 +0,0 @@
-#!/usr/bin/env python3
-"""
-furniture.py — detect recurring page chrome by cross-page repetition; derive content margins;
-reclassify pictures (real figure vs barcode/QR/header furniture). The first-pass mask.
-
-Principle: an item at ~the same (x,y) on many pages is **chrome, not question content**. This
-needs no classifier — pure positional recurrence — and it solves the genuine gap the overlay
-surfaced (the app-generated QR top-right and the foot barcode being mislabelled context_figure),
-including the QR that bleeds past the margin. It also yields the content margins so stage-2 analysis
-can be fed only the question/response region.
-
-Outputs a mask + margins JSON, and an A/B summary (figure false-positives before vs after masking).
-
-Usage:
-  python furniture.py <docling_doc.json> [--freq 0.4] [--out results/furniture.json]
-"""
-import json, argparse
-from collections import defaultdict
-
-GRID = 24          # pt — position quantisation; items sharing a cell across pages are "recurring"
-
-
-def gather(doc):
-    out = []
-    for key in ("texts", "pictures", "tables"):
-        for it in doc.get(key, []):
-            prov = it.get("prov") or []
-            bb = prov[0].get("bbox") if prov else None
-            pg = prov[0].get("page_no") if prov else None
-            if bb and pg:
-                out.append({"page": pg, "kind": key[:-1], "label": it.get("label", key[:-1]),
-                            "bbox": bb, "text": (it.get("text") or "")[:40]})
-    return out
-
-
-def cell(bb):
-    return (round((bb["l"] + bb["r"]) / 2 / GRID), round((bb["t"] + bb["b"]) / 2 / GRID))
-
-
-def detect(items, n_pages, freq):
-    """Flag each item furniture=True if its position-cell appears on >= freq*n_pages pages."""
-    pages_at = defaultdict(set)
-    for it in items:
-        pages_at[cell(it["bbox"])].add(it["page"])
-    fcells = {c: len(p) for c, p in pages_at.items() if len(p) >= freq * n_pages}
-    for it in items:
-        it["furniture"] = cell(it["bbox"]) in fcells
-    return fcells
-
-
-def content_margins(items):
-    """Content x-band + per-page content bbox from NON-furniture items (what stage-2 should see)."""
-    body = [it for it in items if not it["furniture"]]
-    if not body:
-        return None
-    lefts = sorted(it["bbox"]["l"] for it in body)
-    rights = sorted(it["bbox"]["r"] for it in body)
-    band = {"x_left": round(lefts[max(0, len(lefts) // 20)], 1),       # 5th pct — robust to strays
-            "x_right": round(rights[min(len(rights) - 1, len(rights) * 19 // 20)], 1)}
-    per_page = {}
-    bp = defaultdict(list)
-    for it in body:
-        bp[it["page"]].append(it["bbox"])
-    for pg, bbs in bp.items():
-        per_page[pg] = {"top": round(max(b["t"] for b in bbs), 1),
-                        "bottom": round(min(b["b"] for b in bbs), 1),
-                        "left": round(min(b["l"] for b in bbs), 1),
-                        "right": round(max(b["r"] for b in bbs), 1)}
-    return {"content_x_band": band, "per_page": per_page}
-
-
-def main():
-    ap = argparse.ArgumentParser()
-    ap.add_argument("doc")
-    ap.add_argument("--freq", type=float, default=0.40, help="recurrence fraction => furniture")
-    ap.add_argument("--out", default="results/furniture.json")
-    a = ap.parse_args()
-    doc = json.load(open(a.doc))
-    items = gather(doc)
-    n_pages = len({it["page"] for it in items})
-    fcells = detect(items, n_pages, a.freq)
-    margins = content_margins(items)
-
-    pics = [it for it in items if it["kind"] == "picture"]
-    pics_furn = [it for it in pics if it["furniture"]]
-    txt_furn = [it for it in items if it["kind"] == "text" and it["furniture"]]
-    # break furniture pictures down by cell (which recurring object)
-    by_cell = defaultdict(list)
-    for it in pics_furn:
-        by_cell[cell(it["bbox"])].append(it)
-
-    result = {
-        "n_pages": n_pages, "freq_threshold": a.freq,
-        "furniture_cells": {f"{c[0]},{c[1]}": n for c, n in sorted(fcells.items())},
-        "content_margins": margins,
-        "ab_test_figures": {
-            "context_figure_before_mask": len(pics),
-            "context_figure_after_mask": len(pics) - len(pics_furn),
-            "removed_as_furniture": len(pics_furn),
-            "removed_breakdown": {f"cell {c[0]},{c[1]}": len(v) for c, v in sorted(by_cell.items())},
-        },
-        "text_furniture_removed": len(txt_furn),
-        "items": items,   # each carries furniture flag — consumed by overlay.py --furniture
-    }
-    json.dump(result, open(a.out, "w"))
-
-    ab = result["ab_test_figures"]
-    print(f"pages {n_pages}  freq>={a.freq}  furniture cells: {result['furniture_cells']}")
-    print(f"content x-band: {margins['content_x_band'] if margins else None}")
-    print(f"\nA/B — figure (picture) classification:")
-    print(f"  context_figure BEFORE mask : {ab['context_figure_before_mask']}")
-    print(f"  context_figure AFTER  mask : {ab['context_figure_after_mask']}")
-    print(f"  removed as furniture       : {ab['removed_as_furniture']}  {ab['removed_breakdown']}")
-    print(f"  text furniture removed     : {result['text_furniture_removed']} (page numbers / 'Turn over' / headers)")
-    print(f"-> wrote {a.out}")
-
-
-if __name__ == "__main__":
-    main()

api/services/docling/page_roles.py

@@ -1,88 +0,0 @@
-#!/usr/bin/env python3
-"""
-page_roles.py — tag every page with a structural role (the first-pass page-layout pass).
-
-Roles: cover / question / continuation / blank / appendix. Drives two things in the template:
-  * the human sees the paper's shape (which pages are non-question), and
-  * MARGINS are disabled on pages that have no content column (cover, blank) — the override the
-    user asked for ("the front page doesn't have margins").
-
-Signals (deterministic, no GPU): per-page non-space char count, cover/boilerplate keywords, and
-whether the page carries a question band. Output feeds template.py via --page-roles.
-
-Usage:
-  python page_roles.py <docling_doc.json> --bands <bands.json> [--out results/page_roles/x.json]
-"""
-import json, argparse
-from collections import defaultdict
-
-BLANK_MAX = 130          # non-space chars at/below which a page is boilerplate-only (blank)
-COVER_KW = ("time allowed", "instructions", "materials", "information for")
-BLANK_KW = ("blank page", "no questions printed", "no questions are printed")
-APPENDIX_KW = ("data sheet", "formula", "periodic table", "insert", "resource booklet")
-
-# pages where there is no content column -> margins do not apply (the user's override case)
-NO_MARGIN_ROLES = {"cover", "blank"}
-
-
-def page_text(doc):
-    chars, blob = defaultdict(int), defaultdict(list)
-    for t in doc.get("texts", []):
-        prov = t.get("prov") or []
-        pg = prov[0].get("page_no") if prov else None
-        if pg:
-            s = t.get("text") or ""
-            chars[pg] += sum(1 for c in s if not c.isspace())
-            blob[pg].append(s.lower())
-    return chars, {pg: " ".join(v) for pg, v in blob.items()}
-
-
-def tag(doc, qpages):
-    chars, blob = page_text(doc)
-    n = max([*chars, *qpages, 1])
-    first_q = min(qpages) if qpages else n + 1
-    last_q = max(qpages) if qpages else 0
-    roles = {}
-    for pg in range(1, n + 1):
-        b = blob.get(pg, "")
-        if pg in qpages:
-            role = "question"
-        elif pg < first_q and any(k in b for k in COVER_KW):
-            role = "cover"                   # before blank: the cover's instructions mention "blank"
-        elif chars[pg] <= BLANK_MAX or (any(k in b for k in BLANK_KW) and chars[pg] < 300):
-            role = "blank"
-        elif any(k in b for k in APPENDIX_KW):
-            role = "appendix"
-        elif first_q <= pg <= last_q:
-            role = "continuation"           # no question label but inside the question range
-        else:
-            role = "appendix"               # content outside the question range (end-matter/insert)
-        roles[pg] = {"role": role, "chars": chars[pg],
-                     "margins_enabled": role not in NO_MARGIN_ROLES,
-                     "source": "auto", "confirmed": False}
-    return roles
-
-
-def main():
-    ap = argparse.ArgumentParser()
-    ap.add_argument("doc")
-    ap.add_argument("--bands", required=True)
-    ap.add_argument("--out", default="results/page_roles.json")
-    a = ap.parse_args()
-    bands = json.load(open(a.bands))
-    qpages = {int(p) for p in bands["pages"]}
-    roles = tag(json.load(open(a.doc)), qpages)
-    json.dump({"pages": roles}, open(a.out, "w"), indent=2)
-    from collections import Counter
-    c = Counter(v["role"] for v in roles.values())
-    print(f"roles: {dict(c)}")
-    for pg in sorted(roles):
-        r = roles[pg]
-        flag = "" if r["margins_enabled"] else "  (no margins)"
-        if r["role"] != "question":
-            print(f"  p{pg:2d}: {r['role']:12s} chars={r['chars']}{flag}")
-    print(f"-> wrote {a.out}")
-
-
-if __name__ == "__main__":
-    main()

api/services/docling/regions.py

@@ -1,435 +0,0 @@
-"""OpenCV response-region detector for exam template auto-map.
-
-This module is intentionally a best-effort spike.  It detects visual writing
-areas (ruled answer lines and rectangular answer boxes) from rendered exam PDF
-pages and returns mapper-friendly candidate dictionaries.  The caller may ignore
-this output entirely; manual drawing remains the fallback.
-
-Candidate schema (``detect_response_regions_from_pdf`` return item)::
-
-    {
-        "kind": "response",
-        "source": "ai",
-        "confirmed": False,
-        "confidence": 0.0..1.0,
-        "page_index": 0,                 # zero-based PDF page index
-        "bbox": {                        # rendered-page pixel coordinates
-            "x": 72.0, "y": 210.0,
-            "w": 420.0, "h": 86.0,
-            "coord_origin": "TOPLEFT",
-            "unit": "px",
-        },
-        "region_type": "answer_lines" | "answer_box" | "working_space",
-        "detection_method": "opencv_horizontal_lines" | "opencv_contour_box",
-        "line_count": 3,                 # answer_lines only
-        "meta": {...},
-    }
-
-The mapper can persist these as ``exam_response_areas`` with
-``kind='response'``, ``source='ai'``, ``confirmed=false`` after converting the
-rendered-page pixel bbox into the app's canvas coordinate system if needed.
-"""
-
-from __future__ import annotations
-
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any, Iterable
-
-import fitz  # PyMuPDF
-import numpy as np
-from PIL import Image
-
-try:  # OpenCV is an optional runtime dependency until S5 wires regions in.
-    import cv2
-except ImportError as exc:  # pragma: no cover - exercised only in underbuilt envs
-    cv2 = None  # type: ignore[assignment]
-    _CV2_IMPORT_ERROR = exc
-else:  # pragma: no cover - trivial branch
-    _CV2_IMPORT_ERROR = None
-
-
-@dataclass(frozen=True)
-class RegionCandidate:
-    """Internal typed candidate before dict serialization."""
-
-    page_index: int
-    x: float
-    y: float
-    w: float
-    h: float
-    region_type: str
-    confidence: float
-    detection_method: str
-    line_count: int | None = None
-    meta: dict[str, Any] | None = None
-
-    def to_mapper_dict(self) -> dict[str, Any]:
-        candidate: dict[str, Any] = {
-            "kind": "response",
-            "source": "ai",
-            "confirmed": False,
-            "confidence": round(float(self.confidence), 3),
-            "page_index": int(self.page_index),
-            "bbox": {
-                "x": round(float(self.x), 2),
-                "y": round(float(self.y), 2),
-                "w": round(float(self.w), 2),
-                "h": round(float(self.h), 2),
-                "coord_origin": "TOPLEFT",
-                "unit": "px",
-            },
-            "region_type": self.region_type,
-            "detection_method": self.detection_method,
-        }
-        if self.line_count is not None:
-            candidate["line_count"] = int(self.line_count)
-        if self.meta:
-            candidate["meta"] = self.meta
-        return candidate
-
-
-@dataclass(frozen=True)
-class _LineSegment:
-    x: int
-    y: int
-    w: int
-    h: int
-
-    @property
-    def right(self) -> int:
-        return self.x + self.w
-
-    @property
-    def center_y(self) -> float:
-        return self.y + self.h / 2
-
-
-def detect_response_regions_from_pdf(
-    pdf_path: str | Path,
-    *,
-    dpi: int = 144,
-    max_pages: int | None = None,
-    page_indices: Iterable[int] | None = None,
-    min_confidence: float = 0.35,
-) -> list[dict[str, Any]]:
-    """Render a PDF and emit response-area candidate dictionaries.
-
-    Args:
-        pdf_path: Local PDF path.
-        dpi: Render resolution. 144 dpi gives 2 px per PDF point and is a good
-            speed/geometry compromise for the API fast path.
-        max_pages: Optional first-N-pages cap for smoke tests/spikes.
-        page_indices: Optional explicit zero-based page indices. When supplied,
-            ``max_pages`` is ignored.
-        min_confidence: Drop candidates below this confidence.
-
-    Returns:
-        List of mapper-friendly dictionaries documented in the module docstring.
-    """
-
-    if cv2 is None:
-        raise RuntimeError(
-            "OpenCV is required for answer-region detection; install "
-            "opencv-python-headless."
-        ) from _CV2_IMPORT_ERROR
-
-    if dpi <= 0:
-        raise ValueError("dpi must be positive")
-    if not 0 <= min_confidence <= 1:
-        raise ValueError("min_confidence must be between 0 and 1")
-
-    path = Path(pdf_path)
-    if not path.exists():
-        raise FileNotFoundError(path)
-
-    doc = fitz.open(path)
-    try:
-        if page_indices is None:
-            pages = range(len(doc) if max_pages is None else min(len(doc), max_pages))
-        else:
-            pages = list(page_indices)
-        candidates: list[dict[str, Any]] = []
-        zoom = dpi / 72.0
-        matrix = fitz.Matrix(zoom, zoom)
-        for page_index in pages:
-            if page_index < 0 or page_index >= len(doc):
-                continue
-            pix = doc[page_index].get_pixmap(matrix=matrix, alpha=False)
-            image = Image.frombytes("RGB", (pix.width, pix.height), pix.samples)
-            page_candidates = detect_response_regions_from_image(
-                image,
-                page_index=page_index,
-                min_confidence=min_confidence,
-            )
-            for candidate in page_candidates:
-                item = candidate.to_mapper_dict()
-                item.setdefault("meta", {}).update({
-                    "page_width_px": pix.width,
-                    "page_height_px": pix.height,
-                    "page_width_pdf": float(doc[page_index].rect.width),
-                    "page_height_pdf": float(doc[page_index].rect.height),
-                    "render_dpi": dpi,
-                })
-                candidates.append(item)
-        return candidates
-    finally:
-        doc.close()
-
-
-def detect_response_regions_from_image(
-    image: Image.Image | np.ndarray,
-    *,
-    page_index: int = 0,
-    min_confidence: float = 0.35,
-) -> list[RegionCandidate]:
-    """Detect response-area candidates on one rendered page image."""
-
-    if cv2 is None:
-        raise RuntimeError(
-            "OpenCV is required for answer-region detection; install "
-            "opencv-python-headless."
-        ) from _CV2_IMPORT_ERROR
-    if not 0 <= min_confidence <= 1:
-        raise ValueError("min_confidence must be between 0 and 1")
-
-    page = _as_rgb_array(image)
-    gray = cv2.cvtColor(page, cv2.COLOR_RGB2GRAY)
-    binary = _ink_mask(gray)
-
-    height, width = gray.shape[:2]
-    line_candidates = _detect_answer_lines(binary, page_index=page_index, width=width, height=height)
-    box_candidates = _detect_answer_boxes(binary, page_index=page_index, width=width, height=height)
-    candidates = _dedupe_candidates(line_candidates + box_candidates)
-    return [c for c in candidates if c.confidence >= min_confidence]
-
-
-def _as_rgb_array(image: Image.Image | np.ndarray) -> np.ndarray:
-    if isinstance(image, Image.Image):
-        return np.asarray(image.convert("RGB"))
-    array = np.asarray(image)
-    if array.ndim == 2:
-        return np.stack([array, array, array], axis=-1)
-    if array.shape[-1] == 4:
-        return array[:, :, :3]
-    return array
-
-
-def _ink_mask(gray: np.ndarray) -> np.ndarray:
-    """Return a binary mask where printed dark ink is 255."""
-
-    blurred = cv2.GaussianBlur(gray, (3, 3), 0)
-    return cv2.adaptiveThreshold(
-        blurred,
-        255,
-        cv2.ADAPTIVE_THRESH_MEAN_C,
-        cv2.THRESH_BINARY_INV,
-        31,
-        12,
-    )
-
-
-def _detect_answer_lines(binary: np.ndarray, *, page_index: int, width: int, height: int) -> list[RegionCandidate]:
-    # Long horizontal strokes are answer lines. A wide kernel removes text while
-    # retaining ruled lines; min length scales with the page so it works across
-    # A4/letter and DPI values.
-    min_line_width = max(80, int(width * 0.22))
-    kernel_width = max(30, int(width * 0.08))
-    kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (kernel_width, 1))
-    horizontal = cv2.morphologyEx(binary, cv2.MORPH_OPEN, kernel, iterations=1)
-    contours, _ = cv2.findContours(horizontal, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
-
-    segments: list[_LineSegment] = []
-    for contour in contours:
-        x, y, w, h = cv2.boundingRect(contour)
-        if w < min_line_width:
-            continue
-        if h > max(10, int(height * 0.012)):
-            continue
-        # Ignore page borders / header separator lines.
-        if y < height * 0.05 or y > height * 0.96:
-            continue
-        segments.append(_LineSegment(x=x, y=y, w=w, h=max(h, 1)))
-
-    if not segments:
-        return []
-
-    segments.sort(key=lambda seg: (seg.center_y, seg.x))
-    grouped = _group_line_segments(segments, width=width, height=height)
-
-    candidates: list[RegionCandidate] = []
-    for group in grouped:
-        if not group:
-            continue
-        x0 = min(seg.x for seg in group)
-        x1 = max(seg.right for seg in group)
-        y0 = min(seg.y for seg in group)
-        y1 = max(seg.y + seg.h for seg in group)
-        line_count = len(group)
-
-        # Expand vertical bbox so it covers the student-writing band, not just
-        # the 1px strokes. Single underline answers get a modest band above the
-        # line; multi-line answers cover the lines plus inter-line whitespace.
-        if line_count == 1:
-            pad_top = max(18, int(height * 0.018))
-            pad_bottom = max(8, int(height * 0.008))
-        else:
-            gaps = [group[i + 1].center_y - group[i].center_y for i in range(line_count - 1)]
-            median_gap = float(np.median(gaps)) if gaps else height * 0.025
-            pad_top = max(10, int(median_gap * 0.45))
-            pad_bottom = max(8, int(median_gap * 0.35))
-
-        box_x = max(0, x0 - 4)
-        box_y = max(0, y0 - pad_top)
-        box_w = min(width, x1 + 4) - box_x
-        box_h = min(height, y1 + pad_bottom) - box_y
-        if box_w <= 0 or box_h <= 0:
-            continue
-
-        span_ratio = box_w / max(width, 1)
-        count_bonus = min(0.2, max(0, line_count - 1) * 0.05)
-        confidence = min(0.92, 0.42 + span_ratio * 0.35 + count_bonus)
-        region_type = "answer_lines"
-        candidates.append(
-            RegionCandidate(
-                page_index=page_index,
-                x=box_x,
-                y=box_y,
-                w=box_w,
-                h=box_h,
-                region_type=region_type,
-                confidence=confidence,
-                detection_method="opencv_horizontal_lines",
-                line_count=line_count,
-                meta={"line_segments": [{"x": s.x, "y": s.y, "w": s.w, "h": s.h} for s in group]},
-            )
-        )
-    return candidates
-
-
-def _group_line_segments(segments: list[_LineSegment], *, width: int, height: int) -> list[list[_LineSegment]]:
-    groups: list[list[_LineSegment]] = []
-    current: list[_LineSegment] = []
-    max_gap = max(28, int(height * 0.045))
-    min_x_overlap_ratio = 0.35
-
-    for segment in segments:
-        if not current:
-            current = [segment]
-            continue
-        previous = current[-1]
-        y_gap = segment.center_y - previous.center_y
-        overlap = max(0, min(segment.right, previous.right) - max(segment.x, previous.x))
-        narrower = max(1, min(segment.w, previous.w))
-        similar_x = overlap / narrower >= min_x_overlap_ratio or abs(segment.x - previous.x) < width * 0.08
-        if 2 <= y_gap <= max_gap and similar_x:
-            current.append(segment)
-        else:
-            groups.append(current)
-            current = [segment]
-    if current:
-        groups.append(current)
-    return groups
-
-
-def _detect_answer_boxes(binary: np.ndarray, *, page_index: int, width: int, height: int) -> list[RegionCandidate]:
-    # Close gaps in ruled rectangles, then contour them. This catches table-like
-    # working boxes and explicit answer boxes without trying to understand text.
-    kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 5))
-    closed = cv2.morphologyEx(binary, cv2.MORPH_CLOSE, kernel, iterations=1)
-    contours, _ = cv2.findContours(closed, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
-
-    candidates: list[RegionCandidate] = []
-    min_area = width * height * 0.003
-    max_area = width * height * 0.55
-    for contour in contours:
-        x, y, w, h = cv2.boundingRect(contour)
-        area = w * h
-        if area < min_area or area > max_area:
-            continue
-        if w < width * 0.16 or h < height * 0.025:
-            continue
-        if y < height * 0.04 or y + h > height * 0.98:
-            continue
-        aspect = w / max(h, 1)
-        if aspect < 1.2:
-            continue
-
-        contour_area = cv2.contourArea(contour)
-        rectangularity = min(1.0, contour_area / max(area, 1))
-        if rectangularity < 0.03:
-            continue
-        confidence = min(0.88, 0.46 + min(0.24, w / width * 0.24) + min(0.18, h / height * 0.5))
-        region_type = "working_space" if (h > height * 0.12 and rectangularity < 0.18) else "answer_box"
-        padded_x = max(0, x - 2)
-        padded_y = max(0, y - 2)
-        padded_right = min(width, x + w + 2)
-        padded_bottom = min(height, y + h + 2)
-        candidates.append(
-            RegionCandidate(
-                page_index=page_index,
-                x=padded_x,
-                y=padded_y,
-                w=padded_right - padded_x,
-                h=padded_bottom - padded_y,
-                region_type=region_type,
-                confidence=confidence,
-                detection_method="opencv_contour_box",
-                meta={"rectangularity": round(float(rectangularity), 3)},
-            )
-        )
-    return candidates
-
-
-def _dedupe_candidates(candidates: list[RegionCandidate]) -> list[RegionCandidate]:
-    """Remove lower-confidence candidates that substantially overlap."""
-
-    kept: list[RegionCandidate] = []
-    for candidate in sorted(candidates, key=lambda c: c.confidence, reverse=True):
-        if all(_iou(candidate, existing) < 0.55 for existing in kept):
-            kept.append(candidate)
-    kept.sort(key=lambda c: (c.page_index, c.y, c.x))
-    return kept
-
-
-def _iou(a: RegionCandidate, b: RegionCandidate) -> float:
-    if a.page_index != b.page_index:
-        return 0.0
-    ax1, ay1, ax2, ay2 = a.x, a.y, a.x + a.w, a.y + a.h
-    bx1, by1, bx2, by2 = b.x, b.y, b.x + b.w, b.y + b.h
-    ix1, iy1 = max(ax1, bx1), max(ay1, by1)
-    ix2, iy2 = min(ax2, bx2), min(ay2, by2)
-    iw, ih = max(0.0, ix2 - ix1), max(0.0, iy2 - iy1)
-    intersection = iw * ih
-    union = a.w * a.h + b.w * b.h - intersection
-    return intersection / union if union > 0 else 0.0
-
-
-def main() -> None:
-    """Small CLI for smoke testing: python -m api.services.docling.regions PDF."""
-
-    import argparse
-    import json
-
-    parser = argparse.ArgumentParser(description="Detect answer-region candidates in an exam PDF")
-    parser.add_argument("pdf", help="PDF path")
-    parser.add_argument("--dpi", type=int, default=144)
-    parser.add_argument("--max-pages", type=int, default=None)
-    parser.add_argument("--min-confidence", type=float, default=0.35)
-    args = parser.parse_args()
-
-    print(
-        json.dumps(
-            detect_response_regions_from_pdf(
-                args.pdf,
-                dpi=args.dpi,
-                max_pages=args.max_pages,
-                min_confidence=args.min_confidence,
-            ),
-            indent=2,
-        )
-    )
-
-
-if __name__ == "__main__":  # pragma: no cover
-    main()

api/services/docling/scripts/fetch_b1_corpus.py

@@ -1,87 +0,0 @@
-#!/usr/bin/env python3
-"""Populate the gitignored B1 image-only eval corpus from the .94 exam-board store.
-
-The B1 eval papers are NOT committed (third-party copyright; served only via signed URLs).
-This script downloads each B1_GEOMETRY paper's `storage_loc` object from cc.examboards via the
-Storage API into its local `pdf` path (under samples/b1/), so finalize.py --b1-only and the
-B1-2/B1-3 generalization work can run against a real corpus.
-
-Run from api/services/docling/ inside the cc-api-dev container (SUPABASE_URL/SERVICE_ROLE_KEY in env):
-    python3 scripts/fetch_b1_corpus.py            # fetch all B1 papers (skip existing)
-    python3 scripts/fetch_b1_corpus.py --force    # re-download
-    python3 scripts/fetch_b1_corpus.py --only b1-aqa-physics-7408-1-2022jun
-    python3 scripts/fetch_b1_corpus.py --list     # show what would be fetched, no download
-"""
-from __future__ import annotations
-
-import argparse
-import os
-import sys
-
-# Import the canonical B1 corpus definition (slug, storage_loc, local pdf path) from finalize.
-_HERE = os.path.dirname(os.path.abspath(__file__))
-_DOCLING_DIR = os.path.dirname(_HERE)
-sys.path.insert(0, _DOCLING_DIR)
-from finalize import B1_GEOMETRY  # noqa: E402
-
-
-def _split_storage_loc(storage_loc: str) -> tuple[str, str]:
-    """'cc.examboards/aqa/biology/7402/1/2023-jun/qp.pdf' -> ('cc.examboards', 'aqa/.../qp.pdf')."""
-    bucket, _, path = storage_loc.partition("/")
-    if not bucket or not path:
-        raise ValueError(f"malformed storage_loc: {storage_loc!r}")
-    return bucket, path
-
-
-def _entries(only: str | None):
-    for p in B1_GEOMETRY:
-        loc = p.get("storage_loc")
-        pdf = p.get("pdf")
-        if not loc or not pdf:
-            continue
-        if only and p.get("slug") != only:
-            continue
-        yield p["slug"], loc, pdf
-
-
-def main() -> int:
-    ap = argparse.ArgumentParser(description="Fetch the B1 image-only eval corpus from .94 cc.examboards")
-    ap.add_argument("--force", action="store_true", help="re-download even if the local file exists")
-    ap.add_argument("--only", help="fetch a single paper by slug")
-    ap.add_argument("--list", action="store_true", help="list what would be fetched and exit")
-    args = ap.parse_args()
-
-    todo = list(_entries(args.only))
-    if not todo:
-        print("no matching B1 papers", file=sys.stderr)
-        return 1
-
-    if args.list:
-        for slug, loc, pdf in todo:
-            print(f"{slug}\t{loc}\t-> {pdf}")
-        return 0
-
-    from modules.database.supabase.utils.storage import StorageAdmin
-    storage = StorageAdmin()
-
-    ok = skipped = 0
-    for slug, loc, pdf in todo:
-        dest = os.path.join(_DOCLING_DIR, pdf) if not os.path.isabs(pdf) else pdf
-        if os.path.exists(dest) and not args.force:
-            print(f"[skip] {slug} (exists)")
-            skipped += 1
-            continue
-        bucket, path = _split_storage_loc(loc)
-        data = storage.download_file(bucket, path)
-        os.makedirs(os.path.dirname(dest), exist_ok=True)
-        with open(dest, "wb") as fh:
-            fh.write(data)
-        print(f"[ok]   {slug} <- {bucket}/{path} ({len(data)} bytes)")
-        ok += 1
-
-    print(f"fetched {ok}, skipped {skipped}, of {len(todo)}")
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

api/services/docling/scripts/make_b1_gt.py

@@ -1,32 +0,0 @@
-import json, sys
-from pathlib import Path
-base=Path('/app/api/services/docling')
-sys.path.insert(0, str(base))
-import extract
-papers=[
-('b1-aqa-biology-7402-1-2023jun','samples/b1/aqa-biology-7402-1-2023jun.pdf','cc.examboards/aqa/biology/7402/1/2023-jun/qp.pdf'),
-('b1-aqa-chemistry-7405-1-2022jun','samples/b1/aqa-chemistry-7405-1-2022jun.pdf','cc.examboards/aqa/chemistry/7405/1/2022-jun/qp.pdf'),
-('b1-aqa-physics-7408-1-2022jun','samples/b1/aqa-physics-7408-1-2022jun.pdf','cc.examboards/aqa/physics/7408/1/2022-jun/qp.pdf'),
-('b1-aqa-biology-8461-1h-2022jun','samples/b1/aqa-biology-8461-1h-2022jun.pdf','cc.examboards/aqa/biology/8461/1h/2022-jun/qp.pdf'),
-('b1-aqa-chemistry-8462-1h-2022jun','samples/b1/aqa-chemistry-8462-1h-2022jun.pdf','cc.examboards/aqa/chemistry/8462/1h/2022-jun/qp.pdf'),
-('b1-aqa-combined-8464-b1h-2022jun','samples/b1/aqa-combined-8464-b1h-2022jun.pdf','cc.examboards/aqa/combined-science-trilogy/8464/b-1h/2022-jun/qp.pdf'),
-('b1-aqa-combined-8464-c1h-2022jun','samples/b1/aqa-combined-8464-c1h-2022jun.pdf','cc.examboards/aqa/combined-science-trilogy/8464/c-1h/2022-jun/qp.pdf'),
-]
-out={}
-for slug, rel, storage in papers:
-    lines=extract.lines_from_pdftext(str(base/rel))
-    board, code=extract.detect_board(lines)
-    if board != 'aqa':
-        raise RuntimeError(f'{slug}: expected AQA board, detected {board!r} ({code!r})')
-    parts=extract.parse_text_by_board(lines, board)
-    labels=list(parts)
-    out[slug]={
-        'source_pdf': storage,
-        'source_method': 'AQA born-digital text-layer parsed with existing extract.py AQA grammar; used as reproducible GT label set for image-only OCR baseline.',
-        'board_detected': board,
-        'paper_code_detected': code,
-        'labels': labels,
-    }
-    print(slug, board, code, len(labels), labels[:5], labels[-5:])
-Path(base/'fixtures').mkdir(exist_ok=True)
-Path(base/'fixtures/b1_gt_labels.json').write_text(json.dumps(out, indent=2)+"\n")

api/services/docling/scripts/overlay.py

@@ -1,310 +0,0 @@
-#!/usr/bin/env python3
-"""
-overlay.py — human-viewable debug visualisation: draw the extractor's geometry over the rendered
-exam page. Shows WHERE each question/part label was located and where Docling regions
-(figures/tables/MCQ checkboxes) sit, so a reviewer can eyeball whether the structure landed in the
-right place. This is the same geometry the exam-marker app uses to place regions on its canvas.
-
-Coordinates: Docling/RapidOCR bboxes are PDF points with a BOTTOM-LEFT origin. We render the page
-at DPI D (scale = D/72) and flip y against the rendered image height, so we never need the page's
-point-height explicitly: y_top_px = H_px - t*scale.
-
-With --docling, also draws every raw Docling text block (the body/question content the thin
-extractor model discards) so a reviewer can see the FULL detection, not just what we persist.
-Granite tables carry cells but no coordinates; we derive their box by locating the cell-texts in
-the Docling text layer (content+geometry fusion).
-
-Usage:
-  python scripts/overlay.py <structured.json> <source_pdf> [--pages 3,4,5] [--dpi 150] [--out DIR]
-  python scripts/overlay.py <structured.json> <pdf> --docling results/E_tess_full.json --pages 5
-"""
-import os, sys, json, re, argparse, subprocess, tempfile
-from PIL import Image, ImageDraw, ImageFont
-
-PART_COLOR = (211, 47, 47)                          # red — question/part labels
-BODY_COLOR = (150, 150, 150)                         # grey — raw Docling body-text blocks (--docling)
-GRANITE_COLOR = (0, 150, 136)                        # teal — Granite table (geometry derived from cells)
-REGION_COLORS = {                                   # docling region taxonomy -> colour
-    "context_figure": (25, 118, 210),               # blue
-    "context_data": (56, 142, 60),                  # green (tables)
-    "context_caption": (123, 31, 162),              # purple
-    "mcq_option": (245, 124, 0),                     # orange (checkboxes)
-}
-
-
-def _norm(s):
-    return re.sub(r"[^a-z0-9]", "", (s or "").lower())
-
-
-def docling_texts_by_page(doc):
-    """All raw Docling text items -> {page: [(bbox, text, label)]}. The body content we discard."""
-    out = {}
-    for t in doc.get("texts", []):
-        prov = t.get("prov") or []
-        bb = prov[0].get("bbox") if prov else None
-        pg = prov[0].get("page_no") if prov else None
-        if bb and pg:
-            out.setdefault(pg, []).append((bb, t.get("text") or "", t.get("label") or "text"))
-    return out
-
-
-def derive_table_bbox(grid, page_texts):
-    """Granite tables have cells but no coordinates. Locate the cell-texts in the Docling text
-    layer and union their bboxes -> the table's on-page extent.
-
-    Two traps (seen on physics p5): (1) border/maths glyphs ('|','+') normalise to '' and an
-    empty string is a substring of everything; (2) cell WORDS recur in nearby content — the rock
-    names reappear in the MCQ options below the table ('Basalt or chalk'), far left and lower.
-    So we match only blocks whose normalised text is CONTAINED IN a cell (keeps fragments like
-    '2.90'/'Type', rejects the longer 'basaltorchalk'), require length >= 2, then keep the
-    dominant vertical cluster to drop any stray cell-word elsewhere on the page."""
-    import statistics
-    cells = {c for c in (_norm(x) for row in grid for x in row) if len(c) > 1}
-    hit = [bb for bb, txt, _ in page_texts
-           if len(_norm(txt)) > 1 and any(_norm(txt) in c for c in cells)]
-    if len(hit) < 3:
-        return None
-    med = statistics.median(sorted((b["t"] + b["b"]) / 2 for b in hit))
-    hit = [b for b in hit if abs((b["t"] + b["b"]) / 2 - med) <= 120]   # table band only
-    return {"l": min(b["l"] for b in hit), "r": max(b["r"] for b in hit),
-            "t": max(b["t"] for b in hit), "b": min(b["b"] for b in hit)}
-
-
-def _font(sz):
-    for p in ("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",
-              "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"):
-        if os.path.exists(p):
-            return ImageFont.truetype(p, sz)
-    return ImageFont.load_default()
-
-
-MAIN_LINE = (25, 118, 210)                          # blue — main-question y-markers
-PART_LINE = (211, 47, 47)                            # red — part y-markers
-
-
-def _hline(draw, y_pdf, scale, H, W, color, label, width, font, dashed=False, inset=0):
-    """Full-width horizontal marker line at a PDF-point y (BOTTOM-LEFT origin)."""
-    y = H - y_pdf * scale
-    if dashed:
-        x = inset
-        while x < W:
-            draw.line([x, y, min(x + 9, W), y], fill=color, width=width); x += 16
-    else:
-        draw.line([inset, y, W, y], fill=color, width=width)
-    if label:
-        tw = draw.textlength(label, font=font)
-        draw.rectangle([inset, y - 16, inset + tw + 6, y], fill=color)
-        draw.text((inset + 3, y - 15), label, fill=(255, 255, 255), font=font)
-
-
-def _rect(draw, bb, scale, H, color, label, width=3, font=None):
-    """Draw one bbox (BOTTOM-LEFT origin -> image space) + its label."""
-    x0, x1 = bb["l"] * scale, bb["r"] * scale
-    y0, y1 = H - bb["t"] * scale, H - bb["b"] * scale      # t is the higher edge -> smaller y_px
-    draw.rectangle([x0, y0, x1, y1], outline=color, width=width)
-    if label:
-        tw = draw.textlength(label, font=font)
-        draw.rectangle([x0, y0 - 17, x0 + tw + 6, y0], fill=color)
-        draw.text((x0 + 3, y0 - 16), label, fill=(255, 255, 255), font=font)
-
-
-def draw_template(draw, tpl, pg, scale, H, W, font):
-    """Render the editable template for one page: margins/bands as LINES, footprints as BOXES.
-    A confirmed element is drawn solid; an unconfirmed (auto) suggestion is drawn dashed."""
-    MARGIN, MAIN, PART = (0, 150, 136), (25, 118, 210), (211, 47, 47)
-    page = tpl["pages"].get(str(pg)) or tpl["pages"].get(pg) or {}
-    # role banner (top-left); margins suppressed entirely on no-margin pages (cover/blank)
-    role = page.get("role", "question")
-    draw.rectangle([0, 0, 8 + len(role) * 8, 16], fill=(70, 70, 70))
-    draw.text((4, 1), f"role:{role}", fill=(255, 255, 255), font=font)
-    margins_on = page.get("margins_enabled", True)
-    # margins: axis-locked lines (document scope on every page + this page's page-scope lines)
-    for m in (tpl.get("margins", []) if margins_on else []):
-        if m["scope"] == "page" and m.get("page") != pg:
-            continue
-        solid = m.get("confirmed")
-        if m["axis"] == "x":
-            x = m["value"] * scale
-            draw.line([x, 0, x, H], fill=MARGIN, width=2) if solid else _dash_v(draw, x, 0, H, MARGIN, 2)
-        else:
-            y = H - m["value"] * scale
-            draw.line([0, y, W, y], fill=MARGIN, width=2) if solid else _dash_h(draw, 0, W, y, MARGIN, 2)
-    for m in page.get("main_bands", []):
-        if not m.get("is_start", True):          # continuation page: no spurious second "start" line
-            continue
-        _hline(draw, m["y_start"], scale, H, W, MAIN, f"Q{m['question']}", 3, font,
-               dashed=not m.get("confirmed"))
-    for p in page.get("part_bands", []):
-        _hline(draw, p["y_start"], scale, H, W, PART, p["label"], 2, font, inset=90,
-               dashed=not p.get("confirmed"))
-    for f in page.get("furniture", []):
-        if f.get("box"):
-            _rect(draw, f["box"], scale, H, (130, 130, 130), f"furniture:{f.get('kind','')}", 2, font)
-    for g in page.get("figures", []):
-        if g.get("box"):
-            _rect(draw, g["box"], scale, H, (56, 142, 60), "figure", 3, font)
-    for t in page.get("tables", []):
-        if t.get("box"):
-            _rect(draw, t["box"], scale, H, (0, 150, 136),
-                  f"table {t.get('n_rows')}x{t.get('n_cols')}", 3, font)
-
-
-def render_page(pdf, pg, dpi, td):
-    """Render page `pg` and return an image in DOCLING's coordinate space. Docling reports bbox
-    relative to the CropBox, but pdftoppm renders the MediaBox — when CropBox != MediaBox (e.g. the
-    Edexcel 1MA1 papers: media 652x899, crop inset 28.35pt) that mismatch magnifies + shifts every
-    overlaid shape toward a corner. Fix: crop the rendered image to the CropBox so it matches Docling.
-    No-op when CropBox == MediaBox (h556) or when poppler already rendered the CropBox."""
-    base = os.path.join(td, f"p{pg}")
-    subprocess.run(["pdftoppm", "-png", "-r", str(dpi), "-f", str(pg), "-l", str(pg), pdf, base],
-                   check=True)
-    png = next(p for p in (f"{base}-{pg:02d}.png", f"{base}-{pg}.png", f"{base}-{pg:03d}.png")
-               if os.path.exists(p))
-    img = Image.open(png).convert("RGB")
-    try:
-        import pypdf
-        page = pypdf.PdfReader(pdf).pages[pg - 1]
-        mb, cb = page.mediabox, page.cropbox
-        scale = dpi / 72.0
-        mbl, mbt = float(mb.left), float(mb.top)
-        dcrop = any(abs(a - b) > 0.5 for a, b in
-                    ((cb.left, mb.left), (cb.bottom, mb.bottom), (cb.right, mb.right), (cb.top, mb.top)))
-        rendered_mediabox = abs(img.width - (float(mb.right) - mbl) * scale) < 3
-        if dcrop and rendered_mediabox:
-            img = img.crop((round((float(cb.left) - mbl) * scale), round((mbt - float(cb.top)) * scale),
-                            round((float(cb.right) - mbl) * scale), round((mbt - float(cb.bottom)) * scale)))
-    except Exception:
-        pass
-    return img
-
-
-def _dash_v(draw, x, y0, y1, color, w):
-    y = y0
-    while y < y1:
-        draw.line([x, y, x, min(y + 9, y1)], fill=color, width=w); y += 16
-
-
-def _dash_h(draw, x0, x1, y, color, w):
-    x = x0
-    while x < x1:
-        draw.line([x, y, min(x + 9, x1), y], fill=color, width=w); x += 16
-
-
-def main():
-    ap = argparse.ArgumentParser()
-    ap.add_argument("structured"); ap.add_argument("pdf")
-    ap.add_argument("--docling", help="raw Docling doc JSON: also draw every body-text block "
-                                      "(the content the thin model discards) + derive Granite-table boxes")
-    ap.add_argument("--bands", help="bands.py JSON: draw main-question + part start/end y-marker lines")
-    ap.add_argument("--furniture", help="furniture.py JSON: mark recurring furniture vs real figures "
-                                        "+ draw the content x-margins")
-    ap.add_argument("--template", help="template.py JSON: render the editable first-pass template "
-                                       "(margins+bands as lines, furniture/figures as boxes). "
-                                       "When set, draws ONLY the template (the human-review view).")
-    ap.add_argument("--pages", help="comma list, e.g. 3,4,5 (default: all pages with geometry)")
-    ap.add_argument("--dpi", type=int, default=150)
-    ap.add_argument("--out", default="results/overlay")
-    a = ap.parse_args()
-    os.makedirs(a.out, exist_ok=True)
-    scale = a.dpi / 72.0
-    font = _font(14)
-
-    res = json.load(open(a.structured))
-    doc_texts = docling_texts_by_page(json.load(open(a.docling))) if a.docling else {}
-    bands = json.load(open(a.bands))["pages"] if a.bands else {}
-    furn = json.load(open(a.furniture)) if a.furniture else None
-    tpl = json.load(open(a.template)) if a.template else None
-    # gather geometry by page
-    parts_by_pg, regions_by_pg = {}, {}
-    for q in res.get("questions", []):
-        for p in q["parts"]:
-            if p.get("bbox") and p.get("page"):
-                parts_by_pg.setdefault(p["page"], []).append((p["label"], p["bbox"]))
-    for r in res.get("regions", []):
-        if r.get("bbox") and r.get("page"):
-            regions_by_pg.setdefault(r["page"], []).append((r["type"], r["bbox"]))
-    # tables: standard ones carry a bbox; Granite ones don't -> derive from the text layer
-    tables_by_pg = {}
-    for t in res.get("tables", []):
-        pg = t.get("page")
-        if not pg:
-            continue
-        bb = t.get("bbox") or (derive_table_bbox(t.get("grid", []), doc_texts.get(pg, []))
-                               if a.docling else None)
-        if bb:
-            tables_by_pg.setdefault(pg, []).append(
-                (f"table {t.get('source','')} {t.get('n_rows')}x{t.get('n_cols')}", bb))
-
-    want = ([int(x) for x in a.pages.split(",")] if a.pages
-            else (sorted(int(p) for p in tpl["pages"]) if tpl
-                  else sorted(set(parts_by_pg) | set(regions_by_pg) | set(doc_texts))))
-    if not want:
-        sys.exit("no bbox geometry in this result (born-digital text path carries no geometry; "
-                 "use an OCR/rapid-path structured.json)")
-
-    written = []
-    with tempfile.TemporaryDirectory() as td:
-        for pg in want:
-            img = render_page(a.pdf, pg, a.dpi, td)
-            H = img.height
-            draw = ImageDraw.Draw(img)
-            if tpl:                          # template-only render = the human-review view
-                draw_template(draw, tpl, pg, scale, H, img.width, font)
-                out = os.path.join(a.out, f"p{pg:02d}.png")
-                img.save(out); written.append(out)
-                pgd = tpl["pages"].get(str(pg), {})
-                print(f"p{pg}: template — {len(pgd.get('main_bands',[]))} main, "
-                      f"{len(pgd.get('part_bands',[]))} part, {len(pgd.get('furniture',[]))} furn, "
-                      f"{len(pgd.get('figures',[]))} fig -> {out}")
-                continue
-            # layer 0: raw Docling body-text blocks (faint, no label) — the discarded content
-            for bb, txt, lab in doc_texts.get(pg, []):
-                _rect(draw, bb, scale, H, BODY_COLOR, None, 1, font)
-            # layer 1: taxonomy regions
-            for typ, bb in regions_by_pg.get(pg, []):
-                _rect(draw, bb, scale, H, REGION_COLORS.get(typ, (120, 120, 120)), typ, 2, font)
-            # layer 2: tables (Granite-derived boxes in teal)
-            for lab, bb in tables_by_pg.get(pg, []):
-                _rect(draw, bb, scale, H, GRANITE_COLOR, lab, 3, font)
-            # layer 3: part labels on top
-            for lab, bb in parts_by_pg.get(pg, []):
-                _rect(draw, bb, scale, H, PART_COLOR, lab, 3, font)
-            # layer 4: band y-marker lines (main-question = blue, part = red dashed; end = dashed)
-            pb = bands.get(str(pg)) or bands.get(pg)
-            nb = 0
-            if pb:
-                W = img.width
-                for m in pb["main"]:
-                    if not m.get("is_start", True):     # skip continuation-page duplicate
-                        continue
-                    _hline(draw, m["y_start"], scale, H, W, MAIN_LINE,
-                           f"Q{m['question']} ▸ start", 3, font); nb += 1
-                    _hline(draw, m["y_end"], scale, H, W, MAIN_LINE, None, 1, font, dashed=True)
-                for p in pb["part"]:
-                    _hline(draw, p["y_start"], scale, H, W, PART_LINE,
-                           f"{p['label']} start", 2, font, inset=90); nb += 1
-            # layer 5: furniture mask — green=real figure, grey=masked furniture; + content margins
-            if furn:
-                W = img.width
-                for it in furn["items"]:
-                    if it["page"] != pg or it["kind"] != "picture":
-                        continue
-                    if it["furniture"]:
-                        _rect(draw, it["bbox"], scale, H, (130, 130, 130), "furniture", 2, font)
-                    else:
-                        _rect(draw, it["bbox"], scale, H, (56, 142, 60), "figure ✓", 3, font)
-                band = (furn.get("content_margins") or {}).get("content_x_band")
-                if band:
-                    for xk in ("x_left", "x_right"):
-                        x = band[xk] * scale
-                        draw.line([x, 0, x, H], fill=(0, 150, 136), width=2)
-            out = os.path.join(a.out, f"p{pg:02d}.png")
-            img.save(out); written.append(out)
-            print(f"p{pg}: {len(parts_by_pg.get(pg,[]))} part-labels, "
-                  f"{len(regions_by_pg.get(pg,[]))} regions, {len(tables_by_pg.get(pg,[]))} tables, "
-                  f"{len(doc_texts.get(pg,[]))} body-text blocks, {nb} band-lines -> {out}")
-    print(f"-> {len(written)} page(s) in {a.out}/")
-
-
-if __name__ == "__main__":
-    main()

api/services/docling/scripts/rapid_pass.py

@@ -1,69 +0,0 @@
-#!/usr/bin/env python3
-"""
-rapid_pass.py — generalise the proven AQA "RapidOCR margin-pass" (95.2% on the image-only
-8463 paper) to any AQA paper. Born-digital AQA QPs ship a text layer, so we force RapidOCR
-over the *rendered* page (`force_ocr:true`) to simulate the image-only redistribution case
-and recover the boxed `NN.M` question numbers Tesseract shatters.
-
-For each page it writes results/<outdir>/p{N}.json (a full per-page DoclingDocument, the
-shape extract.py's aqa_questions_rapid expects) and a merged.json (for board / front-matter
-detection). All GPU work is serialised + OOM-resilient through dsync.
-
-Usage:
-  python scripts/rapid_pass.py samples/extra/aqa-alevel-physics-7408-1-jun22-qp.pdf rapid_7408
-  python scripts/rapid_pass.py <pdf> <outdir-slug> [first_page] [last_page]
-"""
-import os, sys, json, subprocess, re
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-import dsync
-
-OPTS = {"ocr_engine": "rapidocr", "force_ocr": True}
-
-
-def npages(pdf):
-    out = subprocess.check_output(["pdfinfo", pdf]).decode()
-    return int(out.split("Pages:")[1].split()[0])
-
-
-def main():
-    pdf = sys.argv[1]
-    slug = sys.argv[2]
-    if os.path.isabs(slug) or ".." in slug.split(os.sep) or not re.fullmatch(r"[A-Za-z0-9._/-]+", slug):
-        raise SystemExit(f"unsafe output slug: {slug!r}")
-    n = npages(pdf)
-    first = int(sys.argv[3]) if len(sys.argv) > 3 else 1
-    last = min(int(sys.argv[4]), n) if len(sys.argv) > 4 else n
-    if first > n or first > last:
-        print(f"requested page range {first}-{last} is outside PDF ({n} pages); nothing to do")
-        return
-    outdir = os.path.join("results", slug)
-    os.makedirs(outdir, exist_ok=True)
-
-    r = dsync._redis()
-    print(f"redis: {'connected' if r else 'NO CACHE'}  pdf={pdf}  pages {first}-{last}/{n}")
-    merged = {"texts": [], "tables": [], "pictures": [], "pages": {}, "_failed_pages": []}
-    for pg in range(first, last + 1):
-        page_path = os.path.join(outdir, f"p{pg}.json")
-        if os.path.exists(page_path):
-            doc = json.load(open(page_path))
-            print(f"  p{pg}: file cache HIT ({len(doc.get(texts, []))} texts)")
-        else:
-            doc = dsync.convert_page(pdf, pg, OPTS, r=r)
-        if not doc:
-            merged["_failed_pages"].append(pg)
-            print(f"  p{pg}: FAILED")
-            continue
-        json.dump(doc, open(page_path, "w"))
-        for k in ("texts", "tables", "pictures"):
-            merged[k].extend(doc.get(k, []))
-        merged["pages"].update(doc.get("pages", {}))
-        nmarg = sum(1 for t in doc.get("texts", [])
-                    if (t.get("prov") or [{}])[0].get("bbox", {}).get("l", 999) <= 140)
-        print(f"  p{pg}: {len(doc.get('texts', []))} texts ({nmarg} left-margin)")
-    json.dump(merged, open(os.path.join(outdir, "merged.json"), "w"))
-    print(f"-> {outdir}/  ({last-first+1-len(merged['_failed_pages'])} pages, "
-          f"failed={merged['_failed_pages']})")
-
-
-if __name__ == "__main__":
-    main()

api/services/docling/tables.py

@@ -1,210 +0,0 @@
-#!/usr/bin/env python3
-"""
-tables.py — selective table-cell extraction for the exam extractor (PLAN.md §B).
-
-Two sources, unified into one cell-grid schema:
-  * STANDARD  — the Tesseract+TableFormer backbone already emits `tables[].data.table_cells`
-      (text + row/col offsets + spans + bbox). Free, cached, every run. Good on ruled tables;
-      but it MISSES some data tables and OCRs them as loose tokens (REPORT.md p5).
-  * GRANITE   — Granite-Docling-258M VLM emits `<otsl>` grids in DocTags (clean rows/cols even
-      where the backbone scrambles them). GPU cost, so used SELECTIVELY: only on pages the router
-      flags (a standard table present, or dense picture/checkbox), routed through dsync's GPU lock
-      + Redis cache. Recipe (REPORT.md): {"to_formats":["doctags","json"], "pipeline":"vlm",
-      "vlm_pipeline_model":"granite_docling"}.
-
-Unified table = {page, n_rows, n_cols, grid (2D text), cells, caption, source, is_furniture}.
-"""
-import re, json, os, glob, base64, urllib.request
-
-# ----------------------------------------------------------------- OTSL (Granite DocTags)
-OTSL_BLOCK = re.compile(r"<otsl>(.*?)</otsl>", re.S)
-CAPTION    = re.compile(r"<caption>(?:<loc_\d+>)*(.*?)</caption>", re.S)
-CELL_TOK   = re.compile(r"<(fcel|ecel|ched|rhed|lcel|ucel|xcel|nl)>([^<]*)")
-HEADER_TAGS = {"ched", "rhed"}
-
-
-def parse_otsl(doctags):
-    """Parse every <otsl> block in a DocTags string into unified tables."""
-    out = []
-    for block in OTSL_BLOCK.findall(doctags):
-        cap = None
-        mc = CAPTION.search(block)
-        if mc:
-            cap = re.sub(r"\s+", " ", mc.group(1)).strip()
-        body = CAPTION.sub("", block)
-        body = re.sub(r"<loc_\d+>", "", body)
-        rows, cur = [], []
-        for tag, txt in CELL_TOK.findall(body):
-            if tag == "nl":
-                rows.append(cur); cur = []
-            else:
-                cur.append({"text": txt.strip(), "header": tag in HEADER_TAGS,
-                            "empty": tag == "ecel"})
-        if cur:
-            rows.append(cur)
-        rows = [r for r in rows if r]
-        if not rows:
-            continue
-        n_cols = max(len(r) for r in rows)
-        grid = [[c["text"] for c in r] + [""] * (n_cols - len(r)) for r in rows]
-        out.append({"page": None, "n_rows": len(rows), "n_cols": n_cols, "grid": grid,
-                    "caption": cap, "source": "granite-otsl",
-                    "is_furniture": is_furniture(grid, cap)})
-    return out
-
-
-# ----------------------------------------------------------------- standard TableFormer
-def tables_from_standard(doc):
-    out = []
-    for t in doc.get("tables", []):
-        data = t.get("data", {}) or {}
-        cells = data.get("table_cells", []) or []
-        nr, nc = data.get("num_rows") or 0, data.get("num_cols") or 0
-        grid = [["" for _ in range(nc)] for _ in range(nr)]
-        for c in cells:
-            r0, c0 = c.get("start_row_offset_idx"), c.get("start_col_offset_idx")
-            if r0 is not None and c0 is not None and r0 < nr and c0 < nc and c.get("text"):
-                grid[r0][c0] = c["text"]
-        prov = t.get("prov") or []
-        page = prov[0].get("page_no") if prov else None
-        cap = " ".join(x.get("text", "") for x in (t.get("captions") or []) if isinstance(x, dict)) or None
-        out.append({"page": page, "n_rows": nr, "n_cols": nc, "grid": grid,
-                    "caption": cap, "source": "docling-standard",
-                    "is_furniture": is_furniture(grid, cap)})
-    return out
-
-
-# ----------------------------------------------------------------- furniture filter
-FURNITURE_RE = re.compile(r"examiner|do not write|leave\s+blank|question\s*mark|"
-                          r"for marker|total marks?$", re.I)
-
-
-def is_furniture(grid, caption=None):
-    """A table that is exam scaffolding (mark grid / 'For Examiner's Use'), not question data."""
-    blob = " ".join(cell for row in grid for cell in row) + " " + (caption or "")
-    if FURNITURE_RE.search(blob):
-        return True
-    # a single-column strip of question numbers / blanks = a mark grid
-    flat = [c for row in grid for c in row if c.strip()]
-    if flat and all(re.fullmatch(r"\d{1,2}", c.strip()) for c in flat):
-        return True
-    return False
-
-
-# ----------------------------------------------------------------- Granite via dsync
-VLM_OPTS = {"to_formats": ["doctags", "json"], "pipeline": "vlm",
-            "vlm_pipeline_model": "granite_docling", "image_export_mode": "placeholder"}
-
-
-def _serve_vlm(pdf_b64, fname, page):
-    import dsync
-    opts = {**VLM_OPTS, "page_range": [page, page]}
-    body = {"options": opts,
-            "sources": [{"kind": "file", "base64_string": pdf_b64, "filename": fname}],
-            "target": {"kind": "inbody"}}
-    req = urllib.request.Request(dsync.SERVE + "/v1/convert/source",
-                                 data=json.dumps(body).encode(),
-                                 headers={"Content-Type": "application/json"})
-    for _ in range(4):                                  # tolerate the single-use 404 race
-        try:
-            return json.loads(urllib.request.urlopen(req, timeout=1200).read())
-        except urllib.error.HTTPError as e:
-            if e.code == 404:
-                import time; time.sleep(3); continue
-            raise
-    raise RuntimeError("serve vlm: repeated 404")
-
-
-def _doctags_of(resp):
-    doc = resp.get("document") or {}
-    return doc.get("doctags_content") or doc.get("doc_tags") or doc.get("doctags") or ""
-
-
-def granite_tables(pdf, pages, *, cached_glob=None, retries=4):
-    """Run Granite-Docling on the given pages via dsync (GPU lock + OOM retry + Redis cache),
-    parse <otsl>, tag each table with its page. Falls back to cached *.doctags if serve fails."""
-    import dsync, time
-    cache = _load_cached_doctags(cached_glob) if cached_glob else {}
-    r = dsync._redis()
-    b64 = base64.b64encode(open(pdf, "rb").read()).decode()
-    fname = os.path.basename(pdf)
-    sha = dsync._sha(pdf)
-    out = []
-    for pg in pages:
-        key = f"docling:vlm:{sha}:p{pg}"
-        doctags = None
-        if r and (hit := r.get(key)):
-            doctags = hit if isinstance(hit, str) else hit.decode()
-        if doctags is None:
-            delay = 5
-            for attempt in range(retries):
-                with dsync._GpuLock(r):
-                    resp = _serve_vlm(b64, fname, pg)
-                if dsync._is_oom(resp):
-                    print(f"[granite] p{pg} OOM, backoff {delay}s ({attempt+1}/{retries})")
-                    time.sleep(delay); delay = min(delay * 2, 120); continue
-                doctags = _doctags_of(resp)
-                if r and doctags:
-                    r.set(key, doctags, ex=dsync.CACHE_TTL)
-                break
-        if not doctags and pg in cache:
-            print(f"[granite] p{pg} serve empty -> cached doctags")
-            doctags = cache[pg]
-        for tbl in parse_otsl(doctags or ""):
-            tbl["page"] = pg
-            out.append(tbl)
-    return out
-
-
-def _load_cached_doctags(glob_pat):
-    """Map page_no -> doctags text from files named *p<N>.doctags."""
-    cache = {}
-    for fn in glob.glob(glob_pat):
-        m = re.search(r"p(\d+)\.doctags$", fn)
-        if m:
-            cache[int(m.group(1))] = open(fn, encoding="utf-8", errors="replace").read()
-    return cache
-
-
-# ----------------------------------------------------------------- routing + attach
-def candidate_pages(doc):
-    """Pages the router sends to Granite: a standard table, or a dense picture/checkbox page."""
-    pages = set()
-    for t in doc.get("tables", []):
-        prov = t.get("prov") or []
-        if prov and prov[0].get("page_no"):
-            pages.add(prov[0]["page_no"])
-    chk = {}
-    for it in doc.get("texts", []):
-        if it.get("label", "").startswith("checkbox"):
-            prov = it.get("prov") or []
-            if prov and prov[0].get("page_no"):
-                chk[prov[0]["page_no"]] = chk.get(prov[0]["page_no"], 0) + 1
-    pages |= {p for p, n in chk.items() if n >= 2}
-    return sorted(pages)
-
-
-def attach_to_questions(tables, parts):
-    """Assign each non-furniture table to the nearest preceding part on its page (by y); if no
-    geometry, attach to the first part on that page. Records table refs on the part."""
-    data_tables = [t for t in tables if not t["is_furniture"]]
-    by_page = {}
-    for lab, v in parts.items():
-        by_page.setdefault(v.get("page"), []).append((lab, v))
-    for i, t in enumerate(data_tables):
-        t["id"] = i
-        cands = by_page.get(t["page"], [])
-        if not cands:
-            t["for_part"] = None; continue
-        # best-effort: the part highest on the page (largest bbox top = the page's question stem),
-        # else the earliest part label. (Tables sit under the stem; we don't carry table y here.)
-        with_geo = [(lab, v) for lab, v in cands if v.get("bbox")]
-        if with_geo:
-            lab = max(with_geo, key=lambda kv: (kv[1]["bbox"] or {}).get("t", 0))[0]
-        else:
-            lab = sorted(cands, key=lambda kv: kv[0])[0][0]
-        t["for_part"] = lab
-        parts[lab].setdefault("tables", []).append(
-            {"id": i, "n_rows": t["n_rows"], "n_cols": t["n_cols"],
-             "caption": t["caption"], "source": t["source"]})
-    return data_tables

api/services/docling/template.py

@@ -1,216 +0,0 @@
-#!/usr/bin/env python3
-"""
-template.py — assemble the editable first-pass structural template from the spike's three signal
-sources (extract structured.json + bands.json + furniture.json) into ONE round-trippable JSON the
-human reviewer verifies AND edits before stage-2 generates the final template.
-
-UI principle (user, 2026-06-07): directional LIMITS are draggable LINES (1-DOF, easier to drag);
-object FOOTPRINTS are BOXES. So:
-  * margins      -> four axis-locked LINES: left/right (x), top/bottom (y)
-  * question/part bands -> horizontal LINES: start/end y
-  * furniture / figures / tables -> BOXES (an object's footprint)
-
-Every editable element carries {source: "auto"|"human", confirmed: bool} — the AI-suggestion seam.
-Stage-2 must consume only confirmed elements (or a template marked confirmed at the top level).
-Coordinates are PDF points, BOTTOM-LEFT origin (units in meta); the app maps to its own canvas.
-
-Usage:
-  python template.py --structured S.json --bands B.json --furniture F.json --pdf P.pdf --out T.json
-"""
-import json, argparse, datetime
-
-
-def _line(edge, axis, value, scope, page=None):
-    o = {"edge": edge, "axis": axis, "value": round(value, 1), "scope": scope,
-         "source": "auto", "confirmed": False}
-    if page is not None:
-        o["page"] = page
-    return o
-
-
-def _furn_kind(it):
-    """Best-guess label for a furniture box (human can rename). Position-based, BOTTOM-LEFT origin."""
-    bb = it["bbox"]; cx = (bb["l"] + bb["r"]) / 2; cy = (bb["t"] + bb["b"]) / 2
-    if it["kind"] == "picture":
-        if cx > 430 and cy > 700:
-            return "qr"
-        if cy < 110:
-            return "barcode"
-        return "chrome_picture"
-    if cy < 90:
-        return "footer"
-    if cy > 760:
-        return "header_or_page_number"
-    return "chrome_text"
-
-
-def synthesize_part_box(part_band, content_x_band):
-    """Return the one authoritative S5 part-box projection.
-
-    Parts remain boxes in S5, but the box is a projection rather than intrinsic
-    geometry: document content margins provide the x-extent and the part band
-    provides y. The band end is already bounded by the next part in bands.py;
-    the original label box remains a separate anchor for rendering/review.
-
-    Coordinates stay in the first-pass PDF-point BOTTOMLEFT bbox shape.
-    """
-    if not content_x_band:
-        return None
-    try:
-        x_left = content_x_band["x_left"]
-        x_right = content_x_band["x_right"]
-        y_start = part_band["y_start"]
-        y_end = part_band["y_end"]
-    except KeyError:
-        return None
-    return {
-        "l": round(x_left, 1),
-        "t": round(y_start, 1),
-        "r": round(x_right, 1),
-        "b": round(y_end, 1),
-        "coord_origin": "BOTTOMLEFT",
-    }
-
-
-def build(structured, bands, furniture, pdf=None, page_roles=None):
-    page_roles = page_roles or {}
-    part_bbox = {p["label"]: p.get("bbox")
-                 for q in structured.get("questions", []) for p in q["parts"]}
-    cm = furniture.get("content_margins") or {}
-    xband = cm.get("content_x_band") or {}
-    per_pg_m = cm.get("per_page") or {}
-
-    def margins_on(pg):
-        r = page_roles.get(str(pg)) or page_roles.get(pg)
-        return r.get("margins_enabled", True) if r else True
-
-    # margins as axis-locked LINES — document-level left/right, per-page top/bottom. Per-page
-    # top/bottom are omitted for pages with no content column (cover/blank) — the user's override.
-    margins = []
-    if "x_left" in xband:
-        margins.append(_line("left", "x", xband["x_left"], "document"))
-        margins.append(_line("right", "x", xband["x_right"], "document"))
-    for pg, m in sorted(per_pg_m.items(), key=lambda kv: int(kv[0])):
-        if not margins_on(int(pg)):
-            continue
-        margins.append(_line("top", "y", m["top"], "page", int(pg)))
-        margins.append(_line("bottom", "y", m["bottom"], "page", int(pg)))
-
-    # furniture + figures as BOXES, grouped by page
-    furn_pg, fig_pg = {}, {}
-    for it in furniture.get("items", []):
-        pg = it["page"]
-        if it.get("furniture"):
-            furn_pg.setdefault(pg, []).append(
-                {"box": it["bbox"], "kind": _furn_kind(it), "docling_label": it["label"],
-                 "source": "auto", "confirmed": False})
-        elif it["kind"] == "picture":
-            fig_pg.setdefault(pg, []).append(
-                {"box": it["bbox"], "source": "auto", "confirmed": False})
-
-    tbl_pg = {}
-    for t in structured.get("tables", []):
-        if t.get("page"):
-            tbl_pg.setdefault(t["page"], []).append(
-                {"box": t.get("bbox"), "n_rows": t.get("n_rows"), "n_cols": t.get("n_cols"),
-                 "table_source": t.get("source"), "source": "auto", "confirmed": False})
-
-    # --- reconcile against recovered part labels -------------------------------------------
-    # A part-label position is never furniture or a figure (the label wins), and a "figure" that
-    # covers most of the content area is a Docling page-collapse artifact (the GPU sometimes flags
-    # the whole page as one picture), not a real figure -> drop both. Fixes the Q1.7/Q1.9 clashes
-    # and the full-page "figure" that was masking part labels.
-    part_boxes_pg = {}
-    for q in structured.get("questions", []):
-        for p in q["parts"]:
-            if p.get("bbox") and p.get("page"):
-                part_boxes_pg.setdefault(p["page"], []).append(p["bbox"])
-
-    def _inter(a, b):
-        return not (a["r"] < b["l"] or b["r"] < a["l"] or a["t"] < b["b"] or b["t"] < a["b"])
-
-    def _area(b):
-        return max(0, b["r"] - b["l"]) * max(0, b["t"] - b["b"])
-
-    for pg, items in list(furn_pg.items()):
-        pls = part_boxes_pg.get(pg, [])
-        furn_pg[pg] = [f for f in items if not (f.get("box") and any(_inter(f["box"], pl) for pl in pls))]
-    for pg, items in list(fig_pg.items()):
-        pls = part_boxes_pg.get(pg, [])
-        m = per_pg_m.get(str(pg)) or per_pg_m.get(pg) or {}
-        carea = ((m.get("right", 0) - m.get("left", 0)) * (m.get("top", 0) - m.get("bottom", 0))) or (595 * 842)
-        fig_pg[pg] = [f for f in items if f.get("box")
-                      and _area(f["box"]) <= 0.55 * carea               # not a full-page collapse
-                      and not any(_inter(f["box"], pl) for pl in pls)]  # not clashing a part label
-
-    pages = {}
-    all_pg = (set(bands["pages"]) | {str(p) for p in furn_pg} | {str(p) for p in fig_pg}
-              | {str(p) for p in page_roles})
-    for pgs in sorted(all_pg, key=int):
-        pg = int(pgs)
-        pb = bands["pages"].get(pgs) or bands["pages"].get(pg) or {"main": [], "part": []}
-        main = [{"question": m["question"], "y_start": m["y_start"], "y_end": m["y_end"],
-                 "is_start": m.get("is_start", True),
-                 "source": "auto", "confirmed": False} for m in pb["main"]]
-        part = []
-        for p in pb["part"]:
-            part.append({
-                "label": p["label"], "question": p["question"],
-                "y_start": p["y_start"], "y_end": p["y_end"],
-                "label_box": part_bbox.get(p["label"]),  # anchor, not the part extent
-                "box": synthesize_part_box(p, xband),
-                "marks": p.get("marks"),                 # parsed per-part marks (born-digital)
-                "source": "auto", "confirmed": False,
-            })
-        pr = page_roles.get(pgs) or page_roles.get(pg) or {}
-        pages[pgs] = {
-            "role": pr.get("role", "question"),
-            "role_source": pr.get("source", "default"), "role_confirmed": pr.get("confirmed", False),
-            "margins_enabled": pr.get("margins_enabled", True),   # human-overridable
-            "main_bands": main, "part_bands": part,
-            "furniture": furn_pg.get(pg, []), "figures": fig_pg.get(pg, []),
-            "tables": tbl_pg.get(pg, []),
-        }
-
-    return {
-        "meta": {
-            "schema": "exam-template/first-pass/v1",
-            "board": structured.get("board"), "paper_code": structured.get("paper_code"),
-            "source_pdf": pdf, "n_pages": furniture.get("n_pages"),
-            "coord_origin": "BOTTOMLEFT", "units": "pdf_points",
-            "generated_at": datetime.datetime.now().isoformat(timespec="seconds"),
-            "ui_principle": "directional limits = draggable axis-locked lines; "
-                            "object footprints = boxes",
-            "confirmed": False, "confirmed_by": None, "confirmed_at": None,
-        },
-        "margins": margins,
-        "pages": pages,
-    }
-
-
-def main():
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--structured", required=True)
-    ap.add_argument("--bands", required=True)
-    ap.add_argument("--furniture", required=True)
-    ap.add_argument("--page-roles", dest="page_roles", help="page_roles.py JSON (roles + margin override)")
-    ap.add_argument("--pdf")
-    ap.add_argument("--out", default="results/template.json")
-    a = ap.parse_args()
-    roles = json.load(open(a.page_roles))["pages"] if a.page_roles else {}
-    t = build(json.load(open(a.structured)), json.load(open(a.bands)),
-              json.load(open(a.furniture)), a.pdf, roles)
-    json.dump(t, open(a.out, "w"), indent=2)
-    np = len(t["pages"])
-    nm = sum(len(p["main_bands"]) for p in t["pages"].values())
-    npt = sum(len(p["part_bands"]) for p in t["pages"].values())
-    nf = sum(len(p["furniture"]) for p in t["pages"].values())
-    ng = sum(len(p["figures"]) for p in t["pages"].values())
-    print(f"template {t['meta']['paper_code']} ({t['meta']['board']}): {np} pages, "
-          f"{len(t['margins'])} margin-lines, {nm} main-bands, {npt} part-bands, "
-          f"{nf} furniture-boxes, {ng} figure-boxes")
-    print(f"-> wrote {a.out}")
-
-
-if __name__ == "__main__":
-    main()

api/services/docling/validate.py

@@ -1,222 +0,0 @@
-#!/usr/bin/env python3
-"""
-validate.py — G6 validation/judge: a deterministic consistency pass over an extractor result.
-
-NOT a gate. It never approves or rejects; it attaches confidence + flags so a HUMAN reviewer's
-attention is routed to the parts most likely wrong. A clean paper -> all-green, skim; a flagged
-paper -> the exact items to check, worst-first. Every value stays a *suggestion* a human confirms.
-
-Checks (all deterministic, no GPU, ~free — run on every extraction):
-  C1 marks-sum vs official max      — over-read (sum>max) = error; under (sum<max) = warn
-  C2 part marks plausibility        — marks None / 0 / implausibly high
-  C3 top-level question sequence    — gaps in 1..N (skipped when numbering was OCR-inferred '~')
-  C4 sub-part contiguity            — within a question: a,b,c / .1,.2,.3 with no hole
-  C5 coverage                       — missed parts vs ground truth (when the result carries it)
-
-Usage:
-  python validate.py results/genreport/edexcel1f/ocr_struct_filled.json
-  python validate.py <structured.json> --out report.json
-"""
-import json, re, sys, argparse
-from collections import defaultdict
-
-IMPLAUSIBLE_PART_MARKS = 15        # a single sub-part above this is worth a human glance
-
-
-def _qnum(q):
-    """Numeric value of a top-level question id ('01'->1, '4'->4); None if inferred ('~3') / odd."""
-    if q.startswith("~"):
-        return None
-    m = re.match(r"^0*(\d+)$", q)
-    return int(m.group(1)) if m else None
-
-
-def _subkey(label, q):
-    """The part's own suffix within its question: '01.2'->'2', '4a'->'a', '1bi'->'bi'."""
-    s = label[len(q):] if label.startswith(q) else label
-    return s.lstrip(".").lstrip("~")
-
-
-def validate(result):
-    board = result.get("board")
-    code = result.get("paper_code")
-    flags, checks = [], []
-    parts = [(p["label"], q["question"], p) for q in result.get("questions", []) for p in q["parts"]]
-    conf = {}                                      # label -> high/medium/low
-    low = set()                                    # labels a check has implicated
-
-    def add(cid, severity, status, detail):
-        checks.append({"id": cid, "severity": severity, "status": status, "detail": detail})
-        if status != "ok":
-            flags.append(f"[{severity}] {cid}: {detail}")
-
-    # ---- C1: marks sum vs official maximum -------------------------------------------------
-    mc = result.get("stats", {}).get("marks_check")
-    exp = (mc or {}).get("expected_max") or result.get("front_matter", {}).get("max_marks")
-    msum = (mc or {}).get("sum")
-    if msum is None:
-        msum = sum(p["marks"] for *_, p in parts if p.get("marks") is not None)
-    if exp:
-        if msum > exp:
-            add("C1_marks_sum", "error", "over",
-                f"marks sum {msum} EXCEEDS official max {exp} (+{msum-exp}) — an over-read; check the paper")
-        elif msum < exp:
-            add("C1_marks_sum", "warn", "under",
-                f"marks sum {msum} below official max {exp} (-{exp-msum}) — missing parts or unread marks")
-        else:
-            add("C1_marks_sum", "info", "ok", f"marks sum {msum} == official max {exp}")
-    else:
-        add("C1_marks_sum", "info", "unknown", "no official max available to check the sum against")
-
-    # ---- C2: per-part marks plausibility ---------------------------------------------------
-    none_ct = zero_ct = 0
-    for lab, q, p in parts:
-        mk = p.get("marks")
-        if mk is None:
-            none_ct += 1; low.add(lab)
-        elif mk == 0:
-            zero_ct += 1; low.add(lab)
-        elif mk > IMPLAUSIBLE_PART_MARKS:
-            low.add(lab)
-            add("C2_part_marks", "warn", "implausible",
-                f"part {lab} has {mk} marks (> {IMPLAUSIBLE_PART_MARKS}) — verify it isn't a mis-read")
-    if none_ct or zero_ct:
-        add("C2_part_marks", "warn", "missing",
-            f"{none_ct} part(s) with no mark, {zero_ct} with 0 marks — unread/garbled mark tokens")
-    elif not any(c["id"] == "C2_part_marks" for c in checks):
-        add("C2_part_marks", "info", "ok", "every part carries a plausible mark")
-
-    # ---- C3: top-level question sequence + EXPECTED-question interpolation ------------------
-    # If Q1, Q2 ... Q14 are recovered but 3-13 are not, the paper certainly HAS 3-13 — they were
-    # just missed (e.g. a Docling page-collapse). We emit the full expected sequence with a per-Q
-    # `recovered` flag so a live question-tree view can render the gaps as explicit "needs a second
-    # pass" slots, and a targeted re-OCR knows exactly which questions to chase.
-    qids = [q for q in dict.fromkeys(q for _, q, _ in parts)]
-    nums = sorted({n for n in (_qnum(q) for q in qids) if n is not None})
-    zero_pad = any(len(q) == 2 and q.startswith("0") for q in qids)   # AQA 'NN' vs Edexcel/OCR 'N'
-    question_sequence = []
-    if any(q.startswith("~") for q in qids):
-        add("C3_question_seq", "info", "inferred",
-            "question numbers were OCR-inferred ('~N') — sequence not checkable; treat labels as approximate")
-    elif nums:
-        # isolated high outliers (a content number mis-read as 'Q67' after Q1-10) are likely
-        # spurious top-levels, not 50 missing questions — strip them off the top so the sequence
-        # reflects the real paper, and flag them for review instead of flooding the tree with slots.
-        core, suspect = nums[:], []
-        while len(core) >= 2 and core[-1] - core[-2] > 4:
-            suspect.insert(0, core.pop())
-        hi = core[-1] if core else nums[-1]
-        gaps = [n for n in range(nums[0], hi + 1) if n not in core]
-        question_sequence = [{"n": n, "label": (f"{n:02d}" if zero_pad else str(n)),
-                              "recovered": n in core} for n in range(nums[0], hi + 1)]
-        if suspect:
-            add("C3_question_seq", "warn", "spurious",
-                f"isolated high question number(s) {suspect} after a {nums[0]}-{hi} run — likely a "
-                f"content number mis-read as a top-level question; review/remove")
-        if gaps:
-            add("C3_question_seq", "warn", "gap",
-                f"top-level questions {gaps} missing between {nums[0]}-{hi} — expected but "
-                f"unrecovered; surface as second-pass slots in the question tree")
-        elif not suspect:
-            add("C3_question_seq", "info", "ok", f"questions {nums[0]}-{hi} contiguous")
-
-    # ---- C4: sub-part contiguity within each question --------------------------------------
-    def order(keys):
-        """Map a question's child keys to an ordered scheme + report holes. Handles .N and a/b/c."""
-        dig = sorted(int(k[0]) for k in keys if k[:1].isdigit())
-        let = sorted(k[0] for k in keys if k[:1].isalpha())
-        holes = []
-        if dig:
-            holes += [str(n) for n in range(dig[0], dig[-1] + 1) if n not in dig]
-        if let:
-            lo, hi = ord(let[0]), ord(let[-1])
-            holes += [chr(c) for c in range(lo, hi + 1) if chr(c) not in let]
-        return holes
-    byq = defaultdict(list)
-    for lab, q, p in parts:
-        sk = _subkey(lab, q)
-        if sk:
-            byq[q].append(sk)
-    seq_holes = {}
-    for q, keys in byq.items():
-        firsts = {k[0] for k in keys}            # immediate children only (a / 1 / etc.)
-        h = order(firsts)
-        if h:
-            seq_holes[q] = h
-    if seq_holes:
-        add("C4_subpart_seq", "warn", "gap",
-            "sub-part gaps: " + ", ".join(f"Q{q} missing {hs}" for q, hs in sorted(seq_holes.items())))
-    else:
-        add("C4_subpart_seq", "info", "ok", "sub-parts contiguous within every question")
-
-    # ---- C5: coverage vs ground truth (when present) ---------------------------------------
-    cov = result.get("coverage", {})
-    if cov.get("coverage_pct") is not None:
-        missed = cov.get("missed", [])
-        if missed:
-            add("C5_coverage", "warn", "missed",
-                f"{cov['coverage_pct']}% vs GT ({cov['recovered']}/{cov['total']}); missed {missed[:10]}")
-            low.update(missed)
-        else:
-            add("C5_coverage", "info", "ok", f"100% coverage vs GT ({cov['recovered']}/{cov['total']})")
-
-    # ---- per-part confidence + paper summary -----------------------------------------------
-    sum_mismatch = any(c["id"] == "C1_marks_sum" and c["status"] in ("over", "under") for c in checks)
-    for lab, q, p in parts:
-        if lab in low:
-            conf[lab] = "low"
-        elif sum_mismatch:
-            conf[lab] = "medium"               # paper-level doubt taints every part a little
-        else:
-            conf[lab] = "high"
-    severities = [c["severity"] for c in checks if c["status"] not in ("ok", "info", "unknown")]
-    worst = "error" if "error" in severities else "warn" if "warn" in severities else "clean"
-
-    return {
-        "paper_code": code, "board": board,
-        "summary": {
-            "worst_severity": worst,
-            "needs_priority_review": worst != "clean",
-            "n_flags": len(flags),
-            "marks_sum": msum, "official_max": exp,
-            "parts_total": len(parts),
-            "parts_low_conf": sum(1 for v in conf.values() if v == "low"),
-            "questions_expected": len(question_sequence) or None,
-            "questions_recovered": sum(1 for q in question_sequence if q["recovered"]) or None,
-        },
-        "flags": flags,
-        "checks": checks,
-        "part_confidence": conf,
-        "question_sequence": question_sequence,   # full expected skeleton (recovered + missing slots)
-    }
-
-
-def main():
-    ap = argparse.ArgumentParser()
-    ap.add_argument("structured")
-    ap.add_argument("--out")
-    a = ap.parse_args()
-    rep = validate(json.load(open(a.structured)))
-    s = rep["summary"]
-    print(f"paper       : {rep['paper_code']}  ({rep['board']})")
-    print(f"verdict     : {s['worst_severity'].upper()}  "
-          f"{'-> PRIORITY REVIEW' if s['needs_priority_review'] else '-> all checks clean (still human-reviewable)'}")
-    print(f"marks       : {s['marks_sum']}/{s['official_max']}  | parts {s['parts_total']} "
-          f"({s['parts_low_conf']} low-confidence)")
-    if s.get("questions_expected"):
-        miss = [q["label"] for q in rep["question_sequence"] if not q["recovered"]]
-        print(f"questions   : {s['questions_recovered']}/{s['questions_expected']} recovered"
-              + (f"  | second-pass slots: {miss}" if miss else "  (complete sequence)"))
-    if rep["flags"]:
-        print("flags:")
-        for f in rep["flags"]:
-            print(f"  - {f}")
-    else:
-        print("flags       : none")
-    if a.out:
-        json.dump(rep, open(a.out, "w"), indent=2)
-        print(f"-> wrote {a.out}")
-
-
-if __name__ == "__main__":
-    main()

debug_worker.log

@@ -0,0 +1,36 @@
+ℹ️ 2025-09-19 16:57:46,569 INFO    : start_queue_workers.py:main                :79   >>> Starting 1 queue workers
+ℹ️ 2025-09-19 16:57:46,569 INFO    : start_queue_workers.py:main                :80   >>> Services: ['document_analysis']
+ℹ️ 2025-09-19 16:57:46,569 INFO    : start_queue_workers.py:main                :81   >>> Redis URL: redis://localhost:6379
+ℹ️ 2025-09-19 16:57:46,569 INFO    : queue_system.py     :__init__            :124  >>> Queue initialized with limits: {<ServiceType.TIKA: 'tika'>: 3, <ServiceType.DOCLING: 'docling'>: 2, <ServiceType.LLM: 'llm'>: 5, <ServiceType.SPLIT_MAP: 'split_map'>: 10, <ServiceType.DOCUMENT_ANALYSIS: 'document_analysis'>: 5, <ServiceType.PAGE_IMAGES: 'page_images'>: 3}
+ℹ️ 2025-09-19 16:57:46,601 INFO    : task_processors.py  :__init__            :40   >>> Task processor initialized with service URLs
+ℹ️ 2025-09-19 16:57:46,601 INFO    : queue_system.py     :worker_loop         :414  >>> Starting worker cli-worker-1 for services: ['document_analysis']
+ℹ️ 2025-09-19 16:57:46,601 INFO    : start_queue_workers.py:main                :100  >>> Started workers: ['cli-worker-1']
+❌ 2025-09-19 16:57:58,883 ERROR   : queue_system.py     :worker_loop         :432  >>> Worker cli-worker-1 error: <ServiceType.DOCUMENT_ANALYSIS: 'document_analysis'>
+ℹ️ 2025-09-19 16:58:16,607 INFO    : start_queue_workers.py:main                :124  >>> Queue status - Queued: 0, Processing: 0, Dead: 0
+ℹ️ 2025-09-19 16:58:46,613 INFO    : start_queue_workers.py:main                :124  >>> Queue status - Queued: 0, Processing: 0, Dead: 0
+❌ 2025-09-19 16:59:16,036 ERROR   : queue_system.py     :worker_loop         :432  >>> Worker cli-worker-1 error: <ServiceType.DOCUMENT_ANALYSIS: 'document_analysis'>
+ℹ️ 2025-09-19 16:59:16,618 INFO    : start_queue_workers.py:main                :124  >>> Queue status - Queued: 4, Processing: 2, Dead: 0
+❌ 2025-09-19 16:59:17,041 ERROR   : queue_system.py     :worker_loop         :432  >>> Worker cli-worker-1 error: <ServiceType.PAGE_IMAGES: 'page_images'>
+ℹ️ 2025-09-19 16:59:46,620 INFO    : start_queue_workers.py:main                :124  >>> Queue status - Queued: 2, Processing: 3, Dead: 0
+ℹ️ 2025-09-19 17:00:16,626 INFO    : start_queue_workers.py:main                :124  >>> Queue status - Queued: 3, Processing: 3, Dead: 0
+ℹ️ 2025-09-19 17:00:46,631 INFO    : start_queue_workers.py:main                :124  >>> Queue status - Queued: 3, Processing: 3, Dead: 0
+❌ 2025-09-19 17:00:57,797 ERROR   : queue_system.py     :worker_loop         :432  >>> Worker cli-worker-1 error: Error while reading from localhost:6379 : (54, 'Connection reset by peer')
+❌ 2025-09-19 17:00:58,802 ERROR   : queue_system.py     :worker_loop         :432  >>> Worker cli-worker-1 error: Error 61 connecting to localhost:6379. Connection refused.
+❌ 2025-09-19 17:00:59,806 ERROR   : queue_system.py     :worker_loop         :432  >>> Worker cli-worker-1 error: Error 61 connecting to localhost:6379. Connection refused.
+❌ 2025-09-19 17:01:00,813 ERROR   : queue_system.py     :worker_loop         :432  >>> Worker cli-worker-1 error: Error 61 connecting to localhost:6379. Connection refused.
+❌ 2025-09-19 17:01:01,816 ERROR   : queue_system.py     :worker_loop         :432  >>> Worker cli-worker-1 error: Error 61 connecting to localhost:6379. Connection refused.
+❌ 2025-09-19 17:01:02,822 ERROR   : queue_system.py     :worker_loop         :432  >>> Worker cli-worker-1 error: Error 61 connecting to localhost:6379. Connection refused.
+❌ 2025-09-19 17:01:11,739 ERROR   : queue_system.py     :worker_loop         :432  >>> Worker cli-worker-1 error: <ServiceType.DOCUMENT_ANALYSIS: 'document_analysis'>
+❌ 2025-09-19 17:01:12,742 ERROR   : queue_system.py     :worker_loop         :432  >>> Worker cli-worker-1 error: <ServiceType.PAGE_IMAGES: 'page_images'>
+ℹ️ 2025-09-19 17:01:16,637 INFO    : start_queue_workers.py:main                :124  >>> Queue status - Queued: 3, Processing: 3, Dead: 0
+ℹ️ 2025-09-19 17:01:46,639 INFO    : start_queue_workers.py:main                :124  >>> Queue status - Queued: 2, Processing: 3, Dead: 0
+ℹ️ 2025-09-19 17:02:16,644 INFO    : start_queue_workers.py:main                :124  >>> Queue status - Queued: 3, Processing: 3, Dead: 0
+❌ 2025-09-19 17:02:31,136 ERROR   : queue_system.py     :worker_loop         :432  >>> Worker cli-worker-1 error: <ServiceType.DOCUMENT_ANALYSIS: 'document_analysis'>
+❌ 2025-09-19 17:02:32,140 ERROR   : queue_system.py     :worker_loop         :432  >>> Worker cli-worker-1 error: <ServiceType.PAGE_IMAGES: 'page_images'>
+ℹ️ 2025-09-19 17:02:46,647 INFO    : start_queue_workers.py:main                :124  >>> Queue status - Queued: 6, Processing: 3, Dead: 0
+ℹ️ 2025-09-19 17:03:16,653 INFO    : start_queue_workers.py:main                :124  >>> Queue status - Queued: 6, Processing: 3, Dead: 0
+ℹ️ 2025-09-19 17:03:31,501 INFO    : start_queue_workers.py:signal_handler      :104  >>> Received signal 15, shutting down workers...
+ℹ️ 2025-09-19 17:03:31,501 INFO    : queue_system.py     :shutdown            :451  >>> Shutting down queue workers...
+ℹ️ 2025-09-19 17:03:31,501 INFO    : queue_system.py     :worker_loop         :435  >>> Worker cli-worker-1 shutting down
+ℹ️ 2025-09-19 17:03:31,502 INFO    : queue_system.py     :shutdown            :461  >>> Queue shutdown complete
+ℹ️ 2025-09-19 17:03:31,502 INFO    : start_queue_workers.py:signal_handler      :106  >>> Workers shut down. Exiting.

docker-compose.dev.yml

@@ -1,99 +0,0 @@
-services:
-  # ── Required environment variables (see .env.example) ───────────────────────
-  # APP_BOLT_URL, USER_NEO4J, PASSWORD_NEO4J — Neo4j connection
-  # SUPABASE_URL, SERVICE_ROLE_KEY — Supabase project
-  # REDIS_HOST, REDIS_PORT, REDIS_PASSWORD — Redis (optional auth)
-  # FASTAPI_SECRET_KEY, ADMIN_EMAIL — API config
-  redis-dev:
-    image: redis:7-alpine
-    container_name: cc-redis-dev
-    ports:
-      - "16379:6379"
-    volumes:
-      - redis-dev-data:/data
-    command: redis-server --appendonly yes
-    networks:
-      - kevlarai-network
-    healthcheck:
-      test: [ "CMD", "redis-cli", "ping" ]
-      interval: 5s
-      timeout: 3s
-      retries: 5
-
-  init:
-    image: cc-api-dev:latest
-    container_name: api-init-dev
-    env_file:
-      - .env.dev
-    environment:
-      - REDIS_HOST=redis-dev
-      - RUN_INIT=true
-      - INIT_MODE=${INIT_MODE:-infra}
-      - INIT_ONLY=true
-    command: ["./docker-entrypoint.sh", "init-only"]
-    depends_on:
-      redis-dev:
-        condition: service_healthy
-    networks:
-      - kevlarai-network
-    profiles:
-      - init
-
-  backend-dev:
-    container_name: cc-api-dev
-    image: cc-api-dev:latest
-    env_file:
-      - .env.dev
-    environment:
-      - REDIS_HOST=redis-dev
-      - REDIS_DB_DEV=0
-      - BACKEND_DEV_MODE=true
-      - APP_ENV=development
-      - ENVIRONMENT=development
-      - START_MODE=dev
-      - CC_COMPOSE_PROJECT=api-dev
-      - CC_COMPOSE_SERVICE=backend-dev
-      - RUN_INIT=false
-      - INIT_MODE=infra
-      # P2: route exam auto-map through the spike's full recognition pipeline (extraction service)
-      - EXAM_EXTRACT_URL=${EXAM_EXTRACT_URL:-http://192.168.0.203:8899}
-    ports:
-      - "18000:8000"
-    depends_on:
-      redis-dev:
-        condition: service_healthy
-    networks:
-      - kevlarai-network
-    restart: unless-stopped
-
-  backend-test:
-    image: cc-api-dev:latest
-    env_file:
-      - .env.dev
-    environment:
-      - REDIS_HOST=redis-dev
-      - REDIS_DB_DEV=0
-      - BACKEND_DEV_MODE=true
-      - APP_ENV=development
-      - ENVIRONMENT=development
-      - START_MODE=dev
-      - CC_COMPOSE_PROJECT=api-dev
-      - CC_COMPOSE_SERVICE=backend-test
-      - API_HEALTH_URL=http://192.168.0.64:18000/health
-    depends_on:
-      redis-dev:
-        condition: service_healthy
-    networks:
-      - kevlarai-network
-    entrypoint: ["python", "-m", "pytest"]
-    command: ["-q", "tests"]
-    profiles:
-      - test
-
-volumes:
-  redis-dev-data:
-
-networks:
-  kevlarai-network:
-    external: true
-    name: kevlarai-network

docker-compose.yml

@@ -1,9 +1,4 @@
 services:
-  # ── Required environment variables (see .env.example) ───────────────────────
-  # APP_BOLT_URL, USER_NEO4J, PASSWORD_NEO4J — Neo4j connection
-  # SUPABASE_URL, SERVICE_ROLE_KEY — Supabase project
-  # REDIS_HOST, REDIS_PORT, REDIS_PASSWORD — Redis (optional auth)
-  # FASTAPI_SECRET_KEY, ADMIN_EMAIL — API config
   redis:
     image: redis:7-alpine
     container_name: classroomcopilot-redis
@@ -51,11 +46,6 @@ services:
       - .env
     environment:
       - REDIS_HOST=redis
-      - START_MODE=prod
-      - APP_ENV=production
-      - ENVIRONMENT=production
-      - CC_COMPOSE_PROJECT=api
-      - CC_COMPOSE_SERVICE=backend
       - RUN_INIT=${RUN_INIT:-false} # Set to 'true' to run init on startup
       - INIT_MODE=${INIT_MODE:-infra} # Which init tasks to run
     ports:

docker-entrypoint.sh

@@ -29,7 +29,7 @@ print_error() {
 
 # Check if we should run initialization
 RUN_INIT="${RUN_INIT:-false}"
-INIT_MODE="${INIT_MODE:-infra}"  # Default to 'infra', can be 'infra', 'seed', 'seed-test', 'full', or comma-separated list
+INIT_MODE="${INIT_MODE:-infra}"  # Default to 'infra', can be 'infra', 'full', or comma-separated list
 
 # If RUN_INIT is true, run initialization tasks
 if [ "$RUN_INIT" = "true" ]; then
@@ -51,21 +51,21 @@ if [ "$RUN_INIT" = "true" ]; then
                 }
                 print_success "Infrastructure setup completed"
                 ;;
-            "seed")
-                print_status "Seeding canonical full environment..."
-                python3 main.py --mode seed || {
-                    print_error "Seed failed!"
+            "demo-school")
+                print_status "Creating demo school..."
+                python3 main.py --mode demo-school || {
+                    print_error "Demo school creation failed!"
                     exit 1
                 }
-                print_success "Seed completed"
+                print_success "Demo school creation completed"
                 ;;
-            "seed-test")
-                print_status "Seeding lightweight test environment..."
-                python3 main.py --mode seed-test || {
-                    print_error "Seed test failed!"
+            "demo-users")
+                print_status "Creating demo users..."
+                python3 main.py --mode demo-users || {
+                    print_error "Demo users creation failed!"
                     exit 1
                 }
-                print_success "Seed test completed"
+                print_success "Demo users creation completed"
                 ;;
             "gais-data")
                 print_status "Importing GAIS data..."
@@ -75,18 +75,12 @@ if [ "$RUN_INIT" = "true" ]; then
                 }
                 print_success "GAIS data import completed"
                 ;;
-            "exam-corpus")
-                print_status "Seeding exam-paper corpus (manifest-gated; skips if none configured)..."
-                python3 main.py --mode exam-corpus || {
-                    print_error "Exam corpus seed failed!"
-                    exit 1
-                }
-                print_success "Exam corpus seed completed"
-                ;;
             "full")
                 print_status "Running full initialization..."
                 python3 main.py --mode infra || exit 1
-                python3 main.py --mode seed || exit 1
+                python3 main.py --mode demo-school || exit 1
+                python3 main.py --mode demo-users || exit 1
+                python3 main.py --mode gais-data || exit 1
                 print_success "Full initialization completed"
                 ;;
             *)
@@ -104,13 +98,11 @@ if [ "$RUN_INIT" = "true" ]; then
     fi
 fi
 
-# Start the server (unless init-only mode). Default remains production, but
-# development Compose can set START_MODE=dev so cc-api-dev reports and uses
-# development Redis/config instead of silently booting as prod.
-START_MODE="${START_MODE:-prod}"
+# Start the production server (unless init-only mode)
 if [ "$1" != "init-only" ] && [ -z "$INIT_ONLY" ]; then
-    print_status "Starting ${START_MODE} server..."
-    exec ./start.sh "$START_MODE"
+    print_status "Starting production server..."
+    exec ./start.sh prod
 else
     print_status "Init-only mode - not starting server"
 fi
+

init-production.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 # Helper script to run initialization tasks in production
 # Usage: ./init-production.sh [mode]
-# Modes: infra, seed, seed-test, gais-data, full
+# Modes: infra, demo-school, demo-users, gais-data, full
 
 set -e
 

main.py

@@ -10,7 +10,6 @@ logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH
 from fastapi import FastAPI, HTTPException
 import uvicorn
 import requests
-from urllib.parse import urlparse
 from typing import Dict, Any, Optional
 from modules.database.tools.neo4j_driver_tools import get_driver
 
@@ -23,59 +22,15 @@ from modules.queue_system import ServiceType
 app = FastAPI()
 setup_cors(app)
 
-def _truthy_env(name: str) -> bool:
-    return os.getenv(name, "").lower() in {"1", "true", "yes", "on"}
-
-
-def _runtime_environment() -> str:
-    """Return the API runtime role used for dev/prod backing-service selection."""
-    start_mode = os.getenv("START_MODE", "prod").lower()
-    if start_mode == "dev" or _truthy_env("BACKEND_DEV_MODE"):
-        return "dev"
-    return "prod"
-
-
-def _url_host(url: Optional[str]) -> Optional[str]:
-    if not url:
-        return None
-    parsed = urlparse(url)
-    return parsed.hostname
-
-
-def _runtime_identity() -> Dict[str, Any]:
-    """Non-secret runtime identity for agents and smoke tests.
-
-    This intentionally exposes only modes, labels, and URL hosts. It must not
-    include API keys, passwords, bearer tokens, or full URLs that may embed
-    credentials.
-    """
-    return {
-        "api_runtime_role": _runtime_environment(),
-        "start_mode": os.getenv("START_MODE", "prod"),
-        "app_environment": os.getenv("APP_ENV"),
-        "environment": os.getenv("ENVIRONMENT"),
-        "backend_dev_mode": _truthy_env("BACKEND_DEV_MODE"),
-        "compose_project": os.getenv("COMPOSE_PROJECT_NAME") or os.getenv("CC_COMPOSE_PROJECT"),
-        "compose_service": os.getenv("COMPOSE_SERVICE") or os.getenv("CC_COMPOSE_SERVICE"),
-        "supabase_url_host": _url_host(os.getenv("SUPABASE_URL")),
-    }
-
-
 # Health check endpoint
 @app.get("/health")
 async def health_check() -> Dict[str, Any]:
     """Health check endpoint that verifies all service dependencies"""
-    runtime_identity = _runtime_identity()
     health_status = {
         "status": "healthy",
-        "runtime": runtime_identity,
         "services": {
             "neo4j": {"status": "healthy", "message": "Connected"},
-            "supabase": {
-                "status": "healthy",
-                "message": "Connected",
-                "url_host": runtime_identity["supabase_url_host"],
-            },
+            "supabase": {"status": "healthy", "message": "Connected"},
             "redis": {"status": "healthy", "message": "Connected"}
         }
     }
@@ -90,10 +45,9 @@ async def health_check() -> Dict[str, Any]:
             }
             health_status["status"] = "unhealthy"
     except Exception as e:
-        logger.warning(f"Neo4j health check failed: {e}")
         health_status["services"]["neo4j"] = {
             "status": "unhealthy",
-            "message": "Error checking Neo4j"
+            "message": f"Error checking Neo4j: {str(e)}"
         }
         health_status["status"] = "unhealthy"
     
@@ -113,10 +67,9 @@ async def health_check() -> Dict[str, Any]:
             }
             health_status["status"] = "unhealthy"
     except Exception as e:
-        logger.warning(f"Supabase health check failed: {e}")
         health_status["services"]["supabase"] = {
             "status": "unhealthy",
-            "message": "Error checking Supabase Auth API"
+            "message": f"Error checking Supabase Auth API: {str(e)}"
         }
         health_status["status"] = "unhealthy"
     
@@ -124,8 +77,8 @@ async def health_check() -> Dict[str, Any]:
         # Check Redis using new Redis manager
         from modules.redis_manager import get_redis_manager
         
-        # Determine environment from explicit startup/runtime identity.
-        environment = runtime_identity["api_runtime_role"]
+        # Determine environment
+        environment = 'dev' if os.getenv('BACKEND_DEV_MODE', 'true').lower() == 'true' else 'prod'
         redis_manager = get_redis_manager(environment)
         
         # Get comprehensive health check
@@ -143,10 +96,9 @@ async def health_check() -> Dict[str, Any]:
             health_status["status"] = "unhealthy"
             
     except Exception as e:
-        logger.warning(f"Redis health check failed: {e}")
         health_status["services"]["redis"] = {
             "status": "unhealthy",
-            "message": "Error checking Redis"
+            "message": f"Error checking Redis: {str(e)}"
         }
         health_status["status"] = "unhealthy"
     
@@ -292,20 +244,33 @@ def run_infrastructure_mode():
         logger.error(f"Infrastructure setup failed: {str(e)}")
         return False
 
-def run_seed_mode(test: bool = False):
-    """Run canonical environment seed."""
-    mode = "test" if test else "full"
-    logger.info(f"Running canonical seed mode ({mode})")
+def run_demo_school_mode():
+    """Run demo school creation"""
+    logger.info("Running in demo school mode")
+    logger.info("Starting demo school creation...")
+    
     try:
-        from run.initialization.seed_environment import seed
-        import json
-        result = seed(test=test)
-        print(json.dumps(result, indent=2, default=str))
-        return bool(result.get('success'))
+        from run.initialization import initialize_demo_school_mode
+        initialize_demo_school_mode()
+        logger.info("Demo school creation completed successfully")
+        return True
     except Exception as e:
-        logger.error(f"Seed mode failed: {str(e)}")
+        logger.error(f"Demo school creation failed: {str(e)}")
         return False
 
+def run_demo_users_mode():
+    """Run demo users creation"""
+    logger.info("Running in demo users mode")
+    logger.info("Starting demo users creation...")
+    
+    try:
+        from run.initialization import initialize_demo_users_mode
+        initialize_demo_users_mode()
+        logger.info("Demo users creation completed successfully")
+        return True
+    except Exception as e:
+        logger.error(f"Demo users creation failed: {str(e)}")
+        return False
 
 def run_gais_data_mode():
     """Run GAIS data import"""
@@ -323,52 +288,6 @@ def run_gais_data_mode():
 
 # Old clear_dev_redis_queue function removed - now handled by Redis Manager
 
-def run_exam_corpus_mode():
-    """Seed the public exam-paper corpus from a manifest (optional, gated).
-
-    Env controls:
-      EXAM_CORPUS_MANIFEST    - path to the corpus manifest (required to do anything)
-      EXAM_CORPUS_DRY_RUN     - 'true' to validate + report only
-      EXAM_CORPUS_FORCE       - 'true' to re-upload/overwrite existing objects
-      EXAM_CORPUS_BOARD/_SPEC - filter to one exam_board_code / spec_code
-      EXAM_CORPUS_USER_SUBSET - 'true' to also seed a user-side test subset
-      EXAM_CORPUS_FIRST_SWEEP - 'true' to run the docling/auto-map first pass
-
-    Skips gracefully (success) when no manifest is configured/present, so it is safe
-    in a comma-mode list (e.g. INIT_MODE=infra,seed,exam-corpus) before papers exist.
-    Buckets are NOT created here — infra mode (buckets.py) owns provisioning.
-    """
-    logger.info("Running in exam-corpus seed mode")
-    manifest = os.getenv("EXAM_CORPUS_MANIFEST")
-    if not manifest or not os.path.exists(manifest):
-        logger.warning(
-            f"exam-corpus: no manifest at EXAM_CORPUS_MANIFEST={manifest!r}; skipping (nothing to seed yet)"
-        )
-        return True
-    try:
-        from run.initialization.seed_exam_corpus import load
-        rep = load(
-            manifest,
-            dry_run=_truthy_env("EXAM_CORPUS_DRY_RUN"),
-            force=_truthy_env("EXAM_CORPUS_FORCE"),
-            board_filter=os.getenv("EXAM_CORPUS_BOARD") or None,
-            spec_filter=os.getenv("EXAM_CORPUS_SPEC") or None,
-            user_subset=_truthy_env("EXAM_CORPUS_USER_SUBSET"),
-            do_first_sweep=_truthy_env("EXAM_CORPUS_FIRST_SWEEP"),
-        )
-        if rep.errors:
-            logger.error(f"exam-corpus seed completed with {len(rep.errors)} error(s)")
-            return False
-        logger.info(
-            f"exam-corpus seed ok: specs={rep.specs_upserted} papers={rep.papers_upserted} "
-            f"uploaded={rep.files_uploaded}"
-        )
-        return True
-    except Exception as e:
-        logger.error(f"exam-corpus seed failed: {e}")
-        return False
-
-
 def run_development_mode():
     """Run the server in development mode with auto-reload"""
     logger.info("Running in development mode")
@@ -447,8 +366,8 @@ def parse_arguments():
         epilog="""
 Startup modes:
   infra                    - Setup infrastructure (Neo4j schema, calendar, Supabase buckets)
-  seed                    - Seed canonical full environment (20 school users)
-  seed-test               - Seed lightweight test environment (9 school users)
+  demo-school             - Create demo school (KevlarAI)
+  demo-users              - Create demo users
   gais-data               - Import GAIS data (Edubase, etc.)
   dev                     - Run development server with auto-reload
   prod                    - Run production server (for Docker/containerized deployment)
@@ -457,7 +376,7 @@ Startup modes:
     
     parser.add_argument(
         '--mode', '-m',
-        choices=['infra', 'seed', 'seed-test', 'gais-data', 'exam-corpus', 'dev', 'prod'],
+        choices=['infra', 'demo-school', 'demo-users', 'gais-data', 'dev', 'prod'],
         default='dev',
         help='Startup mode (default: dev)'
     )
@@ -480,24 +399,21 @@ if __name__ == "__main__":
         success = run_infrastructure_mode()
         sys.exit(0 if success else 1)
     
-    elif args.mode == 'seed':
-        success = run_seed_mode(test=False)
+    elif args.mode == 'demo-school':
+        # Run demo school creation
+        success = run_demo_school_mode()
         sys.exit(0 if success else 1)
-
-    elif args.mode == 'seed-test':
-        success = run_seed_mode(test=True)
+    
+    elif args.mode == 'demo-users':
+        # Run demo users creation
+        success = run_demo_users_mode()
         sys.exit(0 if success else 1)
-
+    
     elif args.mode == 'gais-data':
         # Run GAIS data import
         success = run_gais_data_mode()
         sys.exit(0 if success else 1)
-
-    elif args.mode == 'exam-corpus':
-        # Seed the public exam-paper corpus from a manifest (gated; skips if none configured)
-        success = run_exam_corpus_mode()
-        sys.exit(0 if success else 1)
-
+    
     elif args.mode == 'dev':
         # Run development server
         run_development_mode()

modules/auth/platform_admin.py

@@ -1,57 +0,0 @@
-"""
-FastAPI dependencies for platform-level admin access.
-
-Two tiers:
-  require_platform_admin — user must be in admin_profiles
-  require_super_admin    — user must have is_super_admin=True in admin_profiles
-
-Usage:
-    @router.get("/admin/schools")
-    async def list_all_schools(admin=Depends(require_platform_admin)):
-        ...
-
-    @router.post("/admin/provision")
-    async def provision(admin=Depends(require_super_admin)):
-        ...
-"""
-from fastapi import Depends, HTTPException
-from modules.auth.supabase_bearer import SupabaseBearer
-from modules.database.supabase.utils.client import SupabaseServiceRoleClient
-
-
-def _sb() -> SupabaseServiceRoleClient:
-    return SupabaseServiceRoleClient()
-
-
-async def require_platform_admin(
-    credentials: dict = Depends(SupabaseBearer()),
-) -> dict:
-    """Require the caller to be a registered platform admin (in admin_profiles)."""
-    user_id = credentials.get("sub")
-    if not user_id:
-        raise HTTPException(status_code=403, detail="Invalid token")
-    try:
-        sb = _sb()
-        result = (
-            sb.supabase.table("admin_profiles")
-            .select("id,admin_role,is_super_admin")
-            .eq("id", user_id)
-            .single()
-            .execute()
-        )
-        if not result.data:
-            raise HTTPException(status_code=403, detail="Platform admin access required")
-        return {**credentials, "admin_profile": result.data}
-    except HTTPException:
-        raise
-    except Exception:
-        raise HTTPException(status_code=403, detail="Platform admin access required")
-
-
-async def require_super_admin(
-    admin: dict = Depends(require_platform_admin),
-) -> dict:
-    """Require the caller to have is_super_admin=True."""
-    if not admin.get("admin_profile", {}).get("is_super_admin"):
-        raise HTTPException(status_code=403, detail="Super admin access required")
-    return admin

modules/auth/supabase_bearer.py

@@ -20,10 +20,6 @@ class SupabaseBearer(HTTPBearer):
             token = credentials.credentials
             # Decode using the string-based verifier to avoid async dependency conflicts
             payload = verify_supabase_jwt_str(token)
-            # Keep the bearer token available to downstream dependencies that must
-            # call Supabase as the user (RLS/storage policies), without requiring
-            # each router to decode the Authorization header again.
-            payload["_access_token"] = token
             return payload
         except Exception as e:
             logger.error(f"Token verification failed: {str(e)}")

modules/database/schemas/nodes/exams/exam_nodes.py

@@ -1,92 +0,0 @@
-"""
-Neontology node schemas for the cc.public.exams knowledge graph.
-
-cc.public.exams is a dedicated, shared, public Neo4j database — co-primary/authoritative for the
-exam knowledge graph (specs, spec-points, paper→question→part→region structure, ASSESSES links).
-Supabase remains source of truth for operational data (geometry, marks, submissions); the two
-layers join on shared UUIDs:
-
-    exam_questions.id        <-> Question|Part.uuid_string   (container -> Question, leaf -> Part)
-    exam_response_areas.id   <-> Region.uuid_string
-    eb_exams.exam_code       <-> ExamPaper.exam_code
-    eb_specifications.spec_code <-> Specification.spec_code
-
-Ownership: created by an infra-init step; read by all authenticated API calls; written by the API
-service role only (no direct client writes).
-"""
-from typing import ClassVar, Optional
-from ..base_nodes import CCBaseNode
-
-
-class ExamBaseNode(CCBaseNode):
-    __primarylabel__: ClassVar[str] = ''
-
-
-class ExamBoardNode(ExamBaseNode):
-    __primarylabel__: ClassVar[str] = 'ExamBoard'
-    code: str                      # 'AQA'
-    name: str
-
-
-class SpecificationNode(ExamBaseNode):
-    __primarylabel__: ClassVar[str] = 'Specification'
-    spec_code: str                 # 'AQA-PHYS-8463'  (== Supabase eb_specifications.spec_code)
-    exam_board_code: str
-    subject_code: Optional[str] = None
-    award_code: Optional[str] = None
-    title: Optional[str] = None
-
-
-class SpecPointNode(ExamBaseNode):
-    __primarylabel__: ClassVar[str] = 'SpecPoint'
-    ref: str                       # '4.1', '4.2.1'
-    description: str
-    spec_code: str
-    exam_board_code: str
-
-
-class ExamPaperNode(ExamBaseNode):
-    __primarylabel__: ClassVar[str] = 'ExamPaper'
-    exam_code: str                 # == Supabase eb_exams.exam_code
-    spec_code: str
-    paper_code: Optional[str] = None
-    tier: Optional[str] = None
-    session: Optional[str] = None
-    title: Optional[str] = None
-    page_count: Optional[int] = None
-
-
-class QuestionNode(ExamBaseNode):  # roll-up container; uuid_string == exam_questions.id
-    __primarylabel__: ClassVar[str] = 'Question'
-    exam_code: str
-    label: str                     # '01'
-    order: int
-    max_marks: float
-
-
-class PartNode(ExamBaseNode):      # leaf; uuid_string == exam_questions.id
-    __primarylabel__: ClassVar[str] = 'Part'
-    exam_code: str
-    label: str                     # '01.1'
-    order: int
-    max_marks: float
-    answer_type: str
-    mark_scheme_type: str
-
-
-class RegionNode(ExamBaseNode):    # uuid_string == exam_response_areas.id
-    __primarylabel__: ClassVar[str] = 'Region'
-    page: int
-    kind: str                      # 'response' | 'context'
-    response_form: str
-
-
-# Relationship reference (written by the projection / linker, not modelled as classes here):
-#   (:ExamBoard)-[:PUBLISHES]->(:Specification)
-#   (:Specification)-[:HAS_SPEC_POINT]->(:SpecPoint)
-#   (:Specification)-[:HAS_PAPER]->(:ExamPaper)
-#   (:ExamPaper)-[:HAS_QUESTION]->(:Question)
-#   (:Question)-[:HAS_PART]->(:Part)        # nested questions allowed
-#   (:Part)-[:HAS_REGION]->(:Region)
-#   (:Part)-[:ASSESSES]->(:SpecPoint)       # from exam_questions.spec_ref
-#   (:SpecPoint)-[:TEACHES]->(:LearningStatement)   # DEFERRED cross-db bridge

modules/database/schemas/nodes/users.py

@@ -1,13 +1,5 @@
 from typing import ClassVar
-from .base_nodes import UserBaseNode, CCBaseNode
+from .base_nodes import UserBaseNode
 
 class UserNode(UserBaseNode):
     __primarylabel__: ClassVar[str] = 'User'
-
-class JournalNode(CCBaseNode):
-    __primarylabel__: ClassVar[str] = 'Journal'
-    user_id: str
-
-class PlannerNode(CCBaseNode):
-    __primarylabel__: ClassVar[str] = 'Planner'
-    user_id: str

modules/database/services/bootstrap_service.py

@@ -1,370 +0,0 @@
-import os
-from datetime import datetime, timezone
-from typing import Any, Callable, Dict, List, Optional
-
-from modules.logger_tool import initialise_logger
-from modules.database.supabase.utils.client import SupabaseServiceRoleClient
-import modules.database.tools.neo4j_driver_tools as driver_tools
-
-logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), "default", True)
-
-MembershipRow = Dict[str, Any]
-GraphProbe = Callable[..., Dict[str, Any]]
-
-
-ROLE_PERMISSIONS: Dict[str, Dict[str, bool]] = {
-    "school_admin": {
-        "can_manage_school": True,
-        "can_manage_calendar": True,
-        "can_manage_timetable": True,
-        "can_invite_staff": True,
-        "can_manage_classes": True,
-        "can_view_student_data": True,
-    },
-    "department_head": {
-        "can_manage_school": False,
-        "can_manage_calendar": False,
-        "can_manage_timetable": True,
-        "can_invite_staff": False,
-        "can_manage_classes": True,
-        "can_view_student_data": True,
-    },
-    "teacher": {
-        "can_manage_school": False,
-        "can_manage_calendar": False,
-        "can_manage_timetable": False,
-        "can_invite_staff": False,
-        "can_manage_classes": True,
-        "can_view_student_data": False,
-    },
-    "student": {
-        "can_manage_school": False,
-        "can_manage_calendar": False,
-        "can_manage_timetable": False,
-        "can_invite_staff": False,
-        "can_manage_classes": False,
-        "can_view_student_data": False,
-    },
-}
-
-
-BASE_PERMISSIONS: Dict[str, bool] = {
-    "platform_admin": False,
-    "platform_super_admin": False,
-    "can_create_school": True,
-    "can_manage_school": False,
-    "can_manage_calendar": False,
-    "can_manage_timetable": False,
-    "can_invite_staff": False,
-    "can_manage_classes": False,
-    "can_view_student_data": False,
-    "can_use_canvas": True,
-}
-
-
-def utc_now_iso() -> str:
-    return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
-
-
-def _safe_list(result: Any) -> List[Dict[str, Any]]:
-    data = getattr(result, "data", None)
-    if not data:
-        return []
-    if isinstance(data, list):
-        return data
-    return [data]
-
-
-def _safe_one(result: Any) -> Optional[Dict[str, Any]]:
-    data = getattr(result, "data", None)
-    return data if isinstance(data, dict) else None
-
-
-def default_graph_probe(user_id: str, user_email: str, active_institute: Optional[Dict[str, Any]], institute: Optional[Dict[str, Any]]) -> Dict[str, Any]:
-    """Best-effort derived graph status; never authoritative for bootstrap state."""
-    user_db = f"cc.users.teacher.{user_id.replace('-', '')}" if user_id else None
-    neo4j_uuid = (institute or {}).get("neo4j_uuid_string")
-    institute_db = f"cc.institutes.{neo4j_uuid}" if neo4j_uuid else None
-    status = {
-        "available": False,
-        "user_db": user_db,
-        "institute_db": institute_db,
-        "projection_state": "unknown",
-        "needs_rebuild": True,
-        "last_checked_at": utc_now_iso(),
-        "error_code": None,
-    }
-    try:
-        if not user_db and not institute_db:
-            status["projection_state"] = "missing"
-            return status
-        db_to_check = institute_db or user_db
-        with driver_tools.get_session(database=db_to_check) as session:
-            session.run("RETURN 1 AS ok").single()
-        status.update({"available": True, "projection_state": "ready", "needs_rebuild": False})
-        return status
-    except Exception:
-        status.update({"available": False, "projection_state": "error", "needs_rebuild": True, "error_code": "neo4j_unavailable"})
-        return status
-
-
-class BootstrapService:
-    def __init__(self, supabase: Any, graph_probe: Optional[GraphProbe] = None):
-        self.supabase = supabase
-        self.graph_probe = graph_probe or default_graph_probe
-
-    def build(self, credentials: Dict[str, Any]) -> Dict[str, Any]:
-        user_id = credentials.get("sub") or ""
-        user_email = credentials.get("email") or ""
-        if not user_id:
-            raise ValueError("Missing authenticated user id")
-
-        profile = self._get_profile(user_id, user_email)
-        admin_profile = self._get_admin_profile(user_id)
-        memberships = self._get_memberships(user_id)
-        active = self._select_active_institute(profile, memberships)
-        active_institute = self._membership_institute(active, memberships)
-
-        permissions = self._permissions(active, bool(admin_profile), bool((admin_profile or {}).get("is_super_admin")))
-        profile_user_type = "platform_admin" if admin_profile else (profile.get("user_type") or active.get("role") or "unknown")
-        school_status = self._school_status(active, memberships, admin_profile)
-
-        calendar_status = self._calendar_status(active.get("institute_id"))
-        timetable_status = self._timetable_status(user_id, active.get("institute_id"))
-        graph_status = self._graph_status(user_id, user_email, active, active_institute)
-        onboarding = self._onboarding(school_status, permissions, calendar_status, timetable_status, graph_status)
-
-        return {
-            "profile": {
-                "id": user_id,
-                "email": profile.get("email") or user_email,
-                "display_name": profile.get("display_name") or profile.get("full_name") or profile.get("username") or user_email,
-                "user_type": profile_user_type,
-                "school_id": profile.get("school_id"),
-            },
-            "memberships": [self._serialize_membership(row) for row in memberships],
-            "active_institute": {
-                "id": active.get("institute_id"),
-                "source": active.get("source", "none"),
-                "membership_role": active.get("role"),
-            },
-            "permissions": permissions,
-            "school_status": school_status,
-            "onboarding": onboarding,
-            "calendar_status": calendar_status,
-            "timetable_status": timetable_status,
-            "graph_status": graph_status,
-        }
-
-    def _get_profile(self, user_id: str, user_email: str) -> Dict[str, Any]:
-        try:
-            result = (
-                self.supabase.table("profiles")
-                .select("id,email,user_type,username,full_name,display_name,user_db_name,school_db_name,school_id")
-                .eq("id", user_id)
-                .single()
-                .execute()
-            )
-            profile = _safe_one(result) or {}
-        except Exception as exc:
-            logger.warning("Bootstrap profile lookup failed for authenticated user: %s", exc)
-            profile = {}
-        profile.setdefault("id", user_id)
-        profile.setdefault("email", user_email)
-        return profile
-
-    def _get_admin_profile(self, user_id: str) -> Optional[Dict[str, Any]]:
-        try:
-            result = (
-                self.supabase.table("admin_profiles")
-                .select("id,admin_role,is_super_admin")
-                .eq("id", user_id)
-                .single()
-                .execute()
-            )
-            return _safe_one(result)
-        except Exception:
-            return None
-
-    def _get_memberships(self, user_id: str) -> List[MembershipRow]:
-        try:
-            member_rows = _safe_list(
-                self.supabase.table("institute_memberships")
-                .select("profile_id,institute_id,role")
-                .eq("profile_id", user_id)
-                .execute()
-            )
-        except Exception as exc:
-            logger.warning("Bootstrap membership lookup failed: %s", exc)
-            return []
-
-        institute_ids = [str(row.get("institute_id")) for row in member_rows if row.get("institute_id")]
-        institutes_by_id: Dict[str, Dict[str, Any]] = {}
-        if institute_ids:
-            try:
-                inst_rows = _safe_list(
-                    self.supabase.table("institutes")
-                    .select("id,name,urn,website,address,metadata,status,neo4j_uuid_string")
-                    .in_("id", institute_ids)
-                    .execute()
-                )
-                institutes_by_id = {str(row.get("id")): row for row in inst_rows}
-            except Exception as exc:
-                logger.warning("Bootstrap institute lookup failed: %s", exc)
-
-        rows: List[MembershipRow] = []
-        for member in member_rows:
-            inst = institutes_by_id.get(str(member.get("institute_id")), {})
-            if inst and inst.get("status") not in (None, "active"):
-                continue
-            rows.append({**member, "institute": inst, "status": member.get("status") or "active", "is_active": True})
-        return rows
-
-    def _select_active_institute(self, profile: Dict[str, Any], memberships: List[MembershipRow]) -> MembershipRow:
-        active_memberships = [m for m in memberships if m.get("is_active")]
-        profile_school_id = profile.get("school_id")
-        if profile_school_id:
-            for member in active_memberships:
-                if str(member.get("institute_id")) == str(profile_school_id):
-                    return {**member, "source": "profile.school_id"}
-        if len(active_memberships) == 1:
-            return {**active_memberships[0], "source": "single_membership"}
-        return {"institute_id": None, "role": None, "source": "none"}
-
-    def _membership_institute(self, active: MembershipRow, memberships: List[MembershipRow]) -> Optional[Dict[str, Any]]:
-        active_id = active.get("institute_id")
-        if not active_id:
-            return None
-        for member in memberships:
-            if str(member.get("institute_id")) == str(active_id):
-                return member.get("institute") or None
-        return None
-
-    def _serialize_membership(self, row: MembershipRow) -> Dict[str, Any]:
-        inst = row.get("institute") or {}
-        return {
-            "institute_id": row.get("institute_id"),
-            "role": row.get("role") or "teacher",
-            "status": row.get("status") or "active",
-            "is_active": bool(row.get("is_active", True)),
-            "institute": {
-                "id": inst.get("id") or row.get("institute_id"),
-                "name": inst.get("name"),
-                "urn": inst.get("urn"),
-                "website": inst.get("website"),
-                "address": inst.get("address") or {},
-                "metadata": inst.get("metadata") or {},
-            },
-        }
-
-    def _permissions(self, active: MembershipRow, platform_admin: bool, super_admin: bool) -> Dict[str, bool]:
-        permissions = dict(BASE_PERMISSIONS)
-        permissions["platform_admin"] = platform_admin
-        permissions["platform_super_admin"] = super_admin
-        role_permissions = ROLE_PERMISSIONS.get(active.get("role") or "", {})
-        permissions.update(role_permissions)
-        if platform_admin:
-            # Platform authority is additive and must not be reduced by a user's
-            # school membership role (for example a platform admin who also has
-            # a teacher/student membership).
-            permissions.update({
-                "can_create_school": True,
-                "can_manage_school": True,
-                "can_manage_calendar": True,
-                "can_manage_timetable": True,
-                "can_invite_staff": True,
-                "can_manage_classes": True,
-                "can_view_student_data": True,
-            })
-        return permissions
-
-    def _school_status(self, active: MembershipRow, memberships: List[MembershipRow], admin_profile: Optional[Dict[str, Any]]) -> str:
-        if admin_profile:
-            return "platform_admin"
-        if active.get("institute_id"):
-            return "school_admin" if active.get("role") == "school_admin" else "member"
-        if len([m for m in memberships if m.get("is_active")]) > 1:
-            return "multi_school_needs_selection"
-        return "no_school"
-
-    def _calendar_status(self, institute_id: Optional[str]) -> Dict[str, Any]:
-        status = {"available": False, "academic_year_count": 0, "term_count": 0, "current_academic_year_id": None, "needs_setup": True}
-        if not institute_id:
-            return status
-        try:
-            years = _safe_list(self.supabase.table("academic_years").select("id,institute_id").eq("institute_id", institute_id).execute())
-            terms = _safe_list(self.supabase.table("academic_terms").select("id,institute_id").eq("institute_id", institute_id).execute())
-            status["academic_year_count"] = len(years)
-            status["term_count"] = len(terms)
-            status["current_academic_year_id"] = years[0].get("id") if years else None
-            status["available"] = bool(years and terms)
-            status["needs_setup"] = not status["available"]
-        except Exception as exc:
-            logger.warning("Bootstrap calendar status lookup failed: %s", exc)
-        return status
-
-    def _timetable_status(self, user_id: str, institute_id: Optional[str]) -> Dict[str, Any]:
-        status = {"available": False, "teacher_timetable_id": None, "slot_count": 0, "needs_setup": True}
-        if not institute_id:
-            return status
-        try:
-            timetables = _safe_list(
-                self.supabase.table("teacher_timetables")
-                .select("id,profile_id,institute_id")
-                .eq("institute_id", institute_id)
-                .eq("profile_id", user_id)
-                .limit(1)
-                .execute()
-            )
-            if not timetables:
-                # Test/future compatibility for teacher_profile_id naming.
-                timetables = _safe_list(
-                    self.supabase.table("teacher_timetables")
-                    .select("id,teacher_profile_id,institute_id")
-                    .eq("institute_id", institute_id)
-                    .eq("teacher_profile_id", user_id)
-                    .limit(1)
-                    .execute()
-                )
-            if timetables:
-                timetable_id = timetables[0].get("id")
-                slots = _safe_list(
-                    self.supabase.table("teacher_timetable_slots")
-                    .select("id,teacher_timetable_id")
-                    .eq("teacher_timetable_id", timetable_id)
-                    .execute()
-                )
-                status["teacher_timetable_id"] = timetable_id
-                status["slot_count"] = len(slots)
-                status["available"] = bool(slots)
-                status["needs_setup"] = not status["available"]
-        except Exception as exc:
-            logger.warning("Bootstrap timetable status lookup failed: %s", exc)
-        return status
-
-    def _graph_status(self, user_id: str, user_email: str, active: MembershipRow, institute: Optional[Dict[str, Any]]) -> Dict[str, Any]:
-        base = {"available": False, "user_db": None, "institute_db": None, "projection_state": "unknown", "needs_rebuild": True, "last_checked_at": utc_now_iso(), "error_code": None}
-        try:
-            result = self.graph_probe(user_id=user_id, user_email=user_email, active_institute=active, institute=institute)
-            return {**base, **(result or {}), "last_checked_at": (result or {}).get("last_checked_at") or base["last_checked_at"]}
-        except Exception:
-            return {**base, "projection_state": "error", "needs_rebuild": True, "error_code": "neo4j_unavailable"}
-
-    def _onboarding(self, school_status: str, permissions: Dict[str, bool], calendar: Dict[str, Any], timetable: Dict[str, Any], graph: Dict[str, Any]) -> Dict[str, Any]:
-        optional = [] if graph.get("available") else ["graph_rebuild"]
-        if school_status == "no_school":
-            return {"next_step": "create_or_join_school", "required": ["school_membership"], "optional": optional, "message": "Create or join a school to finish setup."}
-        if school_status == "multi_school_needs_selection":
-            return {"next_step": "select_school", "required": ["active_school_selection"], "optional": optional, "message": "Select which school to use for this session."}
-        if permissions.get("can_manage_calendar") and calendar.get("needs_setup"):
-            return {"next_step": "setup_calendar", "required": ["calendar"], "optional": optional, "message": "Set up the school calendar."}
-        if permissions.get("can_manage_timetable") and timetable.get("needs_setup"):
-            return {"next_step": "setup_timetable", "required": ["timetable"], "optional": optional, "message": "Set up the timetable."}
-        if school_status == "school_admin" and permissions.get("can_invite_staff"):
-            return {"next_step": "invite_staff", "required": [], "optional": optional, "message": "Invite staff or continue to your workspace."}
-        return {"next_step": "ready", "required": [], "optional": optional, "message": "Your workspace is ready."}
-
-
-def build_bootstrap_response(credentials: Dict[str, Any]) -> Dict[str, Any]:
-    sb = SupabaseServiceRoleClient()
-    return BootstrapService(sb.supabase).build(credentials)

modules/database/services/exam_projection.py

@@ -1,168 +0,0 @@
-"""Project a saved exam template into the cc.public.exams Neo4j graph (card S4-7).
-
-Supabase is source of truth for the operational template (geometry, marks); cc.public.exams is
-the co-primary knowledge graph (spec §2/S2). On template save the structural skeleton —
-ExamPaper → Question/Part → Region, plus Part-[:ASSESSES]->SpecPoint — is projected here.
-
-Ownership model (R3.5.1): the graph is written by the API SERVICE ROLE only (no client writes),
-so this task reads the template via service role and writes Neo4j with the system driver. It is
-the sanctioned service-role path (documented in the ADR), distinct from the as-user request path.
-
-Join keys (never regenerated):
-    exam_questions.id       -> Question.uuid_string (container) | Part.uuid_string (leaf)
-    exam_response_areas.id  -> Region.uuid_string
-    eb_exams.exam_code      -> ExamPaper.exam_code
-    eb_specifications.spec_code -> Specification.spec_code
-
-Projection is a full re-sync per exam_code (delete this paper's Question/Part/Region, recreate),
-matching the PUT full-replace semantics — idempotent and safe to re-run.
-"""
-import os
-import uuid
-from typing import Any, Dict, List, Optional
-
-from modules.database.supabase.utils.client import SupabaseServiceRoleClient
-from modules.database.tools.neo4j_driver_tools import get_session
-from modules.logger_tool import initialise_logger
-
-logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), "default", True)
-
-# MUST match run/initialization/init_exam_graph.py (shared DB name + deterministic uuid namespace).
-EXAM_DB = "cc.public.exams"
-NS = uuid.UUID("00000000-0000-0000-0000-00000000e8a1")
-
-
-def _uid(*parts: str) -> str:
-    return str(uuid.uuid5(NS, ":".join(parts)))
-
-
-def _rows(result: Any) -> List[Dict[str, Any]]:
-    data = getattr(result, "data", None)
-    if not data:
-        return []
-    return data if isinstance(data, list) else [data]
-
-
-def project_template(template_id: str) -> Dict[str, Any]:
-    """Read the template (service role) and (re)project its structure into cc.public.exams.
-
-    Returns a counts dict. Raises on hard failure (caller decides whether to swallow — a
-    BackgroundTask logs and drops; the manual /neo4j-sync endpoint surfaces the error).
-    """
-    sb = SupabaseServiceRoleClient().supabase
-    template = (sb.table("exam_templates").select("*").eq("id", template_id).limit(1).execute().data or [None])[0]
-    if not template:
-        raise ValueError(f"template {template_id} not found")
-
-    questions = _rows(sb.table("exam_questions").select("*").eq("template_id", template_id).order("order").execute())
-    regions = _rows(sb.table("exam_response_areas").select("*").eq("template_id", template_id).execute())
-
-    # Resolve the paper's exam_code + spec metadata. Catalogue paper → from eb_exams; ad-hoc upload
-    # (no exam_code) → a stable synthetic code so the paper still has a unique graph key.
-    exam_code = template.get("exam_code")
-    spec_code = None
-    paper_meta: Dict[str, Any] = {}
-    if template.get("exam_id"):
-        eb = (sb.table("eb_exams").select("exam_code, spec_code, paper_code, tier, session")
-              .eq("id", template["exam_id"]).limit(1).execute().data or [None])[0]
-        if eb:
-            exam_code = exam_code or eb.get("exam_code")
-            spec_code = eb.get("spec_code")
-            paper_meta = eb
-    if not exam_code:
-        exam_code = f"tpl:{template_id}"
-
-    paper_uid = _uid("ExamPaper", exam_code)
-    counts = {"exam_code": exam_code, "questions": 0, "parts": 0, "regions": 0, "assesses": 0, "spec_linked": False}
-
-    with get_session(database=EXAM_DB) as s:
-        # 1. Clean this paper's existing children (full re-sync), keep the ExamPaper node itself.
-        s.run("MATCH (r:Region {exam_code:$ec}) DETACH DELETE r", ec=exam_code).consume()
-        s.run("MATCH (n {exam_code:$ec}) WHERE n:Question OR n:Part DETACH DELETE n", ec=exam_code).consume()
-
-        # 2. ExamPaper node.
-        s.run(
-            "MERGE (p:ExamPaper {uuid_string:$uid}) "
-            "SET p.exam_code=$ec, p.spec_code=$sc, p.title=$title, p.page_count=$pc, "
-            "    p.paper_code=$paper_code, p.tier=$tier, p.session=$session, p.node_storage_path=$nsp",
-            uid=paper_uid, ec=exam_code, sc=spec_code, title=template.get("title"),
-            pc=template.get("page_count"), paper_code=paper_meta.get("paper_code"),
-            tier=paper_meta.get("tier"), session=paper_meta.get("session"),
-            nsp=f"{EXAM_DB}/ExamPaper/{exam_code}",
-        ).consume()
-
-        # 3. Link to its Specification (seeded separately) when known.
-        if spec_code:
-            r = s.run(
-                "MATCH (sp:Specification {spec_code:$sc}), (p:ExamPaper {exam_code:$ec}) "
-                "MERGE (sp)-[:HAS_PAPER]->(p) RETURN count(*) AS n",
-                sc=spec_code, ec=exam_code,
-            ).single()
-            counts["spec_linked"] = bool(r and r["n"])
-
-        # 4. Question/Part nodes — pass 1: create all nodes (so parents exist before linking).
-        for q in questions:
-            label = "Question" if q.get("is_container") else "Part"
-            s.run(
-                f"MERGE (n:{label} {{uuid_string:$uid}}) "
-                "SET n.exam_code=$ec, n.label=$label, n.order=$order, n.max_marks=$mm, "
-                "    n.answer_type=$at, n.mark_scheme_type=$mst, n.spec_ref=$sref, "
-                "    n.node_storage_path=$nsp",
-                uid=q["id"], ec=exam_code, label=q.get("label"), order=q.get("order") or 0,
-                mm=q.get("max_marks") or 0, at=q.get("answer_type"),
-                mst=(q.get("mark_scheme") or {}).get("type") if isinstance(q.get("mark_scheme"), dict) else None,
-                sref=q.get("spec_ref"), nsp=f"{EXAM_DB}/{label}/{q['id']}",
-            ).consume()
-            counts["parts" if label == "Part" else "questions"] += 1
-
-        # 5. Structural + ASSESSES edges — pass 2.
-        for q in questions:
-            if q.get("parent_id"):
-                s.run(
-                    "MATCH (parent {uuid_string:$pid}), (n {uuid_string:$uid}) MERGE (parent)-[:HAS_PART]->(n)",
-                    pid=q["parent_id"], uid=q["id"],
-                ).consume()
-            else:
-                s.run(
-                    "MATCH (p:ExamPaper {exam_code:$ec}), (n {uuid_string:$uid}) MERGE (p)-[:HAS_QUESTION]->(n)",
-                    ec=exam_code, uid=q["id"],
-                ).consume()
-            if q.get("spec_ref"):
-                # SpecPoints are seeded per spec_code; match within this paper's spec when known.
-                r = s.run(
-                    "MATCH (n {uuid_string:$uid}), (sp:SpecPoint {ref:$ref}) "
-                    + ("WHERE sp.spec_code=$sc " if spec_code else "")
-                    + "MERGE (n)-[:ASSESSES]->(sp) RETURN count(*) AS n",
-                    uid=q["id"], ref=q["spec_ref"], sc=spec_code,
-                ).single()
-                counts["assesses"] += (r["n"] if r else 0)
-
-        # 6. Region nodes + HAS_REGION edges.
-        # Only response/context regions are part of the knowledge graph (RegionNode.kind). The other
-        # S4-9 kinds (question_number, mark_area, reference, furniture) are physical-layer metadata
-        # about the paper, not curriculum structure — they stay in Supabase, out of cc.public.exams.
-        for rg in regions:
-            if rg.get("kind") not in ("response", "context"):
-                continue
-            s.run(
-                "MERGE (r:Region {uuid_string:$uid}) "
-                "SET r.exam_code=$ec, r.page=$page, r.kind=$kind, r.response_form=$rf, r.node_storage_path=$nsp",
-                uid=rg["id"], ec=exam_code, page=rg.get("page"), kind=rg.get("kind"),
-                rf=rg.get("response_form"), nsp=f"{EXAM_DB}/Region/{rg['id']}",
-            ).consume()
-            s.run(
-                "MATCH (q {uuid_string:$qid}), (r:Region {uuid_string:$uid}) MERGE (q)-[:HAS_REGION]->(r)",
-                qid=rg["question_id"], uid=rg["id"],
-            ).consume()
-            counts["regions"] += 1
-
-    logger.info(f"Projected template {template_id} → cc.public.exams: {counts}")
-    return counts
-
-
-def project_template_safe(template_id: str) -> None:
-    """BackgroundTask wrapper: never raises (a failed projection must not break the HTTP save)."""
-    try:
-        project_template(template_id)
-    except Exception as exc:
-        logger.error(f"Background Neo4j projection failed for template {template_id}: {exc}")

modules/database/services/provisioning_service.py

@@ -229,7 +229,6 @@ class ProvisioningService:
             "neo4j_private_db_name": school_db,
             "neo4j_private_sync_status": "ready",
             "neo4j_private_sync_at": datetime.utcnow().isoformat(),
-            "neo4j_uuid_string": self._sanitize_component(institute_id),
         }
         try:
             (
@@ -263,7 +262,6 @@ class ProvisioningService:
             "admin": ("superadmin", "superadmin"),
             "super_admin": ("superadmin", "superadmin"),
             "superadmin": ("superadmin", "superadmin"),
-            "platform_admin": ("superadmin", "superadmin"),
         }
         neo_user_type, worker_type = user_type_map.get(user_type_raw, (user_type_raw or "standard", user_type_raw or "standard"))
 

modules/database/supabase/utils/client.py

@@ -14,39 +14,19 @@ class CreateBucketOptions(TypedDict, total=False):
     allowed_mime_types: List[str]
     name: str
 
-def _create_base_client(url: str, key: str, access_token: Optional[str] = None, options: Optional[Dict[str, Any]] = None) -> Client:
-    """Create a base Supabase client with given configuration.
-    
-    If access_token is provided, it is used as the Authorization header (for per-user RLS).
-    Otherwise, the API key is used (service role bypasses RLS, anon key does not).
-    """
-    # If an access token is provided, use it for Authorization (enables per-user RLS)
-    # Otherwise fall back to the API key
-    auth_header = f"Bearer {access_token}" if access_token else f"Bearer {key}"
-
-    # Only override Authorization here. apikey is supplied to create_client via the `key` arg and
-    # set by supabase-py itself; setting it again here sends a DUPLICATE apikey header that the
-    # Supabase gateway (Kong) rejects with 401 "Duplicate API key found". For a per-user client
-    # apikey stays the anon key (from `key`) while this Authorization carries the user JWT.
-    headers = {
-        "Authorization": auth_header,
-    }
-    if options:
-        headers.update(options.get("headers", {}))
-
+def _create_base_client(url: str, key: str, options: Optional[Dict[str, Any]] = None) -> Client:
+    """Create a base Supabase client with given configuration."""
     client_options = SyncClientOptions(
         schema="public",
         storage=SyncMemoryStorage(),
-        headers=headers,
+        headers={
+            "Authorization": f"Bearer {key}"
+        }
     )
     return create_client(url, key, options=client_options)
 
 class SupabaseServiceRoleClient:
-    """Supabase client for making authenticated requests using the service role key.
-    
-    NOTE: Service role bypasses all RLS policies. Use only for admin operations
-    where you need to read/write any row regardless of ownership.
-    """
+    """Supabase client for making authenticated requests using the service role key"""
     
     def __init__(self, url: Optional[str] = None, service_role_key: Optional[str] = None):
         """Initialize the Supabase client with URL and service role key"""
@@ -56,7 +36,7 @@ class SupabaseServiceRoleClient:
         if not self.url or not self.service_role_key:
             raise ValueError("SUPABASE_URL and SERVICE_ROLE_KEY must be provided")
         
-        # Initialize Supabase client with service role key (bypasses RLS)
+        # Initialize Supabase client with service role key and optional access token
         self.supabase = _create_base_client(self.url, self.service_role_key)
 
     def create_bucket(self, id: str, options: Optional[CreateBucketOptions] = None) -> Dict[str, Any]:
@@ -68,29 +48,17 @@ class SupabaseServiceRoleClient:
         return self.supabase.storage.create_bucket(id, options=options)
 
 class SupabaseAnonClient:
-    """Supabase client for making authenticated requests using the anon key.
-    
-    When initialized with an access_token, per-user RLS policies are enforced
-    via auth.uid() in the JWT. Without an access_token, requests use the anon key
-    which does NOT enforce per-user RLS (only bucket-level storage rules apply).
-    """
+    """Supabase client for making authenticated requests using the anon key"""
     
     def __init__(self, url: Optional[str] = None, anon_key: Optional[str] = None, access_token: Optional[str] = None):
-        """Initialize the Supabase client with URL and anon key.
-        
-        Args:
-            url: Supabase URL
-            anon_key: Anon API key (fallback if no access_token)
-            access_token: User's JWT access token for per-user RLS enforcement
-        """
+        """Initialize the Supabase client with URL and anon key"""
         self.url = url or os.environ.get("SUPABASE_URL", "http://localhost:8000")
         self.anon_key = anon_key or os.environ.get("ANON_KEY")
-        self.access_token = access_token
         
         if not self.url or not self.anon_key:
             raise ValueError("SUPABASE_URL and ANON_KEY must be provided")
         
-        # Initialize Supabase client with anon key and optional access token for RLS
+        # Initialize Supabase client with anon key and optional access token
         self.supabase = _create_base_client(self.url, self.anon_key, access_token=access_token)
 
     def create_bucket(self, id: str, options: Optional[CreateBucketOptions] = None) -> Dict[str, Any]:
@@ -99,13 +67,5 @@ class SupabaseAnonClient:
 
     @classmethod
     def for_user(cls, access_token: str) -> 'SupabaseAnonClient':
-        """Create a client instance for a specific user using their access token.
-        
-        This enables per-user RLS enforcement via auth.uid() in the JWT.
-        """
-        if not access_token or not access_token.strip():
-            raise ValueError("access_token is required for per-user Supabase clients")
-        token = access_token.strip()
-        if token.lower().startswith("bearer "):
-            token = token.split(None, 1)[1]
-        return cls(access_token=token)
+        """Create a client instance for a specific user using their access token"""
+        return cls(access_token=access_token)

modules/database/supabase/utils/storage.py

@@ -110,13 +110,14 @@ class StorageAdmin(StorageManager):
         public: bool = False,
         file_size_limit: Optional[int] = None,
         allowed_mime_types: Optional[List[str]] = None,
-        owner: Optional[str] = None,
-        owner_id: Optional[str] = None
+        owner: Optional[str] = None,  # Kept for backwards compatibility but not used
+        owner_id: Optional[str] = None  # Kept for backwards compatibility but not used
     ) -> Dict[str, Any]:
         """Create a new storage bucket with supported parameters."""
         try:
             self.logger.info(f"Creating bucket {id} with name {name}")
             
+            # Prepare bucket options with only supported parameters
             options: Optional[CreateBucketOptions] = {}
             if public:
                 options["public"] = public
@@ -125,6 +126,7 @@ class StorageAdmin(StorageManager):
             if allowed_mime_types is not None:
                 options["allowed_mime_types"] = allowed_mime_types
             
+            # Create bucket with supported parameters only
             bucket = self.client.supabase.storage.create_bucket(
                 str(id),
                 options=options if options else None
@@ -150,7 +152,7 @@ class StorageAdmin(StorageManager):
                     "public": False,
                     "owner": owner_id,
                     "owner_id": "superadmin",
-                    "file_size_limit": 50 * 1024 * 1024,
+                    "file_size_limit": 50 * 1024 * 1024,  # 50MB
                     "allowed_mime_types": [
                         'image/*', 'video/*', 'application/pdf',
                         'application/msword', 'application/vnd.openxmlformats-officedocument.*',
@@ -163,7 +165,7 @@ class StorageAdmin(StorageManager):
                     "public": False,
                     "owner": owner_id,
                     "owner_id": "superadmin",
-                    "file_size_limit": 50 * 1024 * 1024,
+                    "file_size_limit": 50 * 1024 * 1024,  # 50MB
                     "allowed_mime_types": [
                         'image/*', 'video/*', 'application/pdf',
                         'application/msword', 'application/vnd.openxmlformats-officedocument.*',
@@ -175,7 +177,7 @@ class StorageAdmin(StorageManager):
             results = []
             for bucket in core_buckets:
                 try:
-                    bucket_name = bucket.pop("name")
+                    bucket_name = bucket.pop("name")  # Remove name from options
                     result = self.create_bucket(name=bucket_name, **bucket)
                     results.append({
                         "bucket": bucket["id"],
@@ -208,7 +210,7 @@ class StorageAdmin(StorageManager):
                 public=False,
                 owner=user_id,
                 owner_id=username,
-                file_size_limit=50 * 1024 * 1024,
+                file_size_limit=50 * 1024 * 1024,  # 50MB
                 allowed_mime_types=[
                     'image/*', 'video/*', 'application/pdf',
                     'application/msword', 'application/vnd.openxmlformats-officedocument.*',
@@ -234,7 +236,7 @@ class StorageAdmin(StorageManager):
                     "public": True,
                     "owner": owner_id,
                     "owner_id": school_id,
-                    "file_size_limit": 50 * 1024 * 1024,
+                    "file_size_limit": 50 * 1024 * 1024,  # 50MB
                     "allowed_mime_types": [
                         'image/*', 'video/*', 'application/pdf',
                         'application/msword', 'application/vnd.openxmlformats-officedocument.*',
@@ -247,7 +249,7 @@ class StorageAdmin(StorageManager):
                     "public": False,
                     "owner": owner_id,
                     "owner_id": school_id,
-                    "file_size_limit": 50 * 1024 * 1024,
+                    "file_size_limit": 50 * 1024 * 1024,  # 50MB
                     "allowed_mime_types": [
                         'image/*', 'video/*', 'application/pdf',
                         'application/msword', 'application/vnd.openxmlformats-officedocument.*',
@@ -259,7 +261,7 @@ class StorageAdmin(StorageManager):
             results = {}
             for bucket in school_buckets:
                 try:
-                    bucket_name = bucket.pop("name")
+                    bucket_name = bucket.pop("name")  # Remove name from options
                     result = self.create_bucket(name=bucket_name, **bucket)
                     results[bucket["id"]] = {
                         "status": "success",
@@ -279,20 +281,9 @@ class StorageAdmin(StorageManager):
             raise StorageError(str(e))
 
 class StorageUser(StorageManager):
-    """Storage user class for managing storage with per-user RLS enforcement.
+    """Storage user class for managing storage buckets with user role access."""
     
-    Requires a user access token to enforce Row Level Security policies.
-    Without a token, requests use the anon key which does NOT enforce per-user RLS.
-    """
-    
-    def __init__(self, user_id: Optional[str] = None, access_token: Optional[str] = None):
-        """Initialize StorageUser with user role client.
-        
-        Args:
-            user_id: The user's ID (for logging/context)
-            access_token: User's JWT access token for per-user RLS enforcement
-        """
-        self.user_id = user_id
-        # Pass access_token to enable per-user RLS via auth.uid() in JWT
-        client = SupabaseAnonClient.for_user(access_token) if access_token else SupabaseAnonClient()
-        super().__init__(client)
+    def __init__(self, user_id: Optional[str] = None):
+        """Initialize StorageUser with user role client."""
+        super().__init__(SupabaseAnonClient())
+        self.user_id = user_id

modules/database/tools/neo4j_driver_tools.py

@@ -18,7 +18,7 @@ def _retry_with_backoff(
 ) -> any:
     """
     Helper function to retry operations with exponential backoff.
-
+    
     Args:
         func: Function to retry
         max_attempts: Maximum number of retry attempts
@@ -29,26 +29,26 @@ def _retry_with_backoff(
     attempt = 0
     delay = initial_delay
     start_time = time.time()
-
+    
     while attempt < max_attempts:
         try:
             return func()
         except Exception as e:
             attempt += 1
             elapsed_time = time.time() - start_time
-
+            
             # Check if we've exceeded the maximum total wait time
             if elapsed_time >= max_total_wait:
                 logger.error(f"Exceeded maximum total wait time of {max_total_wait} seconds")
                 raise
-
+            
             if attempt == max_attempts:
                 logger.error(f"Final attempt {attempt} failed: {e}")
                 raise
-
+            
             # Calculate next delay with exponential backoff, but cap it
             delay = min(delay * 2, max_delay)
-
+            
             # If we're in a container initialization scenario, provide more context
             if "Connection refused" in str(e):
                 logger.warning(
@@ -59,7 +59,7 @@ def _retry_with_backoff(
                 )
             else:
                 logger.warning(f"Attempt {attempt} failed: {e}. Retrying in {delay:.1f} seconds...")
-
+            
             time.sleep(delay)
 
 def get_driver(db_name: Optional[str] = None, url: Optional[str] = None, auth: Optional[Tuple[str, str]] = None) -> Optional[Driver]:
@@ -71,7 +71,7 @@ def get_driver(db_name: Optional[str] = None, url: Optional[str] = None, auth: O
             logger.error("Neo4j credentials not found in environment")
             return None
         auth = (username, password)
-
+    
     if auth is None:
         logger.error("No authentication credentials provided")
         return None
@@ -95,7 +95,7 @@ def get_driver(db_name: Optional[str] = None, url: Optional[str] = None, auth: O
     except Exception as e:
         logger.error(f"Failed to establish Neo4j connection after all retries: {e}")
         return None
-
+    
     # Test the connection with the specific database
     if db_name and driver:
         def verify_database():
@@ -127,50 +127,27 @@ def close_driver(driver: Optional[Driver]) -> None:
         logger.info("Closing driver")
         driver.close()
 
-# Global driver instance — None means not yet initialised, _driver_unavailable=True means connection failed
+# Global driver instance
 _driver: Optional[Driver] = None
-_driver_unavailable: bool = False
 
 def get_global_driver() -> Optional[Driver]:
-    """Get or create the global Neo4j driver instance.
-
-    Caches both success and failure so a broken Neo4j connection causes
-    a single 60-second retry at startup, then fast-fails on every
-    subsequent call instead of hanging for 60s each time.
-    """
-    global _driver, _driver_unavailable
-    if _driver_unavailable:
-        return None
+    """Get or create the global Neo4j driver instance."""
+    global _driver
     if _driver is None:
         _driver = get_driver()
-        if _driver is None:
-            _driver_unavailable = True
-            logger.error("Neo4j driver unavailable — all subsequent Neo4j calls will fail fast until process restarts")
     return _driver
 
-def reset_global_driver() -> None:
-    """Reset the cached driver, forcing a reconnection attempt on the next call.
-
-    Call this if Neo4j becomes available after the process started.
-    """
-    global _driver, _driver_unavailable
-    if _driver:
-        close_driver(_driver)
-    _driver = None
-    _driver_unavailable = False
-    logger.info("Global Neo4j driver reset — will reconnect on next request")
-
 @contextmanager
 def get_session(database: Optional[str] = None) -> Generator[Session, None, None]:
     """Get a Neo4j session using the global driver."""
     driver = get_global_driver()
     if driver is None:
         raise Exception("Failed to get Neo4j driver")
-
+    
     session = None
     try:
         session = driver.session(database=database)
         yield session
     finally:
         if session:
-            session.close()
+            session.close()

modules/redis_manager.py

@@ -491,9 +491,7 @@ class RedisManager:
         
         try:
             if not self.client:
-                logger.info("Redis health check has no active client; connecting now")
-                if not self.connect():
-                    raise Exception("No Redis connection")
+                raise Exception("No Redis connection")
             
             # Test connection
             self.client.ping()

modules/services/docling/regions.py

@@ -1,13 +0,0 @@
-"""Compatibility import path for S5 Docling response-region geometry."""
-
-from api.services.docling.regions import (
-    RegionCandidate,
-    detect_response_regions_from_image,
-    detect_response_regions_from_pdf,
-)
-
-__all__ = [
-    "RegionCandidate",
-    "detect_response_regions_from_image",
-    "detect_response_regions_from_pdf",
-]

modules/services/exam_extract.py

@@ -1,86 +0,0 @@
-"""Client for the exam extraction SERVICE (docling-exam-spike, P1).
-
-The service runs the spike's FULL recognition pipeline (textlayer → question tree → OMR/figure/table
-sidecars → structure fusion → analyse) and returns the ghost-region contract the app already consumes.
-This module is a thin HTTP client: POST the paper, poll, return the `analyse` suggestions. The app's
-auto-map merges them (coordinate-adapted, id-remapped) in routers/exam/templates.py.
-
-Config: EXAM_EXTRACT_URL (e.g. http://192.168.0.203:8899). If unset, the app keeps its thin first-pass.
-"""
-from __future__ import annotations
-
-import base64
-import os
-import time
-from typing import Any, Dict, Optional
-
-import requests
-
-from modules.logger_tool import initialise_logger
-
-logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), "default", True)
-
-
-class ExtractError(RuntimeError):
-    pass
-
-
-def service_url() -> Optional[str]:
-    url = os.getenv("EXAM_EXTRACT_URL")
-    return url.rstrip("/") if url else None
-
-
-def is_enabled() -> bool:
-    return bool(service_url())
-
-
-def _get(base: str, slug: str, timeout: int = 30) -> Dict[str, Any]:
-    r = requests.get(f"{base}/api/extract/{slug}", timeout=timeout)
-    r.raise_for_status()
-    return r.json()
-
-
-def get_replica(slug: str, timeout: int = 30) -> Dict[str, Any]:
-    """The digital-replica markdown for a paper (P4): {slug, title, n_questions, total_marks, markdown,
-    questions:[{label, marks, markdown}]}. Raises ExtractError if the paper has no replica yet."""
-    base = service_url()
-    if not base:
-        raise ExtractError("EXAM_EXTRACT_URL not configured")
-    r = requests.get(f"{base}/api/replica/{slug}", timeout=timeout)
-    if r.status_code == 404:
-        raise ExtractError(f"no digital replica for {slug}")
-    r.raise_for_status()
-    return r.json()
-
-
-def extract_suggestions(slug: str, pdf_bytes: bytes, *, force: bool = False,
-                        poll_timeout: int = 1500, poll_interval: int = 5) -> Dict[str, Any]:
-    """POST the paper to the service and poll until the analyse contract is ready.
-
-    Returns the full analyse payload: {status, coordinate_space:'page_fraction', margins,
-    suggestions:{questions[], response_areas[], boundaries[]}, meta}. Raises ExtractError on
-    failure/timeout. A cold paper is ~15 min (Docling per masked page); cached papers return instantly.
-    """
-    base = service_url()
-    if not base:
-        raise ExtractError("EXAM_EXTRACT_URL not configured")
-    payload = {"slug": slug, "pdf_b64": base64.b64encode(pdf_bytes).decode(), "force": force}
-    r = requests.post(f"{base}/api/extract", json=payload, timeout=120)
-    r.raise_for_status()
-    started = r.json()
-    if not started.get("ok", True):
-        raise ExtractError(started.get("error") or "service rejected the request")
-    # cached → fetch the contract straight away; otherwise poll the running job
-    deadline = time.time() + poll_timeout
-    while True:
-        d = _get(base, slug)
-        status = d.get("status")
-        if status == "complete":
-            if not (d.get("suggestions") or {}):
-                raise ExtractError("service returned complete with no suggestions")
-            return d
-        if status == "error":
-            raise ExtractError(f"extraction failed: {d.get('error')}")
-        if time.time() >= deadline:
-            raise ExtractError(f"extraction timed out after {poll_timeout}s (slug={slug})")
-        time.sleep(poll_interval)

modules/upload_validation.py

@@ -1,99 +0,0 @@
-"""Upload boundary validation shared by file-upload endpoints.
-
-E3 hardening: keep user-facing upload routes from buffering arbitrary data and
-from accepting arbitrary MIME/types into Supabase storage.
-"""
-from __future__ import annotations
-
-import os
-from typing import Iterable, Optional
-
-from fastapi import HTTPException, UploadFile
-
-# Conservative defaults: Classroom Copilot uploads are user documents/images.
-# Exam scan uploads already have their own 50 MB PDF-only guard in routers.exam.batches.
-MAX_UPLOAD_BYTES = int(os.getenv("CC_UPLOAD_MAX_BYTES", str(25 * 1024 * 1024)))
-UPLOAD_CHUNK_BYTES = 1024 * 1024
-
-ALLOWED_UPLOAD_MIME_TYPES = frozenset(
-    mt.strip().lower()
-    for mt in os.getenv(
-        "CC_UPLOAD_ALLOWED_MIME_TYPES",
-        ",".join(
-            [
-                "application/pdf",
-                "image/png",
-                "image/jpeg",
-                "image/webp",
-                "image/gif",
-                "text/plain",
-                "text/csv",
-                "text/markdown",
-                "application/msword",
-                "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-                "application/vnd.ms-powerpoint",
-                "application/vnd.openxmlformats-officedocument.presentationml.presentation",
-                "application/vnd.ms-excel",
-                "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-            ]
-        ),
-    ).split(",")
-    if mt.strip()
-)
-
-_PDF_MIME_TYPES = {"application/pdf", "application/x-pdf"}
-
-
-def allowed_upload_mime_types_csv() -> str:
-    """Stable display string for evidence/errors without leaking config internals."""
-    return ", ".join(sorted(ALLOWED_UPLOAD_MIME_TYPES))
-
-
-def _declared_mime(upload: UploadFile) -> str:
-    return (upload.content_type or "application/octet-stream").split(";", 1)[0].strip().lower()
-
-
-def validate_upload_mime(upload: UploadFile, *, allowed_mime_types: Optional[Iterable[str]] = None) -> str:
-    """Validate client-declared upload MIME/type and return its normalised value."""
-    declared = _declared_mime(upload)
-    allowed = {mt.lower() for mt in (allowed_mime_types or ALLOWED_UPLOAD_MIME_TYPES)}
-    if declared not in allowed:
-        raise HTTPException(
-            status_code=415,
-            detail=(
-                f"Unsupported upload type '{declared}'. Allowed MIME types: "
-                f"{', '.join(sorted(allowed))}"
-            ),
-        )
-    return declared
-
-
-async def read_upload_bytes(
-    upload: UploadFile,
-    *,
-    max_bytes: int = MAX_UPLOAD_BYTES,
-    allowed_mime_types: Optional[Iterable[str]] = None,
-) -> tuple[bytes, str]:
-    """Validate MIME and read an UploadFile with a hard size ceiling."""
-    mime_type = validate_upload_mime(upload, allowed_mime_types=allowed_mime_types)
-    chunks: list[bytes] = []
-    total = 0
-    while True:
-        chunk = await upload.read(UPLOAD_CHUNK_BYTES)
-        if not chunk:
-            break
-        total += len(chunk)
-        if total > max_bytes:
-            raise HTTPException(status_code=413, detail=f"Upload exceeds max size ({max_bytes} bytes)")
-        chunks.append(chunk)
-    return b"".join(chunks), mime_type
-
-
-async def read_pdf_upload_bytes(upload: UploadFile, *, max_bytes: int = MAX_UPLOAD_BYTES) -> bytes:
-    """Read a PDF-only upload with size and lightweight magic-header validation."""
-    data, _mime_type = await read_upload_bytes(upload, max_bytes=max_bytes, allowed_mime_types=_PDF_MIME_TYPES)
-    if not data:
-        raise HTTPException(status_code=400, detail="Uploaded PDF is empty")
-    if not data.startswith(b"%PDF-"):
-        raise HTTPException(status_code=415, detail="Uploaded file is not a valid PDF")
-    return data

pytest.ini

@@ -1,4 +0,0 @@
-[pytest]
-testpaths = tests
-python_files = test_*.py
-addopts = -q

requirements.txt

@@ -79,6 +79,4 @@ pdfminer.six
 Pillow
 psutil
 PyPDF2
-PyMuPDF
-# OpenCV answer-region geometry (S5-4)
-opencv-python-headless
+PyMuPDF

routers/database/files/files.py

@@ -12,7 +12,6 @@ from modules.auth.supabase_bearer import SupabaseBearer, verify_supabase_jwt_str
 from modules.logger_tool import initialise_logger
 from modules.database.supabase.utils.client import SupabaseServiceRoleClient
 from modules.database.supabase.utils.storage import StorageAdmin
-from modules.upload_validation import read_upload_bytes
 from modules.document_processor import DocumentProcessor
 from modules.queue_system import (
     enqueue_tika_task, enqueue_docling_task, enqueue_split_map_task,
@@ -37,24 +36,6 @@ DOCLING_NOOCR_TIMEOUT = int(os.getenv('DOCLING_NOOCR_TIMEOUT', '3600'))  # 1 hou
 
 logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
 
-def _user_id_from_payload(payload: Dict[str, Any]) -> str:
-    user_id = payload.get('sub') or payload.get('user_id')
-    if not user_id:
-        raise HTTPException(status_code=401, detail="Invalid token payload")
-    return user_id
-
-def _cabinet_visible_to_user(client: SupabaseServiceRoleClient, cabinet_id: str, user_id: str) -> bool:
-    """Require cabinet ownership before service-role reads file metadata."""
-    owned = (
-        client.supabase.table('file_cabinets')
-        .select('id')
-        .eq('id', cabinet_id)
-        .eq('user_id', user_id)
-        .limit(1)
-        .execute()
-    )
-    return bool(owned.data)
-
 def _safe_filename(name: str) -> str:
     base = os.path.basename(name or 'file')
     return re.sub(r"[^A-Za-z0-9._-]+", "_", base)
@@ -89,13 +70,13 @@ async def upload_file(
     # Stage DB row to get file_id
     staged_path = f"{cabinet_id}/staging/{uuid.uuid4()}"
     name = _safe_filename(path or file.filename)
-    file_bytes, mime_type = await read_upload_bytes(file)
+    file_bytes = await file.read()
     insert_res = client.supabase.table('files').insert({
         'cabinet_id': cabinet_id,
         'name': name,
         'path': staged_path,
         'bucket': bucket,
-        'mime_type': mime_type,
+        'mime_type': file.content_type,
         'uploaded_by': user_id,
         'size_bytes': len(file_bytes),
         'source': 'classroomcopilot-web'
@@ -108,7 +89,7 @@ async def upload_file(
     # Final storage path: bucket/cabinet_id/file_id/file
     final_storage_path = f"{cabinet_id}/{file_id}/{name}"
     try:
-        storage.upload_file(bucket, final_storage_path, file_bytes, mime_type, upsert=True)
+        storage.upload_file(bucket, final_storage_path, file_bytes, file.content_type or 'application/octet-stream', upsert=True)
     except Exception as e:
         # cleanup staged row
         client.supabase.table('files').delete().eq('id', file_id).execute()
@@ -136,10 +117,7 @@ async def upload_file(
 
 @router.get("/files")
 def list_files(cabinet_id: str, payload: Dict[str, Any] = Depends(auth)):
-    user_id = _user_id_from_payload(payload)
     client = SupabaseServiceRoleClient()
-    if not _cabinet_visible_to_user(client, cabinet_id, user_id):
-        return []
     res = client.supabase.table('files').select('*').eq('cabinet_id', cabinet_id).execute()
     return res.data
 

routers/database/files/files_simplified.py

@@ -19,7 +19,6 @@ from fastapi.responses import JSONResponse
 from modules.auth.supabase_bearer import SupabaseBearer
 from modules.database.supabase.utils.client import SupabaseServiceRoleClient
 from modules.database.supabase.utils.storage import StorageAdmin
-from modules.upload_validation import read_upload_bytes
 from modules.logger_tool import initialise_logger
 
 router = APIRouter()
@@ -27,24 +26,6 @@ auth = SupabaseBearer()
 
 logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
 
-def _user_id_from_payload(payload: Dict[str, Any]) -> str:
-    user_id = payload.get('sub') or payload.get('user_id')
-    if not user_id:
-        raise HTTPException(status_code=401, detail="Invalid token payload")
-    return user_id
-
-def _cabinet_visible_to_user(client: SupabaseServiceRoleClient, cabinet_id: str, user_id: str) -> bool:
-    """Require cabinet ownership before service-role reads file metadata."""
-    owned = (
-        client.supabase.table('file_cabinets')
-        .select('id')
-        .eq('id', cabinet_id)
-        .eq('user_id', user_id)
-        .limit(1)
-        .execute()
-    )
-    return bool(owned.data)
-
 def _choose_bucket(scope: str, user_id: str, school_id: Optional[str]) -> str:
     """Choose appropriate bucket based on scope - matches old system logic."""
     scope = (scope or 'teacher').lower()
@@ -73,9 +54,10 @@ async def upload_file(
         if not user_id:
             raise HTTPException(status_code=401, detail="User ID required")
         
-        # Validate MIME/type and read file content with a hard size limit.
-        file_bytes, mime_type = await read_upload_bytes(file)
+        # Read file content
+        file_bytes = await file.read()
         file_size = len(file_bytes)
+        mime_type = file.content_type or 'application/octet-stream'
         filename = file.filename or path
         
         logger.info(f"📤 Simplified upload: {filename} ({file_size} bytes) for user {user_id}")
@@ -152,10 +134,7 @@ async def upload_file(
 @router.get("/files")
 def list_files(cabinet_id: str, payload: Dict[str, Any] = Depends(auth)):
     """List files in a cabinet."""
-    user_id = _user_id_from_payload(payload)
     client = SupabaseServiceRoleClient()
-    if not _cabinet_visible_to_user(client, cabinet_id, user_id):
-        return []
     res = client.supabase.table('files').select('*').eq('cabinet_id', cabinet_id).execute()
     return res.data
 

routers/database/timetable/timetables.py

@@ -1,86 +0,0 @@
-"""
-GET /database/timetable/timetables
-Optional filters: class_id, type, active
-Returns {"timetables": [...]} for the caller's school.
-"""
-import os
-from typing import Any, Dict, List, Optional
-from fastapi import APIRouter, Depends, HTTPException, Query
-from pydantic import BaseModel
-from modules.logger_tool import initialise_logger
-from modules.auth.supabase_bearer import SupabaseBearer
-from modules.database.supabase.utils.client import SupabaseServiceRoleClient
-
-ADMIN_TYPES = ("school_admin", "department_head")
-
-logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
-router = APIRouter()
-
-
-class TimetableResponse(BaseModel):
-    timetables: List[Dict[str, Any]]
-
-
-def _sb() -> SupabaseServiceRoleClient:
-    return SupabaseServiceRoleClient()
-
-
-def _require_institute(user_id: str) -> Optional[str]:
-    try:
-        sb = _sb()
-        p = sb.supabase.table("profiles").select("school_id").eq("id", user_id).single().execute()
-        return str((p.data or {}).get("school_id") or "")
-    except Exception:
-        return None
-
-
-def _is_admin(user_id: str, institute_id: str) -> bool:
-    try:
-        sb = _sb()
-        r = (
-            sb.supabase.table("institute_memberships")
-            .select("role")
-            .eq("profile_id", user_id)
-            .eq("institute_id", institute_id)
-            .in_("role", list(ADMIN_TYPES))
-            .limit(1)
-            .execute()
-        )
-        return bool(r.data)
-    except Exception:
-        return False
-
-
-@router.get("", response_model=TimetableResponse)
-async def list_timetables(
-    class_id: Optional[str] = Query(None),
-    type: Optional[str] = Query(None),
-    active: Optional[bool] = Query(None),
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-
-    if not institute_id:
-        return {"timetables": []}
-
-    sb = _sb()
-
-    if not _is_admin(user_id, institute_id):
-        return {"timetables": []}
-
-    q = (
-        sb.supabase.table("school_timetables")
-        .select("*")
-        .eq("institute_id", institute_id)
-    )
-
-    if class_id:
-        q = q.eq("class_id", class_id)
-    if type:
-        q = q.eq("type", type)
-    if active is not None:
-        q = q.eq("is_active", active)
-
-    res = q.order("created_at", desc=True).execute()
-    return {"timetables": res.data or []}

routers/database/tools/classes_router.py

@@ -1,626 +0,0 @@
-"""
-Classes Router — Supabase-backed CRUD for the `classes` table.
-Institute members can read their institute's classes.
-School admins and teachers can create classes.
-School admins manage teacher/student assignments; teachers can leave/enroll.
-"""
-import os
-from datetime import datetime
-from typing import Any, Dict, List, Optional
-from fastapi import APIRouter, Depends, HTTPException
-from pydantic import BaseModel
-from modules.logger_tool import initialise_logger
-from modules.auth.supabase_bearer import SupabaseBearer
-from modules.database.supabase.utils.client import SupabaseServiceRoleClient
-
-logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
-router = APIRouter()
-
-
-def _sb() -> SupabaseServiceRoleClient:
-    return SupabaseServiceRoleClient()
-
-
-def _resolve_institute_id(user_id: str) -> Optional[str]:
-    """Return the Supabase institute UUID for this user via profiles.school_id."""
-    try:
-        sb = _sb()
-        p = sb.supabase.table("profiles").select("school_id").eq("id", user_id).single().execute()
-        return str((p.data or {}).get("school_id") or "")
-    except Exception:
-        return None
-
-
-def _require_institute(user_id: str) -> Optional[str]:
-    """Return institute_id, or None if the user has no school membership."""
-    return _resolve_institute_id(user_id)
-
-
-def _is_school_admin(user_id: str, institute_id: str) -> bool:
-    try:
-        sb = _sb()
-        r = (
-            sb.supabase.table("institute_memberships")
-            .select("role")
-            .eq("profile_id", user_id)
-            .eq("institute_id", institute_id)
-            .in_("role", ["school_admin", "department_head"])
-            .limit(1)
-            .execute()
-        )
-        return bool(r.data)
-    except Exception:
-        return False
-
-
-# ─── Request models ───────────────────────────────────────────────────────────
-
-class CreateClassRequest(BaseModel):
-    name: str
-    class_code: Optional[str] = None
-    subject: Optional[str] = None
-    key_stage: Optional[str] = None
-    year_group: Optional[str] = None
-    academic_year: Optional[str] = None
-    description: Optional[str] = None
-
-
-class UpdateClassRequest(BaseModel):
-    name: Optional[str] = None
-    class_code: Optional[str] = None
-    subject: Optional[str] = None
-    key_stage: Optional[str] = None
-    year_group: Optional[str] = None
-    academic_year: Optional[str] = None
-    description: Optional[str] = None
-    is_active: Optional[bool] = None
-
-
-class AddTeacherRequest(BaseModel):
-    teacher_id: str
-    is_primary: bool = False
-
-
-class AddStudentRequest(BaseModel):
-    student_id: str
-
-
-# ─── Endpoints ────────────────────────────────────────────────────────────────
-
-@router.get("")
-async def list_classes(
-    subject: Optional[str] = None,
-    school_year: Optional[str] = None,
-    academic_year: Optional[str] = None,
-    key_stage: Optional[str] = None,
-    year_group: Optional[str] = None,
-    search: Optional[str] = None,
-    active_only: bool = True,
-    skip: int = 0,
-    limit: int = 50,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    if not institute_id:
-        return {"classes": [], "total": 0}
-    sb = _sb()
-
-    q = sb.supabase.table("classes").select("*", count="exact").eq("institute_id", institute_id)
-    if active_only:
-        q = q.eq("is_active", True)
-    if subject:
-        q = q.eq("subject", subject)
-    # support both param names
-    yr = academic_year or school_year
-    if yr:
-        q = q.eq("academic_year", yr)
-    if key_stage:
-        q = q.eq("key_stage", key_stage)
-    if year_group:
-        q = q.eq("year_group", year_group)
-    if search:
-        q = q.ilike("name", f"%{search}%")
-
-    q = q.order("name").range(skip, skip + limit - 1)
-    res = q.execute()
-
-    classes = res.data or []
-    total = res.count or 0
-
-    # Attach student/teacher counts
-    class_ids = [c["id"] for c in classes]
-    teacher_counts: Dict[str, int] = {}
-    student_counts: Dict[str, int] = {}
-    if class_ids:
-        try:
-            tc = (
-                sb.supabase.table("class_teachers")
-                .select("class_id", count="exact")
-                .in_("class_id", class_ids)
-                .execute()
-            )
-            for row in (tc.data or []):
-                teacher_counts[row["class_id"]] = teacher_counts.get(row["class_id"], 0) + 1
-
-            sc = (
-                sb.supabase.table("class_students")
-                .select("class_id")
-                .in_("class_id", class_ids)
-                .eq("status", "active")
-                .execute()
-            )
-            for row in (sc.data or []):
-                student_counts[row["class_id"]] = student_counts.get(row["class_id"], 0) + 1
-        except Exception:
-            pass
-
-    enriched = [
-        {**c, "teacher_count": teacher_counts.get(c["id"], 0), "student_count": student_counts.get(c["id"], 0)}
-        for c in classes
-    ]
-    return {"classes": enriched, "total": total}
-
-
-@router.get("/me/teacher")
-async def my_teaching_classes(
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    if not institute_id:
-        return {"classes": []}
-    sb = _sb()
-
-    assigned = (
-        sb.supabase.table("class_teachers")
-        .select("class_id, is_primary")
-        .eq("teacher_id", user_id)
-        .execute()
-        .data or []
-    )
-    if not assigned:
-        return {"classes": []}
-
-    class_ids = [a["class_id"] for a in assigned]
-    is_primary_map = {a["class_id"]: a["is_primary"] for a in assigned}
-
-    res = (
-        sb.supabase.table("classes")
-        .select("*")
-        .in_("id", class_ids)
-        .eq("institute_id", institute_id)
-        .eq("is_active", True)
-        .order("name")
-        .execute()
-        .data or []
-    )
-    enriched = [{**c, "is_primary_teacher": is_primary_map.get(c["id"], False)} for c in res]
-    return {"classes": enriched}
-
-
-@router.get("/me/student")
-async def my_student_classes(
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    if not institute_id:
-        return {"classes": []}
-    sb = _sb()
-
-    enrolled = (
-        sb.supabase.table("class_students")
-        .select("class_id")
-        .eq("student_id", user_id)
-        .eq("status", "active")
-        .execute()
-        .data or []
-    )
-    if not enrolled:
-        return {"classes": []}
-
-    class_ids = [e["class_id"] for e in enrolled]
-    res = (
-        sb.supabase.table("classes")
-        .select("*")
-        .in_("id", class_ids)
-        .eq("institute_id", institute_id)
-        .eq("is_active", True)
-        .order("name")
-        .execute()
-        .data or []
-    )
-    return {"classes": res}
-
-@router.get("/school/students")
-async def list_school_students(
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    """List all students in the caller's school. Used by admin to add students to a class."""
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    if not institute_id:
-        return {"students": []}
-    sb = _sb()
-    members = (
-        sb.supabase.table("institute_memberships")
-        .select("profile_id")
-        .eq("institute_id", institute_id)
-        .eq("role", "student")
-        .execute()
-        .data or []
-    )
-    student_ids = [m["profile_id"] for m in members]
-    if not student_ids:
-        return {"students": []}
-    profiles = (
-        sb.supabase.table("profiles")
-        .select("id, full_name, display_name, email, user_type")
-        .in_("id", student_ids)
-        .order("full_name")
-        .execute()
-        .data or []
-    )
-    return {"students": profiles}
-
-
-
-@router.get("/{class_id}")
-async def get_class(
-    class_id: str,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    sb = _sb()
-
-    cls_res = (
-        sb.supabase.table("classes")
-        .select("*")
-        .eq("id", class_id)
-        .eq("institute_id", institute_id)
-        .single()
-        .execute()
-    )
-    if not cls_res.data:
-        raise HTTPException(status_code=404, detail="Class not found")
-
-    teachers = (
-        sb.supabase.table("class_teachers")
-        .select("teacher_id, is_primary, can_edit, assigned_at")
-        .eq("class_id", class_id)
-        .execute()
-        .data or []
-    )
-    students = (
-        sb.supabase.table("class_students")
-        .select("student_id, status, enrolled_at")
-        .eq("class_id", class_id)
-        .eq("status", "active")
-        .execute()
-        .data or []
-    )
-
-    # Enrich with profile data
-    all_ids = [t["teacher_id"] for t in teachers] + [s["student_id"] for s in students]
-    profile_map: Dict[str, Dict] = {}
-    if all_ids:
-        profiles = (
-            sb.supabase.table("profiles")
-            .select("id, full_name, display_name, email, user_type")
-            .in_("id", list(set(all_ids)))
-            .execute()
-            .data or []
-        )
-        profile_map = {p["id"]: p for p in profiles}
-
-    for t in teachers:
-        t["profile"] = profile_map.get(t["teacher_id"], {})
-    for s in students:
-        s["profile"] = profile_map.get(s["student_id"], {})
-
-    # Enrollment requests (pending)
-    reqs = (
-        sb.supabase.table("enrollment_requests")
-        .select("id, student_id, status, requested_at")
-        .eq("class_id", class_id)
-        .eq("status", "pending")
-        .execute()
-        .data or []
-    )
-    req_student_ids = [r["student_id"] for r in reqs if r.get("student_id")]
-    req_profiles: Dict[str, Dict] = {}
-    if req_student_ids:
-        rp = (
-            sb.supabase.table("profiles")
-            .select("id, full_name, email")
-            .in_("id", req_student_ids)
-            .execute()
-            .data or []
-        )
-        req_profiles = {p["id"]: p for p in rp}
-    for r in reqs:
-        r["profile"] = req_profiles.get(r.get("student_id", ""), {})
-
-    return {
-        **cls_res.data,
-        "teachers": teachers,
-        "students": students,
-        "enrollment_requests": reqs,
-        "student_count": len(students),
-    }
-
-
-@router.post("")
-async def create_class(
-    body: CreateClassRequest,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    sb = _sb()
-
-    row = {
-        "institute_id": institute_id,
-        "name": body.name,
-        "created_by": user_id,
-    }
-    for field in ("class_code", "subject", "key_stage", "year_group", "academic_year", "description"):
-        val = getattr(body, field)
-        if val is not None:
-            row[field] = val
-
-    res = sb.supabase.table("classes").insert(row).execute()
-    new_class = (res.data or [{}])[0]
-    class_id = new_class.get("id")
-
-    # Auto-assign creator as primary teacher if they have teacher role
-    if class_id:
-        try:
-            mem = (
-                sb.supabase.table("institute_memberships")
-                .select("role")
-                .eq("profile_id", user_id)
-                .eq("institute_id", institute_id)
-                .single()
-                .execute()
-            )
-            if (mem.data or {}).get("role") in ("teacher", "department_head"):
-                sb.supabase.table("class_teachers").insert({
-                    "class_id": class_id,
-                    "teacher_id": user_id,
-                    "is_primary": True,
-                    "assigned_by": user_id,
-                }).execute()
-        except Exception:
-            pass
-
-    logger.info(f"Class created: {class_id} '{body.name}' by {user_id}")
-    return new_class
-
-
-@router.patch("/{class_id}")
-async def update_class(
-    class_id: str,
-    body: UpdateClassRequest,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    sb = _sb()
-
-    # Must be school admin OR primary teacher of this class
-    is_admin = _is_school_admin(user_id, institute_id)
-    if not is_admin:
-        ct = (
-            sb.supabase.table("class_teachers")
-            .select("is_primary")
-            .eq("class_id", class_id)
-            .eq("teacher_id", user_id)
-            .single()
-            .execute()
-        )
-        if not (ct.data or {}).get("is_primary"):
-            raise HTTPException(status_code=403, detail="Only school admins or the primary teacher can update this class")
-
-    updates = {k: v for k, v in body.dict().items() if v is not None}
-    if not updates:
-        raise HTTPException(status_code=400, detail="No fields to update")
-    updates["updated_at"] = datetime.utcnow().isoformat()
-
-    res = (
-        sb.supabase.table("classes")
-        .update(updates)
-        .eq("id", class_id)
-        .eq("institute_id", institute_id)
-        .execute()
-    )
-    return (res.data or [{}])[0]
-
-
-@router.delete("/{class_id}")
-async def delete_class(
-    class_id: str,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-
-    if not _is_school_admin(user_id, institute_id):
-        raise HTTPException(status_code=403, detail="Only school admins can delete classes")
-
-    sb = _sb()
-    sb.supabase.table("classes").update({"is_active": False}).eq("id", class_id).eq("institute_id", institute_id).execute()
-    return {"status": "ok"}
-
-
-@router.post("/{class_id}/teachers")
-async def add_teacher(
-    class_id: str,
-    body: AddTeacherRequest,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-
-    if not _is_school_admin(user_id, institute_id):
-        raise HTTPException(status_code=403, detail="Only school admins can assign teachers")
-
-    sb = _sb()
-    res = sb.supabase.table("class_teachers").upsert({
-        "class_id": class_id,
-        "teacher_id": body.teacher_id,
-        "is_primary": body.is_primary,
-        "assigned_by": user_id,
-    }, on_conflict="class_id,teacher_id").execute()
-    return {"status": "ok", "row": (res.data or [{}])[0]}
-
-
-@router.delete("/{class_id}/teachers/{teacher_id}")
-async def remove_teacher(
-    class_id: str,
-    teacher_id: str,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-
-    if not _is_school_admin(user_id, institute_id):
-        raise HTTPException(status_code=403, detail="Only school admins can remove teachers")
-
-    sb = _sb()
-    sb.supabase.table("class_teachers").delete().eq("class_id", class_id).eq("teacher_id", teacher_id).execute()
-    return {"status": "ok"}
-
-
-
-@router.post("/{class_id}/students")
-async def add_student(
-    class_id: str,
-    body: AddStudentRequest,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-
-    if not _is_school_admin(user_id, institute_id):
-        raise HTTPException(status_code=403, detail="Only school admins can enroll students")
-
-    sb = _sb()
-    res = sb.supabase.table("class_students").upsert({
-        "class_id": class_id,
-        "student_id": body.student_id,
-        "status": "active",
-        "enrolled_by": user_id,
-    }, on_conflict="class_id,student_id").execute()
-    return {"status": "ok", "row": (res.data or [{}])[0]}
-
-
-@router.delete("/{class_id}/students/{student_id}")
-async def remove_student(
-    class_id: str,
-    student_id: str,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-
-    if not _is_school_admin(user_id, institute_id):
-        raise HTTPException(status_code=403, detail="Only school admins can remove students")
-
-    sb = _sb()
-    sb.supabase.table("class_students").delete().eq("class_id", class_id).eq("student_id", student_id).execute()
-    return {"status": "ok"}
-
-
-@router.post("/{class_id}/leave")
-async def leave_class(
-    class_id: str,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    sb = _sb()
-    sb.supabase.table("class_students").update({"status": "inactive"}).eq("class_id", class_id).eq("student_id", user_id).execute()
-    return {"status": "ok"}
-
-
-@router.get("/{class_id}/enrollment-requests")
-async def list_enrollment_requests(
-    class_id: str,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-
-    sb = _sb()
-    reqs = (
-        sb.supabase.table("enrollment_requests")
-        .select("*")
-        .eq("class_id", class_id)
-        .eq("status", "pending")
-        .execute()
-        .data or []
-    )
-    return {"requests": reqs}
-
-
-@router.post("/{class_id}/enroll")
-async def request_enrollment(
-    class_id: str,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    sb = _sb()
-    res = sb.supabase.table("enrollment_requests").insert({
-        "class_id": class_id,
-        "student_id": user_id,
-        "status": "pending",
-    }).execute()
-    return {"status": "ok", "request": (res.data or [{}])[0]}
-
-
-@router.patch("/{class_id}/enrollment-requests/{request_id}")
-async def respond_enrollment_request(
-    class_id: str,
-    request_id: str,
-    body: dict,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    """Approve or reject a pending enrollment request. School admin only."""
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    if not _is_school_admin(user_id, institute_id):
-        raise HTTPException(status_code=403, detail="Only school admins can respond to enrollment requests")
-
-    action = body.get("action")
-    if action not in ("approve", "reject"):
-        raise HTTPException(status_code=400, detail="action must be 'approve' or 'reject'")
-
-    sb = _sb()
-    req = (
-        sb.supabase.table("enrollment_requests")
-        .select("student_id, status")
-        .eq("id", request_id)
-        .eq("class_id", class_id)
-        .single()
-        .execute()
-    )
-    if not req.data:
-        raise HTTPException(status_code=404, detail="Request not found")
-    if req.data["status"] != "pending":
-        raise HTTPException(status_code=400, detail="Request is not pending")
-
-    student_id = req.data["student_id"]
-    new_status = "accepted" if action == "approve" else "rejected"
-    sb.supabase.table("enrollment_requests").update({"status": new_status}).eq("id", request_id).execute()
-
-    if action == "approve":
-        sb.supabase.table("class_students").upsert({
-            "class_id": class_id,
-            "student_id": student_id,
-            "status": "active",
-            "enrolled_by": user_id,
-        }, on_conflict="class_id,student_id").execute()
-
-    return {"status": "ok", "action": action, "student_id": student_id}

routers/database/tools/graph_tree_router.py

@@ -1,902 +0,0 @@
-import os
-from typing import Dict, Any, List, Optional, Tuple
-from fastapi import APIRouter, Depends, HTTPException
-from modules.logger_tool import initialise_logger
-from modules.auth.supabase_bearer import SupabaseBearer
-import modules.database.tools.neo4j_driver_tools as driver_tools
-from modules.database.supabase.utils.client import SupabaseServiceRoleClient
-
-logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
-router = APIRouter()
-
-
-# ─── DB helpers ────────────────────────────────────────────────────────────────
-
-def _user_to_teacher_db(user_id: str) -> str:
-    return f"cc.users.teacher.{user_id.replace('-', '')}"
-
-
-def _sb() -> SupabaseServiceRoleClient:
-    return SupabaseServiceRoleClient()
-
-
-def _find_teacher_uuid(db: str, user_email: str) -> Optional[str]:
-    """Query teacher UUID from a known Neo4j institute DB."""
-    try:
-        with driver_tools.get_session(database=db) as session:
-            rec = session.run(
-                'MATCH (t:Teacher) WHERE t.worker_email = $email '
-                'RETURN t.uuid_string AS uuid LIMIT 1',
-                email=user_email,
-            ).single()
-            if rec:
-                return rec['uuid']
-    except Exception:
-        pass
-    return None
-
-
-def _resolve_institute(
-    user_id: str, user_email: str
-) -> tuple:
-    """Returns (supabase_institute_id, neo4j_institute_db, neo4j_teacher_uuid).
-    Supabase-first lookup with Neo4j email-scan fallback."""
-    try:
-        sb = _sb()
-        p = sb.supabase.table('profiles').select('school_id').eq('id', user_id).single().execute()
-        school_id = (p.data or {}).get('school_id')
-        if school_id:
-            i = sb.supabase.table('institutes').select('id,neo4j_uuid_string').eq('id', str(school_id)).single().execute()
-            inst = i.data or {}
-            neo4j_uuid = inst.get('neo4j_uuid_string')
-            if neo4j_uuid:
-                db = f'cc.institutes.{neo4j_uuid}'
-                teacher_uuid = _find_teacher_uuid(db, user_email)
-                return str(school_id), db, teacher_uuid
-    except Exception as e:
-        logger.warning(f'Supabase-first institute resolve failed: {e}')
-    # Fallback: scan Neo4j
-    db, teacher_uuid = _find_teacher_institute(user_email)
-    return None, db, teacher_uuid
-
-
-def _allowed_neo4j_dbs(user_id: str, user_email: str) -> set[str]:
-    """Return Neo4j databases this user may request via lazy graph APIs."""
-    allowed = {f"cc.users.teacher.{user_id.replace('-', '')}"} if user_id else set()
-    if user_id or user_email:
-        _, institute_db, _ = _resolve_institute(user_id, user_email)
-        if institute_db:
-            allowed.add(institute_db)
-            allowed.add(f"{institute_db}.curriculum")
-    return allowed
-
-
-def _require_allowed_neo4j_db(neo4j_db_name: str, node_type: str, section_id: str, user_id: str, user_email: str) -> None:
-    """Reject arbitrary DB traversal from /graph/node/children query params."""
-    if not neo4j_db_name:
-        raise HTTPException(status_code=400, detail="neo4j_db_name is required")
-
-    if neo4j_db_name == "classroomcopilot":
-        if node_type.startswith("Calendar") or section_id == "calendar":
-            return
-        raise HTTPException(status_code=403, detail="Requested graph database is not allowed for this node")
-
-    if neo4j_db_name in _allowed_neo4j_dbs(user_id, user_email):
-        return
-
-    raise HTTPException(status_code=403, detail="Requested graph database is outside the authenticated user's scope")
-
-
-def _find_teacher_institute(user_email: str) -> Tuple[Optional[str], Optional[str]]:
-    """Return (institute_db_name, teacher_uuid) by matching worker_email in all institute DBs."""
-    if not user_email:
-        return None, None
-    try:
-        with driver_tools.get_session(database="system") as session:
-            result = session.run(
-                "SHOW DATABASES YIELD name "
-                "WHERE name STARTS WITH 'cc.institutes.' "
-                "AND NOT name ENDS WITH '.curriculum' "
-                "RETURN name"
-            )
-            dbs = [r["name"] for r in result]
-
-        for db in dbs:
-            try:
-                with driver_tools.get_session(database=db) as session:
-                    rec = session.run(
-                        "MATCH (t:Teacher) WHERE t.worker_email = $email "
-                        "RETURN t.uuid_string AS uuid LIMIT 1",
-                        email=user_email,
-                    ).single()
-                    if rec and rec["uuid"]:
-                        return db, rec["uuid"]
-            except Exception:
-                continue
-    except Exception as e:
-        logger.warning(f"Institute lookup failed: {e}")
-    return None, None
-
-
-# ─── Node queries ───────────────────────────────────────────────────────────────
-
-def _query_user_node(teacher_db: str) -> Optional[Dict]:
-    try:
-        with driver_tools.get_session(database=teacher_db) as session:
-            rec = session.run("MATCH (u:User) RETURN u LIMIT 1").single()
-            if not rec:
-                return None
-            u = rec["u"]
-            return {
-                "neo4j_node_id": u["uuid_string"],
-                "label": u.get("user_name") or u.get("cc_username") or "My Workspace",
-                "node_type": "User",
-                "neo4j_db_name": teacher_db,
-                "is_section": False,
-                "has_children": True,
-            }
-    except Exception as e:
-        logger.warning(f"Could not query User node in {teacher_db}: {e}")
-        return None
-
-
-def _query_calendar_months(year: str) -> List[Dict]:
-    try:
-        with driver_tools.get_session(database="classroomcopilot") as session:
-            result = session.run(
-                "MATCH (y:CalendarYear {year: $year})-[:YEAR_INCLUDES_MONTH]->(m:CalendarMonth) "
-                "RETURN m ORDER BY toInteger(m.month)",
-                year=year,
-            )
-            return [
-                {
-                    "neo4j_node_id": r["m"]["uuid_string"],
-                    "label": r["m"]["month_name"],
-                    "node_type": "CalendarMonth",
-                    "neo4j_db_name": "classroomcopilot",
-                    "is_section": False,
-                    "has_children": True,
-                }
-                for r in result
-            ]
-    except Exception as e:
-        logger.error(f"Error querying calendar months for {year}: {e}")
-        return []
-
-
-def _query_month_days(month_uuid: str) -> List[Dict]:
-    try:
-        with driver_tools.get_session(database="classroomcopilot") as session:
-            result = session.run(
-                "MATCH (m:CalendarMonth {uuid_string: $mid})-[:MONTH_INCLUDES_DAY]->(d:CalendarDay) "
-                "RETURN d ORDER BY d.date",
-                mid=month_uuid,
-            )
-            return [
-                {
-                    "neo4j_node_id": r["d"]["uuid_string"],
-                    "label": f"{r['d']['day_of_week'][:3]} {r['d']['iso_day']}",
-                    "node_type": "CalendarDay",
-                    "neo4j_db_name": "classroomcopilot",
-                    "is_section": False,
-                    "has_children": False,
-                    "neo4j_props": {
-                        "date": str(r["d"].get("date", "")),
-                        "day_of_week": r["d"].get("day_of_week", ""),
-                        "iso_day": r["d"].get("iso_day", ""),
-                    },
-                }
-                for r in result
-            ]
-    except Exception as e:
-        logger.error(f"Error querying days for month {month_uuid}: {e}")
-        return []
-
-
-def _query_teacher_classes(
-    user_id: str, institute_id: str, institute_db: str, section_id: str = ""
-) -> List[Dict]:
-    """Query classes for a teacher or student from Supabase (source of truth)."""
-    try:
-        sb = _sb()
-        teacher_rows = (
-            sb.supabase.table("class_teachers")
-            .select("class_id, is_primary")
-            .eq("teacher_id", user_id)
-            .execute()
-            .data or []
-        )
-        teacher_class_ids = {r["class_id"] for r in teacher_rows}
-        student_rows = (
-            sb.supabase.table("class_students")
-            .select("class_id")
-            .eq("student_id", user_id)
-            .eq("status", "active")
-            .execute()
-            .data or []
-        )
-        student_class_ids = {r["class_id"] for r in student_rows}
-        all_ids = list(teacher_class_ids | student_class_ids)
-        if not all_ids:
-            return []
-        classes = (
-            sb.supabase.table("classes")
-            .select("id, name, class_code, subject")
-            .in_("id", all_ids)
-            .eq("institute_id", institute_id)
-            .eq("is_active", True)
-            .order("name")
-            .execute()
-            .data or []
-        )
-        result = []
-        for c in classes:
-            role = "teacher" if c["id"] in teacher_class_ids else "student"
-            label = c.get("class_code") or c.get("name") or "Class"
-            node: Dict = {
-                "neo4j_node_id": c["id"],
-                "label": label,
-                "node_type": "SubjectClass",
-                "neo4j_db_name": institute_db,
-                "is_section": False,
-                "has_children": True,
-                "neo4j_props": {
-                    "role": role,
-                    "subject": c.get("subject") or "",
-                    "name": c.get("name") or "",
-                    "class_code": c.get("class_code") or "",
-                },
-            }
-            if section_id:
-                node["section_id"] = section_id
-            result.append(node)
-        return result
-    except Exception as e:
-        logger.warning(f"Could not query classes for user {user_id}: {e}")
-        return []
-
-
-# ─── Section builders ───────────────────────────────────────────────────────────
-
-def _section(section_id: str, label: str, db: str, status: str,
-             has_children: bool = False, children: Optional[List] = None) -> Dict:
-    return {
-        "neo4j_node_id": f"section_{section_id}",
-        "label": label,
-        "node_type": "Section",
-        "section_id": section_id,
-        "neo4j_db_name": db,
-        "is_section": True,
-        "has_children": has_children,
-        "status": status,
-        **({"children": children} if children is not None else {}),
-    }
-
-
-def _build_calendar_section() -> Dict:
-    try:
-        with driver_tools.get_session(database="classroomcopilot") as session:
-            rows = session.run(
-                "MATCH (y:CalendarYear) RETURN y ORDER BY toInteger(y.year)"
-            ).data()
-        if not rows:
-            return _section("calendar", "Calendar", "classroomcopilot", "empty")
-        year_nodes = [
-            {
-                "neo4j_node_id": r["y"]["uuid_string"],
-                "label": r["y"].get("year") or r["y"]["uuid_string"],
-                "node_type": "CalendarYear",
-                "neo4j_db_name": "classroomcopilot",
-                "is_section": False,
-                "has_children": True,
-            }
-            for r in rows
-        ]
-        return _section(
-            "calendar", "Calendar", "classroomcopilot", "populated",
-            has_children=True, children=year_nodes,
-        )
-    except Exception as e:
-        logger.warning(f"Calendar section build failed: {e}")
-        return _section("calendar", "Calendar", "classroomcopilot", "empty")
-
-
-def _build_timetable_section(user_id: str, institute_id: Optional[str], institute_db: Optional[str], teacher_uuid: Optional[str]) -> Dict:
-    if not institute_db or not teacher_uuid or not institute_id:
-        return _section("timetable", "My Timetable", "", "no_school")
-
-    try:
-        with driver_tools.get_session(database=institute_db) as session:
-            rec = session.run(
-                "MATCH (t:Teacher {uuid_string: $uuid})-[:HAS_TIMETABLE]->(tt) "
-                "RETURN tt LIMIT 1",
-                uuid=teacher_uuid,
-            ).single()
-            if rec:
-                tt = rec["tt"]
-                tt_uuid = tt["uuid_string"]
-                # Load classes from Supabase (source of truth)
-                classes = _query_teacher_classes(user_id, institute_id, institute_db, section_id="timetable")
-                return {
-                    **_section("timetable", "My Timetable", institute_db, "populated",
-                               has_children=True, children=classes if classes else None),
-                    "neo4j_node_id": tt_uuid,
-                    "node_type": "TeacherTimetable",
-                    "is_section": True,
-                }
-    except Exception as e:
-        logger.warning(f"Timetable query failed: {e}")
-
-    return _section("timetable", "My Timetable", institute_db, "empty")
-
-
-def _build_classes_section(user_id: str, institute_id: Optional[str], institute_db: Optional[str]) -> Dict:
-    if not institute_db or not institute_id:
-        return _section("classes", "My Classes", "", "no_school")
-
-    classes = _query_teacher_classes(user_id, institute_id, institute_db, section_id="classes")
-    if classes:
-        return _section("classes", "My Classes", institute_db, "populated",
-                        has_children=True, children=classes)
-    return _section("classes", "My Classes", institute_db, "empty")
-
-
-def _build_curriculum_section(institute_db: Optional[str]) -> Dict:
-    if not institute_db:
-        return _section("curriculum", "Curriculum", "", "no_school")
-
-    # Check for curriculum DB
-    curriculum_db = f"{institute_db}.curriculum"
-    try:
-        with driver_tools.get_session(database="system") as session:
-            rec = session.run(
-                "SHOW DATABASES YIELD name WHERE name = $db RETURN name",
-                db=curriculum_db,
-            ).single()
-            if not rec:
-                return _section("curriculum", "Curriculum", institute_db, "empty")
-
-        with driver_tools.get_session(database=curriculum_db) as session:
-            rec = session.run("MATCH (n) RETURN count(n) AS cnt").single()
-            cnt = rec["cnt"] if rec else 0
-            if cnt > 0:
-                return _section("curriculum", "Curriculum", curriculum_db, "populated",
-                                has_children=True)
-    except Exception as e:
-        logger.warning(f"Curriculum check failed: {e}")
-
-    return _section("curriculum", "Curriculum", institute_db, "empty")
-
-
-def _build_journal_section(teacher_db: str) -> Dict:
-    try:
-        with driver_tools.get_session(database=teacher_db) as session:
-            rec = session.run("MATCH (j:Journal) RETURN j LIMIT 1").single()
-            if rec:
-                j = rec["j"]
-                return {
-                    **_section("journal", "Journal", teacher_db, "populated", has_children=True),
-                    "neo4j_node_id": j["uuid_string"],
-                    "node_type": "Journal",
-                    "is_section": True,
-                }
-    except Exception as e:
-        logger.warning(f"Journal query failed: {e}")
-
-    return _section("journal", "Journal", teacher_db, "not_initialized")
-
-
-def _build_planner_section(teacher_db: str) -> Dict:
-    try:
-        with driver_tools.get_session(database=teacher_db) as session:
-            rec = session.run("MATCH (p:Planner) RETURN p LIMIT 1").single()
-            if rec:
-                p = rec["p"]
-                return {
-                    **_section("planner", "Planner", teacher_db, "populated", has_children=True),
-                    "neo4j_node_id": p["uuid_string"],
-                    "node_type": "Planner",
-                    "is_section": True,
-                }
-    except Exception as e:
-        logger.warning(f"Planner query failed: {e}")
-
-    return _section("planner", "Planner", teacher_db, "not_initialized")
-
-
-def _build_school_section(institute_db: str) -> Dict:
-    try:
-        with driver_tools.get_session(database=institute_db) as session:
-            rec = session.run("MATCH (s:School) RETURN s LIMIT 1").single()
-            if not rec:
-                return _section("school", "My School", institute_db, "empty")
-            s = rec["s"]
-            name = s.get("name") or "My School"
-            # Academic years under this school
-            ay_rows = session.run(
-                "MATCH (ay:AcademicYear) RETURN ay ORDER BY ay.year"
-            ).data()
-            children = [
-                {
-                    "neo4j_node_id": row["ay"]["uuid_string"],
-                    "label": row["ay"].get("year") or "Academic Year",
-                    "node_type": "AcademicYear",
-                    "neo4j_db_name": institute_db,
-                    "is_section": False,
-                    "has_children": True,
-                }
-                for row in ay_rows
-            ]
-            return {
-                **_section("school", name, institute_db, "populated",
-                           has_children=len(children) > 0,
-                           children=children if children else None),
-                "neo4j_node_id": s["uuid_string"],
-                "node_type": "School",
-                "is_section": True,
-            }
-    except Exception as e:
-        logger.warning(f"School query failed: {e}")
-
-    return _section("school", "My School", institute_db, "empty")
-
-
-# ─── Lazy children for expanded nodes ──────────────────────────────────────────
-
-def _get_children_for_node(
-    neo4j_node_id: str,
-    neo4j_db_name: str,
-    node_type: str,
-    section_id: str = "",
-    user_email: str = "",
-) -> List[Dict]:
-    # Calendar
-    if node_type == "CalendarYear":
-        return _query_calendar_months(neo4j_node_id)
-
-    if node_type == "CalendarMonth":
-        return _query_month_days(neo4j_node_id)
-
-    # TeacherTimetable lazy-load (fallback if not pre-loaded, or for By-Term view)
-    if node_type == "TeacherTimetable" and neo4j_db_name:
-        if section_id in ("", "timetable"):
-            # Classes are pre-loaded in _build_timetable_section via Supabase.
-            # Lazy expansion here is not needed in normal flow.
-            return []
-        if section_id == "timetable-term":
-            try:
-                with driver_tools.get_session(database=neo4j_db_name) as session:
-                    result = session.run(
-                        "MATCH (tt:TeacherTimetable {uuid_string: $id}) "
-                        "-[:ACADEMIC_TIMETABLE_HAS_ACADEMIC_YEAR]->(ay:AcademicYear) "
-                        "-[:ACADEMIC_YEAR_HAS_ACADEMIC_TERM]->(t:AcademicTerm) "
-                        "RETURN t ORDER BY toInteger(t.term_number)",
-                        id=neo4j_node_id,
-                    )
-                    return [
-                        {
-                            "neo4j_node_id": r["t"]["uuid_string"],
-                            "label": r["t"].get("term_name") or "Term {}".format(r["t"].get("term_number", "")),
-                            "node_type": "AcademicTerm",
-                            "neo4j_db_name": neo4j_db_name,
-                            "section_id": "timetable-term",
-                            "is_section": False,
-                            "has_children": True,
-                        }
-                        for r in result
-                    ]
-            except Exception as e:
-                logger.warning(f"TeacherTimetable term children failed: {e}")
-                return []
-
-    # SubjectClass — expand to taught lessons (timetable context) or members (classes context)
-    if node_type == "SubjectClass":
-        class_id = neo4j_node_id  # Supabase class UUID
-        try:
-            sb = _sb()
-            if section_id == "timetable" and user_email:
-                # Resolve teacher profile_id from email
-                prof = sb.supabase.table("profiles").select("id").eq("email", user_email).single().execute()
-                teacher_id = (prof.data or {}).get("id")
-                if teacher_id:
-                    lessons = (
-                        sb.supabase.table("taught_lessons")
-                        .select("id, date, period_code, class_name, subject")
-                        .eq("class_id", class_id)
-                        .eq("teacher_profile_id", teacher_id)
-                        .order("date")
-                        .order("period_code")
-                        .execute()
-                        .data or []
-                    )
-                    return [
-                        {
-                            "neo4j_node_id": tl["id"],
-                            "label": "{} {}".format(
-                                tl.get("period_code") or "",
-                                tl.get("date") or ""
-                            ).strip(),
-                            "node_type": "TaughtLesson",
-                            "neo4j_db_name": neo4j_db_name,
-                            "is_section": False,
-                            "has_children": False,
-                            "section_id": "timetable",
-                        }
-                        for tl in lessons
-                    ]
-                return []
-            if section_id == "classes":
-                # Class members: students enrolled in this class
-                members = (
-                    sb.supabase.table("class_students")
-                    .select("student_id, status, enrolled_at")
-                    .eq("class_id", class_id)
-                    .order("enrolled_at")
-                    .execute()
-                    .data or []
-                )
-                if not members:
-                    return []
-                student_ids = [m["student_id"] for m in members]
-                status_map = {m["student_id"]: m["status"] for m in members}
-                profiles = (
-                    sb.supabase.table("profiles")
-                    .select("id, email, first_name, last_name, user_type")
-                    .in_("id", student_ids)
-                    .order("last_name")
-                    .execute()
-                    .data or []
-                )
-                return [
-                    {
-                        "neo4j_node_id": p["id"],
-                        "label": "{} {} ({})".format(
-                            p.get("first_name") or "",
-                            p.get("last_name") or "",
-                            status_map.get(p["id"], ""),
-                        ).strip(),
-                        "node_type": "Student",
-                        "neo4j_db_name": neo4j_db_name,
-                        "is_section": False,
-                        "has_children": False,
-                        "section_id": "classes",
-                    }
-                    for p in profiles
-                ]
-        except Exception as e:
-            logger.warning(f"SubjectClass children failed: {e}")
-        return []
-
-    # Section containers that need lazy loading
-    if node_type == "Section":
-        if section_id == "timetable" and neo4j_db_name:
-            # Children of timetable section = classes
-            try:
-                with driver_tools.get_session(database=neo4j_db_name) as session:
-                    result = session.run(
-                        "MATCH (tt)-[:TIMETABLE_HAS_CLASS]->(c:SubjectClass) "
-                        "WHERE tt.uuid_string = $id "
-                        "RETURN c ORDER BY c.name",
-                        id=neo4j_node_id,
-                    )
-                    return [
-                        {
-                            "neo4j_node_id": r["c"]["uuid_string"],
-                            "label": r["c"].get("name") or "Class",
-                            "node_type": "SubjectClass",
-                            "neo4j_db_name": neo4j_db_name,
-                            "is_section": False,
-                            "has_children": True,
-                        }
-                        for r in result
-                    ]
-            except Exception as e:
-                logger.warning(f"Timetable children query failed: {e}")
-                return []
-
-        if section_id == "curriculum" and neo4j_db_name:
-            try:
-                with driver_tools.get_session(database=neo4j_db_name) as session:
-                    result = session.run(
-                        "MATCH (s:KeyStageSyllabus) RETURN s ORDER BY s.key_stage"
-                    )
-                    rows = list(result)
-                    if not rows:
-                        result = session.run(
-                            "MATCH (s:YearGroupSyllabus) RETURN s ORDER BY s.year_group"
-                        )
-                        rows = list(result)
-                    return [
-                        {
-                            "neo4j_node_id": r[list(r.keys())[0]]["uuid_string"],
-                            "label": r[list(r.keys())[0]].get("name") or r[list(r.keys())[0]].get("key_stage") or "Syllabus",
-                            "node_type": list(r.keys())[0],
-                            "neo4j_db_name": neo4j_db_name,
-                            "is_section": False,
-                            "has_children": True,
-                        }
-                        for r in rows
-                    ]
-            except Exception as e:
-                logger.warning(f"Curriculum children query failed: {e}")
-                return []
-
-        if section_id == "school" and neo4j_db_name:
-            try:
-                with driver_tools.get_session(database=neo4j_db_name) as session:
-                    result = session.run(
-                        "MATCH (s:School {uuid_string: $id})-[:HAS_DEPARTMENT]->(d) "
-                        "RETURN d ORDER BY d.name",
-                        id=neo4j_node_id,
-                    )
-                    return [
-                        {
-                            "neo4j_node_id": r["d"]["uuid_string"],
-                            "label": r["d"].get("name") or "Department",
-                            "node_type": "Department",
-                            "neo4j_db_name": neo4j_db_name,
-                            "is_section": False,
-                            "has_children": True,
-                        }
-                        for r in result
-                    ]
-            except Exception as e:
-                logger.warning(f"School children query failed: {e}")
-                return []
-
-    # AcademicYear → terms
-    if node_type == "AcademicYear" and neo4j_db_name:
-        try:
-            with driver_tools.get_session(database=neo4j_db_name) as session:
-                result = session.run(
-                    "MATCH (ay:AcademicYear {uuid_string: $id})"
-                    "-[:ACADEMIC_YEAR_HAS_ACADEMIC_TERM]->(t:AcademicTerm) "
-                    "RETURN t ORDER BY toInteger(t.term_number)",
-                    id=neo4j_node_id,
-                )
-                return [
-                    {
-                        "neo4j_node_id": r["t"]["uuid_string"],
-                        "label": r["t"]["term_name"],
-                        "node_type": "AcademicTerm",
-                        "neo4j_db_name": neo4j_db_name,
-                        "is_section": False,
-                        "has_children": True,
-                    }
-                    for r in result
-                ]
-        except Exception as e:
-            logger.warning(f"AcademicYear children failed: {e}")
-        return []
-
-    # AcademicWeek → days (or TaughtLessons in timetable-term context)
-    if node_type == "AcademicWeek" and neo4j_db_name:
-        if section_id == "timetable-term" and user_email:
-            # Supabase: get week date range from Neo4j, then query taught_lessons
-            try:
-                week_start = None
-                with driver_tools.get_session(database=neo4j_db_name) as session:
-                    rec = session.run(
-                        "MATCH (w:AcademicWeek {uuid_string: $id}) "
-                        "RETURN w.start_date AS start_date",
-                        id=neo4j_node_id,
-                    ).single()
-                    if rec:
-                        week_start = rec["start_date"]
-                if week_start:
-                    from datetime import datetime, timedelta
-                    start_dt = datetime.strptime(str(week_start)[:10], "%Y-%m-%d")
-                    end_dt = start_dt + timedelta(days=6)
-                    sb = _sb()
-                    prof = sb.supabase.table("profiles").select("id").eq("email", user_email).single().execute()
-                    teacher_id = (prof.data or {}).get("id")
-                    if teacher_id:
-                        lessons = (
-                            sb.supabase.table("taught_lessons")
-                            .select("id, date, period_code, class_name, subject")
-                            .eq("teacher_profile_id", teacher_id)
-                            .gte("date", start_dt.strftime("%Y-%m-%d"))
-                            .lte("date", end_dt.strftime("%Y-%m-%d"))
-                            .order("date")
-                            .order("period_code")
-                            .execute()
-                            .data or []
-                        )
-                        return [
-                            {
-                                "neo4j_node_id": tl["id"],
-                                "label": "{} — {}".format(
-                                    tl.get("period_code") or "",
-                                    tl.get("class_name") or tl.get("subject") or "Lesson"
-                                ),
-                                "node_type": "TaughtLesson",
-                                "neo4j_db_name": neo4j_db_name,
-                                "is_section": False,
-                                "has_children": False,
-                            }
-                            for tl in lessons
-                        ]
-            except Exception as e:
-                logger.warning(f"AcademicWeek timetable-term Supabase lessons failed: {e}")
-            return []
-        try:
-            with driver_tools.get_session(database=neo4j_db_name) as session:
-                result = session.run(
-                    "MATCH (w:AcademicWeek {uuid_string: $id}) "
-                    "-[:ACADEMIC_WEEK_HAS_ACADEMIC_DAY]->(d:AcademicDay) "
-                    "RETURN d ORDER BY d.date",
-                    id=neo4j_node_id,
-                )
-                return [
-                    {
-                        "neo4j_node_id": r["d"]["uuid_string"],
-                        "label": r["d"].get("date", ""),
-                        "node_type": "AcademicDay",
-                        "neo4j_db_name": neo4j_db_name,
-                        "is_section": False,
-                        "has_children": False,
-                    }
-                    for r in result
-                ]
-        except Exception as e:
-            logger.warning(f"AcademicWeek children failed: {e}")
-        return []
-
-        # AcademicTerm → weeks
-    if node_type == "AcademicTerm" and neo4j_db_name:
-        try:
-            with driver_tools.get_session(database=neo4j_db_name) as session:
-                result = session.run(
-                    "MATCH (t:AcademicTerm {uuid_string: $id})"
-                    "-[:ACADEMIC_TERM_HAS_ACADEMIC_WEEK]->(w:AcademicWeek) "
-                    "RETURN w ORDER BY toInteger(w.academic_week_number)",
-                    id=neo4j_node_id,
-                )
-                return [
-                    {
-                        "neo4j_node_id": r["w"]["uuid_string"],
-                        "label": "Week {}".format(r["w"].get("academic_week_number", r["w"].get("week_number", "?"))),
-                        "node_type": "AcademicWeek",
-                        "neo4j_db_name": neo4j_db_name,
-                        "section_id": section_id if section_id == "timetable-term" else "",
-                        "is_section": False,
-                        "has_children": True,
-                    }
-                    for r in result
-                ]
-        except Exception as e:
-            logger.warning(f"AcademicTerm children failed: {e}")
-        return []
-
-    # SubjectClass → lessons
-    if node_type == "SubjectClass" and neo4j_db_name:
-        try:
-            with driver_tools.get_session(database=neo4j_db_name) as session:
-                result = session.run(
-                    "MATCH (c:SubjectClass {uuid_string: $id})-[:CLASS_HAS_LESSON]->(l) "
-                    "RETURN l ORDER BY l.date, l.start_time LIMIT 50",
-                    id=neo4j_node_id,
-                )
-                return [
-                    {
-                        "neo4j_node_id": r["l"]["uuid_string"],
-                        "label": r["l"].get("title") or r["l"].get("period_code") or "Lesson",
-                        "node_type": "TimetableLesson",
-                        "neo4j_db_name": neo4j_db_name,
-                        "is_section": False,
-                        "has_children": False,
-                    }
-                    for r in result
-                ]
-        except Exception as e:
-            logger.warning(f"Class lessons query failed: {e}")
-
-    return []
-
-
-# ─── Endpoints ──────────────────────────────────────────────────────────────────
-
-@router.get("/tree")
-async def get_teacher_graph_tree(
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    user_email = credentials.get("email", "")
-    if not user_id:
-        raise HTTPException(status_code=403, detail="Could not extract user_id from token")
-
-    teacher_db = _user_to_teacher_db(user_id)
-
-    user_node = _query_user_node(teacher_db) or {
-        "neo4j_node_id": user_id,
-        "label": "My Workspace",
-        "node_type": "User",
-        "neo4j_db_name": teacher_db,
-        "is_section": False,
-        "has_children": True,
-    }
-
-    supabase_institute_id, institute_db, teacher_node_uuid = _resolve_institute(user_id, user_email)
-
-    sections = [
-        _build_calendar_section(),
-        _build_timetable_section(user_id, supabase_institute_id, institute_db, teacher_node_uuid),
-        _build_classes_section(user_id, supabase_institute_id, institute_db),
-        _build_curriculum_section(institute_db),
-        _build_journal_section(teacher_db),
-        _build_planner_section(teacher_db),
-    ]
-
-    if institute_db:
-        sections.append(_build_school_section(institute_db))
-
-    return {
-        "status": "success",
-        "tree": {**user_node, "children": sections},
-        "meta": {
-            "institute_db": institute_db,
-            "teacher_linked": institute_db is not None,
-        },
-    }
-
-
-@router.get("/node/children")
-async def get_node_children(
-    neo4j_node_id: str,
-    neo4j_db_name: str,
-    node_type: str,
-    section_id: str = "",
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    user_email = credentials.get("email", "")
-    if not user_id:
-        raise HTTPException(status_code=403, detail="Could not extract user_id from token")
-    _require_allowed_neo4j_db(neo4j_db_name, node_type, section_id, user_id, user_email)
-    children = _get_children_for_node(neo4j_node_id, neo4j_db_name, node_type, section_id, user_email)
-    return {"status": "success", "children": children}
-
-
-@router.get("/calendar/academic")
-async def get_academic_calendar(credentials: dict = Depends(SupabaseBearer())) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    user_email = credentials.get("email", "")
-    _, institute_db, _ = _resolve_institute(user_id, user_email)
-    if not institute_db:
-        return {"status": "no_school", "terms": []}
-    try:
-        with driver_tools.get_session(database=institute_db) as s:
-            if not s.run("MATCH (ay:AcademicYear) RETURN ay LIMIT 1").single():
-                return {"status": "empty", "terms": [], "institute_db": institute_db}
-            terms = []
-            for tr in s.run(
-                "MATCH (ay:AcademicYear)-[:ACADEMIC_YEAR_HAS_ACADEMIC_TERM]->(t:AcademicTerm) "
-                "RETURN t ORDER BY toInteger(t.term_number)"
-            ):
-                t = tr["t"]
-                with driver_tools.get_session(database=institute_db) as s2:
-                    weeks = [{
-                        "neo4j_node_id": w["w"]["uuid_string"],
-                        "label": f"Week {w['w']['academic_week_number']}",
-                        "node_type": "AcademicWeek",
-                        "neo4j_db_name": institute_db,
-                        "is_section": False,
-                        "has_children": True,
-                    } for w in s2.run(
-                        "MATCH (t:AcademicTerm {uuid_string: $tid})-[:ACADEMIC_TERM_HAS_ACADEMIC_WEEK]->(w:AcademicWeek) "
-                        "RETURN w ORDER BY toInteger(w.academic_week_number)",
-                        tid=t["uuid_string"]
-                    )]
-                terms.append({
-                    "neo4j_node_id": t["uuid_string"],
-                    "label": t["term_name"],
-                    "node_type": "AcademicTerm",
-                    "neo4j_db_name": institute_db,
-                    "is_section": False,
-                    "has_children": len(weeks) > 0,
-                    "children": weeks,
-                })
-            return {"status": "populated", "terms": terms, "institute_db": institute_db}
-    except Exception as e:
-        logger.error(f"Academic calendar query failed: {e}")
-        return {"status": "error", "terms": []}

routers/database/tools/invitations_router.py

@@ -1,372 +0,0 @@
-"""
-Invitations & People Router.
-
-POST   /users/invite               — school admin sends magic-link invitation
-GET    /users/invitations          — list invitations for the school
-DELETE /users/invitations/{id}     — cancel a pending invitation
-POST   /users/invitations/{id}/resend — re-send magic link
-GET    /users/staff                — list current staff members
-GET    /users/students             — list current student members
-"""
-import os
-from datetime import datetime, timezone, timedelta
-from typing import Any, Dict, List, Optional
-from fastapi import APIRouter, Depends, HTTPException, Query
-from pydantic import BaseModel, EmailStr
-from modules.logger_tool import initialise_logger
-from modules.auth.supabase_bearer import SupabaseBearer
-from modules.database.supabase.utils.client import SupabaseServiceRoleClient
-
-logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
-router = APIRouter()
-
-VALID_ROLES = {"teacher", "student", "school_admin", "department_head"}
-STAFF_ROLES = {"teacher", "school_admin", "department_head"}
-
-
-def _sb() -> SupabaseServiceRoleClient:
-    return SupabaseServiceRoleClient()
-
-
-def _resolve_institute_id(user_id: str) -> Optional[str]:
-    try:
-        sb = _sb()
-        p = sb.supabase.table("profiles").select("school_id").eq("id", user_id).single().execute()
-        return str((p.data or {}).get("school_id") or "") or None
-    except Exception:
-        return None
-
-
-def _require_institute(user_id: str) -> str:
-    iid = _resolve_institute_id(user_id)
-    if not iid:
-        raise HTTPException(status_code=400, detail="User is not linked to a school")
-    return iid
-
-
-def _require_school_admin(user_id: str, institute_id: str) -> None:
-    try:
-        sb = _sb()
-        m = (
-            sb.supabase.table("institute_memberships")
-            .select("role")
-            .eq("profile_id", user_id)
-            .eq("institute_id", institute_id)
-            .single()
-            .execute()
-        )
-        role = (m.data or {}).get("role", "")
-        if role not in ("school_admin", "department_head"):
-            raise HTTPException(status_code=403, detail="School admin access required")
-    except HTTPException:
-        raise
-    except Exception:
-        raise HTTPException(status_code=403, detail="Could not verify school admin status")
-
-
-# ─── Request models ───────────────────────────────────────────────────────────
-
-class InviteRequest(BaseModel):
-    email: str
-    role: str                              # teacher | student | school_admin | department_head
-    metadata: Optional[Dict[str, Any]] = None  # year_group, subject, department, etc.
-
-
-# ─── Invite ───────────────────────────────────────────────────────────────────
-
-@router.post("/invite")
-async def invite_user(
-    body: InviteRequest,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    """
-    Create an invitation row and send a Supabase magic-link invite email.
-    Idempotent: if a pending invitation already exists for the same email/school,
-    it is returned (use /resend to refresh the magic link).
-    """
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    _require_school_admin(user_id, institute_id)
-
-    if body.role not in VALID_ROLES:
-        raise HTTPException(status_code=400, detail=f"role must be one of {sorted(VALID_ROLES)}")
-
-    email = body.email.strip().lower()
-    sb = _sb()
-
-    # Check for existing pending invitation
-    existing = (
-        sb.supabase.table("invitations")
-        .select("id,status,email,role,created_at,expires_at")
-        .eq("institute_id", institute_id)
-        .eq("email", email)
-        .eq("status", "pending")
-        .execute()
-        .data or []
-    )
-    if existing:
-        return {
-            "status": "already_pending",
-            "invitation": existing[0],
-            "message": "A pending invitation already exists. Use /resend to refresh the magic link.",
-        }
-
-    # Insert invitation row
-    expires_at = (datetime.now(timezone.utc) + timedelta(days=7)).isoformat()
-    inv_data = {
-        "institute_id": institute_id,
-        "email": email,
-        "role": body.role,
-        "invited_by": user_id,
-        "expires_at": expires_at,
-        "status": "pending",
-        "metadata": body.metadata or {},
-    }
-    inv_res = sb.supabase.table("invitations").insert(inv_data).execute()
-    if not inv_res.data:
-        raise HTTPException(status_code=500, detail="Failed to create invitation record")
-    invitation = inv_res.data[0]
-
-    # Send magic link via Supabase Auth admin
-    try:
-        sb.supabase.auth.admin.invite_user_by_email(
-            email,
-            options={
-                "data": {
-                    "invitation_id": invitation["id"],
-                    "institute_id": institute_id,
-                    "role": body.role,
-                    **(body.metadata or {}),
-                }
-            },
-        )
-        magic_link_sent = True
-    except Exception as e:
-        logger.warning(f"Magic link send failed for {email}: {e}")
-        magic_link_sent = False
-
-    logger.info(f"Invited {email} as {body.role} to school {institute_id} by {user_id}")
-    return {
-        "status": "ok",
-        "invitation": invitation,
-        "magic_link_sent": magic_link_sent,
-    }
-
-
-# ─── List invitations ─────────────────────────────────────────────────────────
-
-@router.get("/invitations")
-async def list_invitations(
-    role: Optional[str] = Query(None),
-    status: Optional[str] = Query(None, description="pending|accepted|expired|cancelled"),
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    _require_school_admin(user_id, institute_id)
-
-    sb = _sb()
-    q = (
-        sb.supabase.table("invitations")
-        .select("id,email,role,status,invited_by,expires_at,created_at,metadata")
-        .eq("institute_id", institute_id)
-        .order("created_at", desc=True)
-    )
-    if role:
-        q = q.eq("role", role)
-    if status:
-        q = q.eq("status", status)
-
-    rows = q.execute().data or []
-
-    # Mark expired rows (status still 'pending' but past expires_at)
-    now = datetime.now(timezone.utc)
-    for row in rows:
-        if row["status"] == "pending":
-            try:
-                exp = datetime.fromisoformat(row["expires_at"].replace("Z", "+00:00"))
-                if exp < now:
-                    row["status"] = "expired"
-            except Exception:
-                pass
-
-    return {"status": "ok", "invitations": rows, "total": len(rows)}
-
-
-# ─── Cancel invitation ────────────────────────────────────────────────────────
-
-@router.delete("/invitations/{invitation_id}")
-async def cancel_invitation(
-    invitation_id: str,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    _require_school_admin(user_id, institute_id)
-
-    sb = _sb()
-    res = (
-        sb.supabase.table("invitations")
-        .update({"status": "cancelled"})
-        .eq("id", invitation_id)
-        .eq("institute_id", institute_id)
-        .eq("status", "pending")
-        .execute()
-    )
-    if not res.data:
-        raise HTTPException(status_code=404, detail="Pending invitation not found")
-    return {"status": "ok", "invitation": res.data[0]}
-
-
-# ─── Resend invitation ────────────────────────────────────────────────────────
-
-@router.post("/invitations/{invitation_id}/resend")
-async def resend_invitation(
-    invitation_id: str,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    """Re-trigger the magic link for a pending (or expired) invitation."""
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    _require_school_admin(user_id, institute_id)
-
-    sb = _sb()
-    inv = (
-        sb.supabase.table("invitations")
-        .select("*")
-        .eq("id", invitation_id)
-        .eq("institute_id", institute_id)
-        .single()
-        .execute()
-    ).data
-    if not inv:
-        raise HTTPException(status_code=404, detail="Invitation not found")
-    if inv["status"] == "accepted":
-        raise HTTPException(status_code=400, detail="Invitation already accepted")
-    if inv["status"] == "cancelled":
-        raise HTTPException(status_code=400, detail="Invitation was cancelled — create a new one")
-
-    # Refresh expiry + status
-    new_expires = (datetime.now(timezone.utc) + timedelta(days=7)).isoformat()
-    sb.supabase.table("invitations").update({
-        "expires_at": new_expires,
-        "status": "pending",
-    }).eq("id", invitation_id).execute()
-
-    # Re-send magic link
-    try:
-        sb.supabase.auth.admin.invite_user_by_email(
-            inv["email"],
-            options={
-                "data": {
-                    "invitation_id": invitation_id,
-                    "institute_id": institute_id,
-                    "role": inv["role"],
-                    **inv.get("metadata", {}),
-                }
-            },
-        )
-        magic_link_sent = True
-    except Exception as e:
-        logger.warning(f"Resend magic link failed for {inv['email']}: {e}")
-        magic_link_sent = False
-
-    return {"status": "ok", "magic_link_sent": magic_link_sent, "new_expires_at": new_expires}
-
-
-# ─── Staff list ───────────────────────────────────────────────────────────────
-
-@router.get("/staff")
-async def list_staff(
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    """List all staff members (teachers, admins, department heads) in the school."""
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    _require_school_admin(user_id, institute_id)
-
-    sb = _sb()
-    members = (
-        sb.supabase.table("institute_memberships")
-        .select("profile_id,role,joined_at")
-        .eq("institute_id", institute_id)
-        .in_("role", list(STAFF_ROLES))
-        .execute()
-        .data or []
-    )
-
-    if not members:
-        return {"status": "ok", "staff": [], "total": 0}
-
-    profile_ids = [m["profile_id"] for m in members]
-    profiles = (
-        sb.supabase.table("profiles")
-        .select("id,email,username,display_name")
-        .in_("id", profile_ids)
-        .execute()
-        .data or []
-    )
-    profile_map = {p["id"]: p for p in profiles}
-
-    staff = []
-    for m in members:
-        p = profile_map.get(m["profile_id"], {})
-        staff.append({
-            "profile_id": m["profile_id"],
-            "email": p.get("email"),
-            "username": p.get("username"),
-            "display_name": p.get("display_name"),
-            "role": m["role"],
-            "joined_at": m["joined_at"],
-        })
-
-    return {"status": "ok", "staff": staff, "total": len(staff)}
-
-
-# ─── Student list ─────────────────────────────────────────────────────────────
-
-@router.get("/students")
-async def list_students(
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    """List all student members in the school."""
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    _require_school_admin(user_id, institute_id)
-
-    sb = _sb()
-    members = (
-        sb.supabase.table("institute_memberships")
-        .select("profile_id,role,joined_at")
-        .eq("institute_id", institute_id)
-        .eq("role", "student")
-        .execute()
-        .data or []
-    )
-
-    if not members:
-        return {"status": "ok", "students": [], "total": 0}
-
-    profile_ids = [m["profile_id"] for m in members]
-    profiles = (
-        sb.supabase.table("profiles")
-        .select("id,email,username,display_name")
-        .in_("id", profile_ids)
-        .execute()
-        .data or []
-    )
-    profile_map = {p["id"]: p for p in profiles}
-
-    students = []
-    for m in members:
-        p = profile_map.get(m["profile_id"], {})
-        students.append({
-            "profile_id": m["profile_id"],
-            "email": p.get("email"),
-            "username": p.get("username"),
-            "display_name": p.get("display_name"),
-            "role": m["role"],
-            "joined_at": m["joined_at"],
-        })
-
-    return {"status": "ok", "students": students, "total": len(students)}

routers/database/tools/lesson_plans_router.py

@@ -1,650 +0,0 @@
-import os
-import uuid
-import aiohttp
-from datetime import datetime
-from typing import Any, Dict, List, Optional
-from fastapi import APIRouter, Depends, HTTPException
-from pydantic import BaseModel
-from modules.logger_tool import initialise_logger
-from modules.auth.supabase_bearer import SupabaseBearer
-from modules.database.supabase.utils.client import SupabaseServiceRoleClient
-
-logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
-router = APIRouter()
-
-_OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434")
-_OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "llama3")
-_LLM_TIMEOUT = 60
-
-VALID_STATUS = {"draft", "ready", "archived"}
-VALID_FIELDS = {"objectives", "activity_description", "title"}
-BLOOM_LEVELS = {"remember", "understand", "apply", "analyse", "evaluate", "create"}
-
-
-# ─── Supabase helpers ─────────────────────────────────────────────────────────
-
-def _sb() -> SupabaseServiceRoleClient:
-    return SupabaseServiceRoleClient()
-
-
-def _resolve_institute_id(user_id: str) -> Optional[str]:
-    try:
-        sb = _sb()
-        p = sb.supabase.table("profiles").select("school_id").eq("id", user_id).single().execute()
-        return str((p.data or {}).get("school_id") or "") or None
-    except Exception:
-        return None
-
-
-def _require_institute(user_id: str) -> str:
-    iid = _resolve_institute_id(user_id)
-    if not iid:
-        raise HTTPException(status_code=400, detail="User is not linked to a school")
-    return iid
-
-
-def _is_creator(plan: Dict[str, Any], user_id: str) -> bool:
-    return str(plan.get("created_by", "")) == str(user_id)
-
-
-def _can_edit_plan(sb: SupabaseServiceRoleClient, plan_id: str, user_id: str) -> bool:
-    try:
-        plan_res = (
-            sb.supabase.table("planned_lessons")
-            .select("created_by")
-            .eq("id", plan_id)
-            .single()
-            .execute()
-        )
-        if not plan_res.data:
-            return False
-        if str(plan_res.data.get("created_by", "")) == str(user_id):
-            return True
-        collab = (
-            sb.supabase.table("lesson_collaborators")
-            .select("can_edit")
-            .eq("planned_lesson_id", plan_id)
-            .eq("profile_id", user_id)
-            .single()
-            .execute()
-        )
-        return bool((collab.data or {}).get("can_edit", False))
-    except Exception:
-        return False
-
-
-# ─── Request models ───────────────────────────────────────────────────────────
-
-class CreatePlanRequest(BaseModel):
-    title: str
-    class_id: Optional[str] = None
-    subject: Optional[str] = None
-    year_group: Optional[str] = None
-    estimated_duration_minutes: Optional[int] = None
-    objectives: Optional[List[Dict[str, Any]]] = None
-    activities: Optional[List[Dict[str, Any]]] = None
-    status: Optional[str] = "draft"
-    tags: Optional[List[str]] = None
-    topic_code: Optional[str] = None
-    whiteboard_room_id: Optional[str] = None
-    course_id: Optional[str] = None
-    sequence_number: Optional[int] = None
-
-
-class UpdatePlanRequest(BaseModel):
-    title: Optional[str] = None
-    class_id: Optional[str] = None
-    subject: Optional[str] = None
-    year_group: Optional[str] = None
-    estimated_duration_minutes: Optional[int] = None
-    objectives: Optional[List[Dict[str, Any]]] = None
-    activities: Optional[List[Dict[str, Any]]] = None
-    status: Optional[str] = None
-    tags: Optional[List[str]] = None
-    topic_code: Optional[str] = None
-    whiteboard_room_id: Optional[str] = None
-    course_id: Optional[str] = None
-    sequence_number: Optional[int] = None
-
-
-class DeliverPlanRequest(BaseModel):
-    taught_lesson_id: Optional[str] = None
-    class_id: Optional[str] = None
-    notes: Optional[str] = None
-
-
-class AddCollaboratorRequest(BaseModel):
-    profile_id: str
-    can_edit: bool = True
-
-
-class SuggestRequest(BaseModel):
-    field: str
-    context: Optional[str] = None
-    activity_section: Optional[str] = None
-    objective_texts: Optional[List[str]] = None
-
-
-# ─── Endpoints ────────────────────────────────────────────────────────────────
-
-@router.get("/plans")
-async def list_plans(
-    class_id: Optional[str] = None,
-    status: Optional[str] = None,
-    subject: Optional[str] = None,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    sb = _sb()
-
-    q = (
-        sb.supabase.table("planned_lessons")
-        .select("*")
-        .eq("institute_id", institute_id)
-    )
-    if class_id:
-        q = q.eq("class_id", class_id)
-    if status:
-        q = q.eq("status", status)
-    if subject:
-        q = q.eq("subject", subject)
-
-    owned = q.eq("created_by", user_id).execute().data or []
-
-    collab_rows = (
-        sb.supabase.table("lesson_collaborators")
-        .select("planned_lesson_id")
-        .eq("profile_id", user_id)
-        .execute()
-        .data or []
-    )
-    collab_plan_ids = [r["planned_lesson_id"] for r in collab_rows]
-
-    collab_plans: List[Dict] = []
-    if collab_plan_ids:
-        q2 = (
-            sb.supabase.table("planned_lessons")
-            .select("*")
-            .eq("institute_id", institute_id)
-            .in_("id", collab_plan_ids)
-        )
-        if class_id:
-            q2 = q2.eq("class_id", class_id)
-        if status:
-            q2 = q2.eq("status", status)
-        if subject:
-            q2 = q2.eq("subject", subject)
-        collab_plans = q2.execute().data or []
-
-    seen_ids: set = set()
-    all_plans: List[Dict] = []
-    for p in owned + collab_plans:
-        if p["id"] not in seen_ids:
-            seen_ids.add(p["id"])
-            all_plans.append(p)
-
-    if not all_plans:
-        return {"plans": []}
-
-    plan_ids = [p["id"] for p in all_plans]
-
-    collab_counts: Dict[str, int] = {}
-    delivery_counts: Dict[str, int] = {}
-
-    try:
-        cc = (
-            sb.supabase.table("lesson_collaborators")
-            .select("planned_lesson_id")
-            .in_("planned_lesson_id", plan_ids)
-            .execute()
-            .data or []
-        )
-        for r in cc:
-            collab_counts[r["planned_lesson_id"]] = collab_counts.get(r["planned_lesson_id"], 0) + 1
-    except Exception:
-        pass
-
-    try:
-        dc = (
-            sb.supabase.table("lesson_deliveries")
-            .select("planned_lesson_id")
-            .in_("planned_lesson_id", plan_ids)
-            .execute()
-            .data or []
-        )
-        for r in dc:
-            delivery_counts[r["planned_lesson_id"]] = delivery_counts.get(r["planned_lesson_id"], 0) + 1
-    except Exception:
-        pass
-
-    enriched = [
-        {
-            **p,
-            "collaborator_count": collab_counts.get(p["id"], 0),
-            "delivery_count": delivery_counts.get(p["id"], 0),
-            "is_owner": str(p.get("created_by", "")) == str(user_id),
-        }
-        for p in all_plans
-    ]
-    return {"plans": enriched}
-
-
-@router.post("/plans")
-async def create_plan(
-    body: CreatePlanRequest,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    sb = _sb()
-
-    if body.status and body.status not in VALID_STATUS:
-        raise HTTPException(status_code=400, detail=f"status must be one of {VALID_STATUS}")
-
-    row: Dict[str, Any] = {
-        "created_by": user_id,
-        "institute_id": institute_id,
-        "title": body.title,
-        "status": body.status or "draft",
-        "objectives": body.objectives or [],
-        "activities": body.activities or [],
-        "tags": body.tags or [],
-    }
-    for field in (
-        "class_id", "subject", "year_group", "estimated_duration_minutes",
-        "topic_code", "whiteboard_room_id", "course_id", "sequence_number",
-    ):
-        val = getattr(body, field)
-        if val is not None:
-            row[field] = val
-
-    res = sb.supabase.table("planned_lessons").insert(row).execute()
-    plan = (res.data or [{}])[0]
-    logger.info(f"Lesson plan created: {plan.get('id')} '{body.title}' by {user_id}")
-    return plan
-
-
-@router.get("/plans/{plan_id}")
-async def get_plan(
-    plan_id: str,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    sb = _sb()
-
-    plan_res = (
-        sb.supabase.table("planned_lessons")
-        .select("*")
-        .eq("id", plan_id)
-        .eq("institute_id", institute_id)
-        .single()
-        .execute()
-    )
-    if not plan_res.data:
-        raise HTTPException(status_code=404, detail="Lesson plan not found")
-
-    plan = plan_res.data
-    if not _is_creator(plan, user_id):
-        collab_check = (
-            sb.supabase.table("lesson_collaborators")
-            .select("can_edit")
-            .eq("planned_lesson_id", plan_id)
-            .eq("profile_id", user_id)
-            .execute()
-            .data
-        )
-        if collab_check is None:
-            raise HTTPException(status_code=403, detail="Access denied")
-
-    collab_rows = (
-        sb.supabase.table("lesson_collaborators")
-        .select("profile_id, can_edit, added_at")
-        .eq("planned_lesson_id", plan_id)
-        .execute()
-        .data or []
-    )
-
-    profile_ids = [r["profile_id"] for r in collab_rows]
-    profile_map: Dict[str, Dict] = {}
-    if profile_ids:
-        try:
-            profiles = (
-                sb.supabase.table("profiles")
-                .select("id, full_name, email")
-                .in_("id", profile_ids)
-                .execute()
-                .data or []
-            )
-            profile_map = {p["id"]: p for p in profiles}
-        except Exception:
-            pass
-
-    collaborators = [
-        {
-            **r,
-            "full_name": profile_map.get(r["profile_id"], {}).get("full_name"),
-            "email": profile_map.get(r["profile_id"], {}).get("email"),
-        }
-        for r in collab_rows
-    ]
-
-    deliveries = (
-        sb.supabase.table("lesson_deliveries")
-        .select("*")
-        .eq("planned_lesson_id", plan_id)
-        .order("started_at", desc=True)
-        .limit(10)
-        .execute()
-        .data or []
-    )
-
-    return {**plan, "collaborators": collaborators, "recent_deliveries": deliveries}
-
-
-@router.patch("/plans/{plan_id}")
-async def update_plan(
-    plan_id: str,
-    body: UpdatePlanRequest,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    sb = _sb()
-
-    if not _can_edit_plan(sb, plan_id, user_id):
-        raise HTTPException(status_code=403, detail="Access denied — not creator or editor collaborator")
-
-    if body.status and body.status not in VALID_STATUS:
-        raise HTTPException(status_code=400, detail=f"status must be one of {VALID_STATUS}")
-
-    updates: Dict[str, Any] = {}
-    for field in (
-        "title", "class_id", "subject", "year_group", "estimated_duration_minutes",
-        "objectives", "activities", "status", "tags", "topic_code",
-        "whiteboard_room_id", "course_id", "sequence_number",
-    ):
-        val = getattr(body, field)
-        if val is not None:
-            updates[field] = val
-
-    if not updates:
-        raise HTTPException(status_code=400, detail="Nothing to update")
-
-    updates["updated_at"] = datetime.utcnow().isoformat()
-    res = (
-        sb.supabase.table("planned_lessons")
-        .update(updates)
-        .eq("id", plan_id)
-        .eq("institute_id", institute_id)
-        .execute()
-    )
-    if not res.data:
-        raise HTTPException(status_code=404, detail="Plan not found")
-    return res.data[0]
-
-
-@router.delete("/plans/{plan_id}")
-async def delete_plan(
-    plan_id: str,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    sb = _sb()
-
-    plan_res = (
-        sb.supabase.table("planned_lessons")
-        .select("created_by")
-        .eq("id", plan_id)
-        .eq("institute_id", institute_id)
-        .single()
-        .execute()
-    )
-    if not plan_res.data:
-        raise HTTPException(status_code=404, detail="Plan not found")
-    if not _is_creator(plan_res.data, user_id):
-        raise HTTPException(status_code=403, detail="Only the creator can delete a plan")
-
-    sb.supabase.table("lesson_collaborators").delete().eq("planned_lesson_id", plan_id).execute()
-    sb.supabase.table("planned_lessons").delete().eq("id", plan_id).execute()
-    logger.info(f"Lesson plan deleted: {plan_id} by {user_id}")
-    return {"status": "ok"}
-
-
-@router.post("/plans/{plan_id}/deliver")
-async def deliver_plan(
-    plan_id: str,
-    body: DeliverPlanRequest,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    sb = _sb()
-
-    plan_res = (
-        sb.supabase.table("planned_lessons")
-        .select("id, whiteboard_room_id")
-        .eq("id", plan_id)
-        .eq("institute_id", institute_id)
-        .single()
-        .execute()
-    )
-    if not plan_res.data:
-        raise HTTPException(status_code=404, detail="Plan not found")
-
-    if not _can_edit_plan(sb, plan_id, user_id):
-        raise HTTPException(status_code=403, detail="Access denied")
-
-    row: Dict[str, Any] = {
-        "planned_lesson_id": plan_id,
-        "delivered_by": user_id,
-        "institute_id": institute_id,
-        "started_at": datetime.utcnow().isoformat(),
-    }
-    if body.taught_lesson_id:
-        row["taught_lesson_id"] = body.taught_lesson_id
-    if body.class_id:
-        row["class_id"] = body.class_id
-    if body.notes:
-        row["notes"] = body.notes
-
-    whiteboard_room_id = plan_res.data.get("whiteboard_room_id")
-    if whiteboard_room_id:
-        row["whiteboard_room_id"] = whiteboard_room_id
-
-    res = sb.supabase.table("lesson_deliveries").insert(row).execute()
-    delivery = (res.data or [{}])[0]
-    logger.info(f"Lesson delivery created: {delivery.get('id')} for plan {plan_id} by {user_id}")
-    return delivery
-
-
-@router.get("/plans/{plan_id}/deliveries")
-async def list_deliveries(
-    plan_id: str,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    sb = _sb()
-
-    plan_res = (
-        sb.supabase.table("planned_lessons")
-        .select("created_by")
-        .eq("id", plan_id)
-        .eq("institute_id", institute_id)
-        .single()
-        .execute()
-    )
-    if not plan_res.data:
-        raise HTTPException(status_code=404, detail="Plan not found")
-
-    deliveries = (
-        sb.supabase.table("lesson_deliveries")
-        .select("*")
-        .eq("planned_lesson_id", plan_id)
-        .order("started_at", desc=True)
-        .execute()
-        .data or []
-    )
-    return {"deliveries": deliveries}
-
-
-@router.post("/plans/{plan_id}/collaborators")
-async def add_collaborator(
-    plan_id: str,
-    body: AddCollaboratorRequest,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    sb = _sb()
-
-    plan_res = (
-        sb.supabase.table("planned_lessons")
-        .select("created_by")
-        .eq("id", plan_id)
-        .eq("institute_id", institute_id)
-        .single()
-        .execute()
-    )
-    if not plan_res.data:
-        raise HTTPException(status_code=404, detail="Plan not found")
-    if not _is_creator(plan_res.data, user_id):
-        raise HTTPException(status_code=403, detail="Only the creator can add collaborators")
-
-    res = (
-        sb.supabase.table("lesson_collaborators")
-        .upsert(
-            {
-                "planned_lesson_id": plan_id,
-                "profile_id": body.profile_id,
-                "can_edit": body.can_edit,
-                "added_at": datetime.utcnow().isoformat(),
-            },
-            on_conflict="planned_lesson_id,profile_id",
-        )
-        .execute()
-    )
-    return {"status": "ok", "row": (res.data or [{}])[0]}
-
-
-@router.delete("/plans/{plan_id}/collaborators/{profile_id}")
-async def remove_collaborator(
-    plan_id: str,
-    profile_id: str,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    sb = _sb()
-
-    plan_res = (
-        sb.supabase.table("planned_lessons")
-        .select("created_by")
-        .eq("id", plan_id)
-        .eq("institute_id", institute_id)
-        .single()
-        .execute()
-    )
-    if not plan_res.data:
-        raise HTTPException(status_code=404, detail="Plan not found")
-    if not _is_creator(plan_res.data, user_id):
-        raise HTTPException(status_code=403, detail="Only the creator can remove collaborators")
-
-    sb.supabase.table("lesson_collaborators").delete().eq("planned_lesson_id", plan_id).eq("profile_id", profile_id).execute()
-    return {"status": "ok"}
-
-
-@router.post("/plans/{plan_id}/suggest")
-async def suggest_field(
-    plan_id: str,
-    body: SuggestRequest,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    sb = _sb()
-
-    if body.field not in VALID_FIELDS:
-        raise HTTPException(status_code=400, detail=f"field must be one of {VALID_FIELDS}")
-
-    plan_res = (
-        sb.supabase.table("planned_lessons")
-        .select("*")
-        .eq("id", plan_id)
-        .eq("institute_id", institute_id)
-        .single()
-        .execute()
-    )
-    if not plan_res.data:
-        raise HTTPException(status_code=404, detail="Plan not found")
-
-    if not _can_edit_plan(sb, plan_id, user_id):
-        raise HTTPException(status_code=403, detail="Access denied")
-
-    plan = plan_res.data
-    subject = plan.get("subject") or "a UK secondary school subject"
-    year_group = plan.get("year_group") or "unspecified year group"
-    title = plan.get("title", "")
-    context = body.context or ""
-
-    context_part = ("Teacher notes: " + context + "\n") if context else ""
-    if body.field == "objectives":
-        existing = ""
-        if body.objective_texts:
-            existing = "\n".join("- " + t for t in body.objective_texts)
-        objectives_part = ("Existing objectives:\n" + existing + "\n") if existing else ""
-        prompt = (
-            "You are an expert UK secondary school teacher.\n"
-            "Lesson: '" + title + "', Subject: " + subject + ", Year: " + year_group + ".\n"
-            + objectives_part + context_part
-            + "Suggest ONE clear, measurable learning objective for this lesson using Bloom's taxonomy. "
-            "Write only the objective text, no bullet point or prefix."
-        )
-    elif body.field == "activity_description":
-        section = body.activity_section or "Main"
-        prompt = (
-            "You are an expert UK secondary school teacher.\n"
-            "Lesson: '" + title + "', Subject: " + subject + ", Year: " + year_group + ".\n"
-            "Activity section: " + section + ".\n"
-            + context_part
-            + "Write a concise, practical description (2-4 sentences) for a " + section + " activity "
-            "suitable for UK secondary pupils. Include what the teacher does and what pupils do."
-        )
-    else:  # title
-        prompt = (
-            "You are an expert UK secondary school teacher.\n"
-            "Subject: " + subject + ", Year: " + year_group + ".\n"
-            + context_part
-            + "Suggest ONE engaging, clear lesson title for a UK secondary school lesson. "
-            "Write only the title, no explanation."
-        )
-
-    try:
-        payload = {
-            "model": _OLLAMA_MODEL,
-            "prompt": prompt,
-            "stream": False,
-        }
-        async with aiohttp.ClientSession() as session:
-            async with session.post(
-                f"{_OLLAMA_URL}/api/generate",
-                json=payload,
-                headers={"Content-Type": "application/json"},
-                timeout=aiohttp.ClientTimeout(total=_LLM_TIMEOUT),
-            ) as resp:
-                if resp.status != 200:
-                    body_text = await resp.text()
-                    logger.error(f"Ollama suggest error ({resp.status}): {body_text}")
-                    raise HTTPException(status_code=502, detail=f"LLM request failed: {resp.status}")
-                data = await resp.json()
-        suggestion = (data.get("response") or "").strip()
-    except HTTPException:
-        raise
-    except Exception as e:
-        logger.error(f"suggest_field LLM call failed for plan {plan_id}: {e}")
-        raise HTTPException(status_code=502, detail="LLM service unavailable")
-
-    return {"suggestion": suggestion}

routers/database/tools/platform_admin_router.py

@@ -1,175 +0,0 @@
-"""
-Platform Admin Router — super_admin / platform_admin operations.
-
-GET  /admin/schools   — list all institutes with member + calendar counts
-GET  /admin/stats     — platform-level summary
-"""
-import os
-from typing import Any, Dict, List
-from fastapi import APIRouter, Depends, HTTPException
-from modules.logger_tool import initialise_logger
-from modules.auth.supabase_bearer import SupabaseBearer
-from modules.auth.platform_admin import require_platform_admin
-from modules.database.supabase.utils.client import SupabaseServiceRoleClient
-
-logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
-router = APIRouter()
-
-
-def _sb() -> SupabaseServiceRoleClient:
-    return SupabaseServiceRoleClient()
-
-
-@router.get("/schools")
-async def list_all_schools(
-    _: dict = Depends(require_platform_admin),
-) -> Dict[str, Any]:
-    """List every institute with basic counts. Platform admin only."""
-    sb = _sb()
-
-    institutes = (
-        sb.supabase.table("institutes")
-        .select("id,name,urn,website,status,created_at,neo4j_uuid_string")
-        .order("name")
-        .execute()
-        .data or []
-    )
-
-    if not institutes:
-        return {"status": "ok", "schools": [], "total": 0}
-
-    inst_ids = [i["id"] for i in institutes]
-
-    # Member counts per institute
-    all_members = (
-        sb.supabase.table("institute_memberships")
-        .select("institute_id,role")
-        .in_("institute_id", inst_ids)
-        .execute()
-        .data or []
-    )
-    from collections import defaultdict
-    member_counts: Dict[str, Dict[str, int]] = defaultdict(lambda: {"staff": 0, "students": 0})
-    staff_roles = {"teacher", "school_admin", "department_head"}
-    for m in all_members:
-        iid = m["institute_id"]
-        if m["role"] in staff_roles:
-            member_counts[iid]["staff"] += 1
-        elif m["role"] == "student":
-            member_counts[iid]["students"] += 1
-
-    # Calendar presence per institute
-    term_rows = (
-        sb.supabase.table("academic_terms")
-        .select("institute_id")
-        .in_("institute_id", inst_ids)
-        .execute()
-        .data or []
-    )
-    has_calendar = {r["institute_id"] for r in term_rows}
-
-    # Pending invitations count
-    inv_rows = (
-        sb.supabase.table("invitations")
-        .select("institute_id")
-        .eq("status", "pending")
-        .in_("institute_id", inst_ids)
-        .execute()
-        .data or []
-    )
-    from collections import Counter
-    inv_counts = Counter(r["institute_id"] for r in inv_rows)
-
-    schools = []
-    for inst in institutes:
-        iid = inst["id"]
-        mc = member_counts.get(iid, {})
-        schools.append({
-            **inst,
-            "staff_count": mc.get("staff", 0),
-            "student_count": mc.get("students", 0),
-            "has_calendar": iid in has_calendar,
-            "pending_invitations": inv_counts.get(iid, 0),
-        })
-
-    return {"status": "ok", "schools": schools, "total": len(schools)}
-
-
-@router.get("/stats")
-async def platform_stats(
-    _: dict = Depends(require_platform_admin),
-) -> Dict[str, Any]:
-    """High-level platform counts. Platform admin only."""
-    sb = _sb()
-
-    inst_count = len(
-        sb.supabase.table("institutes").select("id").execute().data or []
-    )
-    profile_count = len(
-        sb.supabase.table("profiles").select("id").execute().data or []
-    )
-    lesson_count = len(
-        sb.supabase.table("taught_lessons").select("id").execute().data or []
-    )
-    inv_count = len(
-        sb.supabase.table("invitations").select("id").eq("status", "pending").execute().data or []
-    )
-
-    return {
-        "status": "ok",
-        "schools": inst_count,
-        "profiles": profile_count,
-        "taught_lessons": lesson_count,
-        "pending_invitations": inv_count,
-    }
-
-
-@router.post("/reset")
-async def reset_environment(
-    scope: str = "all",
-    _: dict = Depends(require_platform_admin),
-) -> Dict[str, Any]:
-    """DESTRUCTIVE: wipe test data. Platform admin only.
-
-    scope (query param):
-      - all          : full wipe (Neo4j + Supabase data + auth users) AND the entire
-                       exam-marker subsystem below.
-      - exam-corpus  : ONLY the entire exam-marker subsystem, not just public papers:
-                       public corpus/eb_* data, cc.examboards storage objects, exam
-                       templates, template layouts, questions, boundaries, response
-                       areas, marking batches, student submissions, and mark entries
-                       (without touching schools/users).
-      - timetable    : ONLY timetable/calendar materialization tables.
-    """
-    if scope not in ("all", "exam-corpus", "timetable"):
-        raise HTTPException(status_code=400, detail="scope must be one of: all, exam-corpus, timetable")
-    import asyncio
-    import functools
-    from run.initialization.reset_environment import reset as _reset
-    loop = asyncio.get_event_loop()
-    result = await loop.run_in_executor(None, functools.partial(_reset, scope))
-    return {"status": "ok", **result}
-
-
-@router.post("/seed")
-async def seed_environment(
-    _: dict = Depends(require_platform_admin),
-) -> Dict[str, Any]:
-    """Idempotent rebuild: both schools, global calendar, 20 test accounts. Platform admin only."""
-    import asyncio
-    from run.initialization.seed_environment import seed as _seed
-    loop = asyncio.get_event_loop()
-    result = await loop.run_in_executor(None, _seed)
-    return {"status": "ok", **result}
-
-
-@router.post("/seed-timetable")
-async def seed_greenfield_timetable(
-    _: dict = Depends(require_platform_admin),
-) -> Dict[str, Any]:
-    """Seed full timetable + taught lessons for Greenfield Academy. Platform admin only."""
-    import asyncio
-    from run.initialization.seed_greenfield_timetable import seed as _seed
-    loop = asyncio.get_event_loop()
-    result = await loop.run_in_executor(None, _seed)
-    return {"status": "ok", **result}

routers/database/tools/school_router.py

@@ -1,603 +0,0 @@
-"""
-School Router — school status, search, register, and admin-editable info.
-"""
-import os
-import json
-import uuid as uuid_lib
-from typing import Dict, Any, Optional, List
-from fastapi import APIRouter, Depends, Query
-from pydantic import BaseModel
-from modules.logger_tool import initialise_logger
-from modules.auth.supabase_bearer import SupabaseBearer
-from modules.database.supabase.utils.client import SupabaseServiceRoleClient
-import modules.database.tools.neo4j_driver_tools as driver_tools
-
-logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
-router = APIRouter()
-
-
-# ─── Helpers ──────────────────────────────────────────────────────────────────
-
-def _get_sb():
-    return SupabaseServiceRoleClient()
-
-def _institute_db(neo4j_uuid: str) -> str:
-    return f"cc.institutes.{neo4j_uuid}"
-
-def _find_institute_db_by_email(user_email: str) -> Optional[str]:
-    """Fallback: scan all institute DBs for the teacher's email."""
-    try:
-        with driver_tools.get_session(database="system") as s:
-            dbs = [r["name"] for r in s.run(
-                "SHOW DATABASES YIELD name "
-                "WHERE name STARTS WITH 'cc.institutes.' "
-                "AND NOT name ENDS WITH '.curriculum' RETURN name"
-            )]
-        for db in dbs:
-            try:
-                with driver_tools.get_session(database=db) as s:
-                    rec = s.run(
-                        "MATCH (t:Teacher) WHERE t.worker_email = $e "
-                        "RETURN t.uuid_string AS uuid LIMIT 1",
-                        e=user_email
-                    ).single()
-                    if rec:
-                        return db
-            except Exception:
-                continue
-    except Exception as e:
-        logger.warning(f"Institute DB scan failed: {e}")
-    return None
-
-
-# ─── Status endpoint ─────────────────────────────────────────────────────────
-
-@router.get("/status")
-async def get_school_status(credentials: dict = Depends(SupabaseBearer())) -> Dict[str, Any]:
-    """Return the current user's school role, school info, and calendar/timetable setup status."""
-    user_id = credentials.get("sub", "")
-    user_email = credentials.get("email", "")
-
-    try:
-        sb = _get_sb()
-
-        p = sb.supabase.table("profiles").select("school_id").eq("id", user_id).single().execute()
-        school_id = (p.data or {}).get("school_id")
-
-        # Fallback: if profiles.school_id not set, check institute_memberships directly
-        if not school_id:
-            m_fb = sb.supabase.table("institute_memberships").select("institute_id,role").eq("profile_id", user_id).single().execute()
-            if m_fb.data and m_fb.data.get("institute_id"):
-                school_id = m_fb.data["institute_id"]
-                user_role = m_fb.data.get("role") or "teacher"
-                # Self-heal: write school_id back to profile
-                try:
-                    sb.supabase.table("profiles").update({"school_id": str(school_id)}).eq("id", user_id).execute()
-                except Exception:
-                    pass
-            else:
-                return {"status": "no_school"}
-        else:
-            m = sb.supabase.table("institute_memberships").select("role").eq("profile_id", user_id).eq("institute_id", school_id).single().execute()
-            user_role = ((m.data or {}).get("role") or "teacher")
-
-        i = sb.supabase.table("institutes").select("id,name,urn,website,address,metadata,neo4j_uuid_string").eq("id", school_id).single().execute()
-        inst = i.data or {}
-        neo4j_uuid = inst.get("neo4j_uuid_string")
-        meta = inst.get("metadata") or {}
-
-    except Exception as e:
-        logger.warning(f"Supabase school status query failed: {e}")
-        return {"status": "error", "message": str(e)}
-
-    if neo4j_uuid:
-        inst_db = _institute_db(neo4j_uuid)
-    else:
-        inst_db = _find_institute_db_by_email(user_email)
-        if not inst_db:
-            return {"status": "no_school"}
-
-    school_has_calendar = False
-    teacher_has_timetable = False
-    timetable_id = None
-    periods_template = None
-
-    try:
-        with driver_tools.get_session(database=inst_db) as s:
-            ay = s.run("MATCH (ay:AcademicYear) RETURN ay LIMIT 1").single()
-            school_has_calendar = ay is not None
-
-            st = s.run(
-                "MATCH (st:SchoolTimetable) WHERE st.periods_template IS NOT NULL "
-                "RETURN st.periods_template AS p LIMIT 1"
-            ).single()
-            if st:
-                periods_template = json.loads(st["p"])
-
-            t = s.run(
-                "MATCH (t:Teacher) WHERE t.worker_email = $e "
-                "RETURN t.uuid_string AS uuid LIMIT 1",
-                e=user_email
-            ).single()
-            if t:
-                teacher_uuid = t["uuid"]
-                tt = s.run(
-                    "MATCH (t:Teacher {uuid_string: $u})-[:HAS_TIMETABLE]->(tt:TeacherTimetable) "
-                    "RETURN tt.uuid_string AS id LIMIT 1",
-                    u=teacher_uuid
-                ).single()
-                if tt:
-                    teacher_has_timetable = True
-                    timetable_id = tt["id"]
-    except Exception as e:
-        logger.warning(f"Neo4j school status check failed for {inst_db}: {e}")
-
-    return {
-        "status": "ok",
-        "user_role": user_role,
-        "school_id": str(school_id),
-        "institute_db": inst_db,
-        "school_has_calendar": school_has_calendar,
-        "teacher_has_timetable": teacher_has_timetable,
-        "timetable_id": timetable_id,
-        "periods_template": periods_template,
-        "school_info": {
-            "name": inst.get("name", ""),
-            "urn": inst.get("urn", ""),
-            "website": inst.get("website", ""),
-            "address": inst.get("address") or {},
-            "headteacher": meta.get("headteacher", ""),
-            "term_dates_url": meta.get("term_dates_url", ""),
-            "staff_list_url": meta.get("staff_list_url", ""),
-        },
-    }
-
-
-# ─── School info update ───────────────────────────────────────────────────────
-
-class SchoolInfoUpdate(BaseModel):
-    headteacher: Optional[str] = None
-    term_dates_url: Optional[str] = None
-    staff_list_url: Optional[str] = None
-
-@router.patch("/info")
-async def update_school_info(
-    body: SchoolInfoUpdate,
-    credentials: dict = Depends(SupabaseBearer())
-) -> Dict[str, Any]:
-    """Admin: update school metadata fields stored in institutes.metadata jsonb."""
-    user_id = credentials.get("sub", "")
-    try:
-        sb = _get_sb()
-        p = sb.supabase.table("profiles").select("school_id").eq("id", user_id).single().execute()
-        school_id = (p.data or {}).get("school_id")
-        if not school_id:
-            return {"status": "error", "message": "No school linked"}
-
-        m = sb.supabase.table("institute_memberships").select("role").eq("profile_id", user_id).eq("institute_id", school_id).single().execute()
-        role = ((m.data or {}).get("role") or "teacher")
-        if role != "school_admin":
-            return {"status": "error", "message": "school_admin role required"}
-
-        i = sb.supabase.table("institutes").select("metadata").eq("id", school_id).single().execute()
-        meta = dict((i.data or {}).get("metadata") or {})
-        if body.headteacher is not None:
-            meta["headteacher"] = body.headteacher
-        if body.term_dates_url is not None:
-            meta["term_dates_url"] = body.term_dates_url
-        if body.staff_list_url is not None:
-            meta["staff_list_url"] = body.staff_list_url
-
-        sb.supabase.table("institutes").update({"metadata": meta}).eq("id", school_id).execute()
-        return {"status": "ok"}
-    except Exception as e:
-        logger.error(f"School info update failed: {e}")
-        return {"status": "error", "message": str(e)}
-
-
-# ─── GAIS school search ───────────────────────────────────────────────────────
-
-@router.get("/search")
-async def search_schools(
-    q: str = Query(..., min_length=2, description="School name, URN, or postcode"),
-    limit: int = Query(20, ge=1, le=100),
-    status: str = Query("Open", description="Filter by establishment status"),
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    """Search GAIS school reference data by name, URN, or postcode."""
-    try:
-        sb = _get_sb()
-        query = sb.supabase.table("gais_schools").select(
-            "urn,name,status,phase,type,street,locality,town,county,postcode,"
-            "website,telephone,head_title,head_first_name,head_last_name,"
-            "la_code,la_name,number_of_pupils,gender,religious_character,region"
-        )
-        if status:
-            query = query.eq("status", status)
-
-        q_stripped = q.strip()
-        if q_stripped.isdigit():
-            # URN exact match
-            query = query.eq("urn", q_stripped)
-        else:
-            # Name / postcode trigram search — use ilike on name first, fallback includes postcode
-            query = query.ilike("name", f"%{q_stripped}%")
-
-        result = query.limit(limit).execute()
-        return {"status": "ok", "schools": result.data or [], "count": len(result.data or [])}
-    except Exception as e:
-        logger.error(f"School search failed: {e}")
-        return {"status": "error", "message": str(e)}
-
-
-# ─── School registration (onboarding) ────────────────────────────────────────
-
-class SchoolRegisterBody(BaseModel):
-    urn: str
-    name: str
-    address: Optional[Dict[str, Any]] = None
-    website: Optional[str] = None
-    headteacher: Optional[str] = None
-
-@router.post("/register")
-async def register_school(
-    body: SchoolRegisterBody,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    """
-    Onboarding: create an institute record from a GAIS-selected school,
-    provision the Neo4j database, and link the calling user as school_admin.
-    """
-    user_id = credentials.get("sub", "")
-    user_email = credentials.get("email", "")
-
-    try:
-        sb = _get_sb()
-
-        # Prevent duplicate registration
-        existing = sb.supabase.table("institutes").select("id").eq("urn", body.urn).execute()
-        if existing.data:
-            school_id = existing.data[0]["id"]
-            # Still link user if not already linked
-            _ensure_membership(sb, user_id, school_id, "school_admin")
-            sb.supabase.table("profiles").update({"school_id": str(school_id)}).eq("id", user_id).execute()
-            return {"status": "already_exists", "school_id": str(school_id)}
-
-        # Build metadata
-        meta: Dict[str, Any] = {}
-        if body.headteacher:
-            meta["headteacher"] = body.headteacher
-
-        institute_data: Dict[str, Any] = {
-            "name": body.name,
-            "urn": body.urn,
-            "status": "active",
-            "address": body.address or {},
-            "website": body.website or "",
-            "metadata": meta,
-        }
-
-        ins_result = sb.supabase.table("institutes").insert(institute_data).execute()
-        if not ins_result.data:
-            return {"status": "error", "message": "Failed to create institute record"}
-
-        school_id = ins_result.data[0]["id"]
-
-        # Provision Neo4j institute database
-        try:
-            from modules.database.services.provisioning_service import ProvisioningService
-            ps = ProvisioningService()
-            ps.ensure_school(school_id)
-            # Reload institute to get the neo4j_uuid_string set by provisioning
-            inst = sb.supabase.table("institutes").select("neo4j_uuid_string").eq("id", school_id).single().execute()
-            neo4j_uuid = (inst.data or {}).get("neo4j_uuid_string")
-        except Exception as prov_err:
-            logger.warning(f"Neo4j provisioning failed for {school_id}: {prov_err}")
-            neo4j_uuid = None
-
-        # Link user as school_admin
-        _ensure_membership(sb, user_id, school_id, "school_admin")
-        sb.supabase.table("profiles").update({"school_id": str(school_id)}).eq("id", user_id).execute()
-
-        return {
-            "status": "ok",
-            "school_id": str(school_id),
-            "neo4j_uuid": neo4j_uuid,
-        }
-
-    except Exception as e:
-        logger.error(f"School registration failed: {e}")
-        return {"status": "error", "message": str(e)}
-
-
-def _ensure_membership(sb: SupabaseServiceRoleClient, user_id: str, school_id: str, role: str) -> None:
-    existing = sb.supabase.table("institute_memberships").select("id").eq("profile_id", user_id).eq("institute_id", school_id).execute()
-    if not existing.data:
-        sb.supabase.table("institute_memberships").insert({
-            "profile_id": user_id,
-            "institute_id": school_id,
-            "role": role,
-        }).execute()
-
-
-# ─── School Overview ──────────────────────────────────────────────────────────
-
-@router.get("/overview")
-async def get_school_overview(
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    """
-    Summary dashboard for school admins: staff/student/class counts,
-    calendar snapshot (terms, total academic days, current/next term).
-    """
-    user_id = credentials.get("sub", "")
-    sb = _get_sb()
-
-    p = sb.supabase.table("profiles").select("school_id").eq("id", user_id).single().execute()
-    school_id = str((p.data or {}).get("school_id") or "")
-    if not school_id:
-        raise HTTPException(status_code=400, detail="User is not linked to a school")
-
-    # Role check
-    mem = (
-        sb.supabase.table("institute_memberships")
-        .select("role")
-        .eq("profile_id", user_id)
-        .eq("institute_id", school_id)
-        .single()
-        .execute()
-    )
-    user_role = (mem.data or {}).get("role", "teacher")
-
-    # Counts
-    staff_roles = ["teacher", "school_admin", "department_head"]
-    staff_rows = (
-        sb.supabase.table("institute_memberships")
-        .select("profile_id", count="exact")
-        .eq("institute_id", school_id)
-        .in_("role", staff_roles)
-        .execute()
-    )
-    student_rows = (
-        sb.supabase.table("institute_memberships")
-        .select("profile_id", count="exact")
-        .eq("institute_id", school_id)
-        .eq("role", "student")
-        .execute()
-    )
-    class_rows = (
-        sb.supabase.table("classes")
-        .select("id", count="exact")
-        .eq("institute_id", school_id)
-        .eq("is_active", True)
-        .execute()
-    )
-
-    # Calendar snapshot from academic_terms
-    terms = (
-        sb.supabase.table("academic_terms")
-        .select("id,term_name,term_number,start_date,end_date")
-        .eq("institute_id", school_id)
-        .order("term_number")
-        .execute()
-        .data or []
-    )
-
-    # Academic day counts per term
-    if terms:
-        term_ids = [t["id"] for t in terms]
-        day_counts_res = (
-            sb.supabase.table("academic_days")
-            .select("academic_term_id", count="exact")
-            .eq("institute_id", school_id)
-            .eq("day_type", "Academic")
-            .in_("academic_term_id", term_ids)
-            .execute()
-        )
-        # Supabase doesn't group-by server-side; count manually per term
-        all_days = (
-            sb.supabase.table("academic_days")
-            .select("academic_term_id,day_type")
-            .eq("institute_id", school_id)
-            .in_("academic_term_id", term_ids)
-            .execute()
-            .data or []
-        )
-        from collections import defaultdict
-        academic_day_count: Dict[str, int] = defaultdict(int)
-        total_day_count: Dict[str, int] = defaultdict(int)
-        for d in all_days:
-            total_day_count[d["academic_term_id"]] += 1
-            if d["day_type"] == "Academic":
-                academic_day_count[d["academic_term_id"]] += 1
-
-        from datetime import date
-        today_str = str(date.today())
-        for t in terms:
-            t["academic_days"] = academic_day_count.get(t["id"], 0)
-            t["total_days"] = total_day_count.get(t["id"], 0)
-            if t["start_date"] <= today_str <= t["end_date"]:
-                t["is_current"] = True
-            else:
-                t["is_current"] = False
-
-    pending_invites = (
-        sb.supabase.table("invitations")
-        .select("id", count="exact")
-        .eq("institute_id", school_id)
-        .eq("status", "pending")
-        .execute()
-    )
-
-    return {
-        "status": "ok",
-        "user_role": user_role,
-        "counts": {
-            "staff": staff_rows.count or 0,
-            "students": student_rows.count or 0,
-            "classes": class_rows.count or 0,
-            "pending_invitations": pending_invites.count or 0,
-        },
-        "terms": terms,
-        "has_calendar": len(terms) > 0,
-    }
-
-
-# ─── Calendar days (admin view) ───────────────────────────────────────────────
-
-@router.get("/calendar/days")
-async def list_calendar_days(
-    term_id: Optional[str] = None,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    """
-    Return academic_days for the school, optionally filtered by term.
-    Includes week_cycle from the parent academic_week.
-    """
-    user_id = credentials.get("sub", "")
-    sb = _get_sb()
-
-    p = sb.supabase.table("profiles").select("school_id").eq("id", user_id).single().execute()
-    school_id = str((p.data or {}).get("school_id") or "")
-    if not school_id:
-        raise HTTPException(status_code=400, detail="User is not linked to a school")
-
-    q = (
-        sb.supabase.table("academic_days")
-        .select("id,date,day_of_week,day_type,academic_week_id,academic_term_id,academic_day_number,excluded_period_codes")
-        .eq("institute_id", school_id)
-        .order("date")
-    )
-    if term_id:
-        q = q.eq("academic_term_id", term_id)
-
-    days = q.execute().data or []
-
-    # Enrich with week_cycle
-    if days:
-        week_ids = list({d["academic_week_id"] for d in days if d.get("academic_week_id")})
-        weeks = (
-            sb.supabase.table("academic_weeks")
-            .select("id,week_number,week_cycle")
-            .in_("id", week_ids)
-            .execute()
-            .data or []
-        )
-        wk_map = {w["id"]: w for w in weeks}
-        for d in days:
-            wk = wk_map.get(d.get("academic_week_id", ""), {})
-            d["week_cycle"] = wk.get("week_cycle", "")
-            d["week_number"] = wk.get("week_number")
-
-    return {"status": "ok", "days": days, "total": len(days)}
-
-
-@router.patch("/calendar/days/{day_id}")
-async def update_calendar_day(
-    day_id: str,
-    body: Dict[str, Any],
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    """
-    Override day_type for a single academic day (school admin only).
-    Syncs academic_periods: removes periods for non-Academic days,
-    creates periods from periods_template for newly-Academic days.
-    """
-    user_id = credentials.get("sub", "")
-    sb = _get_sb()
-
-    p = sb.supabase.table("profiles").select("school_id").eq("id", user_id).single().execute()
-    school_id = str((p.data or {}).get("school_id") or "")
-    if not school_id:
-        raise HTTPException(status_code=400, detail="User is not linked to a school")
-
-    # Verify admin role
-    mem = (
-        sb.supabase.table("institute_memberships")
-        .select("role")
-        .eq("profile_id", user_id)
-        .eq("institute_id", school_id)
-        .single()
-        .execute()
-    )
-    if (mem.data or {}).get("role") not in ("school_admin", "department_head"):
-        raise HTTPException(status_code=403, detail="School admin access required")
-
-    # Verify day belongs to school
-    day = (
-        sb.supabase.table("academic_days")
-        .select("*")
-        .eq("id", day_id)
-        .eq("institute_id", school_id)
-        .single()
-        .execute()
-    ).data
-    if not day:
-        raise HTTPException(status_code=404, detail="Day not found")
-
-    new_day_type = body.get("day_type", day["day_type"])
-    excluded = body.get("excluded_period_codes", day.get("excluded_period_codes") or [])
-
-    valid_types = {"Academic", "Holiday", "Staff", "OffTimetable"}
-    if new_day_type not in valid_types:
-        raise HTTPException(status_code=400, detail=f"day_type must be one of {sorted(valid_types)}")
-
-    # Update the day
-    sb.supabase.table("academic_days").update({
-        "day_type": new_day_type,
-        "excluded_period_codes": excluded,
-    }).eq("id", day_id).execute()
-
-    old_type = day["day_type"]
-    periods_changed = 0
-
-    if old_type == "Academic" and new_day_type != "Academic":
-        # Remove periods for this day
-        del_res = (
-            sb.supabase.table("academic_periods")
-            .delete()
-            .eq("academic_day_id", day_id)
-            .execute()
-        )
-        periods_changed = -(len(del_res.data or []))
-
-    elif old_type != "Academic" and new_day_type == "Academic":
-        # Create periods from template
-        stt = (
-            sb.supabase.table("school_timetables")
-            .select("periods_template")
-            .eq("institute_id", school_id)
-            .order("created_at", desc=True)
-            .limit(1)
-            .execute()
-            .data or []
-        )
-        template = (stt[0].get("periods_template") or []) if stt else []
-        skip = set(excluded)
-        new_periods = []
-        for period in template:
-            if period.get("code") in skip:
-                continue
-            new_periods.append({
-                "academic_day_id": day_id,
-                "institute_id": school_id,
-                "period_code": period["code"],
-                "period_name": period.get("name", period["code"]),
-                "start_time": period.get("start_time"),
-                "end_time": period.get("end_time"),
-                "period_type": period.get("period_type", "lesson"),
-            })
-        if new_periods:
-            ins_res = (
-                sb.supabase.table("academic_periods")
-                .upsert(new_periods, on_conflict="academic_day_id,period_code")
-                .execute()
-            )
-            periods_changed = len(ins_res.data or [])
-
-    return {
-        "status": "ok",
-        "day_id": day_id,
-        "new_day_type": new_day_type,
-        "periods_changed": periods_changed,
-    }

routers/database/tools/taught_lessons_router.py

@@ -1,601 +0,0 @@
-"""
-Taught Lessons Router — materialization and lesson CRUD.
-
-POST /materialize   — slot template × academic_periods → taught_lessons rows
-GET  /lessons       — teacher's lessons for a date range
-GET  /lessons/{id}  — single lesson detail
-PATCH /lessons/{id} — update lesson_plan, notes, status (teacher-owned)
-"""
-import os
-from collections import defaultdict
-from datetime import datetime, date, timedelta
-from typing import Any, Dict, List, Optional, Tuple
-from fastapi import APIRouter, Depends, HTTPException
-from pydantic import BaseModel
-from modules.logger_tool import initialise_logger
-from modules.auth.supabase_bearer import SupabaseBearer
-from modules.database.supabase.utils.client import SupabaseServiceRoleClient
-
-logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
-router = APIRouter()
-
-
-def _sb() -> SupabaseServiceRoleClient:
-    return SupabaseServiceRoleClient()
-
-
-def _resolve_institute_id(user_id: str) -> Optional[str]:
-    try:
-        sb = _sb()
-        p = sb.supabase.table("profiles").select("school_id").eq("id", user_id).single().execute()
-        return str((p.data or {}).get("school_id") or "") or None
-    except Exception:
-        return None
-
-
-def _require_institute(user_id: str) -> str:
-    iid = _resolve_institute_id(user_id)
-    if not iid:
-        raise HTTPException(status_code=400, detail="User is not linked to a school")
-    return iid
-
-
-# ─── Request models ───────────────────────────────────────────────────────────
-
-class UpdateLessonRequest(BaseModel):
-    lesson_plan: Optional[Dict[str, Any]] = None
-    notes: Optional[str] = None
-    status: Optional[str] = None      # planned | in_progress | completed | cancelled | substituted
-
-
-# ─── Materialize ─────────────────────────────────────────────────────────────
-
-@router.post("/materialize")
-async def materialize_taught_lessons(
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    """
-    Materialize taught_lessons from teacher_timetable_slots × academic_periods.
-
-    For each slot (day_of_week + period_code + week_cycle), find every
-    academic_period that falls on a matching day and week cycle, then
-    UPSERT a taught_lesson row and a whiteboard_room for it.
-
-    Safe to re-run; uses ON CONFLICT DO UPDATE.
-    """
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    sb = _sb()
-
-    # ── 1. Get teacher timetable ──────────────────────────────────────────────
-    tt_rows = (
-        sb.supabase.table("teacher_timetables")
-        .select("id")
-        .eq("profile_id", user_id)
-        .eq("institute_id", institute_id)
-        .limit(1)
-        .execute()
-        .data or []
-    )
-    if not tt_rows:
-        return {"status": "error", "message": "No teacher timetable found — run /timetable/init first"}
-    tt_id = tt_rows[0]["id"]
-
-    # ── 2. Get teacher's timetable slots ──────────────────────────────────────
-    slots = (
-        sb.supabase.table("teacher_timetable_slots")
-        .select("id,day_of_week,period_code,subject_class,start_time,end_time,week_cycle,class_id")
-        .eq("teacher_timetable_id", tt_id)
-        .execute()
-        .data or []
-    )
-    if not slots:
-        return {"status": "ok", "created": 0, "updated": 0, "message": "No timetable slots found"}
-
-    # ── 3. Resolve class_ids for slots (match by subject_class name) ──────────
-    class_name_to_id: Dict[str, str] = {}
-    subject_names = list({s["subject_class"] for s in slots if s.get("subject_class")})
-    if subject_names:
-        try:
-            classes_res = (
-                sb.supabase.table("classes")
-                .select("id,name,class_code")
-                .eq("institute_id", institute_id)
-                .eq("is_active", True)
-                .execute()
-                .data or []
-            )
-            for c in classes_res:
-                if c.get("name"):
-                    class_name_to_id[c["name"]] = c["id"]
-                if c.get("class_code"):
-                    class_name_to_id[c["class_code"]] = c["id"]
-        except Exception as e:
-            logger.warning(f"Could not resolve class names: {e}")
-
-    # slot lookup keyed by (day_of_week, period_code, week_cycle)
-    # week_cycle='' means applies to both A and B weeks
-    slot_map: Dict[Tuple[str, str, str], Dict] = {}
-    for slot in slots:
-        key = (slot["day_of_week"], slot["period_code"], slot.get("week_cycle", ""))
-        slot_map[key] = slot
-
-    # ── 4. Get all academic_periods with day + week info ──────────────────────
-    # Fetch academic_days and academic_weeks separately, then join in Python
-    days_res = (
-        sb.supabase.table("academic_days")
-        .select("id,date,day_of_week,academic_week_id,day_type")
-        .eq("institute_id", institute_id)
-        .execute()
-        .data or []
-    )
-    weeks_res = (
-        sb.supabase.table("academic_weeks")
-        .select("id,week_cycle")
-        .eq("institute_id", institute_id)
-        .execute()
-        .data or []
-    )
-    week_cycle_map = {w["id"]: w["week_cycle"] for w in weeks_res}
-
-    # Only academic days get lesson periods
-    academic_day_map = {
-        d["id"]: {**d, "week_cycle": week_cycle_map.get(d["academic_week_id"], "A")}
-        for d in days_res
-        if d["day_type"] == "Academic"
-    }
-
-    periods_res = (
-        sb.supabase.table("academic_periods")
-        .select("id,academic_day_id,period_code,period_name,start_time,end_time,period_type")
-        .eq("institute_id", institute_id)
-        .execute()
-        .data or []
-    )
-
-    # ── 5. Match slots to periods ─────────────────────────────────────────────
-    to_upsert: List[Dict] = []
-    whiteboard_rows: List[Dict] = []
-
-    for period in periods_res:
-        day_info = academic_day_map.get(period["academic_day_id"])
-        if not day_info:
-            continue
-
-        dow = day_info["day_of_week"]
-        week_cycle = day_info["week_cycle"]
-        pcode = period["period_code"]
-        d = str(day_info["date"])[:10]
-
-        # Check all matching slot patterns: exact week_cycle match or '' (both weeks)
-        matched_slot = (
-            slot_map.get((dow, pcode, week_cycle))
-            or slot_map.get((dow, pcode, ""))
-        )
-        if not matched_slot:
-            continue
-
-        subj_class = matched_slot.get("subject_class", "")
-        # Prefer the slot's own class_id, fall back to name-lookup
-        class_id = (
-            matched_slot.get("class_id")
-            or class_name_to_id.get(subj_class)
-        )
-
-        tl_id_hint = f"tl_{period['id']}"   # deterministic for neo4j_node_id
-        to_upsert.append({
-            "academic_period_id": period["id"],
-            "teacher_timetable_slot_id": matched_slot["id"],
-            "class_id": class_id,            # may be None
-            "teacher_id": user_id,
-            "institute_id": institute_id,
-            "date": d,
-            "period_code": pcode,
-            "week_cycle": week_cycle,
-            "day_of_week": dow,
-            "status": "planned",
-            "neo4j_node_id": tl_id_hint,
-        })
-
-    if not to_upsert:
-        return {"status": "ok", "created": 0, "updated": 0, "message": "No slot-period matches found"}
-
-    # ── 6. Batch-upsert taught_lessons ────────────────────────────────────────
-    BATCH = 100
-    created = 0
-    for i in range(0, len(to_upsert), BATCH):
-        chunk = to_upsert[i : i + BATCH]
-        try:
-            res = (
-                sb.supabase.table("taught_lessons")
-                .upsert(chunk, on_conflict="academic_period_id,teacher_id")
-                .execute()
-            )
-            created += len(res.data or [])
-        except Exception as e:
-            logger.error(f"taught_lessons upsert chunk {i}: {e}")
-
-    # ── 7. Fetch newly-created taught_lesson ids, create whiteboard_rooms ─────
-    try:
-        tl_rows = (
-            sb.supabase.table("taught_lessons")
-            .select("id,date,period_code,class_id")
-            .eq("teacher_id", user_id)
-            .eq("institute_id", institute_id)
-            .is_("whiteboard_room_id", "null")
-            .execute()
-            .data or []
-        )
-
-        # Build whiteboard_room rows
-        wr_rows = []
-        for tl in tl_rows:
-            class_name = class_name_to_id and next(
-                (name for name, cid in class_name_to_id.items() if cid == tl.get("class_id")),
-                tl.get("period_code", "")
-            )
-            wr_rows.append({
-                "user_id": user_id,
-                "institute_id": institute_id,
-                "name": f"{tl['date']} {tl['period_code']}",
-                "context_type": "taught_lesson",
-                "context_id": tl["id"],
-                "storage_path": f"taught_lessons/{tl['id']}",
-            })
-
-        # Batch insert whiteboard_rooms
-        wr_ids: Dict[str, str] = {}  # context_id → room_id
-        for i in range(0, len(wr_rows), BATCH):
-            chunk = wr_rows[i : i + BATCH]
-            try:
-                res = sb.supabase.table("whiteboard_rooms").insert(chunk).execute()
-                for row in (res.data or []):
-                    if row.get("context_id"):
-                        wr_ids[row["context_id"]] = row["id"]
-            except Exception as e:
-                logger.warning(f"whiteboard_rooms batch insert: {e}")
-
-        # Update taught_lessons with their whiteboard_room_id
-        for context_id, room_id in wr_ids.items():
-            try:
-                sb.supabase.table("taught_lessons").update(
-                    {"whiteboard_room_id": room_id}
-                ).eq("id", context_id).execute()
-            except Exception as e:
-                logger.warning(f"whiteboard_room_id update for {context_id}: {e}")
-
-        rooms_created = len(wr_ids)
-    except Exception as e:
-        logger.warning(f"Whiteboard room creation failed (non-fatal): {e}")
-        rooms_created = 0
-
-    logger.info(f"Materialized {created} taught_lessons, {rooms_created} whiteboard_rooms for teacher {user_id}")
-    return {
-        "status": "ok",
-        "lessons_upserted": created,
-        "whiteboard_rooms_created": rooms_created,
-        "total_matches": len(to_upsert),
-    }
-
-
-# ─── Lesson timeline ─────────────────────────────────────────────────────────
-
-@router.get("/lessons")
-async def get_lessons(
-    week_start: Optional[str] = None,   # ISO date "YYYY-MM-DD" (Monday); defaults to current week
-    weeks: int = 1,                      # how many weeks to return (max 4)
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    """
-    Return taught_lessons for the teacher within a date range, grouped by date.
-    Defaults to the current week.
-    """
-    user_id = credentials.get("sub", "")
-    institute_id = _require_institute(user_id)
-    sb = _sb()
-
-    # Resolve week window
-    today = date.today()
-    if week_start:
-        try:
-            monday = datetime.strptime(week_start, "%Y-%m-%d").date()
-        except ValueError:
-            monday = today - timedelta(days=today.weekday())
-    else:
-        monday = today - timedelta(days=today.weekday())
-
-    weeks = min(max(weeks, 1), 4)
-    friday = monday + timedelta(weeks=weeks, days=4)
-
-    lessons = (
-        sb.supabase.table("taught_lessons")
-        .select(
-            "id,date,period_code,week_cycle,day_of_week,status,lesson_plan,notes,whiteboard_room_id,"
-            "class_id,academic_period_id"
-        )
-        .eq("teacher_id", user_id)
-        .eq("institute_id", institute_id)
-        .gte("date", str(monday))
-        .lte("date", str(friday))
-        .order("date")
-        .order("period_code")
-        .execute()
-        .data or []
-    )
-
-    # Enrich with period time and class name
-    period_ids = [l["academic_period_id"] for l in lessons if l.get("academic_period_id")]
-    class_ids = list({l["class_id"] for l in lessons if l.get("class_id")})
-
-    period_map: Dict[str, Dict] = {}
-    class_map: Dict[str, Dict] = {}
-
-    if period_ids:
-        prows = (
-            sb.supabase.table("academic_periods")
-            .select("id,period_name,start_time,end_time")
-            .in_("id", period_ids)
-            .execute()
-            .data or []
-        )
-        period_map = {r["id"]: r for r in prows}
-
-    if class_ids:
-        crows = (
-            sb.supabase.table("classes")
-            .select("id,name,class_code,subject,year_group")
-            .in_("id", class_ids)
-            .execute()
-            .data or []
-        )
-        class_map = {r["id"]: r for r in crows}
-
-    # Group by date
-    days_map: Dict[str, List[Dict]] = defaultdict(list)
-    for lesson in lessons:
-        p = period_map.get(lesson.get("academic_period_id", ""), {})
-        c = class_map.get(lesson.get("class_id", ""), {})
-        enriched = {
-            **lesson,
-            "period_name": p.get("period_name", lesson["period_code"]),
-            "start_time": p.get("start_time"),
-            "end_time": p.get("end_time"),
-            "class_name": c.get("name") or c.get("class_code"),
-            "subject": c.get("subject"),
-            "year_group": c.get("year_group"),
-        }
-        days_map[lesson["date"]].append(enriched)
-
-    # Build ordered list of days
-    days_list = []
-    current = monday
-    while current <= friday:
-        d_str = str(current)
-        days_list.append({
-            "date": d_str,
-            "day_of_week": current.strftime("%A"),
-            "is_today": current == today,
-            "lessons": days_map.get(d_str, []),
-        })
-        current += timedelta(days=1)
-        # Skip weekends
-        if current.weekday() >= 5:
-            current += timedelta(days=7 - current.weekday())
-
-    return {
-        "status": "ok",
-        "week_start": str(monday),
-        "week_end": str(friday),
-        "days": days_list,
-        "total_lessons": len(lessons),
-    }
-
-
-@router.get("/lessons/{lesson_id}")
-async def get_lesson(
-    lesson_id: str,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    sb = _sb()
-
-    res = (
-        sb.supabase.table("taught_lessons")
-        .select("*")
-        .eq("id", lesson_id)
-        .eq("teacher_id", user_id)
-        .single()
-        .execute()
-    )
-    if not res.data:
-        raise HTTPException(status_code=404, detail="Lesson not found")
-    return res.data
-
-
-@router.patch("/lessons/{lesson_id}")
-async def update_lesson(
-    lesson_id: str,
-    body: UpdateLessonRequest,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    """Teacher updates their own lesson content: plan, notes, status."""
-    user_id = credentials.get("sub", "")
-    sb = _sb()
-
-    updates: Dict[str, Any] = {}
-    if body.lesson_plan is not None:
-        updates["lesson_plan"] = body.lesson_plan
-    if body.notes is not None:
-        updates["notes"] = body.notes
-    if body.status is not None:
-        valid = {"planned", "in_progress", "completed", "cancelled", "substituted"}
-        if body.status not in valid:
-            raise HTTPException(status_code=400, detail=f"status must be one of {valid}")
-        updates["status"] = body.status
-
-    if not updates:
-        raise HTTPException(status_code=400, detail="Nothing to update")
-
-    updates["updated_at"] = datetime.utcnow().isoformat()
-    res = (
-        sb.supabase.table("taught_lessons")
-        .update(updates)
-        .eq("id", lesson_id)
-        .eq("teacher_id", user_id)
-        .execute()
-    )
-    if not res.data:
-        raise HTTPException(status_code=404, detail="Lesson not found or access denied")
-    return res.data[0]
-
-
-# ─── Student lesson view ──────────────────────────────────────────────────────
-
-@router.get("/student/lessons")
-async def get_student_lessons(
-    week_start: Optional[str] = None,
-    weeks: int = 1,
-    credentials: dict = Depends(SupabaseBearer()),
-) -> Dict[str, Any]:
-    """
-    Return taught_lessons for a student's enrolled classes for a date range.
-    Grouped by date, Mon-Fri only.
-    """
-    user_id = credentials.get("sub", "")
-    institute_id = _resolve_institute_id(user_id)
-    if not institute_id:
-        return {"status": "error", "message": "Not linked to a school"}
-    sb = _sb()
-
-    today = date.today()
-    if week_start:
-        try:
-            monday = datetime.strptime(week_start, "%Y-%m-%d").date()
-        except ValueError:
-            monday = today - timedelta(days=today.weekday())
-    else:
-        monday = today - timedelta(days=today.weekday())
-
-    weeks = min(max(weeks, 1), 4)
-    friday = monday + timedelta(weeks=weeks, days=4)
-
-    # Get student's active class memberships
-    memberships = (
-        sb.supabase.table("class_students")
-        .select("class_id")
-        .eq("student_id", user_id)
-        .eq("status", "active")
-        .execute()
-        .data or []
-    )
-    class_ids = [m["class_id"] for m in memberships]
-    if not class_ids:
-        days_list = []
-        current = monday
-        while current <= friday:
-            days_list.append({
-                "date": str(current),
-                "day_of_week": current.strftime("%A"),
-                "is_today": current == today,
-                "lessons": [],
-            })
-            current += timedelta(days=1)
-            if current.weekday() >= 5:
-                current += timedelta(days=7 - current.weekday())
-        return {"status": "ok", "week_start": str(monday), "days": days_list, "total_lessons": 0}
-
-    # Query taught_lessons for those classes
-    lessons = (
-        sb.supabase.table("taught_lessons")
-        .select(
-            "id,date,period_code,week_cycle,day_of_week,status,lesson_plan,notes,whiteboard_room_id,"
-            "class_id,academic_period_id,teacher_id"
-        )
-        .in_("class_id", class_ids)
-        .eq("institute_id", institute_id)
-        .gte("date", str(monday))
-        .lte("date", str(friday))
-        .order("date")
-        .order("period_code")
-        .execute()
-        .data or []
-    )
-
-    # Enrich with period times, class name, teacher name
-    period_ids = [l["academic_period_id"] for l in lessons if l.get("academic_period_id")]
-    lesson_class_ids = list({l["class_id"] for l in lessons if l.get("class_id")})
-    teacher_ids = list({l["teacher_id"] for l in lessons if l.get("teacher_id")})
-
-    period_map: Dict[str, Dict] = {}
-    class_map: Dict[str, Dict] = {}
-    teacher_map: Dict[str, Dict] = {}
-
-    if period_ids:
-        prows = (
-            sb.supabase.table("academic_periods")
-            .select("id,period_name,start_time,end_time")
-            .in_("id", period_ids)
-            .execute()
-            .data or []
-        )
-        period_map = {r["id"]: r for r in prows}
-
-    if lesson_class_ids:
-        crows = (
-            sb.supabase.table("classes")
-            .select("id,name,class_code,subject,year_group")
-            .in_("id", lesson_class_ids)
-            .execute()
-            .data or []
-        )
-        class_map = {r["id"]: r for r in crows}
-
-    if teacher_ids:
-        trows = (
-            sb.supabase.table("profiles")
-            .select("id,full_name,display_name")
-            .in_("id", teacher_ids)
-            .execute()
-            .data or []
-        )
-        teacher_map = {r["id"]: r for r in trows}
-
-    from collections import defaultdict
-    days_map: Dict[str, List[Dict]] = defaultdict(list)
-    for lesson in lessons:
-        p = period_map.get(lesson.get("academic_period_id", ""), {})
-        c = class_map.get(lesson.get("class_id", ""), {})
-        t = teacher_map.get(lesson.get("teacher_id", ""), {})
-        enriched = {
-            **lesson,
-            "period_name": p.get("period_name", lesson["period_code"]),
-            "start_time": p.get("start_time"),
-            "end_time": p.get("end_time"),
-            "class_name": c.get("name") or c.get("class_code"),
-            "subject": c.get("subject"),
-            "year_group": c.get("year_group"),
-            "teacher_name": t.get("display_name") or t.get("full_name"),
-        }
-        days_map[lesson["date"]].append(enriched)
-
-    days_list = []
-    current = monday
-    while current <= friday:
-        d_str = str(current)
-        days_list.append({
-            "date": d_str,
-            "day_of_week": current.strftime("%A"),
-            "is_today": current == today,
-            "lessons": days_map.get(d_str, []),
-        })
-        current += timedelta(days=1)
-        if current.weekday() >= 5:
-            current += timedelta(days=7 - current.weekday())
-
-    return {
-        "status": "ok",
-        "week_start": str(monday),
-        "week_end": str(friday),
-        "days": days_list,
-        "total_lessons": len(lessons),
-    }

routers/database/tools/tldraw_supabase_storage.py

@@ -11,117 +11,15 @@ load_dotenv(find_dotenv())
 import os
 import json
 import logging
-from fastapi import APIRouter, Depends, HTTPException, Query
-from typing import Dict, Any, Tuple
+from fastapi import APIRouter, HTTPException, Query
+from typing import Dict, Any
 
-from modules.auth.supabase_bearer import SupabaseBearer
-from modules.database.supabase.utils.client import SupabaseServiceRoleClient
-from modules.database.supabase.utils.storage import StorageError, StorageUser
+from modules.database.supabase.utils.storage import StorageAdmin
 from modules.logger_tool import initialise_logger
 
 router = APIRouter()
 logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
 
-ALLOWED_SNAPSHOT_BUCKETS = {"cc.public.snapshots"}
-PERSONAL_NODE_TYPES = {"User", "Teacher", "Developer", "SuperAdmin", "UserTeacherTimetable"}
-GLOBAL_READONLY_NODE_TYPES = {"CalendarYear", "CalendarMonth", "CalendarWeek", "CalendarDay", "CalendarTimeChunk"}
-
-
-def _sb() -> SupabaseServiceRoleClient:
-    return SupabaseServiceRoleClient()
-
-
-def _parse_snapshot_path(path: str) -> Tuple[str, str, str, str]:
-    """Parse and validate bucket/node_type/node_id into a storage object path."""
-    if not path:
-        raise HTTPException(status_code=400, detail="Path not provided")
-    path_parts = [part for part in path.split('/') if part]
-    if len(path_parts) != 3:
-        raise HTTPException(status_code=400, detail="Invalid path format. Expected: bucket/nodetype/node_id")
-
-    bucket, node_type, node_id = path_parts
-    if bucket not in ALLOWED_SNAPSHOT_BUCKETS:
-        raise HTTPException(status_code=403, detail="Snapshot bucket is not allowed")
-    if any(part in {".", ".."} or ".." in part for part in path_parts):
-        raise HTTPException(status_code=400, detail="Invalid path component")
-    if not node_type.replace("_", "").replace("-", "").isalnum():
-        raise HTTPException(status_code=400, detail="Invalid node type")
-    if not node_id.replace("_", "").replace("-", "").isalnum():
-        raise HTTPException(status_code=400, detail="Invalid node id")
-
-    return bucket, node_type, node_id, f"{node_type}/{node_id}/tldraw_file.json"
-
-
-def _user_scope(user_id: str) -> Dict[str, Any]:
-    """Resolve Supabase/Neo4j scope for the authenticated user."""
-    scope: Dict[str, Any] = {
-        "user_id": user_id,
-        "teacher_db": f"cc.users.teacher.{user_id.replace('-', '')}" if user_id else "",
-        "institute_id": "",
-        "institute_db": "",
-        "curriculum_db": "",
-    }
-    if not user_id:
-        return scope
-    try:
-        sb = _sb()
-        prof = sb.supabase.table("profiles").select("school_id").eq("id", user_id).single().execute()
-        school_id = str((prof.data or {}).get("school_id") or "")
-        scope["institute_id"] = school_id
-        if school_id:
-            inst = sb.supabase.table("institutes").select("neo4j_uuid_string").eq("id", school_id).single().execute()
-            neo4j_uuid = (inst.data or {}).get("neo4j_uuid_string")
-            if neo4j_uuid:
-                scope["institute_db"] = f"cc.institutes.{neo4j_uuid}"
-                scope["curriculum_db"] = f"cc.institutes.{neo4j_uuid}.curriculum"
-    except Exception as exc:
-        logger.warning(f"Could not resolve TLDraw storage scope for user {user_id}: {exc}")
-    return scope
-
-
-def _authorize_snapshot_path(path: str, db_name: str, credentials: Dict[str, Any], write: bool) -> Tuple[str, str, str, str]:
-    """Authorize TLDraw snapshot access before touching Supabase Storage."""
-    user_id = credentials.get("sub", "")
-    if not user_id:
-        raise HTTPException(status_code=403, detail="Could not extract user_id from token")
-    bucket, node_type, node_id, file_path = _parse_snapshot_path(path)
-    scope = _user_scope(user_id)
-    allowed_dbs = {db for db in (scope["teacher_db"], scope["institute_db"], scope["curriculum_db"]) if db}
-
-    if node_type in PERSONAL_NODE_TYPES:
-        if node_id == user_id or (db_name and db_name == scope["teacher_db"]):
-            return bucket, node_type, node_id, file_path
-        raise HTTPException(status_code=403, detail="Snapshot path is outside the authenticated user's workspace")
-
-    if node_type in GLOBAL_READONLY_NODE_TYPES and db_name == "classroomcopilot":
-        if write:
-            raise HTTPException(status_code=403, detail="Global calendar snapshots are read-only")
-        return bucket, node_type, node_id, file_path
-
-    # Institute/curriculum snapshots must be accessed through the caller's institute DB.
-    if db_name and db_name in allowed_dbs:
-        return bucket, node_type, node_id, file_path
-
-    raise HTTPException(status_code=403, detail="Snapshot path is outside the authenticated user's tenant")
-
-
-def _storage_for_user(credentials: Dict[str, Any]) -> StorageUser:
-    access_token = credentials.get("_access_token")
-    if not access_token:
-        raise HTTPException(status_code=403, detail="User access token is required for storage access")
-    return StorageUser(user_id=credentials.get("sub"), access_token=access_token)
-
-
-def _is_valid_tldraw_snapshot(snapshot_data: Any) -> bool:
-    if not isinstance(snapshot_data, dict):
-        return False
-    if not ("document" in snapshot_data and "session" in snapshot_data):
-        return False
-    document = snapshot_data.get("document")
-    if isinstance(document, dict) and "schema" in document:
-        return True
-    return "schemaVersion" in snapshot_data
-
 def create_default_tldraw_content():
     """Create default tldraw content structure."""
     return {
@@ -203,8 +101,7 @@ def create_default_tldraw_content():
 @router.get("/get_tldraw_node_file")
 async def read_tldraw_node_file_from_supabase(
     path: str = Query(..., description="Supabase Storage path (e.g., 'cc.public.snapshots/User/user_id')"),
-    db_name: str = Query(..., description="Database name for context"),
-    credentials: dict = Depends(SupabaseBearer()),
+    db_name: str = Query(..., description="Database name for context")
 ):
     """
     Load TLDraw snapshot from Supabase Storage.
@@ -219,9 +116,26 @@ async def read_tldraw_node_file_from_supabase(
     logger.debug(f"Reading tldraw file from Supabase Storage for path: {path}")
     logger.debug(f"Database name: {db_name}")
     
+    if not path:
+        raise HTTPException(status_code=400, detail="Path not provided")
+    
     try:
-        bucket, node_type, node_id, file_path = _authorize_snapshot_path(path, db_name, credentials, write=False)
-        storage = _storage_for_user(credentials)
+        # Initialize Supabase Storage
+        storage = StorageAdmin()
+        
+        # Parse the path to extract bucket and file path
+        # Expected format: "cc.public.snapshots/User/user_id" or "cc.public.snapshots/Teacher/teacher_id"
+        path_parts = path.split('/')
+        if len(path_parts) < 3:
+            raise HTTPException(status_code=400, detail="Invalid path format. Expected: bucket/nodetype/node_id")
+        
+        bucket = path_parts[0]  # e.g., "cc.public.snapshots"
+        node_type = path_parts[1]  # e.g., "User", "Teacher"
+        node_id = path_parts[2]  # e.g., "cbc309e5-4029-4c34-aab7-0aa33c563cd0"
+        
+        # Construct the file path in Supabase Storage
+        # Format: nodetype/node_id/tldraw_file.json
+        file_path = f"{node_type}/{node_id}/tldraw_file.json"
         
         logger.debug(f"Bucket: {bucket}")
         logger.debug(f"File path: {file_path}")
@@ -233,17 +147,29 @@ async def read_tldraw_node_file_from_supabase(
             # Parse JSON data
             try:
                 snapshot_data = json.loads(file_data.decode('utf-8'))
-            except (UnicodeDecodeError, json.JSONDecodeError) as e:
-                logger.warning(f"Malformed TLDraw snapshot {file_path}; returning default content: {e}")
-                return create_default_tldraw_content()
-
-            logger.info(f"Successfully loaded tldraw snapshot from Supabase Storage: {file_path}")
-            if _is_valid_tldraw_snapshot(snapshot_data):
-                return snapshot_data
-            logger.warning(f"Snapshot data from {file_path} is missing required TLDraw structure. Using default structure.")
-            return create_default_tldraw_content()
+                logger.info(f"Successfully loaded tldraw snapshot from Supabase Storage: {file_path}")
                 
-        except StorageError as e:
+                # Ensure the snapshot has the correct structure for TLDraw
+                if isinstance(snapshot_data, dict) and 'document' in snapshot_data and 'session' in snapshot_data:
+                    # Check if it has the new format (schemaVersion in document.schema)
+                    if 'document' in snapshot_data and isinstance(snapshot_data['document'], dict) and 'schema' in snapshot_data['document']:
+                        return snapshot_data
+                    # Check if it has the old format (schemaVersion at root level)
+                    elif 'schemaVersion' in snapshot_data:
+                        return snapshot_data
+                    else:
+                        # Use default structure if schema is missing
+                        logger.warning(f"Snapshot data from {file_path_in_bucket} is missing schemaVersion. Using default structure.")
+                        return create_default_tldraw_content()
+                else:
+                    # Use default structure if basic structure is missing
+                    logger.warning(f"Snapshot data from {file_path_in_bucket} is missing top-level TLDraw keys. Using default structure.")
+                    return create_default_tldraw_content()
+            except json.JSONDecodeError as e:
+                logger.error(f"Failed to parse JSON from Supabase Storage file: {e}")
+                raise HTTPException(status_code=500, detail="Invalid JSON in file")
+                
+        except Exception as e:
             # File doesn't exist, create default content
             logger.info(f"File not found in Supabase Storage, creating default tldraw content: {file_path}")
             
@@ -273,8 +199,7 @@ async def read_tldraw_node_file_from_supabase(
 async def set_tldraw_node_file_in_supabase(
     path: str = Query(..., description="Supabase Storage path (e.g., 'cc.public.snapshots/User/user_id')"),
     db_name: str = Query(..., description="Database name for context"),
-    data: Dict[str, Any] = None,
-    credentials: dict = Depends(SupabaseBearer()),
+    data: Dict[str, Any] = None
 ):
     """
     Save TLDraw snapshot to Supabase Storage.
@@ -290,12 +215,27 @@ async def set_tldraw_node_file_in_supabase(
     logger.debug(f"Saving tldraw file to Supabase Storage for path: {path}")
     logger.debug(f"Database name: {db_name}")
     
+    if not path:
+        raise HTTPException(status_code=400, detail="Path not provided")
+    
     if not data:
         raise HTTPException(status_code=400, detail="Data not provided")
     
     try:
-        bucket, node_type, node_id, file_path = _authorize_snapshot_path(path, db_name, credentials, write=True)
-        storage = _storage_for_user(credentials)
+        # Initialize Supabase Storage
+        storage = StorageAdmin()
+        
+        # Parse the path to extract bucket and file path
+        path_parts = path.split('/')
+        if len(path_parts) < 3:
+            raise HTTPException(status_code=400, detail="Invalid path format. Expected: bucket/nodetype/node_id")
+        
+        bucket = path_parts[0]  # e.g., "cc.public.snapshots"
+        node_type = path_parts[1]  # e.g., "User", "Teacher"
+        node_id = path_parts[2]  # e.g., "cbc309e5-4029-4c34-aab7-0aa33c563cd0"
+        
+        # Construct the file path in Supabase Storage
+        file_path = f"{node_type}/{node_id}/tldraw_file.json"
         
         logger.debug(f"Bucket: {bucket}")
         logger.debug(f"File path: {file_path}")

routers/database/tools/user_init_router.py

@@ -1,81 +0,0 @@
-import os
-from typing import Dict, Any
-from fastapi import APIRouter, Depends
-from modules.logger_tool import initialise_logger
-from modules.auth.supabase_bearer import SupabaseBearer
-from modules.database.services.provisioning_service import ProvisioningService
-import modules.database.tools.neo4j_driver_tools as driver_tools
-import modules.database.schemas.nodes.users as user_nodes
-import modules.database.tools.neontology_tools as neon
-
-logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
-router = APIRouter()
-
-
-def _teacher_db(user_id: str) -> str:
-    return f"cc.users.teacher.{user_id.replace('-', '')}"
-
-
-def _ensure_journal_planner(user_id: str, teacher_db: str) -> None:
-    neon.init_neontology_connection()
-    try:
-        with driver_tools.get_session(database=teacher_db) as session:
-            has_journal = session.run("MATCH (j:Journal) RETURN count(j) AS n").single()["n"] > 0
-            has_planner = session.run("MATCH (p:Planner) RETURN count(p) AS n").single()["n"] > 0
-
-        if not has_journal:
-            journal = user_nodes.JournalNode(
-                uuid_string=f"{user_id}_journal",
-                node_storage_path=f"users/{user_id}/nodes/journal",
-                user_id=user_id,
-            )
-            neon.create_or_merge_neontology_node(journal, database=teacher_db, operation='merge')
-            logger.info(f"Created Journal node for {user_id}")
-
-        if not has_planner:
-            planner = user_nodes.PlannerNode(
-                uuid_string=f"{user_id}_planner",
-                node_storage_path=f"users/{user_id}/nodes/planner",
-                user_id=user_id,
-            )
-            neon.create_or_merge_neontology_node(planner, database=teacher_db, operation='merge')
-            logger.info(f"Created Planner node for {user_id}")
-    finally:
-        neon.close_neontology_connection()
-
-
-@router.post("/init")
-async def init_user(credentials: dict = Depends(SupabaseBearer())) -> Dict[str, Any]:
-    user_id = credentials.get("sub", "")
-    if not user_id:
-        return {"status": "error", "message": "No user ID in token"}
-
-    db = _teacher_db(user_id)
-
-    # Fast path: check if already fully provisioned
-    try:
-        with driver_tools.get_session(database=db) as session:
-            r = session.run(
-                "MATCH (u:User) "
-                "OPTIONAL MATCH (j:Journal) "
-                "OPTIONAL MATCH (p:Planner) "
-                "RETURN count(u) AS u, count(j) AS j, count(p) AS p"
-            ).single()
-            if r and r["u"] > 0 and r["j"] > 0 and r["p"] > 0:
-                logger.debug(f"User {user_id} already initialized — fast path")
-                return {"status": "ok", "initialized": True, "teacher_db": db}
-    except Exception:
-        pass  # DB doesn't exist yet — fall through to full provisioning
-
-    # Full provisioning
-    try:
-        logger.info(f"Provisioning user {user_id}...")
-        service = ProvisioningService()
-        result = service.ensure_user(user_id)
-        user_db = result.get("user_db_name") or db
-        _ensure_journal_planner(user_id, user_db)
-        logger.info(f"User {user_id} provisioned successfully: {user_db}")
-        return {"status": "ok", "initialized": True, "teacher_db": user_db}
-    except Exception as e:
-        logger.error(f"User init failed for {user_id}: {e}")
-        return {"status": "error", "message": str(e)}

routers/exam/__init__.py

@@ -1,20 +0,0 @@
-"""Exam-marker API package (/api/exam/).
-
-A clean top-level router group (R5.1/E5), deliberately NOT nested under /database/. Every
-endpoint authenticates the JWT and calls Supabase as-the-user so the RLS in
-volumes/db/cc/72-exam-marker.sql is enforced (spec E1/E2 fixes).
-"""
-from fastapi import APIRouter
-
-from routers.exam.templates import router as templates_router
-from routers.exam.batches import router as batches_router
-from routers.exam.bank import router as bank_router
-from routers.exam.corpus import router as corpus_router
-
-router = APIRouter()
-router.include_router(templates_router)
-router.include_router(batches_router)
-router.include_router(bank_router)
-router.include_router(corpus_router)
-
-__all__ = ["router"]

routers/exam/bank.py

@@ -1,152 +0,0 @@
-"""Question bank + custom-paper assembly (/api/exam/bank, /api/exam/custom-papers) — mode 3.
-
-Build-your-own-from-the-spec. The BANK is a browsable index over the leaf questions across the templates
-the caller can see (RLS-scoped, institute-wide); a CUSTOM PAPER is a normal exam_template whose questions
-are COPIES of the selected bank questions, so it flows through the existing setup / marking / projection
-pipeline unchanged. Copy-on-assemble (v1) avoids decoupling question identity from a template; a shared-
-question model is Phase-2. Design: ~/cc/ideas/2026-07-02-mode3-question-bank-design.md.
-
-All access is as-the-user (E1): the bank query returns only questions in templates RLS lets the caller see,
-and a custom paper is created owned by the caller + institute-scoped (writes pass the same RLS with-check).
-"""
-from __future__ import annotations
-
-import os
-import uuid
-from typing import Any, Dict, List, Optional
-
-from fastapi import APIRouter, Depends, HTTPException
-from pydantic import BaseModel
-
-from modules.database.services.exam_projection import project_template_safe
-from modules.logger_tool import initialise_logger
-from routers.exam.dependencies import ExamContext, get_exam_context
-
-logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), "default", True)
-
-router = APIRouter()
-
-
-def _rows(result: Any) -> List[Dict[str, Any]]:
-    data = getattr(result, "data", None)
-    if not data:
-        return []
-    return data if isinstance(data, list) else [data]
-
-
-def _first(result: Any) -> Optional[Dict[str, Any]]:
-    rows = _rows(result)
-    return rows[0] if rows else None
-
-
-@router.get("/bank")
-async def list_bank(
-    spec_ref: Optional[str] = None,
-    subject: Optional[str] = None,
-    ctx: ExamContext = Depends(get_exam_context),
-) -> Dict[str, Any]:
-    """Leaf questions across the caller's institute templates, with their source-paper context.
-
-    Filter by `spec_ref` (spec point) and/or `subject`. Facets over the full visible set drive the UI
-    filters (so filtering by one axis doesn't hide the others' options). This is the spec-planning surface:
-    a teacher picks a spec point → gets every question tagged it across their papers → assembles a paper.
-    """
-    sel = (
-        "id, template_id, label, max_marks, answer_type, spec_ref, bounds, page, "
-        "exam_templates(id, title, subject, exam_code)"
-    )
-    query = ctx.supabase.table("exam_questions").select(sel).eq("is_container", False)
-    if spec_ref:
-        query = query.eq("spec_ref", spec_ref)
-    rows = _rows(query.execute())
-
-    facets_spec: Dict[str, int] = {}
-    facets_subject: Dict[str, int] = {}
-    items: List[Dict[str, Any]] = []
-    for r in rows:
-        tmpl = r.get("exam_templates") or {}
-        subj = tmpl.get("subject")
-        if r.get("spec_ref"):
-            facets_spec[r["spec_ref"]] = facets_spec.get(r["spec_ref"], 0) + 1
-        if subj:
-            facets_subject[subj] = facets_subject.get(subj, 0) + 1
-        if subject and subj != subject:
-            continue
-        items.append({
-            "id": r["id"], "template_id": r["template_id"], "label": r.get("label"),
-            "max_marks": r.get("max_marks"), "answer_type": r.get("answer_type"),
-            "spec_ref": r.get("spec_ref"), "bounds": r.get("bounds"), "page": r.get("page"),
-            "paper": {"id": tmpl.get("id"), "title": tmpl.get("title"),
-                      "subject": subj, "exam_code": tmpl.get("exam_code")},
-        })
-    return {"questions": items, "n": len(items),
-            "facets": {"spec_ref": facets_spec, "subject": facets_subject}}
-
-
-class CustomPaperRequest(BaseModel):
-    title: str
-    subject: Optional[str] = None
-    institute_id: Optional[str] = None
-    question_ids: List[str]
-
-
-@router.post("/custom-papers")
-async def create_custom_paper(
-    body: CustomPaperRequest,
-    ctx: ExamContext = Depends(get_exam_context),
-) -> Dict[str, Any]:
-    """Assemble the selected bank questions into a new template (copy-on-assemble), then project it."""
-    if not body.question_ids:
-        raise HTTPException(status_code=400, detail="question_ids is required")
-    institute_id = ctx.resolve_institute(body.institute_id)
-
-    # RLS makes this return only leaf questions the caller may see; unknown/forbidden ids are silently dropped.
-    srcs = _rows(
-        ctx.supabase.table("exam_questions").select("*")
-        .in_("id", body.question_ids).eq("is_container", False).execute()
-    )
-    by_id = {s["id"]: s for s in srcs}
-    ordered = [by_id[qid] for qid in body.question_ids if qid in by_id]  # preserve the caller's order
-    if not ordered:
-        raise HTTPException(status_code=404, detail="No accessible questions for the given ids")
-
-    template_id = str(uuid.uuid4())
-    ctx.supabase.table("exam_templates").insert({
-        "id": template_id, "title": body.title, "subject": body.subject,
-        "institute_id": institute_id, "teacher_id": ctx.user_id, "status": "draft",
-    }).execute()
-
-    id_map: Dict[str, str] = {}
-    q_rows: List[Dict[str, Any]] = []
-    for order, s in enumerate(ordered):
-        new_id = str(uuid.uuid4())
-        id_map[s["id"]] = new_id
-        q_rows.append({
-            "id": new_id, "template_id": template_id, "parent_id": None,
-            "label": s.get("label"), "order": order, "max_marks": s.get("max_marks") or 0,
-            "answer_type": s.get("answer_type"), "mcq_options": s.get("mcq_options"),
-            "mark_scheme": s.get("mark_scheme") or {}, "is_container": False,
-            "spec_ref": s.get("spec_ref"), "bounds": s.get("bounds"), "page": s.get("page"),
-            "source": "manual", "confirmed": True,
-        })
-    ctx.supabase.table("exam_questions").insert(q_rows).execute()
-
-    # Copy each selected question's response areas onto its new copy (geometry travels with the question).
-    ra_rows: List[Dict[str, Any]] = []
-    if id_map:
-        for ra in _rows(
-            ctx.supabase.table("exam_response_areas").select("*")
-            .in_("question_id", list(id_map.keys())).execute()
-        ):
-            ra_rows.append({
-                "id": str(uuid.uuid4()), "question_id": id_map[ra["question_id"]], "template_id": template_id,
-                "page": ra.get("page"), "bounds": ra.get("bounds"), "kind": ra.get("kind"),
-                "response_form": ra.get("response_form"), "context_type": ra.get("context_type"),
-                "source": "manual", "confirmed": True,
-            })
-        if ra_rows:
-            ctx.supabase.table("exam_response_areas").insert(ra_rows).execute()
-
-    project_template_safe(template_id)  # a custom paper is a normal paper in the graph
-    return {"id": template_id, "title": body.title, "subject": body.subject,
-            "n_questions": len(q_rows), "n_response_areas": len(ra_rows)}

routers/exam/batches.py

@@ -1,404 +0,0 @@
-"""Marking batches, scans, marks, results & CSV (/api/exam/batches..., /api/exam/marks/...) — S4-6.
-
-As with templates, all user-facing access is as-the-user (RLS-enforced; E1). A batch is owned by
-the teacher who creates it (R2.4); colleagues in the same institute can read it
-(marking_batches_read), a teacher in another institute cannot (→ 404, IDOR-safe).
-
-Roster→cohort (R4.3/A7): creating a batch from a class materialises one student_submissions row
-per active enrollee (status='absent'), so every enrolled student is present in results/CSV from
-the start and a no-show is never silently dropped. The roster ids are read AS THE USER from
-class_students (cs_read requires the caller to teach/admin the class); only the display names are
-resolved via service role (profiles is deny-all as-user, E4 — see resolve_student_names).
-
-Scans (R2.3/E3): the upload endpoint enforces a max size and validates that the bytes are a PDF
-before storing. QR-decode + automatic student-matching is a follow-on (no QR'd fixtures exist
-until the PrintGenerator card); v1 supports explicit (manual) and ordered matching.
-"""
-from __future__ import annotations
-
-import csv
-import io
-import os
-import uuid
-from typing import Any, Dict, List, Optional
-
-from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile
-from fastapi.responses import Response
-
-from modules.database.supabase.utils.client import SupabaseServiceRoleClient
-from modules.database.supabase.utils.storage import StorageAdmin
-from modules.logger_tool import initialise_logger
-from routers.exam.dependencies import ExamContext, get_exam_context, resolve_student_names
-from routers.exam.schemas import CreateBatchRequest, MarkUpsertRequest
-
-logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), "default", True)
-
-router = APIRouter()
-
-# E3: bound the upload so a 36-page scan batch can't exhaust memory / be a DoS vector.
-MAX_SCAN_BYTES = int(os.getenv("EXAM_SCAN_MAX_BYTES", str(50 * 1024 * 1024)))  # 50 MB default
-SCANS_BUCKET = os.getenv("EXAM_SCANS_BUCKET", "cc.users")
-SCANS_PREFIX = "exam-submissions"
-
-
-# ─── helpers ─────────────────────────────────────────────────────────────────
-
-def _rows(result: Any) -> List[Dict[str, Any]]:
-    data = getattr(result, "data", None)
-    if not data:
-        return []
-    return data if isinstance(data, list) else [data]
-
-
-def _first(result: Any) -> Optional[Dict[str, Any]]:
-    rows = _rows(result)
-    return rows[0] if rows else None
-
-
-def _fetch_batch_or_404(ctx: ExamContext, batch_id: str) -> Dict[str, Any]:
-    row = _first(ctx.supabase.table("marking_batches").select("*").eq("id", batch_id).limit(1).execute())
-    if not row:
-        raise HTTPException(status_code=404, detail="Batch not found")
-    return row
-
-
-def _require_owner(ctx: ExamContext, batch: Dict[str, Any]) -> None:
-    if batch.get("teacher_id") != ctx.user_id:
-        raise HTTPException(status_code=403, detail="Only the batch owner can modify it")
-
-
-# ─── batches ─────────────────────────────────────────────────────────────────
-
-@router.post("/batches")
-async def create_batch(
-    body: CreateBatchRequest,
-    ctx: ExamContext = Depends(get_exam_context),
-) -> Dict[str, Any]:
-    # The batch inherits the template's institute; reading the template as-user also proves the
-    # caller may see it (RLS) — an unseeable template → 404.
-    template = _first(
-        ctx.supabase.table("exam_templates").select("id, institute_id").eq("id", body.template_id).limit(1).execute()
-    )
-    if not template:
-        raise HTTPException(status_code=404, detail="Template not found")
-
-    batch_row = {
-        "template_id": body.template_id,
-        "class_id": body.class_id,
-        "institute_id": template["institute_id"],
-        "teacher_id": ctx.user_id,
-        "title": body.title,
-        "status": "open",
-    }
-    batch_row = {k: v for k, v in batch_row.items() if v is not None}
-    batch = _first(ctx.supabase.table("marking_batches").insert(batch_row).execute())
-    if not batch:
-        raise HTTPException(status_code=500, detail="Failed to create batch")
-    batch_id = batch["id"]
-
-    seeded = 0
-    if body.class_id:
-        # Roster read is AS THE USER → cs_read requires the caller to teach/admin the class.
-        roster = _rows(
-            ctx.supabase.table("class_students")
-            .select("student_id")
-            .eq("class_id", body.class_id)
-            .eq("status", "active")
-            .execute()
-        )
-        student_ids = [r["student_id"] for r in roster if r.get("student_id")]
-        names = resolve_student_names(student_ids)
-        if student_ids:
-            sub_rows = [
-                {
-                    "batch_id": batch_id,
-                    "student_id": sid,
-                    "student_name": names.get(sid),
-                    "status": "absent",  # A7: present in results until a scan is matched
-                }
-                for sid in student_ids
-            ]
-            ctx.supabase.table("student_submissions").insert(sub_rows).execute()
-            seeded = len(sub_rows)
-
-    logger.info(f"Marking batch {batch_id} created by {ctx.user_id}; {seeded} roster submissions seeded")
-    return {**batch, "submission_count": seeded}
-
-
-@router.get("/batches")
-async def list_batches(
-    include_archived: bool = False,
-    template_id: Optional[str] = None,
-    ctx: ExamContext = Depends(get_exam_context),
-) -> Dict[str, Any]:
-    q = ctx.supabase.table("marking_batches").select("*")
-    if template_id:
-        q = q.eq("template_id", template_id)
-    if not include_archived:
-        q = q.neq("status", "archived")
-    return {"batches": _rows(q.order("created_at", desc=True).execute())}
-
-
-@router.get("/batches/{batch_id}/queue")
-async def batch_queue(
-    batch_id: str,
-    ctx: ExamContext = Depends(get_exam_context),
-) -> Dict[str, Any]:
-    batch = _fetch_batch_or_404(ctx, batch_id)
-    submissions = _rows(
-        ctx.supabase.table("student_submissions").select("*").eq("batch_id", batch_id).execute()
-    )
-    marks = _rows(ctx.supabase.table("mark_entries").select("submission_id").eq("batch_id", batch_id).execute())
-    marked_counts: Dict[str, int] = {}
-    for m in marks:
-        sid = m.get("submission_id")
-        marked_counts[sid] = marked_counts.get(sid, 0) + 1
-
-    enriched = [{**s, "mark_entry_count": marked_counts.get(s["id"], 0)} for s in submissions]
-    progress = {
-        "total": len(submissions),
-        "absent": sum(1 for s in submissions if s.get("status") == "absent"),
-        "complete": sum(1 for s in submissions if s.get("status") == "complete"),
-        "in_progress": sum(1 for s in submissions if s.get("status") in ("matched", "marking")),
-    }
-    return {"batch": batch, "submissions": enriched, "progress": progress}
-
-
-# ─── results & CSV (A7) ──────────────────────────────────────────────────────
-
-def _assemble_results(ctx: ExamContext, batch: Dict[str, Any]) -> Dict[str, Any]:
-    batch_id = batch["id"]
-    questions = _rows(
-        ctx.supabase.table("exam_questions")
-        .select("id, label, max_marks, order")
-        .eq("template_id", batch["template_id"])
-        .order("order")
-        .execute()
-    )
-    submissions = _rows(
-        ctx.supabase.table("student_submissions").select("*").eq("batch_id", batch_id).execute()
-    )
-    marks = _rows(ctx.supabase.table("mark_entries").select("*").eq("batch_id", batch_id).execute())
-
-    by_sub: Dict[str, Dict[str, float]] = {}
-    for m in marks:
-        by_sub.setdefault(m["submission_id"], {})[m["question_id"]] = m.get("awarded_marks")
-
-    results = []
-    for s in submissions:  # every submission incl. absent → A7
-        sub_marks = by_sub.get(s["id"], {})
-        # Blank total ONLY for a genuine no-show (absent AND nothing marked). A student with any
-        # mark gets a real total regardless of status; a present-but-unmarked student totals 0.
-        if sub_marks:
-            total = sum(v or 0 for v in sub_marks.values())
-        elif s.get("status") == "absent":
-            total = None
-        else:
-            total = 0
-        results.append({
-            "submission_id": s["id"],
-            "student_id": s.get("student_id"),
-            "student_name": s.get("student_name"),
-            "status": s.get("status"),
-            "marks": {qid: sub_marks.get(qid) for qid in (q["id"] for q in questions)},
-            "total": total,
-        })
-    return {"batch": batch, "questions": questions, "results": results}
-
-
-@router.get("/batches/{batch_id}/results")
-async def batch_results(
-    batch_id: str,
-    ctx: ExamContext = Depends(get_exam_context),
-) -> Dict[str, Any]:
-    batch = _fetch_batch_or_404(ctx, batch_id)
-    return _assemble_results(ctx, batch)
-
-
-@router.get("/batches/{batch_id}/csv")
-async def batch_csv(
-    batch_id: str,
-    ctx: ExamContext = Depends(get_exam_context),
-) -> Response:
-    batch = _fetch_batch_or_404(ctx, batch_id)
-    data = _assemble_results(ctx, batch)
-    questions = data["questions"]
-
-    buf = io.StringIO()
-    writer = csv.writer(buf)
-    writer.writerow(["student_name", "student_id", "status"] + [q["label"] for q in questions] + ["total"])
-    for r in data["results"]:
-        # Absent students: blank marks + blank total, but the row is ALWAYS present (A7).
-        cells = [
-            "" if r["marks"].get(q["id"]) is None else r["marks"].get(q["id"])
-            for q in questions
-        ]
-        total = "" if r["total"] is None else r["total"]
-        writer.writerow([r.get("student_name") or "", r.get("student_id") or "", r.get("status")] + cells + [total])
-
-    return Response(
-        content=buf.getvalue(),
-        media_type="text/csv",
-        headers={"Content-Disposition": f'attachment; filename="batch-{batch_id}.csv"'},
-    )
-
-
-# ─── marks ───────────────────────────────────────────────────────────────────
-
-def _advance_completion(ctx: ExamContext, batch_id: str, submission_id: str) -> None:
-    """After a mark upsert, advance statuses: a submission with a mark for every markable (leaf)
-    question → complete; a batch whose every non-absent submission is complete → complete. Nothing
-    here regresses a status (only promotes to complete), so it is safe to run on every upsert."""
-    batch = _first(
-        ctx.supabase.table("marking_batches").select("id, template_id, status").eq("id", batch_id).limit(1).execute()
-    )
-    if not batch:
-        return
-    markable = {
-        q["id"] for q in _rows(
-            ctx.supabase.table("exam_questions").select("id, is_container").eq("template_id", batch["template_id"]).execute()
-        ) if not q.get("is_container")
-    }
-    if not markable:
-        return
-    marked = {
-        m["question_id"] for m in _rows(
-            ctx.supabase.table("mark_entries").select("question_id").eq("submission_id", submission_id).execute()
-        )
-    }
-    if not markable.issubset(marked):
-        return
-    ctx.supabase.table("student_submissions").update({"status": "complete"}).eq("id", submission_id).execute()
-    subs = _rows(ctx.supabase.table("student_submissions").select("status").eq("batch_id", batch_id).execute())
-    active = [s for s in subs if s.get("status") != "absent"]
-    if active and all(s.get("status") == "complete" for s in active) and batch.get("status") != "complete":
-        ctx.supabase.table("marking_batches").update({"status": "complete"}).eq("id", batch_id).execute()
-
-
-@router.put("/marks/{mark_id}")
-async def upsert_mark(
-    mark_id: str,
-    body: MarkUpsertRequest,
-    ctx: ExamContext = Depends(get_exam_context),
-) -> Dict[str, Any]:
-    # Derive batch_id from the submission (as-user read → also enforces the caller owns the batch
-    # the submission belongs to). The client never supplies the RLS scoping key directly.
-    submission = _first(
-        ctx.supabase.table("student_submissions").select("id, batch_id, status").eq("id", body.submission_id).limit(1).execute()
-    )
-    if not submission:
-        raise HTTPException(status_code=404, detail="Submission not found")
-
-    # Reject an award that exceeds the question's max (only when a max is actually set; 0/None means
-    # "not scored yet" for AI/unmapped questions, so we can't validate those).
-    question = _first(
-        ctx.supabase.table("exam_questions").select("id, max_marks").eq("id", body.question_id).limit(1).execute()
-    )
-    max_marks = (question or {}).get("max_marks")
-    if isinstance(max_marks, (int, float)) and max_marks > 0 and body.awarded_marks is not None and body.awarded_marks > max_marks:
-        raise HTTPException(status_code=422, detail=f"awarded_marks {body.awarded_marks} exceeds max_marks {max_marks} for this question")
-
-    row = {
-        "id": mark_id,
-        "submission_id": body.submission_id,
-        "question_id": body.question_id,
-        "batch_id": submission["batch_id"],
-        "awarded_marks": body.awarded_marks,
-        "marked_by": "teacher",
-    }
-    if body.mark_scheme_detail is not None:
-        row["mark_scheme_detail"] = body.mark_scheme_detail
-    if body.annotation_shape_ids is not None:
-        row["annotation_shape_ids"] = body.annotation_shape_ids
-    if body.comment is not None:
-        row["comment"] = body.comment
-    if body.confirmed is not None:
-        row["confirmed"] = body.confirmed
-
-    upserted = _first(ctx.supabase.table("mark_entries").upsert(row).execute())
-    if not upserted:
-        raise HTTPException(status_code=500, detail="Failed to upsert mark")
-
-    # A marked student is, by definition, not absent — advance the submission out of the
-    # no-submission states so results/queue reflect that marking has started.
-    if submission.get("status") in ("absent", "unmatched"):
-        ctx.supabase.table("student_submissions").update({"status": "marking"}).eq("id", body.submission_id).execute()
-
-    # Promote the submission/batch to complete once every markable question has a mark.
-    _advance_completion(ctx, submission["batch_id"], body.submission_id)
-
-    return upserted
-
-
-# ─── scans (R2.3 / E3) ───────────────────────────────────────────────────────
-
-@router.post("/batches/{batch_id}/scans")
-async def upload_scan(
-    batch_id: str,
-    file: UploadFile = File(...),
-    student_id: Optional[str] = Form(default=None),
-    matching_method: str = Form(default="manual"),
-    ctx: ExamContext = Depends(get_exam_context),
-) -> Dict[str, Any]:
-    batch = _fetch_batch_or_404(ctx, batch_id)
-    _require_owner(ctx, batch)
-
-    # E3: validate MIME (client-declared) before reading the body.
-    if (file.content_type or "").lower() not in ("application/pdf", "application/x-pdf"):
-        raise HTTPException(status_code=415, detail="Only application/pdf scans are accepted")
-
-    # E3: read with a hard size ceiling instead of buffering an unbounded upload.
-    chunks: List[bytes] = []
-    total = 0
-    while True:
-        chunk = await file.read(1024 * 1024)
-        if not chunk:
-            break
-        total += len(chunk)
-        if total > MAX_SCAN_BYTES:
-            raise HTTPException(status_code=413, detail=f"Scan exceeds max size ({MAX_SCAN_BYTES} bytes)")
-        chunks.append(chunk)
-    data = b"".join(chunks)
-    # E3: content-sniff — declared type can be spoofed; require the PDF magic header.
-    if not data.startswith(b"%PDF-"):
-        raise HTTPException(status_code=415, detail="Uploaded file is not a valid PDF")
-
-    # Store via service role (documented): no submissions-bucket storage RLS exists yet; the
-    # endpoint already authorised the caller as the batch owner above.
-    storage_path = f"{SCANS_PREFIX}/{batch_id}/{uuid.uuid4()}.pdf"
-    try:
-        StorageAdmin().upload_file(SCANS_BUCKET, storage_path, data, "application/pdf", upsert=True)
-    except Exception as exc:
-        logger.error(f"scan storage upload failed (batch={batch_id}): {exc}")
-        raise HTTPException(status_code=502, detail="Failed to store scan")
-
-    sb = ctx.supabase
-    submission: Optional[Dict[str, Any]] = None
-    if matching_method == "manual" and student_id:
-        submission = _first(
-            sb.table("student_submissions").select("*").eq("batch_id", batch_id).eq("student_id", student_id).limit(1).execute()
-        )
-    elif matching_method == "ordered":
-        # Assign to the next not-yet-submitted roster slot.
-        pending = _rows(
-            sb.table("student_submissions").select("*").eq("batch_id", batch_id).in_("status", ["absent", "unmatched"]).execute()
-        )
-        submission = pending[0] if pending else None
-
-    payload = {
-        "scan_url": storage_path,
-        "qr_code": None,
-        "matching_method": matching_method if (student_id or matching_method == "ordered") else "manual",
-        "page_count": None,
-        "status": "matched" if submission else "unmatched",
-    }
-
-    if submission:
-        updated = _first(sb.table("student_submissions").update(payload).eq("id", submission["id"]).execute())
-        return updated or submission
-    # No roster slot matched → create an unmatched submission to be reconciled later.
-    new_row = {"batch_id": batch_id, **payload}
-    created = _first(sb.table("student_submissions").insert(new_row).execute())
-    if not created:
-        raise HTTPException(status_code=500, detail="Failed to record scan submission")
-    return created

routers/exam/corpus.py

@@ -1,98 +0,0 @@
-"""Exam-bank corpus coverage (/api/exam/corpus) — the state of the collected exam bank.
-
-Read-only view over the seeded exam-board catalogue (eb_specifications + eb_exams): board → subject →
-specification → papers, with per-session QP/MS/ER coverage and rollup counts. Shows what the app has
-COLLECTED (question papers, mark schemes, examiner reports) so a teacher/admin can see the bank's state
-and where science coverage is complete vs thin. Catalogue data is public reference — read as-the-user.
-"""
-from __future__ import annotations
-
-import os
-import re
-from typing import Any, Dict, List, Optional
-
-from fastapi import APIRouter, Depends
-
-from modules.logger_tool import initialise_logger
-from routers.exam.dependencies import ExamContext, get_exam_context
-
-logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), "default", True)
-
-router = APIRouter()
-
-DOC_TYPES = ("QP", "MS", "ER")
-
-
-def _rows(result: Any) -> List[Dict[str, Any]]:
-    data = getattr(result, "data", None)
-    if not data:
-        return []
-    return data if isinstance(data, list) else [data]
-
-
-def _award_level(spec: Dict[str, Any]) -> str:
-    """Best-effort GCSE / AS / A-level from the spec code (AQA GCSE = 8xxx, A-level/AS = 7xxx)."""
-    code = re.sub(r"\D", "", spec.get("award_code") or spec.get("spec_code") or "")
-    if code.startswith("8"):
-        return "GCSE"
-    if code.startswith("7"):
-        return "A-level"
-    return spec.get("award_level") or "Other"
-
-
-@router.get("/corpus")
-async def corpus_coverage(ctx: ExamContext = Depends(get_exam_context)) -> Dict[str, Any]:
-    specs = _rows(
-        ctx.supabase.table("eb_specifications")
-        .select("spec_code, exam_board_code, subject_code, award_code, first_teach").execute()
-    )
-    exams = _rows(
-        ctx.supabase.table("eb_exams")
-        .select("exam_code, spec_code, paper_code, tier, session, type_code, storage_loc").execute()
-    )
-
-    # group exam docs → per spec → per paper (paper_code + session) → which doc types are present
-    by_spec: Dict[str, Dict[str, Dict[str, Any]]] = {}
-    for e in exams:
-        sc = e.get("spec_code")
-        if not sc:
-            continue
-        key = f"{e.get('paper_code') or '?'}|{e.get('session') or '?'}"
-        paper = by_spec.setdefault(sc, {}).setdefault(key, {
-            "paper_code": e.get("paper_code"), "session": e.get("session"),
-            "tier": e.get("tier"), "docs": {}, "exam_codes": {},
-        })
-        dt = (e.get("type_code") or "").upper()
-        if dt in DOC_TYPES:
-            paper["docs"][dt] = bool(e.get("storage_loc"))
-            paper["exam_codes"][dt] = e.get("exam_code")
-
-    totals = {"specs": 0, "papers": 0, "sessions": 0, **{d: 0 for d in DOC_TYPES}}
-    boards: Dict[str, Dict[str, Any]] = {}
-    for s in specs:
-        sc = s["spec_code"]
-        papers_map = by_spec.get(sc, {})
-        if not papers_map:
-            continue
-        totals["specs"] += 1
-        board = s.get("exam_board_code") or "?"
-        level = _award_level(s)
-        papers = sorted(papers_map.values(), key=lambda p: (str(p["session"]), str(p["paper_code"])))
-        counts = {d: sum(1 for p in papers if p["docs"].get(d)) for d in DOC_TYPES}
-        for d in DOC_TYPES:
-            totals[d] += counts[d]
-        totals["papers"] += len(papers)
-        totals["sessions"] += len({p["session"] for p in papers})
-        spec_entry = {
-            "spec_code": sc, "subject": (s.get("subject_code") or "").title(), "level": level,
-            "board": board, "first_teach": s.get("first_teach"),
-            "n_papers": len(papers), "counts": counts, "papers": papers,
-        }
-        boards.setdefault(board, {"board": board, "specs": []})["specs"].append(spec_entry)
-
-    board_list = []
-    for board in sorted(boards):
-        specs_sorted = sorted(boards[board]["specs"], key=lambda x: (x["level"], x["subject"], x["spec_code"]))
-        board_list.append({"board": board, "n_specs": len(specs_sorted), "specs": specs_sorted})
-
-    return {"totals": totals, "boards": board_list}

routers/exam/dependencies.py

@@ -1,146 +0,0 @@
-"""Auth + data-access plumbing for the /api/exam/ router.
-
-Per the audit (spec S1/E1): the exam API calls Supabase **as the user** so the RLS in
-72-exam-marker.sql is actually enforced — it does NOT use the service role for user-facing
-reads/writes the way files.py / classes_router.py do. The bearer already attaches the raw
-JWT as payload["_access_token"] (supabase_bearer.py) precisely for this.
-
-Institute resolution is the one wrinkle: institute_memberships and profiles are RLS
-deny-all to a normal authenticated user (E4), so we cannot read them as-user. Instead we
-call public.user_institute_ids() — a SECURITY DEFINER function (71-class-management.sql) that
-PostgREST exposes as an RPC — which returns the caller's institute ids regardless of those
-table policies. This is the same function the RLS policies themselves key off, so the API's
-view of "which institutes is this user in" is guaranteed consistent with what RLS will allow.
-"""
-from __future__ import annotations
-
-import os
-from typing import Any, Dict, List, Optional
-
-from fastapi import Depends, HTTPException
-
-from modules.auth.supabase_bearer import SupabaseBearer
-from modules.database.supabase.utils.client import (
-    SupabaseAnonClient,
-    SupabaseServiceRoleClient,
-)
-from modules.logger_tool import initialise_logger
-
-logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), "default", True)
-
-auth = SupabaseBearer()
-
-
-class ExamContext:
-    """The per-request handle every exam endpoint works through.
-
-    Bundles the caller's id, an as-user Supabase client (RLS-enforced), and the set of
-    institute ids the caller belongs to (for R5.5 institute validation on writes).
-    """
-
-    def __init__(self, user_id: str, access_token: str, supabase: Any, institute_ids: List[str]):
-        self.user_id = user_id
-        self.access_token = access_token
-        self.supabase = supabase
-        self.institute_ids = institute_ids
-
-    def resolve_institute(self, requested: Optional[str]) -> str:
-        """Validate a client-supplied institute_id, or pick the sole membership.
-
-        R5.5: a client-supplied institute_id is never trusted as the authz signal — it must
-        be one the caller actually belongs to. RLS would reject a bad value at write time
-        anyway; resolving here turns that into a clean 400/403 instead of an opaque DB error.
-        """
-        if requested:
-            if requested not in self.institute_ids:
-                raise HTTPException(status_code=403, detail="Not a member of the requested institute")
-            return requested
-        if len(self.institute_ids) == 1:
-            return self.institute_ids[0]
-        if not self.institute_ids:
-            raise HTTPException(status_code=403, detail="Caller has no institute membership")
-        raise HTTPException(
-            status_code=400,
-            detail="institute_id is required when the caller belongs to multiple institutes",
-        )
-
-
-def _extract_institute_ids(rpc_data: Any) -> List[str]:
-    """Normalise the user_institute_ids() RPC result to a list of uuid strings.
-
-    A `returns setof uuid` function comes back from PostgREST as a JSON array of scalars,
-    but tolerate the `[{"user_institute_ids": "..."}]` shape too in case of driver quirks.
-    """
-    out: List[str] = []
-    for row in rpc_data or []:
-        if isinstance(row, dict):
-            val = row.get("user_institute_ids") or next(iter(row.values()), None)
-        else:
-            val = row
-        if val:
-            out.append(str(val))
-    return out
-
-
-async def get_exam_context(payload: Dict[str, Any] = Depends(auth)) -> ExamContext:
-    user_id = payload.get("sub") or payload.get("user_id")
-    access_token = payload.get("_access_token")
-    if not user_id or not access_token:
-        raise HTTPException(status_code=401, detail="Invalid token payload")
-
-    supabase = SupabaseAnonClient.for_user(access_token).supabase
-
-    try:
-        res = supabase.rpc("user_institute_ids").execute()
-        institute_ids = _extract_institute_ids(getattr(res, "data", None))
-    except Exception as exc:
-        logger.error(f"Failed to resolve institute memberships: {exc}")
-        raise HTTPException(status_code=502, detail="Could not resolve institute membership")
-
-    return ExamContext(user_id, access_token, supabase, institute_ids)
-
-
-def resolve_student_names(student_ids: List[str]) -> Dict[str, str]:
-    """Map profile id → display name for roster students (batch-creation denormalisation).
-
-    Documented service-role exception (S1, mirrors lookup_exam_code): `profiles` has no as-user
-    SELECT policy (E4), so the roster's display names can't be read as-the-user. The caller's
-    right to the roster itself is already enforced as-user (class_students.cs_read requires the
-    caller to teach/admin the class); this only resolves names for ids already authorised, and
-    the result is denormalised onto student_submissions so later reads need no profiles access.
-    """
-    if not student_ids:
-        return {}
-    try:
-        sb = SupabaseServiceRoleClient().supabase
-        res = (
-            sb.table("profiles")
-            .select("id, full_name, display_name, email")
-            .in_("id", list(student_ids))
-            .execute()
-        )
-        out: Dict[str, str] = {}
-        for p in getattr(res, "data", None) or []:
-            out[p["id"]] = p.get("full_name") or p.get("display_name") or p.get("email") or ""
-        return out
-    except Exception as exc:
-        logger.warning(f"student name resolution failed: {exc}")
-        return {}
-
-
-def lookup_exam_code(exam_id: str) -> Optional[str]:
-    """Resolve eb_exams.exam_code for a catalogue paper (denormalised onto the template).
-
-    Documented service-role exception (S1): eb_exams is shared exam-board reference data with
-    no as-user SELECT policy (E4), so a normal user cannot read it. This is a read of public
-    catalogue metadata only — not user-scoped data — and is used solely to keep the Neo4j join
-    key (exam_code) correct on the template row.
-    """
-    try:
-        sb = SupabaseServiceRoleClient().supabase
-        res = sb.table("eb_exams").select("exam_code").eq("id", exam_id).limit(1).execute()
-        rows = getattr(res, "data", None) or []
-        return rows[0].get("exam_code") if rows else None
-    except Exception as exc:
-        logger.warning(f"exam_code lookup failed for exam_id={exam_id}: {exc}")
-        return None

routers/exam/schemas.py

@@ -1,166 +0,0 @@
-"""Pydantic request/response models for the /api/exam/ router (S4-5).
-
-Templates are saved from the canvas with a full-replace PUT (R5.2): the client owns
-stable UUIDs for questions / response areas / boundaries so the Supabase ids line up
-with the Neo4j join keys (exam_questions.id ↔ Question|Part.uuid_string,
-exam_response_areas.id ↔ Region.uuid_string — see spec §2). Granular mark-scheme edits
-go through PATCH /api/exam/questions/{qid}.
-
-Models mirror the columns in volumes/db/cc/72-exam-marker.sql. They are intentionally
-permissive (most fields optional) so the canvas can round-trip partial state during
-authoring without the API rejecting work-in-progress.
-"""
-from __future__ import annotations
-
-from typing import Any, Dict, List, Literal, Optional
-
-from pydantic import BaseModel, Field
-
-
-# ─── Templates ─────────────────────────────────────────────────────────────────
-
-class CreateTemplateRequest(BaseModel):
-    title: str
-    subject: Optional[str] = None
-    # Catalogue paper (eb_exams) the template maps, when chosen from the catalogue (R2.2).
-    exam_id: Optional[str] = None
-    # Denormalised onto the template for the Neo4j join (eb_exams.exam_code ↔ ExamPaper.exam_code).
-    # If exam_id is given but exam_code is omitted, the API resolves it from the catalogue.
-    exam_code: Optional[str] = None
-    # Uploaded PDF (files.id) for an ad-hoc paper (R2.2).
-    source_file_id: Optional[str] = None
-    page_count: Optional[int] = None
-    # Active institute (R1.4/R5.5). Validated against the caller's memberships; never trusted
-    # as the authorization signal. Optional when the caller belongs to exactly one institute.
-    institute_id: Optional[str] = None
-
-
-class UpdateTemplateMetaRequest(BaseModel):
-    """Template-level fields that a full-replace PUT may also update alongside the canvas."""
-    title: Optional[str] = None
-    subject: Optional[str] = None
-    page_count: Optional[int] = None
-    status: Optional[Literal["draft", "ready", "archived"]] = None
-
-
-# ─── Canvas entities (children of a template) ────────────────────────────────────
-
-class QuestionPayload(BaseModel):
-    # Client-supplied stable UUID (== Neo4j Question|Part.uuid_string). Optional on first save.
-    id: Optional[str] = None
-    parent_id: Optional[str] = None
-    label: str
-    order: int = 0
-    max_marks: float = 0
-    answer_type: Optional[Literal["written", "mcq", "short", "diagram"]] = None
-    mcq_options: Optional[Any] = None
-    mark_scheme: Dict[str, Any] = Field(default_factory=dict)
-    is_container: bool = False
-    spec_ref: Optional[str] = None
-    # Drawn Part box geometry (73-exam-marker-regions.sql). Null for derived main questions.
-    bounds: Optional[Dict[str, Any]] = None  # {x,y,w,h}
-    page: Optional[int] = None
-    # S5 AI/manual seam + provenance. Existing manual rows default to authoritative.
-    source: Literal["manual", "ai"] = "manual"
-    confirmed: bool = True
-    confidence: Optional[float] = Field(default=None, ge=0, le=1)
-    derivation: Optional[str] = None
-
-
-class ResponseAreaPayload(BaseModel):
-    id: Optional[str] = None  # == Neo4j Region.uuid_string (only response/context project)
-    question_id: str
-    page: int
-    bounds: Dict[str, Any]  # {x,y,w,h}
-    # S4-9 taxonomy (73-exam-marker-regions.sql): response/context graded-or-stimulus;
-    # question_number/mark_area = physical metadata; reference = student resource; furniture = ignore.
-    kind: Literal["response", "context", "question_number", "mark_area", "reference", "furniture"]
-    response_form: Optional[
-        Literal["lines", "answer-box", "working", "diagram", "tick-boxes", "table", "blanks"]
-    ] = None
-    # Optional Context differentiation (v1 generic; future graph/chart/data_table/diagram/code_block/passage).
-    context_type: Optional[str] = None
-    # Rich recognition payload (75-exam-marker-region-meta.sql): figure name/description, OMR geometry, unit…
-    # Carried on canvas save so a named context figure survives a round-trip.
-    meta: Optional[Dict[str, Any]] = None
-    source: Literal["manual", "ai"] = "manual"
-    confirmed: bool = True
-    confidence: Optional[float] = Field(default=None, ge=0, le=1)
-    # Only meaningful for kind='mark_area': part_marks|question_total|grader_box.
-    mark_subtype: Optional[Literal["part_marks", "question_total", "grader_box"]] = None
-    derivation: Optional[str] = None
-
-
-class BoundaryPayload(BaseModel):
-    id: Optional[str] = None
-    question_id: Optional[str] = None
-    label: Optional[str] = None
-    page_index: int
-    y: float
-    bounds: Optional[Dict[str, Any]] = None
-    source: Literal["manual", "ai"] = "manual"
-    confirmed: bool = True
-    confidence: Optional[float] = Field(default=None, ge=0, le=1)
-    derivation: Optional[str] = None
-
-
-class TemplateLayoutPayload(BaseModel):
-    id: Optional[str] = None
-    page_index: int
-    role: Optional[str] = None
-    margin_left: Optional[float] = None
-    margin_right: Optional[float] = None
-    margin_top: Optional[float] = None
-    margin_bottom: Optional[float] = None
-    margins_enabled: bool = True
-    source: Literal["manual", "ai"] = "manual"
-    confirmed: bool = True
-    confidence: Optional[float] = Field(default=None, ge=0, le=1)
-    derivation: Optional[str] = None
-    meta: Dict[str, Any] = Field(default_factory=dict)
-
-
-class TemplateReplaceRequest(BaseModel):
-    """Full-replace canvas save (R5.2 primary path). All children are replaced wholesale."""
-    meta: Optional[UpdateTemplateMetaRequest] = None
-    questions: List[QuestionPayload] = Field(default_factory=list)
-    response_areas: List[ResponseAreaPayload] = Field(default_factory=list)
-    boundaries: List[BoundaryPayload] = Field(default_factory=list)
-    layout: List[TemplateLayoutPayload] = Field(default_factory=list)
-
-
-class PatchQuestionRequest(BaseModel):
-    """Incremental mark-scheme / spec-ref edit (R5.2 granular path)."""
-    label: Optional[str] = None
-    order: Optional[int] = None
-    max_marks: Optional[float] = None
-    answer_type: Optional[Literal["written", "mcq", "short", "diagram"]] = None
-    mcq_options: Optional[Any] = None
-    mark_scheme: Optional[Dict[str, Any]] = None
-    is_container: Optional[bool] = None
-    spec_ref: Optional[str] = None
-
-
-# ─── Marking batches & marks ─────────────────────────────────────────────────
-
-class CreateBatchRequest(BaseModel):
-    template_id: str
-    # When a class is given, the roster (class_students, status='active') is materialised as
-    # student_submissions(status='absent') so every enrolled student appears in results (A7).
-    class_id: Optional[str] = None
-    title: Optional[str] = None
-
-
-class MarkUpsertRequest(BaseModel):
-    """Upsert one mark entry (PUT /marks/{id}; id is the mark_entry uuid).
-
-    batch_id is derived server-side from the submission, so the client never sets the RLS
-    scoping key. submission_id + question_id identify what is being marked.
-    """
-    submission_id: str
-    question_id: str
-    awarded_marks: float = 0
-    mark_scheme_detail: Optional[Dict[str, Any]] = None
-    annotation_shape_ids: Optional[Any] = None
-    comment: Optional[str] = None
-    confirmed: Optional[bool] = None

routers/langchain/neo4j_graph_qa.py

@@ -16,7 +16,7 @@ logging = logger.get_logger(
 from fastapi import APIRouter, HTTPException
 from langchain_classic.chains import GraphCypherQAChain
 from langchain_community.graphs import Neo4jGraph
-from langchain_openai import ChatOpenAI
+from langchain_community.chat_models import ChatOpenAI
 from langchain_classic.prompts.prompt import PromptTemplate
 from routers.llm.private.ollama.ollama_wrapper import OllamaWrapper
 from modules.database.tools.neontology.utils import get_node_types, get_rels_by_type

routers/me/bootstrap_router.py

@@ -1,18 +0,0 @@
-from typing import Any, Dict
-
-from fastapi import APIRouter, Depends, HTTPException
-
-from modules.auth.supabase_bearer import SupabaseBearer
-from modules.database.services.bootstrap_service import build_bootstrap_response
-
-router = APIRouter()
-auth_scheme = SupabaseBearer()
-
-
-@router.get("/bootstrap")
-async def get_me_bootstrap(credentials: Dict[str, Any] = Depends(auth_scheme)) -> Dict[str, Any]:
-    """Authenticated Supabase-first session/onboarding bootstrap contract."""
-    try:
-        return build_bootstrap_response(credentials)
-    except ValueError as exc:
-        raise HTTPException(status_code=403, detail=str(exc)) from exc

routers/simple_upload.py

@@ -26,7 +26,6 @@ from fastapi.responses import JSONResponse
 from modules.auth.supabase_bearer import SupabaseBearer
 from modules.database.supabase.utils.client import SupabaseServiceRoleClient
 from modules.database.supabase.utils.storage import StorageAdmin
-from modules.upload_validation import read_upload_bytes
 from modules.logger_tool import initialise_logger
 
 router = APIRouter()
@@ -60,9 +59,10 @@ async def upload_single_file(
         if not user_id:
             raise HTTPException(status_code=401, detail="User ID required")
         
-        # Validate MIME/type and read file content with a hard size limit.
-        file_bytes, mime_type = await read_upload_bytes(file)
+        # Read file content
+        file_bytes = await file.read()
         file_size = len(file_bytes)
+        mime_type = file.content_type or 'application/octet-stream'
         filename = file.filename or path
         
         logger.info(f"📤 Simple upload: {filename} ({file_size} bytes) for user {user_id}")
@@ -234,9 +234,10 @@ async def upload_directory(
             # Process each file
             for i, (file, relative_path) in enumerate(zip(files, relative_paths)):
                 try:
-                    # Validate MIME/type and read file content with a hard size limit.
-                    file_bytes, mime_type = await read_upload_bytes(file)
+                    # Read file content
+                    file_bytes = await file.read()
                     file_size = len(file_bytes)
+                    mime_type = file.content_type or 'application/octet-stream'
                     filename = file.filename or f"file_{i}"
                     
                     total_size += file_size
@@ -290,8 +291,6 @@ async def upload_directory(
                     
                     logger.info(f"📄 Uploaded file {i+1}/{len(files)}: {relative_path}")
                     
-                except HTTPException:
-                    raise
                 except Exception as e:
                     logger.error(f"Failed to upload file {relative_path}: {e}")
                     # Continue with other files, don't fail entire upload

routers/tlsync_token.py

@@ -1,65 +0,0 @@
-import os
-import time
-from typing import Any, Dict
-from uuid import uuid4
-
-import jwt
-from fastapi import APIRouter, Depends, HTTPException
-
-from modules.auth.supabase_bearer import verify_supabase_token_dep
-
-
-router = APIRouter()
-
-TLSYNC_TOKEN_AUDIENCE = "tlsync"
-DEFAULT_TLSYNC_TOKEN_TTL_SECONDS = 300
-
-
-def _tlsync_token_ttl_seconds() -> int:
-    raw_value = os.getenv("TLSYNC_TOKEN_TTL_SECONDS")
-    if not raw_value:
-        return DEFAULT_TLSYNC_TOKEN_TTL_SECONDS
-
-    try:
-        ttl = int(raw_value)
-    except ValueError as exc:
-        raise HTTPException(status_code=500, detail="TLSync token TTL is misconfigured") from exc
-
-    if ttl <= 0 or ttl > 3600:
-        raise HTTPException(status_code=500, detail="TLSync token TTL is out of range")
-    return ttl
-
-
-def create_tlsync_token(user_claims: Dict[str, Any]) -> Dict[str, Any]:
-    secret = os.getenv("TLSYNC_SECRET")
-    if not secret:
-        raise HTTPException(status_code=503, detail="TLSync authentication is not configured")
-
-    user_id = user_claims.get("sub")
-    if not user_id:
-        raise HTTPException(status_code=401, detail="Authenticated user id is missing")
-
-    ttl_seconds = _tlsync_token_ttl_seconds()
-    issued_at = int(time.time())
-    expires_at = issued_at + ttl_seconds
-    payload = {
-        "sub": user_id,
-        "aud": TLSYNC_TOKEN_AUDIENCE,
-        "iat": issued_at,
-        "exp": expires_at,
-        "jti": uuid4().hex,
-    }
-
-    token = jwt.encode(payload, secret, algorithm="HS256")
-    return {
-        "token": token,
-        "token_type": "Bearer",
-        "expires_in": ttl_seconds,
-        "expires_at": expires_at,
-    }
-
-
-@router.get("/token")
-async def get_tlsync_token(user_claims: Dict[str, Any] = Depends(verify_supabase_token_dep)) -> Dict[str, Any]:
-    """Issue a short-lived TLSync token for an authenticated Supabase user."""
-    return create_tlsync_token(user_claims)

routers/transcribe/sessions.py

@@ -1,13 +1,8 @@
 """Transcription sessions router — CRUD endpoints for transcription sessions and segments."""
 
 from fastapi import APIRouter, Depends, HTTPException, Query
-from fastapi.responses import FileResponse
 from typing import Optional, List
 from datetime import datetime
-import io
-import json
-import tempfile
-import os
 
 from modules.auth.supabase_bearer import SupabaseBearer
 from modules.transcription.models import (
@@ -40,116 +35,6 @@ def get_user_id(credentials=Depends(SupabaseBearer())) -> str:
     return credentials.get("sub", credentials.get("user_id", ""))
 
 
-def seconds_to_srt_timestamp(seconds: float) -> str:
-    """Convert seconds to SRT timestamp format: HH:MM:SS,mmm"""
-    hours = int(seconds // 3600)
-    minutes = int((seconds % 3600) // 60)
-    secs = int(seconds % 60)
-    millis = int((seconds % 1) * 1000)
-    return f"{hours:02d}:{minutes:02d}:{secs:02d},{millis:03d}"
-
-
-def generate_srt(segments: List[dict]) -> str:
-    """Generate SRT (SubRip subtitle) content from segments."""
-    srt_entries = []
-    for idx, seg in enumerate(segments, start=1):
-        start_sec = float(seg.get("start_seconds", 0))
-        end_sec = float(seg.get("end_seconds", 0))
-        text = seg.get("text", "").strip()
-        if not text:
-            continue
-        start_ts = seconds_to_srt_timestamp(start_sec)
-        end_ts = seconds_to_srt_timestamp(end_sec)
-        # Clean text for SRT (no line breaks within a subtitle block)
-        clean_text = text.replace("\n", " ").strip()
-        srt_entries.append(f"{idx}\n{start_ts} --> {end_ts}\n{clean_text}")
-    return "\n\n".join(srt_entries) + "\n" if srt_entries else ""
-
-
-def generate_txt(segments: List[dict]) -> str:
-    """Generate plain text transcript with timestamps from segments."""
-    lines = []
-    for seg in segments:
-        start_sec = float(seg.get("start_seconds", 0))
-        text = seg.get("text", "").strip()
-        if not text:
-            continue
-        ts = seconds_to_srt_timestamp(start_sec)
-        lines.append(f"[{ts}] {text}")
-    return "\n".join(lines) + "\n" if lines else ""
-
-
-def generate_json_export(session: dict, segments: List[dict],
-                         summaries: List[dict],
-                         canvas_events: List[dict]) -> str:
-    """Generate structured JSON export with segments, metadata, and canvas events."""
-    # Build clean segment list (exclude internal DB fields)
-    clean_segments = []
-    for seg in segments:
-        clean_segments.append({
-            "sequence_index": seg.get("sequence_index"),
-            "text": seg.get("text", ""),
-            "start_seconds": float(seg.get("start_seconds", 0)),
-            "end_seconds": float(seg.get("end_seconds", 0)),
-            "is_final": seg.get("is_final", True),
-            "speaker_label": seg.get("speaker_label"),
-            "keyword_matches": seg.get("keyword_matches"),
-        })
-
-    # Build clean summary list
-    clean_summaries = []
-    for s in summaries:
-        clean_summaries.append({
-            "id": s.get("id"),
-            "summary_type": s.get("summary_type"),
-            "content": s.get("content", ""),
-            "llm_provider": s.get("llm_provider"),
-            "llm_model": s.get("llm_model"),
-            "created_at": s.get("created_at"),
-        })
-
-    # Build clean canvas events list
-    clean_events = []
-    for ev in canvas_events:
-        clean_events.append({
-            "id": ev.get("id"),
-            "event_type": ev.get("event_type"),
-            "session_elapsed_seconds": float(ev.get("session_elapsed_seconds", 0)) if ev.get("session_elapsed_seconds") else None,
-            "timestamp": ev.get("timestamp"),
-            "event_payload": ev.get("event_payload", {}),
-        })
-
-    export_data = {
-        "session": {
-            "id": session.get("id"),
-            "title": session.get("title"),
-            "canvas_type": session.get("canvas_type"),
-            "started_at": session.get("started_at"),
-            "ended_at": session.get("ended_at"),
-            "duration_seconds": session.get("duration_seconds"),
-            "timetable_period_id": session.get("timetable_period_id"),
-            "timetable_event_type": session.get("timetable_event_type"),
-            "timetable_event_label": session.get("timetable_event_label"),
-            "auto_tagged": session.get("auto_tagged", False),
-            "llm_provider": session.get("llm_provider"),
-            "llm_model": session.get("llm_model"),
-            "word_count": session.get("word_count", 0),
-            "segment_count": session.get("segment_count", 0),
-        },
-        "segments": clean_segments,
-        "summaries": clean_summaries,
-        "canvas_events": clean_events,
-    }
-
-    return json.dumps(export_data, indent=2, default=str)
-
-
-def sanitize_filename(name: str) -> str:
-    """Remove or replace characters that are unsafe in filenames."""
-    safe = "".join(c if c.isalnum() or c in " _-." else "_" for c in name)
-    return safe[:100] if safe else "export"
-
-
 @router.post("/sessions", response_model=TranscriptionSessionResponse)
 async def create_session(
     session_data: TranscriptionSessionCreate,
@@ -445,65 +330,26 @@ async def export_session(
     export_format: ExportFormat,
     user_id: str = Depends(get_user_id),
 ):
-    """Export session as SRT, TXT, or JSON file download.
-
-    Phase 3E: Full implementation — generates properly formatted files
-    and returns them as downloadable responses. API keys are never stored
-    or logged during export.
-    """
+    """Export session as SRT, TXT, or JSON (Phase 1 stub)."""
     supabase = get_supabase_client()
     
     # Verify ownership
-    session_check = supabase.supabase.table("transcription_sessions").select("*").eq("id", session_id).eq("user_id", user_id).execute()
+    session_check = supabase.supabase.table("transcription_sessions").select("id").eq("id", session_id).eq("user_id", user_id).execute()
     
     if not session_check.data:
         raise HTTPException(status_code=404, detail="Session not found")
     
-    session = session_check.data[0]
-    
     # Get segments
     segments_result = supabase.supabase.table("transcription_segments").select("*").eq("session_id", session_id).order("sequence_index").execute()
     segments = segments_result.data
     
-    # Get summaries (for JSON export)
-    summaries_result = supabase.supabase.table("transcription_summaries").select("*").eq("session_id", session_id).execute()
-    summaries = summaries_result.data
-    
-    # Get canvas events (for JSON export)
-    canvas_result = supabase.supabase.table("canvas_events").select("*").eq("session_id", session_id).order("timestamp").execute()
-    canvas_events = canvas_result.data
-    
-    fmt = export_format.format.lower()
-    
-    if fmt == "srt":
-        content = generate_srt(segments)
-        filename = f"{sanitize_filename(session.get('title', session_id))}_{session.get('started_at', 'export')[:10]}.srt"
-        return FileResponse(
-            io.BytesIO(content.encode("utf-8")),
-            media_type="application/x-subrip",
-            filename=filename,
-            headers={"Content-Disposition": f"attachment; filename*=UTF-8''{filename}"},
-        )
-    
-    elif fmt == "txt":
-        content = generate_txt(segments)
-        filename = f"{sanitize_filename(session.get('title', session_id))}_{session.get('started_at', 'export')[:10]}.txt"
-        return FileResponse(
-            io.BytesIO(content.encode("utf-8")),
-            media_type="text/plain",
-            filename=filename,
-            headers={"Content-Disposition": f"attachment; filename*=UTF-8''{filename}"},
-        )
-    
-    elif fmt == "json":
-        content = generate_json_export(session, segments, summaries, canvas_events)
-        filename = f"{sanitize_filename(session.get('title', session_id))}_{session.get('started_at', 'export')[:10]}.json"
-        return FileResponse(
-            io.BytesIO(content.encode("utf-8")),
-            media_type="application/json",
-            filename=filename,
-            headers={"Content-Disposition": f"attachment; filename*=UTF-8''{filename}"},
-        )
-    
+    if export_format.format == "srt":
+        # Phase 1 stub — implement in Phase 3
+        return {"format": "srt", "content": "[TODO: Generate SRT from segments]"}
+    elif export_format.format == "txt":
+        text = "\n".join(s["text"] for s in segments)
+        return {"format": "txt", "content": text}
+    elif export_format.format == "json":
+        return {"format": "json", "content": {"segments": segments}}
     else:
         raise HTTPException(status_code=400, detail=f"Unsupported format: {export_format.format}")

run/initialization/__init__.py

@@ -1,4 +1,6 @@
 from .infrastructure import initialize_infrastructure
+from .demo_school import initialize_demo_school
+from .demo_users import initialize_demo_users
 from .gais_data import import_gais_data
 from modules.logger_tool import initialise_logger
 import os
@@ -8,32 +10,54 @@ logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH
 def initialize_infrastructure_mode() -> None:
     """Initialize infrastructure: Neo4j schema, calendar, and Supabase buckets"""
     logger.info("Starting infrastructure initialization...")
-
+    
+    # 1. Initialize Neo4j database, schema, and calendar structure
     logger.info("Step 1: Initializing Neo4j infrastructure...")
     from .neo4j import initialize_neo4j
     neo4j_result = initialize_neo4j()
-
+    
     if not neo4j_result["success"]:
         logger.error(f"Neo4j infrastructure initialization failed: {neo4j_result['message']}")
         return
-
+    
+    # 2. Initialize Supabase storage buckets
     logger.info("Step 2: Initializing Supabase storage buckets...")
     from .buckets import initialize_buckets
     buckets_result = initialize_buckets()
-
+    
     if not buckets_result["success"]:
         logger.error(f"Storage buckets initialization failed: {buckets_result['message']}")
         return
-
+    
     logger.info("Infrastructure initialization completed successfully!")
     logger.info(f"Neo4j: {neo4j_result['message']}")
     logger.info(f"Buckets: {buckets_result['message']}")
 
+def initialize_demo_school_mode() -> None:
+    """Initialize demo school (KevlarAI)"""
+    logger.info("Starting demo school initialization...")
+    result = initialize_demo_school()
+    
+    if result["success"]:
+        logger.info("Demo school initialization completed successfully")
+    else:
+        logger.error(f"Demo school initialization failed: {result['message']}")
+
+def initialize_demo_users_mode() -> None:
+    """Initialize demo users"""
+    logger.info("Starting demo users initialization...")
+    result = initialize_demo_users()
+    
+    if result["success"]:
+        logger.info("Demo users initialization completed successfully")
+    else:
+        logger.error(f"Demo users initialization failed: {result['message']}")
+
 def initialize_gais_data_mode() -> None:
     """Initialize GAIS data import (Edubase, etc.)"""
     logger.info("Starting GAIS data import...")
     result = import_gais_data()
-
+    
     if result["success"]:
         logger.info("GAIS data import completed successfully")
     else:
@@ -41,7 +65,11 @@ def initialize_gais_data_mode() -> None:
 
 __all__ = [
     'initialize_infrastructure_mode',
+    'initialize_demo_school_mode',
+    'initialize_demo_users_mode',
     'initialize_gais_data_mode',
     'initialize_infrastructure',
+    'initialize_demo_school',
+    'initialize_demo_users',
     'import_gais_data'
-]
+] 

run/initialization/buckets.py

@@ -46,7 +46,7 @@ def initialize_buckets() -> dict:
                     file_size_limit=1000 * 1024 * 1024,  # 1GB
                 )
             },
-            # Exam Board files (admin-curated public exam corpus: QP/MS/insert/ER + specs)
+            # Exam Board files
             {
                 "id": "cc.examboards",
                 "options": CreateBucketOptions(
@@ -55,34 +55,6 @@ def initialize_buckets() -> dict:
                     file_size_limit=1000 * 1024 * 1024,  # 1GB
                 )
             },
-            # ── Storage taxonomy bins (access scoped by RLS on bucket + leading path segment; RLS = D1) ──
-            # Platform-managed public/shared assets (readable by all authenticated users).
-            {
-                "id": "cc.public",
-                "options": CreateBucketOptions(
-                    name="Classroom Copilot Public",
-                    public=False,
-                    file_size_limit=1000 * 1024 * 1024,  # 1GB
-                )
-            },
-            # Institute-scoped operational assets: cc.institutes/{institute_id}/...
-            {
-                "id": "cc.institutes",
-                "options": CreateBucketOptions(
-                    name="Classroom Copilot Institutes",
-                    public=False,
-                    file_size_limit=1000 * 1024 * 1024,  # 1GB
-                )
-            },
-            # Platform-admin-only assets, seeds, intake/staging for unidentified papers.
-            {
-                "id": "cc.admin",
-                "options": CreateBucketOptions(
-                    name="Classroom Copilot Admin",
-                    public=False,
-                    file_size_limit=1000 * 1024 * 1024,  # 1GB
-                )
-            },
         ]
         
         results = {}
@@ -109,17 +81,11 @@ def initialize_buckets() -> dict:
                     logger.error(f"Failed to create bucket: {bucket['id']}")
                     
             except Exception as e:
-                # Idempotent: an already-existing bucket is not a failure on re-run.
-                if any(s in str(e).lower() for s in ("already exists", "duplicate", "resource already")):
-                    results[bucket["id"]] = {"status": "exists", "result": str(e)}
-                    success_count += 1
-                    logger.info(f"Bucket already exists (ok): {bucket['id']}")
-                else:
-                    results[bucket["id"]] = {
-                        "status": "error",
-                        "error": str(e)
-                    }
-                    logger.error(f"Error creating bucket {bucket['id']}: {str(e)}")
+                results[bucket["id"]] = {
+                    "status": "error",
+                    "error": str(e)
+                }
+                logger.error(f"Error creating bucket {bucket['id']}: {str(e)}")
         
         # Determine overall success
         if success_count == total_count:

run/initialization/demo_school.py

@@ -0,0 +1,181 @@
+"""
+Demo school initialization module for ClassroomCopilot
+Creates the KevlarAI demo school
+"""
+import os
+import json
+import requests
+from typing import Dict, Any
+from modules.logger_tool import initialise_logger
+from modules.database.services.provisioning_service import ProvisioningService
+import time
+
+logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
+
+class DemoSchoolInitializer:
+    """Handles demo school creation"""
+    
+    def __init__(self, supabase_url: str, service_role_key: str):
+        self.supabase_url = supabase_url
+        self.service_role_key = service_role_key
+        self.supabase_headers = {
+            "apikey": service_role_key,
+            "Authorization": f"Bearer {service_role_key}",
+            "Content-Type": "application/json"
+        }
+        self.provisioning_service = ProvisioningService()
+    
+    def create_kevlarai_school(self) -> Dict[str, Any]:
+        """Create the KevlarAI demo school"""
+        logger.info("Creating KevlarAI demo school...")
+        
+        try:
+            # Check if KevlarAI school already exists
+            response = self._supabase_request_with_retry(
+                'get',
+                f"{self.supabase_url}/rest/v1/institutes",
+                headers=self.supabase_headers,
+                params={
+                    "select": "*",
+                    "name": "eq.KevlarAI"
+                }
+            )
+            
+            if response.status_code == 200:
+                existing_schools = response.json()
+                if existing_schools and len(existing_schools) > 0:
+                    logger.info("KevlarAI school already exists")
+                    school = existing_schools[0]
+                    try:
+                        self.provisioning_service.ensure_school(school["id"])
+                    except Exception as provisioning_error:
+                        logger.warning(f"Provisioning KevlarAI school failed: {provisioning_error}")
+                    return {
+                        "success": True,
+                        "message": "KevlarAI school already exists",
+                        "school": school
+                    }
+            
+            # Create KevlarAI school
+            school_data = {
+                "name": "KevlarAI",
+                "urn": "KEVLARAI001",
+                "status": "active",
+                "address": {
+                    "street": "123 Innovation Drive",
+                    "town": "Tech City",
+                    "county": "Digital County",
+                    "postcode": "TC1 2AI",
+                    "country": "United Kingdom"
+                },
+                "website": "https://kevlar.ai",
+                "metadata": {
+                    "school_type": "AI and Technology",
+                    "phase_of_education": "Secondary and Further Education",
+                    "establishment_status": "Open",
+                    "specialization": "Artificial Intelligence, Machine Learning, Robotics"
+                }
+            }
+            
+            # Insert the school
+            response = self._supabase_request_with_retry('post', f"{self.supabase_url}/rest/v1/institutes", headers={**self.supabase_headers, "Prefer": "return=representation"}, json=school_data, params={"select": "*"})
+            
+            logger.info(f"Supabase response status: {response.status_code}")
+            logger.info(f"Supabase response headers: {dict(response.headers)}")
+            logger.info(f"Supabase response text: {response.text}")
+            
+            if response.status_code in (200, 201):
+                try:
+                    data = response.json()
+                    school = data[0] if isinstance(data, list) and data else data
+                    logger.info("Successfully created KevlarAI school")
+                    # Ensure Neo4j provisioning is in place
+                    try:
+                        self.provisioning_service.ensure_school(school["id"])
+                    except Exception as provisioning_error:
+                        logger.warning(f"Provisioning KevlarAI school failed: {provisioning_error}")
+                    return {
+                        "success": True,
+                        "message": "Successfully created KevlarAI school",
+                        "school": school
+                    }
+                except json.JSONDecodeError as e:
+                    logger.error(f"Failed to parse JSON response: {str(e)}")
+                    logger.error(f"Response text: {response.text}")
+                    # If the status code is successful but we can't parse JSON, 
+                    # the school was likely created successfully
+                    return {
+                        "success": True,
+                        "message": "Successfully created KevlarAI school (response not JSON)",
+                        "school": None
+                    }
+            else:
+                logger.error(f"Failed to create KevlarAI school: {response.text}")
+                return {
+                    "success": False,
+                    "message": f"Failed to create KevlarAI school: {response.text}"
+                }
+                
+        except Exception as e:
+            logger.error(f"Error creating KevlarAI school: {str(e)}")
+            return {
+                "success": False,
+                "message": f"Error creating KevlarAI school: {str(e)}"
+            }
+    
+    def _supabase_request_with_retry(self, method, url, **kwargs):
+        """Make a request to Supabase with retry logic"""
+        max_retries = 3
+        retry_delay = 2  # seconds
+        
+        for attempt in range(max_retries):
+            try:
+                if method.lower() == 'get':
+                    response = requests.get(url, **kwargs)
+                elif method.lower() == 'post':
+                    response = requests.post(url, **kwargs)
+                elif method.lower() == 'put':
+                    response = requests.put(url, **kwargs)
+                elif method.lower() == 'delete':
+                    response = requests.delete(url, **kwargs)
+                else:
+                    raise ValueError(f"Unsupported HTTP method: {method}")
+                
+                # If successful or client error (4xx), don't retry
+                if response.status_code < 500:
+                    return response
+                
+                # Server error (5xx), retry after delay
+                logger.warning(f"Supabase server error (attempt {attempt+1}/{max_retries}): {response.status_code} - {response.text}")
+                time.sleep(retry_delay * (attempt + 1))  # Exponential backoff
+                
+            except requests.RequestException as e:
+                logger.warning(f"Supabase request exception (attempt {attempt+1}/{max_retries}): {str(e)}")
+                if attempt == max_retries - 1:
+                    raise
+                time.sleep(retry_delay * (attempt + 1))
+        
+        # If we get here, all retries failed with server errors
+        raise requests.RequestException(f"Failed after {max_retries} attempts to {method} {url}")
+
+def initialize_demo_school() -> Dict[str, Any]:
+    """Initialize demo school (KevlarAI)"""
+    logger.info("Starting demo school initialization...")
+    
+    supabase_url = os.getenv("SUPABASE_URL")
+    service_role_key = os.getenv("SERVICE_ROLE_KEY")
+    
+    if not supabase_url or not service_role_key:
+        return {"success": False, "message": "Missing SUPABASE_URL or SERVICE_ROLE_KEY environment variables"}
+    
+    initializer = DemoSchoolInitializer(supabase_url, service_role_key)
+    
+    # Create KevlarAI school
+    result = initializer.create_kevlarai_school()
+    
+    if result["success"]:
+        logger.info("Demo school initialization completed successfully")
+    else:
+        logger.error(f"Demo school initialization failed: {result['message']}")
+    
+    return result

run/initialization/demo_users.py

@@ -0,0 +1,395 @@
+"""
+Demo users initialization module for ClassroomCopilot
+Creates demo teachers and students
+"""
+import os
+import json
+import requests
+import time
+from typing import Dict, Any
+from modules.logger_tool import initialise_logger
+from modules.database.services.provisioning_service import ProvisioningService
+
+logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
+
+class DemoUsersInitializer:
+    """Handles demo users creation"""
+    
+    def __init__(self, supabase_url: str, service_role_key: str):
+        self.supabase_url = supabase_url
+        self.service_role_key = service_role_key
+        self.supabase_headers = {
+            "apikey": service_role_key,
+            "Authorization": f"Bearer {service_role_key}",
+            "Content-Type": "application/json"
+        }
+        self.provisioning_service = ProvisioningService()
+    
+    def create_demo_users(self) -> Dict[str, Any]:
+        """Create demo teachers and students"""
+        logger.info("Creating demo users...")
+        
+        try:
+            # Define demo users
+            demo_users = [
+                # Demo Teachers
+                {
+                    "email": "[email protected]",
+                    "password": "DemoTeacher123!",
+                    "email_confirm": True,
+                    "user_metadata": {
+                        "name": "Dr. Sarah Chen",
+                        "username": "sarah.chen",
+                        "full_name": "Dr. Sarah Chen",
+                        "display_name": "Dr. Chen",
+                        "user_type": "teacher"
+                    },
+                    "app_metadata": {
+                        "provider": "email",
+                        "providers": ["email"]
+                    }
+                },
+                {
+                    "email": "[email protected]",
+                    "password": "DemoTeacher123!",
+                    "email_confirm": True,
+                    "user_metadata": {
+                        "name": "Prof. Marcus Rodriguez",
+                        "username": "marcus.rodriguez",
+                        "full_name": "Professor Marcus Rodriguez",
+                        "display_name": "Prof. Rodriguez",
+                        "user_type": "teacher"
+                    },
+                    "app_metadata": {
+                        "provider": "email",
+                        "providers": ["email"]
+                    }
+                },
+                # Demo Students
+                {
+                    "email": "[email protected]",
+                    "password": "DemoStudent123!",
+                    "email_confirm": True,
+                    "user_metadata": {
+                        "name": "Alex Thompson",
+                        "username": "alex.thompson",
+                        "full_name": "Alex Thompson",
+                        "display_name": "Alex",
+                        "user_type": "student"
+                    },
+                    "app_metadata": {
+                        "provider": "email",
+                        "providers": ["email"]
+                    }
+                },
+                {
+                    "email": "[email protected]",
+                    "password": "DemoStudent123!",
+                    "email_confirm": True,
+                    "user_metadata": {
+                        "name": "Jordan Lee",
+                        "username": "jordan.lee",
+                        "full_name": "Jordan Lee",
+                        "display_name": "Jordan",
+                        "user_type": "student"
+                    },
+                    "app_metadata": {
+                        "provider": "email",
+                        "providers": ["email"]
+                    }
+                }
+            ]
+            
+            created_users = []
+            failed_users = []
+            
+            for user_data in demo_users:
+                try:
+                    # Create user via Auth API
+                    response = self._supabase_request_with_retry(
+                        'post',
+                        f"{self.supabase_url}/auth/v1/admin/users",
+                        headers=self.supabase_headers,
+                        json=user_data
+                    )
+                    
+                    if response.status_code in (200, 201):
+                        user = response.json()
+                        user_id = user.get("id")
+                        
+                        # Wait a moment for user to be created
+                        time.sleep(1)
+                        
+                        # Create profile
+                        profile_data = {
+                            "id": user_id,
+                            "email": user_data["email"],
+                            "user_type": user_data["user_metadata"]["user_type"],
+                            "username": user_data["user_metadata"]["username"],
+                            "full_name": user_data["user_metadata"]["full_name"],
+                            "display_name": user_data["user_metadata"]["display_name"]
+                        }
+                        
+                        profile_response = self._supabase_request_with_retry(
+                            'post',
+                            f"{self.supabase_url}/rest/v1/profiles",
+                            headers=self.supabase_headers,
+                            json=profile_data
+                        )
+                        
+                        if profile_response.status_code in (200, 201):
+                            created_users.append({
+                                "id": user_id,
+                                "email": user_data["email"],
+                                "user_type": user_data["user_metadata"]["user_type"],
+                                "username": user_data["user_metadata"]["username"]
+                            })
+                            logger.info(f"Successfully created user: {user_data['email']}")
+                        else:
+                            logger.warning(f"Failed to create profile for {user_data['email']}: {profile_response.text}")
+                            failed_users.append({
+                                "email": user_data["email"],
+                                "error": f"Profile creation failed: {profile_response.text}"
+                            })
+                    else:
+                        logger.warning(f"Failed to create user {user_data['email']}: {response.text}")
+                        failed_users.append({
+                            "email": user_data["email"],
+                            "error": f"User creation failed: {response.text}"
+                        })
+                        
+                except Exception as e:
+                    logger.error(f"Error creating user {user_data['email']}: {str(e)}")
+                    failed_users.append({
+                        "email": user_data["email"],
+                        "error": str(e)
+                    })
+            
+            # Create institute memberships for KevlarAI and provision users
+            all_users_to_provision = []
+            
+            # Add newly created users
+            if created_users:
+                all_users_to_provision.extend(created_users)
+                self._create_institute_memberships(created_users)
+            
+            # Also provision existing users that failed due to email_exists
+            existing_users = []
+            for failed_user in failed_users:
+                if "email_exists" in failed_user.get("error", ""):
+                    # Get the existing user ID from Supabase
+                    existing_user_id = self._get_existing_user_id(failed_user["email"])
+                    if existing_user_id:
+                        existing_users.append({
+                            "id": existing_user_id,
+                            "email": failed_user["email"],
+                            "user_type": self._get_user_type_from_email(failed_user["email"]),
+                            "username": self._get_username_from_email(failed_user["email"])
+                        })
+            
+            if existing_users:
+                logger.info(f"Found {len(existing_users)} existing users to provision")
+                all_users_to_provision.extend(existing_users)
+                self._create_institute_memberships(existing_users)
+            
+            # Provision all users (new and existing)
+            if all_users_to_provision:
+                self._provision_users(all_users_to_provision)
+            
+            logger.info(f"Demo users creation completed: {len(created_users)} created, {len(failed_users)} failed")
+            
+            return {
+                "success": True,
+                "message": f"Successfully created {len(created_users)} demo users",
+                "created_users": created_users,
+                "failed_users": failed_users
+            }
+            
+        except Exception as e:
+            logger.error(f"Error creating demo users: {str(e)}")
+            return {
+                "success": False,
+                "message": f"Error creating demo users: {str(e)}"
+            }
+    
+    def _create_institute_memberships(self, users: list) -> None:
+        """Create institute memberships for users in KevlarAI"""
+        logger.info("Creating institute memberships for demo users...")
+        
+        try:
+            # Get KevlarAI institute ID
+            response = self._supabase_request_with_retry(
+                'get',
+                f"{self.supabase_url}/rest/v1/institutes",
+                headers=self.supabase_headers,
+                params={
+                    "select": "id",
+                    "name": "eq.KevlarAI"
+                }
+            )
+            
+            if response.status_code != 200:
+                logger.warning("Could not get KevlarAI institute ID for memberships")
+                return
+            
+            institutes = response.json()
+            if not institutes:
+                logger.warning("KevlarAI institute not found for memberships")
+                return
+            
+            institute_id = institutes[0]["id"]
+            
+            # Get user profile IDs
+            for user in users:
+                try:
+                    profile_response = self._supabase_request_with_retry(
+                        'get',
+                        f"{self.supabase_url}/rest/v1/profiles",
+                        headers=self.supabase_headers,
+                        params={
+                            "select": "id",
+                            "email": f"eq.{user['email']}"
+                        }
+                    )
+                    
+                    if profile_response.status_code == 200:
+                        profiles = profile_response.json()
+                        if profiles:
+                            profile_id = profiles[0]["id"]
+                            
+                            # Create membership
+                            membership_data = {
+                                "profile_id": profile_id,
+                                "institute_id": institute_id,
+                                "role": user["user_type"]
+                            }
+                            
+                            membership_response = self._supabase_request_with_retry(
+                                'post',
+                                f"{self.supabase_url}/rest/v1/institute_memberships",
+                                headers=self.supabase_headers,
+                                json=membership_data
+                            )
+                            
+                            if membership_response.status_code in (200, 201):
+                                logger.info(f"Created membership for {user['email']} in KevlarAI")
+                            else:
+                                logger.warning(f"Failed to create membership for {user['email']}: {membership_response.text}")
+                                
+                except Exception as e:
+                    logger.warning(f"Error creating membership for {user['email']}: {str(e)}")
+                    
+        except Exception as e:
+            logger.warning(f"Error creating institute memberships: {str(e)}")
+
+    def _get_existing_user_id(self, email: str) -> str:
+        """Get the user ID for an existing user by email"""
+        try:
+            response = self._supabase_request_with_retry(
+                'get',
+                f"{self.supabase_url}/rest/v1/profiles",
+                headers=self.supabase_headers,
+                params={
+                    "select": "id",
+                    "email": f"eq.{email}"
+                }
+            )
+            
+            if response.status_code == 200:
+                profiles = response.json()
+                if profiles and len(profiles) > 0:
+                    return profiles[0].get("id")
+            
+            logger.warning(f"Could not find existing user ID for {email}")
+            return None
+            
+        except Exception as e:
+            logger.warning(f"Error getting existing user ID for {email}: {str(e)}")
+            return None
+    
+    def _get_user_type_from_email(self, email: str) -> str:
+        """Get user type from email based on demo user definitions"""
+        if "teacher" in email:
+            return "teacher"
+        elif "student" in email:
+            return "student"
+        return "teacher"  # default
+    
+    def _get_username_from_email(self, email: str) -> str:
+        """Get username from email based on demo user definitions"""
+        username_map = {
+            "[email protected]": "sarah.chen",
+            "[email protected]": "marcus.rodriguez", 
+            "[email protected]": "alex.thompson",
+            "[email protected]": "jordan.lee"
+        }
+        return username_map.get(email, email.split("@")[0])
+
+    def _provision_users(self, users: list) -> None:
+        """Provision Neo4j databases for the created demo users."""
+        for user in users:
+            user_id = user.get("id")
+            if not user_id:
+                continue
+            try:
+                self.provisioning_service.ensure_user(user_id)
+                logger.info(f"Provisioned Neo4j resources for {user.get('email')}")
+            except Exception as exc:
+                logger.warning(f"Failed to provision Neo4j resources for {user.get('email')}: {exc}")
+    
+    def _supabase_request_with_retry(self, method, url, **kwargs):
+        """Make a request to Supabase with retry logic"""
+        max_retries = 3
+        retry_delay = 2  # seconds
+        
+        for attempt in range(max_retries):
+            try:
+                if method.lower() == 'get':
+                    response = requests.get(url, **kwargs)
+                elif method.lower() == 'post':
+                    response = requests.post(url, **kwargs)
+                elif method.lower() == 'put':
+                    response = requests.put(url, **kwargs)
+                elif method.lower() == 'delete':
+                    response = requests.delete(url, **kwargs)
+                else:
+                    raise ValueError(f"Unsupported HTTP method: {method}")
+                
+                # If successful or client error (4xx), don't retry
+                if response.status_code < 500:
+                    return response
+                
+                # Server error (5xx), retry after delay
+                logger.warning(f"Supabase server error (attempt {attempt+1}/{max_retries}): {response.status_code} - {response.text}")
+                time.sleep(retry_delay * (attempt + 1))  # Exponential backoff
+                
+            except requests.RequestException as e:
+                logger.warning(f"Supabase request exception (attempt {attempt+1}/{max_retries}): {str(e)}")
+                if attempt == max_retries - 1:
+                    raise
+                time.sleep(retry_delay * (attempt + 1))
+        
+        # If we get here, all retries failed with server errors
+        raise requests.RequestException(f"Failed after {max_retries} attempts to {method} {url}")
+
+def initialize_demo_users() -> Dict[str, Any]:
+    """Initialize demo users"""
+    logger.info("Starting demo users initialization...")
+    
+    supabase_url = os.getenv("SUPABASE_URL")
+    service_role_key = os.getenv("SERVICE_ROLE_KEY")
+    
+    if not supabase_url or not service_role_key:
+        return {"success": False, "message": "Missing SUPABASE_URL or SERVICE_ROLE_KEY environment variables"}
+    
+    initializer = DemoUsersInitializer(supabase_url, service_role_key)
+    
+    # Create demo users
+    result = initializer.create_demo_users()
+    
+    if result["success"]:
+        logger.info("Demo users initialization completed successfully")
+    else:
+        logger.error(f"Demo users initialization failed: {result['message']}")
+    
+    return result

run/initialization/init_exam_graph.py

@@ -1,162 +0,0 @@
-"""
-init_exam_graph.py — Initialise the cc.public.exams Neo4j knowledge graph.
-
-Creates the shared, public exam database, its uniqueness constraints, and seeds the AQA exam board
-+ the 6 current test specifications (GCSE & A-level Physics/Chemistry/Biology) with their top-level
-topic SpecPoints (44 in total). Idempotent (CREATE DATABASE IF NOT EXISTS / CREATE CONSTRAINT IF NOT
-EXISTS / MERGE).
-
-Run inside the ccapi container:
-    python3 -c "from run.initialization.init_exam_graph import init; import json; print(json.dumps(init()))"
-
-NOTE: only *top-level* topics are seeded (the granularity a teacher plans against). The full sub-point
-breakdown (e.g. 4.1.1.1 ...) is a later data-population task (sourceable from the AQA spec PDF via
-Docling). Seeding all 6 specs means a template's spec_ref finds a matching SpecPoint so
-(:Part)-[:ASSESSES]->(:SpecPoint) fires beyond GCSE Physics; spec_code (e.g. AQA-PHYS-8463) must match
-the eb_exams/eb_specifications seed (card S4-3) and the app's deriveSpecCode.
-"""
-import uuid
-from typing import Dict, Any
-
-from modules.database.tools.neo4j_driver_tools import get_driver
-
-EXAM_DB = "cc.public.exams"
-NS = uuid.UUID("00000000-0000-0000-0000-00000000e8a1")  # stable namespace for deterministic uuids
-
-BOARD = {"code": "AQA", "name": "AQA"}
-SPEC = {
-    "spec_code": "AQA-PHYS-8463",
-    "exam_board_code": "AQA",
-    "subject_code": "PHYS",
-    "award_code": "GCSE",
-    "title": "AQA GCSE Physics (8463)",
-}
-# Real AQA GCSE Physics (8463) top-level topics (ref = topic number).
-SPEC_POINTS = [
-    ("4.1", "Energy"),
-    ("4.2", "Electricity"),
-    ("4.3", "Particle model of matter"),
-    ("4.4", "Atomic structure"),
-    ("4.5", "Forces"),
-    ("4.6", "Waves"),
-    ("4.7", "Magnetism and electromagnetism"),
-    ("4.8", "Space physics"),
-]
-
-# Full AQA catalogue for the current test specs (top-level topics; ref = topic number). Seeding all of
-# them means a template's spec_ref finds a matching SpecPoint so (:Part)-[:ASSESSES]->(:SpecPoint) fires
-# beyond AQA GCSE Physics. Sub-point granularity (e.g. 4.1.1.1) remains a later data-population task.
-SPECIFICATIONS = [
-    {**SPEC, "topics": SPEC_POINTS},
-    {"spec_code": "AQA-CHEM-8462", "exam_board_code": "AQA", "subject_code": "CHEM", "award_code": "GCSE",
-     "title": "AQA GCSE Chemistry (8462)", "topics": [
-        ("4.1", "Atomic structure and the periodic table"), ("4.2", "Bonding, structure, and the properties of matter"),
-        ("4.3", "Quantitative chemistry"), ("4.4", "Chemical changes"), ("4.5", "Energy changes"),
-        ("4.6", "The rate and extent of chemical change"), ("4.7", "Organic chemistry"), ("4.8", "Chemical analysis"),
-        ("4.9", "Chemistry of the atmosphere"), ("4.10", "Using resources")]},
-    {"spec_code": "AQA-BIOL-8461", "exam_board_code": "AQA", "subject_code": "BIOL", "award_code": "GCSE",
-     "title": "AQA GCSE Biology (8461)", "topics": [
-        ("4.1", "Cell biology"), ("4.2", "Organisation"), ("4.3", "Infection and response"), ("4.4", "Bioenergetics"),
-        ("4.5", "Homeostasis and response"), ("4.6", "Inheritance, variation and evolution"), ("4.7", "Ecology")]},
-    {"spec_code": "AQA-PHYS-7408", "exam_board_code": "AQA", "subject_code": "PHYS", "award_code": "A-level",
-     "title": "AQA A-level Physics (7408)", "topics": [
-        ("3.1", "Measurements and their errors"), ("3.2", "Particles and radiation"), ("3.3", "Waves"),
-        ("3.4", "Mechanics and materials"), ("3.5", "Electricity"), ("3.6", "Further mechanics and thermal physics"),
-        ("3.7", "Fields and their consequences"), ("3.8", "Nuclear physics")]},
-    {"spec_code": "AQA-CHEM-7405", "exam_board_code": "AQA", "subject_code": "CHEM", "award_code": "A-level",
-     "title": "AQA A-level Chemistry (7405)", "topics": [
-        ("3.1", "Physical chemistry"), ("3.2", "Inorganic chemistry"), ("3.3", "Organic chemistry")]},
-    {"spec_code": "AQA-BIOL-7402", "exam_board_code": "AQA", "subject_code": "BIOL", "award_code": "A-level",
-     "title": "AQA A-level Biology (7402)", "topics": [
-        ("3.1", "Biological molecules"), ("3.2", "Cells"), ("3.3", "Organisms exchange substances with their environment"),
-        ("3.4", "Genetic information, variation and relationships between organisms"),
-        ("3.5", "Energy transfers in and between organisms"), ("3.6", "Organisms respond to changes"),
-        ("3.7", "Genetics, populations, evolution and ecosystems"), ("3.8", "The control of gene expression")]},
-]
-
-CONSTRAINTS = [
-    "CREATE CONSTRAINT exam_board_uid    IF NOT EXISTS FOR (n:ExamBoard)     REQUIRE n.uuid_string IS UNIQUE",
-    "CREATE CONSTRAINT spec_uid          IF NOT EXISTS FOR (n:Specification) REQUIRE n.uuid_string IS UNIQUE",
-    "CREATE CONSTRAINT specpoint_uid     IF NOT EXISTS FOR (n:SpecPoint)     REQUIRE n.uuid_string IS UNIQUE",
-    "CREATE CONSTRAINT exampaper_uid     IF NOT EXISTS FOR (n:ExamPaper)     REQUIRE n.uuid_string IS UNIQUE",
-    "CREATE CONSTRAINT question_uid      IF NOT EXISTS FOR (n:Question)      REQUIRE n.uuid_string IS UNIQUE",
-    "CREATE CONSTRAINT part_uid          IF NOT EXISTS FOR (n:Part)          REQUIRE n.uuid_string IS UNIQUE",
-    "CREATE CONSTRAINT region_uid        IF NOT EXISTS FOR (n:Region)        REQUIRE n.uuid_string IS UNIQUE",
-    "CREATE CONSTRAINT spec_code_unique  IF NOT EXISTS FOR (n:Specification) REQUIRE n.spec_code   IS UNIQUE",
-    "CREATE CONSTRAINT exam_code_unique  IF NOT EXISTS FOR (n:ExamPaper)     REQUIRE n.exam_code   IS UNIQUE",
-    "CREATE CONSTRAINT board_code_unique IF NOT EXISTS FOR (n:ExamBoard)     REQUIRE n.code        IS UNIQUE",
-]
-
-
-def _uid(*parts: str) -> str:
-    return str(uuid.uuid5(NS, ":".join(parts)))
-
-
-def init() -> Dict[str, Any]:
-    driver = get_driver()
-    result: Dict[str, Any] = {"db": EXAM_DB, "constraints": 0, "spec_points": 0}
-
-    # 1. database
-    with driver.session(database="system") as s:
-        s.run(f"CREATE DATABASE `{EXAM_DB}` IF NOT EXISTS").consume()
-    # wait for availability
-    import time
-    for _ in range(30):
-        with driver.session(database="system") as s:
-            st = s.run("SHOW DATABASE $n YIELD currentStatus RETURN currentStatus", n=EXAM_DB).single()
-        if st and st["currentStatus"] == "online":
-            break
-        time.sleep(1)
-
-    with driver.session(database=EXAM_DB) as s:
-        # 2. constraints
-        for c in CONSTRAINTS:
-            s.run(c).consume()
-            result["constraints"] += 1
-
-        # 3. board (once)
-        board_uid = _uid("ExamBoard", BOARD["code"])
-        s.run(
-            "MERGE (b:ExamBoard {uuid_string:$uid}) "
-            "SET b.code=$code, b.name=$name, b.node_storage_path=$nsp",
-            uid=board_uid, code=BOARD["code"], name=BOARD["name"],
-            nsp=f"{EXAM_DB}/ExamBoard/{BOARD['code']}",
-        ).consume()
-
-        # 4. each specification + its top-level spec points (idempotent MERGE)
-        for spec in SPECIFICATIONS:
-            spec_uid = _uid("Specification", spec["spec_code"])
-            s.run(
-                "MERGE (sp:Specification {uuid_string:$uid}) "
-                "SET sp.spec_code=$sc, sp.exam_board_code=$ebc, sp.subject_code=$subj, "
-                "    sp.award_code=$award, sp.title=$title, sp.node_storage_path=$nsp "
-                "WITH sp MATCH (b:ExamBoard {code:$ebc}) MERGE (b)-[:PUBLISHES]->(sp)",
-                uid=spec_uid, sc=spec["spec_code"], ebc=spec["exam_board_code"],
-                subj=spec["subject_code"], award=spec["award_code"], title=spec["title"],
-                nsp=f"{EXAM_DB}/Specification/{spec['spec_code']}",
-            ).consume()
-            for ref, desc in spec["topics"]:
-                sp_uid = _uid("SpecPoint", spec["spec_code"], ref)
-                s.run(
-                    "MERGE (p:SpecPoint {uuid_string:$uid}) "
-                    "SET p.ref=$ref, p.description=$desc, p.spec_code=$sc, "
-                    "    p.exam_board_code=$ebc, p.node_storage_path=$nsp "
-                    "WITH p MATCH (s:Specification {spec_code:$sc}) MERGE (s)-[:HAS_SPEC_POINT]->(p)",
-                    uid=sp_uid, ref=ref, desc=desc, sc=spec["spec_code"],
-                    ebc=spec["exam_board_code"], nsp=f"{EXAM_DB}/SpecPoint/{spec['spec_code']}/{ref}",
-                ).consume()
-                result["spec_points"] += 1
-
-        counts = s.run(
-            "MATCH (b:ExamBoard) WITH count(b) AS boards "
-            "MATCH (sp:Specification) WITH boards, count(sp) AS specs "
-            "MATCH (p:SpecPoint) RETURN boards, specs, count(p) AS spec_points"
-        ).single()
-        result["verify"] = dict(counts) if counts else {}
-
-    return result
-
-
-if __name__ == "__main__":
-    import json
-    print(json.dumps(init(), indent=2, default=str))

run/initialization/manifests/_corpus_store/.gitignore

@@ -1,3 +0,0 @@
-# Persistent local corpus store — PDFs are NOT committed (re-downloadable from manifest).
-*
-!.gitignore

run/initialization/manifests/generate_corpus_manifest.py

@@ -1,501 +0,0 @@
-#!/usr/bin/env python3
-"""
-generate_corpus_manifest.py — build the public exam-corpus manifest from OFFICIAL sources,
-verifying every source URL is live before it is written.
-
-Output: exam-corpus.yaml (consumed by run/initialization/seed_exam_corpus.py).
-
-Sources (all official exam-board hosts; public past-paper PDFs):
-  AQA      filestore.aqa.org.uk  — fully templatable; enumerated + HEAD-verified here.
-  Edexcel  qualifications.pearson.com — date suffix non-derivable; confirmed URLs embedded.
-  OCR      www.ocr.org.uk/Images — opaque doc-id; confirmed URLs embedded.
-
-Every URL is HEAD/GET-checked (200 + application/pdf) before inclusion, so the committed
-manifest never carries a dead or wrong-cased link. Re-run to refresh as more sessions go public.
-
-Conventions (locked — see ~/cc/ideas/2026-06-07-exam-paper-ingestion.md):
-  session   = "YYYY-Mon"        e.g. 2022-Jun
-  exam_code = BOARD-award-PAPER-SESSIONCOMPACT-ROLE   e.g. AQA-8463-1H-2022JUN-QP
-"""
-from __future__ import annotations
-import concurrent.futures as cf
-import os
-import sys
-import urllib.error
-import urllib.request
-from typing import Any, Dict, List, Optional, Tuple
-
-import yaml
-
-AQA_BASE = "https://filestore.aqa.org.uk/sample-papers-and-mark-schemes"
-ROLE_TOKEN = {"QP": "QP", "MS": "MS", "ER": "WRE"}       # AQA filestore role tokens
-MONTHS = {"JUN": ("june", "Jun"), "NOV": ("november", "Nov")}
-FETCHED = "2026-06-07"
-
-
-def head_ok(url: str, timeout: int = 20) -> bool:
-    """True iff the URL resolves to a real PDF (200 + application/pdf), following redirects.
-    AQA soft-404s redirect to www.aqa.org.uk/req_path=... (text/html), so we check content-type.
-    Uses a tiny Range GET (stdlib urllib) so we never pull the whole PDF just to verify it."""
-    req = urllib.request.Request(url, headers={"Range": "bytes=0-3", "User-Agent": "cc-corpus/1.0"})
-    try:
-        with urllib.request.urlopen(req, timeout=timeout) as r:
-            ctype = (r.headers.get("content-type") or "").lower()
-            return r.status in (200, 206) and "pdf" in ctype
-    except urllib.error.HTTPError as e:
-        # A 206/200 PDF never lands here; 404/redirect-to-html will.
-        ctype = (e.headers.get("content-type") or "").lower() if e.headers else ""
-        return e.code in (200, 206) and "pdf" in ctype
-    except Exception:
-        return False
-
-
-# ─────────────────────────── AQA catalogue ───────────────────────────
-# spec_code, subject, award, award_level, first_teach, [(filestore_papercode, paper_code, tier), ...]
-def _gcse_single(award: str) -> List[Tuple[str, str, Optional[str]]]:
-    out = []
-    for paper in ("1", "2"):
-        for tier in ("F", "H"):
-            out.append((f"{award}{paper}{tier}", f"{award}/{paper}{tier}", tier))
-    return out
-
-def _trilogy(award: str) -> List[Tuple[str, str, Optional[str]]]:
-    out = []
-    for subj in ("B", "C", "P"):
-        for paper in ("1", "2"):
-            for tier in ("F", "H"):
-                out.append((f"{award}{subj}{paper}{tier}", f"{award}/{subj}/{paper}{tier}", tier))
-    return out
-
-def _alevel(award: str, papers=("1", "2", "3")) -> List[Tuple[str, str, Optional[str]]]:
-    return [(f"{award}{p}", f"{award}/{p}", None) for p in papers]
-
-def _subj(award: str, papers, tiers=(None,)) -> List[Tuple[str, str, Optional[str]]]:
-    """Generic GCSE/A-level builder. tiers=('F','H') for tiered subjects (Maths/Science),
-    tiers=(None,) for untiered (English/Geography/CS/Business/Psychology)."""
-    out = []
-    for p in papers:
-        for t in tiers:
-            tl = t or ""
-            out.append((f"{award}{p}{tl}", f"{award}/{p}{tl}", t))
-    return out
-
-def _mfl(award: str) -> List[Tuple[str, str, Optional[str]]]:
-    """AQA MFL: Listening/Reading/Writing papers, each Foundation/Higher (Speaking is teacher-conducted,
-    no public QP). Filestore code encodes skill+tier, e.g. 8658LH = French Listening Higher."""
-    out = []
-    for skill in ("L", "R", "W"):
-        for t in ("F", "H"):
-            out.append((f"{award}{skill}{t}", f"{award}/{skill}{t}", t))
-    return out
-
-AQA_SPECS = [
-    # ── Sciences (round 1 — kept at full depth) ──────────────────────────────────────
-    ("AQA-BIOL-8461", "BIOLOGY", "8461", "GCSE", "2016", _gcse_single("8461")),
-    ("AQA-CHEM-8462", "CHEMISTRY", "8462", "GCSE", "2016", _gcse_single("8462")),
-    ("AQA-PHYS-8463", "PHYSICS", "8463", "GCSE", "2016", _gcse_single("8463")),
-    ("AQA-COMB-8464", "COMBINED SCIENCE TRILOGY", "8464", "GCSE", "2016", _trilogy("8464")),
-    ("AQA-BIOL-7401", "BIOLOGY", "7401", "AS", "2015", _alevel("7401", ("1", "2"))),
-    ("AQA-BIOL-7402", "BIOLOGY", "7402", "A-level", "2015", _alevel("7402")),
-    ("AQA-CHEM-7404", "CHEMISTRY", "7404", "AS", "2015", _alevel("7404", ("1", "2"))),
-    ("AQA-CHEM-7405", "CHEMISTRY", "7405", "A-level", "2015", _alevel("7405")),
-    ("AQA-PHYS-7407", "PHYSICS", "7407", "AS", "2015", _alevel("7407", ("1", "2"))),
-    ("AQA-PHYS-7408", "PHYSICS", "7408", "A-level", "2015", _alevel("7408")),
-    # ── Round 2 breadth — high-volume core (Maths, English) ───────────────────────────
-    ("AQA-MATH-8300", "MATHEMATICS", "8300", "GCSE", "2015", _subj("8300", ("1", "2", "3"), ("F", "H"))),
-    ("AQA-MATH-7357", "MATHEMATICS", "7357", "A-level", "2017", _alevel("7357", ("1", "2", "3"))),
-    ("AQA-MATH-7356", "MATHEMATICS", "7356", "AS", "2017", _alevel("7356", ("1", "2"))),
-    ("AQA-ENGL-8700", "ENGLISH LANGUAGE", "8700", "GCSE", "2015", _subj("8700", ("1", "2"))),
-    ("AQA-ENGLIT-8702", "ENGLISH LITERATURE", "8702", "GCSE", "2015", _subj("8702", ("1", "2"))),
-    ("AQA-ENGL-7702", "ENGLISH LANGUAGE", "7702", "A-level", "2015", _alevel("7702", ("1", "2"))),
-    ("AQA-ENGLIT-7712", "ENGLISH LITERATURE A", "7712", "A-level", "2015", _alevel("7712", ("1", "2"))),
-    # ── Round 2 breadth — humanities / others ─────────────────────────────────────────
-    ("AQA-GEOG-8035", "GEOGRAPHY", "8035", "GCSE", "2016", _subj("8035", ("1", "2", "3"))),
-    ("AQA-GEOG-7037", "GEOGRAPHY", "7037", "A-level", "2016", _alevel("7037", ("1", "2"))),
-    ("AQA-COMP-8525", "COMPUTER SCIENCE", "8525", "GCSE", "2020", _subj("8525", ("1", "2"))),
-    ("AQA-COMP-7517", "COMPUTER SCIENCE", "7517", "A-level", "2015", _alevel("7517", ("1", "2"))),
-    ("AQA-BUS-8132", "BUSINESS", "8132", "GCSE", "2017", _subj("8132", ("1", "2"))),
-    ("AQA-BUS-7132", "BUSINESS", "7132", "A-level", "2015", _alevel("7132", ("1", "2", "3"))),
-    ("AQA-PSYC-8182", "PSYCHOLOGY", "8182", "GCSE", "2017", _subj("8182", ("1", "2"))),
-    ("AQA-PSYC-7182", "PSYCHOLOGY", "7182", "A-level", "2015", _alevel("7182", ("1", "2", "3"))),
-    # ── Round 2 breadth — modern foreign languages (Listening/Reading/Writing, F+H) ───
-    ("AQA-FREN-8658", "FRENCH", "8658", "GCSE", "2016", _mfl("8658")),
-    ("AQA-SPAN-8698", "SPANISH", "8698", "GCSE", "2016", _mfl("8698")),
-    ("AQA-GERM-8668", "GERMAN", "8668", "GCSE", "2016", _mfl("8668")),
-    ("AQA-FREN-7652", "FRENCH", "7652", "A-level", "2016", _alevel("7652", ("1", "2"))),
-    ("AQA-SPAN-7692", "SPANISH", "7692", "A-level", "2016", _alevel("7692", ("1", "2"))),
-    ("AQA-GERM-7662", "GERMAN", "7662", "A-level", "2016", _alevel("7662", ("1", "2"))),
-]
-AQA_SESSIONS = ["JUN18", "JUN19", "NOV20", "NOV21", "JUN22", "JUN23", "JUN24"]
-AQA_ROLES = ["QP", "MS", "ER"]
-
-
-def aqa_url(papercode: str, role: str, session: str) -> Tuple[str, str]:
-    mon = session[:3]
-    yy = session[3:]
-    folder, _ = MONTHS[mon]
-    year = "20" + yy
-    fname = f"AQA-{papercode}-{ROLE_TOKEN[role]}-{session}.PDF"
-    return f"{AQA_BASE}/{year}/{folder}/{fname}", fname
-
-
-def session_pretty(session: str) -> Tuple[str, str]:
-    mon = session[:3]       # "JUN" | "NOV"
-    yy = session[3:]        # "22"
-    _, pretty = MONTHS[mon]
-    # ("2022-Jun" display session, "2022JUN" compact for exam_code — year-first, matches the
-    # locked exam_code convention and the Edexcel/OCR entries).
-    return f"20{yy}-{pretty}", f"20{yy}{mon}"
-
-
-def build_aqa() -> Dict[str, Any]:
-    candidates: List[Tuple[str, str, str, str, str, str, Optional[str], str, str, str]] = []
-    # (spec_code, subject, award, paper_fc, paper_code, tier, role, session, url, fname)
-    spec_meta = {}
-    for spec_code, subject, award, level, first_teach, papers in AQA_SPECS:
-        spec_meta[spec_code] = (subject, award, level, first_teach)
-        for paper_fc, paper_code, tier in papers:
-            for session in AQA_SESSIONS:
-                for role in AQA_ROLES:
-                    url, fname = aqa_url(paper_fc, role, session)
-                    candidates.append((spec_code, subject, award, paper_fc, paper_code, tier,
-                                       role, session, url, fname))
-
-    print(f"[AQA] HEAD-verifying {len(candidates)} candidate URLs...", file=sys.stderr)
-    live: Dict[int, bool] = {}
-    with cf.ThreadPoolExecutor(max_workers=24) as ex:
-        futs = {ex.submit(head_ok, c[8]): i for i, c in enumerate(candidates)}
-        done = 0
-        for fut in cf.as_completed(futs):
-            i = futs[fut]
-            live[i] = fut.result()
-            done += 1
-            if done % 60 == 0:
-                print(f"  ...{done}/{len(candidates)} ({sum(live.values())} live)", file=sys.stderr)
-
-    specs: Dict[str, Dict[str, Any]] = {}
-    for i, c in enumerate(candidates):
-        if not live.get(i):
-            continue
-        spec_code, subject, award, paper_fc, paper_code, tier, role, session, url, fname = c
-        sess_pretty, sess_compact = session_pretty(session)
-        token = paper_fc[len(award):]  # "1H" / "P1H" / "1"
-        exam_code = f"AQA-{award}-{token}-{sess_compact}-{role}"
-        spec = specs.setdefault(spec_code, {"papers": []})
-        spec["papers"].append({
-            "exam_code": exam_code,
-            "paper_code": paper_code,
-            "tier": tier,
-            "session": sess_pretty,
-            "doc_type": role,
-            "file": {
-                "source": f"url:{url}",
-                "original_name": fname,
-                "provenance": {"source_url": url, "fetched": FETCHED,
-                               "license": "AQA public past paper"},
-            },
-        })
-
-    spec_list = []
-    for spec_code, subject, award, level, first_teach, _papers in AQA_SPECS:
-        if spec_code not in specs:
-            continue
-        papers = sorted(specs[spec_code]["papers"], key=lambda p: p["exam_code"])
-        spec_list.append({
-            "spec_code": spec_code, "exam_board_code": "AQA", "subject_code": subject,
-            "award_code": award, "award_level": level, "first_teach": first_teach,
-            "papers": papers,
-        })
-        print(f"[AQA] {spec_code}: {len(papers)} live papers", file=sys.stderr)
-    return {"exam_board_code": "AQA", "specifications": spec_list}
-
-
-# ─────────────── Edexcel / OCR — confirmed direct URLs (re-verified at build) ───────────────
-# These boards aren't templatable (Edexcel has a non-derivable date suffix; OCR uses opaque
-# doc-ids), so confirmed URLs are listed as 6-tuples: (spec_code, paper_code, tier, session, role,
-# url). exam_code is DERIVED (see _mk_exam_code) so it always matches the locked convention.
-EXAM_CODE_PREFIX = {"EDEXCEL": "EDX", "OCR": "OCR"}
-
-def _ec_token(paper_code: str) -> str:
-    t = paper_code.split("/")[-1]
-    return str(int(t)) if t.isdigit() else t   # "01"->"1", "1H"->"1H", "1CH"->"1CH", "11"->"11"
-
-def _mk_exam_code(prefix: str, award: str, paper_code: str, session: str, role: str) -> str:
-    y, m = session.split("-")
-    return f"{prefix}-{award}-{_ec_token(paper_code)}-{y}{m.upper()}-{role}"
-
-_PE = "https://qualifications.pearson.com/content/dam/pdf"
-_EDX = f"{_PE}/GCSE/Science/2016"
-_OCR = "https://www.ocr.org.uk/Images"
-
-EDEXCEL_SPECS = {
-    "EDX-BIOL-1BI0": ("BIOLOGY", "1BI0", "GCSE", "2016"),
-    "EDX-CHEM-1CH0": ("CHEMISTRY", "1CH0", "GCSE", "2016"),
-    "EDX-PHYS-1PH0": ("PHYSICS", "1PH0", "GCSE", "2016"),
-    "EDX-COMB-1SC0": ("COMBINED SCIENCE", "1SC0", "GCSE", "2016"),
-    "EDX-MATH-1MA1": ("MATHEMATICS", "1MA1", "GCSE", "2015"),
-    "EDX-ENGL-1EN0": ("ENGLISH LANGUAGE", "1EN0", "GCSE", "2015"),
-    "EDX-ENGLIT-1ET0": ("ENGLISH LITERATURE", "1ET0", "GCSE", "2015"),
-    "EDX-GEOG-1GA0": ("GEOGRAPHY A", "1GA0", "GCSE", "2016"),
-    "EDX-HIST-1HI0": ("HISTORY", "1HI0", "GCSE", "2016"),
-    "EDX-BUS-1BS0": ("BUSINESS", "1BS0", "GCSE", "2017"),
-    "EDX-COMP-1CP2": ("COMPUTER SCIENCE", "1CP2", "GCSE", "2020"),
-    "EDX-MATH-9MA0": ("MATHEMATICS", "9MA0", "A-level", "2017"),
-    "EDX-ENGL-9EN0": ("ENGLISH LANGUAGE", "9EN0", "A-level", "2015"),
-    "EDX-ENGLIT-9ET0": ("ENGLISH LITERATURE", "9ET0", "A-level", "2015"),
-    "EDX-GEOG-9GE0": ("GEOGRAPHY", "9GE0", "A-level", "2016"),
-}
-EDEXCEL_PAPERS = [
-    # ── Sciences (round 1) ──
-    ("EDX-BIOL-1BI0", "1BI0/1H", "H", "2024-Jun", "QP", f"{_EDX}/Exam-materials/1bi0-1h-que-20240511.pdf"),
-    ("EDX-BIOL-1BI0", "1BI0/2F", "F", "2023-Jun", "QP", f"{_EDX}/Exam-materials/1bi0-2f-que-20230610.pdf"),
-    ("EDX-BIOL-1BI0", "1BI0/2H", "H", "2023-Jun", "QP", f"{_EDX}/Exam-materials/1bi0-2h-que-20230610.pdf"),
-    ("EDX-BIOL-1BI0", "1BI0/1F", "F", "2023-Jun", "MS", f"{_EDX}/Exam-materials/1bi0-1f-rms-20230824.pdf"),
-    ("EDX-BIOL-1BI0", "1BI0/1H", "H", "2024-Jun", "MS", f"{_EDX}/Exam-materials/1bi0-1h-rms-20240822.pdf"),
-    ("EDX-BIOL-1BI0", "1BI0/1H", "H", "2022-Jun", "MS", f"{_EDX}/exam-materials/1bi0-1h-rms-20220825.pdf"),
-    ("EDX-CHEM-1CH0", "1CH0/1F", "F", "2023-Jun", "QP", f"{_EDX}/Exam-materials/1ch0-1f-que-20230523.pdf"),
-    ("EDX-CHEM-1CH0", "1CH0/1H", "H", "2024-Jun", "QP", f"{_EDX}/Exam-materials/1ch0-1h-que-20240518.pdf"),
-    ("EDX-CHEM-1CH0", "1CH0/2H", "H", "2024-Jun", "MS", f"{_EDX}/Exam-materials/1ch0-2h-rms-20240822.pdf"),
-    ("EDX-PHYS-1PH0", "1PH0/1H", "H", "2023-Jun", "QP", f"{_EDX}/Exam-materials/1ph0-1h-que-20230526.pdf"),
-    ("EDX-PHYS-1PH0", "1PH0/2F", "F", "2023-Jun", "QP", f"{_EDX}/Exam-materials/1ph0-2f-que-20230617.pdf"),
-    ("EDX-PHYS-1PH0", "1PH0/1H", "H", "2024-Jun", "QP", f"{_EDX}/Exam-materials/1ph0-1h-que-20240523.pdf"),
-    ("EDX-PHYS-1PH0", "1PH0/2H", "H", "2023-Jun", "MS", f"{_EDX}/Exam-materials/1ph0-2h-rms-20230824.pdf"),
-    ("EDX-PHYS-1PH0", "1PH0/2H", "H", "2022-Jun", "MS", f"{_EDX}/exam-materials/1ph0-2h-rms-20220825.pdf"),
-    ("EDX-COMB-1SC0", "1SC0/1CH", None, "2023-Jun", "MS", f"{_EDX}/Exam-materials/1sc0-1ch-rms-20230824.pdf"),
-    # ── Maths 1MA1 (round 2) ──
-    ("EDX-MATH-1MA1", "1MA1/1H", "H", "2023-Jun", "QP", f"{_PE}/GCSE/Mathematics/2015/Exam-materials/1ma1-1h-que-20230520.pdf"),
-    ("EDX-MATH-1MA1", "1MA1/1H", "H", "2023-Jun", "MS", f"{_PE}/GCSE/Mathematics/2015/Exam-materials/1ma1-1h-rms-20230824.pdf"),
-    ("EDX-MATH-1MA1", "1MA1/1F", "F", "2023-Jun", "MS", f"{_PE}/GCSE/Mathematics/2015/Exam-materials/1ma1-1f-rms-20230824.pdf"),
-    ("EDX-MATH-1MA1", "1MA1/1F", "F", "2024-Jun", "QP", f"{_PE}/GCSE/Mathematics/2015/Exam-materials/1ma1-1f-que-20240517.pdf"),
-    ("EDX-MATH-1MA1", "1MA1/1H", "H", "2024-Jun", "QP", f"{_PE}/GCSE/Mathematics/2015/Exam-materials/1ma1-1h-que-20240517.pdf"),
-    ("EDX-MATH-1MA1", "1MA1/1F", "F", "2024-Jun", "MS", f"{_PE}/GCSE/Mathematics/2015/Exam-materials/1ma1-1f-rms-20240822.pdf"),
-    ("EDX-MATH-1MA1", "1MA1/1H", "H", "2023-Nov", "MS", f"{_PE}/GCSE/Mathematics/2015/Exam-materials/1ma1-1h-rms-20240111.pdf"),
-    ("EDX-MATH-1MA1", "1MA1/1H", "H", "2022-Jun", "MS", f"{_PE}/GCSE/mathematics/2015/exam-materials/1ma1-1h-rms-20220825.pdf"),
-    ("EDX-MATH-1MA1", "1MA1/3H", "H", "2022-Jun", "MS", f"{_PE}/GCSE/mathematics/2015/exam-materials/1ma1-3h-rms-20220825.pdf"),
-    # ── English Language 1EN0 / Literature 1ET0 (round 2) ──
-    ("EDX-ENGL-1EN0", "1EN0/01", None, "2024-Jun", "QP", f"{_PE}/GCSE/English-Language/2015/Exam-materials/1en0-01-que-20240524.pdf"),
-    ("EDX-ENGL-1EN0", "1EN0/01", None, "2023-Nov", "QP", f"{_PE}/GCSE/English-Language/2015/Exam-materials/1en0-01-que-20231108.pdf"),
-    ("EDX-ENGL-1EN0", "1EN0/01", None, "2024-Jun", "MS", f"{_PE}/GCSE/English-Language/2015/Exam-materials/1en0-01-rms-20240822.pdf"),
-    ("EDX-ENGL-1EN0", "1EN0/02", None, "2024-Jun", "MS", f"{_PE}/GCSE/English-Language/2015/Exam-materials/1en0-02-rms-20240822.pdf"),
-    ("EDX-ENGL-1EN0", "1EN0/01", None, "2023-Jun", "MS", f"{_PE}/GCSE/English-Language/2015/Exam-materials/1en0-01-rms-20230824.pdf"),
-    ("EDX-ENGL-1EN0", "1EN0/02", None, "2023-Jun", "MS", f"{_PE}/GCSE/English-Language/2015/Exam-materials/1en0-02-rms-20230824.pdf"),
-    ("EDX-ENGLIT-1ET0", "1ET0/01", None, "2023-Jun", "QP", f"{_PE}/GCSE/English-Literature/2015/Exam-materials/1et0-01-que-20230518.pdf"),
-    ("EDX-ENGLIT-1ET0", "1ET0/02", None, "2023-Jun", "QP", f"{_PE}/GCSE/English-Literature/2015/Exam-materials/1et0-02-que-20230525.pdf"),
-    ("EDX-ENGLIT-1ET0", "1ET0/02", None, "2024-Jun", "QP", f"{_PE}/GCSE/English-Literature/2015/Exam-materials/1et0-02-que-20240521.pdf"),
-    ("EDX-ENGLIT-1ET0", "1ET0/01", None, "2023-Jun", "MS", f"{_PE}/GCSE/English-Literature/2015/Exam-materials/1et0-01-rms-20230824.pdf"),
-    ("EDX-ENGLIT-1ET0", "1ET0/01", None, "2024-Jun", "MS", f"{_PE}/GCSE/English-Literature/2015/Exam-materials/1et0-01-rms-20240822.pdf"),
-    # ── A-level Maths 9MA0 / English 9EN0 / 9ET0 (round 2) ──
-    ("EDX-MATH-9MA0", "9MA0/01", None, "2023-Jun", "QP", f"{_PE}/A-Level/Mathematics/2017/Exam-materials/9ma0-01-que-20230607.pdf"),
-    ("EDX-MATH-9MA0", "9MA0/31", None, "2023-Jun", "QP", f"{_PE}/A-Level/Mathematics/2017/Exam-materials/9ma0-31-que-20230621.pdf"),
-    ("EDX-MATH-9MA0", "9MA0/02", None, "2024-Jun", "QP", f"{_PE}/A-Level/Mathematics/2017/Exam-materials/9ma0-02-que-20240612.pdf"),
-    ("EDX-MATH-9MA0", "9MA0/31", None, "2023-Jun", "MS", f"{_PE}/A-Level/Mathematics/2017/Exam-materials/9ma0-31-rms-20230817.pdf"),
-    ("EDX-MATH-9MA0", "9MA0/01", None, "2024-Jun", "MS", f"{_PE}/A-Level/Mathematics/2017/Exam-materials/9ma0-01-rms-20240815.pdf"),
-    ("EDX-ENGL-9EN0", "9EN0/01", None, "2024-Jun", "MS", f"{_PE}/A-Level/English-Language/2015/Exam-materials/9en0-01-rms-20240815.pdf"),
-    ("EDX-ENGL-9EN0", "9EN0/02", None, "2024-Jun", "MS", f"{_PE}/A-Level/English-Language/2015/Exam-materials/9en0-02-rms-20240815.pdf"),
-    ("EDX-ENGLIT-9ET0", "9ET0/01", None, "2024-Jun", "QP", f"{_PE}/A-Level/English-Literature/2015/Exam-materials/9et0-01-que-20240525.pdf"),
-    ("EDX-ENGLIT-9ET0", "9ET0/01", None, "2023-Jun", "MS", f"{_PE}/A-Level/English-Literature/2015/Exam-materials/9et0-01-rms-20230817.pdf"),
-    ("EDX-ENGLIT-9ET0", "9ET0/03", None, "2023-Jun", "MS", f"{_PE}/A-Level/English-Literature/2015/Exam-materials/9et0-03-rms-20230817.pdf"),
-    # ── Humanities (round 2) ──
-    ("EDX-GEOG-1GA0", "1GA0/01", None, "2023-Jun", "QP", f"{_PE}/GCSE/Geography-A/2016/Exam-materials/1ga0-01-que-20230523.pdf"),
-    ("EDX-GEOG-1GA0", "1GA0/01", None, "2023-Jun", "MS", f"{_PE}/GCSE/Geography-A/2016/Exam-materials/1ga0-01-rms-20230824.pdf"),
-    ("EDX-GEOG-1GA0", "1GA0/02", None, "2023-Jun", "QP", f"{_PE}/GCSE/Geography-A/2016/Exam-materials/1ga0-02-que-20230610.pdf"),
-    ("EDX-GEOG-1GA0", "1GA0/02", None, "2023-Jun", "MS", f"{_PE}/GCSE/Geography-A/2016/Exam-materials/1ga0-02-rms-20230824.pdf"),
-    ("EDX-GEOG-1GA0", "1GA0/01", None, "2024-Jun", "MS", f"{_PE}/GCSE/Geography-A/2016/Exam-materials/1ga0-01-rms-20240822.pdf"),
-    ("EDX-GEOG-1GA0", "1GA0/03", None, "2024-Jun", "QP", f"{_PE}/GCSE/Geography-A/2016/Exam-materials/1ga0-03-que-20240615.pdf"),
-    ("EDX-HIST-1HI0", "1HI0/10", None, "2023-Jun", "QP", f"{_PE}/GCSE/History/2016/Exam-materials/1hi0-10-que-20230519.pdf"),
-    ("EDX-HIST-1HI0", "1HI0/10", None, "2023-Jun", "MS", f"{_PE}/GCSE/History/2016/Exam-materials/1hi0-10-rms-20230824.pdf"),
-    ("EDX-HIST-1HI0", "1HI0/12", None, "2023-Jun", "MS", f"{_PE}/GCSE/History/2016/Exam-materials/1hi0-12-rms-20230824.pdf"),
-    ("EDX-HIST-1HI0", "1HI0/13", None, "2024-Jun", "MS", f"{_PE}/GCSE/History/2016/Exam-materials/1hi0-13-rms-20240822.pdf"),
-    ("EDX-HIST-1HI0", "1HI0/33", None, "2023-Jun", "MS", f"{_PE}/GCSE/History/2016/Exam-materials/1hi0-33-rms-20230824.pdf"),
-    ("EDX-BUS-1BS0", "1BS0/01", None, "2023-Jun", "QP", f"{_PE}/GCSE/Business/2017/Exam-materials/1bs0-01-que-20230519.pdf"),
-    ("EDX-BUS-1BS0", "1BS0/02", None, "2023-Jun", "QP", f"{_PE}/GCSE/Business/2017/Exam-materials/1bs0-02-que-20230613.pdf"),
-    ("EDX-BUS-1BS0", "1BS0/02", None, "2023-Jun", "MS", f"{_PE}/GCSE/Business/2017/Exam-materials/1bs0-02-rms-20230824.pdf"),
-    ("EDX-BUS-1BS0", "1BS0/02", None, "2024-Jun", "QP", f"{_PE}/GCSE/Business/2017/Exam-materials/1bs0-02-que-20240606.pdf"),
-    ("EDX-BUS-1BS0", "1BS0/01", None, "2024-Jun", "MS", f"{_PE}/GCSE/Business/2017/Exam-materials/1bs0-01-rms-20240822.pdf"),
-    ("EDX-COMP-1CP2", "1CP2/01", None, "2023-Jun", "QP", f"{_PE}/GCSE/Computer-science/2020/Exam-materials/1cp2-01-que-20230520.pdf"),
-    ("EDX-COMP-1CP2", "1CP2/01", None, "2023-Jun", "MS", f"{_PE}/GCSE/Computer-science/2020/Exam-materials/1cp2-01-rms-20230824.pdf"),
-    ("EDX-COMP-1CP2", "1CP2/02", None, "2023-Jun", "QP", f"{_PE}/GCSE/Computer-science/2020/Exam-materials/1cp2-02-que-20230526.pdf"),
-    ("EDX-COMP-1CP2", "1CP2/01", None, "2024-Jun", "QP", f"{_PE}/GCSE/Computer-Science/2020/Exam-materials/1cp2-01-que-20240702.pdf"),
-    ("EDX-COMP-1CP2", "1CP2/01", None, "2024-Jun", "MS", f"{_PE}/GCSE/Computer-science/2020/Exam-materials/1cp2-01-rms-20240822.pdf"),
-    ("EDX-GEOG-9GE0", "9GE0/01", None, "2023-Jun", "QP", f"{_PE}/A-Level/Geography/2016/Exam-materials/9ge0-01-que-20230518.pdf"),
-]
-
-OCR_SPECS = {
-    "OCR-BIOL-J247": ("BIOLOGY", "J247", "GCSE", "2016"),
-    "OCR-CHEM-J248": ("CHEMISTRY", "J248", "GCSE", "2016"),
-    "OCR-PHYS-J249": ("PHYSICS", "J249", "GCSE", "2016"),
-    "OCR-COMB-J250": ("COMBINED SCIENCE", "J250", "GCSE", "2016"),
-    "OCR-MATH-J560": ("MATHEMATICS", "J560", "GCSE", "2015"),
-    "OCR-ENGL-J351": ("ENGLISH LANGUAGE", "J351", "GCSE", "2015"),
-    "OCR-ENGLIT-J352": ("ENGLISH LITERATURE", "J352", "GCSE", "2015"),
-    "OCR-COMP-J277": ("COMPUTER SCIENCE", "J277", "GCSE", "2020"),
-    "OCR-GEOG-J383": ("GEOGRAPHY A", "J383", "GCSE", "2016"),
-    "OCR-BUS-J204": ("BUSINESS", "J204", "GCSE", "2017"),
-    "OCR-HIST-J411": ("HISTORY B (SHP)", "J411", "GCSE", "2016"),
-    "OCR-MATH-H240": ("MATHEMATICS A", "H240", "A-level", "2017"),
-    "OCR-ENGLIT-H472": ("ENGLISH LITERATURE", "H472", "A-level", "2015"),
-    "OCR-ENGL-H470": ("ENGLISH LANGUAGE", "H470", "A-level", "2015"),
-}
-OCR_PAPERS = [
-    # ── Sciences (round 1) ──
-    ("OCR-BIOL-J247", "J247/01", "F", "2024-Jun", "QP", f"{_OCR}/727713-question-paper-paper-1.pdf"),
-    ("OCR-BIOL-J247", "J247/01", "F", "2024-Jun", "MS", f"{_OCR}/727745-mark-scheme-paper-1.pdf"),
-    ("OCR-BIOL-J247", "J247/03", "H", "2024-Jun", "QP", f"{_OCR}/727715-question-paper-paper-3.pdf"),
-    ("OCR-BIOL-J247", "J247/03", "H", "2024-Jun", "MS", f"{_OCR}/727747-mark-scheme-paper-3.pdf"),
-    ("OCR-BIOL-J247", "J247/01", "F", "2023-Jun", "QP", f"{_OCR}/704945-question-paper-paper-1.pdf"),
-    ("OCR-BIOL-J247", "J247/03", "H", "2023-Jun", "MS", f"{_OCR}/704979-mark-scheme-paper-3.pdf"),
-    ("OCR-BIOL-J247", "J247/03", "H", "2022-Jun", "QP", f"{_OCR}/678031-question-paper-paper-3.pdf"),
-    ("OCR-BIOL-J247", "J247/01", "F", "2022-Jun", "MS", f"{_OCR}/678076-mark-scheme-paper-1.pdf"),
-    ("OCR-CHEM-J248", "J248/01", "F", "2024-Jun", "QP", f"{_OCR}/727718-question-paper-paper-1.pdf"),
-    ("OCR-CHEM-J248", "J248/03", "H", "2024-Jun", "MS", f"{_OCR}/727751-mark-scheme-paper-3.pdf"),
-    ("OCR-CHEM-J248", "J248/01", "F", "2023-Jun", "QP", f"{_OCR}/704950-question-paper-paper-1.pdf"),
-    ("OCR-CHEM-J248", "J248/03", "H", "2022-Jun", "QP", f"{_OCR}/678036-question-paper-paper-3.pdf"),
-    ("OCR-PHYS-J249", "J249/01", "F", "2024-Jun", "QP", f"{_OCR}/727724-question-paper-paper-1.pdf"),
-    ("OCR-PHYS-J249", "J249/03", "H", "2024-Jun", "MS", f"{_OCR}/727755-mark-scheme-paper-3.pdf"),
-    ("OCR-PHYS-J249", "J249/01", "F", "2023-Jun", "QP", f"{_OCR}/704956-question-paper-paper-1.pdf"),
-    ("OCR-PHYS-J249", "J249/03", "H", "2022-Jun", "MS", f"{_OCR}/678086-mark-scheme-paper-3.pdf"),
-    ("OCR-COMB-J250", "J250/01", "F", "2024-Jun", "QP", f"{_OCR}/727730-question-paper-paper-1.pdf"),
-    ("OCR-COMB-J250", "J250/07", "H", "2024-Jun", "MS", f"{_OCR}/727763-mark-scheme-paper-7.pdf"),
-    # ── Maths J560 (round 2) ──
-    ("OCR-MATH-J560", "J560/01", "F", "2024-Jun", "QP", f"{_OCR}/727817-question-paper-paper-1.pdf"),
-    ("OCR-MATH-J560", "J560/01", "F", "2024-Jun", "MS", f"{_OCR}/727824-mark-scheme-paper-1.pdf"),
-    ("OCR-MATH-J560", "J560/04", "H", "2024-Jun", "QP", f"{_OCR}/727820-question-paper-paper-4.pdf"),
-    ("OCR-MATH-J560", "J560/04", "H", "2024-Jun", "MS", f"{_OCR}/727827-mark-scheme-paper-4.pdf"),
-    ("OCR-MATH-J560", "J560/01", "F", "2023-Jun", "QP", f"{_OCR}/705050-question-paper-paper-1.pdf"),
-    ("OCR-MATH-J560", "J560/01", "F", "2023-Jun", "MS", f"{_OCR}/705057-mark-scheme-paper-1.pdf"),
-    ("OCR-MATH-J560", "J560/04", "H", "2023-Jun", "QP", f"{_OCR}/705053-question-paper-paper-4.pdf"),
-    ("OCR-MATH-J560", "J560/04", "H", "2023-Jun", "MS", f"{_OCR}/705060-mark-scheme-paper-4.pdf"),
-    ("OCR-MATH-J560", "J560/01", "F", "2022-Jun", "QP", f"{_OCR}/678149-question-paper-paper-1.pdf"),
-    ("OCR-MATH-J560", "J560/01", "F", "2022-Jun", "MS", f"{_OCR}/678156-mark-scheme-paper-1.pdf"),
-    ("OCR-MATH-J560", "J560/04", "H", "2022-Jun", "QP", f"{_OCR}/678152-question-paper-paper-4.pdf"),
-    ("OCR-MATH-J560", "J560/04", "H", "2022-Jun", "MS", f"{_OCR}/678159-mark-scheme-paper-4.pdf"),
-    # ── English Language J351 / Literature J352 (round 2) ──
-    ("OCR-ENGL-J351", "J351/01", None, "2024-Jun", "QP", f"{_OCR}/727556-question-paper-communicating-information-and-ideas.pdf"),
-    ("OCR-ENGL-J351", "J351/01", None, "2024-Jun", "MS", f"{_OCR}/727658-mark-scheme-communication-information-and-ideas.pdf"),
-    ("OCR-ENGL-J351", "J351/02", None, "2024-Jun", "QP", f"{_OCR}/727558-question-paper-exploring-effects-and-impact.pdf"),
-    ("OCR-ENGL-J351", "J351/02", None, "2024-Jun", "MS", f"{_OCR}/727659-mark-scheme-exploring-effects-and-impact.pdf"),
-    ("OCR-ENGL-J351", "J351/01", None, "2023-Jun", "QP", f"{_OCR}/704782-question-paper-communicating-information-and-ideas.pdf"),
-    ("OCR-ENGL-J351", "J351/01", None, "2023-Jun", "MS", f"{_OCR}/704888-mark-scheme-communication-information-and-ideas.pdf"),
-    ("OCR-ENGL-J351", "J351/01", None, "2022-Jun", "QP", f"{_OCR}/677852-question-paper-communicating-information-and-ideas.pdf"),
-    ("OCR-ENGL-J351", "J351/01", None, "2022-Jun", "MS", f"{_OCR}/677967-mark-scheme-communication-information-and-ideas.pdf"),
-    ("OCR-ENGLIT-J352", "J352/01", None, "2024-Jun", "QP", f"{_OCR}/727830-question-paper-exploring-modern-and-literary-heritage-texts.pdf"),
-    ("OCR-ENGLIT-J352", "J352/01", None, "2024-Jun", "MS", f"{_OCR}/727832-mark-scheme-exploring-modern-and-literary-heritage-texts.pdf"),
-    ("OCR-ENGLIT-J352", "J352/02", None, "2024-Jun", "QP", f"{_OCR}/727831-question-paper-exploring-poetry-and-shakespeare.pdf"),
-    ("OCR-ENGLIT-J352", "J352/02", None, "2024-Jun", "MS", f"{_OCR}/727833-mark-scheme-exploring-poetry-and-shakespeare.pdf"),
-    ("OCR-ENGLIT-J352", "J352/01", None, "2023-Jun", "QP", f"{_OCR}/705069-question-paper-exploring-modern-and-literary-heritage-texts.pdf"),
-    ("OCR-ENGLIT-J352", "J352/01", None, "2023-Jun", "MS", f"{_OCR}/705075-mark-scheme-exploring-modern-and-literary-heritage-texts.pdf"),
-    # ── A-level Maths H240 / English Lit H472 / Lang H470 (round 2) ──
-    ("OCR-MATH-H240", "H240/01", None, "2024-Jun", "QP", f"{_OCR}/726654-question-paper-pure-mathematics.pdf"),
-    ("OCR-MATH-H240", "H240/01", None, "2024-Jun", "MS", f"{_OCR}/726795-mark-scheme-pure-mathematics.pdf"),
-    ("OCR-MATH-H240", "H240/02", None, "2024-Jun", "QP", f"{_OCR}/726656-question-paper-pure-mathematics-and-statistics.pdf"),
-    ("OCR-MATH-H240", "H240/02", None, "2024-Jun", "MS", f"{_OCR}/726796-mark-scheme-pure-mathematics-and-statistics.pdf"),
-    ("OCR-MATH-H240", "H240/01", None, "2023-Jun", "QP", f"{_OCR}/703866-question-paper-pure-mathematics.pdf"),
-    ("OCR-MATH-H240", "H240/01", None, "2023-Jun", "MS", f"{_OCR}/704008-mark-scheme-pure-mathematics.pdf"),
-    ("OCR-MATH-H240", "H240/01", None, "2022-Jun", "QP", f"{_OCR}/676845-question-paper-pure-mathematics.pdf"),
-    ("OCR-MATH-H240", "H240/01", None, "2022-Jun", "MS", f"{_OCR}/677005-mark-scheme-pure-mathematics.pdf"),
-    ("OCR-ENGLIT-H472", "H472/01", None, "2024-Jun", "QP", f"{_OCR}/726602-question-paper-drama-and-poetry-pre-1900.pdf"),
-    ("OCR-ENGLIT-H472", "H472/01", None, "2024-Jun", "MS", f"{_OCR}/726762-mark-scheme-drama-and-poetry-pre-1900.pdf"),
-    ("OCR-ENGLIT-H472", "H472/01", None, "2023-Jun", "QP", f"{_OCR}/703813-question-paper-drama-and-poetry-pre-1900.pdf"),
-    ("OCR-ENGLIT-H472", "H472/01", None, "2023-Jun", "MS", f"{_OCR}/703974-mark-scheme-drama-and-poetry-pre-1900.pdf"),
-    ("OCR-ENGLIT-H472", "H472/01", None, "2022-Jun", "QP", f"{_OCR}/676783-question-paper-drama-and-poetry-pre-1900.pdf"),
-    ("OCR-ENGLIT-H472", "H472/01", None, "2022-Jun", "MS", f"{_OCR}/676965-mark-scheme-drama-and-poetry-pre-1900.pdf"),
-    ("OCR-ENGL-H470", "H470/01", None, "2024-Jun", "QP", f"{_OCR}/726595-question-paper-exploring-language.pdf"),
-    ("OCR-ENGL-H470", "H470/01", None, "2024-Jun", "MS", f"{_OCR}/726764-mark-scheme-exploring-language.pdf"),
-    ("OCR-ENGL-H470", "H470/01", None, "2023-Jun", "QP", f"{_OCR}/703806-question-paper-exploring-language.pdf"),
-    ("OCR-ENGL-H470", "H470/01", None, "2023-Jun", "MS", f"{_OCR}/703976-mark-scheme-exploring-language.pdf"),
-    ("OCR-ENGL-H470", "H470/01", None, "2022-Jun", "QP", f"{_OCR}/676772-question-paper-exploring-language.pdf"),
-    ("OCR-ENGL-H470", "H470/01", None, "2022-Jun", "MS", f"{_OCR}/676967-mark-scheme-exploring-language.pdf"),
-    # ── Humanities (round 2) ──
-    ("OCR-COMP-J277", "J277/01", None, "2024-Jun", "QP", f"{_OCR}/727534-question-paper-computer-systems.pdf"),
-    ("OCR-COMP-J277", "J277/01", None, "2024-Jun", "MS", f"{_OCR}/727652-mark-scheme-computer-systems.pdf"),
-    ("OCR-COMP-J277", "J277/02", None, "2024-Jun", "QP", f"{_OCR}/727535-question-paper-computational-thinking-algorithms-and-programming.pdf"),
-    ("OCR-COMP-J277", "J277/02", None, "2024-Jun", "MS", f"{_OCR}/727653-mark-scheme-computational-thinking-algorithms-and-programming.pdf"),
-    ("OCR-GEOG-J383", "J383/01", None, "2024-Jun", "QP", f"{_OCR}/727564-question-paper-living-in-the-uk-today.pdf"),
-    ("OCR-GEOG-J383", "J383/01", None, "2024-Jun", "MS", f"{_OCR}/727661-mark-scheme-living-in-the-uk-today.pdf"),
-    ("OCR-GEOG-J383", "J383/02", None, "2024-Jun", "QP", f"{_OCR}/727566-question-paper-the-world-around-us.pdf"),
-    ("OCR-GEOG-J383", "J383/02", None, "2024-Jun", "MS", f"{_OCR}/727662-mark-scheme-the-world-around-us.pdf"),
-    ("OCR-BUS-J204", "J204/01", None, "2024-Jun", "QP", f"{_OCR}/727519-question-paper-business-1-business-activity-marketing-and-people.pdf"),
-    ("OCR-BUS-J204", "J204/01", None, "2024-Jun", "MS", f"{_OCR}/727634-mark-scheme-business-1-business-activity-marketing-and-people.pdf"),
-    ("OCR-BUS-J204", "J204/02", None, "2024-Jun", "QP", f"{_OCR}/727520-question-paper-business-2-operations-finance-and-influences-on-business.pdf"),
-    ("OCR-BUS-J204", "J204/02", None, "2024-Jun", "MS", f"{_OCR}/727635-mark-scheme-business-2-operations-finance-and-influences-on-business.pdf"),
-    ("OCR-BUS-J204", "J204/01", None, "2023-Jun", "QP", f"{_OCR}/704745-question-paper-business-1-business-activity-marketing-and-people.pdf"),
-    ("OCR-BUS-J204", "J204/01", None, "2023-Jun", "MS", f"{_OCR}/704864-mark-scheme-business-1-business-activity-marketing-and-people.pdf"),
-    ("OCR-HIST-J411", "J411/11", None, "2024-Jun", "QP", f"{_OCR}/727590-question-paper-the-people-s-health-c.1250-to-present-with-the-norman-conquest-1065-1087.pdf"),
-    ("OCR-HIST-J411", "J411/11", None, "2024-Jun", "MS", f"{_OCR}/727678-mark-scheme-the-people-s-health-c.1250-to-present-with-the-norman-conquest-1065-1087.pdf"),
-]
-
-
-def build_board(board_code: str, specs_meta: Dict, papers: List) -> Dict[str, Any]:
-    prefix = EXAM_CODE_PREFIX[board_code]
-    print(f"[{board_code}] re-verifying {len(papers)} confirmed URLs...", file=sys.stderr)
-    live: Dict[int, bool] = {}
-    with cf.ThreadPoolExecutor(max_workers=24) as ex:
-        futs = {ex.submit(head_ok, p[5]): i for i, p in enumerate(papers)}
-        for fut in cf.as_completed(futs):
-            live[futs[fut]] = fut.result()
-    by_spec: Dict[str, List[Dict[str, Any]]] = {}
-    for i, (spec_code, paper_code, tier, session, role, url) in enumerate(papers):
-        if not live.get(i):
-            print(f"  DROP (not live): {url}", file=sys.stderr)
-            continue
-        award = specs_meta[spec_code][1]
-        by_spec.setdefault(spec_code, []).append({
-            "exam_code": _mk_exam_code(prefix, award, paper_code, session, role),
-            "paper_code": paper_code, "tier": tier,
-            "session": session, "doc_type": role,
-            "file": {"source": f"url:{url}", "original_name": os.path.basename(url),
-                     "provenance": {"source_url": url, "fetched": FETCHED,
-                                    "license": f"{board_code} public past paper"}},
-        })
-    spec_list = []
-    for spec_code, (subject, award, level, first_teach) in specs_meta.items():
-        if spec_code not in by_spec:
-            continue
-        spec_list.append({
-            "spec_code": spec_code, "exam_board_code": board_code, "subject_code": subject,
-            "award_code": award, "award_level": level, "first_teach": first_teach,
-            "papers": sorted(by_spec[spec_code], key=lambda p: p["exam_code"]),
-        })
-        print(f"[{board_code}] {spec_code}: {len(by_spec[spec_code])} live papers", file=sys.stderr)
-    return {"exam_board_code": board_code, "specifications": spec_list}
-
-
-def main() -> None:
-    out_path = os.path.join(os.path.dirname(__file__), "exam-corpus.yaml")
-    boards = [
-        build_aqa(),
-        build_board("EDEXCEL", EDEXCEL_SPECS, EDEXCEL_PAPERS),
-        build_board("OCR", OCR_SPECS, OCR_PAPERS),
-    ]
-    n_specs = sum(len(b["specifications"]) for b in boards)
-    n_papers = sum(len(s["papers"]) for b in boards for s in b["specifications"])
-    manifest = {
-        "version": 1,
-        "defaults": {"bucket": "cc.examboards"},
-        "provenance": {
-            "collected_by": "kcar",
-            "collected_at": FETCHED,
-            "license_posture": ("Public exam-board past papers downloaded from each board's own "
-                                "official site (AQA filestore, Pearson DAM, OCR Images). Stored in "
-                                "the private dev cc.examboards bucket for internal exam-marker dev/test. "
-                                "Each item records its source_url. Review redistribution rights before "
-                                "any public exposure."),
-            "sources": {
-                "AQA": "https://filestore.aqa.org.uk/sample-papers-and-mark-schemes/",
-                "EDEXCEL": "https://qualifications.pearson.com/en/support/support-topics/exams/past-papers.html",
-                "OCR": "https://www.ocr.org.uk/qualifications/past-paper-finder/",
-            },
-        },
-        # Optional: uncomment + set on dev .94 to exercise user-side flows / first-sweep.
-        # "test_subset": {"user_email": "[email protected]", "papers": 2},
-        # "system_identity": {"user_email": "[email protected]"},
-        "boards": boards,
-    }
-    with open(out_path, "w") as fh:
-        yaml.safe_dump(manifest, fh, sort_keys=False, default_flow_style=False, width=120)
-    print(f"\nWROTE {out_path}: {n_specs} specs, {n_papers} papers across {len(boards)} boards",
-          file=sys.stderr)
-
-
-if __name__ == "__main__":
-    main()

run/initialization/reset_environment.py

@@ -1,429 +0,0 @@
-"""
-reset_environment.py — DESTRUCTIVE wipe of all non-permanent data.
-
-Clears:
-  - Neo4j: drops ALL databases except system, neo4j (including gaisdata, cc.users.*, cc.institutes.*)
-  - Supabase: deletes ALL data tables except gais_local_authorities and gais_schools
-  - Supabase: deletes all auth users except kcar, then re-seeds kcar profile state
-  - Granular scopes can clear exam corpus, timetable data, or --user-subset seed copies
-
-Safe invariants (never touched):
-  - kcar auth account
-  - gais_local_authorities and gais_schools Supabase tables
-  - system / neo4j Neo4j system databases
-
-Run from inside the ccapi container:
-    python3 -c "from run.initialization.reset_environment import reset; reset()"
-"""
-import os
-import time
-import requests
-from typing import List, Dict, Any
-
-from modules.logger_tool import initialise_logger
-import modules.database.tools.neo4j_driver_tools as dt
-
-logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), "default", True)
-
-KCAR_ID    = "d9e1d1a9-04c4-4611-bb05-57babf4a9a28"
-KCAR_EMAIL = "[email protected]"
-
-# Neo4j system databases — never drop these
-NEO4J_SYSTEM_DBS = {"system", "neo4j"}
-
-# Supabase tables to clear, in FK child-first order.
-# gais_local_authorities and gais_schools are intentionally absent.
-SUPABASE_TABLES_TO_CLEAR = [
-    # ── Transcription (deepest children first) ───────────────────────────────
-    "canvas_events",
-    "keyword_events",
-    "transcription_summaries",
-    "transcription_segments",
-    "keyword_watches",
-    "transcription_sessions",
-    # ── Lesson delivery chain ────────────────────────────────────────────────
-    "lesson_deliveries",
-    "lesson_collaborators",
-    # ── Timetable materialization ────────────────────────────────────────────
-    "taught_lessons",
-    # ── Academic calendar (children → parents) ───────────────────────────────
-    "academic_periods",
-    "academic_days",
-    "academic_weeks",
-    "academic_term_breaks",
-    "academic_terms",
-    "academic_years",
-    # ── Teacher timetables ───────────────────────────────────────────────────
-    "teacher_timetable_slots",
-    "teacher_timetables",
-    "school_timetables",
-    # ── Lesson plans ─────────────────────────────────────────────────────────
-    "planned_lessons",
-    # ── Whiteboard rooms ─────────────────────────────────────────────────────
-    "whiteboard_rooms",
-    # ── Classes & enrollment ─────────────────────────────────────────────────
-    "enrollment_requests",
-    "class_students",
-    "class_teachers",
-    "classes",
-    # ── Files & brains ───────────────────────────────────────────────────────
-    "document_artefacts",
-    "brain_files",
-    "cabinet_memberships",
-    "files",
-    "file_cabinets",
-    "brains",
-    # ── Invitations & memberships ────────────────────────────────────────────
-    "invitations",
-    "institute_memberships",
-    "institute_membership_requests",
-    # ── Institutes ───────────────────────────────────────────────────────────
-    "institutes",
-    # ── Profiles (non-kcar cleared separately via auth deletion cascade) ─────
-    "admin_profiles",
-]
-
-# Exam-marker subsystem tables, FK child-first. scope="exam-corpus" is deliberately
-# broader than "public papers": it wipes public corpus eb_* rows, templates, layouts,
-# questions, boundaries, response areas, marking batches, student submissions, and mark
-# entries. NOT in the list above — the previous full reset() never cleared exam data
-# or storage at all; the granular scopes below fold it in.
-EXAM_CORPUS_TABLES = [
-    "mark_entries",
-    "student_submissions",
-    "marking_batches",
-    "exam_response_areas",
-    "exam_boundaries",
-    "exam_template_layout",
-    "exam_questions",
-    "exam_templates",
-    "eb_exams",
-    "eb_specifications",
-]
-
-# Timetable / calendar materialization subset (for scope='timetable').
-TIMETABLE_TABLES = [
-    "lesson_deliveries",
-    "lesson_collaborators",
-    "taught_lessons",
-    "academic_periods",
-    "academic_days",
-    "academic_weeks",
-    "academic_term_breaks",
-    "academic_terms",
-    "academic_years",
-    "teacher_timetable_slots",
-    "teacher_timetables",
-    "school_timetables",
-    "planned_lessons",
-]
-
-# Bucket whose objects scope="exam-corpus" clears for the whole exam-marker subsystem
-# (Storage API — protect_delete blocks raw SQL).
-EXAM_STORAGE_BUCKET = "cc.examboards"
-
-
-def _sb_headers():
-    url = os.environ["SUPABASE_URL"]
-    key = os.environ["SERVICE_ROLE_KEY"]
-    return url, {
-        "apikey": key,
-        "Authorization": f"Bearer {key}",
-        "Content-Type": "application/json",
-        "Prefer": "return=minimal",
-    }
-
-
-# Markers that identify a production Supabase target. Destructive reset against any of these is
-# refused by default (project rule: ".94 only; .156 human-gated") — set RESET_ALLOW_PROD=1 to override.
-PROD_TARGET_MARKERS = ("192.168.0.156", "supabase.classroomcopilot")
-
-
-def _assert_reset_allowed(url: str, scope: str) -> None:
-    """Default-deny destructive reset against a production-looking Supabase target.
-
-    The /admin/reset route and this module both act on os.environ['SUPABASE_URL']; without this guard
-    a platform-admin call on a prod-deployed API would wipe prod data + exam corpus + storage. We refuse
-    when the target matches a known prod marker unless an explicit RESET_ALLOW_PROD opt-in is set.
-    """
-    target = (url or "").lower()
-    looks_prod = any(m in target for m in PROD_TARGET_MARKERS)
-    override = os.environ.get("RESET_ALLOW_PROD", "").strip().lower() in ("1", "true", "yes")
-    if looks_prod and not override:
-        raise RuntimeError(
-            f"refusing destructive reset (scope={scope}) against production-looking target {target!r}; "
-            f"this is human-gated — set RESET_ALLOW_PROD=1 to override."
-        )
-
-
-# ─── Neo4j helpers ────────────────────────────────────────────────────────────
-
-def _neo4j_drop_all_non_system() -> Dict[str, List[str]]:
-    """Drop every Neo4j DB except the system-reserved ones."""
-    with dt.get_session(database="system") as s:
-        all_dbs = [r["name"] for r in s.run("SHOW DATABASES YIELD name RETURN name")]
-
-    to_drop = [db for db in all_dbs if db not in NEO4J_SYSTEM_DBS]
-    dropped = []
-    for db in to_drop:
-        logger.info(f"  DROP DATABASE `{db}`")
-        try:
-            with dt.get_session(database="system") as s:
-                s.run(f"DROP DATABASE `{db}` IF EXISTS")
-            dropped.append(db)
-        except Exception as e:
-            logger.warning(f"  Could not drop `{db}`: {e}")
-    return dropped
-
-
-# ─── Supabase helpers ─────────────────────────────────────────────────────────
-
-# Tables without an uid=1000(kcar) gid=1000(kcar) groups=1000(kcar),27(sudo),119(docker) column — map to the column to use as the delete filter.
-TABLE_FILTER_COLUMN = {
-    "brain_files": "brain_id",
-}
-
-def _sb_clear_table(url: str, headers: dict, table: str) -> int:
-    """Delete all rows from a Supabase table. Returns HTTP status."""
-    col = TABLE_FILTER_COLUMN.get(table, "id")
-    r = requests.delete(
-        f"{url}/rest/v1/{table}",
-        headers=headers,
-        params={col: "not.is.null"},
-    )
-    if r.status_code not in (200, 204):
-        logger.warning(f"  Clear {table}: {r.status_code} {r.text[:120]}")
-    return r.status_code
-
-
-def _supabase_list_auth_users(url: str, headers: dict) -> List[Dict]:
-    r = requests.get(f"{url}/auth/v1/admin/users", headers=headers, params={"per_page": 200})
-    r.raise_for_status()
-    return r.json().get("users", [])
-
-
-def _supabase_delete_auth_user(url: str, headers: dict, uid: str):
-    r = requests.delete(f"{url}/auth/v1/admin/users/{uid}", headers=headers)
-    if r.status_code not in (200, 204):
-        logger.warning(f"  Delete auth user {uid}: {r.status_code} {r.text[:80]}")
-
-
-# ─── Granular helpers ───────────────────────────────────────────────────────────
-
-def _clear_tables(url: str, headers: dict, tables: List[str]) -> "tuple[List[str], List[str]]":
-    cleared, failed = [], []
-    for table in tables:
-        if _sb_clear_table(url, headers, table) in (200, 204):
-            cleared.append(table)
-            logger.info(f"  ✓ {table}")
-        else:
-            failed.append(table)
-    return cleared, failed
-
-
-def _clear_exam_storage() -> Dict[str, Any]:
-    """Remove cc.examboards objects for the exam-marker subsystem.
-
-    scope="exam-corpus" is not limited to public-paper metadata: it also removes the
-    storage objects that back exam board corpus files and any downstream exam-marker
-    artifacts referenced from eb_exams/eb_specifications. Gathers storage_loc from
-    eb_exams/eb_specifications BEFORE the rows are cleared.
-    """
-    try:
-        from modules.database.supabase.utils.client import SupabaseServiceRoleClient
-        from modules.database.supabase.utils.storage import StorageAdmin
-    except Exception as exc:
-        logger.warning(f"  exam storage clear skipped (import): {exc}")
-        return {"removed": 0, "error": str(exc)}
-    sb = SupabaseServiceRoleClient().supabase
-    storage = StorageAdmin()
-    locs: List[str] = []
-    for table in ("eb_exams", "eb_specifications"):
-        try:
-            rows = sb.table(table).select("storage_loc").execute().data or []
-            locs += [r["storage_loc"] for r in rows if r.get("storage_loc")]
-        except Exception as exc:
-            logger.warning(f"  storage_loc gather {table}: {exc}")
-    by_bucket: Dict[str, List[str]] = {}
-    for loc in locs:
-        if "/" in loc:
-            b, _, p = loc.partition("/")
-            by_bucket.setdefault(b, []).append(p)
-    removed = 0
-    for b, paths in by_bucket.items():
-        for i in range(0, len(paths), 100):
-            chunk = paths[i:i + 100]
-            try:
-                storage.client.supabase.storage.from_(b).remove(chunk)
-                removed += len(chunk)
-            except Exception as exc:
-                logger.warning(f"  storage remove {b}: {exc}")
-    logger.info(f"  exam storage removed {removed} objects from {list(by_bucket)}")
-    return {"removed": removed, "buckets": list(by_bucket)}
-
-
-def _clear_user_subset_files() -> Dict[str, Any]:
-    """Remove files rows and cc.users storage objects created by --user-subset seeding.
-
-    Reuses the seed/unseed implementation so reset(scope="user-subset") has the
-    same storage-before-row deletion order and idempotency guarantees as
-    seed_exam_corpus.py --unseed. The helper only targets rows marked by the seeder:
-    bucket='cc.users', source='exam-corpus-seed', path LIKE 'exam-marker/%'.
-    """
-    try:
-        from modules.database.supabase.utils.client import SupabaseServiceRoleClient
-        from modules.database.supabase.utils.storage import StorageAdmin
-        from run.initialization.seed_exam_corpus import LoadReport, _delete_user_subset_files
-    except Exception as exc:
-        logger.warning(f"  user-subset clear skipped (import): {exc}")
-        return {"files_rows_deleted": 0, "storage_objects_removed": 0, "errors": [str(exc)]}
-
-    rep = LoadReport()
-    _delete_user_subset_files(
-        SupabaseServiceRoleClient(),
-        StorageAdmin(),
-        exam_codes=None,
-        rep=rep,
-    )
-    return {
-        "files_rows_deleted": rep.unseed_user_files,
-        "storage_objects_removed": rep.unseed_objects,
-        "errors": rep.errors,
-    }
-
-
-# ─── Main reset ───────────────────────────────────────────────────────────────
-
-def reset(scope: str = "all") -> Dict[str, Any]:
-    """Destructive reset. scope ∈ {all, exam-corpus, timetable, user-subset}.
-
-    - all          : full wipe (Neo4j + Supabase data + auth users) AND the entire
-                     exam-marker subsystem listed below, including --user-subset copies.
-    - exam-corpus  : ONLY the entire exam-marker subsystem, not just public papers:
-                     public corpus/eb_* data, cc.examboards storage objects, exam
-                     templates, template layouts, questions, boundaries, response
-                     areas, marking batches, student submissions, mark entries, and
-                     --user-subset cc.users copies.
-    - timetable    : ONLY timetable/calendar materialization tables.
-    - user-subset  : ONLY files rows and cc.users storage objects created by
-                     seed_exam_corpus.py --user-subset.
-    """
-    scope = (scope or "all").lower()
-    if scope not in ("all", "exam-corpus", "timetable", "user-subset"):
-        raise ValueError(f"invalid scope {scope!r} (want all|exam-corpus|timetable|user-subset)")
-    url, headers = _sb_headers()
-    _assert_reset_allowed(url, scope)
-
-    if scope == "exam-corpus":
-        logger.info("RESET (scope=exam-corpus) — entire exam-marker subsystem: public corpus/eb_* data, cc.examboards storage, templates/layout/questions/boundaries/response areas, marking batches, submissions, mark entries, and --user-subset copies")
-        user_subset = _clear_user_subset_files()
-        storage = _clear_exam_storage()
-        cleared, failed = _clear_tables(url, headers, EXAM_CORPUS_TABLES)
-        return {"scope": scope, "user_subset": user_subset, "exam_storage": storage, "tables_cleared": cleared, "tables_failed": failed}
-
-    if scope == "timetable":
-        logger.info("RESET (scope=timetable) — timetable/calendar tables")
-        cleared, failed = _clear_tables(url, headers, TIMETABLE_TABLES)
-        return {"scope": scope, "tables_cleared": cleared, "tables_failed": failed}
-
-    if scope == "user-subset":
-        logger.info("RESET (scope=user-subset) — --user-subset cc.users storage objects and files rows")
-        user_subset = _clear_user_subset_files()
-        return {"scope": scope, "user_subset": user_subset}
-
-    logger.info("=" * 60)
-    logger.info("RESET ENVIRONMENT — full destructive wipe starting")
-    logger.info("=" * 60)
-    results: Dict[str, Any] = {"scope": scope}
-
-    # ── 1. Neo4j: drop everything except system + neo4j ──────────────────────
-    logger.info("\n[Neo4j] Dropping all non-system databases...")
-    dropped = _neo4j_drop_all_non_system()
-    logger.info(f"  Dropped {len(dropped)}: {dropped}")
-    results["neo4j"] = {"dropped": dropped}
-
-    # ── 2. Supabase: clear all data tables (GAIS preserved) ──────────────────
-    # First remove --user-subset cc.users storage objects (+ their files rows) via the
-    # Storage API, so the generic files-table clear below doesn't strand orphaned objects.
-    results["user_subset"] = _clear_user_subset_files()
-    logger.info("\n[Supabase] Clearing data tables (preserving gais_*)...")
-    url, headers = _sb_headers()
-    cleared, failed = [], []
-    for table in SUPABASE_TABLES_TO_CLEAR:
-        status = _sb_clear_table(url, headers, table)
-        if status in (200, 204):
-            cleared.append(table)
-            logger.info(f"  ✓ {table}")
-        else:
-            failed.append(table)
-    logger.info(f"  Cleared {len(cleared)} tables, {len(failed)} failed")
-
-    # ── 3. Supabase: delete all auth users except kcar ────────────────────────
-    logger.info("\n[Supabase] Deleting test auth users...")
-    all_users = _supabase_list_auth_users(url, headers)
-    deleted_emails = []
-    for u in all_users:
-        if u["email"] == KCAR_EMAIL:
-            continue
-        _supabase_delete_auth_user(url, headers, u["id"])
-        deleted_emails.append(u["email"])
-        time.sleep(0.05)
-    logger.info(f"  Deleted {len(deleted_emails)} auth users")
-
-    # Explicit cleanup in case cascade didn't fire
-    requests.delete(f"{url}/rest/v1/profiles", headers=headers,
-                    params={"id": f"neq.{KCAR_ID}"})
-
-    # ── 4. Reset kcar profile to known-good platform_admin state ──────────────
-    logger.info("\n[Supabase] Resetting kcar profile...")
-    requests.patch(
-        f"{url}/rest/v1/profiles",
-        headers=headers,
-        params={"id": f"eq.{KCAR_ID}"},
-        json={"school_id": None},
-    )
-    logger.info("  kcar → school_id: null ✓")
-
-    # Restore admin_profiles row (wiped with other tables above)
-    requests.post(
-        f"{url}/rest/v1/admin_profiles",
-        headers={**headers, "Prefer": "resolution=merge-duplicates"},
-        json={
-            "id":             KCAR_ID,
-            "email":          KCAR_EMAIL,
-            "display_name":   "Kevin Carroll",
-            "admin_role":     "super_admin",
-            "is_super_admin": True,
-        },
-    )
-    logger.info("  kcar → admin_profiles restored ✓")
-
-    # ── 5. Exam-marker subsystem: storage objects (Storage API) + all exam tables ──
-    # This is the same destructive surface as scope="exam-corpus": public corpus/eb_*
-    # rows, cc.examboards storage, templates/layout/questions/boundaries/response
-    # areas, marking batches, submissions, and mark entries. (The legacy full reset
-    # cleared neither exam tables nor storage — folded in here.)
-    logger.info("\n[Supabase] Clearing entire exam-marker subsystem (public corpus, storage, templates/layout/questions/boundaries/response areas, marking batches, submissions, mark entries)...")
-    exam_storage = _clear_exam_storage()
-    exam_cleared, exam_failed = _clear_tables(url, headers, EXAM_CORPUS_TABLES)
-
-    results["supabase"] = {
-        "tables_cleared": cleared,
-        "tables_failed": failed,
-        "deleted_users": deleted_emails,
-    }
-    results["exam"] = {
-        "storage": exam_storage,
-        "tables_cleared": exam_cleared,
-        "tables_failed": exam_failed,
-    }
-
-    logger.info("\n" + "=" * 60)
-    logger.info("RESET COMPLETE")
-    logger.info("=" * 60)
-    return results
-
-
-if __name__ == "__main__":
-    import json
-    print(json.dumps(reset(), indent=2, default=str))

run/initialization/seed_cohort_9p_ph1.py

@@ -1,218 +0,0 @@
-"""
-seed_cohort_9p_ph1.py — Markable cohort for exam-marker testing.
-
-Creates N student accounts and enrols them ALL into a single class (default the
-Greenfield Year 9 Physics class `9P/Ph1`), so there is a real cohort to mark.
-
-Why: the canonical timetable seeds enrol "one student per year-group band"
-(seed_greenfield_timetable.py), so every class has <=1 student — too few for a
-results table / per-question stats. This seeder fills one class to a usable size.
-
-Mechanics (identical paths to the canonical seeds — nothing bespoke server-side):
-  - auth user:   POST {SUPABASE_URL}/auth/v1/admin/users
-  - profile:     upsert public.profiles  (school_id = institute)
-  - membership:  upsert public.institute_memberships  (role 'student')
-  - enrolment:   POST {API_BASE_URL}/database/timetable/classes/{class_id}/students
-                 (as a school_admin; class_students upsert → idempotent)
-
-Idempotent: re-running skips existing auth users and upserts everything else.
-
-Env required:  SUPABASE_URL, SERVICE_ROLE_KEY   (API_BASE_URL defaults to api-dev)
-Optional env:  COHORT_COUNT, COHORT_CLASS_CODE, COHORT_INSTITUTE_ID,
-               SEED_STUDENT_PASSWORD, SEED_SCHOOL_ADMIN_PASSWORD
-
-Run (dev):
-    SUPABASE_URL=... SERVICE_ROLE_KEY=... API_BASE_URL=http://192.168.0.64:18000 \
-    python3 -c "from run.initialization.seed_cohort_9p_ph1 import seed; seed()"
-"""
-import os
-import time
-import requests
-from typing import Dict, Any, List, Optional
-
-# Greenfield Academy (the school actually populated on dev .94)
-GREENFIELD_ID = os.getenv("COHORT_INSTITUTE_ID", "a1b2c3d4-e5f6-7890-abcd-ef1234567890")
-GREENFIELD_DOMAIN = "greenfieldacademy.test"
-GREENFIELD_ADMIN_EMAIL = f"admin@{GREENFIELD_DOMAIN}"
-
-CLASS_CODE = os.getenv("COHORT_CLASS_CODE", "9P/Ph1")
-COHORT_COUNT = int(os.getenv("COHORT_COUNT", "10"))
-
-# Realistic-ish names so the results table doesn't read "Pupil 01..10".
-COHORT_NAMES = [
-    ("Amelia", "Clarke"), ("Noah", "Bennett"), ("Olivia", "Foster"), ("Leo", "Hughes"),
-    ("Ava", "Patel"), ("Jacob", "Reid"), ("Mia", "Turner"), ("Harry", "Ellis"),
-    ("Isla", "Morgan"), ("Oscar", "Khan"), ("Freya", "Walsh"), ("Theo", "Ndlovu"),
-]
-
-DEFAULT_STUDENT_PASSWORD = "Student@Cc2025!"
-DEFAULT_SCHOOL_ADMIN_PASSWORD = "Admin@Cc2025!"
-
-
-def _ctx() -> Dict[str, str]:
-    return {
-        "supa_url": os.environ["SUPABASE_URL"].rstrip("/"),
-        "service_key": os.environ["SERVICE_ROLE_KEY"],
-        "api_base": os.environ.get("API_BASE_URL", "http://192.168.0.64:18000").rstrip("/"),
-        "student_pw": os.getenv("SEED_STUDENT_PASSWORD", DEFAULT_STUDENT_PASSWORD),
-        "admin_pw": os.getenv("SEED_SCHOOL_ADMIN_PASSWORD", DEFAULT_SCHOOL_ADMIN_PASSWORD),
-    }
-
-
-def _sb_headers(ctx: Dict[str, str]) -> Dict[str, str]:
-    return {
-        "apikey": ctx["service_key"],
-        "Authorization": f"Bearer {ctx['service_key']}",
-        "Content-Type": "application/json",
-    }
-
-
-def _sign_in(ctx: Dict[str, str], email: str, password: str) -> str:
-    r = requests.post(
-        f"{ctx['supa_url']}/auth/v1/token?grant_type=password",
-        headers={"apikey": ctx["service_key"], "Content-Type": "application/json"},
-        json={"email": email, "password": password},
-    )
-    r.raise_for_status()
-    return r.json()["access_token"]
-
-
-def _resolve_class_id(ctx: Dict[str, str]) -> Optional[str]:
-    r = requests.get(
-        f"{ctx['supa_url']}/rest/v1/classes",
-        headers=_sb_headers(ctx),
-        params={"class_code": f"eq.{CLASS_CODE}",
-                "institute_id": f"eq.{GREENFIELD_ID}",
-                "select": "id,name", "limit": "1"},
-    )
-    data = r.json() if r.ok else []
-    return data[0]["id"] if data else None
-
-
-def _existing_auth_users(ctx: Dict[str, str]) -> Dict[str, str]:
-    r = requests.get(
-        f"{ctx['supa_url']}/auth/v1/admin/users",
-        headers=_sb_headers(ctx), params={"per_page": 200},
-    )
-    r.raise_for_status()
-    return {u["email"]: u["id"] for u in r.json().get("users", [])}
-
-
-def _create_auth_user(ctx: Dict[str, str], spec: Dict) -> Optional[str]:
-    r = requests.post(
-        f"{ctx['supa_url']}/auth/v1/admin/users",
-        headers=_sb_headers(ctx),
-        json={
-            "email": spec["email"], "password": ctx["student_pw"], "email_confirm": True,
-            "user_metadata": {
-                "username": spec["username"], "full_name": spec["full_name"],
-                "display_name": spec["display_name"], "user_type": "student",
-            },
-        },
-    )
-    if r.status_code in (200, 201):
-        return r.json()["id"]
-    return None
-
-
-def _upsert(ctx: Dict[str, str], table: str, row: Dict, on_conflict: str) -> bool:
-    h = {**_sb_headers(ctx), "Prefer": "resolution=merge-duplicates,return=minimal"}
-    r = requests.post(f"{ctx['supa_url']}/rest/v1/{table}",
-                      headers=h, json=row, params={"on_conflict": on_conflict})
-    return r.ok
-
-
-def _cohort_specs() -> List[Dict]:
-    specs = []
-    for i in range(1, COHORT_COUNT + 1):
-        first, last = COHORT_NAMES[(i - 1) % len(COHORT_NAMES)]
-        prefix = f"cohort{i:02d}"
-        specs.append({
-            "email": f"{prefix}@{GREENFIELD_DOMAIN}",
-            "username": f"{prefix}.{GREENFIELD_DOMAIN.replace('.', '_')}",
-            "full_name": f"{first} {last}",
-            "display_name": first,
-        })
-    return specs
-
-
-def seed(count: Optional[int] = None) -> Dict[str, Any]:
-    global COHORT_COUNT
-    if count is not None:
-        COHORT_COUNT = count
-    ctx = _ctx()
-    results: Dict[str, Any] = {"class_code": CLASS_CODE, "requested": COHORT_COUNT,
-                              "created": 0, "reused": 0, "enrolled": 0, "errors": []}
-
-    print(f"COHORT SEED → {CLASS_CODE} @ {GREENFIELD_DOMAIN} (target {COHORT_COUNT} students)")
-
-    class_id = _resolve_class_id(ctx)
-    if not class_id:
-        results["errors"].append(f"class {CLASS_CODE} not found for institute {GREENFIELD_ID}")
-        print(f"  ✗ {results['errors'][-1]}")
-        return results
-    print(f"  class_id = {class_id}")
-
-    existing = _existing_auth_users(ctx)
-    specs = _cohort_specs()
-
-    # 1) accounts: auth user + profile + membership
-    uids: Dict[str, str] = {}
-    for spec in specs:
-        email = spec["email"]
-        uid = existing.get(email)
-        if uid:
-            results["reused"] += 1
-        else:
-            uid = _create_auth_user(ctx, spec)
-            if not uid:
-                results["errors"].append(f"create auth user {email}")
-                print(f"  ✗ create {email}")
-                continue
-            results["created"] += 1
-            time.sleep(0.15)
-        uids[email] = uid
-        ok_p = _upsert(ctx, "profiles", {
-            "id": uid, "email": email, "user_type": "student",
-            "username": spec["username"], "full_name": spec["full_name"],
-            "display_name": spec["display_name"], "school_id": GREENFIELD_ID,
-            "neo4j_sync_status": "pending",
-        }, on_conflict="id")
-        ok_m = _upsert(ctx, "institute_memberships", {
-            "profile_id": uid, "institute_id": GREENFIELD_ID, "role": "student", "metadata": {},
-        }, on_conflict="profile_id,institute_id")
-        if not (ok_p and ok_m):
-            results["errors"].append(f"profile/membership {email} (p={ok_p} m={ok_m})")
-
-    # 2) enrol all into the class (via API, as school admin)
-    admin_token = _sign_in(ctx, GREENFIELD_ADMIN_EMAIL, ctx["admin_pw"])
-    for spec in specs:
-        uid = uids.get(spec["email"])
-        if not uid:
-            continue
-        r = requests.post(
-            f"{ctx['api_base']}/database/timetable/classes/{class_id}/students",
-            headers={"Authorization": f"Bearer {admin_token}", "Content-Type": "application/json"},
-            json={"student_id": uid},
-        )
-        body = {}
-        try:
-            body = r.json()
-        except Exception:
-            pass
-        if r.ok and (body.get("status") == "ok" or body.get("row") or body.get("id")):
-            results["enrolled"] += 1
-            print(f"  ✓ {spec['email'].split('@')[0]} → {CLASS_CODE}")
-        else:
-            results["errors"].append(f"enrol {spec['email']}: {r.status_code} {str(body)[:120]}")
-            print(f"  ✗ enrol {spec['email']}: {r.status_code}")
-        time.sleep(0.1)
-
-    print(f"\nDONE: created {results['created']}, reused {results['reused']}, "
-          f"enrolled {results['enrolled']}/{COHORT_COUNT}, errors {len(results['errors'])}")
-    return results
-
-
-if __name__ == "__main__":
-    import json
-    print(json.dumps(seed(), indent=2, default=str))

run/initialization/seed_curriculum.py

@@ -1,389 +0,0 @@
-"""
-seed_curriculum.py — DEPRECATED hardcoded curriculum/exam seeder.
-
-⚠️ SUPERSEDED (2026-06-07) by the manifest-driven corpus loader:
-    run/initialization/seed_exam_corpus.py  (+ manifests/exam-corpus.yaml)
-
-The exam-board parts of this file (eb_specifications / eb_exams) are now seeded from a
-verified, provenance-bearing manifest with real uploaded PDFs — not the hardcoded rows
-below. This module also had a storage_loc inconsistency the overhaul standardises away:
-exam-board files belong in the `cc.examboards` bucket at the canonical path
-`cc.examboards/{board}/{subject}/{award}/{paper}/{session}/{role}.pdf`, NOT under
-`cc.public.snapshots/curriculum/...` (the placeholder rows below still show the old path).
-
-KEEP ONLY for the Neo4j `curriculum_topics` seed (step [3]) which has no replacement yet.
-Do NOT use the eb_specifications/eb_exams blocks for new work — use seed_exam_corpus.py.
-
-Run (Neo4j curriculum topics only is the supported remaining use):
-    python3 -c "from run.initialization.seed_curriculum import seed; seed()"
-"""
-import os
-import time
-import uuid
-import requests
-from typing import Dict, Any, List, Optional
-
-SUPA_URL   = os.environ["SUPABASE_URL"]
-SERVICE_KEY = os.environ["SERVICE_ROLE_KEY"]
-API_BASE   = os.environ.get("API_BASE_URL", "http://localhost:8000")
-
-# ─── School constants ────────────────────────────────────────────────────────
-
-KEVLARAI_ID   = "6585bf91-6ae8-4d72-ab54-cddf3ba4e648"
-GREENFIELD_ID = "a1b2c3d4-e5f6-7890-abcd-ef1234567890"
-
-# ─── Exam board specifications ───────────────────────────────────────────────
-# Realistic UK exam board data for the subjects we teach.
-
-SPECIFICATIONS = [
-    # AQA Physics
-    {
-        "spec_code": "AQA-PHYS-8201",
-        "exam_board_code": "AQA",
-        "award_code": "8201",
-        "subject_code": "PHYSICS",
-        "first_teach": "2016",
-        "spec_ver": "1.3",
-        "storage_loc": "cc.public.snapshots/curriculum/aqa/physics/8201_spec.pdf",
-        "doc_type": "pdf",
-    },
-    {
-        "spec_code": "AQA-PHYS-8203",
-        "exam_board_code": "AQA",
-        "award_code": "8203",
-        "subject_code": "PHYSICS",
-        "first_teach": "2016",
-        "spec_ver": "1.3",
-        "storage_loc": "cc.public.snapshots/curriculum/aqa/physics/8203_spec.pdf",
-        "doc_type": "pdf",
-    },
-    # AQA GCSE Physics 8463 (standalone) — the real spec for the exam-marker test paper
-    # (AQA Physics Paper 1H 2022). Spec graph: cc.public.exams Specification AQA-PHYS-8463.
-    {
-        "spec_code": "AQA-PHYS-8463",
-        "exam_board_code": "AQA",
-        "award_code": "8463",
-        "subject_code": "PHYSICS",
-        "first_teach": "2016",
-        "spec_ver": "1.0",
-        "storage_loc": "cc.examboards/aqa/physics/8463/8463_spec.pdf",  # placeholder (no file yet)
-        "doc_type": "pdf",
-    },
-    # Edexcel Maths
-    {
-        "spec_code": "EDX-MATH-1MA1",
-        "exam_board_code": "EDexcel",
-        "award_code": "1MA1",
-        "subject_code": "MATHEMATICS",
-        "first_teach": "2015",
-        "spec_ver": "2.0",
-        "storage_loc": "cc.public.snapshots/curriculum/edexcel/maths/1MA1_spec.pdf",
-        "doc_type": "pdf",
-    },
-    # OCR Maths
-    {
-        "spec_code": "OCR-MATH-FMH1",
-        "exam_board_code": "OCR",
-        "award_code": "FMH1",
-        "subject_code": "MATHEMATICS",
-        "first_teach": "2017",
-        "spec_ver": "1.1",
-        "storage_loc": "cc.public.snapshots/curriculum/ocr/maths/FMH1_spec.pdf",
-        "doc_type": "pdf",
-    },
-    # AQA Computer Science
-    {
-        "spec_code": "AQA-COMP-7516",
-        "exam_board_code": "AQA",
-        "award_code": "7516",
-        "subject_code": "COMPUTER SCIENCE",
-        "first_teach": "2016",
-        "spec_ver": "1.2",
-        "storage_loc": "cc.public.snapshots/curriculum/aqa/cs/7516_spec.pdf",
-        "doc_type": "pdf",
-    },
-    # Edexcel Computer Science
-    {
-        "spec_code": "EDX-COMP-X042",
-        "exam_board_code": "Edexcel",
-        "award_code": "X042",
-        "subject_code": "COMPUTER SCIENCE",
-        "first_teach": "2016",
-        "spec_ver": "1.0",
-        "storage_loc": "cc.public.snapshots/curriculum/edexcel/cs/X042_spec.pdf",
-        "doc_type": "pdf",
-    },
-]
-
-# ─── Exam papers ─────────────────────────────────────────────────────────────
-# Realistic exam paper references linked to specifications.
-
-EXAMS = [
-    # AQA GCSE Physics 8463/1 Higher — the exam-marker test paper (real PDF uploaded to
-    # cc.examboards). Join key for cc.public.exams ExamPaper.exam_code.
-    {"exam_code": "AQA-PHYS-8463-1H-22-JUN", "spec_code": "AQA-PHYS-8463", "paper_code": "8463/1",
-     "tier": "higher", "session": "June", "type_code": "QP",
-     "storage_loc": "cc.examboards/aqa/physics/8463/AQA-PHYS-8463-1H-22-JUN.pdf"},
-
-    # AQA Physics 8201/1 (Foundation)
-    {"exam_code": "AQA-PHYS-8201-1-23-JUN", "spec_code": "AQA-PHYS-8201", "paper_code": "8201/1",
-     "tier": "foundation", "session": "June", "type_code": "QP"},
-    {"exam_code": "AQA-PHYS-8201-MS-23-JUN", "spec_code": "AQA-PHYS-8201", "paper_code": "8201/1",
-     "tier": "foundation", "session": "June", "type_code": "MS"},
-    {"exam_code": "AQA-PHYS-8201-ER-23-JUN", "spec_code": "AQA-PHYS-8201", "paper_code": "8201/1",
-     "tier": "foundation", "session": "June", "type_code": "ER"},
-
-    # AQA Physics 8201/2 (Higher)
-    {"exam_code": "AQA-PHYS-8201-2-23-JUN", "spec_code": "AQA-PHYS-8201", "paper_code": "8201/2",
-     "tier": "higher", "session": "June", "type_code": "QP"},
-    {"exam_code": "AQA-PHYS-8201-MS-23-JUN-H", "spec_code": "AQA-PHYS-8201", "paper_code": "8201/2",
-     "tier": "higher", "session": "June", "type_code": "MS"},
-
-    # Edexcel Maths 1MA1/1 (Foundation)
-    {"exam_code": "EDX-MATH-1MA1-1-24-JUN", "spec_code": "EDX-MATH-1MA1", "paper_code": "1MA1/1F",
-     "tier": "foundation", "session": "June", "type_code": "QP"},
-    {"exam_code": "EDX-MATH-1MA1-MS-24-JUN", "spec_code": "EDX-MATH-1MA1", "paper_code": "1MA1/1F",
-     "tier": "foundation", "session": "June", "type_code": "MS"},
-
-    # Edexcel Maths 1MA1/2 (Higher)
-    {"exam_code": "EDX-MATH-1MA1-2-24-JUN", "spec_code": "EDX-MATH-1MA1", "paper_code": "1MA1/2H",
-     "tier": "higher", "session": "June", "type_code": "QP"},
-    {"exam_code": "EDX-MATH-1MA1-MS-24-JUN-H", "spec_code": "EDX-MATH-1MA1", "paper_code": "1MA1/2H",
-     "tier": "higher", "session": "June", "type_code": "MS"},
-
-    # OCR Maths FMH1/1
-    {"exam_code": "OCR-MATH-FMH1-1-24-JUN", "spec_code": "OCR-MATH-FMH1", "paper_code": "FMH1/1",
-     "tier": "higher", "session": "June", "type_code": "QP"},
-    {"exam_code": "OCR-MATH-FMH1-MS-24-JUN", "spec_code": "OCR-MATH-FMH1", "paper_code": "FMH1/1",
-     "tier": "higher", "session": "June", "type_code": "MS"},
-
-    # AQA CS 7516/1
-    {"exam_code": "AQA-COMP-7516-1-23-JUN", "spec_code": "AQA-COMP-7516", "paper_code": "7516/1",
-     "tier": None, "session": "June", "type_code": "QP"},
-    {"exam_code": "AQA-COMP-7516-MS-23-JUN", "spec_code": "AQA-COMP-7516", "paper_code": "7516/1",
-     "tier": None, "session": "June", "type_code": "MS"},
-
-    # AQA CS 7516/2
-    {"exam_code": "AQA-COMP-7516-2-23-JUN", "spec_code": "AQA-COMP-7516", "paper_code": "7516/2",
-     "tier": None, "session": "June", "type_code": "QP"},
-    {"exam_code": "AQA-COMP-7516-ER-23-JUN", "spec_code": "AQA-COMP-7516", "paper_code": "7516/2",
-     "tier": None, "session": "June", "type_code": "ER"},
-]
-
-
-# ─── Neo4j curriculum topics ─────────────────────────────────────────────────
-# Curriculum topics stored in Neo4j school databases (not Supabase).
-
-CURRICULUM_TOPICS = {
-    "Physics": [
-        {"topic_code": "PHYS-KS3-01", "title": "Forces", "year_group": "9", "key_stage": "3",
-         "description": "Contact and non-contact forces, resultant forces, moments"},
-        {"topic_code": "PHYS-KS3-02", "title": "Energy", "year_group": "9", "key_stage": "3",
-         "description": "Energy stores, transfers, conservation, dissipation"},
-        {"topic_code": "PHYS-KS3-03", "title": "Waves", "year_group": "9", "key_stage": "3",
-         "description": "Transverse and longitudinal waves, reflection, refraction, diffraction"},
-        {"topic_code": "PHYS-KS4-01", "title": "Electricity", "year_group": "10", "key_stage": "4",
-         "description": "Circuits, current, potential difference, resistance, power"},
-        {"topic_code": "PHYS-KS4-02", "title": "Magnetism and Electromagnetism", "year_group": "10", "key_stage": "4",
-         "description": "Magnetic fields, electromagnets, motors, generators"},
-        {"topic_code": "PHYS-KS4-03", "title": "Atomic Structure", "year_group": "10", "key_stage": "4",
-         "description": "Atoms, isotopes, radioactivity, half-life"},
-        {"topic_code": "PHYS-KS4-04", "title": "Particle Physics", "year_group": "11", "key_stage": "4",
-         "description": "Standard model, quarks, leptons, bosons"},
-        {"topic_code": "PHYS-KS4-05", "title": "Cosmology", "year_group": "11", "key_stage": "4",
-         "description": "Big Bang, stellar evolution, redshift"},
-    ],
-    "Mathematics": [
-        {"topic_code": "MATH-KS3-01", "title": "Number", "year_group": "9", "key_stage": "3",
-         "description": "Integers, fractions, decimals, percentages, ratio, proportion"},
-        {"topic_code": "MATH-KS3-02", "title": "Algebra", "year_group": "9", "key_stage": "3",
-         "description": "Expressions, equations, inequalities, sequences"},
-        {"topic_code": "MATH-KS3-03", "title": "Geometry", "year_group": "9", "key_stage": "3",
-         "description": "Angles, polygons, circles, transformations, constructions"},
-        {"topic_code": "MATH-KS4-01", "title": "Number and Algebra", "year_group": "10", "key_stage": "4",
-         "description": "Surds, indices, standard form, expanding brackets, factorising"},
-        {"topic_code": "MATH-KS4-02", "title": "Graphs and Functions", "year_group": "10", "key_stage": "4",
-         "description": "Linear, quadratic, cubic graphs, gradients, intercepts"},
-        {"topic_code": "MATH-KS4-03", "title": "Statistics and Probability", "year_group": "10", "key_stage": "4",
-         "description": "Data types, charts, expected frequency, tree diagrams, two-way tables"},
-        {"topic_code": "MATH-KS4-04", "title": "Geometry and Measures", "year_group": "10", "key_stage": "4",
-         "description": "Area, volume, surface area, Pythagoras, trigonometry, bearings"},
-        {"topic_code": "MATH-KS4-05", "title": "Simultaneous Equations and Quadratics", "year_group": "11", "key_stage": "4",
-         "description": "Solving simultaneous equations, completing the square, quadratic formula"},
-    ],
-    "Computer Science": [
-        {"topic_code": "CS-KS4-01", "title": "Data Representation", "year_group": "10", "key_stage": "4",
-         "description": "Binary, hexadecimal, bit operations, compression, encryption"},
-        {"topic_code": "CS-KS4-02", "title": "Computer Systems", "year_group": "10", "key_stage": "4",
-         "description": "CPU architecture, memory, storage, networks, topologies"},
-        {"topic_code": "CS-KS4-03", "title": "Algorithms and Programming", "year_group": "10", "key_stage": "4",
-         "description": "Algorithms, flowcharts, pseudocode, debugging, testing"},
-        {"topic_code": "CS-KS4-04", "title": "Data Types and Structures", "year_group": "11", "key_stage": "4",
-         "description": "Strings, arrays, lists, records, 2D arrays"},
-        {"topic_code": "CS-KS4-05", "title": "Boolean Logic and Search", "year_group": "11", "key_stage": "4",
-         "description": "Boolean operators, linear search, binary search, sorting"},
-    ],
-}
-
-
-# ─── Helpers ───────────────────────────────────────────────────────────────────
-
-def _sb_headers() -> Dict:
-    return {
-        "apikey": SERVICE_KEY,
-        "Authorization": f"Bearer {SERVICE_KEY}",
-        "Content-Type": "application/json",
-    }
-
-
-def _sign_in(email: str, password: str) -> str:
-    r = requests.post(
-        f"{SUPA_URL}/auth/v1/token?grant_type=password",
-        headers={"apikey": SERVICE_KEY, "Content-Type": "application/json"},
-        json={"email": email, "password": password},
-    )
-    r.raise_for_status()
-    return r.json()["access_token"]
-
-
-# ─── Main seed ─────────────────────────────────────────────────────────────────
-
-def seed() -> Dict[str, Any]:
-    print("=" * 60)
-    print("Curriculum seed — exam board specs and exams")
-    print("=" * 60)
-    results: Dict[str, Any] = {}
-    errors: List[str] = []
-
-    # ── [1] Seed eb_specifications ──────────────────────────────────────────
-    print("\n[1] Seeding exam board specifications...")
-    specs_created = 0
-    specs_skipped = 0
-
-    for spec in SPECIFICATIONS:
-        r = requests.post(
-            f"{SUPA_URL}/rest/v1/eb_specifications",
-            headers={**_sb_headers(), "Prefer": "return=representation"},
-            json={
-                **spec,
-                "id": str(uuid.uuid4()),
-                "doc_details": {},
-                "docling_docs": {},
-            },
-            params={"on_conflict": "spec_code"},
-        )
-        if r.status_code in (200, 201):
-            specs_created += 1
-            print(f"  ✓ {spec['spec_code']} ({spec['exam_board_code']}/{spec['subject_code']})")
-        elif r.status_code == 409:
-            specs_skipped += 1
-            print(f"  ~ SKIP (exists): {spec['spec_code']}")
-        else:
-            err = f"spec {spec['spec_code']}: {r.status_code} {r.text[:100]}"
-            print(f"  ✗ {err}")
-            errors.append(err)
-
-    results["specifications"] = {"created": specs_created, "skipped": specs_skipped}
-
-    # ── [2] Seed eb_exams ───────────────────────────────────────────────────
-    print("\n[2] Seeding exam papers...")
-    exams_created = 0
-    exams_skipped = 0
-
-    for exam in EXAMS:
-        r = requests.post(
-            f"{SUPA_URL}/rest/v1/eb_exams",
-            headers={**_sb_headers(), "Prefer": "return=representation"},
-            json={
-                **exam,
-                "id": str(uuid.uuid4()),
-                "doc_details": {},
-                "docling_docs": {},
-            },
-            params={"on_conflict": "exam_code"},
-        )
-        if r.status_code in (200, 201):
-            exams_created += 1
-            print(f"  ✓ {exam['exam_code']} ({exam['type_code']})")
-        elif r.status_code == 409:
-            exams_skipped += 1
-            print(f"  ~ SKIP (exists): {exam['exam_code']}")
-        else:
-            err = f"exam {exam['exam_code']}: {r.status_code} {r.text[:100]}"
-            print(f"  ✗ {err}")
-            errors.append(err)
-
-    results["exams"] = {"created": exams_created, "skipped": exams_skipped}
-
-    # ── [3] Seed Neo4j curriculum topics ────────────────────────────────────
-    print("\n[3] Seeding Neo4j curriculum topics...")
-    try:
-        from neo4j import GraphDatabase
-        driver = GraphDatabase.driver("bolt://192.168.0.209:7687", auth=("neo4j", "&%N304j&%"))
-
-        topics_created = 0
-        topics_skipped = 0
-
-        for school_id, school_name in [(KEVLARAI_ID, "KevlarAI"), (GREENFIELD_ID, "Greenfield Academy")]:
-            db_name = f"cc.institutes.{school_id.replace('-', '')}"
-            print(f"\n  [{school_name}] -> {db_name}")
-
-            with driver.session(database=db_name) as s:
-                for subject, topics in CURRICULUM_TOPICS.items():
-                    # Create subject node
-                    s.run(
-                        "MERGE (s:Subject {code: $subject}) "
-                        "SET s.title = $title, s.school_id = $school_id",
-                        subject=subject, title=subject, school_id=school_id,
-                    )
-
-                    for topic in topics:
-                        result = s.run(
-                            "MERGE (t:CurriculumTopic {code: $code}) "
-                            "SET t.title = $title, "
-                            "    t.year_group = $year_group, "
-                            "    t.key_stage = $key_stage, "
-                            "    t.description = $description, "
-                            "    t.subject_code = $subject, "
-                            "    t.school_id = $school_id "
-                            "MERGE (s:Subject {code: $subject}) "
-                            "MERGE (s)-[:CONTAINS_TOPIC]->(t)",
-                            code=topic["topic_code"],
-                            title=topic["title"],
-                            year_group=topic["year_group"],
-                            key_stage=topic["key_stage"],
-                            description=topic["description"],
-                            subject=subject,
-                            school_id=school_id,
-                        )
-                        # Check if it was created or matched
-                        topics_created += 1
-
-            print(f"    ✓ {school_name}: {len(CURRICULUM_TOPICS) * len(list(CURRICULUM_TOPICS.values())[0])} topic nodes")
-
-        driver.close()
-        results["neo4j_topics"] = {"created": topics_created}
-
-    except Exception as e:
-        err = f"neo4j_topics: {e}"
-        print(f"  ✗ {err}")
-        errors.append(err)
-        results["neo4j_topics"] = {"error": str(e)}
-
-    # ── Summary ─────────────────────────────────────────────────────────────
-    print("\n" + "=" * 60)
-    results["success"] = len(errors) == 0
-    results["errors"] = errors
-    print(f"COMPLETE — {specs_created} specs, {exams_created} exams, "
-          f"{results.get('neo4j_topics', {}).get('created', '?')} topics")
-    if errors:
-        print(f"Errors ({len(errors)}):")
-        for e in errors:
-            print(f"  ✗ {e}")
-    print("=" * 60)
-    return results
-
-
-if __name__ == "__main__":
-    import json
-    print(json.dumps(seed(), indent=2, default=str))

run/initialization/seed_environment.py

@@ -1,531 +0,0 @@
-"""
-seed_environment.py — idempotent full-environment rebuild.
-
-Assumes reset_environment.py has already been run (or it's the first boot).
-Safe to re-run: all writes use UPSERT / MERGE.
-
-Schools
--------
-  KevlarAI          6585bf91-6ae8-4d72-ab54-cddf3ba4e648  kevlarai.test
-  Greenfield Academy a1b2c3d4-e5f6-7890-abcd-ef1234567890  greenfieldacademy.test
-
-Uniform accounts per school (10 × 2 = 20 total)
-------------------------------------------------
-  admin@{domain}    school_admin
-  head@{domain}     school_admin
-  physics@{domain}  teacher
-  maths@{domain}    teacher
-  teacher1@{domain} teacher
-  teacher2@{domain} teacher
-  teacher3@{domain} teacher
-  student1@{domain} student
-  student2@{domain} student
-  student3@{domain} student
-
-Run from inside the ccapi container:
-    python3 main.py --mode seed
-    python3 main.py --mode seed-test
-
-Or directly:
-    python3 -c "from run.initialization.seed_environment import seed; seed()"
-    python3 -c "from run.initialization.seed_environment import seed; seed(test=True)"
-"""
-import os
-import time
-import requests
-from datetime import datetime, timedelta
-from typing import Dict, Any, List, Optional
-
-from modules.logger_tool import initialise_logger
-
-logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), "default", True)
-
-# ─── School constants ─────────────────────────────────────────────────────────
-
-KEVLARAI_ID   = "6585bf91-6ae8-4d72-ab54-cddf3ba4e648"
-KEVLARAI_NAME = "KevlarAI"
-KEVLARAI_URN  = "KEVLARAI-001"
-KEVLARAI_DOMAIN = "kevlarai.test"
-
-GREENFIELD_ID   = "a1b2c3d4-e5f6-7890-abcd-ef1234567890"
-GREENFIELD_NAME = "Greenfield Academy"
-GREENFIELD_URN  = "TEST-GFA-001"
-GREENFIELD_DOMAIN = "greenfieldacademy.test"
-
-# ─── Passwords ────────────────────────────────────────────────────────────────
-
-DEFAULT_PLATFORM_ADMIN_PASSWORD = "KevlarAI2025!"
-DEFAULT_SCHOOL_ADMIN_PASSWORD = "Admin@Cc2025!"
-DEFAULT_TEACHER_PASSWORD = "Teacher@Cc2025!"
-DEFAULT_STUDENT_PASSWORD = "Student@Cc2025!"
-
-
-def get_seed_password(role: str) -> str:
-    """Return the seed password for a role, allowing env overrides."""
-    normalized = role.lower().strip()
-    env_by_role = {
-        "platform_admin": ("SEED_PLATFORM_ADMIN_PASSWORD", DEFAULT_PLATFORM_ADMIN_PASSWORD),
-        "school_admin": ("SEED_SCHOOL_ADMIN_PASSWORD", DEFAULT_SCHOOL_ADMIN_PASSWORD),
-        "teacher": ("SEED_TEACHER_PASSWORD", DEFAULT_TEACHER_PASSWORD),
-        "student": ("SEED_STUDENT_PASSWORD", DEFAULT_STUDENT_PASSWORD),
-    }
-    if normalized not in env_by_role:
-        raise ValueError(f"Unknown seed password role: {role}")
-    env_name, default = env_by_role[normalized]
-    return os.getenv(env_name, default)
-
-
-def get_seed_passwords() -> Dict[str, str]:
-    return {
-        "platform_admin": get_seed_password("platform_admin"),
-        "school_admin": get_seed_password("school_admin"),
-        "teacher": get_seed_password("teacher"),
-        "student": get_seed_password("student"),
-    }
-
-
-# ─── Account template ────────────────────────────────────────────────────────
-
-def _school_accounts(domain: str, institute_id: str) -> List[Dict]:
-    passwords = get_seed_passwords()
-    return [
-        # school_admin accounts
-        {
-            "prefix": "admin",  "email": f"admin@{domain}",
-            "full_name": "Alex Admin",    "display_name": "Alex",
-            "username": f"admin.{domain.replace('.', '_')}",
-            "user_type": "teacher", "role": "school_admin", "password": passwords["school_admin"],
-            "institute_id": institute_id,
-        },
-        {
-            "prefix": "head",   "email": f"head@{domain}",
-            "full_name": "Helen Head",    "display_name": "Helen",
-            "username": f"head.{domain.replace('.', '_')}",
-            "user_type": "teacher", "role": "school_admin", "password": passwords["school_admin"],
-            "institute_id": institute_id,
-        },
-        # teacher accounts
-        {
-            "prefix": "physics", "email": f"physics@{domain}",
-            "full_name": "Phil Physics",  "display_name": "Phil",
-            "username": f"physics.{domain.replace('.', '_')}",
-            "user_type": "teacher", "role": "teacher", "password": passwords["teacher"],
-            "institute_id": institute_id,
-        },
-        {
-            "prefix": "maths",   "email": f"maths@{domain}",
-            "full_name": "Mary Maths",    "display_name": "Mary",
-            "username": f"maths.{domain.replace('.', '_')}",
-            "user_type": "teacher", "role": "teacher", "password": passwords["teacher"],
-            "institute_id": institute_id,
-        },
-        {
-            "prefix": "teacher1", "email": f"teacher1@{domain}",
-            "full_name": "Tom Teacher",   "display_name": "Tom",
-            "username": f"teacher1.{domain.replace('.', '_')}",
-            "user_type": "teacher", "role": "teacher", "password": passwords["teacher"],
-            "institute_id": institute_id,
-        },
-        {
-            "prefix": "teacher2", "email": f"teacher2@{domain}",
-            "full_name": "Tara Teach",    "display_name": "Tara",
-            "username": f"teacher2.{domain.replace('.', '_')}",
-            "user_type": "teacher", "role": "teacher", "password": passwords["teacher"],
-            "institute_id": institute_id,
-        },
-        {
-            "prefix": "teacher3", "email": f"teacher3@{domain}",
-            "full_name": "Tim Teachwell", "display_name": "Tim",
-            "username": f"teacher3.{domain.replace('.', '_')}",
-            "user_type": "teacher", "role": "teacher", "password": passwords["teacher"],
-            "institute_id": institute_id,
-        },
-        # student accounts
-        {
-            "prefix": "student1", "email": f"student1@{domain}",
-            "full_name": "Sam Student",   "display_name": "Sam",
-            "username": f"student1.{domain.replace('.', '_')}",
-            "user_type": "student", "role": "student", "password": passwords["student"],
-            "institute_id": institute_id,
-        },
-        {
-            "prefix": "student2", "email": f"student2@{domain}",
-            "full_name": "Sophie Study",  "display_name": "Sophie",
-            "username": f"student2.{domain.replace('.', '_')}",
-            "user_type": "student", "role": "student", "password": passwords["student"],
-            "institute_id": institute_id,
-        },
-        {
-            "prefix": "student3", "email": f"student3@{domain}",
-            "full_name": "Steve Scholar", "display_name": "Steve",
-            "username": f"student3.{domain.replace('.', '_')}",
-            "user_type": "student", "role": "student", "password": passwords["student"],
-            "institute_id": institute_id,
-        },
-    ]
-
-FULL_ACCOUNTS = (
-    _school_accounts(KEVLARAI_DOMAIN, KEVLARAI_ID) +
-    _school_accounts(GREENFIELD_DOMAIN, GREENFIELD_ID)
-)
-
-
-def get_accounts(test: bool = False) -> List[Dict]:
-    """Return full (20-user) or lightweight test (9-user) seed fixtures."""
-    if not test:
-        return list(FULL_ACCOUNTS)
-
-    wanted = {
-        f"student1@{KEVLARAI_DOMAIN}",
-        f"student2@{KEVLARAI_DOMAIN}",
-        f"student3@{KEVLARAI_DOMAIN}",
-        f"admin@{GREENFIELD_DOMAIN}",
-        f"physics@{GREENFIELD_DOMAIN}",
-        f"maths@{GREENFIELD_DOMAIN}",
-        f"teacher1@{GREENFIELD_DOMAIN}",
-        f"student1@{GREENFIELD_DOMAIN}",
-        f"student2@{GREENFIELD_DOMAIN}",
-    }
-    return [account for account in FULL_ACCOUNTS if account["email"] in wanted]
-
-# ─── Supabase helpers ─────────────────────────────────────────────────────────
-
-def _sb_ctx():
-    url = os.environ["SUPABASE_URL"]
-    key = os.environ["SERVICE_ROLE_KEY"]
-    headers = {
-        "apikey": key,
-        "Authorization": f"Bearer {key}",
-        "Content-Type": "application/json",
-    }
-    return url, headers
-
-
-def _auth_post(url, headers, path, data):
-    return requests.post(f"{url}/auth/v1/admin{path}", headers=headers, json=data)
-
-
-def _auth_get(url, headers, path, params=None):
-    r = requests.get(f"{url}/auth/v1/admin{path}", headers=headers, params=params)
-    r.raise_for_status()
-    return r.json()
-
-
-def _rest_upsert(url, headers, table, data, on_conflict):
-    h = {**headers, "Prefer": "resolution=merge-duplicates,return=representation"}
-    r = requests.post(
-        f"{url}/rest/v1/{table}",
-        headers=h,
-        json=data,
-        params={"on_conflict": on_conflict},
-    )
-    return r
-
-
-def _rest_patch(url, headers, table, match_col, match_val, data):
-    r = requests.patch(
-        f"{url}/rest/v1/{table}",
-        headers={**headers, "Prefer": "return=minimal"},
-        params={match_col: f"eq.{match_val}"},
-        json=data,
-    )
-    return r
-
-
-# ─── Main seed function ───────────────────────────────────────────────────────
-
-def seed(test: bool = False) -> Dict[str, Any]:
-    from modules.database.services.provisioning_service import ProvisioningService
-    from modules.database.services.neo4j_service import Neo4jService
-    from modules.database.init.init_calendar import create_calendar
-
-    accounts = get_accounts(test=test)
-    url, headers = _sb_ctx()
-    errors: List[str] = []
-    results: Dict[str, Any] = {"mode": "test" if test else "full", "account_count": len(accounts)}
-
-    # ── Step 1: Fix KevlarAI institute record ─────────────────────────────────
-    logger.info("=" * 60)
-    logger.info(f"SEED ENVIRONMENT ({'test' if test else 'full'} mode)")
-    logger.info("=" * 60)
-    logger.info("\n[1] KevlarAI institute record...")
-    try:
-        r = _rest_upsert(url, headers, "institutes", {
-            "id":      KEVLARAI_ID,
-            "name":    KEVLARAI_NAME,
-            "urn":     KEVLARAI_URN,
-            "status":  "active",
-            "website": "https://kevlarai.test",
-            "address": {"line1": "1 AI Lane", "city": "London", "postcode": "EC1A 1BB"},
-            "metadata": {"headteacher": "Alex Admin", "seeded": True},
-        }, on_conflict="id")
-        if r.status_code in (200, 201):
-            logger.info("  KevlarAI upserted ✓")
-        else:
-            raise Exception(r.text[:200])
-    except Exception as e:
-        errors.append(f"kevlarai_institute: {e}")
-        logger.error(f"  {e}")
-
-    # ── Step 2: Create Greenfield Academy if needed ───────────────────────────
-    logger.info("[2] Greenfield Academy institute record...")
-    try:
-        neo4j_uuid_greenfield = GREENFIELD_ID.replace("-", "")
-        r = _rest_upsert(url, headers, "institutes", {
-            "id":               GREENFIELD_ID,
-            "name":             GREENFIELD_NAME,
-            "urn":              GREENFIELD_URN,
-            "status":           "active",
-            "website":          "https://greenfieldacademy.test",
-            "address":          {"line1": "1 Academy Road", "city": "Testville", "postcode": "TE1 1ST"},
-            "metadata":         {"headteacher": "Alex Admin", "seeded": True},
-            "neo4j_uuid_string": neo4j_uuid_greenfield,
-        }, on_conflict="id")
-        if r.status_code in (200, 201):
-            logger.info("  Greenfield Academy upserted ✓")
-        else:
-            raise Exception(r.text[:200])
-    except Exception as e:
-        errors.append(f"greenfield_institute: {e}")
-        logger.error(f"  {e}")
-
-    # ── Step 3: Provision Neo4j for both schools ──────────────────────────────
-    logger.info("[3] Neo4j school provisioning...")
-    provisioner = ProvisioningService()
-    school_dbs: Dict[str, str] = {}
-
-    for iid, name in [(KEVLARAI_ID, "KevlarAI"), (GREENFIELD_ID, "Greenfield Academy")]:
-        try:
-            result = provisioner.ensure_school(iid)
-            db = result["db_name"]
-            school_dbs[iid] = db
-            logger.info(f"  {name}: {db} ✓")
-        except Exception as e:
-            errors.append(f"ensure_school {name}: {e}")
-            logger.error(f"  {name}: {e}")
-            # derive fallback db name
-            school_dbs[iid] = f"cc.institutes.{iid.replace('-', '')}"
-
-    # ── Step 4: Rebuild classroomcopilot global calendar ─────────────────────
-    logger.info("[4] classroomcopilot global calendar (2024–2028)...")
-    try:
-        neo4j_svc = Neo4jService()
-        neo4j_svc.create_database("classroomcopilot")
-        logger.info("  DB created, waiting 5s for availability...")
-        time.sleep(5)
-        start_dt = datetime(2024, 1, 1)
-        end_dt   = datetime(2028, 12, 31)
-        create_calendar("classroomcopilot", start_dt, end_dt)
-        logger.info("  Calendar built ✓")
-        results["global_calendar"] = "ok"
-    except Exception as e:
-        errors.append(f"global_calendar: {e}")
-        logger.error(f"  {e}")
-        results["global_calendar"] = "error"
-
-    # ── Step 5: Create / verify auth users ────────────────────────────────────
-    logger.info(f"[5] Creating auth users ({len(accounts)} accounts)...")
-    try:
-        existing = _auth_get(url, headers, "/users", {"per_page": 200}).get("users", [])
-        existing_by_email = {u["email"]: u for u in existing}
-    except Exception as e:
-        errors.append(f"list_auth_users: {e}")
-        existing_by_email = {}
-
-    created_users: Dict[str, str] = {}  # email → uid
-    for spec in accounts:
-        email = spec["email"]
-        if email in existing_by_email:
-            created_users[email] = existing_by_email[email]["id"]
-            logger.info(f"  {email}: exists [{created_users[email][:8]}]")
-            continue
-        r = _auth_post(url, headers, "/users", {
-            "email":         email,
-            "password":      spec["password"],
-            "email_confirm": True,
-            "user_metadata": {
-                "username":     spec["username"],
-                "full_name":    spec["full_name"],
-                "display_name": spec["display_name"],
-                "user_type":    spec["user_type"],
-            },
-        })
-        if r.status_code in (200, 201):
-            uid = r.json()["id"]
-            created_users[email] = uid
-            logger.info(f"  {email}: created [{uid[:8]}]")
-        else:
-            errors.append(f"create {email}: {r.text[:150]}")
-            logger.error(f"  {email}: {r.text[:150]}")
-        time.sleep(0.2)
-
-    results["users_created"] = len(created_users)
-
-    # ── Step 6: Upsert profiles and memberships ───────────────────────────────
-    logger.info("[6] Upserting profiles and memberships...")
-    for spec in accounts:
-        uid = created_users.get(spec["email"])
-        if not uid:
-            continue
-        try:
-            _rest_upsert(url, headers, "profiles", {
-                "id":           uid,
-                "email":        spec["email"],
-                "user_type":    spec["user_type"],
-                "username":     spec["username"],
-                "full_name":    spec["full_name"],
-                "display_name": spec["display_name"],
-                "school_id":    spec["institute_id"],
-                "neo4j_sync_status": "pending",
-            }, on_conflict="id")
-            _rest_upsert(url, headers, "institute_memberships", {
-                "profile_id":   uid,
-                "institute_id": spec["institute_id"],
-                "role":         spec["role"],
-                "metadata":     {},
-            }, on_conflict="profile_id,institute_id")
-        except Exception as e:
-            errors.append(f"profile/membership {spec['email']}: {e}")
-            logger.error(f"  {spec['email']}: {e}")
-
-    logger.info("  Profiles and memberships upserted ✓")
-
-    # ── Step 7: Merge Neo4j Teacher/Student nodes ─────────────────────────────
-    logger.info("[7] Merging Neo4j worker nodes...")
-    try:
-        from modules.database.tools.neo4j_driver_tools import close_driver, get_driver
-        bolt_url = os.getenv("NEO4J_BOLT_URL") or os.getenv("APP_BOLT_URL")
-        neo4j_user = os.getenv("NEO4J_USER") or os.getenv("USER_NEO4J")
-        neo4j_password = os.getenv("NEO4J_PASSWORD") or os.getenv("PASSWORD_NEO4J")
-        auth = (neo4j_user, neo4j_password) if neo4j_user and neo4j_password else None
-        driver = get_driver(url=bolt_url, auth=auth) if bolt_url else get_driver()
-        if driver is None:
-            raise RuntimeError("Neo4j driver unavailable; check NEO4J_BOLT_URL/APP_BOLT_URL and NEO4J_PASSWORD/PASSWORD_NEO4J")
-
-        # Group by institute DB
-        by_db: Dict[str, List[Dict]] = {}
-        for spec in accounts:
-            uid = created_users.get(spec["email"])
-            if not uid:
-                continue
-            db = school_dbs.get(spec["institute_id"])
-            if not db:
-                continue
-            by_db.setdefault(db, []).append({**spec, "uid": uid})
-
-        for db, users in by_db.items():
-            with driver.session(database=db) as s:
-                for u in users:
-                    label = "Teacher" if u["user_type"] == "teacher" else "Student"
-                    s.run(
-                        f"MERGE (n:{label} {{uuid_string: $uid}}) "
-                        "SET n.worker_email = $email, "
-                        "    n.worker_name  = $name, "
-                        "    n.worker_type  = $utype",
-                        uid=u["uid"], email=u["email"],
-                        name=u["full_name"], utype=u["user_type"],
-                    )
-            logger.info(f"  [{db[:35]}] {len(users)} nodes merged ✓")
-
-        close_driver(driver)
-        results["neo4j_nodes"] = "ok"
-    except Exception as e:
-        errors.append(f"neo4j_nodes: {e}")
-        logger.error(f"  {e}")
-        results["neo4j_nodes"] = "error"
-
-    # ── Ensure kcar is a platform super-admin ─────────────────────────────────
-    logger.info("[8] Ensuring kcar platform admin record...")
-    KCAR_ID = "d9e1d1a9-04c4-4611-bb05-57babf4a9a28"
-    try:
-        _rest_upsert(url, headers, "admin_profiles", {
-            "id":             KCAR_ID,
-            "email":          "[email protected]",
-            "display_name":   "Kevin Carroll",
-            "admin_role":     "super_admin",
-            "is_super_admin": True,
-            "metadata":       {"seeded": True},
-        }, on_conflict="id")
-        logger.info("  kcar → admin_profiles ✓")
-    except Exception as e:
-        errors.append(f"kcar_admin: {e}")
-        logger.error(f"  {e}")
-
-    # Fix kcar's auth user_metadata and profiles.user_type to "platform_admin".
-    # Without this, POST /user/init assigns kcar to the default school on first login.
-    try:
-        r = requests.put(
-            f"{url}/auth/v1/admin/users/{KCAR_ID}",
-            headers=headers,
-            json={"user_metadata": {"user_type": "platform_admin"}},
-        )
-        if r.status_code in (200, 201):
-            logger.info("  kcar → auth user_metadata: platform_admin ✓")
-        else:
-            logger.warning(f"  kcar user_metadata patch failed ({r.status_code}): {r.text[:120]}")
-    except Exception as e:
-        errors.append(f"kcar_user_type: {e}")
-        logger.error(f"  {e}")
-    try:
-        r = requests.patch(
-            f"{url}/rest/v1/profiles",
-            headers={**headers, "Prefer": "return=minimal"},
-            params={"id": f"eq.{KCAR_ID}"},
-            json={"school_id": None},
-        )
-        if r.status_code in (200, 204):
-            logger.info("  kcar → profiles.school_id: null ✓")
-        else:
-            logger.warning(f"  kcar profiles patch failed ({r.status_code}): {r.text[:120]}")
-    except Exception as e:
-        errors.append(f"kcar_profile: {e}")
-        logger.error(f"  {e}")
-
-    # ── Summary ───────────────────────────────────────────────────────────────
-    results["success"] = len(errors) == 0
-    results["errors"]  = errors
-
-    _print_credential_sheet(created_users, accounts)
-
-    logger.info("\n" + "=" * 60)
-    if errors:
-        logger.info(f"SEED COMPLETE with {len(errors)} error(s)")
-        for e in errors:
-            logger.info(f"  ✗ {e}")
-    else:
-        logger.info("SEED COMPLETE — all steps succeeded")
-    logger.info("=" * 60)
-    return results
-
-
-def _print_credential_sheet(created_users: Dict[str, str], accounts: List[Dict]):
-    PAD = 36
-    include_passwords = os.getenv("PRINT_SEED_CREDENTIALS", "").lower() in {"1", "true", "yes", "on"}
-    logger.info("\n" + "=" * 70)
-    logger.info("CREDENTIAL SHEET" + ("" if include_passwords else " (passwords redacted; set PRINT_SEED_CREDENTIALS=true to print)"))
-    logger.info("=" * 70)
-    logger.info(f"  {'ROLE':<16} {'EMAIL':<{PAD}} PASSWORD")
-    logger.info(f"  {'-'*14}  {'-'*(PAD-2)}  -----------")
-    platform_password = get_seed_password("platform_admin") if include_passwords else "<redacted>"
-    logger.info(f"  {'[platform admin]':<16} {'[email protected]':<{PAD}} {platform_password}")
-    logger.info("")
-
-    for school_id, domain, label in [
-        (KEVLARAI_ID,   KEVLARAI_DOMAIN,   "KevlarAI"),
-        (GREENFIELD_ID, GREENFIELD_DOMAIN, "Greenfield Academy"),
-    ]:
-        logger.info(f"  [{label}]")
-        for spec in accounts:
-            if spec["institute_id"] != school_id:
-                continue
-            uid = created_users.get(spec["email"], "—")
-            status = f"[{uid[:8]}]" if uid != "—" else "[MISSING]"
-            password = spec["password"] if include_passwords else "<redacted>"
-            logger.info(f"  {spec['role']:<16} {spec['email']:<{PAD}} {password}  {status}")
-        logger.info("")
-    logger.info("=" * 70)
-
-
-if __name__ == "__main__":
-    import json
-    print(json.dumps(seed(test="--test" in os.sys.argv), indent=2, default=str))

run/initialization/seed_exam_corpus.py

@@ -1,867 +0,0 @@
-"""
-seed_exam_corpus.py — manifest-driven loader for the public exam-paper corpus.
-
-SCOPE (separate from infra): assumes storage buckets already exist (provisioned by
-run/initialization/buckets.py during infra init). This loader UPLOADS papers and
-SEEDS the catalogue; it does NOT create buckets.
-
-Pipeline per manifest item:
-  validate -> resolve source bytes (local path | url:, cached) -> upload file to
-  cc.examboards (canonical path, skip-if-exists unless --force) -> upsert
-  eb_specifications / eb_exams (catalogue) -> (optional, --user-subset) copy a subset
-  into a test user's exam space so user-side flows are testable -> (optional,
-  --first-sweep) run the docling/auto-map first pass to gather structure.
-
-Manifest template: ~/cc/specs/exam-corpus-manifest.example.yaml
-
-Catalogue columns (real — verified against volumes/db/cc/61-core-schema.sql):
-  eb_specifications(spec_code UNIQUE, exam_board_code, award_code, subject_code,
-                    first_teach, spec_ver, storage_loc, doc_type CHECK(pdf|json|...),
-                    doc_details jsonb, docling_docs jsonb)
-  eb_exams(exam_code UNIQUE, spec_code FK, paper_code, tier, session, type_code,
-           storage_loc, doc_type CHECK(pdf|json|...), doc_details jsonb, docling_docs jsonb)
-
-IMPORTANT schema note: the QP/MS/INSERT/ER *document role* is stored in `type_code`
-(the `/catalogue` endpoint filters `type_code == 'QP'`). The `doc_type` column is the
-*file format* and is CHECK-constrained to {pdf,json,md,html,txt,doctags} — so it is
-always 'pdf' here. (The manifest field is named `doc_type` for the role; the loader
-maps manifest.doc_type -> DB.type_code and sets DB.doc_type = 'pdf'.)
-
-Locked conventions (see ~/cc/ideas/2026-06-07-exam-paper-ingestion.md):
-  session    = "YYYY-Mon"  e.g. "2022-Jun", "2021-Nov"
-  exam_code  = "{BOARD}-{award}-{paper_safe}-{SESSIONCOMPACT}-{ROLE}"  e.g. AQA-8463-1H-2022JUN-QP
-  spec path  = cc.examboards/{board}/{subject}/{award}/spec/{spec_ver}.pdf
-  paper path = cc.examboards/{board}/{subject}/{award}/{paper_safe}/{session}/{role}.pdf
-
-Run inside the api container (env: SUPABASE_URL + SERVICE_ROLE_KEY for dev .94), e.g.:
-  python3 -m run.initialization.seed_exam_corpus --manifest /path/exam-corpus.yaml --dry-run
-  python3 -m run.initialization.seed_exam_corpus --manifest ... --board AQA
-  python3 -m run.initialization.seed_exam_corpus --manifest ... --first-sweep
-"""
-from __future__ import annotations
-import argparse
-import hashlib
-import os
-from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional, Tuple
-
-import requests
-import yaml  # PyYAML
-
-from modules.logger_tool import initialise_logger
-from modules.database.supabase.utils.client import SupabaseServiceRoleClient
-from modules.database.supabase.utils.storage import StorageAdmin, StorageError
-
-logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), "default", True)
-
-EXAM_BUCKET = "cc.examboards"
-# Manifest `doc_type` carries the document ROLE (stored in eb_exams.type_code).
-DOC_ROLES = {"QP", "MS", "INSERT", "ER", "SPECIMEN", "GRADE_BOUNDARIES", "DATA_SHEET"}
-TIERS = {"H", "F", None}
-# Default working dir for cached url: downloads (override with --cache-dir / EXAM_CORPUS_CACHE).
-DEFAULT_CACHE_DIR = os.getenv("EXAM_CORPUS_CACHE", "/tmp/exam-corpus-cache")
-# Persistent, mountable local store laid out exactly like the bucket (download once, seed many,
-# offline-repeatable). Override with --store-dir / EXAM_CORPUS_STORE. Distinct from --cache-dir,
-# which is a throwaway url hash-cache.
-DEFAULT_STORE_DIR = os.getenv(
-    "EXAM_CORPUS_STORE",
-    os.path.join(os.path.dirname(os.path.abspath(__file__)), "manifests", "_corpus_store"),
-)
-
-
-# ─────────────────────────────── canonical storage paths ───────────────────────────────
-def _lc(s: str) -> str:
-    return (s or "").strip().lower().replace(" ", "-")
-
-def _paper_safe(paper_code: str) -> str:
-    # Drop the award prefix, keep all remaining segments so combined-science sub-papers
-    # don't collide on the storage path:
-    #   "8463/1H"    -> "1h"
-    #   "8464/B/1H"  -> "b-1h"   (Trilogy: subject letter + paper + tier)
-    #   "7408/1"     -> "1"
-    parts = _lc(paper_code).split("/")
-    return "-".join(parts[1:]) if len(parts) > 1 else parts[0]
-
-def spec_storage_loc(board: str, subject: str, award: str, spec_ver: str) -> str:
-    # e.g. cc.examboards/aqa/physics/8463/spec/1.1.pdf
-    return f"{EXAM_BUCKET}/{_lc(board)}/{_lc(subject)}/{_lc(award)}/spec/{_lc(spec_ver or 'spec')}.pdf"
-
-def paper_storage_loc(board: str, subject: str, award: str, paper_code: str, session: str, doc_role: str) -> str:
-    # e.g. cc.examboards/aqa/physics/8463/1h/2022-jun/qp.pdf
-    return f"{EXAM_BUCKET}/{_lc(board)}/{_lc(subject)}/{_lc(award)}/{_paper_safe(paper_code)}/{_lc(session)}/{_lc(doc_role)}.pdf"
-
-
-# ─────────────────────────────── report ───────────────────────────────
-@dataclass
-class LoadReport:
-    specs_upserted: int = 0
-    papers_upserted: int = 0
-    files_uploaded: int = 0
-    files_skipped: int = 0
-    files_failed: int = 0
-    user_copies: int = 0
-    swept: int = 0
-    sweep_failed: int = 0
-    downloaded: int = 0
-    download_cached: int = 0
-    unseed_objects: int = 0
-    unseed_user_files: int = 0
-    unseed_exams: int = 0
-    unseed_specs: int = 0
-    unseed_templates: int = 0
-    errors: List[str] = field(default_factory=list)
-
-    def as_dict(self) -> Dict[str, Any]:
-        return {
-            "specs_upserted": self.specs_upserted,
-            "papers_upserted": self.papers_upserted,
-            "downloaded": self.downloaded,
-            "download_cached": self.download_cached,
-            "unseed_objects": self.unseed_objects,
-            "unseed_user_files": self.unseed_user_files,
-            "unseed_exams": self.unseed_exams,
-            "unseed_specs": self.unseed_specs,
-            "unseed_templates": self.unseed_templates,
-            "files_uploaded": self.files_uploaded,
-            "files_skipped": self.files_skipped,
-            "files_failed": self.files_failed,
-            "user_copies": self.user_copies,
-            "swept": self.swept,
-            "sweep_failed": self.sweep_failed,
-            "errors": self.errors,
-        }
-
-
-# ─────────────────────────────── validation ───────────────────────────────
-def validate_manifest(m: Dict[str, Any]) -> List[str]:
-    errs: List[str] = []
-    seen_specs, seen_exams = set(), set()
-    for board in m.get("boards", []):
-        bcode = board.get("exam_board_code")
-        if not bcode:
-            errs.append("board missing exam_board_code")
-        for spec in board.get("specifications", []):
-            sc = spec.get("spec_code")
-            if not sc or sc in seen_specs:
-                errs.append(f"spec_code missing/duplicate: {sc!r}")
-            seen_specs.add(sc)
-            for field_name in ("award_code", "subject_code"):
-                if not spec.get(field_name):
-                    errs.append(f"{sc}: missing {field_name}")
-            for p in spec.get("papers", []):
-                ec = p.get("exam_code")
-                if not ec or ec in seen_exams:
-                    errs.append(f"exam_code missing/duplicate: {ec!r}")
-                seen_exams.add(ec)
-                if p.get("doc_type") not in DOC_ROLES:
-                    errs.append(f"{ec}: bad doc_type/role {p.get('doc_type')!r} (want one of {sorted(DOC_ROLES)})")
-                if p.get("tier") not in TIERS:
-                    errs.append(f"{ec}: bad tier {p.get('tier')!r} (want H|F|null)")
-                if not p.get("paper_code"):
-                    errs.append(f"{ec}: missing paper_code")
-                if not p.get("session"):
-                    errs.append(f"{ec}: missing session")
-                src = (p.get("file") or {}).get("source")
-                if not src:
-                    errs.append(f"{ec}: missing file.source")
-                elif not src.startswith("url:") and not os.path.exists(src):
-                    errs.append(f"{ec}: local source not found: {src}")
-    return errs
-
-
-# ─────────────────────────────── source resolution (local | url:, cached) ───────────────────────────────
-def _resolve_source_bytes(source: str, *, cache_dir: str) -> bytes:
-    """Resolve a manifest file source to bytes.
-
-    'url:https://...'  -> fetch (cached to cache_dir by url hash) ; verifies non-empty.
-    '<local path>'     -> read from disk.
-    """
-    if source.startswith("url:"):
-        url = source[len("url:"):]
-        os.makedirs(cache_dir, exist_ok=True)
-        cache_key = hashlib.sha1(url.encode("utf-8")).hexdigest()
-        cache_path = os.path.join(cache_dir, f"{cache_key}.pdf")
-        if os.path.exists(cache_path) and os.path.getsize(cache_path) > 0:
-            with open(cache_path, "rb") as fh:
-                return fh.read()
-        logger.info(f"[fetch] {url}")
-        resp = requests.get(url, timeout=60, allow_redirects=True)
-        resp.raise_for_status()
-        data = resp.content
-        ctype = resp.headers.get("content-type", "")
-        if not data:
-            raise ValueError(f"empty download: {url}")
-        if "pdf" not in ctype.lower() and not data[:5].startswith(b"%PDF"):
-            raise ValueError(f"not a PDF (content-type={ctype!r}): {url}")
-        tmp = cache_path + ".part"
-        with open(tmp, "wb") as fh:
-            fh.write(data)
-        os.replace(tmp, cache_path)
-        return data
-    with open(source, "rb") as fh:
-        return fh.read()
-
-
-# ─────────────────────── persistent local store (download-once, seed-many) ───────────────────────
-def _store_path(store_dir: str, storage_loc: str) -> str:
-    """Local path mirroring the bucket layout (so the store is directly mountable as the corpus):
-    storage_loc 'cc.examboards/aqa/physics/8463/1h/2022-jun/qp.pdf'
-      -> {store_dir}/aqa/physics/8463/1h/2022-jun/qp.pdf
-    """
-    _, _, path = storage_loc.partition("/")
-    return os.path.join(store_dir, path)
-
-def _item_bytes(source: str, storage_loc: str, *, store_dir: Optional[str], cache_dir: str,
-                populate: bool = True, rep: Optional[LoadReport] = None) -> bytes:
-    """Resolve bytes for an item, preferring the persistent local store when present.
-
-    If store_dir holds the file → read it (offline). Otherwise resolve the source (local|url:) and,
-    when populate=True, write it into the store at its canonical path for future offline runs.
-    """
-    if store_dir:
-        sp = _store_path(store_dir, storage_loc)
-        if os.path.exists(sp) and os.path.getsize(sp) > 0:
-            if rep is not None:
-                rep.download_cached += 1
-            with open(sp, "rb") as fh:
-                return fh.read()
-    data = _resolve_source_bytes(source, cache_dir=cache_dir)
-    if store_dir and populate:
-        sp = _store_path(store_dir, storage_loc)
-        os.makedirs(os.path.dirname(sp), exist_ok=True)
-        tmp = sp + ".part"
-        with open(tmp, "wb") as fh:
-            fh.write(data)
-        os.replace(tmp, sp)
-        if rep is not None:
-            rep.downloaded += 1
-    return data
-
-def download_corpus(m: Dict[str, Any], *, store_dir: str, board_filter: Optional[str],
-                    spec_filter: Optional[str], cache_dir: str, rep: LoadReport) -> None:
-    """--download-only: populate the persistent local store from the manifest. No DB/bucket writes.
-    A later run with the same --store-dir (e.g. mounted into the container) seeds offline from it."""
-    for board in m.get("boards", []):
-        if board_filter and board.get("exam_board_code") != board_filter:
-            continue
-        for spec in board.get("specifications", []):
-            if spec_filter and spec.get("spec_code") != spec_filter:
-                continue
-            sf = spec.get("spec_file")
-            if sf and sf.get("source"):
-                sloc = spec_storage_loc(board["exam_board_code"], spec.get("subject_code", ""),
-                                        spec.get("award_code", ""), spec.get("spec_ver", ""))
-                try:
-                    _item_bytes(sf["source"], sloc, store_dir=store_dir, cache_dir=cache_dir, rep=rep)
-                except Exception as exc:
-                    rep.errors.append(f"download spec {spec.get('spec_code')}: {exc}")
-            for p in spec.get("papers", []):
-                ploc = paper_storage_loc(board["exam_board_code"], spec.get("subject_code", ""),
-                                         spec.get("award_code", ""), p["paper_code"], p["session"], p["doc_type"])
-                try:
-                    _item_bytes(p["file"]["source"], ploc, store_dir=store_dir, cache_dir=cache_dir, rep=rep)
-                except Exception as exc:
-                    rep.errors.append(f"download {p.get('exam_code')}: {exc}")
-    logger.info(f"download-only done: downloaded={rep.downloaded} already_in_store={rep.download_cached} "
-                f"errors={len(rep.errors)} store={store_dir}")
-
-
-# ─────────────────────────────── storage upload (skip-if-exists + sha256) ───────────────────────────────
-def _split_loc(storage_loc: str) -> Tuple[str, str]:
-    bucket, _, path = storage_loc.partition("/")
-    return bucket, path
-
-def _object_exists(storage: StorageAdmin, bucket: str, path: str) -> bool:
-    """Existence check by listing the object's parent folder (Supabase storage has no stat)."""
-    parent, _, name = path.rpartition("/")
-    try:
-        listing = storage.client.supabase.storage.from_(bucket).list(parent)
-    except Exception as exc:
-        logger.warning(f"[exists?] list failed for {bucket}/{parent}: {exc}")
-        return False
-    return any((item.get("name") == name) for item in (listing or []))
-
-def upload_file(storage: StorageAdmin, storage_loc: str, data: bytes, *, force: bool, rep: LoadReport) -> str:
-    """Upload PDF bytes to storage at storage_loc. Returns the sha256 of the bytes.
-
-    Idempotent: if the object already exists and --force was not given, skips the upload
-    (the catalogue upsert still runs and records the checksum). With --force, overwrites.
-    """
-    sha = hashlib.sha256(data).hexdigest()
-    bucket, path = _split_loc(storage_loc)
-    if not force and _object_exists(storage, bucket, path):
-        logger.info(f"[upload] skip-exists {storage_loc} (sha256={sha[:12]})")
-        rep.files_skipped += 1
-        return sha
-    try:
-        storage.upload_file(bucket, path, data, "application/pdf", upsert=True)
-        logger.info(f"[upload] {storage_loc} ({len(data)} bytes, sha256={sha[:12]}) force={force}")
-        rep.files_uploaded += 1
-    except StorageError as exc:
-        logger.error(f"[upload] FAILED {storage_loc}: {exc}")
-        rep.files_failed += 1
-        rep.errors.append(f"upload {storage_loc}: {exc}")
-    return sha
-
-
-# ─────────────────────────────── catalogue upserts ───────────────────────────────
-def upsert_specification(client: SupabaseServiceRoleClient, spec: Dict[str, Any],
-                         storage_loc: Optional[str], sha: Optional[str], rep: LoadReport) -> None:
-    sf = spec.get("spec_file") or {}
-    doc_details = {
-        "award_level": spec.get("award_level"),
-        "provenance": sf.get("provenance"),
-        "original_name": sf.get("original_name"),
-        "sha256": sha,
-    }
-    row = {
-        "spec_code": spec["spec_code"],
-        "exam_board_code": spec["exam_board_code"],
-        "award_code": spec.get("award_code"),
-        "subject_code": spec.get("subject_code"),
-        "first_teach": spec.get("first_teach"),
-        "spec_ver": spec.get("spec_ver"),
-        "storage_loc": storage_loc,
-        "doc_type": "pdf",  # file format (CHECK-constrained); the role lives on eb_exams.type_code
-        "doc_details": {k: v for k, v in doc_details.items() if v is not None},
-    }
-    try:
-        client.supabase.table("eb_specifications").upsert(row, on_conflict="spec_code").execute()
-        logger.info(f"[spec] upsert {row['spec_code']}")
-        rep.specs_upserted += 1
-    except Exception as exc:
-        logger.error(f"[spec] FAILED {row['spec_code']}: {exc}")
-        rep.errors.append(f"spec {row['spec_code']}: {exc}")
-
-def upsert_paper(client: SupabaseServiceRoleClient, spec_code: str, p: Dict[str, Any],
-                 storage_loc: str, sha: Optional[str], rep: LoadReport) -> None:
-    f = p.get("file") or {}
-    doc_role = p["doc_type"]  # manifest role: QP|MS|INSERT|ER...
-    doc_details = {
-        "doc_role": doc_role,                # mirror of type_code for clarity
-        "original_name": f.get("original_name"),
-        "provenance": f.get("provenance"),
-        "sha256": sha,
-    }
-    row = {
-        "exam_code": p["exam_code"],
-        "spec_code": spec_code,
-        "paper_code": p.get("paper_code"),
-        "tier": p.get("tier"),
-        "session": p.get("session"),
-        "type_code": doc_role,               # ROLE goes here (QP/MS/INSERT/ER)
-        "doc_type": "pdf",                   # file format (CHECK-constrained)
-        "storage_loc": storage_loc,
-        "doc_details": {k: v for k, v in doc_details.items() if v is not None},
-    }
-    try:
-        client.supabase.table("eb_exams").upsert(row, on_conflict="exam_code").execute()
-        logger.info(f"[paper] upsert {row['exam_code']} type_code={doc_role}")
-        rep.papers_upserted += 1
-    except Exception as exc:
-        logger.error(f"[paper] FAILED {row['exam_code']}: {exc}")
-        rep.errors.append(f"paper {row['exam_code']}: {exc}")
-
-
-# ─────────────────────────────── user-side test subset ───────────────────────────────
-def _resolve_test_user(client: SupabaseServiceRoleClient, cfg: Dict[str, Any]) -> Optional[Tuple[str, str]]:
-    """Resolve (user_id, institute_id) for the user-side subset from config, with discovery fallback."""
-    user_id = cfg.get("user_id")
-    if not user_id and cfg.get("user_email"):
-        res = client.supabase.table("profiles").select("id").eq("email", cfg["user_email"]).limit(1).execute()
-        rows = getattr(res, "data", None) or []
-        user_id = rows[0]["id"] if rows else None
-    if not user_id:
-        logger.warning("[user-subset] no test user resolvable (set test_subset.user_id or user_email); skipping")
-        return None
-    institute_id = cfg.get("institute_id")
-    if not institute_id:
-        res = client.supabase.table("institute_memberships").select("institute_id").eq("profile_id", user_id).limit(1).execute()
-        rows = getattr(res, "data", None) or []
-        institute_id = rows[0]["institute_id"] if rows else None
-    if not institute_id:
-        logger.warning(f"[user-subset] no institute for user {user_id}; skipping")
-        return None
-    return user_id, institute_id
-
-def copy_user_test_subset(client: SupabaseServiceRoleClient, storage: StorageAdmin,
-                          m: Dict[str, Any], rep: LoadReport) -> None:
-    """Copy a small subset of admin papers into a test user's exam space so user-side flows
-    (upload-as-exam / promote-from-cabinet / mark) are testable.
-
-    Driven by an optional manifest `test_subset:` block:
-        test_subset:
-          user_id: <uuid>            # or user_email: <email>
-          institute_id: <uuid>       # optional; discovered from membership if omitted
-          papers: 2                  # how many QP papers to copy (default 2)
-    Degrades gracefully (logs + skips) if no test user is resolvable on this env.
-    """
-    cfg = m.get("test_subset") or {}
-    resolved = _resolve_test_user(client, cfg)
-    if not resolved:
-        return
-    user_id, institute_id = resolved
-    limit = int(cfg.get("papers", 2))
-
-    # Gather candidate QP papers (admin corpus already uploaded to cc.examboards).
-    candidates: List[Tuple[str, Dict[str, Any]]] = []
-    for board in m.get("boards", []):
-        for spec in board.get("specifications", []):
-            for p in spec.get("papers", []):
-                if p.get("doc_type") == "QP":
-                    candidates.append((board["exam_board_code"], spec, p))
-    candidates = candidates[:limit]
-    if not candidates:
-        logger.info("[user-subset] no QP papers to copy")
-        return
-
-    # Ensure a cabinet for the user.
-    cab_name = "Exam Marker Template Sources"
-    res = client.supabase.table("file_cabinets").select("id").eq("user_id", user_id).eq("name", cab_name).limit(1).execute()
-    rows = getattr(res, "data", None) or []
-    if rows:
-        cabinet_id = rows[0]["id"]
-    else:
-        ins = client.supabase.table("file_cabinets").insert({"user_id": user_id, "name": cab_name}).execute()
-        cabinet_id = (getattr(ins, "data", None) or [{}])[0].get("id")
-    if not cabinet_id:
-        logger.warning("[user-subset] could not ensure cabinet; skipping")
-        return
-
-    import uuid as _uuid
-    for board_code, spec, p in candidates:
-        src_loc = paper_storage_loc(board_code, spec.get("subject_code", ""), spec.get("award_code", ""),
-                                    p["paper_code"], p["session"], p["doc_type"])
-        sbucket, spath = _split_loc(src_loc)
-        try:
-            data = storage.download_file(sbucket, spath)
-        except Exception as exc:
-            logger.warning(f"[user-subset] source missing {src_loc}: {exc}; skipping {p['exam_code']}")
-            continue
-        file_id = str(_uuid.uuid4())
-        safe_name = f"{p['exam_code']}.pdf"
-        dst_bucket = "cc.users"
-        dst_path = f"exam-marker/{institute_id}/{cabinet_id}/{file_id}/{safe_name}"
-        try:
-            storage.upload_file(dst_bucket, dst_path, data, "application/pdf", upsert=True)
-        except Exception as exc:
-            logger.warning(f"[user-subset] copy upload failed {dst_path}: {exc}")
-            continue
-        client.supabase.table("files").upsert({
-            "id": file_id, "cabinet_id": cabinet_id, "name": safe_name, "path": dst_path,
-            "bucket": dst_bucket, "mime_type": "application/pdf", "uploaded_by": user_id,
-            "size_bytes": len(data), "source": "exam-corpus-seed", "is_directory": False,
-            "relative_path": safe_name, "processing_status": "uploaded",
-        }).execute()
-        logger.info(f"[user-subset] copied {p['exam_code']} -> {dst_bucket}/{dst_path}")
-        rep.user_copies += 1
-
-
-# ─────────────────────────────── first sweep (docling auto-map) ───────────────────────────────
-def _resolve_system_identity(client: SupabaseServiceRoleClient, m: Dict[str, Any]) -> Optional[Tuple[str, str]]:
-    cfg = m.get("system_identity") or m.get("test_subset") or {}
-    user_id = cfg.get("teacher_id") or cfg.get("user_id")
-    if not user_id and cfg.get("user_email"):
-        res = client.supabase.table("profiles").select("id").eq("email", cfg["user_email"]).limit(1).execute()
-        rows = getattr(res, "data", None) or []
-        user_id = rows[0]["id"] if rows else None
-    institute_id = cfg.get("institute_id")
-    if user_id and not institute_id:
-        res = client.supabase.table("institute_memberships").select("institute_id").eq("profile_id", user_id).limit(1).execute()
-        rows = getattr(res, "data", None) or []
-        institute_id = rows[0]["institute_id"] if rows else None
-    if not user_id or not institute_id:
-        logger.warning("[first-sweep] no system identity (set system_identity.teacher_id+institute_id); skipping sweep")
-        return None
-    return user_id, institute_id
-
-def first_sweep(client: SupabaseServiceRoleClient, storage: StorageAdmin,
-                m: Dict[str, Any], board_filter: Optional[str], spec_filter: Optional[str],
-                cache_dir: str, rep: LoadReport) -> None:
-    """Run the docling/auto_map first pass over seeded QP papers and persist the resulting
-    template structure (questions/response areas/boundaries/layout) via the same mapping the
-    /auto-map endpoint uses. System-owned exam_templates are created per QP paper.
-
-    Requires a resolvable `system_identity` (teacher_id/user_email + institute_id) on this env.
-    """
-    identity = _resolve_system_identity(client, m)
-    if not identity:
-        return
-    teacher_id, institute_id = identity
-
-    # Import the auto-map mapping helpers lazily (pulls fastapi/router only when sweeping).
-    try:
-        from api.services.docling import auto_map, AutoMapError
-        from routers.exam.templates import _map_first_pass_to_rows
-    except Exception as exc:
-        logger.error(f"[first-sweep] could not import auto-map pipeline: {exc}")
-        rep.errors.append(f"first-sweep import: {exc}")
-        return
-
-    sb = client.supabase
-    for board in m.get("boards", []):
-        if board_filter and board.get("exam_board_code") != board_filter:
-            continue
-        for spec in board.get("specifications", []):
-            if spec_filter and spec.get("spec_code") != spec_filter:
-                continue
-            for p in spec.get("papers", []):
-                if p.get("doc_type") != "QP":
-                    continue
-                # Resolve the seeded eb_exams row (id) for the template join.
-                ex = sb.table("eb_exams").select("id, exam_code").eq("exam_code", p["exam_code"]).limit(1).execute()
-                ex_rows = getattr(ex, "data", None) or []
-                exam_id = ex_rows[0]["id"] if ex_rows else None
-
-                loc = paper_storage_loc(board["exam_board_code"], spec.get("subject_code", ""),
-                                        spec.get("award_code", ""), p["paper_code"], p["session"], p["doc_type"])
-                bkt, path = _split_loc(loc)
-                try:
-                    pdf_bytes = storage.download_file(bkt, path)
-                except Exception as exc:
-                    logger.warning(f"[first-sweep] source missing {loc}: {exc}; skipping {p['exam_code']}")
-                    continue
-
-                # Ensure a system-owned template for this paper (idempotent on exam_code+teacher).
-                tpl = sb.table("exam_templates").select("id").eq("exam_code", p["exam_code"]).eq("teacher_id", teacher_id).limit(1).execute()
-                tpl_rows = getattr(tpl, "data", None) or []
-                if tpl_rows:
-                    template_id = tpl_rows[0]["id"]
-                else:
-                    new_tpl = sb.table("exam_templates").insert({
-                        "exam_id": exam_id, "exam_code": p["exam_code"], "institute_id": institute_id,
-                        "teacher_id": teacher_id, "title": f"{p['exam_code']} (auto-map seed)",
-                        "subject": spec.get("subject_code"), "status": "draft",
-                    }).execute()
-                    template_id = (getattr(new_tpl, "data", None) or [{}])[0].get("id")
-                if not template_id:
-                    logger.warning(f"[first-sweep] could not ensure template for {p['exam_code']}; skipping")
-                    continue
-
-                try:
-                    first_pass = auto_map(pdf_bytes, source_pdf=loc)
-                    rows = _map_first_pass_to_rows(template_id, first_pass, pdf_bytes)
-                except (AutoMapError, ValueError) as exc:
-                    logger.warning(f"[first-sweep] auto-map failed for {p['exam_code']}: {exc}")
-                    rep.sweep_failed += 1
-                    continue
-                except Exception as exc:
-                    logger.exception(f"[first-sweep] unexpected error for {p['exam_code']}: {exc}")
-                    rep.sweep_failed += 1
-                    continue
-
-                # Refresh derived rows. Seed templates are system-owned with no human edits to
-                # preserve, so we clear ALL child rows for the template (not just ai/unconfirmed)
-                # and re-insert id-deduped payloads — idempotent across re-runs and robust to the
-                # deterministic uuid5 ids the mapper can repeat within a batch.
-                for table in ("exam_response_areas", "exam_boundaries", "exam_template_layout", "exam_questions"):
-                    sb.table(table).delete().eq("template_id", template_id).execute()
-                for table, key in (("exam_questions", "questions"), ("exam_response_areas", "response_areas"),
-                                   ("exam_boundaries", "boundaries"), ("exam_template_layout", "layout")):
-                    seen_ids: set = set()
-                    payload = []
-                    for r in (rows.get(key) or []):
-                        rid = r.get("id")
-                        if rid is not None and rid in seen_ids:
-                            continue
-                        if rid is not None:
-                            seen_ids.add(rid)
-                        payload.append(r)
-                    if payload:
-                        sb.table(table).insert(payload).execute()
-                updates = {"page_count": first_pass.get("meta", {}).get("n_pages")}
-                sb.table("exam_templates").update({k: v for k, v in updates.items() if v is not None}).eq("id", template_id).execute()
-                logger.info(f"[first-sweep] swept {p['exam_code']} -> template {template_id} "
-                            f"(q={len(rows.get('questions', []))} ra={len(rows.get('response_areas', []))})")
-                rep.swept += 1
-
-
-# ─────────────────────────────── unseed (inverse of the loader) ───────────────────────────────
-def _chunks(seq: List[Any], n: int = 100):
-    for i in range(0, len(seq), n):
-        yield seq[i:i + n]
-
-def _storage_remove(storage: StorageAdmin, bucket: str, paths: List[str]) -> None:
-    """Remove object paths from a bucket through the Supabase Storage API.
-
-    The python client treats missing objects as a successful no-op, which is useful for
-    unseed idempotency. Any API/permission failure is raised so callers can avoid
-    deleting the matching DB rows while storage may still exist.
-    """
-    result = storage.client.supabase.storage.from_(bucket).remove(paths)
-    error = getattr(result, "error", None)
-    if error:
-        raise StorageError(str(error))
-    if isinstance(result, dict) and result.get("error"):
-        raise StorageError(str(result["error"]))
-
-def _delete_user_subset_files(client: SupabaseServiceRoleClient, storage: StorageAdmin, *,
-                              exam_codes: Optional[List[str]], rep: LoadReport) -> None:
-    """Delete --user-subset files from cc.users storage, then their files rows.
-
-    User-subset seeding writes rows with source='exam-corpus-seed', bucket='cc.users',
-    and paths under exam-marker/. Storage must be removed before the files rows: the
-    files GC trigger also tries to delete storage when rows are deleted, so removing
-    objects first avoids trigger failures and keeps this operation idempotent.
-
-    exam_codes=None means remove all user-subset seed rows (used by unscoped unseed
-    even if the eb_* rows were already removed by a prior partial run).
-    """
-    sb = client.supabase
-    seeded_files: List[Dict[str, Any]] = []
-
-    def _base_query():
-        return sb.table("files").select("id, bucket, path, name, source") \
-            .eq("bucket", "cc.users").eq("source", "exam-corpus-seed") \
-            .like("path", "exam-marker/%")
-
-    if exam_codes is None:
-        seeded_files.extend(getattr(_base_query().execute(), "data", None) or [])
-    elif exam_codes:
-        for chunk in _chunks([f"{code}.pdf" for code in exam_codes if code], 100):
-            seeded_files.extend(getattr(_base_query().in_("name", chunk).execute(), "data", None) or [])
-
-    rows_by_id: Dict[str, Dict[str, Any]] = {}
-    paths_by_bucket: Dict[str, List[str]] = {}
-    seen_paths: set = set()
-    for row in seeded_files:
-        row_id = row.get("id")
-        bucket = row.get("bucket")
-        path = row.get("path")
-        if row_id:
-            rows_by_id[str(row_id)] = row
-        if bucket == "cc.users" and isinstance(path, str) and path.startswith("exam-marker/"):
-            key = (bucket, path)
-            if key not in seen_paths:
-                seen_paths.add(key)
-                paths_by_bucket.setdefault(bucket, []).append(path)
-
-    removable_ids = list(rows_by_id)
-    if not removable_ids and not paths_by_bucket:
-        logger.info("[unseed] no user-subset cc.users files to remove")
-        return
-
-    for bkt, paths in paths_by_bucket.items():
-        for chunk in _chunks(paths, 100):
-            try:
-                _storage_remove(storage, bkt, chunk)
-                rep.unseed_objects += len(chunk)
-            except Exception as exc:
-                logger.warning(f"[unseed] user-subset storage remove failed ({bkt}, {len(chunk)} objs): {exc}")
-                rep.errors.append(f"user-subset storage remove {bkt}: {exc}")
-                return
-
-    for chunk in _chunks(removable_ids, 100):
-        try:
-            sb.table("files").delete().in_("id", chunk).execute()
-            rep.unseed_user_files += len(chunk)
-        except Exception as exc:
-            logger.warning(f"[unseed] user-subset files delete failed: {exc}")
-            rep.errors.append(f"user-subset files delete: {exc}")
-
-def unseed(client: SupabaseServiceRoleClient, storage: StorageAdmin, *,
-           board_filter: Optional[str], spec_filter: Optional[str],
-           drop_specs: bool = True, drop_seed_templates: bool = True, rep: LoadReport) -> None:
-    """Inverse of the loader: remove the seeded public corpus, scoped by --board/--spec (or all).
-
-    Deletes (in FK-safe order): cc.examboards storage objects (via the Storage API, since the
-    protect_delete trigger blocks direct SQL deletes), first-sweep exam_templates created by the
-    seed (title '... (auto-map seed)', cascades children), eb_exams rows, then eb_specifications.
-    """
-    sb = client.supabase
-    q = sb.table("eb_specifications").select("spec_code, storage_loc, exam_board_code")
-    if board_filter:
-        q = q.eq("exam_board_code", board_filter)
-    if spec_filter:
-        q = q.eq("spec_code", spec_filter)
-    specs = getattr(q.execute(), "data", None) or []
-    spec_codes = [s["spec_code"] for s in specs]
-    if not spec_codes:
-        if not board_filter and not spec_filter:
-            _delete_user_subset_files(client, storage, exam_codes=None, rep=rep)
-        logger.info("[unseed] no matching specifications; nothing to do")
-        return
-
-    exams: List[Dict[str, Any]] = []
-    for chunk in _chunks(spec_codes):
-        res = sb.table("eb_exams").select("id, exam_code, storage_loc").in_("spec_code", chunk).execute()
-        exams.extend(getattr(res, "data", None) or [])
-
-    # 1) User-subset storage/rows. Storage is removed before files rows so trg_files_gc has
-    # nothing left to collect when rows are deleted.
-    user_subset_exam_codes = None if not board_filter and not spec_filter else [
-        e.get("exam_code") for e in exams if e.get("exam_code")
-    ]
-    _delete_user_subset_files(client, storage, exam_codes=user_subset_exam_codes, rep=rep)
-
-    # 2) Storage objects (Storage API; batch-remove per bucket). Specs may carry a spec PDF too.
-    by_bucket: Dict[str, List[str]] = {}
-    for row in exams + specs:
-        loc = row.get("storage_loc")
-        if not loc or "/" not in loc:
-            continue
-        bkt, _, path = loc.partition("/")
-        by_bucket.setdefault(bkt, []).append(path)
-    for bkt, paths in by_bucket.items():
-        for chunk in _chunks(paths, 100):
-            try:
-                storage.client.supabase.storage.from_(bkt).remove(chunk)
-                rep.unseed_objects += len(chunk)
-            except Exception as exc:
-                logger.warning(f"[unseed] storage remove failed ({bkt}, {len(chunk)} objs): {exc}")
-
-    # 3) First-sweep templates created by the seed (cascades questions/regions/boundaries/layout).
-    if drop_seed_templates and exams:
-        exam_codes = [e["exam_code"] for e in exams if e.get("exam_code")]
-        for chunk in _chunks(exam_codes, 100):
-            try:
-                res = sb.table("exam_templates").delete(count="exact") \
-                    .in_("exam_code", chunk).like("title", "%(auto-map seed)%").execute()
-                rep.unseed_templates += getattr(res, "count", None) or len(getattr(res, "data", []) or [])
-            except Exception as exc:
-                logger.warning(f"[unseed] template delete failed: {exc}")
-
-    # 4) Catalogue rows: eb_exams (by id), then eb_specifications (by spec_code).
-    exam_ids = [e["id"] for e in exams]
-    for chunk in _chunks(exam_ids, 100):
-        try:
-            sb.table("eb_exams").delete().in_("id", chunk).execute()
-            rep.unseed_exams += len(chunk)
-        except Exception as exc:
-            logger.warning(f"[unseed] eb_exams delete failed: {exc}")
-    if drop_specs:
-        for chunk in _chunks(spec_codes, 100):
-            try:
-                sb.table("eb_specifications").delete().in_("spec_code", chunk).execute()
-                rep.unseed_specs += len(chunk)
-            except Exception as exc:
-                logger.warning(f"[unseed] eb_specifications delete failed: {exc}")
-
-    logger.info(f"unseed done: storage_objects={rep.unseed_objects} user_files={rep.unseed_user_files} "
-                f"templates={rep.unseed_templates} exams={rep.unseed_exams} specs={rep.unseed_specs}")
-
-
-# ─────────────────────────────── orchestration ───────────────────────────────
-def load(manifest_path: str, *, dry_run: bool, force: bool, board_filter: Optional[str],
-         spec_filter: Optional[str], user_subset: bool, do_first_sweep: bool,
-         cache_dir: str = DEFAULT_CACHE_DIR, store_dir: Optional[str] = None) -> LoadReport:
-    with open(manifest_path) as f:
-        m = yaml.safe_load(f)
-    rep = LoadReport()
-
-    errs = validate_manifest(m)
-    if errs:
-        rep.errors = list(errs)
-        logger.error(f"manifest validation failed: {len(errs)} error(s)")
-        for e in errs[:40]:
-            logger.error(f"  - {e}")
-        if not dry_run:
-            return rep
-
-    client = None if dry_run else SupabaseServiceRoleClient()
-    storage = None if dry_run else StorageAdmin()
-
-    for board in m.get("boards", []):
-        if board_filter and board.get("exam_board_code") != board_filter:
-            continue
-        for spec in board.get("specifications", []):
-            if spec_filter and spec.get("spec_code") != spec_filter:
-                continue
-            # Specification document (optional).
-            sloc = None
-            spec_sha = None
-            sf = spec.get("spec_file")
-            if sf and sf.get("source"):
-                sloc = spec_storage_loc(board["exam_board_code"], spec.get("subject_code", ""),
-                                        spec.get("award_code", ""), spec.get("spec_ver", ""))
-                if not dry_run:
-                    try:
-                        spec_sha = upload_file(storage, sloc,
-                                               _item_bytes(sf["source"], sloc, store_dir=store_dir,
-                                                           cache_dir=cache_dir, rep=rep),
-                                               force=force, rep=rep)
-                    except Exception as exc:
-                        logger.error(f"[spec-file] {spec.get('spec_code')}: {exc}")
-                        rep.files_failed += 1
-                        rep.errors.append(f"spec-file {spec.get('spec_code')}: {exc}")
-            if not dry_run:
-                upsert_specification(client, spec, sloc, spec_sha, rep)
-
-            # Papers.
-            for p in spec.get("papers", []):
-                ploc = paper_storage_loc(board["exam_board_code"], spec.get("subject_code", ""),
-                                         spec.get("award_code", ""), p["paper_code"], p["session"], p["doc_type"])
-                if dry_run:
-                    continue
-                psha = None
-                try:
-                    psha = upload_file(storage, ploc,
-                                       _item_bytes(p["file"]["source"], ploc, store_dir=store_dir,
-                                                   cache_dir=cache_dir, rep=rep),
-                                       force=force, rep=rep)
-                except Exception as exc:
-                    logger.error(f"[paper-file] {p.get('exam_code')}: {exc}")
-                    rep.files_failed += 1
-                    rep.errors.append(f"paper-file {p.get('exam_code')}: {exc}")
-                upsert_paper(client, spec["spec_code"], p, ploc, psha, rep)
-
-    if user_subset and not dry_run:
-        copy_user_test_subset(client, storage, m, rep)
-    if do_first_sweep and not dry_run:
-        first_sweep(client, storage, m, board_filter, spec_filter, cache_dir, rep)
-
-    logger.info(f"corpus load done: specs={rep.specs_upserted} papers={rep.papers_upserted} "
-                f"uploaded={rep.files_uploaded} skipped={rep.files_skipped} failed={rep.files_failed} "
-                f"user_copies={rep.user_copies} swept={rep.swept} errors={len(rep.errors)}")
-    return rep
-
-
-def main() -> None:
-    ap = argparse.ArgumentParser(description="Seed (or unseed) the public exam-paper corpus from a manifest.")
-    ap.add_argument("--manifest", help="corpus manifest (required except for --unseed)")
-    ap.add_argument("--dry-run", action="store_true", help="validate + report, no writes")
-    ap.add_argument("--force", action="store_true", help="re-upload/overwrite existing storage objects")
-    ap.add_argument("--board", default=None, help="only this exam_board_code")
-    ap.add_argument("--spec", default=None, help="only this spec_code")
-    ap.add_argument("--user-subset", action="store_true", help="also seed a user-side test subset")
-    ap.add_argument("--first-sweep", action="store_true", help="run docling/auto-map first pass on seeded papers")
-    ap.add_argument("--cache-dir", default=DEFAULT_CACHE_DIR, help="throwaway url-hash cache dir")
-    ap.add_argument("--store-dir", default=DEFAULT_STORE_DIR,
-                    help="persistent, bucket-shaped local store (download-once, seed-many)")
-    ap.add_argument("--no-store", action="store_true",
-                    help="ignore the local store; always fetch from source (don't read/populate the store)")
-    ap.add_argument("--download-only", action="store_true",
-                    help="populate the local store from the manifest; no DB/bucket writes")
-    ap.add_argument("--unseed", action="store_true",
-                    help="INVERSE: remove seeded eb_*/storage/first-sweep templates (scoped by --board/--spec)")
-    a = ap.parse_args()
-    store_dir = None if a.no_store else a.store_dir
-    import json
-
-    if a.unseed:
-        rep = LoadReport()
-        unseed(SupabaseServiceRoleClient(), StorageAdmin(),
-               board_filter=a.board, spec_filter=a.spec, rep=rep)
-        print(json.dumps(rep.as_dict(), indent=2))
-        return
-
-    if not a.manifest:
-        ap.error("--manifest is required unless --unseed is given")
-
-    if a.download_only:
-        with open(a.manifest) as f:
-            m = yaml.safe_load(f)
-        rep = LoadReport()
-        download_corpus(m, store_dir=(a.store_dir), board_filter=a.board, spec_filter=a.spec,
-                        cache_dir=a.cache_dir, rep=rep)
-        print(json.dumps(rep.as_dict(), indent=2))
-        return
-
-    rep = load(a.manifest, dry_run=a.dry_run, force=a.force, board_filter=a.board, spec_filter=a.spec,
-               user_subset=a.user_subset, do_first_sweep=a.first_sweep, cache_dir=a.cache_dir,
-               store_dir=store_dir)
-    print(json.dumps(rep.as_dict(), indent=2))
-
-
-if __name__ == "__main__":
-    main()

run/initialization/seed_file_cabinets.py

@@ -1,423 +0,0 @@
-"""
-seed_file_cabinets.py — Create one file cabinet per class with sample document references.
-
-Creates file_cabinets, files, and cabinet_memberships rows via Supabase REST API
-using the service role key. Also creates document_artefacts entries for sample files.
-
-Each cabinet is owned by the class's primary teacher and shared with students
-in that class via cabinet_memberships.
-
-Tables: file_cabinets, files, cabinet_memberships, document_artefacts
-
-Run inside ccapi container:
-    python3 -c "from run.initialization.seed_file_cabinets import seed; seed()"
-"""
-import os
-import time
-import uuid
-import requests
-from typing import Dict, Any, List, Optional
-
-SUPA_URL   = os.environ["SUPABASE_URL"]
-SERVICE_KEY = os.environ["SERVICE_ROLE_KEY"]
-API_BASE   = os.environ.get("API_BASE_URL", "http://localhost:8000")
-
-# ─── Passwords (standardized from T4) ────────────────────────────────────────
-
-PWD_ADMIN   = "Admin@Cc2025!"
-PWD_TEACHER = "Teacher@Cc2025!"
-PWD_STUDENT = "Student@Cc2025!"
-
-
-# ─── Sample file data ────────────────────────────────────────────────────────
-# Each cabinet gets 2-3 sample files with realistic paths.
-
-SAMPLE_FILES = {
-    # Physics cabinets
-    "lesson_plans": [
-        {"name": "forces_motion_plan.pdf", "path": "cc.public.snapshots/lesson_plans/forces_motion.pdf", "mime_type": "application/pdf", "size": "245KB"},
-        {"name": "electric_circuits_plan.pdf", "path": "cc.public.snapshots/lesson_plans/electric_circuits.pdf", "mime_type": "application/pdf", "size": "312KB"},
-    ],
-    "worksheets": [
-        {"name": "worksheet_fma.pdf", "path": "cc.public.snapshots/worksheets/fma_practice.pdf", "mime_type": "application/pdf", "size": "128KB"},
-        {"name": "worksheet_resistance.pdf", "path": "cc.public.snapshots/worksheets/resistance_calc.pdf", "mime_type": "application/pdf", "size": "95KB"},
-    ],
-    "presentations": [
-        {"name": "intro_forces.pptx", "path": "cc.public.snapshots/presentations/forces_intro.pptx", "mime_type": "application/vnd.openxmlformats-officedocument.presentationml.presentation", "size": "2.1MB"},
-    ],
-    # Maths cabinets
-    "lesson_plans": [
-        {"name": "quadratic_factorisation_plan.pdf", "path": "cc.public.snapshots/lesson_plans/quadratics.pdf", "mime_type": "application/pdf", "size": "278KB"},
-    ],
-    "worksheets": [
-        {"name": "worksheet_quadratics.pdf", "path": "cc.public.snapshots/worksheets/quadratic_practice.pdf", "mime_type": "application/pdf", "size": "156KB"},
-        {"name": "worksheet_tree_diagrams.pdf", "path": "cc.public.snapshots/worksheets/tree_diagrams.pdf", "mime_type": "application/pdf", "size": "134KB"},
-    ],
-    # CS cabinets
-    "lesson_plans": [
-        {"name": "intro_python_plan.pdf", "path": "cc.public.snapshots/lesson_plans/intro_python.pdf", "mime_type": "application/pdf", "size": "198KB"},
-    ],
-    "code_samples": [
-        {"name": "hello_world.py", "path": "cc.public.snapshots/code_samples/hello_world.py", "mime_type": "text/x-python", "size": "0.5KB"},
-        {"name": "variables.py", "path": "cc.public.snapshots/code_samples/variables.py", "mime_type": "text/x-python", "size": "1.2KB"},
-    ],
-}
-
-
-# ─── Helpers ───────────────────────────────────────────────────────────────────
-
-def _sb_headers() -> Dict:
-    return {
-        "apikey": SERVICE_KEY,
-        "Authorization": f"Bearer {SERVICE_KEY}",
-        "Content-Type": "application/json",
-    }
-
-
-def _sign_in(email: str, password: str) -> str:
-    r = requests.post(
-        f"{SUPA_URL}/auth/v1/token?grant_type=password",
-        headers={"apikey": SERVICE_KEY, "Content-Type": "application/json"},
-        json={"email": email, "password": password},
-    )
-    r.raise_for_status()
-    return r.json()["access_token"]
-
-
-def _get_profile_id(email: str) -> Optional[str]:
-    """Look up a profile's UUID by email via Supabase service role."""
-    r = requests.get(
-        f"{SUPA_URL}/rest/v1/profiles",
-        headers=_sb_headers(),
-        params={"email": f"eq.{email}", "select": "id", "limit": "1"},
-    )
-    data = r.json() if r.ok else []
-    return data[0]["id"] if data else None
-
-
-def _get_class_info(admin_token: str, class_code: str) -> Optional[Dict]:
-    """Get class info including teacher and students."""
-    r = requests.get(
-        f"{API_BASE}/database/timetable/classes",
-        headers={"Authorization": f"Bearer {admin_token}"},
-        params={"class_code": class_code},
-    )
-    if not r.ok:
-        return None
-    data = r.json()
-    if isinstance(data, list) and data:
-        return data[0]
-    if isinstance(data, dict):
-        return data
-    return None
-
-
-def _get_class_students(admin_token: str, class_id: str) -> List[str]:
-    """Get student profile IDs enrolled in a class."""
-    r = requests.get(
-        f"{API_BASE}/database/timetable/classes/{class_id}/students",
-        headers={"Authorization": f"Bearer {admin_token}"},
-    )
-    if r.ok:
-        data = r.json()
-        if isinstance(data, list):
-            return [s.get("student_id") or s.get("id") for s in data if s.get("student_id") or s.get("id")]
-    return []
-
-
-# ─── Main seed ─────────────────────────────────────────────────────────────────
-
-def seed() -> Dict[str, Any]:
-    print("=" * 60)
-    print("File cabinets seed — both schools")
-    print("=" * 60)
-    results: Dict[str, Any] = {}
-    errors: List[str] = []
-
-    # ── Sign in as both school admins ───────────────────────────────────────
-    print("\n[1] Signing in as school admins...")
-    admin_tokens = {}
-    for school, email, pwd in [
-        ("KevlarAI", "[email protected]", PWD_ADMIN),
-        ("Greenfield", "[email protected]", PWD_ADMIN),
-    ]:
-        try:
-            token = _sign_in(email, pwd)
-            admin_tokens[school] = token
-            print(f"  ✓ {school} admin signed in")
-        except Exception as e:
-            print(f"  ✗ {school} admin login failed: {e}")
-            errors.append(f"{school}_admin_login: {e}")
-
-    if not admin_tokens:
-        return {"success": False, "error": "No admin tokens obtained"}
-
-    # ── Resolve class codes per school ──────────────────────────────────────
-    print("\n[2] Resolving classes per school...")
-
-    # KevlarAI classes
-    kevlarai_classes = [
-        ("10K/Ph1", "[email protected]"),
-        ("11K/Ph1", "[email protected]"),
-        ("10K/Ma1", "[email protected]"),
-        ("11K/Ma1", "[email protected]"),
-        ("10K/CS1", "[email protected]"),
-        ("11K/CS1", "[email protected]"),
-        ("9K/Ph1",  "[email protected]"),
-        ("9K/Ma1",  "[email protected]"),
-    ]
-
-    # Greenfield classes (subset — just a few for cabinet seeding)
-    greenfield_classes = [
-        ("9P/Ph1",  "[email protected]"),
-        ("10P/Ph2", "[email protected]"),
-        ("9M/Ma1",  "[email protected]"),
-        ("10M/Ma1", "[email protected]"),
-        ("9En/1",   "[email protected]"),
-        ("10Hs/1",  "[email protected]"),
-    ]
-
-    # ── Seed KevlarAI cabinets ──────────────────────────────────────────────
-    print("\n[3] Seeding KevlarAI file cabinets...")
-    results["kevlarai"] = {"cabinets": 0, "files": 0, "memberships": 0}
-
-    for class_code, teacher_email in kevlarai_classes:
-        try:
-            # Get class info
-            class_info = _get_class_info(admin_tokens["KevlarAI"], class_code)
-            if not class_info:
-                print(f"  ✗ class not found: {class_code}")
-                errors.append(f"class_not_found: {class_code}")
-                continue
-
-            class_id = class_info.get("id") or class_info
-            teacher_pid = _get_profile_id(teacher_email)
-            if not teacher_pid:
-                print(f"  ✗ teacher profile not found: {teacher_email}")
-                errors.append(f"teacher_profile_not_found: {teacher_email}")
-                continue
-
-            # Get students in this class
-            student_ids = _get_class_students(admin_tokens["KevlarAI"], str(class_id))
-
-            # Determine file category based on subject
-            subject = (class_info.get("subject") or "").lower()
-            if "physics" in subject:
-                file_category = "lesson_plans"
-            elif "math" in subject:
-                file_category = "worksheets"
-            elif "cs" in subject or "computer" in subject:
-                file_category = "code_samples"
-            else:
-                file_category = "lesson_plans"
-
-            files_list = SAMPLE_FILES.get(file_category, SAMPLE_FILES["lesson_plans"])
-
-            # Create cabinet
-            cabinet_id = str(uuid.uuid4())
-            cabinet_name = f"{class_code} — {class_info.get('name', class_code)}"
-
-            r = requests.post(
-                f"{SUPA_URL}/rest/v1/file_cabinets",
-                headers={**_sb_headers(), "Prefer": "return=representation"},
-                json={"id": cabinet_id, "user_id": teacher_pid, "name": cabinet_name},
-                params={"on_conflict": "id"},
-            )
-            if r.status_code in (200, 201):
-                print(f"  ✓ Cabinet: {cabinet_name}")
-                results["kevlarai"]["cabinets"] += 1
-            else:
-                print(f"  ✗ Cabinet create failed ({class_code}): {r.text[:100]}")
-                errors.append(f"cabinet_create: {class_code}")
-                continue
-
-            # Create files in cabinet
-            for fi in files_list:
-                file_id = str(uuid.uuid4())
-                r = requests.post(
-                    f"{SUPA_URL}/rest/v1/files",
-                    headers={**_sb_headers(), "Prefer": "return=representation"},
-                    json={
-                        "id": file_id,
-                        "cabinet_id": cabinet_id,
-                        "name": fi["name"],
-                        "path": fi["path"],
-                        "bucket": "file-cabinets",
-                        "mime_type": fi.get("mime_type"),
-                        "size": fi.get("size"),
-                        "metadata": {},
-                    },
-                    params={"on_conflict": "id"},
-                )
-                if r.status_code in (200, 201):
-                    results["kevlarai"]["files"] += 1
-
-                    # Create document_artefact for this file
-                    artefact_id = str(uuid.uuid4())
-                    requests.post(
-                        f"{SUPA_URL}/rest/v1/document_artefacts",
-                        headers={**_sb_headers(), "Prefer": "return=representation"},
-                        json={
-                            "id": artefact_id,
-                            "file_id": file_id,
-                            "type": fi.get("mime_type", "application/octet-stream").split("/")[-1],
-                            "rel_path": fi["path"],
-                            "status": "processed",
-                            "extra": {"seeded": True, "source": "seed_file_cabinets"},
-                        },
-                        params={"on_conflict": "id"},
-                    )
-
-                time.sleep(0.05)
-
-            # Create cabinet memberships for students
-            for sid in student_ids:
-                r = requests.post(
-                    f"{SUPA_URL}/rest/v1/cabinet_memberships",
-                    headers={**_sb_headers(), "Prefer": "return=minimal"},
-                    json={
-                        "cabinet_id": cabinet_id,
-                        "profile_id": sid,
-                        "role": "viewer",
-                    },
-                    params={"on_conflict": "cabinet_id,profile_id"},
-                )
-                if r.status_code in (200, 201, 409):
-                    results["kevlarai"]["memberships"] += 1
-
-            time.sleep(0.1)
-
-        except Exception as e:
-            err = f"cabinet seed {class_code}: {e}"
-            print(f"  ✗ {err}")
-            errors.append(err)
-
-    # ── Seed Greenfield cabinets ────────────────────────────────────────────
-    print("\n[4] Seeding Greenfield file cabinets...")
-    results["greenfield"] = {"cabinets": 0, "files": 0, "memberships": 0}
-
-    for class_code, teacher_email in greenfield_classes:
-        try:
-            class_info = _get_class_info(admin_tokens["Greenfield"], class_code)
-            if not class_info:
-                print(f"  ✗ class not found: {class_code}")
-                errors.append(f"class_not_found: {class_code}")
-                continue
-
-            class_id = class_info.get("id") or class_info
-            teacher_pid = _get_profile_id(teacher_email)
-            if not teacher_pid:
-                print(f"  ✗ teacher profile not found: {teacher_email}")
-                errors.append(f"teacher_profile_not_found: {teacher_email}")
-                continue
-
-            student_ids = _get_class_students(admin_tokens["Greenfield"], str(class_id))
-
-            subject = (class_info.get("subject") or "").lower()
-            if "physics" in subject:
-                file_category = "lesson_plans"
-            elif "math" in subject:
-                file_category = "worksheets"
-            elif "english" in subject:
-                file_category = "presentations"
-            elif "history" in subject:
-                file_category = "lesson_plans"
-            else:
-                file_category = "lesson_plans"
-
-            files_list = SAMPLE_FILES.get(file_category, SAMPLE_FILES["lesson_plans"])
-
-            cabinet_id = str(uuid.uuid4())
-            cabinet_name = f"{class_code} — {class_info.get('name', class_code)}"
-
-            r = requests.post(
-                f"{SUPA_URL}/rest/v1/file_cabinets",
-                headers={**_sb_headers(), "Prefer": "return=representation"},
-                json={"id": cabinet_id, "user_id": teacher_pid, "name": cabinet_name},
-                params={"on_conflict": "id"},
-            )
-            if r.status_code in (200, 201):
-                print(f"  ✓ Cabinet: {cabinet_name}")
-                results["greenfield"]["cabinets"] += 1
-            else:
-                print(f"  ✗ Cabinet create failed ({class_code}): {r.text[:100]}")
-                errors.append(f"cabinet_create: {class_code}")
-                continue
-
-            for fi in files_list:
-                file_id = str(uuid.uuid4())
-                r = requests.post(
-                    f"{SUPA_URL}/rest/v1/files",
-                    headers={**_sb_headers(), "Prefer": "return=representation"},
-                    json={
-                        "id": file_id,
-                        "cabinet_id": cabinet_id,
-                        "name": fi["name"],
-                        "path": fi["path"],
-                        "bucket": "file-cabinets",
-                        "mime_type": fi.get("mime_type"),
-                        "size": fi.get("size"),
-                        "metadata": {},
-                    },
-                    params={"on_conflict": "id"},
-                )
-                if r.status_code in (200, 201):
-                    results["greenfield"]["files"] += 1
-
-                    artefact_id = str(uuid.uuid4())
-                    requests.post(
-                        f"{SUPA_URL}/rest/v1/document_artefacts",
-                        headers={**_sb_headers(), "Prefer": "return=representation"},
-                        json={
-                            "id": artefact_id,
-                            "file_id": file_id,
-                            "type": fi.get("mime_type", "application/octet-stream").split("/")[-1],
-                            "rel_path": fi["path"],
-                            "status": "processed",
-                            "extra": {"seeded": True, "source": "seed_file_cabinets"},
-                        },
-                        params={"on_conflict": "id"},
-                    )
-
-                time.sleep(0.05)
-
-            for sid in student_ids:
-                r = requests.post(
-                    f"{SUPA_URL}/rest/v1/cabinet_memberships",
-                    headers={**_sb_headers(), "Prefer": "return=minimal"},
-                    json={
-                        "cabinet_id": cabinet_id,
-                        "profile_id": sid,
-                        "role": "viewer",
-                    },
-                    params={"on_conflict": "cabinet_id,profile_id"},
-                )
-                if r.status_code in (200, 201, 409):
-                    results["greenfield"]["memberships"] += 1
-
-            time.sleep(0.1)
-
-        except Exception as e:
-            err = f"cabinet seed {class_code}: {e}"
-            print(f"  ✗ {err}")
-            errors.append(err)
-
-    # ── Summary ─────────────────────────────────────────────────────────────
-    print("\n" + "=" * 60)
-    results["success"] = len(errors) == 0
-    results["errors"] = errors
-    total_cabinets = results["kevlarai"]["cabinets"] + results["greenfield"]["cabinets"]
-    total_files = results["kevlarai"]["files"] + results["greenfield"]["files"]
-    total_memberships = results["kevlarai"]["memberships"] + results["greenfield"]["memberships"]
-    print(f"COMPLETE — {total_cabinets} cabinets, {total_files} files, {total_memberships} memberships")
-    if errors:
-        print(f"Errors ({len(errors)}):")
-        for e in errors:
-            print(f"  ✗ {e}")
-    print("=" * 60)
-    return results
-
-
-if __name__ == "__main__":
-    import json
-    print(json.dumps(seed(), indent=2, default=str))

run/initialization/seed_greenfield_timetable.py

@@ -1,493 +0,0 @@
-"""
-seed_greenfield_timetable.py — Full timetable + class + student seed for Greenfield Academy.
-
-Flow:
-   1. POST /timetable/setup              — academic year, 3 terms, periods → Supabase
-   2. POST /timetable/materialize-periods — academic_periods rows (days × template)
-   3. Create classes                     — 17 classes with correct metadata
-   4. Add teachers to classes            — primary teacher per class
-   5. POST /timetable/init + slots       — TeacherTimetable + slot assignments
-   6. Patch slot class_ids               — write class_id FK onto teacher_timetable_slots
-   7. Enroll students in classes         — student1→Yr9, student2→Yr10, student3→Yr11
-   8. POST /timetable/materialize        — taught_lessons with class_id populated
-   9. POST /timetable/sync-lessons       — Neo4j TaughtLesson nodes (B.10)
-
-Run inside ccapi container:
-    python3 -c "from run.initialization.seed_greenfield_timetable import seed; seed()"
-"""
-import os
-import time
-import requests
-from typing import Dict, Any, Optional, List
-
-from run.initialization.seed_environment import get_seed_password
-
-GREENFIELD_ADMIN_EMAIL = "[email protected]"
-
-
-def _runtime_context() -> Dict[str, str]:
-    return {
-        "supa_url": os.environ["SUPABASE_URL"],
-        "service_key": os.environ["SERVICE_ROLE_KEY"],
-        "api_base": os.environ.get("API_BASE_URL", "http://localhost:8000"),
-    }
-
-# ─── Period templates ──────────────────────────────────────────────────────────
-
-PERIODS = [
-    {"code": "REG", "name": "Registration", "start_time": "08:45", "end_time": "09:00", "period_type": "registration"},
-    {"code": "P1",  "name": "Period 1",     "start_time": "09:00", "end_time": "10:00", "period_type": "lesson"},
-    {"code": "P2",  "name": "Period 2",     "start_time": "10:00", "end_time": "11:00", "period_type": "lesson"},
-    {"code": "BRK", "name": "Break",        "start_time": "11:00", "end_time": "11:20", "period_type": "break"},
-    {"code": "P3",  "name": "Period 3",     "start_time": "11:20", "end_time": "12:20", "period_type": "lesson"},
-    {"code": "P4",  "name": "Period 4",     "start_time": "12:20", "end_time": "13:20", "period_type": "lesson"},
-    {"code": "LUN", "name": "Lunch",        "start_time": "13:20", "end_time": "14:00", "period_type": "break"},
-    {"code": "P5",  "name": "Period 5",     "start_time": "14:00", "end_time": "15:00", "period_type": "lesson"},
-]
-
-PERIOD_TIMES = {p["code"]: (p["start_time"], p["end_time"]) for p in PERIODS}
-
-# ─── Academic year ─────────────────────────────────────────────────────────────
-
-TERMS = [
-    {"name": "Autumn Term", "term_number": 1, "start_date": "2025-09-03", "end_date": "2025-12-19"},
-    {"name": "Spring Term", "term_number": 2, "start_date": "2026-01-05", "end_date": "2026-04-01"},
-    {"name": "Summer Term", "term_number": 3, "start_date": "2026-04-20", "end_date": "2026-07-17"},
-]
-
-# ─── Class definitions ─────────────────────────────────────────────────────────
-# Covers every unique subject_class code in TEACHER_SLOTS.
-# key_stage: KS3 = years 7-9, KS4 = years 10-11, KS5 = years 12-13
-
-CLASSES = [
-    # Physics
-    {"class_code": "9P/Ph1",  "name": "Year 9 Physics Group 1",   "subject": "Physics",     "year_group": "9",  "key_stage": "3", "teacher": "[email protected]"},
-    {"class_code": "10P/Ph2", "name": "Year 10 Physics Group 2",  "subject": "Physics",     "year_group": "10", "key_stage": "4", "teacher": "[email protected]"},
-    {"class_code": "11P/Ph1", "name": "Year 11 Physics Group 1",  "subject": "Physics",     "year_group": "11", "key_stage": "4", "teacher": "[email protected]"},
-    {"class_code": "12P/Ph1", "name": "Year 12 Physics Group 1",  "subject": "Physics",     "year_group": "12", "key_stage": "5", "teacher": "[email protected]"},
-    # Maths
-    {"class_code": "9M/Ma1",  "name": "Year 9 Maths Group 1",    "subject": "Mathematics", "year_group": "9",  "key_stage": "3", "teacher": "[email protected]"},
-    {"class_code": "10M/Ma1", "name": "Year 10 Maths Group 1",   "subject": "Mathematics", "year_group": "10", "key_stage": "4", "teacher": "[email protected]"},
-    {"class_code": "11M/Ma2", "name": "Year 11 Maths Group 2",   "subject": "Mathematics", "year_group": "11", "key_stage": "4", "teacher": "[email protected]"},
-    # English
-    {"class_code": "7En/1",   "name": "Year 7 English Group 1",  "subject": "English",     "year_group": "7",  "key_stage": "3", "teacher": "[email protected]"},
-    {"class_code": "8En/1",   "name": "Year 8 English Group 1",  "subject": "English",     "year_group": "8",  "key_stage": "3", "teacher": "[email protected]"},
-    {"class_code": "9En/1",   "name": "Year 9 English Group 1",  "subject": "English",     "year_group": "9",  "key_stage": "3", "teacher": "[email protected]"},
-    # History
-    {"class_code": "8Hs/1",   "name": "Year 8 History Group 1",  "subject": "History",     "year_group": "8",  "key_stage": "3", "teacher": "[email protected]"},
-    {"class_code": "9Hs/1",   "name": "Year 9 History Group 1",  "subject": "History",     "year_group": "9",  "key_stage": "3", "teacher": "[email protected]"},
-    {"class_code": "10Hs/1",  "name": "Year 10 History Group 1", "subject": "History",     "year_group": "10", "key_stage": "4", "teacher": "[email protected]"},
-    # Science
-    {"class_code": "7Sc/1",   "name": "Year 7 Science Group 1",  "subject": "Science",     "year_group": "7",  "key_stage": "3", "teacher": "[email protected]"},
-    {"class_code": "8Sc/1",   "name": "Year 8 Science Group 1",  "subject": "Science",     "year_group": "8",  "key_stage": "3", "teacher": "[email protected]"},
-    {"class_code": "9Sc/1",   "name": "Year 9 Science Group 1",  "subject": "Science",     "year_group": "9",  "key_stage": "3", "teacher": "[email protected]"},
-    {"class_code": "10Sc/1",  "name": "Year 10 Science Group 1", "subject": "Science",     "year_group": "10", "key_stage": "4", "teacher": "[email protected]"},
-]
-
-# ─── Teacher slot assignments ──────────────────────────────────────────────────
-
-TEACHER_SLOTS = {
-    "[email protected]": [
-        ("Monday",    "P1", "11P/Ph1"),
-        ("Monday",    "P3", "12P/Ph1"),
-        ("Tuesday",   "P2", "10P/Ph2"),
-        ("Tuesday",   "P4", "9P/Ph1"),
-        ("Wednesday", "P1", "11P/Ph1"),
-        ("Wednesday", "P5", "12P/Ph1"),
-        ("Thursday",  "P3", "10P/Ph2"),
-        ("Thursday",  "P5", "9P/Ph1"),
-        ("Friday",    "P2", "11P/Ph1"),
-        ("Friday",    "P4", "12P/Ph1"),
-    ],
-    "[email protected]": [
-        ("Monday",    "P2", "10M/Ma1"),
-        ("Monday",    "P4", "11M/Ma2"),
-        ("Tuesday",   "P1", "9M/Ma1"),
-        ("Tuesday",   "P3", "10M/Ma1"),
-        ("Wednesday", "P2", "11M/Ma2"),
-        ("Wednesday", "P4", "9M/Ma1"),
-        ("Thursday",  "P1", "10M/Ma1"),
-        ("Thursday",  "P4", "11M/Ma2"),
-        ("Friday",    "P1", "9M/Ma1"),
-        ("Friday",    "P3", "10M/Ma1"),
-    ],
-    "[email protected]": [
-        ("Monday",    "P1", "7En/1"),
-        ("Monday",    "P5", "8En/1"),
-        ("Tuesday",   "P2", "9En/1"),
-        ("Wednesday", "P3", "7En/1"),
-        ("Thursday",  "P2", "8En/1"),
-        ("Friday",    "P5", "9En/1"),
-    ],
-    "[email protected]": [
-        ("Monday",    "P3", "8Hs/1"),
-        ("Tuesday",   "P5", "9Hs/1"),
-        ("Wednesday", "P1", "10Hs/1"),
-        ("Thursday",  "P2", "8Hs/1"),
-        ("Friday",    "P3", "9Hs/1"),
-    ],
-    "[email protected]": [
-        ("Monday",    "P4", "7Sc/1"),
-        ("Tuesday",   "P3", "8Sc/1"),
-        ("Wednesday", "P5", "9Sc/1"),
-        ("Thursday",  "P4", "10Sc/1"),
-        ("Friday",    "P2", "7Sc/1"),
-    ],
-}
-
-# ─── Student enrollments ───────────────────────────────────────────────────────
-# One student per year-group band — enrolled in all subjects for that year.
-
-STUDENT_ENROLLMENTS = {
-    "[email protected]": ["9P/Ph1",  "9M/Ma1",  "9En/1", "9Hs/1",  "9Sc/1"],
-    "[email protected]": ["10P/Ph2", "10M/Ma1",          "10Hs/1", "10Sc/1"],
-    "[email protected]": ["11P/Ph1", "11M/Ma2"],
-}
-
-
-# ─── Helpers ───────────────────────────────────────────────────────────────────
-
-def _sb_headers() -> Dict:
-    service_key = _runtime_context()["service_key"]
-    return {
-        "apikey": service_key,
-        "Authorization": f"Bearer {service_key}",
-        "Content-Type": "application/json",
-        "Prefer": "return=representation",
-    }
-
-
-def _sign_in(email: str, password: str) -> str:
-    ctx = _runtime_context()
-    r = requests.post(
-        f"{ctx['supa_url']}/auth/v1/token?grant_type=password",
-        headers={"apikey": ctx["service_key"], "Content-Type": "application/json"},
-        json={"email": email, "password": password},
-    )
-    r.raise_for_status()
-    return r.json()["access_token"]
-
-
-def _api(token: str, method: str, path: str, body: Dict = None) -> Dict:
-    api_base = _runtime_context()["api_base"]
-    h = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
-    r = getattr(requests, method)(f"{api_base}{path}", headers=h, json=body)
-    try:
-        return r.json()
-    except Exception:
-        return {"_raw": r.text, "_status": r.status_code}
-
-
-def _get_profile_id(email: str) -> Optional[str]:
-    """Look up a profile's UUID by email via Supabase service role."""
-    supa_url = _runtime_context()["supa_url"]
-    r = requests.get(
-        f"{supa_url}/rest/v1/profiles",
-        headers=_sb_headers(),
-        params={"email": f"eq.{email}", "select": "id", "limit": "1"},
-    )
-    data = r.json() if r.ok else []
-    return data[0]["id"] if data else None
-
-
-def _get_teacher_timetable_id(profile_id: str) -> Optional[str]:
-    """Return the Supabase teacher_timetables.id for a given profile."""
-    supa_url = _runtime_context()["supa_url"]
-    r = requests.get(
-        f"{supa_url}/rest/v1/teacher_timetables",
-        headers=_sb_headers(),
-        params={"profile_id": f"eq.{profile_id}", "select": "id", "limit": "1"},
-    )
-    data = r.json() if r.ok else []
-    return data[0]["id"] if data else None
-
-
-def _patch_slot_class_ids(teacher_tt_sb_id: str, class_code_to_id: Dict[str, str]) -> int:
-    """Update class_id FK on teacher_timetable_slots rows via Supabase service role."""
-    supa_url = _runtime_context()["supa_url"]
-    patched = 0
-    for code, class_uuid in class_code_to_id.items():
-        r = requests.patch(
-            f"{supa_url}/rest/v1/teacher_timetable_slots",
-            headers=_sb_headers(),
-            params={
-                "teacher_timetable_id": f"eq.{teacher_tt_sb_id}",
-                "subject_class": f"eq.{code}",
-            },
-            json={"class_id": class_uuid},
-        )
-        if r.ok:
-            patched += 1
-    return patched
-
-
-# ─── Main seed ─────────────────────────────────────────────────────────────────
-
-def seed() -> Dict[str, Any]:
-    print("=" * 60)
-    print("Greenfield Academy — full timetable + class + student seed")
-    print("=" * 60)
-    results: Dict[str, Any] = {}
-    errors: List[str] = []
-
-    # ── [1] Sign in as Greenfield admin ───────────────────────────────────────
-    print("\n[1] Signing in as [email protected]...")
-    try:
-        admin_token = _sign_in(GREENFIELD_ADMIN_EMAIL, get_seed_password("school_admin"))
-        print("  ✓ signed in")
-    except Exception as e:
-        return {"success": False, "error": str(e)}
-
-    # ── [2] POST /timetable/setup ─────────────────────────────────────────────
-    print("\n[2] Setting up school timetable (academic year + terms + periods)...")
-    r = _api(admin_token, "post", "/timetable/setup", {
-        "year_start": "2025-09-03",
-        "year_end":   "2026-07-17",
-        "terms":   TERMS,
-        "periods": PERIODS,
-    })
-    if r.get("status") == "ok" or r.get("school_timetable_id") or r.get("timetable_id"):
-        print(f"  ✓ timetable: {r.get('school_timetable_id') or r.get('timetable_id')}")
-        results["setup"] = "ok"
-    else:
-        err = f"timetable/setup: {r}"
-        print(f"  ✗ {err}")
-        errors.append(err)
-        results["setup"] = "error"
-
-    # ── [3] POST /timetable/materialize-periods ───────────────────────────────
-    print("\n[3] Materializing academic_periods (days × periods_template)...")
-    r = _api(admin_token, "post", "/timetable/materialize-periods", None)
-    if r.get("status") == "ok":
-        print(f"  ✓ {r.get('created')} periods created across {r.get('academic_days')} academic days")
-        results["materialize_periods"] = "ok"
-    else:
-        err = f"materialize-periods: {r}"
-        print(f"  ✗ {err}")
-        errors.append(err)
-        results["materialize_periods"] = "error"
-
-    # ── [5] Build profile-ID lookup for all teachers + students ───────────────
-    print("\n[3] Resolving profile IDs for teachers and students...")
-    all_emails = (
-        list(TEACHER_SLOTS.keys())
-        + list(STUDENT_ENROLLMENTS.keys())
-    )
-    profile_ids: Dict[str, str] = {}
-    for email in all_emails:
-        pid = _get_profile_id(email)
-        if pid:
-            profile_ids[email] = pid
-            print(f"  ✓ {email} → {pid[:8]}…")
-        else:
-            print(f"  ✗ profile not found for {email}")
-            errors.append(f"profile_not_found: {email}")
-
-    # ── [6] Create classes ────────────────────────────────────────────────────
-    print(f"\n[4] Creating {len(CLASSES)} classes...")
-    results["classes"] = {}
-    class_code_to_id: Dict[str, str] = {}
-
-    for cls in CLASSES:
-        r = _api(admin_token, "post", "/database/timetable/classes", {
-            "name":          cls["name"],
-            "class_code":    cls["class_code"],
-            "subject":       cls["subject"],
-            "year_group":    cls["year_group"],
-            "key_stage":     cls["key_stage"],
-            "academic_year": "2025-2026",
-        })
-        class_id = r.get("id") or (r.get("class", {}) or {}).get("id")
-        if class_id:
-            class_code_to_id[cls["class_code"]] = class_id
-            results["classes"][cls["class_code"]] = "ok"
-            print(f"  ✓ {cls['class_code']} → {class_id[:8]}…")
-        else:
-            err = f"create class {cls['class_code']}: {r}"
-            print(f"  ✗ {err}")
-            errors.append(err)
-            results["classes"][cls["class_code"]] = "error"
-        time.sleep(0.1)
-
-    # ── [7] Add teachers to their classes ────────────────────────────────────
-    print("\n[5] Adding teachers to classes...")
-    results["class_teachers"] = {}
-    for cls in CLASSES:
-        class_id = class_code_to_id.get(cls["class_code"])
-        teacher_pid = profile_ids.get(cls["teacher"])
-        if not class_id or not teacher_pid:
-            results["class_teachers"][cls["class_code"]] = "skip"
-            continue
-        r = _api(admin_token, "post", f"/database/timetable/classes/{class_id}/teachers", {
-            "teacher_id": teacher_pid,
-            "is_primary": True,
-        })
-        if r.get("status") == "ok" or r.get("id"):
-            print(f"  ✓ {cls['teacher'].split('@')[0]} → {cls['class_code']}")
-            results["class_teachers"][cls["class_code"]] = "ok"
-        else:
-            err = f"add teacher {cls['class_code']}: {r}"
-            print(f"  ✗ {err}")
-            errors.append(err)
-            results["class_teachers"][cls["class_code"]] = "error"
-        time.sleep(0.1)
-
-    # ── [8] Teacher timetable init + slots ────────────────────────────────────
-    print("\n[6] Initialising TeacherTimetable and setting slots for each teacher...")
-    results["init"] = {}
-    results["slots"] = {}
-    teacher_tt_sb_ids: Dict[str, str] = {}  # email → teacher_timetables.id
-
-    for teacher_email, slot_tuples in TEACHER_SLOTS.items():
-        try:
-            teacher_token = _sign_in(teacher_email, get_seed_password("teacher"))
-        except Exception as e:
-            err = f"login {teacher_email}: {e}"
-            print(f"  ✗ {err}")
-            errors.append(err)
-            results["init"][teacher_email] = "error"
-            results["slots"][teacher_email] = "error"
-            continue
-
-        # 6a: init TeacherTimetable
-        r = _api(teacher_token, "post", "/timetable/init", None)
-        if r.get("status") == "ok":
-            print(f"  ✓ init {teacher_email}")
-            results["init"][teacher_email] = "ok"
-        else:
-            print(f"  ~ init {teacher_email}: {r.get('message', r)} (may already exist)")
-            results["init"][teacher_email] = "warn"
-
-        # 6b: get timetable_id (Neo4j uuid_string for slot FK)
-        status_r = _api(teacher_token, "get", "/timetable/status", None)
-        timetable_id = status_r.get("timetable_id")
-        if not timetable_id:
-            err = f"no timetable_id for {teacher_email}: {status_r}"
-            print(f"  ✗ {err}")
-            errors.append(err)
-            results["slots"][teacher_email] = "error"
-            continue
-
-        # 6c: save slots (subject_class text; class_id patched separately)
-        slot_list = [
-            {
-                "day_of_week":   day,
-                "period_code":   code,
-                "subject_class": cls,
-                "start_time":    PERIOD_TIMES[code][0],
-                "end_time":      PERIOD_TIMES[code][1],
-            }
-            for day, code, cls in slot_tuples
-        ]
-        r = _api(teacher_token, "post", "/timetable/slots", {
-            "timetable_id": timetable_id,
-            "slots": slot_list,
-        })
-        if r.get("status") == "ok" or r.get("created") is not None:
-            count = r.get("created") or len(slot_list)
-            print(f"  ✓ {teacher_email}: {count} slots")
-            results["slots"][teacher_email] = "ok"
-        else:
-            err = f"slots {teacher_email}: {r}"
-            print(f"  ✗ {err}")
-            errors.append(err)
-            results["slots"][teacher_email] = "error"
-
-        # record Supabase teacher_timetable FK for patching
-        teacher_pid = profile_ids.get(teacher_email)
-        if teacher_pid:
-            tt_sb_id = _get_teacher_timetable_id(teacher_pid)
-            if tt_sb_id:
-                teacher_tt_sb_ids[teacher_email] = tt_sb_id
-
-        time.sleep(0.3)
-
-    # ── [9] Patch teacher_timetable_slots.class_id ────────────────────────────
-    print("\n[7] Patching class_id onto teacher_timetable_slots...")
-    results["slot_patch"] = {}
-    for teacher_email, slot_tuples in TEACHER_SLOTS.items():
-        tt_sb_id = teacher_tt_sb_ids.get(teacher_email)
-        if not tt_sb_id:
-            results["slot_patch"][teacher_email] = "skip"
-            continue
-        teacher_codes = {cls for _, _, cls in slot_tuples}
-        relevant_map = {code: uid for code, uid in class_code_to_id.items() if code in teacher_codes}
-        n = _patch_slot_class_ids(tt_sb_id, relevant_map)
-        print(f"  ✓ {teacher_email}: {n} slots patched")
-        results["slot_patch"][teacher_email] = n
-
-    # ── [10] Enroll students in classes ───────────────────────────────────────
-    print("\n[8] Enrolling students in classes...")
-    results["enrollments"] = {}
-    for student_email, class_codes in STUDENT_ENROLLMENTS.items():
-        student_pid = profile_ids.get(student_email)
-        results["enrollments"][student_email] = {}
-        if not student_pid:
-            results["enrollments"][student_email] = "no_profile"
-            continue
-        for code in class_codes:
-            class_id = class_code_to_id.get(code)
-            if not class_id:
-                results["enrollments"][student_email][code] = "no_class"
-                continue
-            r = _api(admin_token, "post", f"/database/timetable/classes/{class_id}/students", {
-                "student_id": student_pid,
-            })
-            if r.get("status") == "ok" or r.get("id"):
-                print(f"  ✓ {student_email.split('@')[0]} → {code}")
-                results["enrollments"][student_email][code] = "ok"
-            else:
-                err = f"enroll {student_email} → {code}: {r}"
-                print(f"  ✗ {err}")
-                errors.append(err)
-                results["enrollments"][student_email][code] = "error"
-            time.sleep(0.1)
-
-    # ── [11] Materialize taught lessons ────────────────────────────────────────
-    print("\n[9] Materializing taught lessons for each teacher...")
-    results["materialize"] = {}
-    for teacher_email in TEACHER_SLOTS:
-        try:
-            teacher_token = _sign_in(teacher_email, get_seed_password("teacher"))
-        except Exception as e:
-            err = f"login {teacher_email}: {e}"
-            print(f"  ✗ {err}")
-            errors.append(err)
-            continue
-        r = _api(teacher_token, "post", "/timetable/materialize", None)
-        if r.get("status") == "ok":
-            print(f"  ✓ {teacher_email}: {r.get('lessons_upserted', '?')} lessons, "
-                  f"{r.get('whiteboard_rooms_created', '?')} rooms")
-            results["materialize"][teacher_email] = "ok"
-        else:
-            err = f"materialize {teacher_email}: {r}"
-            print(f"  ✗ {err}")
-            errors.append(err)
-            results["materialize"][teacher_email] = "error"
-        time.sleep(0.3)
-
-    # ── [12] Neo4j sync (B.10) ────────────────────────────────────────────────
-    print("\n[10] Syncing Neo4j TaughtLesson nodes (B.10)...")
-    r = _api(admin_token, "post", "/timetable/sync-lessons", None)
-    if r.get("status") == "ok":
-        print(f"  ✓ Neo4j sync: {r.get('taught_lessons')} lessons, "
-              f"{r.get('teacher_timetables')} timetables, {r.get('slots')} slots")
-        results["neo4j_sync"] = "ok"
-    else:
-        err = f"sync-lessons: {r}"
-        print(f"  ✗ {err}")
-        errors.append(err)
-        results["neo4j_sync"] = "error"
-
-    # ── Summary ───────────────────────────────────────────────────────────────
-    print("\n" + "=" * 60)
-    results["success"] = len(errors) == 0
-    results["errors"] = errors
-    if errors:
-        print(f"COMPLETE with {len(errors)} error(s):")
-        for e in errors:
-            print(f"  ✗ {e}")
-    else:
-        print("COMPLETE — all steps succeeded")
-    print("=" * 60)
-    return results

run/initialization/seed_kevlarai_timetable.py

@@ -1,456 +0,0 @@
-"""
-seed_kevlarai_timetable.py — Full timetable + class + student seed for KevlarAI school.
-
-Mirrors Greenfield's structure so both schools are testable.
-KevlarAI gets 8 classes across 3 subjects (Physics, Maths, Computer Science),
-2 teachers, and 2 students.
-
-Flow:
-   1. POST /timetable/setup              — academic year, 3 terms, periods → Supabase
-   2. POST /timetable/materialize-periods — academic_periods rows (days x template)
-   3. Create classes                     — 8 classes with correct metadata
-   4. Add teachers to classes            — primary teacher per class
-   5. POST /timetable/init + slots       — TeacherTimetable + slot assignments
-   6. Patch slot class_ids               — write class_id FK onto teacher_timetable_slots
-   7. Enroll students in classes         — student1→Yr10, student2→Yr11
-   8. POST /timetable/materialize        — taught_lessons with class_id populated
-   9. POST /timetable/sync-lessons       — Neo4j TaughtLesson nodes (B.10)
-
-Run inside ccapi container:
-    python3 -c "from run.initialization.seed_kevlarai_timetable import seed; seed()"
-"""
-import os
-import time
-import requests
-from typing import Dict, Any, Optional, List
-
-SUPA_URL   = os.environ["SUPABASE_URL"]
-SERVICE_KEY = os.environ["SERVICE_ROLE_KEY"]
-API_BASE   = os.environ.get("API_BASE_URL", "http://localhost:8000")
-
-KEVLARAI_ADMIN_EMAIL = "[email protected]"
-KEVLARAI_ADMIN_PWD   = "Admin@Cc2025!"
-PWD_TEACHER = "Teacher@Cc2025!"
-PWD_STUDENT = "Student@Cc2025!"
-
-# ─── Period templates (same as Greenfield) ─────────────────────────────────────
-
-PERIODS = [
-    {"code": "REG", "name": "Registration", "start_time": "08:45", "end_time": "09:00", "period_type": "registration"},
-    {"code": "P1",  "name": "Period 1",     "start_time": "09:00", "end_time": "10:00", "period_type": "lesson"},
-    {"code": "P2",  "name": "Period 2",     "start_time": "10:00", "end_time": "11:00", "period_type": "lesson"},
-    {"code": "BRK", "name": "Break",        "start_time": "11:00", "end_time": "11:20", "period_type": "break"},
-    {"code": "P3",  "name": "Period 3",     "start_time": "11:20", "end_time": "12:20", "period_type": "lesson"},
-    {"code": "P4",  "name": "Period 4",     "start_time": "12:20", "end_time": "13:20", "period_type": "lesson"},
-    {"code": "LUN", "name": "Lunch",        "start_time": "13:20", "end_time": "14:00", "period_type": "break"},
-    {"code": "P5",  "name": "Period 5",     "start_time": "14:00", "end_time": "15:00", "period_type": "lesson"},
-]
-
-PERIOD_TIMES = {p["code"]: (p["start_time"], p["end_time"]) for p in PERIODS}
-
-# ─── Academic year ─────────────────────────────────────────────────────────────
-
-TERMS = [
-    {"name": "Autumn Term", "term_number": 1, "start_date": "2025-09-03", "end_date": "2025-12-19"},
-    {"name": "Spring Term", "term_number": 2, "start_date": "2026-01-05", "end_date": "2026-04-01"},
-    {"name": "Summer Term", "term_number": 3, "start_date": "2026-04-20", "end_date": "2026-07-17"},
-]
-
-# ─── Class definitions ─────────────────────────────────────────────────────────
-# KevlarAI: 8 classes across Physics, Maths, Computer Science
-
-CLASSES = [
-    # Physics
-    {"class_code": "10K/Ph1",  "name": "Year 10 Physics Group 1",  "subject": "Physics",     "year_group": "10", "key_stage": "4", "teacher": "[email protected]"},
-    {"class_code": "11K/Ph1",  "name": "Year 11 Physics Group 1",  "subject": "Physics",     "year_group": "11", "key_stage": "4", "teacher": "[email protected]"},
-    # Maths
-    {"class_code": "10K/Ma1",  "name": "Year 10 Maths Group 1",    "subject": "Mathematics", "year_group": "10", "key_stage": "4", "teacher": "[email protected]"},
-    {"class_code": "11K/Ma1",  "name": "Year 11 Maths Group 1",    "subject": "Mathematics", "year_group": "11", "key_stage": "4", "teacher": "[email protected]"},
-    # Computer Science
-    {"class_code": "10K/CS1",  "name": "Year 10 CS Group 1",       "subject": "Computer Science", "year_group": "10", "key_stage": "4", "teacher": "[email protected]"},
-    {"class_code": "11K/CS1",  "name": "Year 11 CS Group 1",       "subject": "Computer Science", "year_group": "11", "key_stage": "4", "teacher": "[email protected]"},
-    # Additional KS3 classes for breadth
-    {"class_code": "9K/Ph1",   "name": "Year 9 Physics Group 1",   "subject": "Physics",     "year_group": "9",  "key_stage": "3", "teacher": "[email protected]"},
-    {"class_code": "9K/Ma1",   "name": "Year 9 Maths Group 1",     "subject": "Mathematics", "year_group": "9",  "key_stage": "3", "teacher": "[email protected]"},
-]
-
-# ─── Teacher slot assignments ──────────────────────────────────────────────────
-
-TEACHER_SLOTS = {
-    "[email protected]": [
-        ("Monday",    "P1", "10K/Ph1"),
-        ("Monday",    "P3", "11K/Ph1"),
-        ("Tuesday",   "P2", "10K/CS1"),
-        ("Tuesday",   "P4", "9K/Ph1"),
-        ("Wednesday", "P1", "11K/Ph1"),
-        ("Wednesday", "P5", "10K/Ph1"),
-        ("Thursday",  "P3", "10K/CS1"),
-        ("Thursday",  "P5", "9K/Ph1"),
-    ],
-    "[email protected]": [
-        ("Monday",    "P2", "10K/Ma1"),
-        ("Monday",    "P4", "11K/Ma1"),
-        ("Tuesday",   "P1", "9K/Ma1"),
-        ("Tuesday",   "P3", "10K/Ma1"),
-        ("Wednesday", "P2", "11K/Ma1"),
-        ("Wednesday", "P4", "9K/Ma1"),
-        ("Thursday",  "P1", "10K/Ma1"),
-        ("Thursday",  "P4", "11K/Ma1"),
-        ("Friday",    "P1", "9K/Ma1"),
-        ("Friday",    "P3", "10K/Ma1"),
-    ],
-}
-
-# ─── Student enrollments ───────────────────────────────────────────────────────
-
-STUDENT_ENROLLMENTS = {
-    "[email protected]": ["10K/Ph1", "10K/Ma1", "10K/CS1"],
-    "[email protected]": ["11K/Ph1", "11K/Ma1"],
-}
-
-
-# ─── Helpers ───────────────────────────────────────────────────────────────────
-
-def _sb_headers() -> Dict:
-    return {
-        "apikey": SERVICE_KEY,
-        "Authorization": f"Bearer {SERVICE_KEY}",
-        "Content-Type": "application/json",
-        "Prefer": "return=representation",
-    }
-
-
-def _sign_in(email: str, password: str) -> str:
-    r = requests.post(
-        f"{SUPA_URL}/auth/v1/token?grant_type=password",
-        headers={"apikey": SERVICE_KEY, "Content-Type": "application/json"},
-        json={"email": email, "password": password},
-    )
-    r.raise_for_status()
-    return r.json()["access_token"]
-
-
-def _api(token: str, method: str, path: str, body: Optional[Dict] = None) -> Dict:
-    h = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
-    r = getattr(requests, method)(f"{API_BASE}{path}", headers=h, json=body)
-    try:
-        return r.json()
-    except Exception:
-        return {"_raw": r.text, "_status": r.status_code}
-
-
-def _get_profile_id(email: str) -> Optional[str]:
-    """Look up a profile's UUID by email via Supabase service role."""
-    r = requests.get(
-        f"{SUPA_URL}/rest/v1/profiles",
-        headers=_sb_headers(),
-        params={"email": f"eq.{email}", "select": "id", "limit": "1"},
-    )
-    data = r.json() if r.ok else []
-    return data[0]["id"] if data else None
-
-
-def _get_teacher_timetable_id(profile_id: str) -> Optional[str]:
-    """Return the Supabase teacher_timetables.id for a given profile."""
-    r = requests.get(
-        f"{SUPA_URL}/rest/v1/teacher_timetables",
-        headers=_sb_headers(),
-        params={"profile_id": f"eq.{profile_id}", "select": "id", "limit": "1"},
-    )
-    data = r.json() if r.ok else []
-    return data[0]["id"] if data else None
-
-
-def _patch_slot_class_ids(teacher_tt_sb_id: str, class_code_to_id: Dict[str, str]) -> int:
-    """Update class_id FK on teacher_timetable_slots rows via Supabase service role."""
-    patched = 0
-    for code, class_uuid in class_code_to_id.items():
-        r = requests.patch(
-            f"{SUPA_URL}/rest/v1/teacher_timetable_slots",
-            headers=_sb_headers(),
-            params={
-                "teacher_timetable_id": f"eq.{teacher_tt_sb_id}",
-                "subject_class": f"eq.{code}",
-            },
-            json={"class_id": class_uuid},
-        )
-        if r.ok:
-            patched += 1
-    return patched
-
-
-# ─── Main seed ─────────────────────────────────────────────────────────────────
-
-def seed() -> Dict[str, Any]:
-    print("=" * 60)
-    print("KevlarAI — full timetable + class + student seed")
-    print("=" * 60)
-    results: Dict[str, Any] = {}
-    errors: List[str] = []
-
-    # ── [1] Sign in as KevlarAI admin ───────────────────────────────────────
-    print("\n[1] Signing in as [email protected]...")
-    try:
-        admin_token = _sign_in(KEVLARAI_ADMIN_EMAIL, KEVLARAI_ADMIN_PWD)
-        print("  ✓ signed in")
-    except Exception as e:
-        return {"success": False, "error": str(e)}
-
-    # ── [2] POST /timetable/setup ─────────────────────────────────────────────
-    print("\n[2] Setting up school timetable (academic year + terms + periods)...")
-    r = _api(admin_token, "post", "/timetable/setup", {
-        "year_start": "2025-09-03",
-        "year_end":   "2026-07-17",
-        "terms":   TERMS,
-        "periods": PERIODS,
-    })
-    if r.get("status") == "ok" or r.get("school_timetable_id") or r.get("timetable_id"):
-        print(f"  ✓ timetable: {r.get('school_timetable_id') or r.get('timetable_id')}")
-        results["setup"] = "ok"
-    else:
-        err = f"timetable/setup: {r}"
-        print(f"  ✗ {err}")
-        errors.append(err)
-        results["setup"] = "error"
-
-    # ── [3] POST /timetable/materialize-periods ───────────────────────────────
-    print("\n[3] Materializing academic_periods (days x periods_template)...")
-    r = _api(admin_token, "post", "/timetable/materialize-periods", None)
-    if r.get("status") == "ok":
-        print(f"  ✓ {r.get('created')} periods created across {r.get('academic_days')} academic days")
-        results["materialize_periods"] = "ok"
-    else:
-        err = f"materialize-periods: {r}"
-        print(f"  ✗ {err}")
-        errors.append(err)
-        results["materialize_periods"] = "error"
-
-    # ── [5] Build profile-ID lookup for all teachers + students ───────────────
-    print("\n[3] Resolving profile IDs for teachers and students...")
-    all_emails = (
-        list(TEACHER_SLOTS.keys())
-        + list(STUDENT_ENROLLMENTS.keys())
-    )
-    profile_ids: Dict[str, str] = {}
-    for email in all_emails:
-        pid = _get_profile_id(email)
-        if pid:
-            profile_ids[email] = pid
-            print(f"  ✓ {email} -> {pid[:8]}...")
-        else:
-            print(f"  ✗ profile not found for {email}")
-            errors.append(f"profile_not_found: {email}")
-
-    # ── [6] Create classes ────────────────────────────────────────────────────
-    print(f"\n[4] Creating {len(CLASSES)} classes...")
-    results["classes"] = {}
-    class_code_to_id: Dict[str, str] = {}
-
-    for cls in CLASSES:
-        r = _api(admin_token, "post", "/database/timetable/classes", {
-            "name":          cls["name"],
-            "class_code":    cls["class_code"],
-            "subject":       cls["subject"],
-            "year_group":    cls["year_group"],
-            "key_stage":     cls["key_stage"],
-            "academic_year": "2025-2026",
-        })
-        class_id = r.get("id") or (r.get("class", {}) or {}).get("id")
-        if class_id:
-            class_code_to_id[cls["class_code"]] = class_id
-            results["classes"][cls["class_code"]] = "ok"
-            print(f"  ✓ {cls['class_code']} -> {class_id[:8]}...")
-        else:
-            err = f"create class {cls['class_code']}: {r}"
-            print(f"  ✗ {err}")
-            errors.append(err)
-            results["classes"][cls["class_code"]] = "error"
-        time.sleep(0.1)
-
-    # ── [7] Add teachers to their classes ────────────────────────────────────
-    print("\n[5] Adding teachers to classes...")
-    results["class_teachers"] = {}
-    for cls in CLASSES:
-        class_id = class_code_to_id.get(cls["class_code"])
-        teacher_pid = profile_ids.get(cls["teacher"])
-        if not class_id or not teacher_pid:
-            results["class_teachers"][cls["class_code"]] = "skip"
-            continue
-        r = _api(admin_token, "post", f"/database/timetable/classes/{class_id}/teachers", {
-            "teacher_id": teacher_pid,
-            "is_primary": True,
-        })
-        if r.get("status") == "ok" or r.get("id"):
-            print(f"  ✓ {cls['teacher'].split('@')[0]} -> {cls['class_code']}")
-            results["class_teachers"][cls["class_code"]] = "ok"
-        else:
-            err = f"add teacher {cls['class_code']}: {r}"
-            print(f"  ✗ {err}")
-            errors.append(err)
-            results["class_teachers"][cls["class_code"]] = "error"
-        time.sleep(0.1)
-
-    # ── [8] Teacher timetable init + slots ────────────────────────────────────
-    print("\n[6] Initialising TeacherTimetable and setting slots for each teacher...")
-    results["init"] = {}
-    results["slots"] = {}
-    teacher_tt_sb_ids: Dict[str, str] = {}  # email -> teacher_timetables.id
-
-    for teacher_email, slot_tuples in TEACHER_SLOTS.items():
-        try:
-            teacher_token = _sign_in(teacher_email, PWD_TEACHER)
-        except Exception as e:
-            err = f"login {teacher_email}: {e}"
-            print(f"  ✗ {err}")
-            errors.append(err)
-            results["init"][teacher_email] = "error"
-            results["slots"][teacher_email] = "error"
-            continue
-
-        # 6a: init TeacherTimetable
-        r = _api(teacher_token, "post", "/timetable/init", None)
-        if r.get("status") == "ok":
-            print(f"  ✓ init {teacher_email}")
-            results["init"][teacher_email] = "ok"
-        else:
-            print(f"  ~ init {teacher_email}: {r.get('message', r)} (may already exist)")
-            results["init"][teacher_email] = "warn"
-
-        # 6b: get timetable_id (Neo4j uuid_string for slot FK)
-        status_r = _api(teacher_token, "get", "/timetable/status", None)
-        timetable_id = status_r.get("timetable_id")
-        if not timetable_id:
-            err = f"no timetable_id for {teacher_email}: {status_r}"
-            print(f"  ✗ {err}")
-            errors.append(err)
-            results["slots"][teacher_email] = "error"
-            continue
-
-        # 6c: save slots (subject_class text; class_id patched separately)
-        slot_list = [
-            {
-                "day_of_week":   day,
-                "period_code":   code,
-                "subject_class": cls,
-                "start_time":    PERIOD_TIMES[code][0],
-                "end_time":      PERIOD_TIMES[code][1],
-            }
-            for day, code, cls in slot_tuples
-        ]
-        r = _api(teacher_token, "post", "/timetable/slots", {
-            "timetable_id": timetable_id,
-            "slots": slot_list,
-        })
-        if r.get("status") == "ok" or r.get("created") is not None:
-            count = r.get("created") or len(slot_list)
-            print(f"  ✓ {teacher_email}: {count} slots")
-            results["slots"][teacher_email] = "ok"
-        else:
-            err = f"slots {teacher_email}: {r}"
-            print(f"  ✗ {err}")
-            errors.append(err)
-            results["slots"][teacher_email] = "error"
-
-        # record Supabase teacher_timetable FK for patching
-        teacher_pid = profile_ids.get(teacher_email)
-        if teacher_pid:
-            tt_sb_id = _get_teacher_timetable_id(teacher_pid)
-            if tt_sb_id:
-                teacher_tt_sb_ids[teacher_email] = tt_sb_id
-
-        time.sleep(0.3)
-
-    # ── [9] Patch teacher_timetable_slots.class_id ────────────────────────────
-    print("\n[7] Patching class_id onto teacher_timetable_slots...")
-    results["slot_patch"] = {}
-    for teacher_email, slot_tuples in TEACHER_SLOTS.items():
-        tt_sb_id = teacher_tt_sb_ids.get(teacher_email)
-        if not tt_sb_id:
-            results["slot_patch"][teacher_email] = "skip"
-            continue
-        teacher_codes = {cls for _, _, cls in slot_tuples}
-        relevant_map = {code: uid for code, uid in class_code_to_id.items() if code in teacher_codes}
-        n = _patch_slot_class_ids(tt_sb_id, relevant_map)
-        print(f"  ✓ {teacher_email}: {n} slots patched")
-        results["slot_patch"][teacher_email] = n
-
-    # ── [10] Enroll students in classes ───────────────────────────────────────
-    print("\n[8] Enrolling students in classes...")
-    results["enrollments"] = {}
-    for student_email, class_codes in STUDENT_ENROLLMENTS.items():
-        student_pid = profile_ids.get(student_email)
-        results["enrollments"][student_email] = {}
-        if not student_pid:
-            results["enrollments"][student_email] = "no_profile"
-            continue
-        for code in class_codes:
-            class_id = class_code_to_id.get(code)
-            if not class_id:
-                results["enrollments"][student_email][code] = "no_class"
-                continue
-            r = _api(admin_token, "post", f"/database/timetable/classes/{class_id}/students", {
-                "student_id": student_pid,
-            })
-            if r.get("status") == "ok" or r.get("id"):
-                print(f"  ✓ {student_email.split('@')[0]} -> {code}")
-                results["enrollments"][student_email][code] = "ok"
-            else:
-                err = f"enroll {student_email} -> {code}: {r}"
-                print(f"  ✗ {err}")
-                errors.append(err)
-                results["enrollments"][student_email][code] = "error"
-            time.sleep(0.1)
-
-    # ── [11] Materialize taught lessons ────────────────────────────────────────
-    print("\n[9] Materializing taught lessons for each teacher...")
-    results["materialize"] = {}
-    for teacher_email in TEACHER_SLOTS:
-        try:
-            teacher_token = _sign_in(teacher_email, PWD_TEACHER)
-        except Exception as e:
-            err = f"login {teacher_email}: {e}"
-            print(f"  ✗ {err}")
-            errors.append(err)
-            continue
-        r = _api(teacher_token, "post", "/timetable/materialize", None)
-        if r.get("status") == "ok":
-            print(f"  ✓ {teacher_email}: {r.get('lessons_upserted', '?')} lessons, "
-                  f"{r.get('whiteboard_rooms_created', '?')} rooms")
-            results["materialize"][teacher_email] = "ok"
-        else:
-            err = f"materialize {teacher_email}: {r}"
-            print(f"  ✗ {err}")
-            errors.append(err)
-            results["materialize"][teacher_email] = "error"
-        time.sleep(0.3)
-
-    # ── [12] Neo4j sync (B.10) ────────────────────────────────────────────────
-    print("\n[10] Syncing Neo4j TaughtLesson nodes (B.10)...")
-    r = _api(admin_token, "post", "/timetable/sync-lessons", None)
-    if r.get("status") == "ok":
-        print(f"  ✓ Neo4j sync: {r.get('taught_lessons')} lessons, "
-              f"{r.get('teacher_timetables')} timetables, {r.get('slots')} slots")
-        results["neo4j_sync"] = "ok"
-    else:
-        err = f"sync-lessons: {r}"
-        print(f"  ✗ {err}")
-        errors.append(err)
-        results["neo4j_sync"] = "error"
-
-    # ── Summary ───────────────────────────────────────────────────────────────
-    print("\n" + "=" * 60)
-    results["success"] = len(errors) == 0
-    results["errors"] = errors
-    if errors:
-        print(f"COMPLETE with {len(errors)} error(s):")
-        for e in errors:
-            print(f"  ✗ {e}")
-    else:
-        print("COMPLETE — all steps succeeded")
-    print("=" * 60)
-    return results
-
-
-if __name__ == "__main__":
-    import json
-    print(json.dumps(seed(), indent=2, default=str))

run/initialization/seed_planned_lessons.py

@@ -1,385 +0,0 @@
-"""
-seed_planned_lessons.py — Create 2-3 planned lessons per teacher across both schools.
-
-Uses the /lessons/plans API endpoint (POST) to create lesson plans.
-Each plan is linked to a class, subject, and year group where possible.
-Plans are idempotent: checks for existing plans by title+subject before creating.
-
-Tables: planned_lessons, lesson_collaborators, lesson_deliveries
-
-Run inside ccapi container:
-    python3 -c "from run.initialization.seed_planned_lessons import seed; seed()"
-"""
-import os
-import time
-import requests
-from typing import Dict, Any, List, Optional
-
-SUPA_URL   = os.environ["SUPABASE_URL"]
-SERVICE_KEY = os.environ["SERVICE_ROLE_KEY"]
-API_BASE   = os.environ.get("API_BASE_URL", "http://localhost:8000")
-
-# ─── Passwords (standardized from T4) ────────────────────────────────────────
-
-PWD_ADMIN   = "Admin@Cc2025!"
-PWD_TEACHER = "Teacher@Cc2025!"
-
-# ─── Planned lesson templates per school ──────────────────────────────────────
-# Each entry: (teacher_email, title, subject, year_group, class_code, objectives, activities)
-
-KEVLARAI_PLANS = [
-    {
-        "teacher": "[email protected]",
-        "title": "Introduction to Forces and Motion",
-        "subject": "Physics",
-        "year_group": "10",
-        "class_code": "10K/Ph1",
-        "objectives": [
-            {"text": "Define force, mass, and acceleration", "bloom": "remember"},
-            {"text": "Apply F=ma to solve simple problems", "bloom": "apply"},
-        ],
-        "activities": [
-            {"type": "demo", "description": "Demonstrate forces with spring scales"},
-            {"type": "worksheet", "description": "F=ma calculation practice (10 problems)"},
-        ],
-        "duration_minutes": 60,
-    },
-    {
-        "teacher": "[email protected]",
-        "title": "Electric Circuits Basics",
-        "subject": "Physics",
-        "year_group": "11",
-        "class_code": "11K/Ph1",
-        "objectives": [
-            {"text": "Identify series and parallel circuit components", "bloom": "understand"},
-            {"text": "Calculate total resistance in series circuits", "bloom": "apply"},
-        ],
-        "activities": [
-            {"type": "lab", "description": "Build series circuit with resistors"},
-            {"type": "quiz", "description": "Resistance calculation quiz (5 questions)"},
-        ],
-        "duration_minutes": 60,
-    },
-    {
-        "teacher": "[email protected]",
-        "title": "Quadratic Equations — Factorisation Method",
-        "subject": "Mathematics",
-        "year_group": "10",
-        "class_code": "10K/Ma1",
-        "objectives": [
-            {"text": "Factorise quadratic expressions of the form x²+bx+c", "bloom": "apply"},
-            {"text": "Solve quadratic equations by factorisation", "bloom": "analyse"},
-        ],
-        "activities": [
-            {"type": "direct_instruction", "description": "Walk through 3 worked examples"},
-            {"type": "pair_work", "description": "Factorise 8 quadratics with a partner"},
-        ],
-        "duration_minutes": 60,
-    },
-    {
-        "teacher": "[email protected]",
-        "title": "Probability — Tree Diagrams",
-        "subject": "Mathematics",
-        "year_group": "11",
-        "class_code": "11K/Ma1",
-        "objectives": [
-            {"text": "Construct tree diagrams for two-stage events", "bloom": "apply"},
-            {"text": "Calculate combined probabilities from tree diagrams", "bloom": "analyse"},
-        ],
-        "activities": [
-            {"type": "demo", "description": "Coin toss tree diagram on whiteboard"},
-            {"type": "worksheet", "description": "5 tree diagram probability problems"},
-        ],
-        "duration_minutes": 60,
-    },
-]
-
-GREENFIELD_PLANS = [
-    {
-        "teacher": "[email protected]",
-        "title": "Waves and Sound",
-        "subject": "Physics",
-        "year_group": "9",
-        "class_code": "9P/Ph1",
-        "objectives": [
-            {"text": "Describe properties of transverse and longitudinal waves", "bloom": "remember"},
-            {"text": "Calculate wave speed using v=fλ", "bloom": "apply"},
-        ],
-        "activities": [
-            {"type": "demo", "description": "Slinky wave demonstrations"},
-            {"type": "worksheet", "description": "Wave speed calculations (8 problems)"},
-        ],
-        "duration_minutes": 60,
-    },
-    {
-        "teacher": "[email protected]",
-        "title": "Energy Transfers and Conservation",
-        "subject": "Physics",
-        "year_group": "10",
-        "class_code": "10P/Ph2",
-        "objectives": [
-            {"text": "Identify energy stores and transfer pathways", "bloom": "understand"},
-            {"text": "Apply conservation of energy to real-world scenarios", "bloom": "analyse"},
-        ],
-        "activities": [
-            {"type": "group_work", "description": "Energy audit of a household"},
-            {"type": "presentation", "description": "Present findings on energy efficiency"},
-        ],
-        "duration_minutes": 60,
-    },
-    {
-        "teacher": "[email protected]",
-        "title": "Algebra — Expanding Brackets",
-        "subject": "Mathematics",
-        "year_group": "9",
-        "class_code": "9M/Ma1",
-        "objectives": [
-            {"text": "Expand single brackets: a(b+c)", "bloom": "apply"},
-            {"text": "Expand double brackets: (a+b)(c+d)", "bloom": "analyse"},
-        ],
-        "activities": [
-            {"type": "direct_instruction", "description": "Area model for expanding brackets"},
-            {"type": "worksheet", "description": "15 expansion problems (graded difficulty)"},
-        ],
-        "duration_minutes": 60,
-    },
-    {
-        "teacher": "[email protected]",
-        "title": "Simultaneous Equations — Elimination Method",
-        "subject": "Mathematics",
-        "year_group": "10",
-        "class_code": "10M/Ma1",
-        "objectives": [
-            {"text": "Solve simultaneous equations by elimination", "bloom": "apply"},
-            {"text": "Choose between substitution and elimination strategically", "bloom": "evaluate"},
-        ],
-        "activities": [
-            {"type": "direct_instruction", "description": "Walk through 3 elimination examples"},
-            {"type": "pair_work", "description": "Solve 6 simultaneous equation pairs"},
-        ],
-        "duration_minutes": 60,
-    },
-    {
-        "teacher": "[email protected]",
-        "title": "Shakespeare — Macbeth Act 1 Analysis",
-        "subject": "English",
-        "year_group": "9",
-        "class_code": "9En/1",
-        "objectives": [
-            {"text": "Identify key themes in Act 1", "bloom": "understand"},
-            {"text": "Analyse Shakespeare's use of imagery and language", "bloom": "analyse"},
-        ],
-        "activities": [
-            {"type": "close_reading", "description": "Close read Act 1, Scene 3 (witches' prophecy)"},
-            {"type": "essay", "description": "Short paragraph: How does Shakespeare create tension?"},
-        ],
-        "duration_minutes": 60,
-    },
-    {
-        "teacher": "[email protected]",
-        "title": "The Tudors — Henry VIII's Reforms",
-        "subject": "History",
-        "year_group": "10",
-        "class_code": "10Hs/1",
-        "objectives": [
-            {"text": "Describe Henry VIII's religious reforms", "bloom": "remember"},
-            {"text": "Evaluate the political motivations behind the reforms", "bloom": "evaluate"},
-        ],
-        "activities": [
-            {"type": "source_analysis", "description": "Analyze Act of Supremacy 1534"},
-            {"type": "debate", "description": "Was Henry's break with Rome politically necessary?"},
-        ],
-        "duration_minutes": 60,
-    },
-]
-
-
-# ─── Helpers ───────────────────────────────────────────────────────────────────
-
-def _sign_in(email: str, password: str) -> str:
-    r = requests.post(
-        f"{SUPA_URL}/auth/v1/token?grant_type=password",
-        headers={"apikey": SERVICE_KEY, "Content-Type": "application/json"},
-        json={"email": email, "password": password},
-    )
-    r.raise_for_status()
-    return r.json()["access_token"]
-
-
-def _api(token: str, method: str, path: str, body: Optional[Dict] = None) -> Dict:
-    h = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
-    r = getattr(requests, method)(f"{API_BASE}{path}", headers=h, json=body)
-    try:
-        return r.json()
-    except Exception:
-        return {"_raw": r.text, "_status": r.status_code}
-
-
-def _get_profile_id(email: str) -> Optional[str]:
-    """Look up a profile's UUID by email via Supabase service role."""
-    r = requests.get(
-        f"{SUPA_URL}/rest/v1/profiles",
-        headers={
-            "apikey": SERVICE_KEY,
-            "Authorization": f"Bearer {SERVICE_KEY}",
-            "Content-Type": "application/json",
-        },
-        params={"email": f"eq.{email}", "select": "id", "limit": "1"},
-    )
-    data = r.json() if r.ok else []
-    return data[0]["id"] if data else None
-
-
-def _get_class_id(class_code: str, admin_token: str) -> Optional[str]:
-    """Look up a class UUID by class_code via the API."""
-    r = requests.get(
-        f"{API_BASE}/database/timetable/classes",
-        headers={"Authorization": f"Bearer {admin_token}"},
-        params={"class_code": class_code},
-    )
-    data = r.json() if r.ok else []
-    if isinstance(data, list) and data:
-        return data[0].get("id") or data[0]
-    if isinstance(data, dict):
-        return data.get("id") or data.get("class", {}).get("id")
-    return None
-
-
-def _existing_plans_for_teacher(token: str, teacher_email: str) -> List[str]:
-    """Return list of existing plan titles for a teacher (to check idempotency)."""
-    r = requests.get(
-        f"{API_BASE}/lessons/plans",
-        headers={"Authorization": f"Bearer {token}"},
-    )
-    if r.ok:
-        plans = r.json().get("plans", [])
-        return [p.get("title", "") for p in plans]
-    return []
-
-
-# ─── Main seed ─────────────────────────────────────────────────────────────────
-
-def seed() -> Dict[str, Any]:
-    print("=" * 60)
-    print("Planned lessons seed — both schools")
-    print("=" * 60)
-    results: Dict[str, Any] = {}
-    errors: List[str] = []
-
-    # ── Sign in as both school admins ───────────────────────────────────────
-    print("\n[1] Signing in as school admins...")
-    admin_tokens = {}
-    for school, email, pwd in [
-        ("KevlarAI", "[email protected]", PWD_ADMIN),
-        ("Greenfield", "[email protected]", PWD_ADMIN),
-    ]:
-        try:
-            token = _sign_in(email, pwd)
-            admin_tokens[school] = token
-            print(f"  ✓ {school} admin signed in")
-        except Exception as e:
-            print(f"  ✗ {school} admin login failed: {e}")
-            errors.append(f"{school}_admin_login: {e}")
-
-    if not admin_tokens:
-        return {"success": False, "error": "No admin tokens obtained"}
-
-    # ── Resolve class IDs ───────────────────────────────────────────────────
-    print("\n[2] Resolving class IDs...")
-    all_class_codes = set()
-    for plans in [KEVLARAI_PLANS, GREENFIELD_PLANS]:
-        for p in plans:
-            if p.get("class_code"):
-                all_class_codes.add(p["class_code"])
-
-    class_code_to_id: Dict[str, str] = {}
-    for code in all_class_codes:
-        # Try KevlarAI first, then Greenfield
-        for school in ["KevlarAI", "Greenfield"]:
-            cid = _get_class_id(code, admin_tokens[school])
-            if cid:
-                class_code_to_id[code] = cid
-                print(f"  ✓ {code} -> {cid[:8]}...")
-                break
-        else:
-            print(f"  ✗ class not found: {code}")
-            errors.append(f"class_not_found: {code}")
-
-    # ── Seed planned lessons ────────────────────────────────────────────────
-    print("\n[3] Creating planned lessons...")
-    created_count = 0
-    skipped_count = 0
-
-    for school, plans in [("KevlarAI", KEVLARAI_PLANS), ("Greenfield", GREENFIELD_PLANS)]:
-        admin_token = admin_tokens[school]
-        print(f"\n  [{school}]")
-
-        for plan_spec in plans:
-            teacher_email = plan_spec["teacher"]
-            title = plan_spec["title"]
-
-            # Check idempotency
-            try:
-                teacher_token = _sign_in(teacher_email, PWD_TEACHER)
-            except Exception as e:
-                err = f"login {teacher_email}: {e}"
-                print(f"    ✗ {err}")
-                errors.append(err)
-                continue
-
-            existing = _existing_plans_for_teacher(teacher_token, teacher_email)
-            if title in existing:
-                print(f"    ~ SKIP (exists): {title}")
-                skipped_count += 1
-                continue
-
-            # Build class_id
-            class_id = None
-            cc = plan_spec.get("class_code")
-            if cc:
-                class_id = class_code_to_id.get(cc)
-
-            body = {
-                "title": title,
-                "subject": plan_spec["subject"],
-                "year_group": plan_spec["year_group"],
-                "estimated_duration_minutes": plan_spec.get("duration_minutes", 60),
-                "objectives": plan_spec["objectives"],
-                "activities": plan_spec["activities"],
-                "status": "draft",
-                "tags": [plan_spec["subject"].lower(), f"yr{plan_spec['year_group']}"],
-            }
-            if class_id:
-                body["class_id"] = class_id
-
-            r = _api(teacher_token, "post", "/lessons/plans", body)
-            plan_id = r.get("id") or (r.get("planned_lesson", {}) or {}).get("id")
-            if plan_id:
-                print(f"    ✓ {title} [{plan_id[:8]}...]")
-                created_count += 1
-            else:
-                err = f"create plan '{title}': {r}"
-                print(f"    ✗ {err}")
-                errors.append(err)
-
-            time.sleep(0.2)
-
-    # ── Summary ─────────────────────────────────────────────────────────────
-    print("\n" + "=" * 60)
-    results["success"] = len(errors) == 0
-    results["errors"] = errors
-    results["created"] = created_count
-    results["skipped"] = skipped_count
-    if errors:
-        print(f"COMPLETE with {len(errors)} error(s):")
-        for e in errors:
-            print(f"  ✗ {e}")
-    else:
-        print(f"COMPLETE — {created_count} created, {skipped_count} skipped (idempotent)")
-    print("=" * 60)
-    return results
-
-
-if __name__ == "__main__":
-    import json
-    print(json.dumps(seed(), indent=2, default=str))

run/initialization/seed_test_environment.py

@@ -1,15 +0,0 @@
-"""Compatibility wrapper for the canonical seed environment test mode."""
-from typing import Any, Dict
-
-from run.initialization.seed_environment import seed
-
-
-def seed_test_environment() -> Dict[str, Any]:
-    """Seed the lightweight 9-user test environment."""
-    return seed(test=True)
-
-
-if __name__ == "__main__":
-    import json
-
-    print(json.dumps(seed_test_environment(), indent=2, default=str))

run/initialization/sync_users_neo4j.py

@@ -1,238 +0,0 @@
-"""Sync Supabase profile users into the central Neo4j cc.users database.
-
-This script is intentionally idempotent. It defaults to --dry-run so it can be
-used safely during diagnostics. Use --apply to write/update Neo4j nodes.
-"""
-
-from __future__ import annotations
-
-import argparse
-import os
-import sys
-from pathlib import Path
-from dataclasses import dataclass
-from typing import Any
-
-SCRIPT_DIR = str(Path(__file__).resolve().parent)
-if SCRIPT_DIR in sys.path:
-    sys.path.remove(SCRIPT_DIR)
-
-import requests
-from dotenv import load_dotenv
-from neo4j import GraphDatabase
-
-
-@dataclass(frozen=True)
-class UserRecord:
-    uuid_string: str
-    user_email: str
-    cc_username: str
-    user_type: str
-    user_name: str
-    display_name: str | None
-    institute_id: str | None
-    institute_name: str | None
-    institute_role: str | None
-    institute_neo4j_uuid_string: str | None
-    user_db_name: str
-    institute_db_name: str | None
-    node_storage_path: str
-
-
-def load_environment(env_file: str | None) -> None:
-    if env_file:
-        load_dotenv(env_file, override=True)
-    else:
-        load_dotenv(override=False)
-
-
-def supabase_headers() -> dict[str, str]:
-    key = os.getenv('SERVICE_ROLE_KEY') or os.getenv('ANON_KEY')
-    if not key:
-        raise RuntimeError('SERVICE_ROLE_KEY or ANON_KEY is required')
-    return {'apikey': key, 'Authorization': f'Bearer {key}'}
-
-
-def fetch_table(table: str, select: str) -> list[dict[str, Any]]:
-    base = os.getenv('SUPABASE_URL')
-    if not base:
-        raise RuntimeError('SUPABASE_URL is required')
-    response = requests.get(
-        f'{base.rstrip("/")}/rest/v1/{table}',
-        headers=supabase_headers(),
-        params={'select': select},
-        timeout=30,
-    )
-    if response.status_code != 200:
-        raise RuntimeError(f'Supabase {table} query failed: {response.status_code} {response.text[:500]}')
-    return response.json()
-
-
-def normalize_uuid(value: str | None) -> str | None:
-    return value.replace('-', '') if value else None
-
-
-def build_records() -> list[UserRecord]:
-    profiles = fetch_table('profiles', 'id,email,username,full_name,display_name,user_type,metadata,user_db_name,school_db_name')
-    memberships = fetch_table('institute_memberships', 'profile_id,institute_id,role')
-    institutes = fetch_table('institutes', 'id,name,urn,neo4j_uuid_string')
-    membership_by_profile = {m['profile_id']: m for m in memberships}
-    institute_by_id = {i['id']: i for i in institutes}
-    records: list[UserRecord] = []
-    for profile in profiles:
-        profile_id = profile['id']
-        user_type = profile.get('user_type') or 'unknown'
-        uuid_no_dash = normalize_uuid(profile_id)
-        email = profile.get('email') or ''
-        username = profile.get('username') or (email.split('@', 1)[0] if email else uuid_no_dash)
-        user_name = profile.get('full_name') or profile.get('display_name') or username or email or profile_id
-        membership = membership_by_profile.get(profile_id, {})
-        institute = institute_by_id.get(membership.get('institute_id'), {}) if membership else {}
-        institute_uuid = institute.get('neo4j_uuid_string') or normalize_uuid(institute.get('id'))
-        user_db_name = profile.get('user_db_name') or f'cc.users.{user_type}.{uuid_no_dash}'
-        institute_db_name = profile.get('school_db_name') or (f'cc.institutes.{institute_uuid}' if institute_uuid else None)
-        records.append(UserRecord(
-            uuid_string=profile_id,
-            user_email=email,
-            cc_username=username or profile_id,
-            user_type=user_type,
-            user_name=user_name,
-            display_name=profile.get('display_name'),
-            institute_id=membership.get('institute_id') if membership else None,
-            institute_name=institute.get('name') if institute else None,
-            institute_role=membership.get('role') if membership else None,
-            institute_neo4j_uuid_string=institute_uuid,
-            user_db_name=user_db_name,
-            institute_db_name=institute_db_name,
-            node_storage_path=f'neo4j://{user_db_name}/User/{profile_id}',
-        ))
-    return records
-
-
-def neo4j_driver():
-    url = os.getenv('APP_BOLT_URL')
-    username = os.getenv('USER_NEO4J')
-    password = os.getenv('PASSWORD_NEO4J')
-    if not url or not username or not password:
-        raise RuntimeError('APP_BOLT_URL, USER_NEO4J, and PASSWORD_NEO4J are required')
-    return GraphDatabase.driver(url, auth=(username, password))
-
-
-def ensure_schema(session) -> None:
-    statements = [
-        'CREATE CONSTRAINT user_uuid_unique IF NOT EXISTS FOR (u:User) REQUIRE u.uuid_string IS UNIQUE',
-        'CREATE INDEX user_email_index IF NOT EXISTS FOR (u:User) ON (u.user_email)',
-        'CREATE INDEX user_username_index IF NOT EXISTS FOR (u:User) ON (u.cc_username)',
-        'CREATE INDEX user_type_index IF NOT EXISTS FOR (u:User) ON (u.user_type)',
-        'CREATE INDEX user_institute_index IF NOT EXISTS FOR (u:User) ON (u.institute_id)',
-        'CREATE CONSTRAINT institute_uuid_unique IF NOT EXISTS FOR (i:Institute) REQUIRE i.uuid_string IS UNIQUE',
-    ]
-    for statement in statements:
-        session.run(statement).consume()
-
-
-def merge_user(session, record: UserRecord) -> None:
-    labels = ':User'
-    if record.user_type == 'teacher':
-        labels = ':User:Teacher'
-    elif record.user_type == 'student':
-        labels = ':User:Student'
-    session.run(
-        f"""
-        MERGE (u{labels} {{uuid_string: $uuid_string}})
-        SET u.user_email = $user_email,
-            u.cc_username = $cc_username,
-            u.user_type = $user_type,
-            u.user_name = $user_name,
-            u.display_name = $display_name,
-            u.institute_id = $institute_id,
-            u.institute_name = $institute_name,
-            u.institute_role = $institute_role,
-            u.institute_neo4j_uuid_string = $institute_neo4j_uuid_string,
-            u.user_db_name = $user_db_name,
-            u.institute_db_name = $institute_db_name,
-            u.node_storage_path = $node_storage_path,
-            u.merged = true,
-            u.source = 'supabase.profiles',
-            u.synced_at = datetime()
-        """,
-        **record.__dict__,
-    ).consume()
-    if record.institute_neo4j_uuid_string:
-        session.run(
-            """
-            MATCH (u:User {uuid_string: $uuid_string})
-            MERGE (i:Institute {uuid_string: $institute_neo4j_uuid_string})
-            SET i.supabase_id = $institute_id,
-                i.name = $institute_name
-            MERGE (u)-[r:MEMBER_OF]->(i)
-            SET r.role = $institute_role,
-                r.synced_at = datetime()
-            """,
-            **record.__dict__,
-        ).consume()
-
-
-def verify(session) -> dict[str, Any]:
-    result: dict[str, Any] = {}
-    result['users'] = session.run('MATCH (u:User) RETURN count(u) AS c').single()['c']
-    result['teachers'] = session.run("MATCH (u:User {user_type: 'teacher'}) RETURN count(u) AS c").single()['c']
-    result['students'] = session.run("MATCH (u:User {user_type: 'student'}) RETURN count(u) AS c").single()['c']
-    result['bad_users'] = session.run("""
-        MATCH (u:User)
-        WHERE u.uuid_string IS NULL OR u.user_email IS NULL OR u.cc_username IS NULL
-           OR u.user_type IS NULL OR u.user_name IS NULL OR u.user_db_name IS NULL
-        RETURN count(u) AS c
-    """).single()['c']
-    result['duplicate_uuids'] = session.run("""
-        MATCH (u:User)
-        WITH u.uuid_string AS uuid, count(*) AS c
-        WHERE c > 1
-        RETURN count(*) AS c
-    """).single()['c']
-    result['memberships'] = session.run('MATCH (:User)-[r:MEMBER_OF]->(:Institute) RETURN count(r) AS c').single()['c']
-    result['institutes'] = session.run('MATCH (i:Institute) RETURN count(i) AS c').single()['c']
-    return result
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--env-file', default=None)
-    parser.add_argument('--database', default='cc.users')
-    parser.add_argument('--apply', action='store_true', help='Write to Neo4j. Defaults to dry-run.')
-    args = parser.parse_args()
-    load_environment(args.env_file)
-    records = build_records()
-    by_type: dict[str, int] = {}
-    for record in records:
-        by_type[record.user_type] = by_type.get(record.user_type, 0) + 1
-    print(f'Supabase source records: {len(records)} {by_type}')
-    if not args.apply:
-        for record in records[:5]:
-            print(f'DRY RUN {record.uuid_string} {record.user_email} -> {record.user_db_name}')
-        print('Dry run only. Re-run with --apply to write Neo4j cc.users.')
-        return 0
-    with neo4j_driver() as driver:
-        with driver.session(database=args.database) as session:
-            ensure_schema(session)
-            for record in records:
-                merge_user(session, record)
-            result = verify(session)
-    print(f'Neo4j verification: {result}')
-    expected = {
-        'users': len(records),
-        'teachers': by_type.get('teacher', 0),
-        'students': by_type.get('student', 0),
-        'bad_users': 0,
-        'duplicate_uuids': 0,
-        'memberships': len(records),
-        'institutes': 2,
-    }
-    for key, value in expected.items():
-        if result.get(key) != value:
-            raise RuntimeError(f'Verification failed for {key}: expected {value}, got {result.get(key)}')
-    return 0
-
-
-if __name__ == '__main__':
-    raise SystemExit(main())

run/routers.py

@@ -9,15 +9,6 @@ from routers.msgraph import router_onenote
 from routers.dev.tests import timetable_test
 from routers.database.init import entity_init, calendar, timetables, curriculum, get_data, schools
 from routers.database.tools import get_nodes, get_nodes_and_edges, tldraw_filesystem, tldraw_supabase_storage, get_events, calendar_structure_router, default_nodes_router, worker_structure_router
-from routers.database.tools.graph_tree_router import router as graph_tree_router
-from routers.database.tools.user_init_router import router as user_init_router
-from routers.database.tools.timetable_builder_router import router as timetable_builder_router
-from routers.database.tools.school_router import router as school_router
-from routers.database.tools.classes_router import router as classes_router
-from routers.database.tools.taught_lessons_router import router as taught_lessons_router
-from routers.database.tools.invitations_router import router as invitations_router
-from routers.database.tools.platform_admin_router import router as platform_admin_router
-from routers.database.tools.lesson_plans_router import router as lesson_plans_router
 from routers.database.files import cabinets as cabinets_router
 from routers.database.files import files as files_router
 from routers.simple_upload import router as simple_upload_router
@@ -38,9 +29,6 @@ from routers import provisioning as provisioning_router
 from routers.transcribe.sessions import router as sessions_router
 from routers.transcribe.canvas_events import router as canvas_events_router
 from routers.transcribe.keywords import router as keywords_router
-from routers.me.bootstrap_router import router as me_bootstrap_router
-from routers import tlsync_token as tlsync_token_router
-from routers.exam import router as exam_router
 
 def register_routes(app: FastAPI):
     logger.info("Starting to register routes...")
@@ -59,28 +47,12 @@ def register_routes(app: FastAPI):
     app.include_router(entity_init.router, prefix="/database/entity", tags=["Entity"])
     app.include_router(calendar.router, prefix="/database/calendar", tags=["Calendar"])
     app.include_router(schools.router, prefix="/database/schools", tags=["Schools"])
-    from routers.database.timetable.timetables import router as timetable_router
     app.include_router(timetables.router, prefix="/database/timetables", tags=["Timetables"])
-    app.include_router(timetable_router, prefix="/database/timetable/timetables", tags=["Timetables"])
     app.include_router(curriculum.router, prefix="/database/curriculum", tags=["Curriculum"])
     
     # Navigation Routes
     app.include_router(calendar_structure_router.router, prefix="/database/calendar-structure", tags=["Calendar"])
     app.include_router(worker_structure_router.router, prefix="/database/worker-structure", tags=["Worker"])
-
-    # Session/bootstrap routes
-    app.include_router(me_bootstrap_router, prefix="/me", tags=["Bootstrap"])
-
-    # Graph navigation
-    app.include_router(graph_tree_router, prefix="/graph", tags=["Graph Navigation"])
-    app.include_router(user_init_router, prefix="/user", tags=["User"])
-    app.include_router(timetable_builder_router, prefix="/timetable", tags=["Timetable"])
-    app.include_router(school_router, prefix="/school", tags=["School"])
-    app.include_router(classes_router, prefix="/database/timetable/classes", tags=["Classes"])
-    app.include_router(taught_lessons_router, prefix="/timetable", tags=["Taught Lessons"])
-    app.include_router(invitations_router, prefix="/users", tags=["People"])
-    app.include_router(platform_admin_router, prefix="/admin", tags=["Platform Admin"])
-    app.include_router(lesson_plans_router, prefix="/lessons", tags=["Lesson Plans"])
     app.include_router(default_nodes_router.router, prefix="/database/tools", tags=["Navigation"])
 
     # Database Filesystem Routes
@@ -132,12 +104,6 @@ def register_routes(app: FastAPI):
     # Provisioning Routes
     app.include_router(provisioning_router.router)
 
-    # TLSync auth token route
-    app.include_router(tlsync_token_router.router, prefix="/api/tlsync", tags=["TLSync"])
-
-    # Exam-marker Routes (as-user Supabase, RLS-enforced; spec §4)
-    app.include_router(exam_router, prefix="/api/exam", tags=["Exam"])
-
     # Transcription Routes (CIS Phase 1)
     app.include_router(sessions_router, prefix="/transcribe", tags=["Transcription Sessions"])
     app.include_router(canvas_events_router, prefix="/transcribe", tags=["Transcription Canvas Events"])

start.sh

@@ -2,7 +2,7 @@
 
 # ClassroomCopilot Startup Script
 # Usage: ./start.sh [start_mode]
-# start_mode options: infra, seed, seed-test, gais-data, full, dev, prod
+# start_mode options: infra, demo-school, demo-users, gais-data, full, dev, prod
 
 set -e
 
@@ -14,10 +14,10 @@ show_help() {
     echo ""
     echo "Start modes:"
     echo "  infra                    - Setup infrastructure (Neo4j schema, calendar, Supabase buckets)"
-    echo "  seed                    - Seed canonical full environment (20 school users)"
-    echo "  seed-test               - Seed lightweight test environment (9 school users)"
+    echo "  demo-school             - Create demo school (KevlarAI)"
+    echo "  demo-users              - Create demo users"
     echo "  gais-data               - Import GAIS data (Edubase, etc.)"
-    echo "  full                    - Run full initialization (infra → seed)"
+    echo "  full                    - Run full initialization (infra → demo-school → demo-users → gais-data)"
     echo "  nuke                    - 💥 NUKE Redis - Clear all queue data for fresh start"
     echo "  dev                     - Run development server with auto-reload"
     echo "  prod                    - Run production server (for Docker/containerized deployment)"
@@ -25,8 +25,8 @@ show_help() {
     echo "Examples:"
     echo "  ./start.sh                    # Run in dev mode (default)"
     echo "  ./start.sh infra              # Setup infrastructure"
-    echo "  ./start.sh seed               # Seed canonical full environment"
-    echo "  ./start.sh seed-test          # Seed lightweight test environment"
+    echo "  ./start.sh demo-school        # Create demo school"
+    echo "  ./start.sh demo-users         # Create demo users"
     echo "  ./start.sh gais-data          # Import GAIS data"
     echo "  ./start.sh full               # Run full initialization"
     echo "  ./start.sh full --yes         # Run full initialization without prompts"
@@ -133,32 +133,54 @@ run_infra() {
     fi
 }
 
-# Function to run canonical environment seed
-run_seed() {
-    local mode=${1:-seed}
-    if [[ "$mode" == "seed-test" ]]; then
-        print_status "Running lightweight seed-test mode (9 school users)..."
-    else
-        print_status "Running canonical seed mode (20 school users)..."
-    fi
+# Function to run demo school creation
+run_demo_school() {
+    print_status "Running demo school creation mode..."
+    print_status "This will create the KevlarAI demo school."
     
     # Check if we should proceed
     if [[ "$AUTO_YES" != true ]]; then
-        read -p "Do you want to continue with $mode? (y/N): " -n 1 -r
+        read -p "Do you want to continue with demo school creation? (y/N): " -n 1 -r
         echo
         if [[ ! $REPLY =~ ^[Yy]$ ]]; then
-            print_status "$mode cancelled."
+            print_status "Demo school creation cancelled."
             exit 0
         fi
     fi
     
-    print_status "Starting $mode process..."
-    $PYTHON_CMD main.py --mode "$mode"
+    print_status "Starting demo school creation process..."
+    $PYTHON_CMD main.py --mode demo-school
     
     if [ $? -eq 0 ]; then
-        print_success "$mode completed successfully!"
+        print_success "Demo school creation completed successfully!"
     else
-        print_error "$mode failed!"
+        print_error "Demo school creation failed!"
+        exit 1
+    fi
+}
+
+# Function to run demo users creation
+run_demo_users() {
+    print_status "Running demo users creation mode..."
+    print_status "This will create demo users for testing."
+    
+    # Check if we should proceed
+    if [[ "$AUTO_YES" != true ]]; then
+        read -p "Do you want to continue with demo users creation? (y/N): " -n 1 -r
+        echo
+        if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+            print_status "Demo users creation cancelled."
+            exit 0
+        fi
+    fi
+    
+    print_status "Starting demo users creation process..."
+    $PYTHON_CMD main.py --mode demo-users
+    
+    if [ $? -eq 0 ]; then
+        print_success "Demo users creation completed successfully!"
+    else
+        print_error "Demo users creation failed!"
         exit 1
     fi
 }
@@ -252,7 +274,7 @@ except Exception as e:
 
 # Function to run full initialization (all steps in order)
 run_full() {
-    print_status "Running full initialization (infra → seed)..."
+    print_status "Running full initialization (infra → demo-school → demo-users → gais-data)..."
 
     # Single confirmation for the whole flow
     if [[ "$AUTO_YES" != true ]]; then
@@ -267,8 +289,14 @@ run_full() {
     # Run infra
     run_infra || { print_error "Full init aborted during infra."; exit 1; }
 
-    # Run canonical full seed
-    run_seed seed || { print_error "Full init aborted during seed."; exit 1; }
+    # Run demo school
+    run_demo_school || { print_error "Full init aborted during demo-school."; exit 1; }
+
+    # Run demo users
+    run_demo_users || { print_error "Full init aborted during demo-users."; exit 1; }
+
+    # Run GAIS data import
+    run_gais_data || { print_error "Full init aborted during gais-data."; exit 1; }
 
     print_success "Full initialization completed successfully!"
 }
@@ -355,11 +383,11 @@ main() {
         "infra")
             run_infra
             ;;
-        "seed")
-            run_seed seed
+        "demo-school")
+            run_demo_school
             ;;
-        "seed-test")
-            run_seed seed-test
+        "demo-users")
+            run_demo_users
             ;;
         "gais-data")
             run_gais_data
@@ -378,7 +406,7 @@ main() {
             ;;
         *)
             print_error "Invalid start mode: $START_MODE"
-            print_status "Valid modes: infra, seed, seed-test, gais-data, full, nuke, dev, prod"
+            print_status "Valid modes: infra, demo-school, demo-users, gais-data, nuke, dev, prod"
             print_status "Usage: ./start.sh [start_mode]"
             print_status "Use './start.sh --help' for more information"
             exit 1

tests/test_dev_stack.py

@@ -1,86 +0,0 @@
-import os
-
-import requests
-
-
-def _supabase_headers():
-    key = os.getenv('SERVICE_ROLE_KEY') or os.getenv('ANON_KEY')
-    assert key, 'SERVICE_ROLE_KEY or ANON_KEY must be set for Supabase integration tests'
-    return {
-        'apikey': key,
-        'Authorization': f'Bearer {key}',
-        'Prefer': 'count=exact',
-    }
-
-
-def _rest_count(table: str) -> int:
-    supabase_url = os.getenv('SUPABASE_URL')
-    assert supabase_url, 'SUPABASE_URL must be set'
-    response = requests.get(
-        f'{supabase_url.rstrip("/")}/rest/v1/{table}',
-        headers=_supabase_headers(),
-        params={'select': 'id'},
-        timeout=15,
-    )
-    assert response.status_code in (200, 206), response.text[:500]
-    content_range = response.headers.get('content-range', '')
-    assert '/' in content_range, f'missing exact content-range count for {table}: {content_range!r}'
-    return int(content_range.rsplit('/', 1)[1])
-
-
-def test_dev_environment_points_at_dev_supabase():
-    assert os.getenv('SUPABASE_URL') == 'http://192.168.0.94:8000'
-
-
-def test_dev_api_health_endpoint_is_healthy():
-    health_url = os.getenv('API_HEALTH_URL', 'http://192.168.0.64:18000/health')
-    response = requests.get(health_url, timeout=15)
-    assert response.status_code == 200
-    payload = response.json()
-    assert payload['status'] == 'healthy'
-
-    runtime = payload['runtime']
-    assert runtime['api_runtime_role'] == 'dev'
-    assert runtime['start_mode'] == 'dev'
-    assert runtime['app_environment'] == 'development'
-    assert runtime['environment'] == 'development'
-    assert runtime['backend_dev_mode'] is True
-    assert runtime['compose_project'] == 'api-dev'
-    assert runtime['supabase_url_host'] == '192.168.0.94'
-
-    assert payload['services']['supabase']['status'] == 'healthy'
-    assert payload['services']['supabase']['url_host'] == '192.168.0.94'
-    assert payload['services']['redis']['status'] == 'healthy'
-    assert payload['services']['redis']['environment'] == 'dev'
-    assert payload['services']['redis']['database'] == 0
-
-
-# NOTE: these are >= baselines, not exact counts. The greenfield seed produces this floor;
-# additive exam-marker fixtures (S4-4 cohort adds ~10 students/memberships; ad-hoc classes) push
-# the live .94 counts above it. Exact == froze a snapshot that any new fixture breaks, while >=
-# still catches a broken or missing seed.
-def test_supabase_dev_seed_core_counts():
-    assert _rest_count('profiles') >= 21
-    assert _rest_count('institute_memberships') >= 21
-    assert _rest_count('institutes') >= 2
-
-
-def test_supabase_dev_seed_timetable_counts():
-    assert _rest_count('classes') >= 17
-    assert _rest_count('taught_lessons') >= 1462
-
-
-def test_runtime_identity_does_not_expose_secret_values():
-    health_url = os.getenv('API_HEALTH_URL', 'http://192.168.0.64:18000/health')
-    response = requests.get(health_url, timeout=15)
-    assert response.status_code == 200
-    payload_text = response.text
-    for secret_name in ('SERVICE_ROLE_KEY', 'ANON_KEY', 'SUPABASE_JWT_SECRET', 'REDIS_PASSWORD'):
-        secret_value = os.getenv(secret_name)
-        if secret_value:
-            assert secret_value not in payload_text
-
-    runtime = response.json()['runtime']
-    assert 'supabase_url' not in runtime
-    assert 'service_role_key' not in runtime
-    assert 'anon_key' not in runtime

tests/test_docling_auto_map.py

@@ -1,53 +0,0 @@
-import json
-import os
-from pathlib import Path
-
-import pytest
-
-from api.services.docling import FIRST_PASS_SCHEMA, auto_map
-
-
-SPIKE_ROOT = Path(os.environ.get("DOCLING_SPIKE_ROOT", "/home/kcar/dev/docling-exam-spike"))
-PHYSICS_PDF = SPIKE_ROOT / "samples" / "AQA-Physics-Paper-1H-2022-with-qr.pdf"
-PHYSICS_TEMPLATE = SPIKE_ROOT / "results" / "template" / "physics.json"
-BORN_DIGITAL_PDF = SPIKE_ROOT / "samples" / "physics-p1h-2022-qp.pdf"
-
-
-@pytest.mark.skipif(not (PHYSICS_PDF.exists() and PHYSICS_TEMPLATE.exists()), reason="spike corpus not present")
-def test_auto_map_matches_spike_physics_template_shape():
-    expected = json.loads(PHYSICS_TEMPLATE.read_text())
-    result = auto_map(PHYSICS_PDF.read_bytes(), spike_root=SPIKE_ROOT)
-
-    assert result["meta"]["schema"] == FIRST_PASS_SCHEMA
-    assert result["meta"]["schema"] == expected["meta"]["schema"]
-    assert set(result.keys()) == set(expected.keys())
-    assert result["meta"]["board"] == expected["meta"]["board"]
-    assert result["meta"]["paper_code"] == expected["meta"]["paper_code"]
-    assert len(result["margins"]) == len(expected["margins"])
-    assert set(result["pages"].keys()) == set(expected["pages"].keys())
-    assert result["pages"]["2"]["role"] == expected["pages"]["2"]["role"]
-    part_band = result["pages"]["2"]["part_bands"][0]
-    assert set(expected["pages"]["2"]["part_bands"][0].keys()).issubset(part_band.keys())
-    assert part_band["box"]
-
-
-@pytest.mark.skipif(not BORN_DIGITAL_PDF.exists(), reason="born-digital spike PDF not present")
-def test_auto_map_fast_path_without_cache_produces_first_pass_template():
-    result = auto_map(
-        BORN_DIGITAL_PDF.read_bytes(),
-        source_pdf="samples/physics-p1h-2022-qp.pdf",
-        spike_root=SPIKE_ROOT,
-        prefer_cache=False,
-    )
-
-    assert result["meta"]["schema"] == FIRST_PASS_SCHEMA
-    assert result["meta"]["board"] == "aqa"
-    assert result["meta"]["paper_code"] == "8463/1"
-    assert result["meta"]["source_pdf"] == "samples/physics-p1h-2022-qp.pdf"
-    assert result["margins"]
-    assert result["pages"]
-
-
-def test_auto_map_rejects_empty_pdf_bytes():
-    with pytest.raises(ValueError):
-        auto_map(b"")