feat(seed): expand corpus to 1178 papers + download-only/unseed/granular reset
PRIMARY — corpus breadth (505->1178 papers, 18->60 specs, all URLs HEAD-verified): - AQA (enumerated): Maths, English Lang/Lit, Geography, Computer Science, Business, Psychology, MFL (French/Spanish/German), GCSE + A-level, on top of round-1 sciences. - Edexcel + OCR (confirmed direct URLs via research): Maths, English, Geography, History, Business, Computer Science, GCSE + A-level. - generate_corpus_manifest.py: _subj/_mfl AQA builders, Edexcel/OCR spec+URL tables, derived exam_code (_mk_exam_code) matching the locked convention, concurrent re-verify. Verified on dev .94: eb_specifications=60, eb_exams=1178, QP=469, doc_type all 'pdf', seed idempotent (uploaded=673 new, skipped=505), failed=0. SECONDARY: - --download-only + persistent bucket-shaped local store (manifests/_corpus_store/, gitignored): download-once, seed-many, offline-repeatable; --store-dir/--no-store. (_store_path/_item_bytes/ download_corpus). Verified: store populated, seed reads offline (download_cached). - --unseed [--board/--spec]: inverse loader — storage objects (Storage API; protect_delete blocks raw SQL), first-sweep seed templates, eb_exams, eb_specifications. Verified reversible on .94. - Granular admin reset: POST /admin/reset?scope=all|exam-corpus|timetable. reset_environment.reset(scope) adds EXAM_CORPUS_TABLES (10) + cc.examboards storage cleanup + TIMETABLE_TABLES (13); 'all' now also clears the exam subsystem the legacy reset missed. No schema migration required. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
CC Worker
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
5750413f43
commit
cdc105ae54
@@ -82,6 +82,41 @@ SUPABASE_TABLES_TO_CLEAR = [
|
||||
"admin_profiles",
|
||||
]
|
||||
|
||||
# Exam subsystem tables, FK child-first. NOT in the list above — the previous full reset()
|
||||
# never cleared exam data or storage at all; the granular scopes below fold it in.
|
||||
EXAM_CORPUS_TABLES = [
|
||||
"mark_entries",
|
||||
"student_submissions",
|
||||
"marking_batches",
|
||||
"exam_response_areas",
|
||||
"exam_boundaries",
|
||||
"exam_template_layout",
|
||||
"exam_questions",
|
||||
"exam_templates",
|
||||
"eb_exams",
|
||||
"eb_specifications",
|
||||
]
|
||||
|
||||
# Timetable / calendar materialization subset (for scope='timetable').
|
||||
TIMETABLE_TABLES = [
|
||||
"lesson_deliveries",
|
||||
"lesson_collaborators",
|
||||
"taught_lessons",
|
||||
"academic_periods",
|
||||
"academic_days",
|
||||
"academic_weeks",
|
||||
"academic_term_breaks",
|
||||
"academic_terms",
|
||||
"academic_years",
|
||||
"teacher_timetable_slots",
|
||||
"teacher_timetables",
|
||||
"school_timetables",
|
||||
"planned_lessons",
|
||||
]
|
||||
|
||||
# Buckets whose objects the exam-corpus reset clears (Storage API — protect_delete blocks raw SQL).
|
||||
EXAM_STORAGE_BUCKET = "cc.examboards"
|
||||
|
||||
|
||||
def _sb_headers():
|
||||
url = os.environ["SUPABASE_URL"]
|
||||
@@ -146,13 +181,84 @@ def _supabase_delete_auth_user(url: str, headers: dict, uid: str):
|
||||
logger.warning(f" Delete auth user {uid}: {r.status_code} {r.text[:80]}")
|
||||
|
||||
|
||||
# ─── Granular helpers ───────────────────────────────────────────────────────────
|
||||
|
||||
def _clear_tables(url: str, headers: dict, tables: List[str]) -> "tuple[List[str], List[str]]":
|
||||
cleared, failed = [], []
|
||||
for table in tables:
|
||||
if _sb_clear_table(url, headers, table) in (200, 204):
|
||||
cleared.append(table)
|
||||
logger.info(f" ✓ {table}")
|
||||
else:
|
||||
failed.append(table)
|
||||
return cleared, failed
|
||||
|
||||
|
||||
def _clear_exam_storage() -> Dict[str, Any]:
|
||||
"""Remove cc.examboards objects via the Storage API (protect_delete blocks raw SQL deletes).
|
||||
Gathers storage_loc from eb_exams/eb_specifications BEFORE the rows are cleared."""
|
||||
try:
|
||||
from modules.database.supabase.utils.client import SupabaseServiceRoleClient
|
||||
from modules.database.supabase.utils.storage import StorageAdmin
|
||||
except Exception as exc:
|
||||
logger.warning(f" exam storage clear skipped (import): {exc}")
|
||||
return {"removed": 0, "error": str(exc)}
|
||||
sb = SupabaseServiceRoleClient().supabase
|
||||
storage = StorageAdmin()
|
||||
locs: List[str] = []
|
||||
for table in ("eb_exams", "eb_specifications"):
|
||||
try:
|
||||
rows = sb.table(table).select("storage_loc").execute().data or []
|
||||
locs += [r["storage_loc"] for r in rows if r.get("storage_loc")]
|
||||
except Exception as exc:
|
||||
logger.warning(f" storage_loc gather {table}: {exc}")
|
||||
by_bucket: Dict[str, List[str]] = {}
|
||||
for loc in locs:
|
||||
if "/" in loc:
|
||||
b, _, p = loc.partition("/")
|
||||
by_bucket.setdefault(b, []).append(p)
|
||||
removed = 0
|
||||
for b, paths in by_bucket.items():
|
||||
for i in range(0, len(paths), 100):
|
||||
chunk = paths[i:i + 100]
|
||||
try:
|
||||
storage.client.supabase.storage.from_(b).remove(chunk)
|
||||
removed += len(chunk)
|
||||
except Exception as exc:
|
||||
logger.warning(f" storage remove {b}: {exc}")
|
||||
logger.info(f" exam storage removed {removed} objects from {list(by_bucket)}")
|
||||
return {"removed": removed, "buckets": list(by_bucket)}
|
||||
|
||||
|
||||
# ─── Main reset ───────────────────────────────────────────────────────────────
|
||||
|
||||
def reset() -> Dict[str, Any]:
|
||||
def reset(scope: str = "all") -> Dict[str, Any]:
|
||||
"""Destructive reset. scope ∈ {all, exam-corpus, timetable}.
|
||||
|
||||
- all : full wipe (Neo4j + Supabase data + auth users) AND the exam subsystem + storage.
|
||||
- exam-corpus : ONLY eb_*/exam_* tables + cc.examboards storage objects (load/unload the corpus).
|
||||
- timetable : ONLY timetable/calendar materialization tables.
|
||||
"""
|
||||
scope = (scope or "all").lower()
|
||||
if scope not in ("all", "exam-corpus", "timetable"):
|
||||
raise ValueError(f"invalid scope {scope!r} (want all|exam-corpus|timetable)")
|
||||
url, headers = _sb_headers()
|
||||
|
||||
if scope == "exam-corpus":
|
||||
logger.info("RESET (scope=exam-corpus) — exam tables + cc.examboards storage")
|
||||
storage = _clear_exam_storage()
|
||||
cleared, failed = _clear_tables(url, headers, EXAM_CORPUS_TABLES)
|
||||
return {"scope": scope, "exam_storage": storage, "tables_cleared": cleared, "tables_failed": failed}
|
||||
|
||||
if scope == "timetable":
|
||||
logger.info("RESET (scope=timetable) — timetable/calendar tables")
|
||||
cleared, failed = _clear_tables(url, headers, TIMETABLE_TABLES)
|
||||
return {"scope": scope, "tables_cleared": cleared, "tables_failed": failed}
|
||||
|
||||
logger.info("=" * 60)
|
||||
logger.info("RESET ENVIRONMENT — full destructive wipe starting")
|
||||
logger.info("=" * 60)
|
||||
results: Dict[str, Any] = {}
|
||||
results: Dict[str, Any] = {"scope": scope}
|
||||
|
||||
# ── 1. Neo4j: drop everything except system + neo4j ──────────────────────
|
||||
logger.info("\n[Neo4j] Dropping all non-system databases...")
|
||||
@@ -213,11 +319,22 @@ def reset() -> Dict[str, Any]:
|
||||
)
|
||||
logger.info(" kcar → admin_profiles restored ✓")
|
||||
|
||||
# ── 5. Exam subsystem: storage objects (Storage API) + exam tables ───────────
|
||||
# (The legacy full reset cleared neither exam tables nor storage — folded in here.)
|
||||
logger.info("\n[Supabase] Clearing exam subsystem (storage + eb_*/exam_* tables)...")
|
||||
exam_storage = _clear_exam_storage()
|
||||
exam_cleared, exam_failed = _clear_tables(url, headers, EXAM_CORPUS_TABLES)
|
||||
|
||||
results["supabase"] = {
|
||||
"tables_cleared": cleared,
|
||||
"tables_failed": failed,
|
||||
"deleted_users": deleted_emails,
|
||||
}
|
||||
results["exam"] = {
|
||||
"storage": exam_storage,
|
||||
"tables_cleared": exam_cleared,
|
||||
"tables_failed": exam_failed,
|
||||
}
|
||||
|
||||
logger.info("\n" + "=" * 60)
|
||||
logger.info("RESET COMPLETE")
|
||||
|
||||
Reference in New Issue
Block a user