ClassroomCopilot/api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

latest

Browse Source
This commit is contained in:
kcar
2025-11-14 14:47:19 +00:00
parent 2a85845835
commit 46b2319e2d
199 changed files with 607543 additions and 11147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
routers/database/files/__init__.py
+3
View File
@@ -0,0 +1,3 @@
from . import cabinets, files
routers/database/files/cabinets.py
+75
View File
@@ -0,0 +1,75 @@
import os
from fastapi import APIRouter, Depends, HTTPException
from typing import Any, Dict
from modules.auth.supabase_bearer import SupabaseBearer
from modules.database.supabase.utils.client import SupabaseServiceRoleClient
router = APIRouter()
auth = SupabaseBearer()
@router.get("/cabinets")
def list_cabinets(payload: Dict[str, Any] = Depends(auth)):
user_id = payload.get('sub') or payload.get('user_id')
if not user_id:
raise HTTPException(status_code=401, detail="Invalid token payload")
client = SupabaseServiceRoleClient()
# Owned + shared via membership
owned = client.supabase.table('file_cabinets').select('*').eq('user_id', user_id).execute().data
shared = client.supabase.table('cabinet_memberships').select('cabinet_id').eq('profile_id', user_id).execute().data
shared_ids = [m['cabinet_id'] for m in (shared or [])]
shared_rows = client.supabase.table('file_cabinets').select('*').in_('id', shared_ids).execute().data if shared_ids else []
return {"owned": owned or [], "shared": shared_rows or []}
@router.post("/cabinets")
def create_cabinet(body: Dict[str, Any], payload: Dict[str, Any] = Depends(auth)):
user_id = payload.get('sub') or payload.get('user_id')
name = (body or {}).get('name')
if not user_id or not name:
raise HTTPException(status_code=400, detail="name is required")
client = SupabaseServiceRoleClient()
res = client.supabase.table('file_cabinets').insert({
'user_id': user_id,
'name': name
}).execute()
return res.data
@router.patch("/cabinets/{cabinet_id}")
def rename_cabinet(cabinet_id: str, body: Dict[str, Any], payload: Dict[str, Any] = Depends(auth)):
name = (body or {}).get('name')
if not name:
raise HTTPException(status_code=400, detail="name is required")
client = SupabaseServiceRoleClient()
res = client.supabase.table('file_cabinets').update({'name': name}).eq('id', cabinet_id).execute()
return res.data
@router.delete("/cabinets/{cabinet_id}")
def delete_cabinet(cabinet_id: str, payload: Dict[str, Any] = Depends(auth)):
client = SupabaseServiceRoleClient()
res = client.supabase.table('file_cabinets').delete().eq('id', cabinet_id).execute()
return res.data
@router.post("/cabinets/{cabinet_id}/members")
def add_member(cabinet_id: str, body: Dict[str, Any], payload: Dict[str, Any] = Depends(auth)):
target_profile_id = (body or {}).get('profile_id')
role = (body or {}).get('role', 'viewer')
if not target_profile_id:
raise HTTPException(status_code=400, detail="profile_id required")
client = SupabaseServiceRoleClient()
# Insert membership (RLS will ensure only owner can do it)
res = client.supabase.table('cabinet_memberships').upsert({
'cabinet_id': cabinet_id,
'profile_id': target_profile_id,
'role': role
}).execute()
return res.data
@router.delete("/cabinets/{cabinet_id}/members/{profile_id}")
def remove_member(cabinet_id: str, profile_id: str, payload: Dict[str, Any] = Depends(auth)):
client = SupabaseServiceRoleClient()
res = client.supabase.table('cabinet_memberships').delete().match({
'cabinet_id': cabinet_id,
'profile_id': profile_id
}).execute()
return res.data
routers/database/files/files.py
+1000
View File
File diff suppressed because it is too large Load Diff
routers/database/files/files_simplified.py
+256
View File
@@ -0,0 +1,256 @@
"""
Simplified Files Router
======================
Simplified version of the files router with auto-processing removed.
Keeps only essential functionality for file management and manual processing triggers.
This replaces the complex auto-processing system with simple file storage.
"""
import os
import uuid
import logging
from typing import Dict, List, Optional, Any
from pathlib import Path
from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Form, BackgroundTasks
from fastapi.responses import JSONResponse
from modules.auth.supabase_bearer import SupabaseBearer
from modules.database.supabase.utils.client import SupabaseServiceRoleClient
from modules.database.supabase.utils.storage import StorageAdmin
from modules.logger_tool import initialise_logger
router = APIRouter()
auth = SupabaseBearer()
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
def _choose_bucket(scope: str, user_id: str, school_id: Optional[str]) -> str:
"""Choose appropriate bucket based on scope - matches old system logic."""
scope = (scope or 'teacher').lower()
if scope == 'school' and school_id:
return f"cc.institutes.{school_id}.private"
# teacher / student fall back to users bucket for now
return 'cc.users'
@router.post("/files/upload")
async def upload_file(
cabinet_id: str = Form(...),
path: str = Form(...),
scope: str = Form(...),
file: UploadFile = File(...),
payload: Dict[str, Any] = Depends(auth)
):
"""
SIMPLIFIED file upload - no automatic processing.
Just stores the file and creates a database record.
This is the legacy endpoint maintained for backward compatibility.
"""
try:
user_id = payload.get('sub') or payload.get('user_id')
if not user_id:
raise HTTPException(status_code=401, detail="User ID required")
# Read file content
file_bytes = await file.read()
file_size = len(file_bytes)
mime_type = file.content_type or 'application/octet-stream'
filename = file.filename or path
logger.info(f"📤 Simplified upload: {filename} ({file_size} bytes) for user {user_id}")
# Initialize services
client = SupabaseServiceRoleClient()
storage = StorageAdmin()
# Generate file ID and storage path
file_id = str(uuid.uuid4())
# Use same bucket logic as old system for consistency
bucket = _choose_bucket('teacher', user_id, None)
storage_path = f"{cabinet_id}/{file_id}/{filename}"
# Store file in Supabase storage
try:
storage.upload_file(bucket, storage_path, file_bytes, mime_type, upsert=True)
except Exception as e:
logger.error(f"Storage upload failed for {file_id}: {e}")
raise HTTPException(status_code=500, detail=f"Storage upload failed: {str(e)}")
# Create database record
try:
insert_res = client.supabase.table('files').insert({
'id': file_id,
'name': filename,
'cabinet_id': cabinet_id,
'bucket': bucket,
'path': storage_path,
'mime_type': mime_type,
'uploaded_by': user_id,
'size_bytes': file_size,
'source': 'classroomcopilot-web',
'is_directory': False,
'processing_status': 'uploaded', # No auto-processing
'relative_path': filename
}).execute()
if not insert_res.data:
# Clean up storage on DB failure
try:
storage.delete_file(bucket, storage_path)
except:
pass
raise HTTPException(status_code=500, detail="Failed to create file record")
file_record = insert_res.data[0]
except Exception as e:
logger.error(f"Database insert failed for {file_id}: {e}")
# Clean up storage
try:
storage.delete_file(bucket, storage_path)
except:
pass
raise HTTPException(status_code=500, detail=f"Database error: {str(e)}")
logger.info(f"✅ Simplified upload completed: {file_id}")
return {
'status': 'success',
'message': 'File uploaded successfully (no auto-processing)',
'file': file_record,
'auto_processing_disabled': True,
'next_steps': 'Use manual processing endpoints if needed'
}
except HTTPException:
raise
except Exception as e:
logger.error(f"Upload error: {e}")
raise HTTPException(status_code=500, detail=f"Upload failed: {str(e)}")
@router.get("/files")
def list_files(cabinet_id: str, payload: Dict[str, Any] = Depends(auth)):
"""List files in a cabinet."""
client = SupabaseServiceRoleClient()
res = client.supabase.table('files').select('*').eq('cabinet_id', cabinet_id).execute()
return res.data
@router.get("/files/{file_id}")
def get_file(file_id: str, payload: Dict[str, Any] = Depends(auth)):
"""Get file details."""
client = SupabaseServiceRoleClient()
res = client.supabase.table('files').select('*').eq('id', file_id).single().execute()
if not res.data:
raise HTTPException(status_code=404, detail="File not found")
return res.data
@router.delete("/files/{file_id}")
def delete_file(file_id: str, payload: Dict[str, Any] = Depends(auth)):
"""Delete a file."""
client = SupabaseServiceRoleClient()
storage = StorageAdmin()
# Get file info first
res = client.supabase.table('files').select('*').eq('id', file_id).single().execute()
if not res.data:
raise HTTPException(status_code=404, detail="File not found")
file_data = res.data
# Delete from storage
try:
storage.delete_file(file_data['bucket'], file_data['path'])
except Exception as e:
logger.warning(f"Failed to delete file from storage: {e}")
# Delete from database
delete_res = client.supabase.table('files').delete().eq('id', file_id).execute()
logger.info(f"🗑️ Deleted file: {file_id}")
return {
'status': 'success',
'message': 'File deleted successfully'
}
@router.post("/files/{file_id}/process-manual")
async def trigger_manual_processing(
file_id: str,
processing_type: str = Form('basic'), # basic, advanced, custom
payload: Dict[str, Any] = Depends(auth)
):
"""
Trigger manual processing for a file.
This is where users can manually start processing when they want it.
"""
# TODO: Implement manual processing triggers
# This would call the archived processing logic when the user explicitly requests it
logger.info(f"🔧 Manual processing requested for file {file_id} (type: {processing_type})")
return {
'status': 'accepted',
'message': f'Manual processing queued for file {file_id}',
'processing_type': processing_type,
'note': 'Manual processing not yet implemented - will use archived auto-processing logic'
}
@router.get("/files/{file_id}/status")
def get_processing_status(file_id: str, payload: Dict[str, Any] = Depends(auth)):
"""Get processing status for a file."""
client = SupabaseServiceRoleClient()
res = client.supabase.table('files').select('processing_status, error_message, extra').eq('id', file_id).single().execute()
if not res.data:
raise HTTPException(status_code=404, detail="File not found")
return {
'file_id': file_id,
'status': res.data.get('processing_status', 'unknown'),
'error': res.data.get('error_message'),
'details': res.data.get('extra', {})
}
# Keep existing artefacts endpoints for backward compatibility
@router.get("/files/{file_id}/artefacts")
def list_file_artefacts(file_id: str, payload: Dict[str, Any] = Depends(auth)):
"""List artefacts for a file."""
client = SupabaseServiceRoleClient()
res = client.supabase.table('document_artefacts').select('*').eq('file_id', file_id).execute()
return res.data or []
@router.get("/files/{file_id}/viewer-artefacts")
def list_viewer_artefacts(file_id: str, payload: Dict[str, Any] = Depends(auth)):
"""List artefacts organized for the viewer."""
client = SupabaseServiceRoleClient()
# Get all artefacts
res = client.supabase.table('document_artefacts').select('*').eq('file_id', file_id).execute()
artefacts = res.data or []
# Simple organization - no complex bundle logic
organized = {
'document_analysis': [],
'processing_bundles': [],
'raw_data': []
}
for artefact in artefacts:
artefact_type = artefact.get('type', '')
if 'analysis' in artefact_type.lower():
organized['document_analysis'].append(artefact)
elif any(bundle_type in artefact_type for bundle_type in ['docling', 'bundle']):
organized['processing_bundles'].append(artefact)
else:
organized['raw_data'].append(artefact)
return organized
routers/database/files/split_map.py
+588
View File
@@ -0,0 +1,588 @@
# api/routers/database/files/split_map.py
"""
Automatic split_map.json generator for uploaded documents.
This module creates chapter/section boundaries for documents using existing artefacts
(Tika JSON, Docling frontmatter OCR) and optional PDF outline extraction.
Strategy (waterfall, stop at confidence ≥ 0.7):
1. PDF Outline/Bookmarks (best): confidence ≈ 0.95
2. Headings from Docling JSON: confidence ≈ 0.8
3. TOC from Tika text: confidence ≈ 0.7-0.8
4. Fixed windows: confidence ≈ 0.2
Hard constraints:
- For any fallback Docling "no-OCR" call: limit page_range to [1, min(30, page_count)]
- Never process more than 30 pages in one Docling request
- Use existing artefacts whenever possible
"""
import re
import json
import uuid
import datetime
import os
import requests
from typing import List, Dict, Any, Optional, Tuple
from modules.database.supabase.utils.client import SupabaseServiceRoleClient
from modules.database.supabase.utils.storage import StorageAdmin
from modules.logger_tool import initialise_logger
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
# ---------- Utilities
def _now_iso():
"""Return current UTC timestamp in ISO format."""
return datetime.datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
def _load_artefact_json(storage: StorageAdmin, bucket: str, rel_path: str) -> Optional[Dict[str, Any]]:
"""Load JSON artefact from storage."""
try:
raw = storage.download_file(bucket, rel_path)
return json.loads(raw.decode("utf-8"))
except Exception as e:
logger.debug(f"Failed to load artefact {rel_path}: {e}")
return None
def _page_count_from_tika(tika_json: Dict[str, Any]) -> Optional[int]:
"""Extract page count from Tika JSON metadata."""
for k in ("xmpTPg:NPages", "Page-Count", "pdf:PageCount", "pdf:pagecount"):
v = tika_json.get(k) or tika_json.get(k.lower())
try:
if v is not None:
return int(v)
except Exception:
pass
return None
# ---------- A) Outline via PyMuPDF (optional but recommended)
def _try_outline(pdf_bytes: bytes) -> Optional[List[Tuple[str, int]]]:
"""
Extract PDF outline/bookmarks using PyMuPDF.
Returns [(title, start_page)] for level-1 bookmarks only.
"""
try:
import fitz # PyMuPDF
doc = fitz.open(stream=pdf_bytes, filetype="pdf")
toc = doc.get_toc(simple=True) # list of [level, title, page]
doc.close()
# Keep level-1 only, ensure valid pages
out = []
for level, title, page in toc:
if level == 1 and page >= 1:
clean_title = title.strip()
if clean_title and len(clean_title) > 1:
out.append((clean_title, page))
return out if len(out) >= 2 else None # Need at least 2 chapters
except ImportError:
logger.debug("PyMuPDF not available, skipping outline extraction")
return None
except Exception as e:
logger.debug(f"Outline extraction failed: {e}")
return None
# ---------- B) Headings from Docling JSON
def _try_headings(docling_json: Dict[str, Any]) -> Optional[List[Tuple[str, int, int]]]:
"""
Extract headings from Docling JSON.
Returns [(title, start_page, level)] — we only return starts; end pages are computed later.
"""
if not docling_json:
return None
# Handle different Docling JSON structures
blocks = (docling_json.get("blocks") or
docling_json.get("elements") or
docling_json.get("body", {}).get("blocks") or [])
candidates: List[Tuple[str, int, int]] = []
for b in blocks:
# Check if this is a heading block
role = (b.get("role") or b.get("type") or "").lower()
if not ("heading" in role or role in ("h1", "h2", "title", "section-header")):
continue
# Extract text content
text = (b.get("text") or b.get("content") or "").strip()
if not text or len(text) < 3:
continue
# Extract page number with robust handling of 0-based pageIndex
p = None
if b.get("pageIndex") is not None:
try:
p = int(b.get("pageIndex")) + 1
except Exception:
p = None
if p is None:
for key in ("page", "page_no", "page_number"):
if b.get(key) is not None:
try:
p = int(b.get(key))
except Exception:
p = None
break
if p is None or p < 1:
continue
# Determine heading level
level = 1 # default
if "1" in role or "h1" in role:
level = 1
elif "2" in role or "h2" in role:
level = 2
# Chapter regex boosts to level 1
if re.match(r"^\s*(chapter|ch\.?|section|part)\s+\d+", text, re.I):
level = 1
candidates.append((text, p, level))
if not candidates:
return None
# Prefer level 1; if none, promote level 2 to level 1
l1 = [(t, p, l) for (t, p, l) in candidates if l == 1]
if not l1:
l1 = [(t, p, 1) for (t, p, _) in candidates]
# Sort by page and keep strictly increasing pages only
l1_sorted = []
seen = set()
for (t, p, l) in sorted(l1, key=lambda x: x[1]):
if p not in seen and p >= 1:
l1_sorted.append((t, p, l))
seen.add(p)
return l1_sorted if len(l1_sorted) >= 2 else None
def _try_headings_fallback(file_id: str, cabinet_id: str, bucket: str,
processing_bytes: bytes, processing_mime: str,
page_count: int) -> Optional[List[Tuple[str, int, int]]]:
"""
Make a limited Docling no-OCR call (max 30 pages) to extract headings.
This is used only when existing artefacts don't have sufficient heading data.
"""
try:
docling_url = os.getenv('DOCLING_URL') or os.getenv('NEOFS_DOCLING_URL')
if not docling_url:
logger.debug("No Docling URL configured for headings fallback")
return None
# Strictly limit to first 30 pages
max_pages = min(30, page_count)
logger.info(f"Headings fallback: limited Docling call for file_id={file_id}, pages=1-{max_pages}")
# Build Docling request (no-OCR, limited pages)
docling_api_key = os.getenv('DOCLING_API_KEY')
headers = {'Accept': 'application/json'}
if docling_api_key:
headers['X-Api-Key'] = docling_api_key
form_data = [
('target_type', 'inbody'),
('to_formats', 'json'),
('do_ocr', 'false'),
('force_ocr', 'false'),
('image_export_mode', 'embedded'),
('pdf_backend', 'dlparse_v4'),
('table_mode', 'fast'),
('page_range', '1'),
('page_range', str(max_pages))
]
files = [('files', ('file', processing_bytes, processing_mime))]
# Make the request with timeout
timeout = int(os.getenv('DOCLING_HEADINGS_TIMEOUT', '1800')) # 30 minutes default
resp = requests.post(
f"{docling_url.rstrip('/')}/v1/convert/file",
files=files,
data=form_data,
headers=headers,
timeout=timeout
)
resp.raise_for_status()
docling_json = resp.json()
logger.debug(f"Headings fallback: received Docling response for file_id={file_id}")
return _try_headings(docling_json)
except Exception as e:
logger.error(f"Headings fallback failed for file_id={file_id}: {e}")
return None
# ---------- C) TOC from Tika text (dot leaders & page num)
TOC_LINE = re.compile(r"^\s*(.+?)\s?(\.{2,}|\s{3,})\s*(\d{1,4})\s*$")
def _try_toc_text(tika_text: str) -> Optional[List[Tuple[str, int]]]:
"""
Parse TOC from Tika text using dot leaders and page numbers.
Returns [(title, start_page)] if successful.
"""
if not tika_text:
return None
# Heuristic: only scan first ~1500 lines (roughly first 15 pages)
head = "\n".join(tika_text.splitlines()[:1500])
pairs = []
for line in head.splitlines():
m = TOC_LINE.match(line)
if not m:
continue
title = m.group(1).strip()
try:
page = int(m.group(3))
except Exception:
continue
# Reject obvious junk
if len(title) < 3 or page < 1 or page > 9999:
continue
# Skip common false positives
if any(skip in title.lower() for skip in ['copyright', 'isbn', 'published', 'printed']):
continue
pairs.append((title, page))
# Require at least 5 entries and monotonic pages
if len(pairs) >= 5:
pages = [p for _, p in pairs]
if pages == sorted(pages):
logger.debug(f"TOC extraction found {len(pairs)} entries")
return pairs
return None
# ---------- Build entries with ends, apply smoothing
def _entries_from_starts(starts: List[Tuple[str, int, int]], page_count: int, source: str = "headings") -> List[Dict[str, Any]]:
"""
Build entries from start points with computed end pages.
starts: [(title, page, level)]
"""
entries = []
base_confidence = 0.8 if source == "headings" else 0.75
for i, (title, start, level) in enumerate(starts):
end = (starts[i + 1][1] - 1) if i + 1 < len(starts) else page_count
entries.append({
"id": f"sec{i + 1:02d}",
"title": title,
"level": level,
"start_page": int(start),
"end_page": int(end),
"source": source,
"confidence": base_confidence
})
# Merge tiny sections (< 3 pages) into previous
merged = []
for e in entries:
section_size = e["end_page"] - e["start_page"] + 1
if merged and section_size < 3:
# Merge into previous section
merged[-1]["end_page"] = e["end_page"]
merged[-1]["title"] += " / " + e["title"]
merged[-1]["confidence"] *= 0.95 # Slight confidence penalty for merging
else:
merged.append(e)
return merged
def _entries_from_pairs(pairs: List[Tuple[str, int]], page_count: int, source: str = "outline") -> List[Dict[str, Any]]:
"""
Build entries from (title, start_page) pairs.
"""
entries = []
base_confidence = 0.95 if source == "outline" else (0.8 if source == "toc" else 0.75)
for i, (title, start) in enumerate(pairs):
end = (pairs[i + 1][1] - 1) if i + 1 < len(pairs) else page_count
entries.append({
"id": f"sec{i + 1:02d}",
"title": title,
"level": 1,
"start_page": int(start),
"end_page": int(end),
"source": source,
"confidence": base_confidence
})
# Apply same merging logic for tiny sections
merged = []
for e in entries:
section_size = e["end_page"] - e["start_page"] + 1
if merged and section_size < 3:
merged[-1]["end_page"] = e["end_page"]
merged[-1]["title"] += " / " + e["title"]
merged[-1]["confidence"] *= 0.95
else:
merged.append(e)
return merged
# ---------- Post-processing normalization
def _normalize_entries(entries: List[Dict[str, Any]], page_count: int) -> List[Dict[str, Any]]:
"""Normalize entries to ensure:
- coverage from page 1
- 1 <= start_page <= end_page <= page_count
- strictly increasing, non-overlapping ranges
- fill initial gap with a synthetic front matter section if needed
"""
if not entries:
return entries
# Sanitize and sort by start_page
safe: List[Dict[str, Any]] = []
for e in entries:
try:
s = int(e.get("start_page", 1))
t = int(e.get("end_page", s))
except Exception:
continue
s = max(1, min(s, page_count))
t = max(1, min(t, page_count))
if t < s:
t = s
ne = dict(e)
ne["start_page"], ne["end_page"] = s, t
safe.append(ne)
safe.sort(key=lambda x: (x["start_page"], x.get("level", 1)))
# De-overlap by adjusting starts; ensure monotonic ranges
normalized: List[Dict[str, Any]] = []
for e in safe:
if not normalized:
normalized.append(e)
continue
prev = normalized[-1]
if e["start_page"] <= prev["end_page"]:
e["start_page"] = prev["end_page"] + 1
if e["start_page"] > page_count:
continue
if e["end_page"] < e["start_page"]:
e["end_page"] = e["start_page"]
e["end_page"] = min(e["end_page"], page_count)
normalized.append(e)
# Insert synthetic front matter if first start > 1
if normalized and normalized[0]["start_page"] > 1:
front = {
"id": "sec00",
"title": "Front matter",
"level": 1,
"start_page": 1,
"end_page": normalized[0]["start_page"] - 1,
"source": "synthetic",
"confidence": 0.6,
}
normalized.insert(0, front)
# Ensure last section ends at page_count
if normalized and normalized[-1]["end_page"] < page_count:
normalized[-1]["end_page"] = page_count
# Renumber ids sequentially
out: List[Dict[str, Any]] = []
for idx, e in enumerate(normalized, start=1):
ne = dict(e)
ne["id"] = f"sec{idx:02d}"
out.append(ne)
return out
# ---------- Main entry point
def create_split_map_for_file(file_id: str) -> Dict[str, Any]:
"""
Create split_map.json for a file using waterfall strategy:
1. PDF outline (best)
2. Docling headings (from existing or limited fallback)
3. Tika TOC parsing
4. Fixed windows (fallback)
"""
logger.info(f"Creating split_map for file_id={file_id}")
client = SupabaseServiceRoleClient()
storage = StorageAdmin()
# 1) Lookup file row & bucket
fr = client.supabase.table('files').select('id,bucket,cabinet_id,name,path,mime_type').eq('id', file_id).single().execute()
file_row = fr.data or {}
bucket = file_row.get('bucket')
cabinet_id = file_row.get('cabinet_id')
# 2) Find artefacts
arts = client.supabase.table('document_artefacts') \
.select('*').eq('file_id', file_id).order('created_at', desc=True).execute().data or []
def find_art(t):
for a in arts:
if a.get('type') == t:
return a
return None
a_pdf = find_art('document_pdf') # if converted to PDF
a_tika = find_art('tika_json')
a_noocr = find_art('docling_noocr_json')
a_fm = find_art('docling_frontmatter_json')
# 3) Load JSON/text
tika_json = _load_artefact_json(storage, bucket, a_tika['rel_path']) if a_tika else None
docling_noocr = _load_artefact_json(storage, bucket, a_noocr['rel_path']) if a_noocr else None
docling_fm = _load_artefact_json(storage, bucket, a_fm['rel_path']) if a_fm else None
# Get page count
page_count = _page_count_from_tika(tika_json or {}) or 100 # reasonable default
# Get PDF bytes for outline extraction
pdf_bytes = None
processing_bytes = None
processing_mime = None
if a_pdf:
# Use converted PDF
pdf_bytes = storage.download_file(bucket, a_pdf['rel_path'])
processing_bytes = pdf_bytes
processing_mime = 'application/pdf'
else:
# Check if original file is PDF
if file_row.get('mime_type') == 'application/pdf':
pdf_bytes = storage.download_file(bucket, file_row['path'])
processing_bytes = pdf_bytes
processing_mime = 'application/pdf'
# 4) Try methods in waterfall order
method = "fixed"
confidence = 0.2
entries: List[Dict[str, Any]] = []
# A) PDF Outline/Bookmarks (best)
if pdf_bytes and not entries:
logger.debug(f"Trying outline extraction for file_id={file_id}")
pairs = _try_outline(pdf_bytes)
if pairs:
entries = _entries_from_pairs(pairs, page_count, source="outline")
method, confidence = "outline", 0.95
logger.info(f"Split map: outline method found {len(entries)} sections")
# B) Headings from existing Docling JSON
if not entries:
logger.debug(f"Trying headings from existing Docling JSON for file_id={file_id}")
# Try no-OCR first, then frontmatter
for docling_json, source_name in [(docling_noocr, "noocr"), (docling_fm, "frontmatter")]:
if docling_json:
starts = _try_headings(docling_json)
if starts:
entries = _entries_from_starts(starts, page_count, source="headings")
method, confidence = "headings", 0.8
logger.info(f"Split map: headings method ({source_name}) found {len(entries)} sections")
break
# B2) Headings fallback with limited Docling call (if we have processing bytes)
if not entries and processing_bytes and processing_mime:
logger.debug(f"Trying headings fallback with limited Docling call for file_id={file_id}")
starts = _try_headings_fallback(file_id, cabinet_id, bucket, processing_bytes, processing_mime, page_count)
if starts:
entries = _entries_from_starts(starts, page_count, source="headings")
method, confidence = "headings", 0.75 # Slightly lower confidence for fallback
logger.info(f"Split map: headings fallback found {len(entries)} sections")
# C) TOC from Tika text
if not entries and tika_json:
logger.debug(f"Trying TOC extraction from Tika text for file_id={file_id}")
# Try common Tika text keys
text = tika_json.get("X-TIKA:content") or tika_json.get("content") or ""
pairs = _try_toc_text(text)
if pairs:
entries = _entries_from_pairs(pairs, page_count, source="toc")
method, confidence = "toc", 0.75
logger.info(f"Split map: TOC method found {len(entries)} sections")
# D) Fixed windows (fallback)
if not entries:
logger.info(f"Using fixed window fallback for file_id={file_id}")
step = max(10, min(20, page_count // 10)) # Adaptive step size
pairs = []
for i in range(1, page_count + 1, step):
end_page = min(i + step - 1, page_count)
title = f"Pages {i}-{end_page}" if i != end_page else f"Page {i}"
pairs.append((title, i))
entries = _entries_from_pairs(pairs, page_count, source="fixed")
method, confidence = "fixed", 0.2
logger.info(f"Split map: fixed method created {len(entries)} sections")
# 5) Normalize entries and build split_map.json
entries = _normalize_entries(entries, page_count)
split_map = {
"version": 1,
"file_id": file_id,
"source_pdf_artefact_id": a_pdf['id'] if a_pdf else None,
"sources": {
"docling_noocr_json": a_noocr['id'] if a_noocr else None,
"docling_frontmatter_json": a_fm['id'] if a_fm else None,
"tika_json": a_tika['id'] if a_tika else None
},
"method": method,
"confidence": confidence,
"page_count": page_count,
"entries": entries,
"created_at": _now_iso(),
"notes": f"auto-generated using {method} method; user can edit in Split Marker UI"
}
# 6) Store as artefact
artefact_id = str(uuid.uuid4())
rel_path = f"{cabinet_id}/{file_id}/{artefact_id}/split_map.json"
storage.upload_file(
bucket,
rel_path,
json.dumps(split_map, ensure_ascii=False, indent=2).encode("utf-8"),
"application/json",
upsert=True
)
# Enhanced metadata for UI display
enhanced_extra = {
"method": method,
"confidence": confidence,
"entries_count": len(entries),
"display_name": "Document Structure Map",
"bundle_label": "Split Map",
"section_title": "Document Structure Map",
"page_count": page_count,
"bundle_type": "split_map_json",
"processing_mode": "document_analysis",
"pipeline": "structure_analysis",
"is_structure_map": True,
"ui_category": "document_analysis",
"ui_order": 2,
"description": f"Document section boundaries identified using {method} method with {confidence:.1%} confidence ({len(entries)} sections)",
"viewer_type": "json"
}
client.supabase.table('document_artefacts').insert({
"id": artefact_id,
"file_id": file_id,
"type": "split_map_json",
"rel_path": rel_path,
"extra": enhanced_extra,
"status": "completed"
}).execute()
logger.info(f"Split map stored: file_id={file_id}, method={method}, confidence={confidence:.2f}, entries={len(entries)}")
return split_map
routers/database/init/curriculum.py
+6 -7
View File
@@ -31,20 +31,19 @@ async def upload_curriculum(file: UploadFile = File(...), db_name: str = Form(..
async def upload_school_curriculum(
file: UploadFile = File(...),
db_name: str = Form(...),
school_uuid: str = Form(...),
school_uuid_string: str = Form(...),
school_name: str = Form(...),
school_website: str = Form(...),
school_path: str = Form(...)
school_node_storage_path: str = Form(...)
):
if file.content_type != 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':
return {"status": "Error", "message": "Invalid file format"}
logging.info(f"Uploading curriculum for school {school_name} in {db_name}")
dataframes = xl.create_dataframes_from_fastapiuploadfile(file)
school_node = SchoolNode(
unique_id=f'School_{school_uuid}',
school_uuid=school_uuid,
school_name=school_name,
school_website=school_website,
path=school_path
uuid_string=school_uuid_string,
name=school_name,
website=school_website,
node_storage_path=school_node_storage_path
)
return init_school_curriculum.create_curriculum(db_name, dataframes, school_node)
routers/database/init/entity_init.py
+50 -46
View File
@@ -23,9 +23,9 @@ def initialise_schools_from_config():
"""Initialize a school with the configuration provided from env variables
"""
default_config = {
"school_uuid": "kevlarai",
"school_name": "KevlarAI School",
"school_website": "https://kevlarai.com",
"uuid_string": "kevlarai-dev",
"name": "KevlarAI School",
"website": "https://kevlarai.com",
"timetable_file": "kevlarai_data/kevlarai_timetable.xlsx",
"curriculum_file": "kevlarai_data/kevlarai_curriculum.xlsx"
}
@@ -33,10 +33,10 @@ def initialise_schools_from_config():
# school_config_str = os.getenv("SCHOOL_CONFIG") # TODO: Implement this
school_config = default_config
db_name = f"cc.institutes.{school_config['school_uuid']}"
db_name = f"cc.institutes.{school_config['uuid_string']}"
curriculum_db_name = f"{db_name}.curriculum"
logger.info(f"Creating database for {school_config['school_name']} using db_name: {db_name}")
logger.info(f"Creating database for {school_config['name']} using db_name: {db_name}")
driver = driver_tools.get_driver()
if driver is None:
logger.error("Failed to connect to Neo4j")
@@ -54,7 +54,7 @@ def initialise_schools_from_config():
# Add filesystem path debugging
base_path = os.getenv("NODE_FILESYSTEM_PATH")
schools_path = os.path.join(base_path, "schools")
school_path = os.path.join(schools_path, f"cc.institutes.{school_config['school_uuid']}")
school_path = os.path.join(schools_path, f"cc.institutes.{school_config['uuid_string']}")
logger.debug("Filesystem paths:", {
"base_path": base_path,
@@ -70,29 +70,34 @@ def initialise_schools_from_config():
})
# Create database entry for school without timetable or curriculum
logger.info(f"Creating school entry for {school_config['school_name']} in database {db_name} without timetable or curriculum")
logger.info(f"Creating school entry for {school_config['name']} in database {db_name} without timetable or curriculum")
school_uuid_string=school_config["uuid_strig"]
school_name=school_config["name"]
school_website=school_config["website"]
result = init_school.create_school(
db_name=db_name,
school_uuid=school_config["school_uuid"],
school_name=school_config["school_name"],
school_website=school_config["school_website"]
school_type="development",
uuid_string=school_uuid_string,
name=school_name,
website=school_website
)
logger.success(f"{school_config['school_name']} school entry created successfully")
logger.success(f"{school_config['name']} school entry created successfully")
# Create school node from result
school_node = result['school_node']
refreshed_school_node = SchoolNode(
unique_id=school_node.unique_id,
school_uuid=school_node.school_uuid,
school_name=school_node.school_name,
school_website=school_node.school_website,
path=school_node.path
school_type="development",
uuid_string=school_node.uuid_string,
name=school_node.name,
website=school_node.website
)
# Create timetable entries for school from Excel file
timetable_file = os.path.join(os.getenv("BACKEND_INIT_PATH"), school_config["timetable_file"])
logger.info(f"Creating timetable entries for {school_config['school_name']} using timetable file: {timetable_file}.")
logger.info(f"Creating timetable entries for {school_config['name']} using timetable file: {timetable_file}.")
school_timetable_dataframes = xl.create_dataframes(timetable_file)
@@ -107,7 +112,7 @@ def initialise_schools_from_config():
curriculum_file = os.path.join(os.getenv("BACKEND_INIT_PATH"), school_config["curriculum_file"])
school_curriculum_dataframes = xl.create_dataframes(curriculum_file)
logger.info(f"Creating curriculum entries for {school_config['school_name']} using curriculum file: {curriculum_file}.")
logger.info(f"Creating curriculum entries for {school_config['name']} using curriculum file: {curriculum_file}.")
init_school_curriculum.create_curriculum(
dataframes=school_curriculum_dataframes,
db_name=db_name,
@@ -123,18 +128,18 @@ async def create_user(
user_type: str = Form(...),
user_name: str = Form(...),
user_email: str = Form(...),
school_uuid: str = Form(None),
school_uuid_string: str = Form(None),
school_name: str = Form(None),
school_website: str = Form(None),
school_path: str = Form(None),
school_node_storage_path: str = Form(None),
worker_data: str = Form(None)
):
logger.info(f"Creating user with user_id: {user_id}, user_type: {user_type}, user_name: {user_name}, user_email: {user_email}")
if school_uuid:
logger.info(f"School UUID provided: {school_uuid}")
if school_uuid_string:
logger.info(f"School UUID string provided: {school_uuid_string}")
else:
logger.info(f"No school UUID provided")
logger.info(f"No school UUID string provided")
if school_name:
logger.info(f"School name provided: {school_name}")
@@ -146,8 +151,8 @@ async def create_user(
else:
logger.info(f"No school website provided")
if school_path:
logger.info(f"School path provided: {school_path}")
if school_node_storage_path:
logger.info(f"School path provided: {school_node_storage_path}")
else:
logger.info(f"No school path provided")
@@ -169,13 +174,12 @@ async def create_user(
# Create school node if school data provided
school_node = None
if all([school_uuid, school_name, school_website, school_path]):
if all([school_uuid_string, school_name, school_website, school_node_storage_path]):
school_node = SchoolNode(
unique_id=f'School_{school_uuid}',
school_uuid=school_uuid,
school_name=school_name,
school_website=school_website,
path=school_path
uuid_string=school_uuid_string,
name=school_name,
website=school_website,
node_storage_path=school_node_storage_path
)
# Create user with single database reference
@@ -219,23 +223,23 @@ async def create_schools():
@router.post("/create-department")
async def create_department(
db_name: str = Form(...),
unique_id: str = Form(...),
uuid_string: str = Form(...),
department_name: str = Form(...),
department_code: str = Form(...),
path: str = Form(...)
department_node_storage_path: str = Form(...)
):
if db_name is None or unique_id is None or department_name is None or department_code is None or path is None:
logging.error(f"Invalid department data: {db_name}, {unique_id}, {department_name}, {department_code}, {path}")
if db_name is None or uuid_string is None or department_name is None or department_code is None or department_node_storage_path is None:
logging.error(f"Invalid department data: {db_name}, {uuid_string}, {department_name}, {department_code}, {department_node_storage_path}")
raise HTTPException(status_code=400, detail="Invalid department data")
department = DepartmentNode(
unique_id=unique_id,
uuid_string=uuid_string,
department_name=department_name,
department_code=department_code,
path=path
node_storage_path=department_node_storage_path
)
logger.info(f"Creating department {department_name} with unique_id {unique_id}")
logger.info(f"Creating department {department_name} with uuid_string {uuid_string}")
try:
result = init_school.create_department(db_name, department)
return JSONResponse(content={"status": "success", "data": result})
@@ -246,20 +250,20 @@ async def create_department(
@router.post("/create-class")
async def create_class(
db_name: str = Form(...),
unique_id: str = Form(...),
uuid_string: str = Form(...),
subject_class_code: str = Form(...),
year_group: str = Form(...),
subject: str = Form(...),
subject_code: str = Form(...),
path: str = Form(...)
subject_node_storage_path: str = Form(...)
):
subject_class_node = SubjectClassNode(
unique_id=unique_id,
uuid_string=uuid_string,
subject_class_code=subject_class_code,
year_group=year_group,
subject=subject,
subject_code=subject_code,
path=path
node_storage_path=subject_node_storage_path
)
# Implementation for creating a class
pass
@@ -267,14 +271,14 @@ async def create_class(
@router.post("/create-room")
async def create_room(
db_name: str = Form(...),
room_unique_id: str = Form(...),
room_uuid_string: str = Form(...),
room_code: str = Form(...),
path: str = Form(...)
room_node_storage_path: str = Form(...)
):
room = RoomNode(
room_unique_id=room_unique_id,
room_uuid_string=room_uuid_string,
room_code=room_code,
path=path
node_storage_path=room_node_storage_path
)
# Implementation for creating a room
pass
routers/database/init/schools.py
+10 -12
View File
@@ -15,7 +15,7 @@ logging = logger.get_logger(
from fastapi import APIRouter, File, UploadFile, Form, HTTPException, BackgroundTasks
import pandas as pd
import modules.database.tools.neo4j_driver_tools as driver
from modules.database.tools.neo4j_session_tools import get_node_by_unique_id
from modules.database.tools.neo4j_session_tools import get_node_by_uuid_string
import modules.database.init.init_school_timetable as init_school_timetable
import modules.database.init.init_worker_timetable as init_worker_timetable
from modules.database.schemas.nodes.schools.schools import SchoolNode
@@ -28,18 +28,16 @@ router = APIRouter()
async def upload_school_timetable(
file: UploadFile = File(...),
db_name: str = Form(...),
unique_id: str = Form(...),
school_uuid: str = Form(...),
school_uuid_string: str = Form(...),
school_name: str = Form(...),
school_website: str = Form(...),
path: str = Form(...)
school_node_storage_path: str = Form(...)
):
school_node = SchoolNode(
unique_id=unique_id,
school_uuid=school_uuid,
school_name=school_name,
school_website=school_website,
path=path
uuid_string=school_uuid_string,
name=school_name,
website=school_website,
node_storage_path=school_node_storage_path
)
if file.content_type != 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':
return {"status": "Error", "message": "Invalid file format"}
@@ -91,13 +89,13 @@ async def process_worker_timetable(file_content, worker_node_data):
timetable_df = pd.read_excel(BytesIO(file_content))
# Get the school version of the worker node
logging.info(f"Getting school worker node for {worker_node_data['unique_id']} from {worker_node_data['worker_db_name']}")
logging.info(f"Getting school worker node for {worker_node_data['uuid_string']} from {worker_node_data['worker_db_name']}")
with neo_driver.session(database=worker_node_data['worker_db_name']) as neo_session:
school_worker_node = get_node_by_unique_id(session=neo_session, unique_id=worker_node_data['unique_id'])
school_worker_node = get_node_by_uuid_string(session=neo_session, uuid_string=worker_node_data['uuid_string'])
if school_worker_node is None:
error_msg = f"School worker node not found for unique_id: {worker_node_data['unique_id']}"
error_msg = f"School worker node not found for uuid_string: {worker_node_data['uuid_string']}"
logging.error(error_msg)
raise Exception(error_msg)
routers/database/init/timetables.py
+14 -16
View File
@@ -15,7 +15,7 @@ logging = logger.get_logger(
from fastapi import APIRouter, File, UploadFile, Form, HTTPException, BackgroundTasks
import pandas as pd
import modules.database.tools.neo4j_driver_tools as driver
from modules.database.tools.neo4j_session_tools import get_node_by_unique_id
from modules.database.tools.neo4j_session_tools import get_node_by_uuid_string
import modules.database.init.init_school_timetable as init_school_timetable
import modules.database.init.init_worker_timetable as init_worker_timetable
from modules.database.schemas.nodes.users import UserNode
@@ -31,18 +31,16 @@ router = APIRouter()
async def upload_school_timetable(
file: UploadFile = File(...),
db_name: str = Form(...),
unique_id: str = Form(...),
school_uuid: str = Form(...),
school_uuid_string: str = Form(...),
school_name: str = Form(...),
school_website: str = Form(...),
path: str = Form(...)
school_node_storage_path: str = Form(...)
):
school_node = SchoolNode(
unique_id=unique_id,
school_uuid=school_uuid,
school_name=school_name,
school_website=school_website,
path=path
uuid_string=school_uuid_string,
name=school_name,
website=school_website,
node_storage_path=school_node_storage_path
)
if file.content_type != 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':
return {"status": "Error", "message": "Invalid file format"}
@@ -101,13 +99,13 @@ async def process_worker_timetable(file_content, user_node_data, worker_node_dat
timetable_df = pd.read_excel(BytesIO(file_content))
# Get the school version of the worker node
logging.info(f"Getting school worker node for {worker_node_data['unique_id']} from {worker_node_data['worker_db_name']}")
logging.info(f"Getting school worker node for {worker_node_data['uuid_string']} from {worker_node_data['worker_db_name']}")
with neo_driver.session(database=worker_node_data['worker_db_name']) as neo_session:
school_worker_node = get_node_by_unique_id(session=neo_session, unique_id=worker_node_data['unique_id'])
school_worker_node = get_node_by_uuid_string(session=neo_session, uuid_string=worker_node_data['uuid_string'])
if school_worker_node is None:
error_msg = f"School worker node not found for unique_id: {worker_node_data['unique_id']}"
error_msg = f"School worker node not found for uuid_string: {worker_node_data['uuid_string']}"
logging.error(error_msg)
raise Exception(error_msg)
@@ -127,23 +125,23 @@ async def process_worker_timetable(file_content, user_node_data, worker_node_dat
# Create TeacherNode from worker_node_data
user_worker_node = TeacherNode(
unique_id=worker_node_data['unique_id'],
uuid_string=worker_node_data['uuid_string'],
teacher_code=worker_node_data['teacher_code'],
teacher_name_formal=worker_node_data['teacher_name_formal'],
teacher_email=worker_node_data['teacher_email'],
path=worker_node_data['path'],
node_storage_path=worker_node_data['node_storage_path'],
worker_db_name=worker_node_data['worker_db_name'],
user_db_name=worker_node_data['user_db_name']
)
# Create user node
user_node = UserNode(
unique_id=user_node_data['unique_id'],
uuid_string=user_node_data['uuid_string'],
user_id=user_node_data['user_id'],
user_type=user_node_data['user_type'],
user_name=user_node_data['user_name'],
user_email=user_node_data['user_email'],
path=user_node_data['path'],
node_storage_path=user_node_data['node_storage_path'],
worker_node_data=user_node_data['worker_node_data']
)
routers/database/tools/calendar_structure_router.py
+23 -23
View File
@@ -27,29 +27,29 @@ async def get_calendar_structure(db_name: str) -> Dict[str, Any]:
// Collect all nodes with dates converted to strings
RETURN {
years: collect(DISTINCT {
id: y.unique_id,
path: y.path,
id: y.uuid_string,
path: y.node_storage_path,
date: toString(y.date),
__primarylabel__: 'CalendarYear'
}),
months: collect(DISTINCT {
id: m.unique_id,
path: m.path,
id: m.uuid_string,
path: m.node_storage_path,
date: toString(m.date),
__primarylabel__: 'CalendarMonth'
}),
weeks: collect(DISTINCT {
id: w.unique_id,
path: w.path,
id: w.uuid_string,
path: w.node_storage_path,
date: toString(w.date),
__primarylabel__: 'CalendarWeek'
}),
days: collect(DISTINCT {
id: d.unique_id,
path: d.path,
id: d.uuid_string,
path: d.node_storage_path,
date: toString(d.date),
week_id: w.unique_id,
month_id: m.unique_id,
week_id: w.uuid_string,
month_id: m.uuid_string,
__primarylabel__: 'CalendarDay'
})
} as structure
@@ -98,11 +98,11 @@ async def get_calendar_days(db_name: str, start_date: str, end_date: str) -> Dic
OPTIONAL MATCH (w:CalendarWeek)-[:WEEK_INCLUDES_DAY]->(d)
OPTIONAL MATCH (m:CalendarMonth)-[:MONTH_INCLUDES_DAY]->(d)
RETURN {
id: d.unique_id,
path: d.path,
id: d.uuid_string,
path: d.node_storage_path,
date: d.date,
week_id: w.unique_id,
month_id: m.unique_id,
week_id: w.uuid_string,
month_id: m.uuid_string,
__primarylabel__: 'CalendarDay'
} as day
ORDER BY d.date
@@ -132,10 +132,10 @@ async def get_calendar_weeks(db_name: str, start_date: str, end_date: str) -> Di
WHERE date(w.date) >= date($start_date) AND date(w.date) <= date($end_date)
WITH w, collect(d) as days
RETURN {
id: w.unique_id,
path: w.path,
id: w.uuid_string,
path: w.node_storage_path,
date: w.date,
day_ids: [day in days | day.unique_id],
day_ids: [day in days | day.uuid_string],
__primarylabel__: 'CalendarWeek'
} as week
ORDER BY w.date
@@ -165,10 +165,10 @@ async def get_calendar_months(db_name: str, start_date: str, end_date: str) -> D
WHERE date(m.date) >= date($start_date) AND date(m.date) <= date($end_date)
WITH m, collect(d) as days
RETURN {
id: m.unique_id,
path: m.path,
id: m.uuid_string,
path: m.node_storage_path,
date: m.date,
day_ids: [day in days | day.unique_id],
day_ids: [day in days | day.uuid_string],
__primarylabel__: 'CalendarMonth'
} as month
ORDER BY m.date
@@ -197,10 +197,10 @@ async def get_calendar_years(db_name: str) -> Dict[str, Any]:
MATCH (y:CalendarYear)-[:YEAR_INCLUDES_MONTH]->(m:CalendarMonth)
WITH y, collect(m) as months
RETURN {
id: y.unique_id,
path: y.path,
id: y.uuid_string,
path: y.node_storage_path,
date: y.date,
month_ids: [month in months | month.unique_id],
month_ids: [month in months | month.uuid_string],
__primarylabel__: 'CalendarYear'
} as year
ORDER BY y.date
routers/database/tools/default_nodes_router.py
+37 -3
View File
@@ -47,7 +47,7 @@ def get_default_node_week(db_name: str) -> Dict[str, Any]:
return {
"status": "success",
"node": {
"id": node["unique_id"],
"id": node["uuid_string"],
"path": node["path"],
"type": "CalendarWeek",
"label": node.get("title", "Calendar Week"),
@@ -81,7 +81,7 @@ def get_default_node_month(db_name: str) -> Dict[str, Any]:
return {
"status": "success",
"node": {
"id": node["unique_id"],
"id": node["uuid_string"],
"path": node["path"],
"type": "CalendarMonth",
"label": node.get("title", "Calendar Month"),
@@ -89,6 +89,39 @@ def get_default_node_month(db_name: str) -> Dict[str, Any]:
}
}
@router.get("/debug-list-nodes")
async def debug_list_nodes(db_name: str) -> Dict[str, Any]:
"""Debug endpoint to list all nodes in a database."""
try:
with driver_tools.get_session(database=db_name) as session:
query = """
MATCH (n)
RETURN labels(n) as labels, n.uuid_string as uuid, n.user_name as name, n.cc_username as username
LIMIT 20
"""
result = session.run(query)
nodes = []
for record in result:
nodes.append({
"labels": list(record["labels"]),
"uuid": record["uuid"],
"name": record["name"],
"username": record["username"]
})
return {
"status": "success",
"db_name": db_name,
"node_count": len(nodes),
"nodes": nodes
}
except Exception as e:
return {
"status": "error",
"db_name": db_name,
"error": str(e)
}
@router.get("/get-default-node/{context}")
async def get_default_node(context: str, db_name: str, base_context: str | None = None) -> Dict[str, Any]:
"""Get the default node for a given context."""
@@ -244,8 +277,9 @@ async def get_default_node(context: str, db_name: str, base_context: str | None
return {
"status": "success",
"node": {
"id": node["unique_id"],
"id": node["uuid_string"],
"path": node["path"],
"node_storage_path": node.get("node_storage_path", node["path"]),
"type": list(node.labels)[0],
"label": node.get("title", ""),
"data": converted_data
routers/database/tools/get_events.py
+6 -6
View File
@@ -51,10 +51,10 @@ router = APIRouter()
@router.get("/get_teacher_timetable_events")
async def get_teacher_timetable_events(
unique_id: str,
uuid_string: str,
worker_db_name: str
):
logging.info(f"Getting timetable events for teacher {unique_id} from database {worker_db_name}")
logging.info(f"Getting timetable events for teacher {uuid_string} from database {worker_db_name}")
neo_driver = driver.get_driver(db_name=worker_db_name)
if neo_driver is None:
return {"status": "error", "message": "Failed to connect to the database"}
@@ -62,9 +62,9 @@ async def get_teacher_timetable_events(
try:
with neo_driver.session(database=worker_db_name) as neo_session:
query = """
MATCH (t:Teacher {unique_id: $unique_id})-[:TEACHER_HAS_TIMETABLE]->(tt:TeacherTimetable)
MATCH (t:Teacher {uuid_string: $uuid_string})-[:TEACHER_HAS_TIMETABLE]->(tt:TeacherTimetable)
-[:TIMETABLE_HAS_CLASS]->(sc:SubjectClass)-[:CLASS_HAS_LESSON]->(tl:TimetableLesson)
RETURN tl.unique_id as id,
RETURN tl.uuid_string as id,
tl.period_code as period_code,
COALESCE(sc.subject_class_code, 'Untitled Class') as subject_class,
tl.date as date,
@@ -72,7 +72,7 @@ async def get_teacher_timetable_events(
tl.end_time as end_time,
tl.path as path
"""
result = neo_session.run(query, unique_id=unique_id)
result = neo_session.run(query, uuid_string=uuid_string)
events = []
for record in result:
@@ -92,7 +92,7 @@ async def get_teacher_timetable_events(
"path": record['path']
}
})
logging.info(f"Found {len(events)} events for teacher {unique_id}")
logging.info(f"Found {len(events)} events for teacher {uuid_string}")
return {"status": "success", "events": events}
except Exception as e:
logging.error(f"Error fetching events: {str(e)}")
routers/database/tools/get_nodes.py
+49 -45
View File
@@ -25,18 +25,18 @@ from fastapi import APIRouter, HTTPException, Query
router = APIRouter()
@router.get("/get-node")
async def get_node(unique_id: str = Query(...), db_name: str = Query(...)):
logging.info(f"Getting node for {unique_id} from database {db_name}")
async def get_node(uuid_string: str = Query(...), db_name: str = Query(...)):
logging.info(f"Getting node for {uuid_string} from database {db_name}")
neo_driver = driver.get_driver(db_name=db_name)
if neo_driver is None:
return {"status": "error", "message": "Failed to connect to the database"}
try:
with neo_driver.session(database=db_name) as neo_session:
query = """
MATCH (n {unique_id: $unique_id})
MATCH (n {uuid_string: $uuid_string})
RETURN n
"""
result = neo_session.run(query, unique_id=unique_id)
result = neo_session.run(query, uuid_string=uuid_string)
record = result.single()
if record:
@@ -47,11 +47,23 @@ async def get_node(unique_id: str = Query(...), db_name: str = Query(...)):
try:
# Convert node based on its type
node_type = node_labels[0] if node_labels else "Unknown"
if node_type in globals():
node_class = globals()[f"{node_type}Node"]
logging.debug(f"Attempting to convert node of type: {node_type}")
logging.debug(f"Available node classes: {[name for name in globals() if name.endswith('Node')]}")
logging.debug(f"UserNode in globals: {'UserNode' in globals()}")
logging.debug(f"UserNode class: {UserNode}")
logging.debug(f"UserNode class name: {UserNode.__name__}")
# Try to find the node class
node_class_name = f"{node_type}Node"
if node_class_name in globals():
node_class = globals()[node_class_name]
logging.debug(f"Found node class: {node_class}")
node_object = node_class(**node_data)
node_dict = node_object.to_dict()
logging.debug(f"Successfully converted node to dict: {node_dict}")
else:
logging.warning(f"No node class found for type: {node_type} (looking for {node_class_name}), using raw data")
logging.debug(f"Available classes: {[name for name in globals() if 'Node' in name]}")
node_dict = node_data
return {
@@ -101,8 +113,8 @@ async def get_user_node(user_id: str = Query(...)):
driver.close_driver(neo_driver)
@router.get("/get-connected-nodes")
async def get_connected_nodes(unique_id: str = Query(...), db_name: str = Query(...)):
logging.info(f"Getting connected nodes for {unique_id} from database {db_name}")
async def get_connected_nodes(uuid_string: str = Query(...), db_name: str = Query(...)):
logging.info(f"Getting connected nodes for {uuid_string} from database {db_name}")
neo_driver = driver.get_driver(db_name=db_name)
if neo_driver is None:
return {"status": "error", "message": "Failed to connect to the database"}
@@ -110,11 +122,11 @@ async def get_connected_nodes(unique_id: str = Query(...), db_name: str = Query(
try:
with neo_driver.session(database=db_name) as neo_session:
query = """
MATCH (n {unique_id: $unique_id})
MATCH (n {uuid_string: $uuid_string})
OPTIONAL MATCH (n)-[]-(connected)
RETURN n, collect(connected) as connected_nodes
"""
result = neo_session.run(query, unique_id=unique_id)
result = neo_session.run(query, uuid_string=uuid_string)
record = result.single()
if record:
main_node = record['n']
@@ -171,15 +183,15 @@ async def get_connected_nodes(unique_id: str = Query(...), db_name: str = Query(
driver.close_driver(neo_driver)
@router.get("/get-user-connected-nodes")
async def get_user_connected_nodes(unique_id: str = Query(...)):
logging.info(f"Getting user adjacent nodes for node {unique_id}")
async def get_user_connected_nodes(uuid_string: str = Query(...)):
logging.info(f"Getting user adjacent nodes for node {uuid_string}")
db_name = os.getenv("NEO4J_DB_NAME", "cc.institutes.kevlarai") # TODO: This function needs to be able to take a db_name as a parameter
neo_driver = driver.get_driver(db_name=db_name)
if neo_driver is None:
raise HTTPException(status_code=500, detail="Failed to connect to the database")
try:
with neo_driver.session(database=db_name) as neo_session:
user_node_and_connected_nodes = session.get_node_by_unique_id_and_adjacent_nodes(neo_session, unique_id)
user_node_and_connected_nodes = session.get_node_by_uuid_string_and_adjacent_nodes(neo_session, uuid_string)
user_node = user_node_and_connected_nodes['node']
connected_nodes = user_node_and_connected_nodes['connected_nodes']
try:
@@ -251,15 +263,15 @@ async def get_user_connected_nodes(unique_id: str = Query(...)):
driver.close_driver(neo_driver)
@router.get("/get-worker-connected-nodes")
async def get_worker_connected_nodes(unique_id: str = Query(...)):
logging.info(f"Getting worker adjacent nodes for node {unique_id}")
async def get_worker_connected_nodes(uuid_string: str = Query(...)):
logging.info(f"Getting worker adjacent nodes for node {uuid_string}")
db_name = os.getenv("NEO4J_DB_NAME", "cc.institutes.kevlarai") # TODO: This function needs to be able to take a db_name as a parameter
neo_driver = driver.get_driver(db_name=db_name)
if neo_driver is None:
raise HTTPException(status_code=500, detail="Failed to connect to the database")
try:
with neo_driver.session(database=db_name) as neo_session:
node_and_connected_nodes = session.get_node_by_unique_id_and_adjacent_nodes(neo_session, unique_id)
node_and_connected_nodes = session.get_node_by_uuid_string_and_adjacent_nodes(neo_session, uuid_string)
worker_node = node_and_connected_nodes['node']
connected_nodes = node_and_connected_nodes['connected_nodes']
try:
@@ -319,9 +331,9 @@ async def get_worker_connected_nodes(unique_id: str = Query(...)):
driver.close_driver(neo_driver)
@router.get("/get-calendar-connected-nodes")
async def get_calendar_connected_nodes(unique_id: str = Query(...)):
async def get_calendar_connected_nodes(uuid_string: str = Query(...)):
db_name = os.getenv("NEO4J_DB_NAME", "cc.institutes.kevlarai")
logging.info(f"Getting connected nodes for calendar {unique_id} from database {db_name}")
logging.info(f"Getting connected nodes for calendar {uuid_string} from database {db_name}")
neo_driver = driver.get_driver(db_name=db_name)
if neo_driver is None:
return {"status": "error", "message": "Failed to connect to the database"}
@@ -330,11 +342,11 @@ async def get_calendar_connected_nodes(unique_id: str = Query(...)):
with neo_driver.session(database=db_name) as neo_session:
query = """
MATCH (n)
WHERE n.unique_id = $unique_id AND (n:Calendar OR n:CalendarYear OR n:CalendarMonth OR n:CalendarWeek OR n:CalendarDay OR n:CalendarTimeChunk)
WHERE n.uuid_string = $uuid_string AND (n:Calendar OR n:CalendarYear OR n:CalendarMonth OR n:CalendarWeek OR n:CalendarDay OR n:CalendarTimeChunk)
OPTIONAL MATCH (n)-[]-(connected)
RETURN n, collect(connected) as connected_nodes
"""
result = neo_session.run(query, unique_id=unique_id)
result = neo_session.run(query, uuid_string=uuid_string)
record = result.single()
if record:
calendar_node = record['n']
@@ -369,9 +381,9 @@ async def get_calendar_connected_nodes(unique_id: str = Query(...)):
driver.close_driver(neo_driver)
@router.get("/get-teacher-timetable-connected-nodes")
async def get_teacher_timetable_connected_nodes(unique_id: str = Query(...)):
async def get_teacher_timetable_connected_nodes(uuid_string: str = Query(...)):
db_name = os.getenv("NEO4J_DB_NAME", "cc.institutes.kevlarai")
logging.info(f"Getting connected nodes for teacher timetable {unique_id} from database {db_name}")
logging.info(f"Getting connected nodes for teacher timetable {uuid_string} from database {db_name}")
neo_driver = driver.get_driver(db_name=db_name)
if neo_driver is None:
return {"status": "error", "message": "Failed to connect to the database"}
@@ -379,11 +391,11 @@ async def get_teacher_timetable_connected_nodes(unique_id: str = Query(...)):
try:
with neo_driver.session(database=db_name) as neo_session:
query = """
MATCH (n:TeacherTimetable {unique_id: $unique_id})
MATCH (n:TeacherTimetable {uuid_string: $uuid_string})
OPTIONAL MATCH (n)-[]-(connected)
RETURN n, collect(connected) as connected_nodes
"""
result = neo_session.run(query, unique_id=unique_id)
result = neo_session.run(query, uuid_string=uuid_string)
record = result.single()
if record:
teacher_timetable_node = record['n']
@@ -422,9 +434,9 @@ async def get_teacher_timetable_connected_nodes(unique_id: str = Query(...)):
driver.close_driver(neo_driver)
@router.get("/get-school-timetable-connected-nodes")
async def get_school_timetable_connected_nodes(unique_id: str = Query(...)):
async def get_school_timetable_connected_nodes(uuid_string: str = Query(...)):
db_name = os.getenv("NEO4J_DB_NAME", "cc.institutes.kevlarai")
logging.info(f"Getting connected nodes for school timetable {unique_id} from database {db_name}")
logging.info(f"Getting connected nodes for school timetable {uuid_string} from database {db_name}")
neo_driver = driver.get_driver(db_name=db_name)
if neo_driver is None:
return {"status": "error", "message": "Failed to connect to the database"}
@@ -432,11 +444,11 @@ async def get_school_timetable_connected_nodes(unique_id: str = Query(...)):
try:
with neo_driver.session(database=db_name) as neo_session:
query = """
MATCH (n:SchoolTimetable {unique_id: $unique_id})
MATCH (n:SchoolTimetable {uuid_string: $uuid_string})
OPTIONAL MATCH (n)-[]-(connected)
RETURN n, collect(connected) as connected_nodes
"""
result = neo_session.run(query, unique_id=unique_id)
result = neo_session.run(query, uuid_string=uuid_string)
record = result.single()
if record:
school_timetable_node = record['n']
@@ -483,9 +495,9 @@ async def get_school_timetable_connected_nodes(unique_id: str = Query(...)):
driver.close_driver(neo_driver)
@router.get("/get-curriculum-connected-nodes")
async def get_curriculum_connected_nodes(unique_id: str = Query(...)):
async def get_curriculum_connected_nodes(uuid_string: str = Query(...)):
db_name = os.getenv("NEO4J_DB_NAME", "cc.institutes.kevlarai")
logging.info(f"Getting connected nodes for curriculum {unique_id} from database {db_name}")
logging.info(f"Getting connected nodes for curriculum {uuid_string} from database {db_name}")
neo_driver = driver.get_driver(db_name=db_name)
if neo_driver is None:
return {"status": "error", "message": "Failed to connect to the database"}
@@ -494,11 +506,11 @@ async def get_curriculum_connected_nodes(unique_id: str = Query(...)):
with neo_driver.session(database=db_name) as neo_session:
query = """
MATCH (n)
WHERE n.unique_id = $unique_id AND (n:PastoralStructure OR n:YearGroup OR n:CurriculumStructure OR n:KeyStage OR n:KeyStageSyllabus OR n:YearGroupSyllabus OR n:Subject OR n:Topic OR n:TopicLesson OR n:LearningStatement OR n:ScienceLab)
WHERE n.uuid_string = $uuid_string AND (n:PastoralStructure OR n:YearGroup OR n:CurriculumStructure OR n:KeyStage OR n:KeyStageSyllabus OR n:YearGroupSyllabus OR n:Subject OR n:Topic OR n:TopicLesson OR n:LearningStatement OR n:ScienceLab)
OPTIONAL MATCH (n)-[]-(connected)
RETURN n, collect(connected) as connected_nodes
"""
result = neo_session.run(query, unique_id=unique_id)
result = neo_session.run(query, uuid_string=uuid_string)
record = result.single()
if record:
curriculum_node = record['n']
@@ -533,26 +545,18 @@ async def get_curriculum_connected_nodes(unique_id: str = Query(...)):
driver.close_driver(neo_driver)
@router.get("/get-school-node")
async def get_school_node(school_uuid: str = Query(...)):
logging.info(f"Getting school node for school {school_uuid}...")
db_name = f"cc.institutes.{school_uuid}"
async def get_school_node(school_uuid_string: str = Query(...)):
logging.info(f"Getting school node for school {school_uuid_string}...")
db_name = f"cc.institutes.{school_uuid_string}"
neo_driver = driver.get_driver(db_name=db_name)
if neo_driver is None:
return {"status": "error", "message": "Failed to connect to the database"}
try:
with neo_driver.session(database=db_name) as neo_session:
nodes = session.find_nodes_by_label_and_properties(neo_session, "School", {"school_uuid": school_uuid})
nodes = session.find_nodes_by_label_and_properties(neo_session, "School", {"uuid_string": school_uuid_string})
if nodes:
school_node = nodes[0]
data = SchoolNode(
unique_id=school_node["unique_id"],
school_uuid=school_node["school_uuid"],
school_name=school_node["school_name"],
school_website=school_node["school_website"],
path=school_node["path"]
)
school_node_data = data.to_dict()
school_node_data = SchoolNode(**nodes[0]).to_dict()
return {"status": "success", "school_node": school_node_data, "school_node_raw": nodes}
else:
return {"status": "not_found", "message": "School node not found"}
routers/database/tools/get_nodes_and_edges.py
+4 -4
View File
@@ -89,8 +89,8 @@ async def get_all_nodes_and_edges():
@router.get("/get-connected-nodes-and-edges")
async def get_connected_nodes_and_edges(unique_id: str = Query(...), db_name: str = Query(...)):
logging.info(f"Getting connected nodes and edges for {unique_id} from database {db_name}")
async def get_connected_nodes_and_edges(uuid_string: str = Query(...), db_name: str = Query(...)):
logging.info(f"Getting connected nodes and edges for {uuid_string} from database {db_name}")
neo_driver = driver.get_driver(db_name=db_name)
if neo_driver is None:
return {"status": "error", "message": "Failed to connect to the database"}
@@ -98,11 +98,11 @@ async def get_connected_nodes_and_edges(unique_id: str = Query(...), db_name: st
try:
with neo_driver.session(database=db_name) as neo_session:
query = """
MATCH (n {unique_id: $unique_id})
MATCH (n {uuid_string: $uuid_string})
OPTIONAL MATCH (n)-[r]-(connected)
RETURN n, collect(connected) as connected_nodes, collect(r) as relationships
"""
result = neo_session.run(query, unique_id=unique_id)
result = neo_session.run(query, uuid_string=uuid_string)
record = result.single()
if record:
main_node = record['n']
routers/database/tools/tldraw_filesystem.py
+218 -51
View File
@@ -34,20 +34,24 @@ async def read_tldraw_user_node_file(user_node: UserNode):
logging.debug(f"Filesystem root path: {fs.root_path}")
# Handle path based on environment
if os.getenv("DEV_MODE") == "true":
# In dev mode, use the full system path from the node
if not user_node.path:
raise HTTPException(status_code=400, detail="Node path not found")
logging.debug(f"Using DEV_MODE path: {user_node.path}")
base_path = os.path.normpath(user_node.path)
# Use the path directly as provided - it represents the structure from root
if not user_node.node_storage_path:
raise HTTPException(status_code=400, detail="Node path not found")
# The path might already contain parts of the filesystem structure
# We need to construct the full path carefully
if user_node.node_storage_path.startswith("users/"):
# If path starts with users/, remove it since filesystem already has users/ structure
base_path = user_node.node_storage_path[6:] # Remove "users/" prefix
logging.debug(f"Removed 'users/' prefix, base_path is now: {base_path}")
else:
# In prod mode, construct path using formatted email
logging.warning(f"Using db_name as base path not ready in prod: {db_name}")
base_path = formatted_email
base_path = user_node.node_storage_path
logging.debug(f"No 'users/' prefix found, using path as-is: {base_path}")
base_path = os.path.normpath(base_path)
logging.debug(f"Using base path: {base_path}")
# Construct final path including tldraw file
logging.debug(f"Base path: {base_path}")
file_path = os.path.join(base_path, "tldraw_file.json")
logging.debug(f"File path: {file_path}")
file_location = os.path.normpath(os.path.join(fs.root_path, file_path))
@@ -68,8 +72,30 @@ async def read_tldraw_user_node_file(user_node: UserNode):
logging.error(f"Error reading file: {e}")
raise HTTPException(status_code=500, detail="Error reading file")
else:
logging.debug(f"File does not exist: {file_location}")
raise HTTPException(status_code=404, detail="File not found")
# Check if directory exists
directory_location = os.path.dirname(file_location)
if os.path.exists(directory_location):
logging.debug(f"Directory exists but file doesn't, creating default tldraw file at: {file_location}")
try:
# Create default tldraw content
default_tldraw_content = create_default_tldraw_content()
# Ensure directory exists (should already exist, but just in case)
os.makedirs(directory_location, exist_ok=True)
# Write the default file
with open(file_location, "w") as file:
json.dump(default_tldraw_content, file, indent=4)
logging.info(f"Default tldraw file created at: {file_location}")
return default_tldraw_content
except Exception as e:
logging.error(f"Error creating default tldraw file: {e}")
raise HTTPException(status_code=500, detail="Error creating default tldraw file")
else:
logging.debug(f"Neither directory nor file exists: {directory_location}")
raise HTTPException(status_code=404, detail="Directory not found")
@router.post("/set_tldraw_user_node_file")
async def set_tldraw_user_node_file(user_node: UserNode, data: Dict):
@@ -81,15 +107,22 @@ async def set_tldraw_user_node_file(user_node: UserNode, data: Dict):
fs = ClassroomCopilotFilesystem(db_name=db_name, init_run_type="user")
# Handle path based on environment
if os.getenv("ENVIRONMENT") == "dev":
# In dev mode, use the full system path from the node
if not user_node.path:
raise HTTPException(status_code=400, detail="Node path not found")
base_path = os.path.normpath(user_node.path)
# Use the path directly as provided - it represents the structure from root
if not user_node.node_storage_path:
raise HTTPException(status_code=400, detail="Node path not found")
# The path might already contain parts of the filesystem structure
# We need to construct the full path carefully
if user_node.node_storage_path.startswith("users/"):
# If path starts with users/, remove it since filesystem already has users/ structure
base_path = user_node.node_storage_path[6:] # Remove "users/" prefix
logging.debug(f"Removed 'users/' prefix, base_path is now: {base_path}")
else:
# In prod mode, construct path using formatted email
base_path = formatted_email
base_path = user_node.node_storage_path
logging.debug(f"No 'users/' prefix found, using path as-is: {base_path}")
base_path = os.path.normpath(base_path)
logging.debug(f"Using base path: {base_path}")
# Construct final path including tldraw file
file_path = os.path.join(base_path, "tldraw_file.json")
@@ -99,11 +132,15 @@ async def set_tldraw_user_node_file(user_node: UserNode, data: Dict):
try:
# Ensure directory exists
os.makedirs(os.path.dirname(file_location), exist_ok=True)
directory_location = os.path.dirname(file_location)
os.makedirs(directory_location, exist_ok=True)
logging.debug(f"Ensured directory exists: {directory_location}")
# Write the file
with open(file_location, "w") as file:
json.dump(data, file)
json.dump(data, file, indent=4)
logging.info(f"tldraw file successfully written to: {file_location}")
return {"status": "success"}
except Exception as e:
logging.error(f"Error writing file: {e}")
@@ -112,29 +149,38 @@ async def set_tldraw_user_node_file(user_node: UserNode, data: Dict):
@router.get("/get_tldraw_node_file")
async def read_tldraw_node_file(path: str, db_name: str):
logging.debug(f"Reading tldraw file for path: {path}")
logging.debug(f"Database name: {db_name}")
fs = ClassroomCopilotFilesystem(db_name=db_name, init_run_type="user")
logging.debug(f"Filesystem root path: {fs.root_path}")
# Handle path based on environment
if os.getenv("DEV_MODE") == "true":
# In dev mode, use the full system path from the node
if not path:
raise HTTPException(status_code=400, detail="Path not provided")
logging.debug(f"Using DEV_MODEpath: {path}")
base_path = os.path.normpath(path)
# Use the path directly as provided - it represents the structure from root
if not path:
raise HTTPException(status_code=400, detail="Path not provided")
# The path might already contain parts of the filesystem structure
# We need to construct the full path carefully
if path.startswith("users/"):
# If path starts with users/, remove it since filesystem already has users/ structure
base_path = path[6:] # Remove "users/" prefix
logging.debug(f"Removed 'users/' prefix, base_path is now: {base_path}")
else:
# In prod mode, construct path
logging.warning(f"Using db_name as base path not ready in prod: {db_name}")
base_path = db_name
base_path = path
logging.debug(f"No 'users/' prefix found, using path as-is: {base_path}")
base_path = os.path.normpath(base_path)
logging.debug(f"Using base path: {base_path}")
# Construct final path including tldraw file
logging.debug(f"Base path: {base_path}")
file_path = os.path.join(base_path, "tldraw_file.json")
logging.debug(f"File path: {file_path}")
file_location = os.path.normpath(os.path.join(fs.root_path, file_path))
logging.debug(f"File location: {file_location}")
logging.debug(f"Final file location: {file_location}")
# Debug: Check what directories exist
logging.debug(f"Checking if root path exists: {fs.root_path} - {os.path.exists(fs.root_path)}")
logging.debug(f"Checking if base path exists: {os.path.join(fs.root_path, base_path)} - {os.path.exists(os.path.join(fs.root_path, base_path))}")
logging.debug(f"Attempting to read file at: {file_location}")
@@ -151,8 +197,44 @@ async def read_tldraw_node_file(path: str, db_name: str):
logging.error(f"Error reading file: {e}")
raise HTTPException(status_code=500, detail="Error reading file")
else:
logging.debug(f"File does not exist: {file_location}")
raise HTTPException(status_code=404, detail="File not found")
# Check if directory exists
directory_location = os.path.dirname(file_location)
logging.debug(f"Checking if directory exists: {directory_location} - {os.path.exists(directory_location)}")
if os.path.exists(directory_location):
logging.debug(f"Directory exists but file doesn't, creating default tldraw file at: {file_location}")
try:
# Create default tldraw content
default_tldraw_content = create_default_tldraw_content()
# Ensure directory exists (should already exist, but just in case)
os.makedirs(directory_location, exist_ok=True)
# Write the default file
with open(file_location, "w") as file:
json.dump(default_tldraw_content, file, indent=4)
logging.info(f"Default tldraw file created at: {file_location}")
return default_tldraw_content
except Exception as e:
logging.error(f"Error creating default tldraw file: {e}")
raise HTTPException(status_code=500, detail="Error creating default tldraw file")
else:
logging.debug(f"Neither directory nor file exists: {directory_location}")
# List contents of parent directories to help debug
parent_dir = os.path.dirname(directory_location)
if os.path.exists(parent_dir):
logging.debug(f"Parent directory exists: {parent_dir}")
try:
contents = os.listdir(parent_dir)
logging.debug(f"Parent directory contents: {contents}")
except Exception as e:
logging.debug(f"Could not list parent directory contents: {e}")
else:
logging.debug(f"Parent directory does not exist: {parent_dir}")
raise HTTPException(status_code=404, detail="Directory not found")
@router.post("/set_tldraw_node_file")
async def set_tldraw_node_file(path: str, db_name: str, data: Dict):
@@ -162,22 +244,25 @@ async def set_tldraw_node_file(path: str, db_name: str, data: Dict):
logging.debug(f"Filesystem root path: {fs.root_path}")
# Handle path based on environment
if os.getenv("DEV_MODE") == "true":
# In dev mode, use the full system path from the node
if not path:
raise HTTPException(status_code=400, detail="Path not provided")
logging.debug(f"Using DEV_MODEpath: {path}")
base_path = os.path.normpath(path)
# Use the path directly as provided - it represents the structure from root
if not path:
raise HTTPException(status_code=400, detail="Path not provided")
# The path might already contain parts of the filesystem structure
# We need to construct the full path carefully
if path.startswith("users/"):
# If path starts with users/, remove it since filesystem already has users/ structure
base_path = path[6:] # Remove "users/" prefix
logging.debug(f"Removed 'users/' prefix, base_path is now: {base_path}")
else:
# In prod mode, construct path
logging.warning(f"Using db_name as base path not ready in prod: {db_name}")
base_path = db_name
base_path = path
logging.debug(f"No 'users/' prefix found, using path as-is: {base_path}")
base_path = os.path.normpath(base_path)
logging.debug(f"Using base path: {base_path}")
# Construct final path including tldraw file
logging.debug(f"Base path: {base_path}")
file_path = os.path.join(base_path, "tldraw_file.json")
logging.debug(f"File path: {file_path}")
file_location = os.path.normpath(os.path.join(fs.root_path, file_path))
logging.debug(f"File location: {file_location}")
@@ -185,12 +270,94 @@ async def set_tldraw_node_file(path: str, db_name: str, data: Dict):
try:
# Ensure directory exists
os.makedirs(os.path.dirname(file_location), exist_ok=True)
directory_location = os.path.dirname(file_location)
os.makedirs(directory_location, exist_ok=True)
logging.debug(f"Ensured directory exists: {directory_location}")
# Write the file
with open(file_location, "w") as file:
json.dump(data, file)
json.dump(data, file, indent=4)
logging.info(f"tldraw file successfully written to: {file_location}")
return {"status": "success"}
except Exception as e:
logging.error(f"Error writing file: {e}")
raise HTTPException(status_code=500, detail="Error writing file")
def create_default_tldraw_content():
"""Create default tldraw content structure."""
return {
"document": {
"store": {
"document:document": {
"gridSize": 10,
"name": "",
"meta": {},
"id": "document:document",
"typeName": "document"
},
"page:page": {
"meta": {},
"id": "page:page",
"name": "Page 1",
"index": "a1",
"typeName": "page"
}
},
"schema": {
"schemaVersion": 2,
"sequences": {
"com.tldraw.store": 4,
"com.tldraw.asset": 1,
"com.tldraw.camera": 1,
"com.tldraw.document": 2,
"com.tldraw.instance": 25,
"com.tldraw.instance_page_state": 5,
"com.tldraw.page": 1,
"com.tldraw.instance_presence": 5,
"com.tldraw.pointer": 1,
"com.tldraw.shape": 4,
"com.tldraw.asset.bookmark": 2,
"com.tldraw.asset.image": 5,
"com.tldraw.asset.video": 5,
"com.tldraw.shape.arrow": 5,
"com.tldraw.shape.bookmark": 2,
"com.tldraw.shape.draw": 2,
"com.tldraw.shape.embed": 4,
"com.tldraw.shape.frame": 0,
"com.tldraw.shape.geo": 9,
"com.tldraw.shape.group": 0,
"com.tldraw.shape.highlight": 1,
"com.tldraw.shape.image": 4,
"com.tldraw.shape.line": 5,
"com.tldraw.shape.note": 8,
"com.tldraw.shape.text": 2,
"com.tldraw.shape.video": 2,
"com.tldraw.binding.arrow": 0
}
},
"recordVersions": {
"asset": {"version": 1, "subTypeKey": "type", "subTypeVersions": {}},
"camera": {"version": 1},
"document": {"version": 2},
"instance": {"version": 21},
"instance_page_state": {"version": 5},
"page": {"version": 1},
"shape": {"version": 3, "subTypeKey": "type", "subTypeVersions": {}},
"instance_presence": {"version": 5},
"pointer": {"version": 1}
},
"rootShapeIds": [],
"bindings": [],
"assets": []
},
"session": {
"version": 0,
"currentPageId": "page:page",
"pageStates": [{
"pageId": "page:page",
"camera": {"x": 0, "y": 0, "z": 1},
"selectedShapeIds": []
}]
}
}
routers/database/tools/tldraw_supabase_storage.py
+265
View File
@@ -0,0 +1,265 @@
"""
TLDraw Supabase Storage Router
=============================
Handles TLDraw snapshot operations using Supabase Storage instead of local filesystem.
This replaces the old filesystem-based tldraw_filesystem.py router.
"""
from dotenv import load_dotenv, find_dotenv
load_dotenv(find_dotenv())
import os
import json
import logging
from fastapi import APIRouter, HTTPException, Query
from typing import Dict, Any
from modules.database.supabase.utils.storage import StorageAdmin
from modules.logger_tool import initialise_logger
router = APIRouter()
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
def create_default_tldraw_content():
"""Create default tldraw content structure."""
return {
"document": {
"store": {
"document:document": {
"gridSize": 10,
"name": "",
"meta": {},
"id": "document:document",
"typeName": "document"
},
"page:page": {
"meta": {},
"id": "page:page",
"name": "Page 1",
"index": "a1",
"typeName": "page"
}
},
"schema": {
"schemaVersion": 2,
"sequences": {
"com.tldraw.store": 4,
"com.tldraw.asset": 1,
"com.tldraw.camera": 1,
"com.tldraw.document": 2,
"com.tldraw.instance": 25,
"com.tldraw.instance_page_state": 5,
"com.tldraw.page": 1,
"com.tldraw.instance_presence": 5,
"com.tldraw.pointer": 1,
"com.tldraw.shape": 4,
"com.tldraw.asset.bookmark": 2,
"com.tldraw.asset.image": 5,
"com.tldraw.asset.video": 5,
"com.tldraw.shape.arrow": 5,
"com.tldraw.shape.bookmark": 2,
"com.tldraw.shape.draw": 2,
"com.tldraw.shape.embed": 4,
"com.tldraw.shape.frame": 0,
"com.tldraw.shape.geo": 9,
"com.tldraw.shape.group": 0,
"com.tldraw.shape.highlight": 1,
"com.tldraw.shape.image": 4,
"com.tldraw.shape.line": 5,
"com.tldraw.shape.note": 8,
"com.tldraw.shape.text": 2,
"com.tldraw.shape.video": 2,
"com.tldraw.binding.arrow": 0
}
},
"recordVersions": {
"asset": {"version": 1, "subTypeKey": "type", "subTypeVersions": {}},
"camera": {"version": 1},
"document": {"version": 2},
"instance": {"version": 21},
"instance_page_state": {"version": 5},
"page": {"version": 1},
"shape": {"version": 3, "subTypeKey": "type", "subTypeVersions": {}},
"instance_presence": {"version": 5},
"pointer": {"version": 1}
},
"rootShapeIds": [],
"bindings": [],
"assets": []
},
"session": {
"version": 0,
"currentPageId": "page:page",
"pageStates": [{
"pageId": "page:page",
"camera": {"x": 0, "y": 0, "z": 1},
"selectedShapeIds": []
}]
}
}
@router.get("/get_tldraw_node_file")
async def read_tldraw_node_file_from_supabase(
path: str = Query(..., description="Supabase Storage path (e.g., 'cc.public.snapshots/User/user_id')"),
db_name: str = Query(..., description="Database name for context")
):
"""
Load TLDraw snapshot from Supabase Storage.
Args:
path: Supabase Storage path in format 'bucket/nodetype/node_id'
db_name: Database name for context (used for logging)
Returns:
TLDraw snapshot data
"""
logger.debug(f"Reading tldraw file from Supabase Storage for path: {path}")
logger.debug(f"Database name: {db_name}")
if not path:
raise HTTPException(status_code=400, detail="Path not provided")
try:
# Initialize Supabase Storage
storage = StorageAdmin()
# Parse the path to extract bucket and file path
# Expected format: "cc.public.snapshots/User/user_id" or "cc.public.snapshots/Teacher/teacher_id"
path_parts = path.split('/')
if len(path_parts) < 3:
raise HTTPException(status_code=400, detail="Invalid path format. Expected: bucket/nodetype/node_id")
bucket = path_parts[0] # e.g., "cc.public.snapshots"
node_type = path_parts[1] # e.g., "User", "Teacher"
node_id = path_parts[2] # e.g., "cbc309e5-4029-4c34-aab7-0aa33c563cd0"
# Construct the file path in Supabase Storage
# Format: nodetype/node_id/tldraw_file.json
file_path = f"{node_type}/{node_id}/tldraw_file.json"
logger.debug(f"Bucket: {bucket}")
logger.debug(f"File path: {file_path}")
try:
# Try to download the file from Supabase Storage
file_data = storage.download_file(bucket, file_path)
# Parse JSON data
try:
snapshot_data = json.loads(file_data.decode('utf-8'))
logger.info(f"Successfully loaded tldraw snapshot from Supabase Storage: {file_path}")
# Ensure the snapshot has the correct structure for TLDraw
if isinstance(snapshot_data, dict) and 'document' in snapshot_data and 'session' in snapshot_data:
# Check if it has the new format (schemaVersion in document.schema)
if 'document' in snapshot_data and isinstance(snapshot_data['document'], dict) and 'schema' in snapshot_data['document']:
return snapshot_data
# Check if it has the old format (schemaVersion at root level)
elif 'schemaVersion' in snapshot_data:
return snapshot_data
else:
# Use default structure if schema is missing
logger.warning(f"Snapshot data from {file_path_in_bucket} is missing schemaVersion. Using default structure.")
return create_default_tldraw_content()
else:
# Use default structure if basic structure is missing
logger.warning(f"Snapshot data from {file_path_in_bucket} is missing top-level TLDraw keys. Using default structure.")
return create_default_tldraw_content()
except json.JSONDecodeError as e:
logger.error(f"Failed to parse JSON from Supabase Storage file: {e}")
raise HTTPException(status_code=500, detail="Invalid JSON in file")
except Exception as e:
# File doesn't exist, create default content
logger.info(f"File not found in Supabase Storage, creating default tldraw content: {file_path}")
# Create default tldraw content
default_content = create_default_tldraw_content()
try:
# Upload default content to Supabase Storage
json_data = json.dumps(default_content, indent=2).encode('utf-8')
storage.upload_file(bucket, file_path, json_data, 'application/json', upsert=True)
logger.info(f"Default tldraw file created in Supabase Storage: {file_path}")
return default_content
except Exception as upload_error:
logger.error(f"Error creating default tldraw file in Supabase Storage: {upload_error}")
raise HTTPException(status_code=500, detail="Error creating default tldraw file")
except HTTPException:
# Re-raise HTTP exceptions
raise
except Exception as e:
logger.error(f"Unexpected error loading tldraw file from Supabase Storage: {e}")
raise HTTPException(status_code=500, detail=f"Error loading file: {str(e)}")
@router.post("/set_tldraw_node_file")
async def set_tldraw_node_file_in_supabase(
path: str = Query(..., description="Supabase Storage path (e.g., 'cc.public.snapshots/User/user_id')"),
db_name: str = Query(..., description="Database name for context"),
data: Dict[str, Any] = None
):
"""
Save TLDraw snapshot to Supabase Storage.
Args:
path: Supabase Storage path in format 'bucket/nodetype/node_id'
db_name: Database name for context (used for logging)
data: TLDraw snapshot data to save
Returns:
Success status
"""
logger.debug(f"Saving tldraw file to Supabase Storage for path: {path}")
logger.debug(f"Database name: {db_name}")
if not path:
raise HTTPException(status_code=400, detail="Path not provided")
if not data:
raise HTTPException(status_code=400, detail="Data not provided")
try:
# Initialize Supabase Storage
storage = StorageAdmin()
# Parse the path to extract bucket and file path
path_parts = path.split('/')
if len(path_parts) < 3:
raise HTTPException(status_code=400, detail="Invalid path format. Expected: bucket/nodetype/node_id")
bucket = path_parts[0] # e.g., "cc.public.snapshots"
node_type = path_parts[1] # e.g., "User", "Teacher"
node_id = path_parts[2] # e.g., "cbc309e5-4029-4c34-aab7-0aa33c563cd0"
# Construct the file path in Supabase Storage
file_path = f"{node_type}/{node_id}/tldraw_file.json"
logger.debug(f"Bucket: {bucket}")
logger.debug(f"File path: {file_path}")
# Convert data to JSON
try:
json_data = json.dumps(data, indent=2).encode('utf-8')
except (TypeError, ValueError) as e:
logger.error(f"Failed to serialize data to JSON: {e}")
raise HTTPException(status_code=400, detail="Invalid data format")
# Upload to Supabase Storage
try:
storage.upload_file(bucket, file_path, json_data, 'application/json', upsert=True)
logger.info(f"Successfully saved tldraw snapshot to Supabase Storage: {file_path}")
return {"status": "success", "message": "File saved successfully"}
except Exception as upload_error:
logger.error(f"Error uploading file to Supabase Storage: {upload_error}")
raise HTTPException(status_code=500, detail="Error saving file")
except HTTPException:
# Re-raise HTTP exceptions
raise
except Exception as e:
logger.error(f"Unexpected error saving tldraw file to Supabase Storage: {e}")
raise HTTPException(status_code=500, detail=f"Error saving file: {str(e)}")
routers/database/tools/worker_structure_router.py
+16 -16
View File
@@ -29,34 +29,34 @@ async def get_worker_structure(db_name: str) -> Dict[str, Any]:
// Collect all nodes
RETURN {
timetables: collect(DISTINCT {
id: tt.unique_id,
path: tt.path,
id: tt.uuid_string,
path: tt.node_storage_path,
title: tt.title,
type: tt.__primarylabel__,
startTime: toString(tt.start_date),
endTime: toString(tt.end_date)
}),
classes: collect(DISTINCT {
id: c.unique_id,
path: c.path,
id: c.uuid_string,
path: c.node_storage_path,
title: c.title,
type: c.__primarylabel__
}),
lessons: collect(DISTINCT {
id: l.unique_id,
path: l.path,
id: l.uuid_string,
path: l.node_storage_path,
title: l.title,
type: l.__primarylabel__
}),
journals: collect(DISTINCT {
id: j.unique_id,
path: j.path,
id: j.uuid_string,
path: j.node_storage_path,
title: j.title,
type: j.__primarylabel__
}),
planners: collect(DISTINCT {
id: p.unique_id,
path: p.path,
id: p.uuid_string,
path: p.node_storage_path,
title: p.title,
type: p.__primarylabel__
})
@@ -106,8 +106,8 @@ async def get_timetables(db_name: str, start_date: str, end_date: str) -> Dict[s
MATCH (tt:UserTeacherTimetable)
WHERE date(tt.start_date) >= date($start_date) AND date(tt.end_date) <= date($end_date)
RETURN {
id: tt.unique_id,
path: tt.path,
id: tt.uuid_string,
path: tt.node_storage_path,
title: tt.title,
type: tt.__primarylabel__,
startTime: toString(tt.start_date),
@@ -138,8 +138,8 @@ async def get_journals(db_name: str) -> Dict[str, Any]:
query = """
MATCH (j:Journal)
RETURN {
id: j.unique_id,
path: j.path,
id: j.uuid_string,
path: j.node_storage_path,
title: j.title,
type: j.__primarylabel__
} as journal
@@ -168,8 +168,8 @@ async def get_planners(db_name: str) -> Dict[str, Any]:
query = """
MATCH (p:Planner)
RETURN {
id: p.unique_id,
path: p.path,
id: p.uuid_string,
path: p.node_storage_path,
title: p.title,
type: p.__primarylabel__
} as planner