ClassroomCopilot/api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

latest

Browse Source
This commit is contained in:
kcar
2025-11-14 14:47:19 +00:00
parent 2a85845835
commit 46b2319e2d
199 changed files with 607543 additions and 11147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
routers/admin/admin_panel.py
-2769
View File
File diff suppressed because it is too large Load Diff
routers/admin_panel_routes.py
-113
View File
@@ -1,113 +0,0 @@
from fastapi import APIRouter, Request, Depends, HTTPException
from fastapi.responses import HTMLResponse
from fastapi.templating import Jinja2Templates
import os
from modules.logger_tool import initialise_logger
from modules.database.services.school_admin_service import SchoolAdminService
from modules.database.supabase.utils.storage import StorageManager
from .auth import verify_admin
from typing import Dict
router = APIRouter()
templates = Jinja2Templates(directory="templates")
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
# Initialize services
school_service = SchoolAdminService()
storage_manager = StorageManager()
@router.get("/schools/manage", response_class=HTMLResponse)
async def manage_schools(request: Request, admin: Dict = Depends(verify_admin)):
"""Manage schools page"""
return templates.TemplateResponse(
"admin/schools/manage.html",
{"request": request, "admin": admin}
)
@router.get("/storage/manage", response_class=HTMLResponse)
async def manage_storage(request: Request, admin: Dict = Depends(verify_admin)):
"""Storage management page"""
try:
# Get list of storage buckets with correct IDs
buckets = [
{
"id": "cc.institutes",
"name": "School Files",
"public": False,
"file_size_limit": 50 * 1024 * 1024, # 50MB
"allowed_mime_types": [
"image/*",
"video/*",
"application/pdf",
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.ms-excel",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.ms-powerpoint",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"text/plain",
"text/csv",
"application/json"
]
},
{
"id": "cc.users",
"name": "User Files",
"public": False,
"file_size_limit": 50 * 1024 * 1024, # 50MB
"allowed_mime_types": [
"image/*",
"video/*",
"application/pdf",
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.ms-excel",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.ms-powerpoint",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"text/plain",
"text/csv",
"application/json"
]
}
]
return templates.TemplateResponse(
"admin/storage/manage.html",
{"request": request, "admin": admin, "buckets": buckets}
)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.get("/schema", response_class=HTMLResponse)
async def manage_schema(request: Request, admin: Dict = Depends(verify_admin)):
"""Schema management page"""
return templates.TemplateResponse(
"admin/schema/manage.html",
{"request": request, "admin": admin}
)
@router.get("/storage/{bucket_id}/contents")
async def list_bucket_contents(
request: Request,
bucket_id: str,
path: str = "",
admin: Dict = Depends(verify_admin)
):
"""List contents of a storage bucket"""
try:
contents = storage_manager.list_bucket_contents(bucket_id, path)
bucket = {"id": bucket_id, "name": bucket_id.replace("_", " ").title()}
return templates.TemplateResponse(
"admin/storage/contents.html",
{
"request": request,
"admin": admin,
"bucket": bucket,
"contents": contents,
"current_path": path
}
)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
routers/admin_routes.py
-498
View File
@@ -1,498 +0,0 @@
from fastapi import APIRouter, Request, Depends, HTTPException, File, UploadFile, Form
from fastapi.responses import HTMLResponse, JSONResponse
from fastapi.templating import Jinja2Templates
from typing import Dict
import os
from modules.logger_tool import initialise_logger
from modules.database.services.admin_service import AdminService, AdminProfileBase
from modules.database.services.school_admin_service import SchoolAdminService
from modules.database.supabase.utils.client import SupabaseAnonClient
from modules.database.supabase.utils.storage import StorageManager
from .auth import verify_admin
import csv
import io
router = APIRouter()
templates = Jinja2Templates(directory="templates")
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
# Initialize services
admin_service = AdminService()
school_service = SchoolAdminService()
storage_manager = StorageManager(SupabaseAnonClient)
@router.get("/", response_class=HTMLResponse)
async def admin_dashboard(request: Request, admin: Dict = Depends(verify_admin)):
"""Render admin dashboard"""
return templates.TemplateResponse(
"admin/dashboard/index.html",
{
"request": request,
"admin": admin,
"app_version": os.getenv("APP_VERSION", "Unknown")
}
)
@router.get("/users")
async def list_users(request: Request, admin: Dict = Depends(verify_admin)):
"""List all users"""
return templates.TemplateResponse(
"admin/users/list.html",
{"request": request, "admin": admin}
)
@router.get("/users/{user_id}")
async def get_user(request: Request, user_id: str, admin: Dict = Depends(verify_admin)):
"""Get user details"""
return templates.TemplateResponse(
"admin/users/detail.html",
{"request": request, "admin": admin, "user_id": user_id}
)
@router.get("/admins")
async def list_admins(request: Request, admin: Dict = Depends(verify_admin)):
"""List all admins"""
if not admin.get("is_super_admin"):
raise HTTPException(status_code=403, detail="Only super admins can view admin list")
admins = admin_service.list_admins()
return templates.TemplateResponse(
"admin/users/admins.html",
{"request": request, "admin": admin, "admins": admins}
)
@router.post("/admins")
async def create_admin(admin_data: AdminProfileBase, current_admin: Dict = Depends(verify_admin)):
"""Create a new admin"""
try:
result = admin_service.create_admin(admin_data, current_admin)
return JSONResponse(content={"status": "success", "admin": result})
except Exception as e:
raise HTTPException(status_code=400, detail=str(e))
@router.get("/schools/manage", response_class=HTMLResponse)
async def manage_schools(request: Request, admin: Dict = Depends(verify_admin)):
"""Manage schools page"""
try:
# Fetch schools from Supabase
result = admin_service.supabase.table("schools").select("*").execute()
schools = result.data if result else []
# Sort schools by establishment_name
schools.sort(key=lambda x: x.get("establishment_name", ""))
return templates.TemplateResponse(
"admin/schools/manage.html",
{
"request": request,
"admin": admin,
"schools": schools,
"schools_count": len(schools)
}
)
except Exception as e:
logger.error(f"Error fetching schools: {str(e)}")
return templates.TemplateResponse(
"admin/schools/manage.html",
{
"request": request,
"admin": admin,
"schools": [],
"schools_count": 0,
"error": str(e)
}
)
@router.post("/schools/import")
async def import_schools(
file: UploadFile = File(...),
admin: Dict = Depends(verify_admin)
):
"""Import schools from CSV file"""
if not file.filename.endswith('.csv'):
raise HTTPException(status_code=400, detail="Please upload a CSV file")
try:
# Process the CSV file
content = await file.read()
csv_text = content.decode('utf-8-sig') # Handle BOM if present
csv_reader = csv.DictReader(io.StringIO(csv_text))
# Prepare data for batch insert
schools_data = []
for row in csv_reader:
school_data = {
"urn": row.get("URN"),
"la_code": row.get("LA (code)"),
"la_name": row.get("LA (name)"),
"establishment_number": row.get("EstablishmentNumber"),
"establishment_name": row.get("EstablishmentName"),
"establishment_type": row.get("TypeOfEstablishment (name)"),
"establishment_type_group": row.get("EstablishmentTypeGroup (name)"),
"establishment_status": row.get("EstablishmentStatus (name)"),
"reason_establishment_opened": row.get("ReasonEstablishmentOpened (name)"),
"open_date": row.get("OpenDate"),
"reason_establishment_closed": row.get("ReasonEstablishmentClosed (name)"),
"close_date": row.get("CloseDate"),
"phase_of_education": row.get("PhaseOfEducation (name)"),
"statutory_low_age": row.get("StatutoryLowAge"),
"statutory_high_age": row.get("StatutoryHighAge"),
"boarders": row.get("Boarders (name)"),
"nursery_provision": row.get("NurseryProvision (name)"),
"official_sixth_form": row.get("OfficialSixthForm (name)"),
"gender": row.get("Gender (name)"),
"religious_character": row.get("ReligiousCharacter (name)"),
"religious_ethos": row.get("ReligiousEthos (name)"),
"diocese": row.get("Diocese (name)"),
"admissions_policy": row.get("AdmissionsPolicy (name)"),
"school_capacity": row.get("SchoolCapacity"),
"special_classes": row.get("SpecialClasses (name)"),
"census_date": row.get("CensusDate"),
"number_of_pupils": row.get("NumberOfPupils"),
"number_of_boys": row.get("NumberOfBoys"),
"number_of_girls": row.get("NumberOfGirls"),
"percentage_fsm": row.get("PercentageFSM"),
"trust_school_flag": row.get("TrustSchoolFlag (name)"),
"trusts_name": row.get("Trusts (name)"),
"school_sponsor_flag": row.get("SchoolSponsorFlag (name)"),
"school_sponsors_name": row.get("SchoolSponsors (name)"),
"federation_flag": row.get("FederationFlag (name)"),
"federations_name": row.get("Federations (name)"),
"ukprn": row.get("UKPRN"),
"fehe_identifier": row.get("FEHEIdentifier"),
"further_education_type": row.get("FurtherEducationType (name)"),
"ofsted_last_inspection": row.get("OfstedLastInsp"),
"last_changed_date": row.get("LastChangedDate"),
"street": row.get("Street"),
"locality": row.get("Locality"),
"address3": row.get("Address3"),
"town": row.get("Town"),
"county": row.get("County (name)"),
"postcode": row.get("Postcode"),
"school_website": row.get("SchoolWebsite"),
"telephone_num": row.get("TelephoneNum"),
"head_title": row.get("HeadTitle (name)"),
"head_first_name": row.get("HeadFirstName"),
"head_last_name": row.get("HeadLastName"),
"head_preferred_job_title": row.get("HeadPreferredJobTitle"),
"gssla_code": row.get("GSSLACode (name)"),
"parliamentary_constituency": row.get("ParliamentaryConstituency (name)"),
"urban_rural": row.get("UrbanRural (name)"),
"rsc_region": row.get("RSCRegion (name)"),
"country": row.get("Country (name)"),
"uprn": row.get("UPRN"),
"sen_stat": row.get("SENStat") == "true",
"sen_no_stat": row.get("SENNoStat") == "true",
"sen_unit_on_roll": row.get("SenUnitOnRoll"),
"sen_unit_capacity": row.get("SenUnitCapacity"),
"resourced_provision_on_roll": row.get("ResourcedProvisionOnRoll"),
"resourced_provision_capacity": row.get("ResourcedProvisionCapacity"),
}
# Clean up empty strings and convert types
for key, value in school_data.items():
if value == "":
school_data[key] = None
elif key in ["statutory_low_age", "statutory_high_age", "school_capacity",
"number_of_pupils", "number_of_boys", "number_of_girls",
"sen_unit_on_roll", "sen_unit_capacity",
"resourced_provision_on_roll", "resourced_provision_capacity"]:
if value:
try:
float_val = float(value)
int_val = int(float_val)
school_data[key] = int_val
except (ValueError, TypeError):
school_data[key] = None
elif key == "percentage_fsm":
if value:
try:
school_data[key] = float(value)
except (ValueError, TypeError):
school_data[key] = None
elif key in ["open_date", "close_date", "census_date",
"ofsted_last_inspection", "last_changed_date"]:
if value:
try:
# Convert date from DD-MM-YYYY to YYYY-MM-DD
parts = value.split("-")
if len(parts) == 3:
school_data[key] = f"{parts[2]}-{parts[1]}-{parts[0]}"
else:
school_data[key] = None
except:
school_data[key] = None
schools_data.append(school_data)
# Batch insert schools using admin service's Supabase client
if schools_data:
result = admin_service.supabase.table("schools").upsert(
schools_data,
on_conflict="urn" # Update if URN already exists
).execute()
logger.info(f"Imported {len(schools_data)} schools")
return {"status": "success", "imported_count": len(schools_data)}
else:
raise HTTPException(status_code=400, detail="No valid school data found in CSV")
except Exception as e:
logger.error(f"Error importing schools: {str(e)}")
raise HTTPException(status_code=500, detail=str(e))
@router.post("/initialize-schools-database")
async def initialize_schools_database(admin: Dict = Depends(verify_admin)):
"""Initialize schools database"""
if not admin.get("is_super_admin"):
raise HTTPException(status_code=403, detail="Only super admins can initialize database")
result = school_service.create_schools_database()
if result["status"] == "error":
raise HTTPException(status_code=500, detail=result["message"])
return result
@router.get("/check-schools-database")
async def check_schools_database(admin: Dict = Depends(verify_admin)):
"""Check schools database status"""
try:
# Use SchoolService to check if database exists and has required nodes/relationships
result = school_service.check_schools_database()
return {"exists": result["status"] == "success"}
except Exception as e:
logger.error(f"Error checking schools database: {str(e)}")
return {"exists": False, "error": str(e)}
@router.get("/storage", response_class=HTMLResponse)
async def storage_management(request: Request, admin: Dict = Depends(verify_admin)):
"""Storage management page"""
try:
# Get list of storage buckets with correct IDs
buckets = [
{
"id": "cc.institutes",
"name": "School Files",
"public": False,
"file_size_limit": 50 * 1024 * 1024, # 50MB
"allowed_mime_types": [
"image/*",
"video/*",
"application/pdf",
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.ms-excel",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.ms-powerpoint",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"text/plain",
"text/csv",
"application/json"
]
},
{
"id": "cc.users",
"name": "User Files",
"public": False,
"file_size_limit": 50 * 1024 * 1024, # 50MB
"allowed_mime_types": [
"image/*",
"video/*",
"application/pdf",
"application/msword",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"application/vnd.ms-excel",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"application/vnd.ms-powerpoint",
"application/vnd.openxmlformats-officedocument.presentationml.presentation",
"text/plain",
"text/csv",
"application/json"
]
}
]
return templates.TemplateResponse(
"admin/storage/manage.html",
{"request": request, "admin": admin, "buckets": buckets}
)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.get("/storage/{bucket_id}/contents")
async def list_bucket_contents(
request: Request,
bucket_id: str,
path: str = "",
admin: Dict = Depends(verify_admin)
):
"""List contents of a storage bucket"""
try:
contents = storage_manager.list_bucket_contents(bucket_id, path)
bucket = {"id": bucket_id, "name": bucket_id.replace("_", " ").title()}
return templates.TemplateResponse(
"admin/storage/contents.html",
{
"request": request,
"admin": admin,
"bucket": bucket,
"contents": contents,
"current_path": path
}
)
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.get("/storage/{bucket_id}/download/{file_path:path}")
async def download_file(
bucket_id: str,
file_path: str,
admin: Dict = Depends(verify_admin)
):
"""Get download URL for a file"""
try:
url = storage_manager.create_signed_url(bucket_id, file_path)
return {"url": url}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.delete("/storage/{bucket_id}/objects/{object_path:path}")
async def delete_object(
bucket_id: str,
object_path: str,
admin: Dict = Depends(verify_admin)
):
"""Delete an object from storage"""
try:
storage_manager.delete_file(bucket_id, object_path)
return {"status": "success"}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.get("/check-storage")
async def check_storage(admin: Dict = Depends(verify_admin)):
"""Check storage buckets status"""
try:
# Use the same bucket IDs as defined in initialize_storage
buckets = [
{"id": "cc.users", "name": "User Files"},
{"id": "cc.institutes", "name": "School Files"}
]
results = []
for bucket in buckets:
exists = storage_manager.check_bucket_exists(bucket["id"])
results.append({
"id": bucket["id"],
"name": bucket["name"],
"exists": exists
})
return {"buckets": results}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.post("/initialize-storage")
async def initialize_storage(admin: Dict = Depends(verify_admin)):
"""Initialize storage buckets and policies for schools"""
try:
# Verify super admin status
if not admin.get('is_super_admin'):
raise HTTPException(status_code=403, detail="Only super admins can initialize storage")
# Use the storage manager to initialize storage
storage_manager = StorageManager(SupabaseAnonClient)
return storage_manager.initialize_storage()
except Exception as e:
logger.error(f"Error initializing storage: {str(e)}")
raise HTTPException(status_code=500, detail=str(e))
@router.get("/check-schema")
async def check_schema(admin: Dict = Depends(verify_admin)):
"""Check Neo4j schema status"""
try:
from modules.database.services.graph_service import GraphService
graph_service = GraphService()
# Get actual schema status
schema_status = graph_service.check_schema_status()
# Return status with proper validation
return {
"constraints_valid": schema_status["constraints_count"] > 0,
"constraints_count": schema_status["constraints_count"],
"indexes_valid": schema_status["indexes_count"] > 0,
"indexes_count": schema_status["indexes_count"],
"labels_valid": schema_status["labels_count"] > 0,
"labels_count": schema_status["labels_count"]
}
except Exception as e:
logger.error(f"Error checking schema: {str(e)}")
return {
"constraints_valid": False,
"constraints_count": 0,
"indexes_valid": False,
"indexes_count": 0,
"labels_valid": False,
"labels_count": 0,
"error": str(e)
}
@router.post("/initialize-schema")
async def initialize_schema(admin: Dict = Depends(verify_admin)):
"""Initialize Neo4j schema (constraints and indexes)"""
if not admin.get("is_super_admin"):
raise HTTPException(status_code=403, detail="Only super admins can initialize schema")
try:
from modules.database.services.graph_service import GraphService
graph_service = GraphService()
# Initialize schema
result = graph_service.initialize_schema()
if result["status"] == "error":
raise HTTPException(status_code=500, detail=result["message"])
return result
except Exception as e:
logger.error(f"Error initializing schema: {str(e)}")
raise HTTPException(status_code=500, detail=str(e))
@router.get("/schools/{school_id}")
async def view_school(request: Request, school_id: str, admin: Dict = Depends(verify_admin)):
"""View school details"""
try:
# Fetch school details from Supabase
result = admin_service.supabase.table("schools").select("*").eq("id", school_id).single().execute()
school = result.data if result else None
if not school:
raise HTTPException(status_code=404, detail="School not found")
return templates.TemplateResponse(
"admin/schools/detail.html",
{"request": request, "admin": admin, "school": school}
)
except Exception as e:
logger.error(f"Error fetching school details: {str(e)}")
raise HTTPException(status_code=500, detail=str(e))
@router.delete("/schools/{school_id}")
async def delete_school(school_id: str, admin: Dict = Depends(verify_admin)):
"""Delete a school"""
try:
# Verify super admin status
if not admin.get("is_super_admin"):
raise HTTPException(status_code=403, detail="Only super admins can delete schools")
# Delete the school from Supabase
result = admin_service.supabase.table("schools").delete().eq("id", school_id).execute()
if not result.data:
raise HTTPException(status_code=404, detail="School not found")
return {"status": "success", "message": "School deleted successfully"}
except Exception as e:
logger.error(f"Error deleting school: {str(e)}")
raise HTTPException(status_code=500, detail=str(e))
routers/assets/docling/__init__.py
+7
View File
@@ -0,0 +1,7 @@
from fastapi import APIRouter
router = APIRouter()
@router.get("/ping")
def ping():
return {"status": "ok"}
routers/auth.py
+1 -1
View File
@@ -45,7 +45,7 @@ async def login_page(
# If no super admin and init flag is true, show initialization form
if not has_super_admin:
expected_email = os.getenv("VITE_SUPER_ADMIN_EMAIL")
expected_email = os.getenv("ADMIN_EMAIL")
return templates.TemplateResponse(
"admin/login.html",
{
routers/database/files/__init__.py
+3
View File
@@ -0,0 +1,3 @@
from . import cabinets, files
routers/database/files/cabinets.py
+75
View File
@@ -0,0 +1,75 @@
import os
from fastapi import APIRouter, Depends, HTTPException
from typing import Any, Dict
from modules.auth.supabase_bearer import SupabaseBearer
from modules.database.supabase.utils.client import SupabaseServiceRoleClient
router = APIRouter()
auth = SupabaseBearer()
@router.get("/cabinets")
def list_cabinets(payload: Dict[str, Any] = Depends(auth)):
user_id = payload.get('sub') or payload.get('user_id')
if not user_id:
raise HTTPException(status_code=401, detail="Invalid token payload")
client = SupabaseServiceRoleClient()
# Owned + shared via membership
owned = client.supabase.table('file_cabinets').select('*').eq('user_id', user_id).execute().data
shared = client.supabase.table('cabinet_memberships').select('cabinet_id').eq('profile_id', user_id).execute().data
shared_ids = [m['cabinet_id'] for m in (shared or [])]
shared_rows = client.supabase.table('file_cabinets').select('*').in_('id', shared_ids).execute().data if shared_ids else []
return {"owned": owned or [], "shared": shared_rows or []}
@router.post("/cabinets")
def create_cabinet(body: Dict[str, Any], payload: Dict[str, Any] = Depends(auth)):
user_id = payload.get('sub') or payload.get('user_id')
name = (body or {}).get('name')
if not user_id or not name:
raise HTTPException(status_code=400, detail="name is required")
client = SupabaseServiceRoleClient()
res = client.supabase.table('file_cabinets').insert({
'user_id': user_id,
'name': name
}).execute()
return res.data
@router.patch("/cabinets/{cabinet_id}")
def rename_cabinet(cabinet_id: str, body: Dict[str, Any], payload: Dict[str, Any] = Depends(auth)):
name = (body or {}).get('name')
if not name:
raise HTTPException(status_code=400, detail="name is required")
client = SupabaseServiceRoleClient()
res = client.supabase.table('file_cabinets').update({'name': name}).eq('id', cabinet_id).execute()
return res.data
@router.delete("/cabinets/{cabinet_id}")
def delete_cabinet(cabinet_id: str, payload: Dict[str, Any] = Depends(auth)):
client = SupabaseServiceRoleClient()
res = client.supabase.table('file_cabinets').delete().eq('id', cabinet_id).execute()
return res.data
@router.post("/cabinets/{cabinet_id}/members")
def add_member(cabinet_id: str, body: Dict[str, Any], payload: Dict[str, Any] = Depends(auth)):
target_profile_id = (body or {}).get('profile_id')
role = (body or {}).get('role', 'viewer')
if not target_profile_id:
raise HTTPException(status_code=400, detail="profile_id required")
client = SupabaseServiceRoleClient()
# Insert membership (RLS will ensure only owner can do it)
res = client.supabase.table('cabinet_memberships').upsert({
'cabinet_id': cabinet_id,
'profile_id': target_profile_id,
'role': role
}).execute()
return res.data
@router.delete("/cabinets/{cabinet_id}/members/{profile_id}")
def remove_member(cabinet_id: str, profile_id: str, payload: Dict[str, Any] = Depends(auth)):
client = SupabaseServiceRoleClient()
res = client.supabase.table('cabinet_memberships').delete().match({
'cabinet_id': cabinet_id,
'profile_id': profile_id
}).execute()
return res.data
routers/database/files/files.py
+1000
View File
File diff suppressed because it is too large Load Diff
routers/database/files/files_simplified.py
+256
View File
@@ -0,0 +1,256 @@
"""
Simplified Files Router
======================
Simplified version of the files router with auto-processing removed.
Keeps only essential functionality for file management and manual processing triggers.
This replaces the complex auto-processing system with simple file storage.
"""
import os
import uuid
import logging
from typing import Dict, List, Optional, Any
from pathlib import Path
from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Form, BackgroundTasks
from fastapi.responses import JSONResponse
from modules.auth.supabase_bearer import SupabaseBearer
from modules.database.supabase.utils.client import SupabaseServiceRoleClient
from modules.database.supabase.utils.storage import StorageAdmin
from modules.logger_tool import initialise_logger
router = APIRouter()
auth = SupabaseBearer()
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
def _choose_bucket(scope: str, user_id: str, school_id: Optional[str]) -> str:
"""Choose appropriate bucket based on scope - matches old system logic."""
scope = (scope or 'teacher').lower()
if scope == 'school' and school_id:
return f"cc.institutes.{school_id}.private"
# teacher / student fall back to users bucket for now
return 'cc.users'
@router.post("/files/upload")
async def upload_file(
cabinet_id: str = Form(...),
path: str = Form(...),
scope: str = Form(...),
file: UploadFile = File(...),
payload: Dict[str, Any] = Depends(auth)
):
"""
SIMPLIFIED file upload - no automatic processing.
Just stores the file and creates a database record.
This is the legacy endpoint maintained for backward compatibility.
"""
try:
user_id = payload.get('sub') or payload.get('user_id')
if not user_id:
raise HTTPException(status_code=401, detail="User ID required")
# Read file content
file_bytes = await file.read()
file_size = len(file_bytes)
mime_type = file.content_type or 'application/octet-stream'
filename = file.filename or path
logger.info(f"📤 Simplified upload: {filename} ({file_size} bytes) for user {user_id}")
# Initialize services
client = SupabaseServiceRoleClient()
storage = StorageAdmin()
# Generate file ID and storage path
file_id = str(uuid.uuid4())
# Use same bucket logic as old system for consistency
bucket = _choose_bucket('teacher', user_id, None)
storage_path = f"{cabinet_id}/{file_id}/{filename}"
# Store file in Supabase storage
try:
storage.upload_file(bucket, storage_path, file_bytes, mime_type, upsert=True)
except Exception as e:
logger.error(f"Storage upload failed for {file_id}: {e}")
raise HTTPException(status_code=500, detail=f"Storage upload failed: {str(e)}")
# Create database record
try:
insert_res = client.supabase.table('files').insert({
'id': file_id,
'name': filename,
'cabinet_id': cabinet_id,
'bucket': bucket,
'path': storage_path,
'mime_type': mime_type,
'uploaded_by': user_id,
'size_bytes': file_size,
'source': 'classroomcopilot-web',
'is_directory': False,
'processing_status': 'uploaded', # No auto-processing
'relative_path': filename
}).execute()
if not insert_res.data:
# Clean up storage on DB failure
try:
storage.delete_file(bucket, storage_path)
except:
pass
raise HTTPException(status_code=500, detail="Failed to create file record")
file_record = insert_res.data[0]
except Exception as e:
logger.error(f"Database insert failed for {file_id}: {e}")
# Clean up storage
try:
storage.delete_file(bucket, storage_path)
except:
pass
raise HTTPException(status_code=500, detail=f"Database error: {str(e)}")
logger.info(f"✅ Simplified upload completed: {file_id}")
return {
'status': 'success',
'message': 'File uploaded successfully (no auto-processing)',
'file': file_record,
'auto_processing_disabled': True,
'next_steps': 'Use manual processing endpoints if needed'
}
except HTTPException:
raise
except Exception as e:
logger.error(f"Upload error: {e}")
raise HTTPException(status_code=500, detail=f"Upload failed: {str(e)}")
@router.get("/files")
def list_files(cabinet_id: str, payload: Dict[str, Any] = Depends(auth)):
"""List files in a cabinet."""
client = SupabaseServiceRoleClient()
res = client.supabase.table('files').select('*').eq('cabinet_id', cabinet_id).execute()
return res.data
@router.get("/files/{file_id}")
def get_file(file_id: str, payload: Dict[str, Any] = Depends(auth)):
"""Get file details."""
client = SupabaseServiceRoleClient()
res = client.supabase.table('files').select('*').eq('id', file_id).single().execute()
if not res.data:
raise HTTPException(status_code=404, detail="File not found")
return res.data
@router.delete("/files/{file_id}")
def delete_file(file_id: str, payload: Dict[str, Any] = Depends(auth)):
"""Delete a file."""
client = SupabaseServiceRoleClient()
storage = StorageAdmin()
# Get file info first
res = client.supabase.table('files').select('*').eq('id', file_id).single().execute()
if not res.data:
raise HTTPException(status_code=404, detail="File not found")
file_data = res.data
# Delete from storage
try:
storage.delete_file(file_data['bucket'], file_data['path'])
except Exception as e:
logger.warning(f"Failed to delete file from storage: {e}")
# Delete from database
delete_res = client.supabase.table('files').delete().eq('id', file_id).execute()
logger.info(f"🗑️ Deleted file: {file_id}")
return {
'status': 'success',
'message': 'File deleted successfully'
}
@router.post("/files/{file_id}/process-manual")
async def trigger_manual_processing(
file_id: str,
processing_type: str = Form('basic'), # basic, advanced, custom
payload: Dict[str, Any] = Depends(auth)
):
"""
Trigger manual processing for a file.
This is where users can manually start processing when they want it.
"""
# TODO: Implement manual processing triggers
# This would call the archived processing logic when the user explicitly requests it
logger.info(f"🔧 Manual processing requested for file {file_id} (type: {processing_type})")
return {
'status': 'accepted',
'message': f'Manual processing queued for file {file_id}',
'processing_type': processing_type,
'note': 'Manual processing not yet implemented - will use archived auto-processing logic'
}
@router.get("/files/{file_id}/status")
def get_processing_status(file_id: str, payload: Dict[str, Any] = Depends(auth)):
"""Get processing status for a file."""
client = SupabaseServiceRoleClient()
res = client.supabase.table('files').select('processing_status, error_message, extra').eq('id', file_id).single().execute()
if not res.data:
raise HTTPException(status_code=404, detail="File not found")
return {
'file_id': file_id,
'status': res.data.get('processing_status', 'unknown'),
'error': res.data.get('error_message'),
'details': res.data.get('extra', {})
}
# Keep existing artefacts endpoints for backward compatibility
@router.get("/files/{file_id}/artefacts")
def list_file_artefacts(file_id: str, payload: Dict[str, Any] = Depends(auth)):
"""List artefacts for a file."""
client = SupabaseServiceRoleClient()
res = client.supabase.table('document_artefacts').select('*').eq('file_id', file_id).execute()
return res.data or []
@router.get("/files/{file_id}/viewer-artefacts")
def list_viewer_artefacts(file_id: str, payload: Dict[str, Any] = Depends(auth)):
"""List artefacts organized for the viewer."""
client = SupabaseServiceRoleClient()
# Get all artefacts
res = client.supabase.table('document_artefacts').select('*').eq('file_id', file_id).execute()
artefacts = res.data or []
# Simple organization - no complex bundle logic
organized = {
'document_analysis': [],
'processing_bundles': [],
'raw_data': []
}
for artefact in artefacts:
artefact_type = artefact.get('type', '')
if 'analysis' in artefact_type.lower():
organized['document_analysis'].append(artefact)
elif any(bundle_type in artefact_type for bundle_type in ['docling', 'bundle']):
organized['processing_bundles'].append(artefact)
else:
organized['raw_data'].append(artefact)
return organized
routers/database/files/split_map.py
+588
View File
@@ -0,0 +1,588 @@
# api/routers/database/files/split_map.py
"""
Automatic split_map.json generator for uploaded documents.
This module creates chapter/section boundaries for documents using existing artefacts
(Tika JSON, Docling frontmatter OCR) and optional PDF outline extraction.
Strategy (waterfall, stop at confidence ≥ 0.7):
1. PDF Outline/Bookmarks (best): confidence ≈ 0.95
2. Headings from Docling JSON: confidence ≈ 0.8
3. TOC from Tika text: confidence ≈ 0.7-0.8
4. Fixed windows: confidence ≈ 0.2
Hard constraints:
- For any fallback Docling "no-OCR" call: limit page_range to [1, min(30, page_count)]
- Never process more than 30 pages in one Docling request
- Use existing artefacts whenever possible
"""
import re
import json
import uuid
import datetime
import os
import requests
from typing import List, Dict, Any, Optional, Tuple
from modules.database.supabase.utils.client import SupabaseServiceRoleClient
from modules.database.supabase.utils.storage import StorageAdmin
from modules.logger_tool import initialise_logger
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
# ---------- Utilities
def _now_iso():
"""Return current UTC timestamp in ISO format."""
return datetime.datetime.utcnow().replace(microsecond=0).isoformat() + "Z"
def _load_artefact_json(storage: StorageAdmin, bucket: str, rel_path: str) -> Optional[Dict[str, Any]]:
"""Load JSON artefact from storage."""
try:
raw = storage.download_file(bucket, rel_path)
return json.loads(raw.decode("utf-8"))
except Exception as e:
logger.debug(f"Failed to load artefact {rel_path}: {e}")
return None
def _page_count_from_tika(tika_json: Dict[str, Any]) -> Optional[int]:
"""Extract page count from Tika JSON metadata."""
for k in ("xmpTPg:NPages", "Page-Count", "pdf:PageCount", "pdf:pagecount"):
v = tika_json.get(k) or tika_json.get(k.lower())
try:
if v is not None:
return int(v)
except Exception:
pass
return None
# ---------- A) Outline via PyMuPDF (optional but recommended)
def _try_outline(pdf_bytes: bytes) -> Optional[List[Tuple[str, int]]]:
"""
Extract PDF outline/bookmarks using PyMuPDF.
Returns [(title, start_page)] for level-1 bookmarks only.
"""
try:
import fitz # PyMuPDF
doc = fitz.open(stream=pdf_bytes, filetype="pdf")
toc = doc.get_toc(simple=True) # list of [level, title, page]
doc.close()
# Keep level-1 only, ensure valid pages
out = []
for level, title, page in toc:
if level == 1 and page >= 1:
clean_title = title.strip()
if clean_title and len(clean_title) > 1:
out.append((clean_title, page))
return out if len(out) >= 2 else None # Need at least 2 chapters
except ImportError:
logger.debug("PyMuPDF not available, skipping outline extraction")
return None
except Exception as e:
logger.debug(f"Outline extraction failed: {e}")
return None
# ---------- B) Headings from Docling JSON
def _try_headings(docling_json: Dict[str, Any]) -> Optional[List[Tuple[str, int, int]]]:
"""
Extract headings from Docling JSON.
Returns [(title, start_page, level)] — we only return starts; end pages are computed later.
"""
if not docling_json:
return None
# Handle different Docling JSON structures
blocks = (docling_json.get("blocks") or
docling_json.get("elements") or
docling_json.get("body", {}).get("blocks") or [])
candidates: List[Tuple[str, int, int]] = []
for b in blocks:
# Check if this is a heading block
role = (b.get("role") or b.get("type") or "").lower()
if not ("heading" in role or role in ("h1", "h2", "title", "section-header")):
continue
# Extract text content
text = (b.get("text") or b.get("content") or "").strip()
if not text or len(text) < 3:
continue
# Extract page number with robust handling of 0-based pageIndex
p = None
if b.get("pageIndex") is not None:
try:
p = int(b.get("pageIndex")) + 1
except Exception:
p = None
if p is None:
for key in ("page", "page_no", "page_number"):
if b.get(key) is not None:
try:
p = int(b.get(key))
except Exception:
p = None
break
if p is None or p < 1:
continue
# Determine heading level
level = 1 # default
if "1" in role or "h1" in role:
level = 1
elif "2" in role or "h2" in role:
level = 2
# Chapter regex boosts to level 1
if re.match(r"^\s*(chapter|ch\.?|section|part)\s+\d+", text, re.I):
level = 1
candidates.append((text, p, level))
if not candidates:
return None
# Prefer level 1; if none, promote level 2 to level 1
l1 = [(t, p, l) for (t, p, l) in candidates if l == 1]
if not l1:
l1 = [(t, p, 1) for (t, p, _) in candidates]
# Sort by page and keep strictly increasing pages only
l1_sorted = []
seen = set()
for (t, p, l) in sorted(l1, key=lambda x: x[1]):
if p not in seen and p >= 1:
l1_sorted.append((t, p, l))
seen.add(p)
return l1_sorted if len(l1_sorted) >= 2 else None
def _try_headings_fallback(file_id: str, cabinet_id: str, bucket: str,
processing_bytes: bytes, processing_mime: str,
page_count: int) -> Optional[List[Tuple[str, int, int]]]:
"""
Make a limited Docling no-OCR call (max 30 pages) to extract headings.
This is used only when existing artefacts don't have sufficient heading data.
"""
try:
docling_url = os.getenv('DOCLING_URL') or os.getenv('NEOFS_DOCLING_URL')
if not docling_url:
logger.debug("No Docling URL configured for headings fallback")
return None
# Strictly limit to first 30 pages
max_pages = min(30, page_count)
logger.info(f"Headings fallback: limited Docling call for file_id={file_id}, pages=1-{max_pages}")
# Build Docling request (no-OCR, limited pages)
docling_api_key = os.getenv('DOCLING_API_KEY')
headers = {'Accept': 'application/json'}
if docling_api_key:
headers['X-Api-Key'] = docling_api_key
form_data = [
('target_type', 'inbody'),
('to_formats', 'json'),
('do_ocr', 'false'),
('force_ocr', 'false'),
('image_export_mode', 'embedded'),
('pdf_backend', 'dlparse_v4'),
('table_mode', 'fast'),
('page_range', '1'),
('page_range', str(max_pages))
]
files = [('files', ('file', processing_bytes, processing_mime))]
# Make the request with timeout
timeout = int(os.getenv('DOCLING_HEADINGS_TIMEOUT', '1800')) # 30 minutes default
resp = requests.post(
f"{docling_url.rstrip('/')}/v1/convert/file",
files=files,
data=form_data,
headers=headers,
timeout=timeout
)
resp.raise_for_status()
docling_json = resp.json()
logger.debug(f"Headings fallback: received Docling response for file_id={file_id}")
return _try_headings(docling_json)
except Exception as e:
logger.error(f"Headings fallback failed for file_id={file_id}: {e}")
return None
# ---------- C) TOC from Tika text (dot leaders & page num)
TOC_LINE = re.compile(r"^\s*(.+?)\s?(\.{2,}|\s{3,})\s*(\d{1,4})\s*$")
def _try_toc_text(tika_text: str) -> Optional[List[Tuple[str, int]]]:
"""
Parse TOC from Tika text using dot leaders and page numbers.
Returns [(title, start_page)] if successful.
"""
if not tika_text:
return None
# Heuristic: only scan first ~1500 lines (roughly first 15 pages)
head = "\n".join(tika_text.splitlines()[:1500])
pairs = []
for line in head.splitlines():
m = TOC_LINE.match(line)
if not m:
continue
title = m.group(1).strip()
try:
page = int(m.group(3))
except Exception:
continue
# Reject obvious junk
if len(title) < 3 or page < 1 or page > 9999:
continue
# Skip common false positives
if any(skip in title.lower() for skip in ['copyright', 'isbn', 'published', 'printed']):
continue
pairs.append((title, page))
# Require at least 5 entries and monotonic pages
if len(pairs) >= 5:
pages = [p for _, p in pairs]
if pages == sorted(pages):
logger.debug(f"TOC extraction found {len(pairs)} entries")
return pairs
return None
# ---------- Build entries with ends, apply smoothing
def _entries_from_starts(starts: List[Tuple[str, int, int]], page_count: int, source: str = "headings") -> List[Dict[str, Any]]:
"""
Build entries from start points with computed end pages.
starts: [(title, page, level)]
"""
entries = []
base_confidence = 0.8 if source == "headings" else 0.75
for i, (title, start, level) in enumerate(starts):
end = (starts[i + 1][1] - 1) if i + 1 < len(starts) else page_count
entries.append({
"id": f"sec{i + 1:02d}",
"title": title,
"level": level,
"start_page": int(start),
"end_page": int(end),
"source": source,
"confidence": base_confidence
})
# Merge tiny sections (< 3 pages) into previous
merged = []
for e in entries:
section_size = e["end_page"] - e["start_page"] + 1
if merged and section_size < 3:
# Merge into previous section
merged[-1]["end_page"] = e["end_page"]
merged[-1]["title"] += " / " + e["title"]
merged[-1]["confidence"] *= 0.95 # Slight confidence penalty for merging
else:
merged.append(e)
return merged
def _entries_from_pairs(pairs: List[Tuple[str, int]], page_count: int, source: str = "outline") -> List[Dict[str, Any]]:
"""
Build entries from (title, start_page) pairs.
"""
entries = []
base_confidence = 0.95 if source == "outline" else (0.8 if source == "toc" else 0.75)
for i, (title, start) in enumerate(pairs):
end = (pairs[i + 1][1] - 1) if i + 1 < len(pairs) else page_count
entries.append({
"id": f"sec{i + 1:02d}",
"title": title,
"level": 1,
"start_page": int(start),
"end_page": int(end),
"source": source,
"confidence": base_confidence
})
# Apply same merging logic for tiny sections
merged = []
for e in entries:
section_size = e["end_page"] - e["start_page"] + 1
if merged and section_size < 3:
merged[-1]["end_page"] = e["end_page"]
merged[-1]["title"] += " / " + e["title"]
merged[-1]["confidence"] *= 0.95
else:
merged.append(e)
return merged
# ---------- Post-processing normalization
def _normalize_entries(entries: List[Dict[str, Any]], page_count: int) -> List[Dict[str, Any]]:
"""Normalize entries to ensure:
- coverage from page 1
- 1 <= start_page <= end_page <= page_count
- strictly increasing, non-overlapping ranges
- fill initial gap with a synthetic front matter section if needed
"""
if not entries:
return entries
# Sanitize and sort by start_page
safe: List[Dict[str, Any]] = []
for e in entries:
try:
s = int(e.get("start_page", 1))
t = int(e.get("end_page", s))
except Exception:
continue
s = max(1, min(s, page_count))
t = max(1, min(t, page_count))
if t < s:
t = s
ne = dict(e)
ne["start_page"], ne["end_page"] = s, t
safe.append(ne)
safe.sort(key=lambda x: (x["start_page"], x.get("level", 1)))
# De-overlap by adjusting starts; ensure monotonic ranges
normalized: List[Dict[str, Any]] = []
for e in safe:
if not normalized:
normalized.append(e)
continue
prev = normalized[-1]
if e["start_page"] <= prev["end_page"]:
e["start_page"] = prev["end_page"] + 1
if e["start_page"] > page_count:
continue
if e["end_page"] < e["start_page"]:
e["end_page"] = e["start_page"]
e["end_page"] = min(e["end_page"], page_count)
normalized.append(e)
# Insert synthetic front matter if first start > 1
if normalized and normalized[0]["start_page"] > 1:
front = {
"id": "sec00",
"title": "Front matter",
"level": 1,
"start_page": 1,
"end_page": normalized[0]["start_page"] - 1,
"source": "synthetic",
"confidence": 0.6,
}
normalized.insert(0, front)
# Ensure last section ends at page_count
if normalized and normalized[-1]["end_page"] < page_count:
normalized[-1]["end_page"] = page_count
# Renumber ids sequentially
out: List[Dict[str, Any]] = []
for idx, e in enumerate(normalized, start=1):
ne = dict(e)
ne["id"] = f"sec{idx:02d}"
out.append(ne)
return out
# ---------- Main entry point
def create_split_map_for_file(file_id: str) -> Dict[str, Any]:
"""
Create split_map.json for a file using waterfall strategy:
1. PDF outline (best)
2. Docling headings (from existing or limited fallback)
3. Tika TOC parsing
4. Fixed windows (fallback)
"""
logger.info(f"Creating split_map for file_id={file_id}")
client = SupabaseServiceRoleClient()
storage = StorageAdmin()
# 1) Lookup file row & bucket
fr = client.supabase.table('files').select('id,bucket,cabinet_id,name,path,mime_type').eq('id', file_id).single().execute()
file_row = fr.data or {}
bucket = file_row.get('bucket')
cabinet_id = file_row.get('cabinet_id')
# 2) Find artefacts
arts = client.supabase.table('document_artefacts') \
.select('*').eq('file_id', file_id).order('created_at', desc=True).execute().data or []
def find_art(t):
for a in arts:
if a.get('type') == t:
return a
return None
a_pdf = find_art('document_pdf') # if converted to PDF
a_tika = find_art('tika_json')
a_noocr = find_art('docling_noocr_json')
a_fm = find_art('docling_frontmatter_json')
# 3) Load JSON/text
tika_json = _load_artefact_json(storage, bucket, a_tika['rel_path']) if a_tika else None
docling_noocr = _load_artefact_json(storage, bucket, a_noocr['rel_path']) if a_noocr else None
docling_fm = _load_artefact_json(storage, bucket, a_fm['rel_path']) if a_fm else None
# Get page count
page_count = _page_count_from_tika(tika_json or {}) or 100 # reasonable default
# Get PDF bytes for outline extraction
pdf_bytes = None
processing_bytes = None
processing_mime = None
if a_pdf:
# Use converted PDF
pdf_bytes = storage.download_file(bucket, a_pdf['rel_path'])
processing_bytes = pdf_bytes
processing_mime = 'application/pdf'
else:
# Check if original file is PDF
if file_row.get('mime_type') == 'application/pdf':
pdf_bytes = storage.download_file(bucket, file_row['path'])
processing_bytes = pdf_bytes
processing_mime = 'application/pdf'
# 4) Try methods in waterfall order
method = "fixed"
confidence = 0.2
entries: List[Dict[str, Any]] = []
# A) PDF Outline/Bookmarks (best)
if pdf_bytes and not entries:
logger.debug(f"Trying outline extraction for file_id={file_id}")
pairs = _try_outline(pdf_bytes)
if pairs:
entries = _entries_from_pairs(pairs, page_count, source="outline")
method, confidence = "outline", 0.95
logger.info(f"Split map: outline method found {len(entries)} sections")
# B) Headings from existing Docling JSON
if not entries:
logger.debug(f"Trying headings from existing Docling JSON for file_id={file_id}")
# Try no-OCR first, then frontmatter
for docling_json, source_name in [(docling_noocr, "noocr"), (docling_fm, "frontmatter")]:
if docling_json:
starts = _try_headings(docling_json)
if starts:
entries = _entries_from_starts(starts, page_count, source="headings")
method, confidence = "headings", 0.8
logger.info(f"Split map: headings method ({source_name}) found {len(entries)} sections")
break
# B2) Headings fallback with limited Docling call (if we have processing bytes)
if not entries and processing_bytes and processing_mime:
logger.debug(f"Trying headings fallback with limited Docling call for file_id={file_id}")
starts = _try_headings_fallback(file_id, cabinet_id, bucket, processing_bytes, processing_mime, page_count)
if starts:
entries = _entries_from_starts(starts, page_count, source="headings")
method, confidence = "headings", 0.75 # Slightly lower confidence for fallback
logger.info(f"Split map: headings fallback found {len(entries)} sections")
# C) TOC from Tika text
if not entries and tika_json:
logger.debug(f"Trying TOC extraction from Tika text for file_id={file_id}")
# Try common Tika text keys
text = tika_json.get("X-TIKA:content") or tika_json.get("content") or ""
pairs = _try_toc_text(text)
if pairs:
entries = _entries_from_pairs(pairs, page_count, source="toc")
method, confidence = "toc", 0.75
logger.info(f"Split map: TOC method found {len(entries)} sections")
# D) Fixed windows (fallback)
if not entries:
logger.info(f"Using fixed window fallback for file_id={file_id}")
step = max(10, min(20, page_count // 10)) # Adaptive step size
pairs = []
for i in range(1, page_count + 1, step):
end_page = min(i + step - 1, page_count)
title = f"Pages {i}-{end_page}" if i != end_page else f"Page {i}"
pairs.append((title, i))
entries = _entries_from_pairs(pairs, page_count, source="fixed")
method, confidence = "fixed", 0.2
logger.info(f"Split map: fixed method created {len(entries)} sections")
# 5) Normalize entries and build split_map.json
entries = _normalize_entries(entries, page_count)
split_map = {
"version": 1,
"file_id": file_id,
"source_pdf_artefact_id": a_pdf['id'] if a_pdf else None,
"sources": {
"docling_noocr_json": a_noocr['id'] if a_noocr else None,
"docling_frontmatter_json": a_fm['id'] if a_fm else None,
"tika_json": a_tika['id'] if a_tika else None
},
"method": method,
"confidence": confidence,
"page_count": page_count,
"entries": entries,
"created_at": _now_iso(),
"notes": f"auto-generated using {method} method; user can edit in Split Marker UI"
}
# 6) Store as artefact
artefact_id = str(uuid.uuid4())
rel_path = f"{cabinet_id}/{file_id}/{artefact_id}/split_map.json"
storage.upload_file(
bucket,
rel_path,
json.dumps(split_map, ensure_ascii=False, indent=2).encode("utf-8"),
"application/json",
upsert=True
)
# Enhanced metadata for UI display
enhanced_extra = {
"method": method,
"confidence": confidence,
"entries_count": len(entries),
"display_name": "Document Structure Map",
"bundle_label": "Split Map",
"section_title": "Document Structure Map",
"page_count": page_count,
"bundle_type": "split_map_json",
"processing_mode": "document_analysis",
"pipeline": "structure_analysis",
"is_structure_map": True,
"ui_category": "document_analysis",
"ui_order": 2,
"description": f"Document section boundaries identified using {method} method with {confidence:.1%} confidence ({len(entries)} sections)",
"viewer_type": "json"
}
client.supabase.table('document_artefacts').insert({
"id": artefact_id,
"file_id": file_id,
"type": "split_map_json",
"rel_path": rel_path,
"extra": enhanced_extra,
"status": "completed"
}).execute()
logger.info(f"Split map stored: file_id={file_id}, method={method}, confidence={confidence:.2f}, entries={len(entries)}")
return split_map
routers/database/init/curriculum.py
+6 -7
View File
@@ -31,20 +31,19 @@ async def upload_curriculum(file: UploadFile = File(...), db_name: str = Form(..
async def upload_school_curriculum(
file: UploadFile = File(...),
db_name: str = Form(...),
school_uuid: str = Form(...),
school_uuid_string: str = Form(...),
school_name: str = Form(...),
school_website: str = Form(...),
school_path: str = Form(...)
school_node_storage_path: str = Form(...)
):
if file.content_type != 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':
return {"status": "Error", "message": "Invalid file format"}
logging.info(f"Uploading curriculum for school {school_name} in {db_name}")
dataframes = xl.create_dataframes_from_fastapiuploadfile(file)
school_node = SchoolNode(
unique_id=f'School_{school_uuid}',
school_uuid=school_uuid,
school_name=school_name,
school_website=school_website,
path=school_path
uuid_string=school_uuid_string,
name=school_name,
website=school_website,
node_storage_path=school_node_storage_path
)
return init_school_curriculum.create_curriculum(db_name, dataframes, school_node)
routers/database/init/entity_init.py
+50 -46
View File
@@ -23,9 +23,9 @@ def initialise_schools_from_config():
"""Initialize a school with the configuration provided from env variables
"""
default_config = {
"school_uuid": "kevlarai",
"school_name": "KevlarAI School",
"school_website": "https://kevlarai.com",
"uuid_string": "kevlarai-dev",
"name": "KevlarAI School",
"website": "https://kevlarai.com",
"timetable_file": "kevlarai_data/kevlarai_timetable.xlsx",
"curriculum_file": "kevlarai_data/kevlarai_curriculum.xlsx"
}
@@ -33,10 +33,10 @@ def initialise_schools_from_config():
# school_config_str = os.getenv("SCHOOL_CONFIG") # TODO: Implement this
school_config = default_config
db_name = f"cc.institutes.{school_config['school_uuid']}"
db_name = f"cc.institutes.{school_config['uuid_string']}"
curriculum_db_name = f"{db_name}.curriculum"
logger.info(f"Creating database for {school_config['school_name']} using db_name: {db_name}")
logger.info(f"Creating database for {school_config['name']} using db_name: {db_name}")
driver = driver_tools.get_driver()
if driver is None:
logger.error("Failed to connect to Neo4j")
@@ -54,7 +54,7 @@ def initialise_schools_from_config():
# Add filesystem path debugging
base_path = os.getenv("NODE_FILESYSTEM_PATH")
schools_path = os.path.join(base_path, "schools")
school_path = os.path.join(schools_path, f"cc.institutes.{school_config['school_uuid']}")
school_path = os.path.join(schools_path, f"cc.institutes.{school_config['uuid_string']}")
logger.debug("Filesystem paths:", {
"base_path": base_path,
@@ -70,29 +70,34 @@ def initialise_schools_from_config():
})
# Create database entry for school without timetable or curriculum
logger.info(f"Creating school entry for {school_config['school_name']} in database {db_name} without timetable or curriculum")
logger.info(f"Creating school entry for {school_config['name']} in database {db_name} without timetable or curriculum")
school_uuid_string=school_config["uuid_strig"]
school_name=school_config["name"]
school_website=school_config["website"]
result = init_school.create_school(
db_name=db_name,
school_uuid=school_config["school_uuid"],
school_name=school_config["school_name"],
school_website=school_config["school_website"]
school_type="development",
uuid_string=school_uuid_string,
name=school_name,
website=school_website
)
logger.success(f"{school_config['school_name']} school entry created successfully")
logger.success(f"{school_config['name']} school entry created successfully")
# Create school node from result
school_node = result['school_node']
refreshed_school_node = SchoolNode(
unique_id=school_node.unique_id,
school_uuid=school_node.school_uuid,
school_name=school_node.school_name,
school_website=school_node.school_website,
path=school_node.path
school_type="development",
uuid_string=school_node.uuid_string,
name=school_node.name,
website=school_node.website
)
# Create timetable entries for school from Excel file
timetable_file = os.path.join(os.getenv("BACKEND_INIT_PATH"), school_config["timetable_file"])
logger.info(f"Creating timetable entries for {school_config['school_name']} using timetable file: {timetable_file}.")
logger.info(f"Creating timetable entries for {school_config['name']} using timetable file: {timetable_file}.")
school_timetable_dataframes = xl.create_dataframes(timetable_file)
@@ -107,7 +112,7 @@ def initialise_schools_from_config():
curriculum_file = os.path.join(os.getenv("BACKEND_INIT_PATH"), school_config["curriculum_file"])
school_curriculum_dataframes = xl.create_dataframes(curriculum_file)
logger.info(f"Creating curriculum entries for {school_config['school_name']} using curriculum file: {curriculum_file}.")
logger.info(f"Creating curriculum entries for {school_config['name']} using curriculum file: {curriculum_file}.")
init_school_curriculum.create_curriculum(
dataframes=school_curriculum_dataframes,
db_name=db_name,
@@ -123,18 +128,18 @@ async def create_user(
user_type: str = Form(...),
user_name: str = Form(...),
user_email: str = Form(...),
school_uuid: str = Form(None),
school_uuid_string: str = Form(None),
school_name: str = Form(None),
school_website: str = Form(None),
school_path: str = Form(None),
school_node_storage_path: str = Form(None),
worker_data: str = Form(None)
):
logger.info(f"Creating user with user_id: {user_id}, user_type: {user_type}, user_name: {user_name}, user_email: {user_email}")
if school_uuid:
logger.info(f"School UUID provided: {school_uuid}")
if school_uuid_string:
logger.info(f"School UUID string provided: {school_uuid_string}")
else:
logger.info(f"No school UUID provided")
logger.info(f"No school UUID string provided")
if school_name:
logger.info(f"School name provided: {school_name}")
@@ -146,8 +151,8 @@ async def create_user(
else:
logger.info(f"No school website provided")
if school_path:
logger.info(f"School path provided: {school_path}")
if school_node_storage_path:
logger.info(f"School path provided: {school_node_storage_path}")
else:
logger.info(f"No school path provided")
@@ -169,13 +174,12 @@ async def create_user(
# Create school node if school data provided
school_node = None
if all([school_uuid, school_name, school_website, school_path]):
if all([school_uuid_string, school_name, school_website, school_node_storage_path]):
school_node = SchoolNode(
unique_id=f'School_{school_uuid}',
school_uuid=school_uuid,
school_name=school_name,
school_website=school_website,
path=school_path
uuid_string=school_uuid_string,
name=school_name,
website=school_website,
node_storage_path=school_node_storage_path
)
# Create user with single database reference
@@ -219,23 +223,23 @@ async def create_schools():
@router.post("/create-department")
async def create_department(
db_name: str = Form(...),
unique_id: str = Form(...),
uuid_string: str = Form(...),
department_name: str = Form(...),
department_code: str = Form(...),
path: str = Form(...)
department_node_storage_path: str = Form(...)
):
if db_name is None or unique_id is None or department_name is None or department_code is None or path is None:
logging.error(f"Invalid department data: {db_name}, {unique_id}, {department_name}, {department_code}, {path}")
if db_name is None or uuid_string is None or department_name is None or department_code is None or department_node_storage_path is None:
logging.error(f"Invalid department data: {db_name}, {uuid_string}, {department_name}, {department_code}, {department_node_storage_path}")
raise HTTPException(status_code=400, detail="Invalid department data")
department = DepartmentNode(
unique_id=unique_id,
uuid_string=uuid_string,
department_name=department_name,
department_code=department_code,
path=path
node_storage_path=department_node_storage_path
)
logger.info(f"Creating department {department_name} with unique_id {unique_id}")
logger.info(f"Creating department {department_name} with uuid_string {uuid_string}")
try:
result = init_school.create_department(db_name, department)
return JSONResponse(content={"status": "success", "data": result})
@@ -246,20 +250,20 @@ async def create_department(
@router.post("/create-class")
async def create_class(
db_name: str = Form(...),
unique_id: str = Form(...),
uuid_string: str = Form(...),
subject_class_code: str = Form(...),
year_group: str = Form(...),
subject: str = Form(...),
subject_code: str = Form(...),
path: str = Form(...)
subject_node_storage_path: str = Form(...)
):
subject_class_node = SubjectClassNode(
unique_id=unique_id,
uuid_string=uuid_string,
subject_class_code=subject_class_code,
year_group=year_group,
subject=subject,
subject_code=subject_code,
path=path
node_storage_path=subject_node_storage_path
)
# Implementation for creating a class
pass
@@ -267,14 +271,14 @@ async def create_class(
@router.post("/create-room")
async def create_room(
db_name: str = Form(...),
room_unique_id: str = Form(...),
room_uuid_string: str = Form(...),
room_code: str = Form(...),
path: str = Form(...)
room_node_storage_path: str = Form(...)
):
room = RoomNode(
room_unique_id=room_unique_id,
room_uuid_string=room_uuid_string,
room_code=room_code,
path=path
node_storage_path=room_node_storage_path
)
# Implementation for creating a room
pass
routers/database/init/schools.py
+10 -12
View File
@@ -15,7 +15,7 @@ logging = logger.get_logger(
from fastapi import APIRouter, File, UploadFile, Form, HTTPException, BackgroundTasks
import pandas as pd
import modules.database.tools.neo4j_driver_tools as driver
from modules.database.tools.neo4j_session_tools import get_node_by_unique_id
from modules.database.tools.neo4j_session_tools import get_node_by_uuid_string
import modules.database.init.init_school_timetable as init_school_timetable
import modules.database.init.init_worker_timetable as init_worker_timetable
from modules.database.schemas.nodes.schools.schools import SchoolNode
@@ -28,18 +28,16 @@ router = APIRouter()
async def upload_school_timetable(
file: UploadFile = File(...),
db_name: str = Form(...),
unique_id: str = Form(...),
school_uuid: str = Form(...),
school_uuid_string: str = Form(...),
school_name: str = Form(...),
school_website: str = Form(...),
path: str = Form(...)
school_node_storage_path: str = Form(...)
):
school_node = SchoolNode(
unique_id=unique_id,
school_uuid=school_uuid,
school_name=school_name,
school_website=school_website,
path=path
uuid_string=school_uuid_string,
name=school_name,
website=school_website,
node_storage_path=school_node_storage_path
)
if file.content_type != 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':
return {"status": "Error", "message": "Invalid file format"}
@@ -91,13 +89,13 @@ async def process_worker_timetable(file_content, worker_node_data):
timetable_df = pd.read_excel(BytesIO(file_content))
# Get the school version of the worker node
logging.info(f"Getting school worker node for {worker_node_data['unique_id']} from {worker_node_data['worker_db_name']}")
logging.info(f"Getting school worker node for {worker_node_data['uuid_string']} from {worker_node_data['worker_db_name']}")
with neo_driver.session(database=worker_node_data['worker_db_name']) as neo_session:
school_worker_node = get_node_by_unique_id(session=neo_session, unique_id=worker_node_data['unique_id'])
school_worker_node = get_node_by_uuid_string(session=neo_session, uuid_string=worker_node_data['uuid_string'])
if school_worker_node is None:
error_msg = f"School worker node not found for unique_id: {worker_node_data['unique_id']}"
error_msg = f"School worker node not found for uuid_string: {worker_node_data['uuid_string']}"
logging.error(error_msg)
raise Exception(error_msg)
routers/database/init/timetables.py
+14 -16
View File
@@ -15,7 +15,7 @@ logging = logger.get_logger(
from fastapi import APIRouter, File, UploadFile, Form, HTTPException, BackgroundTasks
import pandas as pd
import modules.database.tools.neo4j_driver_tools as driver
from modules.database.tools.neo4j_session_tools import get_node_by_unique_id
from modules.database.tools.neo4j_session_tools import get_node_by_uuid_string
import modules.database.init.init_school_timetable as init_school_timetable
import modules.database.init.init_worker_timetable as init_worker_timetable
from modules.database.schemas.nodes.users import UserNode
@@ -31,18 +31,16 @@ router = APIRouter()
async def upload_school_timetable(
file: UploadFile = File(...),
db_name: str = Form(...),
unique_id: str = Form(...),
school_uuid: str = Form(...),
school_uuid_string: str = Form(...),
school_name: str = Form(...),
school_website: str = Form(...),
path: str = Form(...)
school_node_storage_path: str = Form(...)
):
school_node = SchoolNode(
unique_id=unique_id,
school_uuid=school_uuid,
school_name=school_name,
school_website=school_website,
path=path
uuid_string=school_uuid_string,
name=school_name,
website=school_website,
node_storage_path=school_node_storage_path
)
if file.content_type != 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':
return {"status": "Error", "message": "Invalid file format"}
@@ -101,13 +99,13 @@ async def process_worker_timetable(file_content, user_node_data, worker_node_dat
timetable_df = pd.read_excel(BytesIO(file_content))
# Get the school version of the worker node
logging.info(f"Getting school worker node for {worker_node_data['unique_id']} from {worker_node_data['worker_db_name']}")
logging.info(f"Getting school worker node for {worker_node_data['uuid_string']} from {worker_node_data['worker_db_name']}")
with neo_driver.session(database=worker_node_data['worker_db_name']) as neo_session:
school_worker_node = get_node_by_unique_id(session=neo_session, unique_id=worker_node_data['unique_id'])
school_worker_node = get_node_by_uuid_string(session=neo_session, uuid_string=worker_node_data['uuid_string'])
if school_worker_node is None:
error_msg = f"School worker node not found for unique_id: {worker_node_data['unique_id']}"
error_msg = f"School worker node not found for uuid_string: {worker_node_data['uuid_string']}"
logging.error(error_msg)
raise Exception(error_msg)
@@ -127,23 +125,23 @@ async def process_worker_timetable(file_content, user_node_data, worker_node_dat
# Create TeacherNode from worker_node_data
user_worker_node = TeacherNode(
unique_id=worker_node_data['unique_id'],
uuid_string=worker_node_data['uuid_string'],
teacher_code=worker_node_data['teacher_code'],
teacher_name_formal=worker_node_data['teacher_name_formal'],
teacher_email=worker_node_data['teacher_email'],
path=worker_node_data['path'],
node_storage_path=worker_node_data['node_storage_path'],
worker_db_name=worker_node_data['worker_db_name'],
user_db_name=worker_node_data['user_db_name']
)
# Create user node
user_node = UserNode(
unique_id=user_node_data['unique_id'],
uuid_string=user_node_data['uuid_string'],
user_id=user_node_data['user_id'],
user_type=user_node_data['user_type'],
user_name=user_node_data['user_name'],
user_email=user_node_data['user_email'],
path=user_node_data['path'],
node_storage_path=user_node_data['node_storage_path'],
worker_node_data=user_node_data['worker_node_data']
)
routers/database/tools/calendar_structure_router.py
+23 -23
View File
@@ -27,29 +27,29 @@ async def get_calendar_structure(db_name: str) -> Dict[str, Any]:
// Collect all nodes with dates converted to strings
RETURN {
years: collect(DISTINCT {
id: y.unique_id,
path: y.path,
id: y.uuid_string,
path: y.node_storage_path,
date: toString(y.date),
__primarylabel__: 'CalendarYear'
}),
months: collect(DISTINCT {
id: m.unique_id,
path: m.path,
id: m.uuid_string,
path: m.node_storage_path,
date: toString(m.date),
__primarylabel__: 'CalendarMonth'
}),
weeks: collect(DISTINCT {
id: w.unique_id,
path: w.path,
id: w.uuid_string,
path: w.node_storage_path,
date: toString(w.date),
__primarylabel__: 'CalendarWeek'
}),
days: collect(DISTINCT {
id: d.unique_id,
path: d.path,
id: d.uuid_string,
path: d.node_storage_path,
date: toString(d.date),
week_id: w.unique_id,
month_id: m.unique_id,
week_id: w.uuid_string,
month_id: m.uuid_string,
__primarylabel__: 'CalendarDay'
})
} as structure
@@ -98,11 +98,11 @@ async def get_calendar_days(db_name: str, start_date: str, end_date: str) -> Dic
OPTIONAL MATCH (w:CalendarWeek)-[:WEEK_INCLUDES_DAY]->(d)
OPTIONAL MATCH (m:CalendarMonth)-[:MONTH_INCLUDES_DAY]->(d)
RETURN {
id: d.unique_id,
path: d.path,
id: d.uuid_string,
path: d.node_storage_path,
date: d.date,
week_id: w.unique_id,
month_id: m.unique_id,
week_id: w.uuid_string,
month_id: m.uuid_string,
__primarylabel__: 'CalendarDay'
} as day
ORDER BY d.date
@@ -132,10 +132,10 @@ async def get_calendar_weeks(db_name: str, start_date: str, end_date: str) -> Di
WHERE date(w.date) >= date($start_date) AND date(w.date) <= date($end_date)
WITH w, collect(d) as days
RETURN {
id: w.unique_id,
path: w.path,
id: w.uuid_string,
path: w.node_storage_path,
date: w.date,
day_ids: [day in days | day.unique_id],
day_ids: [day in days | day.uuid_string],
__primarylabel__: 'CalendarWeek'
} as week
ORDER BY w.date
@@ -165,10 +165,10 @@ async def get_calendar_months(db_name: str, start_date: str, end_date: str) -> D
WHERE date(m.date) >= date($start_date) AND date(m.date) <= date($end_date)
WITH m, collect(d) as days
RETURN {
id: m.unique_id,
path: m.path,
id: m.uuid_string,
path: m.node_storage_path,
date: m.date,
day_ids: [day in days | day.unique_id],
day_ids: [day in days | day.uuid_string],
__primarylabel__: 'CalendarMonth'
} as month
ORDER BY m.date
@@ -197,10 +197,10 @@ async def get_calendar_years(db_name: str) -> Dict[str, Any]:
MATCH (y:CalendarYear)-[:YEAR_INCLUDES_MONTH]->(m:CalendarMonth)
WITH y, collect(m) as months
RETURN {
id: y.unique_id,
path: y.path,
id: y.uuid_string,
path: y.node_storage_path,
date: y.date,
month_ids: [month in months | month.unique_id],
month_ids: [month in months | month.uuid_string],
__primarylabel__: 'CalendarYear'
} as year
ORDER BY y.date
routers/database/tools/default_nodes_router.py
+37 -3
View File
@@ -47,7 +47,7 @@ def get_default_node_week(db_name: str) -> Dict[str, Any]:
return {
"status": "success",
"node": {
"id": node["unique_id"],
"id": node["uuid_string"],
"path": node["path"],
"type": "CalendarWeek",
"label": node.get("title", "Calendar Week"),
@@ -81,7 +81,7 @@ def get_default_node_month(db_name: str) -> Dict[str, Any]:
return {
"status": "success",
"node": {
"id": node["unique_id"],
"id": node["uuid_string"],
"path": node["path"],
"type": "CalendarMonth",
"label": node.get("title", "Calendar Month"),
@@ -89,6 +89,39 @@ def get_default_node_month(db_name: str) -> Dict[str, Any]:
}
}
@router.get("/debug-list-nodes")
async def debug_list_nodes(db_name: str) -> Dict[str, Any]:
"""Debug endpoint to list all nodes in a database."""
try:
with driver_tools.get_session(database=db_name) as session:
query = """
MATCH (n)
RETURN labels(n) as labels, n.uuid_string as uuid, n.user_name as name, n.cc_username as username
LIMIT 20
"""
result = session.run(query)
nodes = []
for record in result:
nodes.append({
"labels": list(record["labels"]),
"uuid": record["uuid"],
"name": record["name"],
"username": record["username"]
})
return {
"status": "success",
"db_name": db_name,
"node_count": len(nodes),
"nodes": nodes
}
except Exception as e:
return {
"status": "error",
"db_name": db_name,
"error": str(e)
}
@router.get("/get-default-node/{context}")
async def get_default_node(context: str, db_name: str, base_context: str | None = None) -> Dict[str, Any]:
"""Get the default node for a given context."""
@@ -244,8 +277,9 @@ async def get_default_node(context: str, db_name: str, base_context: str | None
return {
"status": "success",
"node": {
"id": node["unique_id"],
"id": node["uuid_string"],
"path": node["path"],
"node_storage_path": node.get("node_storage_path", node["path"]),
"type": list(node.labels)[0],
"label": node.get("title", ""),
"data": converted_data
routers/database/tools/get_events.py
+6 -6
View File
@@ -51,10 +51,10 @@ router = APIRouter()
@router.get("/get_teacher_timetable_events")
async def get_teacher_timetable_events(
unique_id: str,
uuid_string: str,
worker_db_name: str
):
logging.info(f"Getting timetable events for teacher {unique_id} from database {worker_db_name}")
logging.info(f"Getting timetable events for teacher {uuid_string} from database {worker_db_name}")
neo_driver = driver.get_driver(db_name=worker_db_name)
if neo_driver is None:
return {"status": "error", "message": "Failed to connect to the database"}
@@ -62,9 +62,9 @@ async def get_teacher_timetable_events(
try:
with neo_driver.session(database=worker_db_name) as neo_session:
query = """
MATCH (t:Teacher {unique_id: $unique_id})-[:TEACHER_HAS_TIMETABLE]->(tt:TeacherTimetable)
MATCH (t:Teacher {uuid_string: $uuid_string})-[:TEACHER_HAS_TIMETABLE]->(tt:TeacherTimetable)
-[:TIMETABLE_HAS_CLASS]->(sc:SubjectClass)-[:CLASS_HAS_LESSON]->(tl:TimetableLesson)
RETURN tl.unique_id as id,
RETURN tl.uuid_string as id,
tl.period_code as period_code,
COALESCE(sc.subject_class_code, 'Untitled Class') as subject_class,
tl.date as date,
@@ -72,7 +72,7 @@ async def get_teacher_timetable_events(
tl.end_time as end_time,
tl.path as path
"""
result = neo_session.run(query, unique_id=unique_id)
result = neo_session.run(query, uuid_string=uuid_string)
events = []
for record in result:
@@ -92,7 +92,7 @@ async def get_teacher_timetable_events(
"path": record['path']
}
})
logging.info(f"Found {len(events)} events for teacher {unique_id}")
logging.info(f"Found {len(events)} events for teacher {uuid_string}")
return {"status": "success", "events": events}
except Exception as e:
logging.error(f"Error fetching events: {str(e)}")
routers/database/tools/get_nodes.py
+49 -45
View File
@@ -25,18 +25,18 @@ from fastapi import APIRouter, HTTPException, Query
router = APIRouter()
@router.get("/get-node")
async def get_node(unique_id: str = Query(...), db_name: str = Query(...)):
logging.info(f"Getting node for {unique_id} from database {db_name}")
async def get_node(uuid_string: str = Query(...), db_name: str = Query(...)):
logging.info(f"Getting node for {uuid_string} from database {db_name}")
neo_driver = driver.get_driver(db_name=db_name)
if neo_driver is None:
return {"status": "error", "message": "Failed to connect to the database"}
try:
with neo_driver.session(database=db_name) as neo_session:
query = """
MATCH (n {unique_id: $unique_id})
MATCH (n {uuid_string: $uuid_string})
RETURN n
"""
result = neo_session.run(query, unique_id=unique_id)
result = neo_session.run(query, uuid_string=uuid_string)
record = result.single()
if record:
@@ -47,11 +47,23 @@ async def get_node(unique_id: str = Query(...), db_name: str = Query(...)):
try:
# Convert node based on its type
node_type = node_labels[0] if node_labels else "Unknown"
if node_type in globals():
node_class = globals()[f"{node_type}Node"]
logging.debug(f"Attempting to convert node of type: {node_type}")
logging.debug(f"Available node classes: {[name for name in globals() if name.endswith('Node')]}")
logging.debug(f"UserNode in globals: {'UserNode' in globals()}")
logging.debug(f"UserNode class: {UserNode}")
logging.debug(f"UserNode class name: {UserNode.__name__}")
# Try to find the node class
node_class_name = f"{node_type}Node"
if node_class_name in globals():
node_class = globals()[node_class_name]
logging.debug(f"Found node class: {node_class}")
node_object = node_class(**node_data)
node_dict = node_object.to_dict()
logging.debug(f"Successfully converted node to dict: {node_dict}")
else:
logging.warning(f"No node class found for type: {node_type} (looking for {node_class_name}), using raw data")
logging.debug(f"Available classes: {[name for name in globals() if 'Node' in name]}")
node_dict = node_data
return {
@@ -101,8 +113,8 @@ async def get_user_node(user_id: str = Query(...)):
driver.close_driver(neo_driver)
@router.get("/get-connected-nodes")
async def get_connected_nodes(unique_id: str = Query(...), db_name: str = Query(...)):
logging.info(f"Getting connected nodes for {unique_id} from database {db_name}")
async def get_connected_nodes(uuid_string: str = Query(...), db_name: str = Query(...)):
logging.info(f"Getting connected nodes for {uuid_string} from database {db_name}")
neo_driver = driver.get_driver(db_name=db_name)
if neo_driver is None:
return {"status": "error", "message": "Failed to connect to the database"}
@@ -110,11 +122,11 @@ async def get_connected_nodes(unique_id: str = Query(...), db_name: str = Query(
try:
with neo_driver.session(database=db_name) as neo_session:
query = """
MATCH (n {unique_id: $unique_id})
MATCH (n {uuid_string: $uuid_string})
OPTIONAL MATCH (n)-[]-(connected)
RETURN n, collect(connected) as connected_nodes
"""
result = neo_session.run(query, unique_id=unique_id)
result = neo_session.run(query, uuid_string=uuid_string)
record = result.single()
if record:
main_node = record['n']
@@ -171,15 +183,15 @@ async def get_connected_nodes(unique_id: str = Query(...), db_name: str = Query(
driver.close_driver(neo_driver)
@router.get("/get-user-connected-nodes")
async def get_user_connected_nodes(unique_id: str = Query(...)):
logging.info(f"Getting user adjacent nodes for node {unique_id}")
async def get_user_connected_nodes(uuid_string: str = Query(...)):
logging.info(f"Getting user adjacent nodes for node {uuid_string}")
db_name = os.getenv("NEO4J_DB_NAME", "cc.institutes.kevlarai") # TODO: This function needs to be able to take a db_name as a parameter
neo_driver = driver.get_driver(db_name=db_name)
if neo_driver is None:
raise HTTPException(status_code=500, detail="Failed to connect to the database")
try:
with neo_driver.session(database=db_name) as neo_session:
user_node_and_connected_nodes = session.get_node_by_unique_id_and_adjacent_nodes(neo_session, unique_id)
user_node_and_connected_nodes = session.get_node_by_uuid_string_and_adjacent_nodes(neo_session, uuid_string)
user_node = user_node_and_connected_nodes['node']
connected_nodes = user_node_and_connected_nodes['connected_nodes']
try:
@@ -251,15 +263,15 @@ async def get_user_connected_nodes(unique_id: str = Query(...)):
driver.close_driver(neo_driver)
@router.get("/get-worker-connected-nodes")
async def get_worker_connected_nodes(unique_id: str = Query(...)):
logging.info(f"Getting worker adjacent nodes for node {unique_id}")
async def get_worker_connected_nodes(uuid_string: str = Query(...)):
logging.info(f"Getting worker adjacent nodes for node {uuid_string}")
db_name = os.getenv("NEO4J_DB_NAME", "cc.institutes.kevlarai") # TODO: This function needs to be able to take a db_name as a parameter
neo_driver = driver.get_driver(db_name=db_name)
if neo_driver is None:
raise HTTPException(status_code=500, detail="Failed to connect to the database")
try:
with neo_driver.session(database=db_name) as neo_session:
node_and_connected_nodes = session.get_node_by_unique_id_and_adjacent_nodes(neo_session, unique_id)
node_and_connected_nodes = session.get_node_by_uuid_string_and_adjacent_nodes(neo_session, uuid_string)
worker_node = node_and_connected_nodes['node']
connected_nodes = node_and_connected_nodes['connected_nodes']
try:
@@ -319,9 +331,9 @@ async def get_worker_connected_nodes(unique_id: str = Query(...)):
driver.close_driver(neo_driver)
@router.get("/get-calendar-connected-nodes")
async def get_calendar_connected_nodes(unique_id: str = Query(...)):
async def get_calendar_connected_nodes(uuid_string: str = Query(...)):
db_name = os.getenv("NEO4J_DB_NAME", "cc.institutes.kevlarai")
logging.info(f"Getting connected nodes for calendar {unique_id} from database {db_name}")
logging.info(f"Getting connected nodes for calendar {uuid_string} from database {db_name}")
neo_driver = driver.get_driver(db_name=db_name)
if neo_driver is None:
return {"status": "error", "message": "Failed to connect to the database"}
@@ -330,11 +342,11 @@ async def get_calendar_connected_nodes(unique_id: str = Query(...)):
with neo_driver.session(database=db_name) as neo_session:
query = """
MATCH (n)
WHERE n.unique_id = $unique_id AND (n:Calendar OR n:CalendarYear OR n:CalendarMonth OR n:CalendarWeek OR n:CalendarDay OR n:CalendarTimeChunk)
WHERE n.uuid_string = $uuid_string AND (n:Calendar OR n:CalendarYear OR n:CalendarMonth OR n:CalendarWeek OR n:CalendarDay OR n:CalendarTimeChunk)
OPTIONAL MATCH (n)-[]-(connected)
RETURN n, collect(connected) as connected_nodes
"""
result = neo_session.run(query, unique_id=unique_id)
result = neo_session.run(query, uuid_string=uuid_string)
record = result.single()
if record:
calendar_node = record['n']
@@ -369,9 +381,9 @@ async def get_calendar_connected_nodes(unique_id: str = Query(...)):
driver.close_driver(neo_driver)
@router.get("/get-teacher-timetable-connected-nodes")
async def get_teacher_timetable_connected_nodes(unique_id: str = Query(...)):
async def get_teacher_timetable_connected_nodes(uuid_string: str = Query(...)):
db_name = os.getenv("NEO4J_DB_NAME", "cc.institutes.kevlarai")
logging.info(f"Getting connected nodes for teacher timetable {unique_id} from database {db_name}")
logging.info(f"Getting connected nodes for teacher timetable {uuid_string} from database {db_name}")
neo_driver = driver.get_driver(db_name=db_name)
if neo_driver is None:
return {"status": "error", "message": "Failed to connect to the database"}
@@ -379,11 +391,11 @@ async def get_teacher_timetable_connected_nodes(unique_id: str = Query(...)):
try:
with neo_driver.session(database=db_name) as neo_session:
query = """
MATCH (n:TeacherTimetable {unique_id: $unique_id})
MATCH (n:TeacherTimetable {uuid_string: $uuid_string})
OPTIONAL MATCH (n)-[]-(connected)
RETURN n, collect(connected) as connected_nodes
"""
result = neo_session.run(query, unique_id=unique_id)
result = neo_session.run(query, uuid_string=uuid_string)
record = result.single()
if record:
teacher_timetable_node = record['n']
@@ -422,9 +434,9 @@ async def get_teacher_timetable_connected_nodes(unique_id: str = Query(...)):
driver.close_driver(neo_driver)
@router.get("/get-school-timetable-connected-nodes")
async def get_school_timetable_connected_nodes(unique_id: str = Query(...)):
async def get_school_timetable_connected_nodes(uuid_string: str = Query(...)):
db_name = os.getenv("NEO4J_DB_NAME", "cc.institutes.kevlarai")
logging.info(f"Getting connected nodes for school timetable {unique_id} from database {db_name}")
logging.info(f"Getting connected nodes for school timetable {uuid_string} from database {db_name}")
neo_driver = driver.get_driver(db_name=db_name)
if neo_driver is None:
return {"status": "error", "message": "Failed to connect to the database"}
@@ -432,11 +444,11 @@ async def get_school_timetable_connected_nodes(unique_id: str = Query(...)):
try:
with neo_driver.session(database=db_name) as neo_session:
query = """
MATCH (n:SchoolTimetable {unique_id: $unique_id})
MATCH (n:SchoolTimetable {uuid_string: $uuid_string})
OPTIONAL MATCH (n)-[]-(connected)
RETURN n, collect(connected) as connected_nodes
"""
result = neo_session.run(query, unique_id=unique_id)
result = neo_session.run(query, uuid_string=uuid_string)
record = result.single()
if record:
school_timetable_node = record['n']
@@ -483,9 +495,9 @@ async def get_school_timetable_connected_nodes(unique_id: str = Query(...)):
driver.close_driver(neo_driver)
@router.get("/get-curriculum-connected-nodes")
async def get_curriculum_connected_nodes(unique_id: str = Query(...)):
async def get_curriculum_connected_nodes(uuid_string: str = Query(...)):
db_name = os.getenv("NEO4J_DB_NAME", "cc.institutes.kevlarai")
logging.info(f"Getting connected nodes for curriculum {unique_id} from database {db_name}")
logging.info(f"Getting connected nodes for curriculum {uuid_string} from database {db_name}")
neo_driver = driver.get_driver(db_name=db_name)
if neo_driver is None:
return {"status": "error", "message": "Failed to connect to the database"}
@@ -494,11 +506,11 @@ async def get_curriculum_connected_nodes(unique_id: str = Query(...)):
with neo_driver.session(database=db_name) as neo_session:
query = """
MATCH (n)
WHERE n.unique_id = $unique_id AND (n:PastoralStructure OR n:YearGroup OR n:CurriculumStructure OR n:KeyStage OR n:KeyStageSyllabus OR n:YearGroupSyllabus OR n:Subject OR n:Topic OR n:TopicLesson OR n:LearningStatement OR n:ScienceLab)
WHERE n.uuid_string = $uuid_string AND (n:PastoralStructure OR n:YearGroup OR n:CurriculumStructure OR n:KeyStage OR n:KeyStageSyllabus OR n:YearGroupSyllabus OR n:Subject OR n:Topic OR n:TopicLesson OR n:LearningStatement OR n:ScienceLab)
OPTIONAL MATCH (n)-[]-(connected)
RETURN n, collect(connected) as connected_nodes
"""
result = neo_session.run(query, unique_id=unique_id)
result = neo_session.run(query, uuid_string=uuid_string)
record = result.single()
if record:
curriculum_node = record['n']
@@ -533,26 +545,18 @@ async def get_curriculum_connected_nodes(unique_id: str = Query(...)):
driver.close_driver(neo_driver)
@router.get("/get-school-node")
async def get_school_node(school_uuid: str = Query(...)):
logging.info(f"Getting school node for school {school_uuid}...")
db_name = f"cc.institutes.{school_uuid}"
async def get_school_node(school_uuid_string: str = Query(...)):
logging.info(f"Getting school node for school {school_uuid_string}...")
db_name = f"cc.institutes.{school_uuid_string}"
neo_driver = driver.get_driver(db_name=db_name)
if neo_driver is None:
return {"status": "error", "message": "Failed to connect to the database"}
try:
with neo_driver.session(database=db_name) as neo_session:
nodes = session.find_nodes_by_label_and_properties(neo_session, "School", {"school_uuid": school_uuid})
nodes = session.find_nodes_by_label_and_properties(neo_session, "School", {"uuid_string": school_uuid_string})
if nodes:
school_node = nodes[0]
data = SchoolNode(
unique_id=school_node["unique_id"],
school_uuid=school_node["school_uuid"],
school_name=school_node["school_name"],
school_website=school_node["school_website"],
path=school_node["path"]
)
school_node_data = data.to_dict()
school_node_data = SchoolNode(**nodes[0]).to_dict()
return {"status": "success", "school_node": school_node_data, "school_node_raw": nodes}
else:
return {"status": "not_found", "message": "School node not found"}
routers/database/tools/get_nodes_and_edges.py
+4 -4
View File
@@ -89,8 +89,8 @@ async def get_all_nodes_and_edges():
@router.get("/get-connected-nodes-and-edges")
async def get_connected_nodes_and_edges(unique_id: str = Query(...), db_name: str = Query(...)):
logging.info(f"Getting connected nodes and edges for {unique_id} from database {db_name}")
async def get_connected_nodes_and_edges(uuid_string: str = Query(...), db_name: str = Query(...)):
logging.info(f"Getting connected nodes and edges for {uuid_string} from database {db_name}")
neo_driver = driver.get_driver(db_name=db_name)
if neo_driver is None:
return {"status": "error", "message": "Failed to connect to the database"}
@@ -98,11 +98,11 @@ async def get_connected_nodes_and_edges(unique_id: str = Query(...), db_name: st
try:
with neo_driver.session(database=db_name) as neo_session:
query = """
MATCH (n {unique_id: $unique_id})
MATCH (n {uuid_string: $uuid_string})
OPTIONAL MATCH (n)-[r]-(connected)
RETURN n, collect(connected) as connected_nodes, collect(r) as relationships
"""
result = neo_session.run(query, unique_id=unique_id)
result = neo_session.run(query, uuid_string=uuid_string)
record = result.single()
if record:
main_node = record['n']
routers/database/tools/tldraw_filesystem.py
+218 -51
View File
@@ -34,20 +34,24 @@ async def read_tldraw_user_node_file(user_node: UserNode):
logging.debug(f"Filesystem root path: {fs.root_path}")
# Handle path based on environment
if os.getenv("DEV_MODE") == "true":
# In dev mode, use the full system path from the node
if not user_node.path:
raise HTTPException(status_code=400, detail="Node path not found")
logging.debug(f"Using DEV_MODE path: {user_node.path}")
base_path = os.path.normpath(user_node.path)
# Use the path directly as provided - it represents the structure from root
if not user_node.node_storage_path:
raise HTTPException(status_code=400, detail="Node path not found")
# The path might already contain parts of the filesystem structure
# We need to construct the full path carefully
if user_node.node_storage_path.startswith("users/"):
# If path starts with users/, remove it since filesystem already has users/ structure
base_path = user_node.node_storage_path[6:] # Remove "users/" prefix
logging.debug(f"Removed 'users/' prefix, base_path is now: {base_path}")
else:
# In prod mode, construct path using formatted email
logging.warning(f"Using db_name as base path not ready in prod: {db_name}")
base_path = formatted_email
base_path = user_node.node_storage_path
logging.debug(f"No 'users/' prefix found, using path as-is: {base_path}")
base_path = os.path.normpath(base_path)
logging.debug(f"Using base path: {base_path}")
# Construct final path including tldraw file
logging.debug(f"Base path: {base_path}")
file_path = os.path.join(base_path, "tldraw_file.json")
logging.debug(f"File path: {file_path}")
file_location = os.path.normpath(os.path.join(fs.root_path, file_path))
@@ -68,8 +72,30 @@ async def read_tldraw_user_node_file(user_node: UserNode):
logging.error(f"Error reading file: {e}")
raise HTTPException(status_code=500, detail="Error reading file")
else:
logging.debug(f"File does not exist: {file_location}")
raise HTTPException(status_code=404, detail="File not found")
# Check if directory exists
directory_location = os.path.dirname(file_location)
if os.path.exists(directory_location):
logging.debug(f"Directory exists but file doesn't, creating default tldraw file at: {file_location}")
try:
# Create default tldraw content
default_tldraw_content = create_default_tldraw_content()
# Ensure directory exists (should already exist, but just in case)
os.makedirs(directory_location, exist_ok=True)
# Write the default file
with open(file_location, "w") as file:
json.dump(default_tldraw_content, file, indent=4)
logging.info(f"Default tldraw file created at: {file_location}")
return default_tldraw_content
except Exception as e:
logging.error(f"Error creating default tldraw file: {e}")
raise HTTPException(status_code=500, detail="Error creating default tldraw file")
else:
logging.debug(f"Neither directory nor file exists: {directory_location}")
raise HTTPException(status_code=404, detail="Directory not found")
@router.post("/set_tldraw_user_node_file")
async def set_tldraw_user_node_file(user_node: UserNode, data: Dict):
@@ -81,15 +107,22 @@ async def set_tldraw_user_node_file(user_node: UserNode, data: Dict):
fs = ClassroomCopilotFilesystem(db_name=db_name, init_run_type="user")
# Handle path based on environment
if os.getenv("ENVIRONMENT") == "dev":
# In dev mode, use the full system path from the node
if not user_node.path:
raise HTTPException(status_code=400, detail="Node path not found")
base_path = os.path.normpath(user_node.path)
# Use the path directly as provided - it represents the structure from root
if not user_node.node_storage_path:
raise HTTPException(status_code=400, detail="Node path not found")
# The path might already contain parts of the filesystem structure
# We need to construct the full path carefully
if user_node.node_storage_path.startswith("users/"):
# If path starts with users/, remove it since filesystem already has users/ structure
base_path = user_node.node_storage_path[6:] # Remove "users/" prefix
logging.debug(f"Removed 'users/' prefix, base_path is now: {base_path}")
else:
# In prod mode, construct path using formatted email
base_path = formatted_email
base_path = user_node.node_storage_path
logging.debug(f"No 'users/' prefix found, using path as-is: {base_path}")
base_path = os.path.normpath(base_path)
logging.debug(f"Using base path: {base_path}")
# Construct final path including tldraw file
file_path = os.path.join(base_path, "tldraw_file.json")
@@ -99,11 +132,15 @@ async def set_tldraw_user_node_file(user_node: UserNode, data: Dict):
try:
# Ensure directory exists
os.makedirs(os.path.dirname(file_location), exist_ok=True)
directory_location = os.path.dirname(file_location)
os.makedirs(directory_location, exist_ok=True)
logging.debug(f"Ensured directory exists: {directory_location}")
# Write the file
with open(file_location, "w") as file:
json.dump(data, file)
json.dump(data, file, indent=4)
logging.info(f"tldraw file successfully written to: {file_location}")
return {"status": "success"}
except Exception as e:
logging.error(f"Error writing file: {e}")
@@ -112,29 +149,38 @@ async def set_tldraw_user_node_file(user_node: UserNode, data: Dict):
@router.get("/get_tldraw_node_file")
async def read_tldraw_node_file(path: str, db_name: str):
logging.debug(f"Reading tldraw file for path: {path}")
logging.debug(f"Database name: {db_name}")
fs = ClassroomCopilotFilesystem(db_name=db_name, init_run_type="user")
logging.debug(f"Filesystem root path: {fs.root_path}")
# Handle path based on environment
if os.getenv("DEV_MODE") == "true":
# In dev mode, use the full system path from the node
if not path:
raise HTTPException(status_code=400, detail="Path not provided")
logging.debug(f"Using DEV_MODEpath: {path}")
base_path = os.path.normpath(path)
# Use the path directly as provided - it represents the structure from root
if not path:
raise HTTPException(status_code=400, detail="Path not provided")
# The path might already contain parts of the filesystem structure
# We need to construct the full path carefully
if path.startswith("users/"):
# If path starts with users/, remove it since filesystem already has users/ structure
base_path = path[6:] # Remove "users/" prefix
logging.debug(f"Removed 'users/' prefix, base_path is now: {base_path}")
else:
# In prod mode, construct path
logging.warning(f"Using db_name as base path not ready in prod: {db_name}")
base_path = db_name
base_path = path
logging.debug(f"No 'users/' prefix found, using path as-is: {base_path}")
base_path = os.path.normpath(base_path)
logging.debug(f"Using base path: {base_path}")
# Construct final path including tldraw file
logging.debug(f"Base path: {base_path}")
file_path = os.path.join(base_path, "tldraw_file.json")
logging.debug(f"File path: {file_path}")
file_location = os.path.normpath(os.path.join(fs.root_path, file_path))
logging.debug(f"File location: {file_location}")
logging.debug(f"Final file location: {file_location}")
# Debug: Check what directories exist
logging.debug(f"Checking if root path exists: {fs.root_path} - {os.path.exists(fs.root_path)}")
logging.debug(f"Checking if base path exists: {os.path.join(fs.root_path, base_path)} - {os.path.exists(os.path.join(fs.root_path, base_path))}")
logging.debug(f"Attempting to read file at: {file_location}")
@@ -151,8 +197,44 @@ async def read_tldraw_node_file(path: str, db_name: str):
logging.error(f"Error reading file: {e}")
raise HTTPException(status_code=500, detail="Error reading file")
else:
logging.debug(f"File does not exist: {file_location}")
raise HTTPException(status_code=404, detail="File not found")
# Check if directory exists
directory_location = os.path.dirname(file_location)
logging.debug(f"Checking if directory exists: {directory_location} - {os.path.exists(directory_location)}")
if os.path.exists(directory_location):
logging.debug(f"Directory exists but file doesn't, creating default tldraw file at: {file_location}")
try:
# Create default tldraw content
default_tldraw_content = create_default_tldraw_content()
# Ensure directory exists (should already exist, but just in case)
os.makedirs(directory_location, exist_ok=True)
# Write the default file
with open(file_location, "w") as file:
json.dump(default_tldraw_content, file, indent=4)
logging.info(f"Default tldraw file created at: {file_location}")
return default_tldraw_content
except Exception as e:
logging.error(f"Error creating default tldraw file: {e}")
raise HTTPException(status_code=500, detail="Error creating default tldraw file")
else:
logging.debug(f"Neither directory nor file exists: {directory_location}")
# List contents of parent directories to help debug
parent_dir = os.path.dirname(directory_location)
if os.path.exists(parent_dir):
logging.debug(f"Parent directory exists: {parent_dir}")
try:
contents = os.listdir(parent_dir)
logging.debug(f"Parent directory contents: {contents}")
except Exception as e:
logging.debug(f"Could not list parent directory contents: {e}")
else:
logging.debug(f"Parent directory does not exist: {parent_dir}")
raise HTTPException(status_code=404, detail="Directory not found")
@router.post("/set_tldraw_node_file")
async def set_tldraw_node_file(path: str, db_name: str, data: Dict):
@@ -162,22 +244,25 @@ async def set_tldraw_node_file(path: str, db_name: str, data: Dict):
logging.debug(f"Filesystem root path: {fs.root_path}")
# Handle path based on environment
if os.getenv("DEV_MODE") == "true":
# In dev mode, use the full system path from the node
if not path:
raise HTTPException(status_code=400, detail="Path not provided")
logging.debug(f"Using DEV_MODEpath: {path}")
base_path = os.path.normpath(path)
# Use the path directly as provided - it represents the structure from root
if not path:
raise HTTPException(status_code=400, detail="Path not provided")
# The path might already contain parts of the filesystem structure
# We need to construct the full path carefully
if path.startswith("users/"):
# If path starts with users/, remove it since filesystem already has users/ structure
base_path = path[6:] # Remove "users/" prefix
logging.debug(f"Removed 'users/' prefix, base_path is now: {base_path}")
else:
# In prod mode, construct path
logging.warning(f"Using db_name as base path not ready in prod: {db_name}")
base_path = db_name
base_path = path
logging.debug(f"No 'users/' prefix found, using path as-is: {base_path}")
base_path = os.path.normpath(base_path)
logging.debug(f"Using base path: {base_path}")
# Construct final path including tldraw file
logging.debug(f"Base path: {base_path}")
file_path = os.path.join(base_path, "tldraw_file.json")
logging.debug(f"File path: {file_path}")
file_location = os.path.normpath(os.path.join(fs.root_path, file_path))
logging.debug(f"File location: {file_location}")
@@ -185,12 +270,94 @@ async def set_tldraw_node_file(path: str, db_name: str, data: Dict):
try:
# Ensure directory exists
os.makedirs(os.path.dirname(file_location), exist_ok=True)
directory_location = os.path.dirname(file_location)
os.makedirs(directory_location, exist_ok=True)
logging.debug(f"Ensured directory exists: {directory_location}")
# Write the file
with open(file_location, "w") as file:
json.dump(data, file)
json.dump(data, file, indent=4)
logging.info(f"tldraw file successfully written to: {file_location}")
return {"status": "success"}
except Exception as e:
logging.error(f"Error writing file: {e}")
raise HTTPException(status_code=500, detail="Error writing file")
def create_default_tldraw_content():
"""Create default tldraw content structure."""
return {
"document": {
"store": {
"document:document": {
"gridSize": 10,
"name": "",
"meta": {},
"id": "document:document",
"typeName": "document"
},
"page:page": {
"meta": {},
"id": "page:page",
"name": "Page 1",
"index": "a1",
"typeName": "page"
}
},
"schema": {
"schemaVersion": 2,
"sequences": {
"com.tldraw.store": 4,
"com.tldraw.asset": 1,
"com.tldraw.camera": 1,
"com.tldraw.document": 2,
"com.tldraw.instance": 25,
"com.tldraw.instance_page_state": 5,
"com.tldraw.page": 1,
"com.tldraw.instance_presence": 5,
"com.tldraw.pointer": 1,
"com.tldraw.shape": 4,
"com.tldraw.asset.bookmark": 2,
"com.tldraw.asset.image": 5,
"com.tldraw.asset.video": 5,
"com.tldraw.shape.arrow": 5,
"com.tldraw.shape.bookmark": 2,
"com.tldraw.shape.draw": 2,
"com.tldraw.shape.embed": 4,
"com.tldraw.shape.frame": 0,
"com.tldraw.shape.geo": 9,
"com.tldraw.shape.group": 0,
"com.tldraw.shape.highlight": 1,
"com.tldraw.shape.image": 4,
"com.tldraw.shape.line": 5,
"com.tldraw.shape.note": 8,
"com.tldraw.shape.text": 2,
"com.tldraw.shape.video": 2,
"com.tldraw.binding.arrow": 0
}
},
"recordVersions": {
"asset": {"version": 1, "subTypeKey": "type", "subTypeVersions": {}},
"camera": {"version": 1},
"document": {"version": 2},
"instance": {"version": 21},
"instance_page_state": {"version": 5},
"page": {"version": 1},
"shape": {"version": 3, "subTypeKey": "type", "subTypeVersions": {}},
"instance_presence": {"version": 5},
"pointer": {"version": 1}
},
"rootShapeIds": [],
"bindings": [],
"assets": []
},
"session": {
"version": 0,
"currentPageId": "page:page",
"pageStates": [{
"pageId": "page:page",
"camera": {"x": 0, "y": 0, "z": 1},
"selectedShapeIds": []
}]
}
}
routers/database/tools/tldraw_supabase_storage.py
+265
View File
@@ -0,0 +1,265 @@
"""
TLDraw Supabase Storage Router
=============================
Handles TLDraw snapshot operations using Supabase Storage instead of local filesystem.
This replaces the old filesystem-based tldraw_filesystem.py router.
"""
from dotenv import load_dotenv, find_dotenv
load_dotenv(find_dotenv())
import os
import json
import logging
from fastapi import APIRouter, HTTPException, Query
from typing import Dict, Any
from modules.database.supabase.utils.storage import StorageAdmin
from modules.logger_tool import initialise_logger
router = APIRouter()
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
def create_default_tldraw_content():
"""Create default tldraw content structure."""
return {
"document": {
"store": {
"document:document": {
"gridSize": 10,
"name": "",
"meta": {},
"id": "document:document",
"typeName": "document"
},
"page:page": {
"meta": {},
"id": "page:page",
"name": "Page 1",
"index": "a1",
"typeName": "page"
}
},
"schema": {
"schemaVersion": 2,
"sequences": {
"com.tldraw.store": 4,
"com.tldraw.asset": 1,
"com.tldraw.camera": 1,
"com.tldraw.document": 2,
"com.tldraw.instance": 25,
"com.tldraw.instance_page_state": 5,
"com.tldraw.page": 1,
"com.tldraw.instance_presence": 5,
"com.tldraw.pointer": 1,
"com.tldraw.shape": 4,
"com.tldraw.asset.bookmark": 2,
"com.tldraw.asset.image": 5,
"com.tldraw.asset.video": 5,
"com.tldraw.shape.arrow": 5,
"com.tldraw.shape.bookmark": 2,
"com.tldraw.shape.draw": 2,
"com.tldraw.shape.embed": 4,
"com.tldraw.shape.frame": 0,
"com.tldraw.shape.geo": 9,
"com.tldraw.shape.group": 0,
"com.tldraw.shape.highlight": 1,
"com.tldraw.shape.image": 4,
"com.tldraw.shape.line": 5,
"com.tldraw.shape.note": 8,
"com.tldraw.shape.text": 2,
"com.tldraw.shape.video": 2,
"com.tldraw.binding.arrow": 0
}
},
"recordVersions": {
"asset": {"version": 1, "subTypeKey": "type", "subTypeVersions": {}},
"camera": {"version": 1},
"document": {"version": 2},
"instance": {"version": 21},
"instance_page_state": {"version": 5},
"page": {"version": 1},
"shape": {"version": 3, "subTypeKey": "type", "subTypeVersions": {}},
"instance_presence": {"version": 5},
"pointer": {"version": 1}
},
"rootShapeIds": [],
"bindings": [],
"assets": []
},
"session": {
"version": 0,
"currentPageId": "page:page",
"pageStates": [{
"pageId": "page:page",
"camera": {"x": 0, "y": 0, "z": 1},
"selectedShapeIds": []
}]
}
}
@router.get("/get_tldraw_node_file")
async def read_tldraw_node_file_from_supabase(
path: str = Query(..., description="Supabase Storage path (e.g., 'cc.public.snapshots/User/user_id')"),
db_name: str = Query(..., description="Database name for context")
):
"""
Load TLDraw snapshot from Supabase Storage.
Args:
path: Supabase Storage path in format 'bucket/nodetype/node_id'
db_name: Database name for context (used for logging)
Returns:
TLDraw snapshot data
"""
logger.debug(f"Reading tldraw file from Supabase Storage for path: {path}")
logger.debug(f"Database name: {db_name}")
if not path:
raise HTTPException(status_code=400, detail="Path not provided")
try:
# Initialize Supabase Storage
storage = StorageAdmin()
# Parse the path to extract bucket and file path
# Expected format: "cc.public.snapshots/User/user_id" or "cc.public.snapshots/Teacher/teacher_id"
path_parts = path.split('/')
if len(path_parts) < 3:
raise HTTPException(status_code=400, detail="Invalid path format. Expected: bucket/nodetype/node_id")
bucket = path_parts[0] # e.g., "cc.public.snapshots"
node_type = path_parts[1] # e.g., "User", "Teacher"
node_id = path_parts[2] # e.g., "cbc309e5-4029-4c34-aab7-0aa33c563cd0"
# Construct the file path in Supabase Storage
# Format: nodetype/node_id/tldraw_file.json
file_path = f"{node_type}/{node_id}/tldraw_file.json"
logger.debug(f"Bucket: {bucket}")
logger.debug(f"File path: {file_path}")
try:
# Try to download the file from Supabase Storage
file_data = storage.download_file(bucket, file_path)
# Parse JSON data
try:
snapshot_data = json.loads(file_data.decode('utf-8'))
logger.info(f"Successfully loaded tldraw snapshot from Supabase Storage: {file_path}")
# Ensure the snapshot has the correct structure for TLDraw
if isinstance(snapshot_data, dict) and 'document' in snapshot_data and 'session' in snapshot_data:
# Check if it has the new format (schemaVersion in document.schema)
if 'document' in snapshot_data and isinstance(snapshot_data['document'], dict) and 'schema' in snapshot_data['document']:
return snapshot_data
# Check if it has the old format (schemaVersion at root level)
elif 'schemaVersion' in snapshot_data:
return snapshot_data
else:
# Use default structure if schema is missing
logger.warning(f"Snapshot data from {file_path_in_bucket} is missing schemaVersion. Using default structure.")
return create_default_tldraw_content()
else:
# Use default structure if basic structure is missing
logger.warning(f"Snapshot data from {file_path_in_bucket} is missing top-level TLDraw keys. Using default structure.")
return create_default_tldraw_content()
except json.JSONDecodeError as e:
logger.error(f"Failed to parse JSON from Supabase Storage file: {e}")
raise HTTPException(status_code=500, detail="Invalid JSON in file")
except Exception as e:
# File doesn't exist, create default content
logger.info(f"File not found in Supabase Storage, creating default tldraw content: {file_path}")
# Create default tldraw content
default_content = create_default_tldraw_content()
try:
# Upload default content to Supabase Storage
json_data = json.dumps(default_content, indent=2).encode('utf-8')
storage.upload_file(bucket, file_path, json_data, 'application/json', upsert=True)
logger.info(f"Default tldraw file created in Supabase Storage: {file_path}")
return default_content
except Exception as upload_error:
logger.error(f"Error creating default tldraw file in Supabase Storage: {upload_error}")
raise HTTPException(status_code=500, detail="Error creating default tldraw file")
except HTTPException:
# Re-raise HTTP exceptions
raise
except Exception as e:
logger.error(f"Unexpected error loading tldraw file from Supabase Storage: {e}")
raise HTTPException(status_code=500, detail=f"Error loading file: {str(e)}")
@router.post("/set_tldraw_node_file")
async def set_tldraw_node_file_in_supabase(
path: str = Query(..., description="Supabase Storage path (e.g., 'cc.public.snapshots/User/user_id')"),
db_name: str = Query(..., description="Database name for context"),
data: Dict[str, Any] = None
):
"""
Save TLDraw snapshot to Supabase Storage.
Args:
path: Supabase Storage path in format 'bucket/nodetype/node_id'
db_name: Database name for context (used for logging)
data: TLDraw snapshot data to save
Returns:
Success status
"""
logger.debug(f"Saving tldraw file to Supabase Storage for path: {path}")
logger.debug(f"Database name: {db_name}")
if not path:
raise HTTPException(status_code=400, detail="Path not provided")
if not data:
raise HTTPException(status_code=400, detail="Data not provided")
try:
# Initialize Supabase Storage
storage = StorageAdmin()
# Parse the path to extract bucket and file path
path_parts = path.split('/')
if len(path_parts) < 3:
raise HTTPException(status_code=400, detail="Invalid path format. Expected: bucket/nodetype/node_id")
bucket = path_parts[0] # e.g., "cc.public.snapshots"
node_type = path_parts[1] # e.g., "User", "Teacher"
node_id = path_parts[2] # e.g., "cbc309e5-4029-4c34-aab7-0aa33c563cd0"
# Construct the file path in Supabase Storage
file_path = f"{node_type}/{node_id}/tldraw_file.json"
logger.debug(f"Bucket: {bucket}")
logger.debug(f"File path: {file_path}")
# Convert data to JSON
try:
json_data = json.dumps(data, indent=2).encode('utf-8')
except (TypeError, ValueError) as e:
logger.error(f"Failed to serialize data to JSON: {e}")
raise HTTPException(status_code=400, detail="Invalid data format")
# Upload to Supabase Storage
try:
storage.upload_file(bucket, file_path, json_data, 'application/json', upsert=True)
logger.info(f"Successfully saved tldraw snapshot to Supabase Storage: {file_path}")
return {"status": "success", "message": "File saved successfully"}
except Exception as upload_error:
logger.error(f"Error uploading file to Supabase Storage: {upload_error}")
raise HTTPException(status_code=500, detail="Error saving file")
except HTTPException:
# Re-raise HTTP exceptions
raise
except Exception as e:
logger.error(f"Unexpected error saving tldraw file to Supabase Storage: {e}")
raise HTTPException(status_code=500, detail=f"Error saving file: {str(e)}")
routers/database/tools/worker_structure_router.py
+16 -16
View File
@@ -29,34 +29,34 @@ async def get_worker_structure(db_name: str) -> Dict[str, Any]:
// Collect all nodes
RETURN {
timetables: collect(DISTINCT {
id: tt.unique_id,
path: tt.path,
id: tt.uuid_string,
path: tt.node_storage_path,
title: tt.title,
type: tt.__primarylabel__,
startTime: toString(tt.start_date),
endTime: toString(tt.end_date)
}),
classes: collect(DISTINCT {
id: c.unique_id,
path: c.path,
id: c.uuid_string,
path: c.node_storage_path,
title: c.title,
type: c.__primarylabel__
}),
lessons: collect(DISTINCT {
id: l.unique_id,
path: l.path,
id: l.uuid_string,
path: l.node_storage_path,
title: l.title,
type: l.__primarylabel__
}),
journals: collect(DISTINCT {
id: j.unique_id,
path: j.path,
id: j.uuid_string,
path: j.node_storage_path,
title: j.title,
type: j.__primarylabel__
}),
planners: collect(DISTINCT {
id: p.unique_id,
path: p.path,
id: p.uuid_string,
path: p.node_storage_path,
title: p.title,
type: p.__primarylabel__
})
@@ -106,8 +106,8 @@ async def get_timetables(db_name: str, start_date: str, end_date: str) -> Dict[s
MATCH (tt:UserTeacherTimetable)
WHERE date(tt.start_date) >= date($start_date) AND date(tt.end_date) <= date($end_date)
RETURN {
id: tt.unique_id,
path: tt.path,
id: tt.uuid_string,
path: tt.node_storage_path,
title: tt.title,
type: tt.__primarylabel__,
startTime: toString(tt.start_date),
@@ -138,8 +138,8 @@ async def get_journals(db_name: str) -> Dict[str, Any]:
query = """
MATCH (j:Journal)
RETURN {
id: j.unique_id,
path: j.path,
id: j.uuid_string,
path: j.node_storage_path,
title: j.title,
type: j.__primarylabel__
} as journal
@@ -168,8 +168,8 @@ async def get_planners(db_name: str) -> Dict[str, Any]:
query = """
MATCH (p:Planner)
RETURN {
id: p.unique_id,
path: p.path,
id: p.uuid_string,
path: p.node_storage_path,
title: p.title,
type: p.__primarylabel__
} as planner
routers/langchain/neo4j_graph_qa.py
+27 -31
View File
@@ -1,3 +1,4 @@
from weakref import ref
from dotenv import load_dotenv, find_dotenv
load_dotenv(find_dotenv())
import os
@@ -18,36 +19,21 @@ from langchain_community.graphs import Neo4jGraph
from langchain_community.chat_models import ChatOpenAI
from langchain.prompts.prompt import PromptTemplate
from routers.llm.private.ollama.ollama_wrapper import OllamaWrapper
from modules.database.tools.neontology.utils import get_node_types, get_rels_by_type
from modules.database.tools.neontology.basenode import BaseNode
from modules.database.tools.neontology.baserelationship import BaseRelationship
router = APIRouter()
# Define the schema for nodes and relationships
node_types = {
"KeyStage": ["merged", "key_stage_name", "unique_id", "created"],
"KeyStageSyllabus": ["ks_syllabus_name", "unique_id", "created", "merged", "ks_syllabus_key_stage", "ks_syllabus_subject"],
"YearGroup": ["created", "merged", "unique_id", "year_group_name"],
"YearGroupSyllabus": ["created", "merged", "yr_syllabus_name", "yr_syllabus_year_group", "yr_syllabus_id", "yr_syllabus_subject"],
"Topic": ["topic_type", "topic_assessment_type", "created", "merged", "unique_id", "topic_id", "total_number_of_lessons_for_topic", "topic_title"],
"Lesson": ["topic_lesson_id", "topic_lesson_type", "created", "merged", "topic_lesson_title", "topic_lesson_length", "topic_lesson_suggested_activities", "topic_lesson_weblinks", "topic_lesson_skills_learned"],
"LearningStatement": ["created", "merged", "lesson_learning_statement", "lesson_learning_statement_id", "lesson_learning_statement_type"]
}
relationship_types = {
"KEY_STAGE_INCLUDES_KEY_STAGE_SYLLABUS": ["created", "merged"],
"KEY_STAGE_SYLLABUS_INCLUDES_YEAR_GROUP_SYLLABUS": ["created", "merged"],
"YEAR_GROUP_FOLLOWS_YEAR_GROUP": ["created", "merged"],
"KEY_STAGE_FOLLOWS_KEY_STAGE": ["created", "merged"],
"YEAR_SYLLABUS_INCLUDES_TOPIC": ["created", "merged"],
"TOPIC_INCLUDES_LESSON": ["created", "merged"],
"LESSON_INCLUDES_LEARNING_STATEMENT": ["created", "merged"],
"LESSON_FOLLOWS_LESSON": ["created", "merged"]
}
node_types = get_node_types(BaseNode)
relationship_types = get_rels_by_type(BaseRelationship)
@router.get("/prompt")
async def query_graph(
database: str, prompt: str, top_k: int = 30, model: str = "gpt-4o", temperature: float = 0,
database: str, prompt: str, top_k: int = 30, model: str = "qwen2.5-coder:3b", temperature: float = 0,
verbose: bool = False, return_intermediate_steps: bool = False, exclude_types: list = None, include_types: list = None,
return_direct: bool = False, validate_cypher: bool = False, model_type: str = "openai"
return_direct: bool = False, validate_cypher: bool = False, model_type: str = "ollama"
):
logging.info(f"Received request with prompt: {prompt}")
if exclude_types is None:
@@ -70,7 +56,9 @@ async def query_graph(
url=os.environ['APP_BOLT_URL'],
username=os.environ['USER_NEO4J'],
password=os.environ['PASSWORD_NEO4J'],
database=database
database=database,
enhanced_schema=True,
sanitize=True,
)
logging.info("Refreshing schema...")
@@ -79,14 +67,18 @@ async def query_graph(
schema = graph.schema
logging.info(f"Schema: {schema}")
CYPHER_GENERATION_TEMPLATE = """Task: Generate a Cypher statement to query a graph database for timetable information.
CYPHER_GENERATION_TEMPLATE = """Task: Generate a Cypher statement to query a graph database.
Role:
You are an assistant in a school for teachers, specializing in querying graph databases to find answers to questions.
The teacher will ask you questions about their timetable.
You are an assistant specializing in querying graph databases to find answers to questions about establishments, schools, and related data.
The user will ask you questions about the graph database
Instructions:
1. Use only the provided relationship types and properties in the schema.
2. Do not use any other relationship types or properties that are not provided.
3. When querying for geographic entities like counties, towns, or countries, use the 'name' property, not 'code'.
4. To find relationship types, use: MATCH (n:NodeType)-[r]->(m) RETURN DISTINCT type(r)
5. Relationship labels are in uppercase, e.g. LOCATED_IN_COUNTRY
6. For broad queries use OPTIONAL MATCH to allow for null results, e.g. OPTIONAL MATCH (n:NodeType) RETURN n.name
Schema:
{schema}
@@ -95,6 +87,7 @@ async def query_graph(
1. Do not include any explanations or apologies in your responses.
2. Do not respond to any questions that might ask anything else than for you to construct a Cypher statement.
3. Do not include any text except the generated Cypher statement.
4. Do not include line break characters n other formatting keys.
The question is:
{question}"""
@@ -105,11 +98,11 @@ async def query_graph(
)
if model_type == "ollama":
ollama_host = os.getenv("OLLAMA_URL")
ollama_port = os.getenv("OLLAMA_PORT")
ollama_host = os.getenv("HOST_OLLAMA")
ollama_port = os.getenv("PORT_OLLAMA")
if not ollama_host or not ollama_port:
raise HTTPException(status_code=500, detail="Ollama host or port not set")
client = OllamaWrapper(host=f'http://{ollama_host}:{ollama_port}')
client = OllamaWrapper(host=f'{ollama_host}:{ollama_port}', model=model)
cypher_llm = client
qa_llm = client
else:
@@ -127,7 +120,8 @@ async def query_graph(
exclude_types=exclude_types,
include_types=include_types,
return_direct=return_direct,
validate_cypher=validate_cypher
validate_cypher=validate_cypher,
allow_dangerous_requests=True
)
formatted_prompt = CYPHER_GENERATION_PROMPT.format(schema=schema, question=prompt)
@@ -150,4 +144,6 @@ async def query_graph(
logging.info(f"Cypher chain: \n{chain}\n")
logging.info("==================================================")
return chain(prompt)
return chain(prompt)
routers/langchain/test.ipynb
+4 -4
View File
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"execution_count": null,
"metadata": {},
"outputs": [
{
@@ -47,7 +47,7 @@
"import json\n",
"\n",
"# Define the URL of your FastAPI server\n",
"BASE_URL = \"http://localhost:8000\" # Adjust this if your server is running on a different port or host\n",
"BASE_URL = \"http://localhost:8001\" # Adjust this if your server is running on a different port or host\n",
"\n",
"# Define the endpoint\n",
"ENDPOINT = f\"{BASE_URL}/api/langchain/interactive_langgraph_query/query\"\n",
@@ -81,7 +81,7 @@
"\n",
"def test_followup_queries(model=\"openai\"):\n",
" queries = [\n",
" \"What is the latest local news from a particular town?\"\n",
" \"Tell me everything you can about schools in Medway\"\n",
" ]\n",
" \n",
" print(f\"Testing queries requiring follow-up using {model} model:\")\n",
@@ -117,7 +117,7 @@
"#test_simple_queries(\"ollama\")\n",
"\n",
"print(\"\\nRunning simple query tests with OpenAI:\\n\")\n",
"test_simple_queries(\"openai\")\n",
"test_simple_queries(\"ollama\")\n",
"\n",
"#print(\"\\nRunning follow-up query tests with Ollama:\\n\")\n",
"#test_followup_queries(\"ollama\")\n",
routers/llm/private/ollama/ollama_wrapper.py
+4 -2
View File
@@ -4,14 +4,16 @@ from langchain_core.runnables.base import Runnable
from langchain.prompts.base import StringPromptValue
class OllamaWrapper(Runnable):
def __init__(self, host: str):
def __init__(self, host: str, model: str = "llama3.2:latest"):
self.client = Client(host=host)
self.model = model
def invoke(self, prompt: Any, config: Dict[str, Any] = None, **kwargs: Any) -> str:
if isinstance(prompt, StringPromptValue):
prompt = prompt.to_string()
model_name = kwargs.get("model", "llama3")
# Use the model from constructor, but allow override via kwargs
model_name = kwargs.get("model", self.model)
options = {
"temperature": kwargs.get("temperature"),
"top_p": kwargs.get("top_p"),
routers/maintenance/redis_admin.py
+168
View File
@@ -0,0 +1,168 @@
import os
from typing import Any, Dict, List, Optional
from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel
import redis
from modules.logger_tool import initialise_logger
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
router = APIRouter()
def _redis_client() -> redis.Redis:
host = os.getenv('REDIS_HOST', '127.0.0.1')
port = int(os.getenv('REDIS_PORT', '6379'))
return redis.Redis(host=host, port=port, decode_responses=True)
@router.get("/ping")
def ping() -> Dict[str, Any]:
try:
r = _redis_client()
pong = r.ping()
return {"ok": True, "pong": pong}
except Exception as e:
logger.error(f"Redis ping failed: {e}")
raise HTTPException(status_code=500, detail=f"Redis ping failed: {e}")
@router.get("/info")
def info(section: Optional[str] = Query(None, description="Optional INFO section, e.g. memory, server, clients, keyspace")) -> Dict[str, Any]:
try:
r = _redis_client()
data = r.info(section) if section else r.info()
return {"ok": True, "info": data}
except Exception as e:
logger.error(f"Redis info failed: {e}")
raise HTTPException(status_code=500, detail=f"Redis info failed: {e}")
@router.get("/scan")
def scan(cursor: int = 0, pattern: Optional[str] = None, count: int = Query(100, ge=1, le=10000)) -> Dict[str, Any]:
try:
r = _redis_client()
next_cursor, keys = r.scan(cursor=cursor, match=pattern, count=count)
return {"ok": True, "cursor": next_cursor, "keys": keys, "count": len(keys)}
except Exception as e:
logger.error(f"Redis scan failed: {e}")
raise HTTPException(status_code=500, detail=f"Redis scan failed: {e}")
@router.get("/keys")
def list_keys(pattern: str = Query("*", description="Glob-style pattern to match keys"), limit: int = Query(200, ge=1, le=5000)) -> Dict[str, Any]:
try:
r = _redis_client()
keys: List[str] = []
cursor = 0
while True and len(keys) < limit:
cursor, batch = r.scan(cursor=cursor, match=pattern, count=min(1000, limit - len(keys)))
keys.extend(batch)
if cursor == 0:
break
return {"ok": True, "keys": keys[:limit], "count": len(keys[:limit])}
except Exception as e:
logger.error(f"Redis keys failed: {e}")
raise HTTPException(status_code=500, detail=f"Redis keys failed: {e}")
@router.get("/key")
def get_key(key: str) -> Dict[str, Any]:
try:
r = _redis_client()
t = r.type(key)
value: Any = None
if t == 'string':
value = r.get(key)
elif t == 'hash':
value = r.hgetall(key)
elif t == 'list':
value = r.lrange(key, 0, 99)
elif t == 'set':
value = list(r.smembers(key))
elif t == 'zset':
value = r.zrange(key, 0, 99, withscores=True)
ttl = r.ttl(key)
return {"ok": True, "type": t, "ttl": ttl, "value": value}
except Exception as e:
logger.error(f"Redis get key failed: {e}")
raise HTTPException(status_code=500, detail=f"Redis get key failed: {e}")
class DeleteKeysBody(BaseModel):
keys: Optional[List[str]] = None
pattern: Optional[str] = None
@router.post("/delete")
def delete_keys(body: DeleteKeysBody) -> Dict[str, Any]:
if not body.keys and not body.pattern:
raise HTTPException(status_code=400, detail="Provide 'keys' or 'pattern'")
try:
r = _redis_client()
to_delete: List[str] = body.keys or []
if body.pattern:
# Resolve pattern to keys via SCAN to avoid blocking
keys: List[str] = []
cursor = 0
while True:
cursor, batch = r.scan(cursor=cursor, match=body.pattern, count=1000)
keys.extend(batch)
if cursor == 0:
break
to_delete.extend(keys)
deleted = 0
for k in set(to_delete):
try:
deleted += r.delete(k)
except Exception:
pass
return {"ok": True, "deleted": deleted}
except Exception as e:
logger.error(f"Redis delete failed: {e}")
raise HTTPException(status_code=500, detail=f"Redis delete failed: {e}")
@router.post("/flushdb")
def flush_db(confirm: bool = Query(False, description="Must be true to flush the current DB")) -> Dict[str, Any]:
if not confirm:
raise HTTPException(status_code=400, detail="Set confirm=true to flush the current Redis DB")
try:
r = _redis_client()
r.flushdb()
return {"ok": True, "message": "Flushed current Redis DB"}
except Exception as e:
logger.error(f"Redis FLUSHDB failed: {e}")
raise HTTPException(status_code=500, detail=f"Redis FLUSHDB failed: {e}")
@router.get("/queues")
def queue_stats() -> Dict[str, Any]:
"""Report queue sizes and basic counters used by DocumentProcessingQueue."""
try:
r = _redis_client()
keys = {
"high": "queue:high",
"normal": "queue:normal",
"low": "queue:low",
"processing": "processing",
"dead_letter": "dead_letter",
"metrics": "metrics",
}
data = {
"queues": {
"high": r.llen(keys["high"]),
"normal": r.llen(keys["normal"]),
"low": r.llen(keys["low"]),
},
"processing": r.hgetall(keys["processing"]),
"dead_letter": r.llen(keys["dead_letter"]),
"metrics": r.hgetall(keys["metrics"]),
}
return {"ok": True, **data}
except Exception as e:
logger.error(f"Queue stats failed: {e}")
raise HTTPException(status_code=500, detail=f"Queue stats failed: {e}")
routers/provisioning.py
+53
View File
@@ -0,0 +1,53 @@
from datetime import date
from typing import Optional
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel
from modules.auth.supabase_bearer import SupabaseBearer
from modules.database.services.provisioning_service import ProvisioningService
router = APIRouter(prefix="/provisioning", tags=["Provisioning"])
auth = SupabaseBearer()
class ProvisionUserRequest(BaseModel):
user_id: str
class ProvisionUserResponse(BaseModel):
user_db_name: str
worker_db_name: Optional[str]
worker_type: Optional[str]
class ProvisionSchoolRequest(BaseModel):
institute_id: str
class ProvisionSchoolResponse(BaseModel):
db_name: str
curriculum_db_name: str
@router.post("/users", response_model=ProvisionUserResponse)
def provision_user(payload: ProvisionUserRequest, token=Depends(auth)):
"""Ensure a user's Neo4j resources exist."""
# Basic authorization: require matching subject or service role token
if token.get('role') not in ('service_role', 'admin') and token.get('sub') != payload.user_id:
raise HTTPException(status_code=403, detail="Forbidden")
service = ProvisioningService()
result = service.ensure_user(payload.user_id)
return ProvisionUserResponse(**result)
@router.post("/schools", response_model=ProvisionSchoolResponse)
def provision_school(payload: ProvisionSchoolRequest, token=Depends(auth)):
"""Ensure a school's Neo4j resources exist."""
if token.get('role') not in ('service_role', 'admin'):
raise HTTPException(status_code=403, detail="Forbidden")
service = ProvisioningService()
result = service.ensure_school(payload.institute_id)
return ProvisionSchoolResponse(db_name=result['db_name'], curriculum_db_name=result['curriculum_db_name'])
routers/queue_management.py
+313
View File
@@ -0,0 +1,313 @@
"""
Queue Management API
Provides endpoints for monitoring and managing the document processing queue.
"""
from fastapi import APIRouter, HTTPException, Depends
from typing import Dict, Any, List, Optional
from modules.queue_system import get_queue, TaskPriority, ServiceType
from modules.task_processors import get_processor
from modules.auth.supabase_bearer import SupabaseBearer
import os
router = APIRouter()
auth = SupabaseBearer()
@router.get("/queue/stats")
def get_queue_stats(payload: Dict[str, Any] = Depends(auth)):
"""Get comprehensive queue statistics."""
queue = get_queue()
stats = queue.get_queue_stats()
return stats
@router.get("/queue/health")
def queue_health():
"""Check queue health status."""
try:
queue = get_queue()
stats = queue.get_queue_stats()
# Basic health checks
total_processing = stats['total_processing']
total_queued = sum(stats['queues'].values())
dead_letter_count = stats['dead_letter_count']
status = "healthy"
issues = []
if dead_letter_count > 10:
issues.append(f"High dead letter count: {dead_letter_count}")
status = "degraded"
if total_processing == 0 and total_queued > 0:
issues.append("Tasks queued but no workers processing")
status = "degraded"
# Check Redis connectivity
queue.redis_client.ping()
return {
"status": status,
"total_processing": total_processing,
"total_queued": total_queued,
"dead_letter_count": dead_letter_count,
"issues": issues,
"timestamp": queue.redis_client.time()[0]
}
except Exception as e:
return {
"status": "unhealthy",
"error": str(e),
"timestamp": None
}
@router.post("/queue/workers/start")
def start_workers(
worker_count: int = 1,
services: Optional[List[str]] = None,
payload: Dict[str, Any] = Depends(auth)
):
"""Start queue workers."""
if not payload.get('role') == 'service_role': # Admin only
raise HTTPException(status_code=403, detail="Admin access required")
processor = get_processor()
# Convert service names to enums
service_enums = []
if services:
for service_name in services:
try:
service_enums.append(ServiceType(service_name))
except ValueError:
raise HTTPException(status_code=400, detail=f"Invalid service: {service_name}")
else:
service_enums = list(ServiceType)
worker_ids = []
for i in range(worker_count):
worker_id = processor.start_worker(services=service_enums)
worker_ids.append(worker_id)
return {
"message": f"Started {worker_count} workers",
"worker_ids": worker_ids,
"services": [s.value for s in service_enums]
}
@router.post("/queue/workers/stop")
def stop_workers(timeout: int = 30, payload: Dict[str, Any] = Depends(auth)):
"""Stop all queue workers."""
if not payload.get('role') == 'service_role': # Admin only
raise HTTPException(status_code=403, detail="Admin access required")
processor = get_processor()
processor.shutdown(timeout=timeout)
return {"message": "Workers shutdown initiated"}
@router.get("/queue/tasks/{task_id}")
def get_task_status(task_id: str, payload: Dict[str, Any] = Depends(auth)):
"""Get status of a specific task."""
queue = get_queue()
# Get task data from Redis
task_key = queue._get_task_key(task_id)
task_data = queue.redis_client.hgetall(task_key)
if not task_data:
raise HTTPException(status_code=404, detail="Task not found")
return task_data
@router.get("/queue/tasks/by-file/{file_id}")
def list_tasks_by_file(file_id: str, limit: int = 200, payload: Dict[str, Any] = Depends(auth)):
"""List recent queue tasks for a given file_id (best-effort scan)."""
queue = get_queue()
results: List[Dict[str, Any]] = []
count = 0
import json
try:
for key in queue.redis_client.scan_iter(match="task:*"):
task_data = queue.redis_client.hgetall(key)
if not task_data:
continue
if task_data.get('file_id') != file_id:
continue
# Build a brief
try:
payload_raw = task_data.get('payload') or '{}'
payload_obj = json.loads(payload_raw)
except Exception:
payload_obj = {}
results.append({
'id': task_data.get('id') or key.split(':', 1)[-1],
'service': task_data.get('service'),
'task_type': task_data.get('task_type'),
'status': task_data.get('status', 'pending'),
'priority': task_data.get('priority'),
'created_at': float(task_data.get('created_at') or 0),
'scheduled_at': float(task_data.get('scheduled_at') or 0),
'depends_on': payload_obj.get('depends_on') or []
})
count += 1
if count >= max(1, int(limit)):
break
except Exception as e:
raise HTTPException(status_code=500, detail=f"Failed scanning tasks: {e}")
# Sort by created_at desc
results.sort(key=lambda x: x.get('created_at', 0), reverse=True)
return {
'file_id': file_id,
'count': len(results),
'tasks': results
}
@router.get("/queue/tasks/{task_id}/dependencies")
def get_task_dependencies(task_id: str, payload: Dict[str, Any] = Depends(auth)):
"""Inspect a task's dependency state (direct depends_on only)."""
queue = get_queue()
task_key = queue._get_task_key(task_id)
task_data = queue.redis_client.hgetall(task_key)
if not task_data:
raise HTTPException(status_code=404, detail="Task not found")
# Parse payload JSON stored in Redis
import json
try:
payload_raw = task_data.get('payload') or '{}'
payload_obj = json.loads(payload_raw)
except Exception:
payload_obj = {}
depends_on = payload_obj.get('depends_on') or []
if not isinstance(depends_on, list):
depends_on = []
details: List[Dict[str, Any]] = []
missing: List[str] = []
all_completed = True
for dep_id in depends_on:
if not dep_id:
continue
dep_key = queue._get_task_key(dep_id)
dep_data = queue.redis_client.hgetall(dep_key)
if not dep_data:
missing.append(dep_id)
all_completed = False
details.append({
'task_id': dep_id,
'status': 'missing'
})
continue
status = dep_data.get('status', 'pending')
if status != 'completed':
all_completed = False
details.append({
'task_id': dep_id,
'status': status,
'service': dep_data.get('service'),
'task_type': dep_data.get('task_type'),
'created_at': dep_data.get('created_at'),
'scheduled_at': dep_data.get('scheduled_at')
})
return {
'task_id': task_id,
'depends_on': depends_on,
'all_completed': all_completed,
'missing': missing,
'details': details
}
@router.delete("/queue/dead-letter/{task_id}")
def remove_dead_task(task_id: str, payload: Dict[str, Any] = Depends(auth)):
"""Remove a task from the dead letter queue."""
if not payload.get('role') == 'service_role': # Admin only
raise HTTPException(status_code=403, detail="Admin access required")
queue = get_queue()
# Remove from dead letter queue
removed = queue.redis_client.lrem(queue.dead_letter_key, 1, task_id)
if removed == 0:
raise HTTPException(status_code=404, detail="Task not found in dead letter queue")
# Clean up task data
queue.redis_client.delete(queue._get_task_key(task_id))
return {"message": f"Removed task {task_id} from dead letter queue"}
@router.post("/queue/dead-letter/{task_id}/retry")
def retry_dead_task(task_id: str, payload: Dict[str, Any] = Depends(auth)):
"""Retry a task from the dead letter queue."""
if not payload.get('role') == 'service_role': # Admin only
raise HTTPException(status_code=403, detail="Admin access required")
queue = get_queue()
# Get task data
task_key = queue._get_task_key(task_id)
task_data = queue.redis_client.hgetall(task_key)
if not task_data:
raise HTTPException(status_code=404, detail="Task not found")
# Remove from dead letter queue
removed = queue.redis_client.lrem(queue.dead_letter_key, 1, task_id)
if removed == 0:
raise HTTPException(status_code=404, detail="Task not found in dead letter queue")
# Reset task for retry
queue.redis_client.hset(
task_key,
mapping={
'attempts': 0,
'status': 'pending',
'scheduled_at': queue.redis_client.time()[0]
}
)
# Re-queue task
priority = TaskPriority(task_data['priority'])
queue_key = queue.queue_keys[priority]
queue.redis_client.lpush(queue_key, task_id)
return {"message": f"Task {task_id} requeued for retry"}
@router.get("/queue/metrics")
def get_queue_metrics(hours: int = 1, payload: Dict[str, Any] = Depends(auth)):
"""Get queue processing metrics over time."""
queue = get_queue()
# This is a simplified version - in production you'd want more sophisticated metrics
current_time = queue.redis_client.time()[0]
start_time = current_time - (hours * 3600)
# Scan for metric keys in the time range
metrics = {}
pattern = f"{queue.metrics_key}:*"
for key in queue.redis_client.scan_iter(match=pattern):
# Parse key format: metrics:action:service:priority:timestamp_minute
parts = key.split(':')
if len(parts) >= 5:
action = parts[1]
service = parts[2]
priority = parts[3]
timestamp_minute = int(parts[4])
if timestamp_minute >= (start_time // 60):
count = int(queue.redis_client.get(key) or 0)
metric_key = f"{action}_{service}_{priority}"
if metric_key not in metrics:
metrics[metric_key] = 0
metrics[metric_key] += count
return {
"time_range_hours": hours,
"metrics": metrics
}
routers/queue_monitor.py
+174
View File
@@ -0,0 +1,174 @@
"""
Queue Monitoring Router
Provides endpoints to monitor queue status, service limits, and processing counts
to help debug issues like Docling server overload.
"""
from fastapi import APIRouter, Depends
from typing import Dict, Any, List
from modules.auth.supabase_bearer import SupabaseBearer
from modules.queue_system import get_queue
from modules.redis_manager import get_redis_manager
from modules.logger_tool import initialise_logger
import os
router = APIRouter()
auth = SupabaseBearer()
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
@router.get("/queue/status")
def get_queue_status(payload: Dict[str, Any] = Depends(auth)):
"""Get comprehensive queue status including service limits and current processing counts."""
try:
queue = get_queue()
stats = queue.get_queue_stats()
# Add environment configuration
config = {
'QUEUE_DOCLING_LIMIT': int(os.getenv('QUEUE_DOCLING_LIMIT', '2')),
'QUEUE_TIKA_LIMIT': int(os.getenv('QUEUE_TIKA_LIMIT', '3')),
'QUEUE_LLM_LIMIT': int(os.getenv('QUEUE_LLM_LIMIT', '5')),
'BUNDLE_ARCHITECTURE_ENABLED': True,
'AUTO_DOCLING_OCR': os.getenv('AUTO_DOCLING_OCR', 'true'),
'AUTO_DOCLING_NO_OCR': os.getenv('AUTO_DOCLING_NO_OCR', 'true'),
'AUTO_DOCLING_VLM': os.getenv('AUTO_DOCLING_VLM', 'false')
}
return {
'queue_statistics': stats,
'configuration': config,
'redis_host': queue.redis_host,
'redis_port': queue.redis_port,
'service_limits': dict(queue.service_limits),
'rate_limits': dict(queue.rate_limits)
}
except Exception as e:
logger.error(f"Failed to get queue status: {e}")
return {
'error': str(e),
'queue_available': False
}
@router.get("/queue/tasks/by-file/{file_id}")
def get_file_queue_tasks(file_id: str, payload: Dict[str, Any] = Depends(auth)):
"""Get all queue tasks for a specific file (for debugging)."""
try:
# This would require extending the queue system to track tasks by file_id
# For now, return a placeholder that indicates this feature needs implementation
return {
'message': 'File-specific task tracking not yet implemented',
'file_id': file_id,
'suggestion': 'Check Redis directly or extend queue system with file_id indexing'
}
except Exception as e:
logger.error(f"Failed to get tasks for file {file_id}: {e}")
return {
'error': str(e),
'file_id': file_id
}
@router.get("/queue/health")
def get_comprehensive_queue_health(payload: Dict[str, Any] = Depends(auth)):
"""Get comprehensive queue and Redis health information with environment details."""
try:
# Determine environment
environment = 'dev' if os.getenv('BACKEND_DEV_MODE', 'true').lower() == 'true' else 'prod'
# Get Redis manager health
redis_manager = get_redis_manager(environment)
redis_health = redis_manager.health_check()
# Get queue stats
queue_stats = {}
queue_available = False
try:
queue = get_queue()
queue_stats = queue.get_queue_stats()
queue_available = True
except Exception as e:
queue_stats = {'error': str(e)}
# Environment configuration
env_config = {
'current_environment': environment,
'redis_database': redis_health.get('database', 'unknown'),
'persistence_enabled': os.getenv(f'REDIS_PERSIST_{environment.upper()}', 'unknown'),
'task_ttl': os.getenv(f'REDIS_TASK_TTL_{environment.upper()}', 'unknown'),
'service_limits': {
'docling': int(os.getenv('QUEUE_DOCLING_LIMIT', '2')),
'tika': int(os.getenv('QUEUE_TIKA_LIMIT', '3')),
'llm': int(os.getenv('QUEUE_LLM_LIMIT', '5')),
'split_map': int(os.getenv('QUEUE_SPLIT_MAP_LIMIT', '10')),
'document_analysis': int(os.getenv('QUEUE_DOCUMENT_ANALYSIS_LIMIT', '5')),
'page_images': int(os.getenv('QUEUE_PAGE_IMAGES_LIMIT', '3'))
},
'workers': int(os.getenv('QUEUE_WORKERS', '1'))
}
# Overall health status
overall_status = 'healthy'
issues = []
if redis_health['status'] != 'healthy':
overall_status = 'unhealthy'
issues.append(f"Redis: {redis_health.get('error', 'Unknown issue')}")
if not queue_available:
overall_status = 'degraded'
issues.append(f"Queue system: {queue_stats.get('error', 'Not accessible')}")
# Check for concerning queue states
if queue_available and queue_stats.get('dead_letter_count', 0) > 0:
overall_status = 'warning'
issues.append(f"Dead letter queue has {queue_stats['dead_letter_count']} failed tasks")
return {
'status': overall_status,
'issues': issues,
'timestamp': redis_health.get('timestamp'),
'environment': env_config,
'redis': redis_health,
'queue': queue_stats,
'recommendations': _get_health_recommendations(redis_health, queue_stats, environment)
}
except Exception as e:
logger.error(f"Failed to get comprehensive queue health: {e}")
return {
'status': 'error',
'error': str(e),
'timestamp': None
}
def _get_health_recommendations(redis_health: Dict, queue_stats: Dict, environment: str) -> List[str]:
"""Generate health recommendations based on current state."""
recommendations = []
# Redis recommendations
if redis_health.get('status') != 'healthy':
recommendations.append("Check Redis service status and connectivity")
# Queue recommendations
if queue_stats.get('dead_letter_count', 0) > 5:
recommendations.append("High number of failed tasks - check task processing logic")
# Processing recommendations
processing_counts = queue_stats.get('processing', {})
for service, count in processing_counts.items():
if count < 0:
recommendations.append(f"Negative processing counter for {service} - restart server to reset")
# Environment-specific recommendations
if environment == 'dev':
recommendations.append("Development mode: Database will be cleared on restart")
else:
recommendations.append("Production mode: Tasks will be recovered on restart")
if not recommendations:
recommendations.append("System appears healthy - no specific recommendations")
return recommendations
routers/simple_upload.py
+543
View File
@@ -0,0 +1,543 @@
"""
Simple Upload Router
===================
Handles file and directory uploads without automatic processing.
Just stores files in Supabase storage and creates database records.
Features:
- Single file upload
- Directory/folder upload with manifest
- No automatic processing
- Immediate response to users
- Directory structure preservation
"""
import os
import uuid
import json
import tempfile
import logging
from typing import Dict, List, Optional, Any
from pathlib import Path
from fastapi import APIRouter, Depends, HTTPException, UploadFile, File, Form, BackgroundTasks
from fastapi.responses import JSONResponse
from modules.auth.supabase_bearer import SupabaseBearer
from modules.database.supabase.utils.client import SupabaseServiceRoleClient
from modules.database.supabase.utils.storage import StorageAdmin
from modules.logger_tool import initialise_logger
router = APIRouter()
auth = SupabaseBearer()
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
def _choose_bucket(scope: str, user_id: str, school_id: Optional[str]) -> str:
"""Choose appropriate bucket based on scope - matches old system logic."""
scope = (scope or 'teacher').lower()
if scope == 'school' and school_id:
return f"cc.institutes.{school_id}.private"
# teacher / student fall back to users bucket for now
return 'cc.users'
@router.post("/files/upload")
async def upload_single_file(
cabinet_id: str = Form(...),
path: str = Form(...),
scope: str = Form(...),
file: UploadFile = File(...),
payload: Dict[str, Any] = Depends(auth)
):
"""
Simple single file upload - no automatic processing.
Just stores the file and creates a database record.
"""
try:
user_id = payload.get('sub') or payload.get('user_id')
if not user_id:
raise HTTPException(status_code=401, detail="User ID required")
# Read file content
file_bytes = await file.read()
file_size = len(file_bytes)
mime_type = file.content_type or 'application/octet-stream'
filename = file.filename or path
logger.info(f"📤 Simple upload: {filename} ({file_size} bytes) for user {user_id}")
# Initialize services
client = SupabaseServiceRoleClient()
storage = StorageAdmin()
# Generate file ID and storage path
file_id = str(uuid.uuid4())
# Use same bucket logic as old system for consistency
bucket = _choose_bucket('teacher', user_id, None) # Default to teacher scope
storage_path = f"{cabinet_id}/{file_id}/{filename}"
# Store file in Supabase storage
try:
storage.upload_file(bucket, storage_path, file_bytes, mime_type, upsert=True)
except Exception as e:
logger.error(f"Storage upload failed for {file_id}: {e}")
raise HTTPException(status_code=500, detail=f"Storage upload failed: {str(e)}")
# Create database record
try:
insert_res = client.supabase.table('files').insert({
'id': file_id,
'name': filename,
'cabinet_id': cabinet_id,
'bucket': bucket,
'path': storage_path,
'mime_type': mime_type,
'uploaded_by': user_id,
'size_bytes': file_size,
'source': 'classroomcopilot-web',
'is_directory': False,
'processing_status': 'uploaded',
'relative_path': filename # For single files, relative path is just the filename
}).execute()
if not insert_res.data:
# Clean up storage on DB failure
try:
storage.delete_file(bucket, storage_path)
except:
pass
raise HTTPException(status_code=500, detail="Failed to create file record")
file_record = insert_res.data[0]
except Exception as e:
logger.error(f"Database insert failed for {file_id}: {e}")
# Clean up storage
try:
storage.delete_file(bucket, storage_path)
except:
pass
raise HTTPException(status_code=500, detail=f"Database error: {str(e)}")
logger.info(f"✅ Simple upload completed: {file_id}")
return {
'status': 'success',
'message': 'File uploaded successfully',
'file': file_record,
'processing_required': False, # No automatic processing
'next_steps': 'File is ready for manual processing if needed'
}
except HTTPException:
raise
except Exception as e:
logger.error(f"Upload error: {e}")
raise HTTPException(status_code=500, detail=f"Upload failed: {str(e)}")
@router.post("/files/upload-directory")
async def upload_directory(
cabinet_id: str = Form(...),
scope: str = Form(...),
directory_name: str = Form(...),
files: List[UploadFile] = File(...),
file_paths: str = Form(...), # JSON string of relative paths
payload: Dict[str, Any] = Depends(auth)
):
"""
Upload a complete directory/folder with all files.
Preserves directory structure and creates a manifest.
"""
try:
user_id = payload.get('sub') or payload.get('user_id')
if not user_id:
raise HTTPException(status_code=401, detail="User ID required")
# Parse file paths
try:
relative_paths = json.loads(file_paths)
except json.JSONDecodeError:
raise HTTPException(status_code=400, detail="Invalid file_paths JSON")
if len(files) != len(relative_paths):
raise HTTPException(status_code=400, detail="Files and paths count mismatch")
logger.info(f"📁 Directory upload: {directory_name} ({len(files)} files) for user {user_id}")
# Initialize services
client = SupabaseServiceRoleClient()
storage = StorageAdmin()
# Generate session ID for this directory upload
upload_session_id = str(uuid.uuid4())
directory_id = str(uuid.uuid4())
# Use same bucket logic as old system for consistency
bucket = _choose_bucket('teacher', user_id, None)
# Calculate total size and build manifest
total_size = 0
directory_structure = {}
uploaded_files = []
directory_records = {} # Track created directories: {relative_path: directory_id}
# First, analyze all paths to determine directory structure
all_directories = set()
for relative_path in relative_paths:
# Get all parent directories for this file
path_parts = relative_path.split('/')
for i in range(len(path_parts) - 1): # Exclude the filename
dir_path = '/'.join(path_parts[:i+1])
all_directories.add(dir_path)
logger.info(f"📁 Creating directory structure with {len(all_directories)} directories: {sorted(all_directories)}")
try:
# Create directory records for all directories (sorted to create parents first)
sorted_directories = sorted(all_directories, key=lambda x: (len(x.split('/')), x))
for dir_path in sorted_directories:
dir_id = str(uuid.uuid4())
path_parts = dir_path.split('/')
dir_name = path_parts[-1] # Last part is the directory name
# Determine parent directory
parent_id = None
if len(path_parts) > 1:
parent_path = '/'.join(path_parts[:-1])
parent_id = directory_records.get(parent_path)
directory_record = {
'id': dir_id,
'name': dir_name,
'cabinet_id': cabinet_id,
'bucket': bucket,
'path': f"{cabinet_id}/{dir_id}/",
'mime_type': 'inode/directory',
'uploaded_by': user_id,
'size_bytes': 0,
'source': 'classroomcopilot-web',
'is_directory': True,
'parent_directory_id': parent_id,
'upload_session_id': upload_session_id,
'processing_status': 'uploaded',
'relative_path': dir_path
}
directory_records[dir_path] = dir_id
# Insert directory record
result = client.supabase.table('files').insert(directory_record).execute()
logger.info(f"📁 Created directory: {dir_path} (ID: {dir_id}, Parent: {parent_id})")
# Process each file
for i, (file, relative_path) in enumerate(zip(files, relative_paths)):
try:
# Read file content
file_bytes = await file.read()
file_size = len(file_bytes)
mime_type = file.content_type or 'application/octet-stream'
filename = file.filename or f"file_{i}"
total_size += file_size
# Generate file ID and determine parent directory
file_id = str(uuid.uuid4())
# Find the correct parent directory for this file
path_parts = relative_path.split('/')
if len(path_parts) > 1:
parent_dir_path = '/'.join(path_parts[:-1])
parent_directory_id = directory_records.get(parent_dir_path)
else:
parent_directory_id = None # File is in root cabinet
# Use parent directory ID for storage path if exists, otherwise use a generated path
if parent_directory_id:
storage_path = f"{cabinet_id}/{parent_directory_id}/{path_parts[-1]}"
else:
storage_path = f"{cabinet_id}/{file_id}"
# Store file in Supabase storage
storage.upload_file(bucket, storage_path, file_bytes, mime_type, upsert=True)
# Create file record
file_record = {
'id': file_id,
'name': filename,
'cabinet_id': cabinet_id,
'bucket': bucket,
'path': storage_path,
'mime_type': mime_type,
'uploaded_by': user_id,
'size_bytes': file_size,
'source': 'classroomcopilot-web',
'is_directory': False,
'parent_directory_id': parent_directory_id,
'relative_path': relative_path,
'upload_session_id': upload_session_id,
'processing_status': 'uploaded'
}
uploaded_files.append(file_record)
# Build directory structure for manifest
_add_to_directory_structure(directory_structure, relative_path, {
'size': file_size,
'mime_type': mime_type,
'file_id': file_id
})
logger.info(f"📄 Uploaded file {i+1}/{len(files)}: {relative_path}")
except Exception as e:
logger.error(f"Failed to upload file {relative_path}: {e}")
# Continue with other files, don't fail entire upload
continue
# Create directory manifest
directory_manifest = {
'total_files': len(uploaded_files),
'total_size_bytes': total_size,
'directory_structure': directory_structure,
'upload_timestamp': '2024-09-23T12:00:00Z', # TODO: Use actual timestamp
'upload_method': 'directory_picker',
'upload_session_id': upload_session_id
}
# Update root directory with manifest and total size (if root directory exists)
root_directory_id = directory_records.get(sorted_directories[0]) if sorted_directories else None
if root_directory_id:
update_res = client.supabase.table('files').update({
'size_bytes': total_size,
'directory_manifest': directory_manifest
}).eq('id', root_directory_id).execute()
if not update_res.data:
logger.warning("Failed to update root directory with manifest")
# Insert all file records in batch
if uploaded_files:
files_insert_res = client.supabase.table('files').insert(uploaded_files).execute()
if not files_insert_res.data:
logger.warning("Some file records failed to insert")
logger.info(f"✅ Directory upload completed: {root_directory_id} ({len(uploaded_files)} files, {len(sorted_directories)} directories)")
return {
'status': 'success',
'message': f'Directory uploaded successfully with {len(uploaded_files)} files in {len(sorted_directories)} directories',
'directories_created': len(sorted_directories),
'files_count': len(uploaded_files),
'total_size_bytes': total_size,
'root_directory_id': root_directory_id,
'upload_session_id': upload_session_id,
'processing_required': False, # No automatic processing
'next_steps': 'Files are ready for manual processing if needed'
}
except Exception as e:
logger.error(f"Directory upload failed: {e}")
# TODO: Implement cleanup of partially uploaded files
raise HTTPException(status_code=500, detail=f"Directory upload failed: {str(e)}")
except HTTPException:
raise
except Exception as e:
logger.error(f"Directory upload error: {e}")
raise HTTPException(status_code=500, detail=f"Upload failed: {str(e)}")
def _add_to_directory_structure(structure: Dict, relative_path: str, file_info: Dict):
"""Add a file to the directory structure manifest."""
path_parts = relative_path.split('/')
current_level = structure
# Navigate/create directory structure
for i, part in enumerate(path_parts):
if i == len(path_parts) - 1:
# This is the file itself
current_level[part] = file_info
else:
# This is a directory
if part not in current_level:
current_level[part] = {}
current_level = current_level[part]
@router.get("/files")
def list_files(
cabinet_id: str,
include_directories: bool = True,
parent_directory_id: Optional[str] = None,
page: int = 1,
per_page: int = 20,
search: Optional[str] = None,
sort_by: str = 'created_at',
sort_order: str = 'desc',
payload: Dict[str, Any] = Depends(auth)
):
"""
List files with pagination, search, and sorting support.
Args:
cabinet_id: Cabinet to list files from
include_directories: Whether to include directory entries
parent_directory_id: Filter by parent directory
page: Page number (1-based)
per_page: Items per page (max 100)
search: Search term for filename
sort_by: Field to sort by (name, size_bytes, created_at, processing_status)
sort_order: Sort order (asc, desc)
"""
try:
client = SupabaseServiceRoleClient()
# Validate pagination parameters
page = max(1, page)
per_page = min(max(1, per_page), 100) # Limit to 100 items per page
offset = (page - 1) * per_page
# Validate sort parameters
valid_sort_fields = ['name', 'size_bytes', 'created_at', 'processing_status', 'mime_type']
if sort_by not in valid_sort_fields:
sort_by = 'created_at'
if sort_order.lower() not in ['asc', 'desc']:
sort_order = 'desc'
# Build base query
query = client.supabase.table('files').select('*').eq('cabinet_id', cabinet_id)
count_query = client.supabase.table('files').select('id', count='exact').eq('cabinet_id', cabinet_id)
# Apply filters
if parent_directory_id:
query = query.eq('parent_directory_id', parent_directory_id)
count_query = count_query.eq('parent_directory_id', parent_directory_id)
elif not include_directories:
query = query.eq('is_directory', False)
count_query = count_query.eq('is_directory', False)
# Apply search filter
if search:
search_term = f"%{search}%"
query = query.ilike('name', search_term)
count_query = count_query.ilike('name', search_term)
# Get total count
count_res = count_query.execute()
total_count = count_res.count if hasattr(count_res, 'count') else len(count_res.data or [])
# Apply sorting and pagination
query = query.order(sort_by, desc=(sort_order.lower() == 'desc'))
query = query.range(offset, offset + per_page - 1)
res = query.execute()
files = res.data or []
# Calculate pagination metadata
total_pages = (total_count + per_page - 1) // per_page
has_next = page < total_pages
has_prev = page > 1
return {
'files': files,
'pagination': {
'page': page,
'per_page': per_page,
'total_count': total_count,
'total_pages': total_pages,
'has_next': has_next,
'has_prev': has_prev,
'offset': offset
},
'filters': {
'search': search,
'sort_by': sort_by,
'sort_order': sort_order,
'include_directories': include_directories,
'parent_directory_id': parent_directory_id
}
}
except Exception as e:
logger.error(f"List files error: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.get("/files/{file_id}")
def get_file_details(file_id: str, payload: Dict[str, Any] = Depends(auth)):
"""Get detailed information about a file or directory."""
try:
client = SupabaseServiceRoleClient()
res = client.supabase.table('files').select('*').eq('id', file_id).single().execute()
if not res.data:
raise HTTPException(status_code=404, detail="File not found")
file_data = res.data
# If it's a directory, also get its contents
if file_data.get('is_directory'):
contents_res = client.supabase.table('files').select('*').eq('parent_directory_id', file_id).execute()
file_data['contents'] = contents_res.data or []
return file_data
except HTTPException:
raise
except Exception as e:
logger.error(f"Get file details error: {e}")
raise HTTPException(status_code=500, detail=str(e))
@router.delete("/files/{file_id}")
def delete_file(file_id: str, payload: Dict[str, Any] = Depends(auth)):
"""Delete a file or directory and its contents."""
try:
client = SupabaseServiceRoleClient()
storage = StorageAdmin()
# Get file info
res = client.supabase.table('files').select('*').eq('id', file_id).single().execute()
if not res.data:
raise HTTPException(status_code=404, detail="File not found")
file_data = res.data
# If it's a directory, delete all contents first
if file_data.get('is_directory'):
contents_res = client.supabase.table('files').select('*').eq('parent_directory_id', file_id).execute()
# Delete each file in the directory
for content_file in contents_res.data or []:
try:
# Delete from storage
storage.delete_file(content_file['bucket'], content_file['path'])
except Exception as e:
logger.warning(f"Failed to delete file from storage: {content_file['path']}: {e}")
# Delete all directory contents from database
client.supabase.table('files').delete().eq('parent_directory_id', file_id).execute()
else:
# Delete single file from storage
try:
storage.delete_file(file_data['bucket'], file_data['path'])
except Exception as e:
logger.warning(f"Failed to delete file from storage: {file_data['path']}: {e}")
# Delete the main record
delete_res = client.supabase.table('files').delete().eq('id', file_id).execute()
logger.info(f"🗑️ Deleted {'directory' if file_data.get('is_directory') else 'file'}: {file_id}")
return {
'status': 'success',
'message': f"{'Directory' if file_data.get('is_directory') else 'File'} deleted successfully"
}
except HTTPException:
raise
except Exception as e:
logger.error(f"Delete file error: {e}")
raise HTTPException(status_code=500, detail=str(e))