Initial commit

routers/admin_panel_routes.py

@@ -0,0 +1,113 @@
+from fastapi import APIRouter, Request, Depends, HTTPException
+from fastapi.responses import HTMLResponse
+from fastapi.templating import Jinja2Templates
+import os
+from modules.logger_tool import initialise_logger
+from modules.database.services.school_admin_service import SchoolAdminService
+from modules.database.supabase.utils.storage import StorageManager
+from .auth import verify_admin
+from typing import Dict
+
+router = APIRouter()
+templates = Jinja2Templates(directory="templates")
+logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
+
+# Initialize services
+school_service = SchoolAdminService()
+storage_manager = StorageManager()
+
+@router.get("/schools/manage", response_class=HTMLResponse)
+async def manage_schools(request: Request, admin: Dict = Depends(verify_admin)):
+    """Manage schools page"""
+    return templates.TemplateResponse(
+        "admin/schools/manage.html",
+        {"request": request, "admin": admin}
+    )
+
+@router.get("/storage/manage", response_class=HTMLResponse)
+async def manage_storage(request: Request, admin: Dict = Depends(verify_admin)):
+    """Storage management page"""
+    try:
+        # Get list of storage buckets with correct IDs
+        buckets = [
+            {
+                "id": "cc.institutes",
+                "name": "School Files",
+                "public": False,
+                "file_size_limit": 50 * 1024 * 1024,  # 50MB
+                "allowed_mime_types": [
+                    "image/*",
+                    "video/*",
+                    "application/pdf",
+                    "application/msword",
+                    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+                    "application/vnd.ms-excel",
+                    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+                    "application/vnd.ms-powerpoint",
+                    "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+                    "text/plain",
+                    "text/csv",
+                    "application/json"
+                ]
+            },
+            {
+                "id": "cc.users",
+                "name": "User Files",
+                "public": False,
+                "file_size_limit": 50 * 1024 * 1024,  # 50MB
+                "allowed_mime_types": [
+                    "image/*",
+                    "video/*",
+                    "application/pdf",
+                    "application/msword",
+                    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+                    "application/vnd.ms-excel",
+                    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+                    "application/vnd.ms-powerpoint",
+                    "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+                    "text/plain",
+                    "text/csv",
+                    "application/json"
+                ]
+            }
+        ]
+        
+        return templates.TemplateResponse(
+            "admin/storage/manage.html",
+            {"request": request, "admin": admin, "buckets": buckets}
+        )
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/schema", response_class=HTMLResponse)
+async def manage_schema(request: Request, admin: Dict = Depends(verify_admin)):
+    """Schema management page"""
+    return templates.TemplateResponse(
+        "admin/schema/manage.html",
+        {"request": request, "admin": admin}
+    )
+
+@router.get("/storage/{bucket_id}/contents")
+async def list_bucket_contents(
+    request: Request,
+    bucket_id: str,
+    path: str = "",
+    admin: Dict = Depends(verify_admin)
+):
+    """List contents of a storage bucket"""
+    try:
+        contents = storage_manager.list_bucket_contents(bucket_id, path)
+        bucket = {"id": bucket_id, "name": bucket_id.replace("_", " ").title()}
+        
+        return templates.TemplateResponse(
+            "admin/storage/contents.html",
+            {
+                "request": request,
+                "admin": admin,
+                "bucket": bucket,
+                "contents": contents,
+                "current_path": path
+            }
+        )
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))

routers/admin_routes.py

@@ -0,0 +1,498 @@
+from fastapi import APIRouter, Request, Depends, HTTPException, File, UploadFile, Form
+from fastapi.responses import HTMLResponse, JSONResponse
+from fastapi.templating import Jinja2Templates
+from typing import Dict
+import os
+from modules.logger_tool import initialise_logger
+from modules.database.services.admin_service import AdminService, AdminProfileBase
+from modules.database.services.school_admin_service import SchoolAdminService
+from modules.database.supabase.utils.client import SupabaseAnonClient
+from modules.database.supabase.utils.storage import StorageManager
+from .auth import verify_admin
+import csv
+import io
+
+router = APIRouter()
+templates = Jinja2Templates(directory="templates")
+logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
+
+# Initialize services
+admin_service = AdminService()
+school_service = SchoolAdminService()
+storage_manager = StorageManager(SupabaseAnonClient)
+
+@router.get("/", response_class=HTMLResponse)
+async def admin_dashboard(request: Request, admin: Dict = Depends(verify_admin)):
+    """Render admin dashboard"""
+    return templates.TemplateResponse(
+        "admin/dashboard/index.html",
+        {
+            "request": request, 
+            "admin": admin,
+            "app_version": os.getenv("APP_VERSION", "Unknown")
+        }
+    )
+
+@router.get("/users")
+async def list_users(request: Request, admin: Dict = Depends(verify_admin)):
+    """List all users"""
+    return templates.TemplateResponse(
+        "admin/users/list.html",
+        {"request": request, "admin": admin}
+    )
+
+@router.get("/users/{user_id}")
+async def get_user(request: Request, user_id: str, admin: Dict = Depends(verify_admin)):
+    """Get user details"""
+    return templates.TemplateResponse(
+        "admin/users/detail.html",
+        {"request": request, "admin": admin, "user_id": user_id}
+    )
+
+@router.get("/admins")
+async def list_admins(request: Request, admin: Dict = Depends(verify_admin)):
+    """List all admins"""
+    if not admin.get("is_super_admin"):
+        raise HTTPException(status_code=403, detail="Only super admins can view admin list")
+    
+    admins = admin_service.list_admins()
+    return templates.TemplateResponse(
+        "admin/users/admins.html",
+        {"request": request, "admin": admin, "admins": admins}
+    )
+
+@router.post("/admins")
+async def create_admin(admin_data: AdminProfileBase, current_admin: Dict = Depends(verify_admin)):
+    """Create a new admin"""
+    try:
+        result = admin_service.create_admin(admin_data, current_admin)
+        return JSONResponse(content={"status": "success", "admin": result})
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+@router.get("/schools/manage", response_class=HTMLResponse)
+async def manage_schools(request: Request, admin: Dict = Depends(verify_admin)):
+    """Manage schools page"""
+    try:
+        # Fetch schools from Supabase
+        result = admin_service.supabase.table("schools").select("*").execute()
+        schools = result.data if result else []
+        
+        # Sort schools by establishment_name
+        schools.sort(key=lambda x: x.get("establishment_name", ""))
+        
+        return templates.TemplateResponse(
+            "admin/schools/manage.html",
+            {
+                "request": request, 
+                "admin": admin,
+                "schools": schools,
+                "schools_count": len(schools)
+            }
+        )
+    except Exception as e:
+        logger.error(f"Error fetching schools: {str(e)}")
+        return templates.TemplateResponse(
+            "admin/schools/manage.html",
+            {
+                "request": request,
+                "admin": admin,
+                "schools": [],
+                "schools_count": 0,
+                "error": str(e)
+            }
+        )
+
+@router.post("/schools/import")
+async def import_schools(
+    file: UploadFile = File(...),
+    admin: Dict = Depends(verify_admin)
+):
+    """Import schools from CSV file"""
+    if not file.filename.endswith('.csv'):
+        raise HTTPException(status_code=400, detail="Please upload a CSV file")
+    
+    try:
+        # Process the CSV file
+        content = await file.read()
+        csv_text = content.decode('utf-8-sig')  # Handle BOM if present
+        csv_reader = csv.DictReader(io.StringIO(csv_text))
+        
+        # Prepare data for batch insert
+        schools_data = []
+        for row in csv_reader:
+            school_data = {
+                "urn": row.get("URN"),
+                "la_code": row.get("LA (code)"),
+                "la_name": row.get("LA (name)"),
+                "establishment_number": row.get("EstablishmentNumber"),
+                "establishment_name": row.get("EstablishmentName"),
+                "establishment_type": row.get("TypeOfEstablishment (name)"),
+                "establishment_type_group": row.get("EstablishmentTypeGroup (name)"),
+                "establishment_status": row.get("EstablishmentStatus (name)"),
+                "reason_establishment_opened": row.get("ReasonEstablishmentOpened (name)"),
+                "open_date": row.get("OpenDate"),
+                "reason_establishment_closed": row.get("ReasonEstablishmentClosed (name)"),
+                "close_date": row.get("CloseDate"),
+                "phase_of_education": row.get("PhaseOfEducation (name)"),
+                "statutory_low_age": row.get("StatutoryLowAge"),
+                "statutory_high_age": row.get("StatutoryHighAge"),
+                "boarders": row.get("Boarders (name)"),
+                "nursery_provision": row.get("NurseryProvision (name)"),
+                "official_sixth_form": row.get("OfficialSixthForm (name)"),
+                "gender": row.get("Gender (name)"),
+                "religious_character": row.get("ReligiousCharacter (name)"),
+                "religious_ethos": row.get("ReligiousEthos (name)"),
+                "diocese": row.get("Diocese (name)"),
+                "admissions_policy": row.get("AdmissionsPolicy (name)"),
+                "school_capacity": row.get("SchoolCapacity"),
+                "special_classes": row.get("SpecialClasses (name)"),
+                "census_date": row.get("CensusDate"),
+                "number_of_pupils": row.get("NumberOfPupils"),
+                "number_of_boys": row.get("NumberOfBoys"),
+                "number_of_girls": row.get("NumberOfGirls"),
+                "percentage_fsm": row.get("PercentageFSM"),
+                "trust_school_flag": row.get("TrustSchoolFlag (name)"),
+                "trusts_name": row.get("Trusts (name)"),
+                "school_sponsor_flag": row.get("SchoolSponsorFlag (name)"),
+                "school_sponsors_name": row.get("SchoolSponsors (name)"),
+                "federation_flag": row.get("FederationFlag (name)"),
+                "federations_name": row.get("Federations (name)"),
+                "ukprn": row.get("UKPRN"),
+                "fehe_identifier": row.get("FEHEIdentifier"),
+                "further_education_type": row.get("FurtherEducationType (name)"),
+                "ofsted_last_inspection": row.get("OfstedLastInsp"),
+                "last_changed_date": row.get("LastChangedDate"),
+                "street": row.get("Street"),
+                "locality": row.get("Locality"),
+                "address3": row.get("Address3"),
+                "town": row.get("Town"),
+                "county": row.get("County (name)"),
+                "postcode": row.get("Postcode"),
+                "school_website": row.get("SchoolWebsite"),
+                "telephone_num": row.get("TelephoneNum"),
+                "head_title": row.get("HeadTitle (name)"),
+                "head_first_name": row.get("HeadFirstName"),
+                "head_last_name": row.get("HeadLastName"),
+                "head_preferred_job_title": row.get("HeadPreferredJobTitle"),
+                "gssla_code": row.get("GSSLACode (name)"),
+                "parliamentary_constituency": row.get("ParliamentaryConstituency (name)"),
+                "urban_rural": row.get("UrbanRural (name)"),
+                "rsc_region": row.get("RSCRegion (name)"),
+                "country": row.get("Country (name)"),
+                "uprn": row.get("UPRN"),
+                "sen_stat": row.get("SENStat") == "true",
+                "sen_no_stat": row.get("SENNoStat") == "true",
+                "sen_unit_on_roll": row.get("SenUnitOnRoll"),
+                "sen_unit_capacity": row.get("SenUnitCapacity"),
+                "resourced_provision_on_roll": row.get("ResourcedProvisionOnRoll"),
+                "resourced_provision_capacity": row.get("ResourcedProvisionCapacity"),
+            }
+            
+            # Clean up empty strings and convert types
+            for key, value in school_data.items():
+                if value == "":
+                    school_data[key] = None
+                elif key in ["statutory_low_age", "statutory_high_age", "school_capacity", 
+                           "number_of_pupils", "number_of_boys", "number_of_girls",
+                           "sen_unit_on_roll", "sen_unit_capacity",
+                           "resourced_provision_on_roll", "resourced_provision_capacity"]:
+                    if value:
+                        try:
+                            float_val = float(value)
+                            int_val = int(float_val)
+                            school_data[key] = int_val
+                        except (ValueError, TypeError):
+                            school_data[key] = None
+                elif key == "percentage_fsm":
+                    if value:
+                        try:
+                            school_data[key] = float(value)
+                        except (ValueError, TypeError):
+                            school_data[key] = None
+                elif key in ["open_date", "close_date", "census_date", 
+                           "ofsted_last_inspection", "last_changed_date"]:
+                    if value:
+                        try:
+                            # Convert date from DD-MM-YYYY to YYYY-MM-DD
+                            parts = value.split("-")
+                            if len(parts) == 3:
+                                school_data[key] = f"{parts[2]}-{parts[1]}-{parts[0]}"
+                            else:
+                                school_data[key] = None
+                        except:
+                            school_data[key] = None
+            
+            schools_data.append(school_data)
+        
+        # Batch insert schools using admin service's Supabase client
+        if schools_data:
+            result = admin_service.supabase.table("schools").upsert(
+                schools_data, 
+                on_conflict="urn"  # Update if URN already exists
+            ).execute()
+            
+            logger.info(f"Imported {len(schools_data)} schools")
+            return {"status": "success", "imported_count": len(schools_data)}
+        else:
+            raise HTTPException(status_code=400, detail="No valid school data found in CSV")
+            
+    except Exception as e:
+        logger.error(f"Error importing schools: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.post("/initialize-schools-database")
+async def initialize_schools_database(admin: Dict = Depends(verify_admin)):
+    """Initialize schools database"""
+    if not admin.get("is_super_admin"):
+        raise HTTPException(status_code=403, detail="Only super admins can initialize database")
+    
+    result = school_service.create_schools_database()
+    if result["status"] == "error":
+        raise HTTPException(status_code=500, detail=result["message"])
+    return result
+
+@router.get("/check-schools-database")
+async def check_schools_database(admin: Dict = Depends(verify_admin)):
+    """Check schools database status"""
+    try:
+        # Use SchoolService to check if database exists and has required nodes/relationships
+        result = school_service.check_schools_database()
+        return {"exists": result["status"] == "success"}
+    except Exception as e:
+        logger.error(f"Error checking schools database: {str(e)}")
+        return {"exists": False, "error": str(e)}
+
+@router.get("/storage", response_class=HTMLResponse)
+async def storage_management(request: Request, admin: Dict = Depends(verify_admin)):
+    """Storage management page"""
+    try:
+        # Get list of storage buckets with correct IDs
+        buckets = [
+            {
+                "id": "cc.institutes",
+                "name": "School Files",
+                "public": False,
+                "file_size_limit": 50 * 1024 * 1024,  # 50MB
+                "allowed_mime_types": [
+                    "image/*",
+                    "video/*",
+                    "application/pdf",
+                    "application/msword",
+                    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+                    "application/vnd.ms-excel",
+                    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+                    "application/vnd.ms-powerpoint",
+                    "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+                    "text/plain",
+                    "text/csv",
+                    "application/json"
+                ]
+            },
+            {
+                "id": "cc.users",
+                "name": "User Files",
+                "public": False,
+                "file_size_limit": 50 * 1024 * 1024,  # 50MB
+                "allowed_mime_types": [
+                    "image/*",
+                    "video/*",
+                    "application/pdf",
+                    "application/msword",
+                    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+                    "application/vnd.ms-excel",
+                    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+                    "application/vnd.ms-powerpoint",
+                    "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+                    "text/plain",
+                    "text/csv",
+                    "application/json"
+                ]
+            }
+        ]
+        
+        return templates.TemplateResponse(
+            "admin/storage/manage.html",
+            {"request": request, "admin": admin, "buckets": buckets}
+        )
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/storage/{bucket_id}/contents")
+async def list_bucket_contents(
+    request: Request,
+    bucket_id: str,
+    path: str = "",
+    admin: Dict = Depends(verify_admin)
+):
+    """List contents of a storage bucket"""
+    try:
+        contents = storage_manager.list_bucket_contents(bucket_id, path)
+        bucket = {"id": bucket_id, "name": bucket_id.replace("_", " ").title()}
+        
+        return templates.TemplateResponse(
+            "admin/storage/contents.html",
+            {
+                "request": request,
+                "admin": admin,
+                "bucket": bucket,
+                "contents": contents,
+                "current_path": path
+            }
+        )
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/storage/{bucket_id}/download/{file_path:path}")
+async def download_file(
+    bucket_id: str,
+    file_path: str,
+    admin: Dict = Depends(verify_admin)
+):
+    """Get download URL for a file"""
+    try:
+        url = storage_manager.create_signed_url(bucket_id, file_path)
+        return {"url": url}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.delete("/storage/{bucket_id}/objects/{object_path:path}")
+async def delete_object(
+    bucket_id: str,
+    object_path: str,
+    admin: Dict = Depends(verify_admin)
+):
+    """Delete an object from storage"""
+    try:
+        storage_manager.delete_file(bucket_id, object_path)
+        return {"status": "success"}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/check-storage")
+async def check_storage(admin: Dict = Depends(verify_admin)):
+    """Check storage buckets status"""
+    try:
+        # Use the same bucket IDs as defined in initialize_storage
+        buckets = [
+            {"id": "cc.users", "name": "User Files"},
+            {"id": "cc.institutes", "name": "School Files"}
+        ]
+        
+        results = []
+        for bucket in buckets:
+            exists = storage_manager.check_bucket_exists(bucket["id"])
+            results.append({
+                "id": bucket["id"],
+                "name": bucket["name"],
+                "exists": exists
+            })
+        
+        return {"buckets": results}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.post("/initialize-storage")
+async def initialize_storage(admin: Dict = Depends(verify_admin)):
+    """Initialize storage buckets and policies for schools"""
+    try:
+        # Verify super admin status
+        if not admin.get('is_super_admin'):
+            raise HTTPException(status_code=403, detail="Only super admins can initialize storage")
+
+        # Use the storage manager to initialize storage
+        storage_manager = StorageManager(SupabaseAnonClient)
+        return storage_manager.initialize_storage()
+    except Exception as e:
+        logger.error(f"Error initializing storage: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/check-schema")
+async def check_schema(admin: Dict = Depends(verify_admin)):
+    """Check Neo4j schema status"""
+    try:
+        from modules.database.services.graph_service import GraphService
+        graph_service = GraphService()
+        
+        # Get actual schema status
+        schema_status = graph_service.check_schema_status()
+        
+        # Return status with proper validation
+        return {
+            "constraints_valid": schema_status["constraints_count"] > 0,
+            "constraints_count": schema_status["constraints_count"],
+            "indexes_valid": schema_status["indexes_count"] > 0,
+            "indexes_count": schema_status["indexes_count"],
+            "labels_valid": schema_status["labels_count"] > 0,
+            "labels_count": schema_status["labels_count"]
+        }
+    except Exception as e:
+        logger.error(f"Error checking schema: {str(e)}")
+        return {
+            "constraints_valid": False,
+            "constraints_count": 0,
+            "indexes_valid": False,
+            "indexes_count": 0,
+            "labels_valid": False,
+            "labels_count": 0,
+            "error": str(e)
+        }
+
+@router.post("/initialize-schema")
+async def initialize_schema(admin: Dict = Depends(verify_admin)):
+    """Initialize Neo4j schema (constraints and indexes)"""
+    if not admin.get("is_super_admin"):
+        raise HTTPException(status_code=403, detail="Only super admins can initialize schema")
+    
+    try:
+        from modules.database.services.graph_service import GraphService
+        graph_service = GraphService()
+        
+        # Initialize schema
+        result = graph_service.initialize_schema()
+        
+        if result["status"] == "error":
+            raise HTTPException(status_code=500, detail=result["message"])
+            
+        return result
+    except Exception as e:
+        logger.error(f"Error initializing schema: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/schools/{school_id}")
+async def view_school(request: Request, school_id: str, admin: Dict = Depends(verify_admin)):
+    """View school details"""
+    try:
+        # Fetch school details from Supabase
+        result = admin_service.supabase.table("schools").select("*").eq("id", school_id).single().execute()
+        school = result.data if result else None
+        
+        if not school:
+            raise HTTPException(status_code=404, detail="School not found")
+        
+        return templates.TemplateResponse(
+            "admin/schools/detail.html",
+            {"request": request, "admin": admin, "school": school}
+        )
+    except Exception as e:
+        logger.error(f"Error fetching school details: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.delete("/schools/{school_id}")
+async def delete_school(school_id: str, admin: Dict = Depends(verify_admin)):
+    """Delete a school"""
+    try:
+        # Verify super admin status
+        if not admin.get("is_super_admin"):
+            raise HTTPException(status_code=403, detail="Only super admins can delete schools")
+        
+        # Delete the school from Supabase
+        result = admin_service.supabase.table("schools").delete().eq("id", school_id).execute()
+        
+        if not result.data:
+            raise HTTPException(status_code=404, detail="School not found")
+        
+        return {"status": "success", "message": "School deleted successfully"}
+    except Exception as e:
+        logger.error(f"Error deleting school: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))

routers/assets/pdf.py

@@ -0,0 +1,423 @@
+import os
+from modules.logger_tool import initialise_logger
+logger = initialise_logger(log_name="pdf", log_level=os.getenv("LOG_LEVEL"), log_dir=os.getenv("LOG_PATH"), log_format="default", runtime=True)
+
+from fastapi import APIRouter, UploadFile, File, HTTPException
+from fastapi.responses import JSONResponse
+from pathlib import Path
+import tempfile
+from PIL import Image
+import io
+import base64
+import traceback
+import sys
+import subprocess
+from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError
+import asyncio
+import psutil
+import math
+import time
+from pdfminer.high_level import extract_pages
+from pdfminer.layout import LTTextContainer, LTChar, LTLine, LTRect, LTFigure, LTTextBox, LTTextBoxHorizontal, LTTextLine
+import re
+
+router = APIRouter()
+
+# Global semaphore to control total concurrent PDF processing
+MAX_CONCURRENT_PROCESSING = 4  # Adjust based on server capacity
+processing_semaphore = asyncio.Semaphore(MAX_CONCURRENT_PROCESSING)
+
+def calculate_optimal_workers():
+    """Calculate optimal number of worker threads based on system resources."""
+    cpu_count = os.cpu_count() or 4
+    available_memory = psutil.virtual_memory().available
+    memory_per_worker = 500 * 1024 * 1024  # 500MB per worker estimate
+    
+    # Calculate workers based on CPU and memory constraints
+    cpu_based_workers = max(1, cpu_count - 1)  # Leave one core free
+    memory_based_workers = max(1, int(available_memory / memory_per_worker))
+    
+    # Take the minimum of CPU and memory-based calculations
+    optimal_workers = min(cpu_based_workers, memory_based_workers)
+    
+    # Cap at a reasonable maximum
+    final_workers = min(optimal_workers, 8)  # Maximum 8 workers per process
+
+    logger.info("Resource utilization:", {
+        "total_cpus": cpu_count,
+        "available_memory_gb": available_memory / (1024**3),
+        "cpu_based_workers": cpu_based_workers,
+        "memory_based_workers": memory_based_workers,
+        "final_workers": final_workers
+    })
+    
+    return final_workers
+
+def is_heading(textbox, page_height):
+    """Determine if a textbox is likely a heading based on font size and position."""
+    if not isinstance(textbox, LTTextContainer):
+        return False, 0
+
+    # Get the most common font size in the textbox
+    font_sizes = []
+    for text_line in textbox._objs:
+        if isinstance(text_line, LTTextLine):
+            font_sizes.extend(
+                char.size
+                for char in text_line._objs
+                if isinstance(char, LTChar)
+            )
+    if not font_sizes:
+        return False, 0
+
+    most_common_size = max(set(font_sizes), key=font_sizes.count)
+
+    # Position near top of page suggests a heading
+    is_near_top = textbox.y1 > (page_height - 100)
+
+    # Determine heading level based on font size and position
+    if most_common_size > 20 or is_near_top:
+        return True, 1
+    elif most_common_size > 16:
+        return True, 2
+    elif most_common_size > 14:
+        return True, 3
+
+    return False, 0
+
+def clean_text(text):
+    """Clean and normalize text content."""
+    # Remove multiple spaces and newlines
+    text = re.sub(r'\s+', ' ', text)
+    # Remove special characters often found in PDFs
+    text = re.sub(r'[^\x00-\x7F]+', '', text)
+    return text.strip()
+
+def extract_page_text(page):
+    """Extract text from a PDF page and format as markdown."""
+    page_height = page.height
+    text_elements = []
+    current_list_items = []
+    
+    # First pass: collect all text elements and identify their roles
+    for element in page:
+        if isinstance(element, LTTextContainer):
+            text = clean_text(element.get_text())
+            if not text:
+                continue
+            
+            is_head, level = is_heading(element, page_height)
+            
+            # Check if this looks like a list item
+            is_list_item = bool(re.match(r'^[\u2022\u2023\u25E6\u2043\u2219•\-*]\s', text))
+            
+            if is_head:
+                # If we have pending list items, add them first
+                if current_list_items:
+                    text_elements.extend(current_list_items)
+                    current_list_items = []
+                text_elements.append((f"{'#' * level} {text.lstrip('1234567890.-* ')}", element.y1))
+            elif is_list_item:
+                current_list_items.append((f"* {text.lstrip('1234567890.-* ')}", element.y1))
+            else:
+                # If this is regular text and we have pending list items
+                if current_list_items:
+                    # Check if this text is part of the same list (similar y-position)
+                    if any(abs(item[1] - element.y1) < 20 for item in current_list_items):
+                        current_list_items.append((f"* {text}", element.y1))
+                        continue
+                    else:
+                        # Add pending list items before adding this text
+                        text_elements.extend(current_list_items)
+                        current_list_items = []
+                text_elements.append((text, element.y1))
+    
+    # Add any remaining list items
+    if current_list_items:
+        text_elements.extend(current_list_items)
+    
+    # Sort elements by vertical position (top to bottom)
+    text_elements.sort(key=lambda x: -x[1])
+    
+    # Return just the text parts, properly formatted
+    return '\n\n'.join(element[0] for element in text_elements)
+
+def process_page(temp_dir: str, pdf_path: str, page_info: tuple, timeout: int = 30) -> dict:
+    """
+    Worker function to process a single page and maintain A4 proportions.
+    Args:
+        temp_dir: Path to temporary directory
+        pdf_path: Path to PDF file
+        page_info: Tuple of (index, page_number)
+        timeout: Maximum time in seconds to process a single page
+    Returns:
+        dict: Processed page information
+    """
+    i, page_idx = page_info
+    page_num = page_idx + 1  # PDF pages are 1-indexed
+    output_prefix = str(Path(temp_dir) / f"page_{page_num}")
+
+    try:
+        # Extract text from PDF page
+        pages = list(extract_pages(pdf_path, page_numbers=[page_idx]))
+        page_text = extract_page_text(pages[0]) if pages else ""
+        # Convert PDF page to PNG with timeout
+        process = subprocess.Popen(
+            [
+                'pdftoppm',
+                '-png',
+                '-singlefile',
+                '-f',
+                str(page_num),
+                '-l',
+                str(page_num),
+                '-r',
+                '600',  # High resolution for better quality
+                pdf_path,
+                output_prefix,
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE
+        )
+
+        try:
+            stdout, stderr = process.communicate(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            process.kill()
+            raise TimeoutError(f"Page {page_num} processing timed out after {timeout} seconds")
+
+        if process.returncode != 0:
+            raise Exception(f"pdftoppm failed for page {page_num}: {stderr.decode()}")
+
+        output_file = f"{output_prefix}.png"
+        if not Path(output_file).exists():
+            raise Exception(f"Could not find output file for page {page_num}")
+
+        # Open and process the image
+        with Image.open(output_file) as img:
+            result = _process_image(img, i)
+            if result['success']:
+                result['meta'] = {
+                    'text': page_text,
+                    'format': 'markdown'
+                }
+            return result
+    except Exception as e:
+        logger.error(f"Error processing page {page_num}: {str(e)}")
+        return {
+            "index": i,
+            "error": str(e),
+            "success": False,
+        }
+
+def _process_image(img: Image.Image, index: int) -> dict:
+    """Process a single image, maintaining A4 proportions."""
+    try:
+        # Determine orientation and target dimensions
+        is_portrait = img.height > img.width
+        target_height = 720  # Fixed height to match frontend slide height
+        
+        if is_portrait:
+            # A4 portrait ratio is 210:297
+            target_width = int(target_height * (210/297))
+        else:
+            # A4 landscape ratio is 297:210
+            target_width = int(target_height * (297/210))
+
+        # Resize image maintaining aspect ratio
+        img = img.resize((target_width, target_height), Image.Resampling.LANCZOS)
+
+        # Convert to base64
+        buffered = io.BytesIO()
+        img.save(buffered, format="PNG", optimize=True)
+        img_str = base64.b64encode(buffered.getvalue()).decode()
+
+        return {
+            "index": index,
+            "data": f"data:image/png;base64,{img_str}",
+            "success": True,
+            "dimensions": {
+                "width": target_width,
+                "height": target_height,
+                "orientation": "portrait" if is_portrait else "landscape"
+            }
+        }
+    except Exception as e:
+        logger.error(f"Error processing image for page {index}: {str(e)}")
+        return {
+            "index": index,
+            "error": str(e),
+            "success": False,
+        }
+
+async def process_pages_in_chunks(temp_dir: str, pdf_path: str, visible_pages: list, chunk_size: int = 5):
+    """Process pages in chunks to manage memory better."""
+    all_processed_pages = []
+    num_workers = calculate_optimal_workers()
+    total_chunks = math.ceil(len(visible_pages) / chunk_size)
+    
+    logger.info("Starting page processing:", {
+        "total_pages": len(visible_pages),
+        "chunk_size": chunk_size,
+        "total_chunks": total_chunks,
+        "workers_per_chunk": num_workers
+    })
+
+    # Process pages in chunks
+    for chunk_index in range(0, len(visible_pages), chunk_size):
+        chunk = visible_pages[chunk_index:chunk_index + chunk_size]
+        processed_chunk = []
+        current_chunk_num = (chunk_index // chunk_size) + 1
+
+        logger.info(f"Processing chunk {current_chunk_num}/{total_chunks}", {
+            "chunk_size": len(chunk),
+            "chunk_start_index": chunk_index,
+            "memory_usage_gb": psutil.Process().memory_info().rss / (1024**3)
+        })
+
+        start_time = time.time()
+        with ThreadPoolExecutor(max_workers=num_workers) as executor:
+            # Submit chunk of tasks
+            future_to_page = {
+                executor.submit(
+                    process_page, temp_dir, pdf_path, page_info
+                ): page_info
+                for page_info in chunk
+            }
+
+            # Process completed tasks as they finish
+            for future in as_completed(future_to_page):
+                try:
+                    result = future.result(timeout=60)  # Increased timeout to 60 seconds per page
+                    if result.get('success', False):
+                        processed_chunk.append(result)
+                    page_info = future_to_page[future]
+                    logger.debug(f"Processed page {page_info[1] + 1}", {
+                        "success": result.get('success', False),
+                        "processing_time": time.time() - start_time
+                    })
+                except TimeoutError:
+                    page_info = future_to_page[future]
+                    logger.error(f"Timeout processing page {page_info[1] + 1}")
+                except Exception as e:
+                    page_info = future_to_page[future]
+                    logger.error(f"Error processing page {page_info[1] + 1}: {str(e)}")
+
+        chunk_time = time.time() - start_time
+        logger.info(f"Completed chunk {current_chunk_num}/{total_chunks}", {
+            "processed_pages": len(processed_chunk),
+            "chunk_processing_time": chunk_time,
+            "avg_time_per_page": chunk_time / len(chunk) if chunk else 0
+        })
+
+        all_processed_pages.extend(processed_chunk)
+
+        # Small delay between chunks to allow other tasks to process
+        await asyncio.sleep(0.1)
+
+    return all_processed_pages
+
+@router.post("/convert")
+async def convert_pdf_to_images(file: UploadFile = File(...)):
+    try:
+        async with processing_semaphore:  # Control concurrent processing
+            start_time = time.time()
+            # Log request details
+            logger.info(
+                "Received file upload request",
+                {
+                    "filename": file.filename,
+                    "content_type": file.content_type,
+                    "current_memory_usage_gb": psutil.Process()
+                    .memory_info()
+                    .rss
+                    / (1024**3),
+                    "cpu_percent": psutil.cpu_percent(interval=1),
+                },
+            )
+
+            # Validate file
+            if not file.filename.endswith('.pdf'):
+                logger.error("Invalid file type")
+                return JSONResponse({
+                    "status": "error",
+                    "message": "Invalid file type. Please upload a .pdf file"
+                }, status_code=400)
+
+            # Create a temporary directory to store the PDF file
+            with tempfile.TemporaryDirectory() as temp_dir:
+                pdf_path = Path(temp_dir) / "document.pdf"
+                logger.debug(f"Saving file to temporary path: {pdf_path}")
+
+                try:
+                    # Save uploaded file
+                    content = await file.read()
+                    logger.debug(f"Read file content, size: {len(content)} bytes")
+
+                    with open(pdf_path, "wb") as buffer:
+                        buffer.write(content)
+                    logger.debug("File saved successfully")
+
+                    if not pdf_path.exists() or pdf_path.stat().st_size == 0:
+                        raise Exception("Failed to save file or file is empty")
+
+                    # Get number of pages using pdfinfo
+                    result = subprocess.run(['pdfinfo', str(pdf_path)], capture_output=True, text=True)
+                    pages_line = [line for line in result.stdout.split('\n') if line.startswith('Pages:')][0]
+                    num_pages = int(pages_line.split(':')[1].strip())
+
+                    visible_pages = [(i, i) for i in range(num_pages)]
+
+                    if num_pages == 0:
+                        logger.warning("No pages found in document")
+                        return JSONResponse({
+                            "status": "error",
+                            "message": "No pages found in document"
+                        }, status_code=400)
+
+                    logger.info(f"Processing {num_pages} pages")
+
+                    # Calculate chunk size based on number of pages
+                    chunk_size = min(5, max(2, math.ceil(num_pages / 4)))
+                    processed_pages = await process_pages_in_chunks(str(temp_dir), str(pdf_path), visible_pages, chunk_size)
+
+                    if not processed_pages:
+                        raise Exception("Failed to process any pages successfully")
+
+                    # Sort pages by index
+                    processed_pages.sort(key=lambda x: x['index'])
+
+                    logger.info(f"Successfully processed {len(processed_pages)} pages")
+
+                    # After processing all pages
+                    total_time = time.time() - start_time
+                    logger.info("PDF processing completed", {
+                        "total_processing_time": total_time,
+                        "pages_processed": len(processed_pages),
+                        "avg_time_per_page": total_time / len(processed_pages) if processed_pages else 0,
+                        "final_memory_usage_gb": psutil.Process().memory_info().rss / (1024**3)
+                    })
+
+                    return JSONResponse({
+                        "status": "success",
+                        "slides": processed_pages,  # Using same format as PowerPoint for consistency
+                        "processing_stats": {
+                            "total_time": total_time,
+                            "pages_processed": len(processed_pages),
+                            "avg_time_per_page": total_time / len(processed_pages) if processed_pages else 0
+                        }
+                    })
+
+                except Exception as inner_error:
+                    logger.error(f"Inner error: {str(inner_error)}")
+                    logger.error(traceback.format_exc())
+                    raise
+
+    except Exception as e:
+        logger.error(f"Error processing PDF: {str(e)}")
+        logger.error(f"Python version: {sys.version}")
+        logger.error(f"Traceback: {traceback.format_exc()}")
+        return JSONResponse({
+            "status": "error",
+            "message": f"Failed to process PDF: {str(e)}"
+        }, status_code=500)

routers/assets/powerpoint.py

@@ -0,0 +1,398 @@
+import os
+from modules.logger_tool import initialise_logger
+logger = initialise_logger(log_name="powerpoint", log_level=os.getenv("LOG_LEVEL"), log_dir=os.getenv("LOG_PATH"), log_format="default", runtime=True)
+
+from fastapi import APIRouter, UploadFile, File, HTTPException
+from fastapi.responses import JSONResponse
+from pathlib import Path
+import tempfile
+from pptx import Presentation
+from PIL import Image
+import io
+import base64
+import traceback
+import sys
+import subprocess
+from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError
+import asyncio
+import psutil
+import math
+import time
+
+router = APIRouter()
+
+# Global semaphore to control total concurrent PowerPoint processing
+MAX_CONCURRENT_PROCESSING = 4  # Adjust based on server capacity
+processing_semaphore = asyncio.Semaphore(MAX_CONCURRENT_PROCESSING)
+
+def calculate_optimal_workers():
+    """Calculate optimal number of worker threads based on system resources."""
+    cpu_count = os.cpu_count() or 4
+    available_memory = psutil.virtual_memory().available
+    memory_per_worker = 500 * 1024 * 1024  # 500MB per worker estimate
+    
+    # Calculate workers based on CPU and memory constraints
+    cpu_based_workers = max(1, cpu_count - 1)  # Leave one core free
+    memory_based_workers = max(1, int(available_memory / memory_per_worker))
+    
+    # Take the minimum of CPU and memory-based calculations
+    optimal_workers = min(cpu_based_workers, memory_based_workers)
+    
+    # Cap at a reasonable maximum
+    final_workers = min(optimal_workers, 8)  # Maximum 8 workers per process
+
+    # Log resource information
+    logger.info("Resource utilization:", {
+        "total_cpus": cpu_count,
+        "available_memory_gb": available_memory / (1024**3),
+        "cpu_based_workers": cpu_based_workers,
+        "memory_based_workers": memory_based_workers,
+        "final_workers": final_workers
+    })
+    
+    return final_workers
+
+def extract_text_from_shape(shape):
+    """Extract text from a PowerPoint shape."""
+    if hasattr(shape, 'text') and shape.text.strip():
+        return shape.text.strip()
+
+    # Handle tables
+    if shape.has_table:
+        table_text = []
+        for row in shape.table.rows:
+            row_text = []
+            row_text.extend(cell.text.strip() for cell in row.cells if cell.text.strip())
+            if row_text:
+                table_text.append('| ' + ' | '.join(row_text) + ' |')
+        if table_text:
+            # Add markdown table header separator
+            table_text.insert(1, '|' + '---|' * (len(table_text[0].split('|')) - 2))
+            return '\n'.join(table_text)
+
+    # Handle grouped shapes
+    if hasattr(shape, 'shapes'):
+        group_text = []
+        for subshape in shape.shapes:
+            if text := extract_text_from_shape(subshape):
+                group_text.append(text)
+        return '\n'.join(group_text) if group_text else ''
+
+    return ''
+
+def extract_slide_text(slide):
+    """Extract text from a PowerPoint slide and format as markdown."""
+    slide_text = []
+
+    # Extract title if present
+    if slide.shapes.title and slide.shapes.title.text.strip():
+        slide_text.append(f"# {slide.shapes.title.text.strip()}")
+
+    # Process all shapes
+    for shape in slide.shapes:
+        if shape != slide.shapes.title:  # Skip title as we've already processed it
+            if text := extract_text_from_shape(shape):
+                slide_text.append(text)
+
+    return '\n\n'.join(slide_text)
+
+def process_slide(temp_dir: str, pdf_path: str, pptx_path: str, slide_info: tuple, timeout: int = 30) -> dict:
+    """
+    Worker function to process a single slide and enforce 16:9 aspect ratio.
+    Args:
+        temp_dir: Path to temporary directory
+        pdf_path: Path to PDF file
+        pptx_path: Path to PowerPoint file
+        slide_info: Tuple of (index, slide_number)
+        timeout: Maximum time in seconds to process a single slide
+    Returns:
+        dict: Processed slide information
+    """
+    i, slide_idx = slide_info
+    slide_num = slide_idx + 1  # PDF pages are 1-indexed
+    output_prefix = str(Path(temp_dir) / f"slide_{slide_num}")
+
+    try:
+        # Extract text from PowerPoint slide
+        prs = Presentation(pptx_path)
+        slide_text = extract_slide_text(prs.slides[slide_idx])
+
+        # Convert PDF page to PNG with timeout
+        process = subprocess.Popen(
+            [
+                'pdftoppm',
+                '-png',
+                '-singlefile',
+                '-f',
+                str(slide_num),
+                '-l',
+                str(slide_num),
+                '-r',
+                '600',  # High resolution for better quality
+                pdf_path,
+                output_prefix,
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE
+        )
+        
+        try:
+            stdout, stderr = process.communicate(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            process.kill()
+            raise TimeoutError(f"Slide {slide_num} processing timed out after {timeout} seconds")
+
+        if process.returncode != 0:
+            raise Exception(f"pdftoppm failed for slide {slide_num}: {stderr.decode()}")
+
+        output_file = f"{output_prefix}.png"
+        if not Path(output_file).exists():
+            raise Exception(f"Could not find output file for slide {slide_num}")
+
+        # Open and process the image
+        with Image.open(output_file) as img:
+            result = _process_image(img, i)
+            if result['success']:
+                result['meta'] = {
+                    'text': slide_text,
+                    'format': 'markdown'
+                }
+            return result
+    except Exception as e:
+        logger.error(f"Error processing slide {slide_num}: {str(e)}")
+        return {
+            "index": i,
+            "error": str(e),
+            "success": False,
+        }
+
+def _process_image(img: Image.Image, index: int) -> dict:
+    """Process a single image, enforcing aspect ratio and size constraints."""
+    try:
+        # Enforce 16:9 aspect ratio
+        target_aspect_ratio = 16 / 9
+        img_aspect_ratio = img.width / img.height
+
+        if img_aspect_ratio > target_aspect_ratio:  # Wider than 16:9
+            new_width = int(img.height * target_aspect_ratio)
+            offset = (img.width - new_width) // 2
+            img = img.crop((offset, 0, offset + new_width, img.height))
+        elif img_aspect_ratio < target_aspect_ratio:  # Taller than 16:9
+            new_height = int(img.width / target_aspect_ratio)
+            offset = (img.height - new_height) // 2
+            img = img.crop((0, offset, img.width, offset + new_height))
+
+        # Resize to target resolution (2560x1440)
+        img = img.resize((2560, 1440), Image.Resampling.LANCZOS)
+
+        # Convert to base64
+        buffered = io.BytesIO()
+        img.save(buffered, format="PNG", optimize=True)
+        img_str = base64.b64encode(buffered.getvalue()).decode()
+
+        return {
+            "index": index,
+            "data": f"data:image/png;base64,{img_str}",
+            "success": True,
+        }
+    except Exception as e:
+        logger.error(f"Error processing image for slide {index}: {str(e)}")
+        return {
+            "index": index,
+            "error": str(e),
+            "success": False,
+        }
+
+async def process_slides_in_chunks(temp_dir: str, pdf_path: str, pptx_path: str, visible_slides: list, chunk_size: int = 5):
+    """Process slides in chunks to manage memory better."""
+    all_processed_slides = []
+    num_workers = calculate_optimal_workers()
+    total_chunks = math.ceil(len(visible_slides) / chunk_size)
+    
+    logger.info("Starting slide processing:", {
+        "total_slides": len(visible_slides),
+        "chunk_size": chunk_size,
+        "total_chunks": total_chunks,
+        "workers_per_chunk": num_workers
+    })
+
+    # Process slides in chunks
+    for chunk_index in range(0, len(visible_slides), chunk_size):
+        chunk = visible_slides[chunk_index:chunk_index + chunk_size]
+        processed_chunk = []
+        current_chunk_num = (chunk_index // chunk_size) + 1
+
+        logger.info(f"Processing chunk {current_chunk_num}/{total_chunks}", {
+            "chunk_size": len(chunk),
+            "chunk_start_index": chunk_index,
+            "memory_usage_gb": psutil.Process().memory_info().rss / (1024**3)
+        })
+
+        start_time = time.time()
+        with ThreadPoolExecutor(max_workers=num_workers) as executor:
+            # Submit chunk of tasks
+            future_to_slide = {
+                executor.submit(
+                    process_slide, temp_dir, pdf_path, pptx_path, slide_info
+                ): slide_info
+                for slide_info in chunk
+            }
+
+            # Process completed tasks as they finish
+            for future in as_completed(future_to_slide):
+                try:
+                    result = future.result(timeout=60)  # Increased timeout to 60 seconds per slide
+                    if result.get('success', False):
+                        processed_chunk.append(result)
+                    slide_info = future_to_slide[future]
+                    logger.debug(f"Processed slide {slide_info[1] + 1}", {
+                        "success": result.get('success', False),
+                        "processing_time": time.time() - start_time
+                    })
+                except TimeoutError:
+                    slide_info = future_to_slide[future]
+                    logger.error(f"Timeout processing slide {slide_info[1] + 1}")
+                except Exception as e:
+                    slide_info = future_to_slide[future]
+                    logger.error(f"Error processing slide {slide_info[1] + 1}: {str(e)}")
+
+        chunk_time = time.time() - start_time
+        logger.info(f"Completed chunk {current_chunk_num}/{total_chunks}", {
+            "processed_slides": len(processed_chunk),
+            "chunk_processing_time": chunk_time,
+            "avg_time_per_slide": chunk_time / len(chunk) if chunk else 0
+        })
+
+        all_processed_slides.extend(processed_chunk)
+
+        # Small delay between chunks to allow other tasks to process
+        await asyncio.sleep(0.1)
+
+    return all_processed_slides
+
+@router.post("/convert")
+async def convert_pptx_to_images(file: UploadFile = File(...)):
+    try:
+        async with processing_semaphore:  # Control concurrent processing
+            start_time = time.time()
+            # Log request details
+            logger.info(
+                "Received file upload request",
+                {
+                    "filename": file.filename,
+                    "content_type": file.content_type,
+                    "current_memory_usage_gb": psutil.Process()
+                    .memory_info()
+                    .rss
+                    / (1024**3),
+                    "cpu_percent": psutil.cpu_percent(interval=1),
+                },
+            )
+
+            # Validate file
+            if not file.filename.endswith('.pptx'):
+                logger.error("Invalid file type")
+                return JSONResponse({
+                    "status": "error",
+                    "message": "Invalid file type. Please upload a .pptx file"
+                }, status_code=400)
+
+            # Create a temporary directory to store the PowerPoint file
+            with tempfile.TemporaryDirectory() as temp_dir:
+                pptx_path = Path(temp_dir) / "presentation.pptx"
+                logger.debug(f"Saving file to temporary path: {pptx_path}")
+
+                try:
+                    # Save uploaded file
+                    content = await file.read()
+                    logger.debug(f"Read file content, size: {len(content)} bytes")
+
+                    with open(pptx_path, "wb") as buffer:
+                        buffer.write(content)
+                    logger.debug("File saved successfully")
+
+                    if not pptx_path.exists() or pptx_path.stat().st_size == 0:
+                        raise Exception("Failed to save file or file is empty")
+
+                    # Open the presentation and get visible slides
+                    prs = Presentation(str(pptx_path))
+                    visible_slides = [
+                        (i, slide_idx)
+                        for i, (slide_idx, _) in enumerate(
+                            (i, slide)
+                            for i, slide in enumerate(prs.slides)
+                            if not hasattr(slide, 'show') or slide.show
+                        )
+                    ]
+                    num_slides = len(visible_slides)
+
+                    if num_slides == 0:
+                        logger.warning("No visible slides found in presentation")
+                        return JSONResponse({
+                            "status": "error",
+                            "message": "No visible slides found in presentation"
+                        }, status_code=400)
+
+                    logger.info(f"Processing {num_slides} visible slides")
+
+                    # Convert PowerPoint to PDF
+                    pdf_path = Path(temp_dir) / "presentation.pdf"
+                    logger.debug("Converting PowerPoint to PDF")
+
+                    result = subprocess.run([
+                        'soffice',
+                        '--headless',
+                        '--convert-to', 'pdf',
+                        '--outdir', str(temp_dir),
+                        str(pptx_path)
+                    ], check=True, capture_output=True, text=True)
+
+                    if not pdf_path.exists():
+                        raise Exception("PDF file was not created")
+
+                    logger.debug(f"PDF created successfully at {pdf_path}, size: {pdf_path.stat().st_size} bytes")
+
+                    # Calculate chunk size based on number of slides
+                    chunk_size = min(5, max(2, math.ceil(num_slides / 4)))
+                    processed_slides = await process_slides_in_chunks(str(temp_dir), str(pdf_path), str(pptx_path), visible_slides, chunk_size)
+
+                    if not processed_slides:
+                        raise Exception("Failed to process any slides successfully")
+
+                    # Sort slides by index
+                    processed_slides.sort(key=lambda x: x['index'])
+
+                    logger.info(f"Successfully processed {len(processed_slides)} slides")
+
+                    # After processing all slides
+                    total_time = time.time() - start_time
+                    logger.info("PowerPoint processing completed", {
+                        "total_processing_time": total_time,
+                        "slides_processed": len(processed_slides),
+                        "avg_time_per_slide": total_time / len(processed_slides) if processed_slides else 0,
+                        "final_memory_usage_gb": psutil.Process().memory_info().rss / (1024**3)
+                    })
+
+                    return JSONResponse({
+                        "status": "success",
+                        "slides": processed_slides,
+                        "processing_stats": {
+                            "total_time": total_time,
+                            "slides_processed": len(processed_slides),
+                            "avg_time_per_slide": total_time / len(processed_slides) if processed_slides else 0
+                        }
+                    })
+
+                except Exception as inner_error:
+                    logger.error(f"Inner error: {str(inner_error)}")
+                    logger.error(traceback.format_exc())
+                    raise
+
+    except Exception as e:
+        logger.error(f"Error processing PowerPoint: {str(e)}")
+        logger.error(f"Python version: {sys.version}")
+        logger.error(f"Traceback: {traceback.format_exc()}")
+        return JSONResponse({
+            "status": "error",
+            "message": f"Failed to process PowerPoint: {str(e)}"
+        }, status_code=500)

routers/assets/word.py

@@ -0,0 +1,418 @@
+import os
+from modules.logger_tool import initialise_logger
+logger = initialise_logger(log_name="word", log_level=os.getenv("LOG_LEVEL"), log_dir=os.getenv("LOG_PATH"), log_format="default", runtime=True)
+
+from fastapi import APIRouter, UploadFile, File, HTTPException
+from fastapi.responses import JSONResponse
+from pathlib import Path
+import tempfile
+from PIL import Image
+import io
+import base64
+import traceback
+import sys
+import subprocess
+from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError
+import asyncio
+import psutil
+import math
+import time
+from docx import Document
+
+router = APIRouter()
+
+# Global semaphore to control total concurrent Word processing
+MAX_CONCURRENT_PROCESSING = 4  # Adjust based on server capacity
+processing_semaphore = asyncio.Semaphore(MAX_CONCURRENT_PROCESSING)
+
+def calculate_optimal_workers():
+    """Calculate optimal number of worker threads based on system resources."""
+    cpu_count = os.cpu_count() or 4
+    available_memory = psutil.virtual_memory().available
+    memory_per_worker = 500 * 1024 * 1024  # 500MB per worker estimate
+    
+    # Calculate workers based on CPU and memory constraints
+    cpu_based_workers = max(1, cpu_count - 1)  # Leave one core free
+    memory_based_workers = max(1, int(available_memory / memory_per_worker))
+    
+    # Take the minimum of CPU and memory-based calculations
+    optimal_workers = min(cpu_based_workers, memory_based_workers)
+    
+    # Cap at a reasonable maximum
+    final_workers = min(optimal_workers, 8)  # Maximum 8 workers per process
+
+    logger.info("Resource utilization:", {
+        "total_cpus": cpu_count,
+        "available_memory_gb": available_memory / (1024**3),
+        "cpu_based_workers": cpu_based_workers,
+        "memory_based_workers": memory_based_workers,
+        "final_workers": final_workers
+    })
+    
+    return final_workers
+
+def extract_text_from_paragraph(paragraph):
+    """Extract text from a Word paragraph and format as markdown."""
+    text = paragraph.text.strip()
+    if not text:
+        return ''
+    
+    # Handle different heading levels
+    if paragraph.style.name.startswith('Heading'):
+        level = int(paragraph.style.name[-1])
+        return f"{'#' * level} {text}"
+    
+    # Handle lists
+    if paragraph._element.pPr is not None and paragraph._element.pPr.numPr is not None:
+        return f"* {text}"
+    
+    return text
+
+def extract_text_from_table(table):
+    """Extract text from a Word table and format as markdown."""
+    # Process header row
+    header_row = []
+    header_row.extend((cell.text.strip() or ' ') for cell in table.rows[0].cells)
+    table_text = [
+        '| ' + ' | '.join(header_row) + ' |',
+        '|' + '---|' * (len(header_row) - 1) + '---|',
+    ]
+    # Process remaining rows
+    for row in table.rows[1:]:
+        row_text = []
+        row_text.extend((cell.text.strip() or ' ') for cell in row.cells)
+        table_text.append('| ' + ' | '.join(row_text) + ' |')
+
+    return '\n'.join(table_text)
+
+def extract_page_text(doc, page_index):
+    """Extract text from a Word document page and format as markdown."""
+    # Note: python-docx doesn't provide direct page access, so we'll use a heuristic
+    # to group paragraphs into pages based on content length
+    CHARS_PER_PAGE = 3000  # Approximate characters per page
+
+    all_blocks = []
+    current_chars = 0
+    current_page = 0
+
+    for element in doc.element.body:
+        if current_page > page_index:
+            break
+
+        if element.tag.endswith('p'):
+            paragraph = doc.paragraphs[len(all_blocks)]
+            if text := extract_text_from_paragraph(paragraph):
+                current_chars += len(text)
+                if current_page == page_index:
+                    all_blocks.append(text)
+        elif element.tag.endswith('tbl'):
+            table = doc.tables[sum(isinstance(b, str) for b in all_blocks)]
+            if text := extract_text_from_table(table):
+                current_chars += len(text)
+                if current_page == page_index:
+                    all_blocks.append(text)
+
+        if current_chars >= CHARS_PER_PAGE:
+            current_page += 1
+            current_chars = 0
+
+    return '\n\n'.join(all_blocks)
+
+def process_page(temp_dir: str, pdf_path: str, docx_path: str, page_info: tuple, timeout: int = 30) -> dict:
+    """
+    Worker function to process a single page and maintain A4 proportions.
+    Args:
+        temp_dir: Path to temporary directory
+        pdf_path: Path to PDF file
+        docx_path: Path to Word file
+        page_info: Tuple of (index, page_number)
+        timeout: Maximum time in seconds to process a single page
+    Returns:
+        dict: Processed page information
+    """
+    i, page_idx = page_info
+    page_num = page_idx + 1  # PDF pages are 1-indexed
+    output_prefix = str(Path(temp_dir) / f"page_{page_num}")
+
+    try:
+        # Extract text from Word document
+        doc = Document(docx_path)
+        page_text = extract_page_text(doc, page_idx)
+
+        # Convert PDF page to PNG with timeout
+        process = subprocess.Popen(
+            [
+                'pdftoppm',
+                '-png',
+                '-singlefile',
+                '-f',
+                str(page_num),
+                '-l',
+                str(page_num),
+                '-r',
+                '600',  # High resolution for better quality
+                pdf_path,
+                output_prefix,
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE
+        )
+        
+        try:
+            stdout, stderr = process.communicate(timeout=timeout)
+        except subprocess.TimeoutExpired:
+            process.kill()
+            raise TimeoutError(f"Page {page_num} processing timed out after {timeout} seconds")
+
+        if process.returncode != 0:
+            raise Exception(f"pdftoppm failed for page {page_num}: {stderr.decode()}")
+
+        output_file = f"{output_prefix}.png"
+        if not Path(output_file).exists():
+            raise Exception(f"Could not find output file for page {page_num}")
+
+        # Open and process the image
+        with Image.open(output_file) as img:
+            result = _process_image(img, i)
+            if result['success']:
+                result['meta'] = {
+                    'text': page_text,
+                    'format': 'markdown'
+                }
+            return result
+    except Exception as e:
+        logger.error(f"Error processing page {page_num}: {str(e)}")
+        return {
+            "index": i,
+            "error": str(e),
+            "success": False,
+        }
+
+def _process_image(img: Image.Image, index: int) -> dict:
+    """Process a single image, maintaining A4 proportions."""
+    try:
+        # Determine orientation and target dimensions
+        is_portrait = img.height > img.width
+        target_height = 720  # Fixed height to match frontend slide height
+        
+        if is_portrait:
+            # A4 portrait ratio is 210:297
+            target_width = int(target_height * (210/297))
+        else:
+            # A4 landscape ratio is 297:210
+            target_width = int(target_height * (297/210))
+
+        # Resize image maintaining aspect ratio
+        img = img.resize((target_width, target_height), Image.Resampling.LANCZOS)
+
+        # Convert to base64
+        buffered = io.BytesIO()
+        img.save(buffered, format="PNG", optimize=True)
+        img_str = base64.b64encode(buffered.getvalue()).decode()
+
+        return {
+            "index": index,
+            "data": f"data:image/png;base64,{img_str}",
+            "success": True,
+            "dimensions": {
+                "width": target_width,
+                "height": target_height,
+                "orientation": "portrait" if is_portrait else "landscape"
+            }
+        }
+    except Exception as e:
+        logger.error(f"Error processing image for page {index}: {str(e)}")
+        return {
+            "index": index,
+            "error": str(e),
+            "success": False,
+        }
+
+async def process_pages_in_chunks(temp_dir: str, pdf_path: str, docx_path: str, visible_pages: list, chunk_size: int = 5):
+    """Process pages in chunks to manage memory better."""
+    all_processed_pages = []
+    num_workers = calculate_optimal_workers()
+    total_chunks = math.ceil(len(visible_pages) / chunk_size)
+    
+    logger.info("Starting page processing:", {
+        "total_pages": len(visible_pages),
+        "chunk_size": chunk_size,
+        "total_chunks": total_chunks,
+        "workers_per_chunk": num_workers
+    })
+
+    # Process pages in chunks
+    for chunk_index in range(0, len(visible_pages), chunk_size):
+        chunk = visible_pages[chunk_index:chunk_index + chunk_size]
+        processed_chunk = []
+        current_chunk_num = (chunk_index // chunk_size) + 1
+
+        logger.info(f"Processing chunk {current_chunk_num}/{total_chunks}", {
+            "chunk_size": len(chunk),
+            "chunk_start_index": chunk_index,
+            "memory_usage_gb": psutil.Process().memory_info().rss / (1024**3)
+        })
+
+        start_time = time.time()
+        with ThreadPoolExecutor(max_workers=num_workers) as executor:
+            # Submit chunk of tasks
+            future_to_page = {
+                executor.submit(
+                    process_page, temp_dir, pdf_path, docx_path, page_info
+                ): page_info
+                for page_info in chunk
+            }
+
+            # Process completed tasks as they finish
+            for future in as_completed(future_to_page):
+                try:
+                    result = future.result(timeout=60)  # Increased timeout to 60 seconds per page
+                    if result.get('success', False):
+                        processed_chunk.append(result)
+                    page_info = future_to_page[future]
+                    logger.debug(f"Processed page {page_info[1] + 1}", {
+                        "success": result.get('success', False),
+                        "processing_time": time.time() - start_time
+                    })
+                except TimeoutError:
+                    page_info = future_to_page[future]
+                    logger.error(f"Timeout processing page {page_info[1] + 1}")
+                except Exception as e:
+                    page_info = future_to_page[future]
+                    logger.error(f"Error processing page {page_info[1] + 1}: {str(e)}")
+
+        chunk_time = time.time() - start_time
+        logger.info(f"Completed chunk {current_chunk_num}/{total_chunks}", {
+            "processed_pages": len(processed_chunk),
+            "chunk_processing_time": chunk_time,
+            "avg_time_per_page": chunk_time / len(chunk) if chunk else 0
+        })
+
+        all_processed_pages.extend(processed_chunk)
+
+        # Small delay between chunks to allow other tasks to process
+        await asyncio.sleep(0.1)
+
+    return all_processed_pages
+
+@router.post("/convert")
+async def convert_docx_to_images(file: UploadFile = File(...)):
+    try:
+        async with processing_semaphore:  # Control concurrent processing
+            start_time = time.time()
+            # Log request details
+            logger.info(
+                "Received file upload request",
+                {
+                    "filename": file.filename,
+                    "content_type": file.content_type,
+                    "current_memory_usage_gb": psutil.Process()
+                    .memory_info()
+                    .rss
+                    / (1024**3),
+                    "cpu_percent": psutil.cpu_percent(interval=1),
+                },
+            )
+
+            # Validate file
+            if not file.filename.endswith('.docx'):
+                logger.error("Invalid file type")
+                return JSONResponse({
+                    "status": "error",
+                    "message": "Invalid file type. Please upload a .docx file"
+                }, status_code=400)
+
+            # Create a temporary directory to store the Word file
+            with tempfile.TemporaryDirectory() as temp_dir:
+                docx_path = Path(temp_dir) / "document.docx"
+                pdf_path = Path(temp_dir) / "document.pdf"
+
+                logger.debug(f"Saving file to temporary path: {docx_path}")
+
+                try:
+                    # Save uploaded file
+                    content = await file.read()
+                    logger.debug(f"Read file content, size: {len(content)} bytes")
+
+                    with open(docx_path, "wb") as buffer:
+                        buffer.write(content)
+                    logger.debug("File saved successfully")
+
+                    if not docx_path.exists() or docx_path.stat().st_size == 0:
+                        raise Exception("Failed to save file or file is empty")
+
+                    # Convert Word to PDF using LibreOffice
+                    logger.debug("Converting Word to PDF")
+                    result = subprocess.run([
+                        'soffice',
+                        '--headless',
+                        '--convert-to', 'pdf',
+                        '--outdir', str(temp_dir),
+                        str(docx_path)
+                    ], check=True, capture_output=True, text=True)
+
+                    if not pdf_path.exists():
+                        raise Exception("PDF file was not created")
+
+                    logger.debug(f"PDF created successfully at {pdf_path}, size: {pdf_path.stat().st_size} bytes")
+
+                    # Get number of pages using pdfinfo
+                    result = subprocess.run(['pdfinfo', str(pdf_path)], capture_output=True, text=True)
+                    pages_line = [line for line in result.stdout.split('\n') if line.startswith('Pages:')][0]
+                    num_pages = int(pages_line.split(':')[1].strip())
+
+                    visible_pages = [(i, i) for i in range(num_pages)]
+
+                    if num_pages == 0:
+                        logger.warning("No pages found in document")
+                        return JSONResponse({
+                            "status": "error",
+                            "message": "No pages found in document"
+                        }, status_code=400)
+
+                    logger.info(f"Processing {num_pages} pages")
+
+                    # Calculate chunk size based on number of pages
+                    chunk_size = min(5, max(2, math.ceil(num_pages / 4)))
+                    processed_pages = await process_pages_in_chunks(str(temp_dir), str(pdf_path), str(docx_path), visible_pages, chunk_size)
+
+                    if not processed_pages:
+                        raise Exception("Failed to process any pages successfully")
+
+                    # Sort pages by index
+                    processed_pages.sort(key=lambda x: x['index'])
+
+                    logger.info(f"Successfully processed {len(processed_pages)} pages")
+
+                    # After processing all pages
+                    total_time = time.time() - start_time
+                    logger.info("Word document processing completed", {
+                        "total_processing_time": total_time,
+                        "pages_processed": len(processed_pages),
+                        "avg_time_per_page": total_time / len(processed_pages) if processed_pages else 0,
+                        "final_memory_usage_gb": psutil.Process().memory_info().rss / (1024**3)
+                    })
+
+                    return JSONResponse({
+                        "status": "success",
+                        "slides": processed_pages,  # Using same format as PowerPoint for consistency
+                        "processing_stats": {
+                            "total_time": total_time,
+                            "pages_processed": len(processed_pages),
+                            "avg_time_per_page": total_time / len(processed_pages) if processed_pages else 0
+                        }
+                    })
+
+                except Exception as inner_error:
+                    logger.error(f"Inner error: {str(inner_error)}")
+                    logger.error(traceback.format_exc())
+                    raise
+
+    except Exception as e:
+        logger.error(f"Error processing Word document: {str(e)}")
+        logger.error(f"Python version: {sys.version}")
+        logger.error(f"Traceback: {traceback.format_exc()}")
+        return JSONResponse({
+            "status": "error",
+            "message": f"Failed to process Word document: {str(e)}"
+        }, status_code=500)

routers/auth.py

@@ -0,0 +1,139 @@
+from fastapi import APIRouter, Request, Response, HTTPException, Form, Body
+from fastapi.responses import HTMLResponse, RedirectResponse
+from fastapi.templating import Jinja2Templates
+from typing import Dict
+import os
+from modules.logger_tool import initialise_logger
+from modules.database.services.admin_service import AdminService
+from modules.database.services.auth_service import auth_service
+
+router = APIRouter()
+templates = Jinja2Templates(directory="templates")
+logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
+
+# Initialize services
+admin_service = AdminService()
+
+async def verify_admin(request: Request) -> Dict:
+    """Verify that the user is an admin and has necessary permissions"""
+    session = request.cookies.get("sb-access-token")
+    return await auth_service.verify_admin(session)
+
+@router.get("/admin", response_class=HTMLResponse)
+async def admin_root(request: Request):
+    """Root admin route - redirects to login or dashboard"""
+    try:
+        admin = await verify_admin(request)
+        return RedirectResponse(url="/api/admin/", status_code=303)
+    except HTTPException:
+        # Check if super admin exists
+        has_super_admin = await auth_service.check_super_admin_exists()
+        if not has_super_admin:
+            return RedirectResponse(url="/api/admin/login?init=true", status_code=303)
+        return RedirectResponse(url="/api/admin/login", status_code=303)
+
+@router.get("/admin/login", response_class=HTMLResponse)
+async def login_page(
+    request: Request,
+    error: str = None,
+    success: str = None,
+    init: bool = False
+):
+    """Render admin login page"""
+    # Check if super admin exists
+    has_super_admin = await auth_service.check_super_admin_exists()
+    
+    # If no super admin and init flag is true, show initialization form
+    if not has_super_admin:
+        expected_email = os.getenv("VITE_SUPER_ADMIN_EMAIL")
+        return templates.TemplateResponse(
+            "admin/login.html",
+            {
+                "request": request,
+                "error": error,
+                "success": success,
+                "init_super_admin": True,
+                "expected_super_admin_email": expected_email
+            }
+        )
+    
+    return templates.TemplateResponse(
+        "admin/login.html",
+        {
+            "request": request,
+            "error": error,
+            "success": success,
+            "init_super_admin": False
+        }
+    )
+
+@router.post("/admin/login")
+async def login(
+    request: Request,
+    response: Response,
+    email: str = Form(...),
+    password: str = Form(...)
+):
+    """Handle admin login"""
+    try:
+        # Login with auth service
+        auth_result = await auth_service.login_admin(email, password)
+        
+        # Set session cookie and redirect
+        response = RedirectResponse(url="/api/admin/", status_code=303)
+        response.set_cookie(
+            "sb-access-token",
+            auth_result["access_token"],
+            httponly=True,
+            secure=True
+        )
+        return response
+
+    except HTTPException as e:
+        return RedirectResponse(
+            url=f"/api/admin/login?error={str(e.detail)}",
+            status_code=303
+        )
+    except Exception as e:
+        logger.error(f"Login error: {str(e)}")
+        return RedirectResponse(
+            url=f"/api/admin/login?error={str(e)}",
+            status_code=303
+        )
+
+@router.post("/admin/logout")
+async def logout(response: Response):
+    """Handle admin logout"""
+    try:
+        response = RedirectResponse(url="/api/admin/login", status_code=303)
+        response.delete_cookie("sb-access-token")
+        return response
+    except Exception as e:
+        logger.error(f"Logout error: {str(e)}")
+        raise HTTPException(status_code=500, detail="Logout failed")
+
+@router.post("/admin/initialize-super-admin")
+async def initialize_super_admin(
+    admin_data: Dict = Body(...),
+    request: Request = None
+):
+    """Initialize the super admin account"""
+    try:
+        # Validate required fields
+        required_fields = ["email", "password", "display_name"]
+        for field in required_fields:
+            if field not in admin_data:
+                raise HTTPException(status_code=400, detail=f"Missing required field: {field}")
+
+        # Set up super admin
+        admin_service = AdminService()
+        result = admin_service.setup_super_admin(admin_data)
+        
+        return {
+            "status": "success",
+            "message": "Super admin account created successfully! Please log in with your credentials.",
+            "admin": result
+        }
+    except Exception as e:
+        logger.error(f"Error initializing super admin: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))

routers/connections/arbor_router.py

@@ -0,0 +1,29 @@
+from fastapi import APIRouter, HTTPException
+import os
+import requests
+from base64 import b64decode
+
+router = APIRouter()
+
+def get_basic_auth_header(token: str) -> dict:
+    """Decode the base64 token and return the appropriate header."""
+    decoded_token = b64decode(token).decode('utf-8')
+    return {"Authorization": f"Basic {token}"}
+
+@router.get("/data/{id}")
+async def fetch_arbor_data(id: int, token: str):
+    url_mapping = {
+        1: os.environ["KS3_COURSE_CLASS_MEMBERSHIP_URL"],
+        2: os.environ["TEACHING_GROUP_MEMBERSHIPS_2023_2024_URL"],
+        3: os.environ["SCHEDULED_TIMETABLE_SLOTS_URL"],
+        4: os.environ["BEHAVIOURAL_INCIDENTS_REPORTING_URL"],
+        5: os.environ["Y7_LESSON_TIMETABLE_URL"]
+    }
+    if id not in url_mapping:
+        raise HTTPException(status_code=404, detail="Data ID not supported")
+
+    headers = get_basic_auth_header(token)
+    response = requests.get(url_mapping[id], headers=headers)
+    if response.status_code != 200:
+        raise HTTPException(status_code=response.status_code, detail="Failed to fetch data from Arbor")
+    return response.json()

routers/connections/handleID_3.py

@@ -0,0 +1,19 @@
+import sys
+import json
+
+def filter_by_staff(data, staff_name="Kevin Carter"):
+    return [entry for entry in data if entry.get("Staff") == staff_name]
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        staff_name = sys.argv[1]
+    else:
+        staff_name = "Kevin Carter"
+    
+    input_data = sys.stdin.read()
+    try:
+        data = json.loads(input_data)
+        filtered_data = filter_by_staff(data, staff_name)
+        print(json.dumps(filtered_data, indent=4))
+    except json.JSONDecodeError:
+        print("Invalid JSON input", file=sys.stderr)

routers/connections/ollama_format_timetable.py

@@ -0,0 +1,34 @@
+import os
+import sys
+import json
+import requests
+
+def format_timetable_with_ollama(timetable_data):
+    url = f"{os.environ.get('APP_API_URL')}/llm/private/ollama/ollama_generate"
+    headers = {"Content-Type": "application/json"}
+    prompt = (
+        "Create a markdown formatted table of the following timetable data. "
+        "The table should have columns for 'Day', 'Time Slot', 'Effective Dates', 'Event', 'Room', and 'Staff':\n\n"
+        f"{json.dumps(timetable_data, indent=4)}"
+    )
+    payload = {
+        "model": "llama3",  # Adjust the model name if necessary
+        "prompt": prompt
+    }
+
+    response = requests.post(url, headers=headers, json=payload)
+    if response.status_code == 200:
+        return response.json().get("response")
+    else:
+        raise Exception(f"Failed to get response from Ollama: {response.status_code} {response.text}")
+
+if __name__ == "__main__":
+    input_data = sys.stdin.read()
+    try:
+        timetable_data = json.loads(input_data)
+        markdown_table = format_timetable_with_ollama(timetable_data)
+        print(markdown_table)
+    except json.JSONDecodeError:
+        print("Invalid JSON input", file=sys.stderr)
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)

routers/connections/openai_format_timetable.py

@@ -0,0 +1,45 @@
+import sys
+import os
+import json
+import requests
+
+def format_timetable_with_openai(timetable_data):
+    url = f"{os.environ.get('APP_API_URL')}/llm/public/openai/openai_general_prompt"
+    headers = {"Content-Type": "application/json"}
+    prompt = (
+        "Create a markdown formatted table of the following timetable data. "
+        "The table should have columns for 'Day', 'Time Slot', 'Effective Dates', 'Event', 'Room', and 'Staff':\n\n"
+        f"{json.dumps(timetable_data, indent=4)}"
+    )
+    payload = {
+        "model": "gpt-4-turbo",  # Adjust the model name if necessary
+        "prompt": prompt,
+        "max_tokens": 1500,
+        "temperature": 0.7,
+        "top_p": 1.0,
+        "n": 1,
+        "stop": None
+    }
+
+    response = requests.post(url, headers=headers, json=payload)
+    if response.status_code == 200:
+        return response.json().get("response")
+    else:
+        raise Exception(f"Failed to get response from OpenAI: {response.status_code} {response.text}")
+
+if __name__ == "__main__":
+    input_data = sys.stdin.read()
+    try:
+        timetable_data = json.loads(input_data)
+        markdown_table = format_timetable_with_openai(timetable_data)
+        
+        # Save the markdown table to a .md file
+        output_file = "timetable.md"
+        with open(output_file, "w") as file:
+            file.write(markdown_table)
+        
+        print(f"Markdown table saved to {output_file}")
+    except json.JSONDecodeError:
+        print("Invalid JSON input", file=sys.stderr)
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)

routers/database/department.py

@@ -0,0 +1,4 @@
+from fastapi import APIRouter, Depends, File, UploadFile
+from backend.app.run.dependencies import admin_dependency
+
+router = APIRouter()

routers/database/handle_connection.py

@@ -0,0 +1,14 @@
+from fastapi import APIRouter, Depends
+from backend.app.run.dependencies import admin_dependency
+
+import modules.database.tools.neo4j_driver_tools as driver
+import modules.database.tools.neo4j_session_tools as session
+import modules.database.tools.neo4j_http_tools as http
+import modules.database.tools.queries as query
+
+router = APIRouter()
+
+# Handle neo4j driver
+@router.post("/create-driver")
+async def create_driver(driver: driver.Neo4jDriver = Depends(driver.get_neo4j_driver)):
+    return driver

routers/database/init/calendar.py

@@ -0,0 +1,32 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'api_routers_database_init_calendar'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+from modules.database.tools.neontology.basenode import BaseNode
+import modules.database.init.init_calendar as init_calendar
+from fastapi import APIRouter
+from datetime import date
+from fastapi import HTTPException
+
+router = APIRouter()
+
+@router.post("/create-calendar")
+async def create_calendar(db_name: str, start_date: date, end_date: date, attach_to_calendar_node: bool = False, entity_node: BaseNode = None):
+    try:
+        logging.info(f"Creating calendar for {db_name} from {start_date} to {end_date}")
+        if entity_node is None:
+            logging.info("No user entity node provided, proceeding without attaching to user entity.")
+        return init_calendar.create_calendar(db_name, start_date, end_date, attach_to_calendar_node, entity_node)
+    except Exception as e:
+        logging.error(f"Error processing request: {e}")
+        raise HTTPException(status_code=422, detail=str(e))

routers/database/init/classes.py

@@ -0,0 +1,16 @@
+import os
+from modules.logger_tool import initialise_logger
+logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
+
+from fastapi import APIRouter, File, UploadFile, Form, BackgroundTasks
+
+router = APIRouter()
+
+@router.post("/upload-class-list")
+async def upload_class_list(
+    background_tasks: BackgroundTasks,
+    file: UploadFile = File(...),
+    user_node: str = Form(...),
+    worker_node: str = Form(...)
+):
+    pass

routers/database/init/curriculum.py

@@ -0,0 +1,50 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'api_routers_database_init_curriculum'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+import modules.database.init.xl_tools as xl
+import modules.database.init.init_school_curriculum as init_school_curriculum
+from modules.database.schemas.nodes.schools.schools import SchoolNode
+from fastapi import APIRouter, File, UploadFile, Form
+
+router = APIRouter()
+
+@router.post("/upload-curriculum")
+async def upload_curriculum(file: UploadFile = File(...), db_name: str = Form(...)):
+    if file.content_type != 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':
+        return {"status": "Error", "message": "Invalid file format"}
+    logging.info(f"Uploading curriculum for {db_name}")
+    dataframes = xl.create_dataframes_from_fastapiuploadfile(file)
+    return init_school_curriculum.create_curriculum(db_name, dataframes)
+
+@router.post("/upload-school-curriculum")
+async def upload_school_curriculum(
+    file: UploadFile = File(...),
+    db_name: str = Form(...),
+    school_uuid: str = Form(...),
+    school_name: str = Form(...),
+    school_website: str = Form(...),
+    school_path: str = Form(...)
+):
+    if file.content_type != 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':
+        return {"status": "Error", "message": "Invalid file format"}
+    logging.info(f"Uploading curriculum for school {school_name} in {db_name}")
+    dataframes = xl.create_dataframes_from_fastapiuploadfile(file)
+    school_node = SchoolNode(
+        unique_id=f'School_{school_uuid}',
+        school_uuid=school_uuid,
+        school_name=school_name,
+        school_website=school_website,
+        path=school_path
+    )
+    return init_school_curriculum.create_curriculum(db_name, dataframes, school_node)

routers/database/init/entity_init.py

@@ -0,0 +1,280 @@
+import os
+from modules.logger_tool import initialise_logger
+logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
+import modules.database.tools.neo4j_driver_tools as driver_tools
+import modules.database.tools.neo4j_session_tools as session_tools
+import modules.database.init.init_user as init_user
+from modules.database.tools.neo4j_db_formatter import format_user_email_for_neo_db
+import modules.database.init.init_school as init_school
+import modules.database.init.init_school_timetable as init_school_timetable
+import modules.database.init.init_school_curriculum as init_school_curriculum
+import modules.database.init.xl_tools as xl
+from modules.database.schemas.nodes.schools.schools import SchoolNode, SubjectClassNode, RoomNode, DepartmentNode
+from fastapi import APIRouter, Form, HTTPException
+from fastapi.responses import JSONResponse
+import json
+
+VALID_USER_TYPES = ['admin', 'cc_admin', 'cc_email_school_admin', 'cc_ms_school_admin', 'email_school_admin', 'ms_school_admin', 'cc_email_teacher', 'cc_ms_teacher', 'cc_email_student', 'cc_ms_student', 'email_teacher', 'ms_teacher', 'email_student', 'ms_student', 'ms_federated_teacher', 'ms_federated_student', 'standard', 'developer'] # TODO: Implement dev_ user types for pytests, consider use of cc_ user types
+
+router = APIRouter()
+
+# Helpers
+def initialise_schools_from_config():
+        """Initialize a school with the configuration provided from env variables
+        """
+        default_config = {
+            "school_uuid": "kevlarai",
+            "school_name": "KevlarAI School",
+            "school_website": "https://kevlarai.com",
+            "timetable_file": "kevlarai_data/kevlarai_timetable.xlsx",
+            "curriculum_file": "kevlarai_data/kevlarai_curriculum.xlsx"
+        }
+        
+        # school_config_str = os.getenv("SCHOOL_CONFIG") # TODO: Implement this
+        school_config = default_config
+    
+        db_name = f"cc.institutes.{school_config['school_uuid']}"
+        curriculum_db_name = f"{db_name}.curriculum"
+        
+        logger.info(f"Creating database for {school_config['school_name']} using db_name: {db_name}")
+        driver = driver_tools.get_driver()
+        if driver is None:
+            logger.error("Failed to connect to Neo4j")
+            return
+        
+        with driver.session() as session:
+            # Create main school database
+            session_tools.create_database(session, db_name)
+            logger.debug(f"Database {db_name} created")
+            
+            # Create curriculum database
+            session_tools.create_database(session, curriculum_db_name)
+            logger.debug(f"Curriculum database {curriculum_db_name} created")
+        
+        # Add filesystem path debugging
+        base_path = os.getenv("NODE_FILESYSTEM_PATH")
+        schools_path = os.path.join(base_path, "schools")
+        school_path = os.path.join(schools_path, f"cc.institutes.{school_config['school_uuid']}")
+        
+        logger.debug("Filesystem paths:", {
+            "base_path": base_path,
+            "schools_path": schools_path,
+            "school_path": school_path
+        })
+        
+        # Check if directories exist
+        logger.debug("Directory existence check:", {
+            "base_exists": os.path.exists(base_path),
+            "schools_exists": os.path.exists(schools_path),
+            "school_exists": os.path.exists(school_path)
+        })
+        
+        # Create database entry for school without timetable or curriculum
+        logger.info(f"Creating school entry for {school_config['school_name']} in database {db_name} without timetable or curriculum")
+        result = init_school.create_school(
+            db_name=db_name,
+            school_uuid=school_config["school_uuid"],
+            school_name=school_config["school_name"],
+            school_website=school_config["school_website"]
+        )
+        logger.success(f"{school_config['school_name']} school entry created successfully")
+        
+        # Create school node from result
+        school_node = result['school_node']
+        refreshed_school_node = SchoolNode(
+            unique_id=school_node.unique_id,
+            school_uuid=school_node.school_uuid,
+            school_name=school_node.school_name,
+            school_website=school_node.school_website,
+            path=school_node.path
+        )
+
+        # Create timetable entries for school from Excel file
+        timetable_file = os.path.join(os.getenv("BACKEND_INIT_PATH"), school_config["timetable_file"])
+        
+        logger.info(f"Creating timetable entries for {school_config['school_name']} using timetable file: {timetable_file}.")
+        school_timetable_dataframes = xl.create_dataframes(timetable_file)
+        
+        
+        init_school_timetable.create_school_timetable(
+            dataframes=school_timetable_dataframes, 
+            db_name=db_name, 
+            school_node=refreshed_school_node
+        )
+        logger.success("Timetable entries created successfully")
+
+        # Create curriculum entries for school from Excel file in both databases
+        curriculum_file = os.path.join(os.getenv("BACKEND_INIT_PATH"), school_config["curriculum_file"])
+        school_curriculum_dataframes = xl.create_dataframes(curriculum_file)
+        
+        logger.info(f"Creating curriculum entries for {school_config['school_name']} using curriculum file: {curriculum_file}.")
+        init_school_curriculum.create_curriculum(
+            dataframes=school_curriculum_dataframes, 
+            db_name=db_name,
+            curriculum_db_name=curriculum_db_name,
+            school_node=refreshed_school_node
+        )
+        logger.success("Curriculum entries created successfully")
+
+
+@router.post("/create-user")
+async def create_user(
+    user_id: str = Form(...),
+    user_type: str = Form(...),
+    user_name: str = Form(...),
+    user_email: str = Form(...),
+    school_uuid: str = Form(None),
+    school_name: str = Form(None),
+    school_website: str = Form(None),
+    school_path: str = Form(None),
+    worker_data: str = Form(None)
+):
+    logger.info(f"Creating user with user_id: {user_id}, user_type: {user_type}, user_name: {user_name}, user_email: {user_email}")
+    
+    if school_uuid:
+        logger.info(f"School UUID provided: {school_uuid}")
+    else:
+        logger.info(f"No school UUID provided")
+        
+    if school_name:
+        logger.info(f"School name provided: {school_name}")
+    else:
+        logger.info(f"No school name provided")
+    
+    if school_website:
+        logger.info(f"School website provided: {school_website}")
+    else:
+        logger.info(f"No school website provided")
+        
+    if school_path:
+        logger.info(f"School path provided: {school_path}")
+    else:
+        logger.info(f"No school path provided")
+    
+    if worker_data:
+        logger.info(f"Worker data provided: {worker_data}")
+    else:
+        logger.info(f"No worker data provided")
+    
+    # Validate inputs
+    if any(param is None for param in (user_type, user_name, user_email, user_id)):
+        raise HTTPException(status_code=400, detail=f"Invalid user data")
+    
+    if user_type not in VALID_USER_TYPES:
+        raise HTTPException(status_code=400, detail=f"Invalid user type: {user_type}")
+    
+    try:
+        # Parse worker data
+        worker_data_dict = json.loads(worker_data) if worker_data else None
+        
+        # Create school node if school data provided
+        school_node = None
+        if all([school_uuid, school_name, school_website, school_path]):
+            school_node = SchoolNode(
+                unique_id=f'School_{school_uuid}',
+                school_uuid=school_uuid,
+                school_name=school_name,
+                school_website=school_website,
+                path=school_path
+            )
+        
+        # Create user with single database reference
+        formatted_email = format_user_email_for_neo_db(user_email)
+        user_db_name = f"cc.users.{formatted_email}"
+        
+        result = init_user.create_user(
+            db_name=user_db_name,
+            user_id=user_id,
+            user_type=user_type,
+            username=user_name,
+            email=user_email,
+            school_node=school_node,
+            worker_data=worker_data_dict
+        )
+        
+        # Ensure the result is JSON serializable
+        response_data = {
+            "status": "success",
+            "data": {
+                "user_node": result['user_node'],
+                "worker_node": result['worker_node'],
+                "calendar_nodes": result.get('calendar_nodes')
+            }
+        }
+        
+        return JSONResponse(content=response_data)
+        
+    except Exception as e:
+        logger.error(f"Error creating user in Neo4j: {str(e)}", exc_info=True)
+        return JSONResponse(
+            content={"status": "error", "message": str(e)},
+            status_code=500
+        )
+
+@router.post("/create-schools")
+async def create_schools():
+    initialise_schools_from_config()
+    return JSONResponse(content={"status": "success", "message": "Schools created successfully"})
+
+@router.post("/create-department")
+async def create_department(
+    db_name: str = Form(...),
+    unique_id: str = Form(...),
+    department_name: str = Form(...),
+    department_code: str = Form(...),
+    path: str = Form(...)
+):
+    if db_name is None or unique_id is None or department_name is None or department_code is None or path is None:
+        logging.error(f"Invalid department data: {db_name}, {unique_id}, {department_name}, {department_code}, {path}")
+        raise HTTPException(status_code=400, detail="Invalid department data")
+
+    department = DepartmentNode(
+        unique_id=unique_id,
+        department_name=department_name,
+        department_code=department_code,
+        path=path
+    )
+
+    logger.info(f"Creating department {department_name} with unique_id {unique_id}")
+    try:
+        result = init_school.create_department(db_name, department)
+        return JSONResponse(content={"status": "success", "data": result})
+    except Exception as e:
+        logger.error(f"Error creating department: {str(e)}")
+        return JSONResponse(content={"status": "error", "message": str(e)}, status_code=500)
+
+@router.post("/create-class")
+async def create_class(
+    db_name: str = Form(...),
+    unique_id: str = Form(...),
+    subject_class_code: str = Form(...),
+    year_group: str = Form(...),
+    subject: str = Form(...),
+    subject_code: str = Form(...),
+    path: str = Form(...)
+):
+    subject_class_node = SubjectClassNode(
+        unique_id=unique_id,
+        subject_class_code=subject_class_code,
+        year_group=year_group,
+        subject=subject,
+        subject_code=subject_code,
+        path=path
+    )
+    # Implementation for creating a class
+    pass
+
+@router.post("/create-room")
+async def create_room(
+    db_name: str = Form(...),
+    room_unique_id: str = Form(...),
+    room_code: str = Form(...),
+    path: str = Form(...)
+):
+    room = RoomNode(
+        room_unique_id=room_unique_id,
+        room_code=room_code,
+        path=path
+    )
+    # Implementation for creating a room
+    pass

routers/database/init/get_data.py

@@ -0,0 +1,28 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'api_routers_database_init_get_data'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+import modules.database.init.xl_tools as xl
+from fastapi import APIRouter, File, UploadFile
+
+router = APIRouter()
+
+@router.post("/get-dataframes-from-xl")
+async def get_dataframes_from_xl(file: UploadFile = File(...)):
+    if file.content_type != 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':
+        return {"status": "Error", "message": "Invalid file format"}
+    try:
+        logging.info(f"Getting dataframes from {file.filename}")
+        return xl.create_dataframes(await file.read())
+    except Exception as e:
+        return {"status": "Error", "message": str(e)}

routers/database/init/schools.py

@@ -0,0 +1,115 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'api_routers_database_init_schools'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+from fastapi import APIRouter, File, UploadFile, Form, HTTPException, BackgroundTasks
+import pandas as pd
+import modules.database.tools.neo4j_driver_tools as driver
+from modules.database.tools.neo4j_session_tools import get_node_by_unique_id
+import modules.database.init.init_school_timetable as init_school_timetable
+import modules.database.init.init_worker_timetable as init_worker_timetable
+from modules.database.schemas.nodes.schools.schools import SchoolNode
+import modules.database.init.xl_tools as xl
+import json
+
+router = APIRouter()
+
+@router.post("/upload-school-timetable")
+async def upload_school_timetable(
+    file: UploadFile = File(...),
+    db_name: str = Form(...),
+    unique_id: str = Form(...),
+    school_uuid: str = Form(...),
+    school_name: str = Form(...),
+    school_website: str = Form(...),
+    path: str = Form(...)
+):
+    school_node = SchoolNode(
+        unique_id=unique_id,
+        school_uuid=school_uuid,
+        school_name=school_name,
+        school_website=school_website,
+        path=path
+    )
+    if file.content_type != 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':
+        return {"status": "Error", "message": "Invalid file format"}
+    logging.info(f"Uploading timetable for {db_name} from {file.filename}")
+    dataframes = xl.create_dataframes_from_fastapiuploadfile(file)
+    return init_school_timetable.create_school_timetable(dataframes, db_name, school_node)
+
+@router.post("/upload-worker-timetable")
+async def upload_worker_timetable(
+    background_tasks: BackgroundTasks,
+    file: UploadFile = File(...),
+    worker_node: str = Form(...)
+):
+    if file.content_type != 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':
+        raise HTTPException(status_code=422, detail="Invalid file format")
+    
+    try:
+        worker_node_data = json.loads(worker_node)
+        logging.info(f"Uploading worker timetable for {worker_node_data['teacher_code']} from {file.filename} for {worker_node_data['worker_db_name']}")
+        logging.debug(f"Worker node data: {worker_node_data}")
+        
+        # Read file content into memory
+        file_content = await file.read()
+        
+        # Schedule the processing of the timetable in the background
+        background_tasks.add_task(
+            process_worker_timetable,
+            file_content,
+            worker_node_data
+        )
+        
+        return {
+            "status": "Accepted",
+            "message": "Processing of teacher timetable started"
+        }
+    except Exception as e:
+        logging.error(f"Error handling timetable upload: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+async def process_worker_timetable(file_content, worker_node_data):
+    neo_driver = driver.get_driver(db_name=worker_node_data['worker_db_name'])
+    if neo_driver is None:
+        logging.error(f"Failed to connect to the database {worker_node_data['worker_db_name']}")
+        return
+    
+    try:
+        # Create a DataFrame from the file content
+        from io import BytesIO
+        timetable_df = pd.read_excel(BytesIO(file_content))
+        
+        # Get the school version of the worker node
+        logging.info(f"Getting school worker node for {worker_node_data['unique_id']} from {worker_node_data['worker_db_name']}")
+        
+        with neo_driver.session(database=worker_node_data['worker_db_name']) as neo_session:
+            school_worker_node = get_node_by_unique_id(session=neo_session, unique_id=worker_node_data['unique_id'])
+            
+            if school_worker_node is None:
+                error_msg = f"School worker node not found for unique_id: {worker_node_data['unique_id']}"
+                logging.error(error_msg)
+                raise Exception(error_msg)
+                
+            logging.debug(f"School worker node found: {school_worker_node}")
+            
+            logging.info(f"Initializing worker timetable for school worker: {school_worker_node['teacher_code']}")
+            init_worker_timetable.init_worker_timetable(timetable_df, school_worker_node)
+            logging.info(f"Worker timetable initialized for school worker: {school_worker_node['teacher_code']}")
+            
+    except Exception as e:
+        logging.error(f"Error processing worker timetable: {str(e)}")
+        raise
+    finally:
+        logging.info(f"Closing driver for {worker_node_data['worker_db_name']}")
+        driver.close_driver(neo_driver)

routers/database/init/timetables.py

@@ -0,0 +1,166 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'api_routers_database_init_timetables'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+from fastapi import APIRouter, File, UploadFile, Form, HTTPException, BackgroundTasks
+import pandas as pd
+import modules.database.tools.neo4j_driver_tools as driver
+from modules.database.tools.neo4j_session_tools import get_node_by_unique_id
+import modules.database.init.init_school_timetable as init_school_timetable
+import modules.database.init.init_worker_timetable as init_worker_timetable
+from modules.database.schemas.nodes.users import UserNode
+from modules.database.schemas.nodes.schools.schools import SchoolNode
+from modules.database.schemas.nodes.workers.workers import TeacherNode
+import modules.database.init.xl_tools as xl
+import json
+import modules.database.tools.neontology_tools as neon
+
+router = APIRouter()
+
+@router.post("/upload-school-timetable")
+async def upload_school_timetable(
+    file: UploadFile = File(...),
+    db_name: str = Form(...),
+    unique_id: str = Form(...),
+    school_uuid: str = Form(...),
+    school_name: str = Form(...),
+    school_website: str = Form(...),
+    path: str = Form(...)
+):
+    school_node = SchoolNode(
+        unique_id=unique_id,
+        school_uuid=school_uuid,
+        school_name=school_name,
+        school_website=school_website,
+        path=path
+    )
+    if file.content_type != 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':
+        return {"status": "Error", "message": "Invalid file format"}
+    logging.info(f"Uploading timetable for {db_name} from {file.filename}")
+    dataframes = xl.create_dataframes_from_fastapiuploadfile(file)
+    return init_school_timetable.create_school_timetable(dataframes, db_name, school_node)
+
+@router.post("/upload-worker-timetable")
+async def upload_worker_timetable(
+    background_tasks: BackgroundTasks,
+    file: UploadFile = File(...),
+    user_node: str = Form(...),
+    worker_node: str = Form(...)
+):
+    if file.content_type != 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':
+        raise HTTPException(status_code=422, detail="Invalid file format")
+    
+    try:
+        worker_node_data = json.loads(worker_node)
+        user_node_data = json.loads(user_node)
+        logging.info(f"Uploading worker timetable for {worker_node_data['teacher_code']} from {file.filename} for {worker_node_data['worker_db_name']}")
+        logging.debug(f"Worker node data: {worker_node_data}")
+        logging.debug(f"User node data: {user_node_data}")
+
+        # Read file content into memory
+        file_content = await file.read()
+        
+        # Schedule the processing of the timetable in the background
+        background_tasks.add_task(
+            process_worker_timetable,
+            file_content,
+            user_node_data,
+            worker_node_data
+        )
+        
+        return {
+            "status": "Accepted",
+            "message": "Processing of teacher timetable started"
+        }
+    except Exception as e:
+        logging.error(f"Error handling timetable upload: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+async def process_worker_timetable(file_content, user_node_data, worker_node_data):
+    # Initialize neontology connection first
+    neon.init_neontology_connection()
+    
+    neo_driver = driver.get_driver(db_name=worker_node_data['worker_db_name'])
+    if neo_driver is None:
+        logging.error(f"Failed to connect to the database {worker_node_data['worker_db_name']}")
+        return
+    
+    try:
+        # Create a DataFrame from the file content
+        from io import BytesIO
+        timetable_df = pd.read_excel(BytesIO(file_content))
+        
+        # Get the school version of the worker node
+        logging.info(f"Getting school worker node for {worker_node_data['unique_id']} from {worker_node_data['worker_db_name']}")
+        
+        with neo_driver.session(database=worker_node_data['worker_db_name']) as neo_session:
+            school_worker_node = get_node_by_unique_id(session=neo_session, unique_id=worker_node_data['unique_id'])
+            
+            if school_worker_node is None:
+                error_msg = f"School worker node not found for unique_id: {worker_node_data['unique_id']}"
+                logging.error(error_msg)
+                raise Exception(error_msg)
+                
+            logging.debug(f"School worker node found: {school_worker_node}")
+            
+            # Create timetable in school database
+            logging.info(f"Initializing worker timetable for school worker: {school_worker_node['teacher_code']}")
+            init_worker_timetable.init_worker_timetable(timetable_df, school_worker_node)
+            logging.info(f"Worker timetable initialized for school worker: {school_worker_node['teacher_code']}")
+            
+            # Create timetable in user database
+            if 'user_db_name' in worker_node_data:
+                from modules.database.init.init_user_timetable import create_user_worker_timetable
+                from modules.database.schemas.nodes.workers.workers import TeacherNode
+                
+                logging.info(f"Creating user timetable structure in {worker_node_data['user_db_name']}")
+                
+                # Create TeacherNode from worker_node_data
+                user_worker_node = TeacherNode(
+                    unique_id=worker_node_data['unique_id'],
+                    teacher_code=worker_node_data['teacher_code'],
+                    teacher_name_formal=worker_node_data['teacher_name_formal'],
+                    teacher_email=worker_node_data['teacher_email'],
+                    path=worker_node_data['path'],
+                    worker_db_name=worker_node_data['worker_db_name'],
+                    user_db_name=worker_node_data['user_db_name']
+                )
+                
+                # Create user node
+                user_node = UserNode(
+                    unique_id=user_node_data['unique_id'],
+                    user_id=user_node_data['user_id'],
+                    user_type=user_node_data['user_type'],
+                    user_name=user_node_data['user_name'],
+                    user_email=user_node_data['user_email'],
+                    path=user_node_data['path'],
+                    worker_node_data=user_node_data['worker_node_data']
+                )
+                
+                # Create user timetable structure
+                create_user_worker_timetable(
+                    user_node=user_node,
+                    user_worker_node=user_worker_node,
+                    school_db_name=worker_node_data['worker_db_name']
+                )
+
+                logging.info(f"User timetable structure created in {worker_node_data['user_db_name']}")
+            else:
+                logging.warning("No user_db_name provided, skipping user timetable creation")
+            
+    except Exception as e:
+        logging.error(f"Error processing worker timetable: {str(e)}")
+        raise
+    finally:
+        logging.info(f"Closing driver for {worker_node_data['worker_db_name']}")
+        driver.close_driver(neo_driver)

routers/database/schools.py

@@ -0,0 +1,99 @@
+from fastapi import APIRouter, Depends, File, UploadFile
+from backend.app.run.dependencies import admin_dependency
+from pydantic import BaseModel
+
+router = APIRouter()
+
+class NodeBase(BaseModel):
+    Name: str
+
+class LocalAuthority(NodeBase):
+    pass
+
+class SchoolNode(NodeBase):
+    Type: str
+    Status: str
+
+class ParliamentaryConstituency(NodeBase):
+    pass
+
+class AdministrativeWard(NodeBase):
+    pass
+
+class RelationshipBase(BaseModel):
+    start_node: NodeBase
+    end_node: NodeBase
+    relationship_type: str
+
+class HasParliamentaryConstituency(RelationshipBase):
+    pass
+
+class HasAdministrativeWard(RelationshipBase):
+    pass
+
+class HasSchool(RelationshipBase):
+    pass
+
+@router.post("/batch-create-schools")
+async def add_school_to_global(file: UploadFile = File(...)):
+    if file is None:
+        return {"status": "Error", "message": "No file received"}
+
+    try:
+        import pandas as pd
+        from io import BytesIO
+        from app.modules.driver_tools import create_node_http, create_relationship_http
+        data = pd.read_csv(BytesIO(await file.read()), usecols=["LA (name)", "ParliamentaryConstituency (name)", "AdministrativeWard (name)", "EstablishmentName", "TypeOfEstablishment (name)", "EstablishmentStatus (name)"])
+        unique_las = data["LA (name)"].unique()
+        for la_name in unique_las:
+            la_node = {"Name": la_name}
+            la_id = create_node_http("LocalAuthority", la_node, db="GlobalSchools")
+            constituencies = data[data["LA (name)"] == la_name]["ParliamentaryConstituency (name)"].unique()
+            for constituency in constituencies:
+                constituency_node = {"Name": constituency}
+                constituency_id = create_node_http("ParliamentaryConstituency", constituency_node, db="GlobalSchools")
+                create_relationship_http({"start_node": {"id": la_id}, "end_node": {"id": constituency_id}, "relationship_type": "HAS_PARLIAMENTARY_CONSTITUENCY"}, db="GlobalSchools")
+                wards = data[(data["LA (name)"] == la_name) & (data["ParliamentaryConstituency (name)"] == constituency)]["AdministrativeWard (name)"].unique()
+                for ward in wards:
+                    ward_node = {"Name": ward}
+                    ward_id = create_node_http("AdministrativeWard", ward_node, db="GlobalSchools")
+                    create_relationship_http({"start_node": {"id": constituency_id}, "end_node": {"id": ward_id}, "relationship_type": "HAS_ADMINISTRATIVE_WARD"}, db="GlobalSchools")
+                    schools = data[(data["LA (name)"] == la_name) & (data["ParliamentaryConstituency (name)"] == constituency) & (data["AdministrativeWard (name)"] == ward)]
+                    for index, school in schools.iterrows():
+                        school_node = {
+                            "Name": school["EstablishmentName"],
+                            "Type": school["TypeOfEstablishment (name)"],
+                            "Status": school["EstablishmentStatus (name)"]
+                        }
+                        school_id = create_node_http("School", school_node, db="GlobalSchools")
+                        create_relationship_http({"start_node": {"id": ward_id}, "end_node": {"id": school_id}, "relationship_type": "HAS_SCHOOL"}, db="GlobalSchools")
+        return {"status": "Success", "message": "Graph structure updated successfully"}
+    except Exception as e:
+        print("Failed to process file:", e)
+        return {"status": "Error", "message": "Failed to process file"}
+
+@router.post("/create-school")
+async def add_school_to_global(file: UploadFile = File(...)):
+    if file is None:
+        return {"status": "Error", "message": "No file received"}
+
+    try:
+        import pandas as pd
+        from io import BytesIO
+        data = pd.read_excel(BytesIO(await file.read()), usecols=[0], nrows=5).squeeze()
+        print("Data read from file:", data)
+        if len(data) < 5:
+            return {"status": "Error", "message": "Insufficient data in file"}
+        school_data = {
+            "name": data[0],
+            "address": data[1],
+            "ofsted_number": data[2],
+            "website": data[3],
+            "geo_location": data[4]
+        }
+        from app.modules.driver_tools import create_node_http
+        response = create_node_http("globalschools", "School", school_data)
+        return {"status": "School added to global school db via HTTP", "school_data": school_data, "response": response}
+    except Exception as e:
+        print("Failed to process file:", e)
+        return {"status": "Error", "message": "Failed to process file"}

routers/database/student.py

@@ -0,0 +1,5 @@
+from fastapi import APIRouter, Depends, File, UploadFile
+from backend.app.run.dependencies import admin_dependency
+
+router = APIRouter()
+

routers/database/teacher.py

@@ -0,0 +1,5 @@
+from fastapi import APIRouter, Depends, File, UploadFile
+from backend.app.run.dependencies import admin_dependency
+
+router = APIRouter()
+

routers/database/tools/calendar_structure_router.py

@@ -0,0 +1,220 @@
+import os
+from fastapi import APIRouter, HTTPException
+from typing import List, Dict, Any, Optional
+from datetime import datetime, timedelta
+from modules.logger_tool import initialise_logger
+from modules.database.tools import neo4j_driver_tools as driver_tools
+
+logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
+router = APIRouter()
+
+@router.get("/get-calendar-structure")
+async def get_calendar_structure(db_name: str) -> Dict[str, Any]:
+    """
+    Get the complete calendar structure including years, months, weeks, and days.
+    """
+    try:
+        # Get all calendar nodes in a single query
+        query = """
+        // Match all calendar-related nodes
+        MATCH (y:CalendarYear)
+        OPTIONAL MATCH (y)-[:YEAR_INCLUDES_MONTH]->(m:CalendarMonth)
+        OPTIONAL MATCH (m)-[:MONTH_INCLUDES_DAY]->(d:CalendarDay)
+        OPTIONAL MATCH (w:CalendarWeek)-[:WEEK_INCLUDES_DAY]->(d)
+        WITH y, m, w, d
+        ORDER BY y.date, m.date, w.date, d.date
+
+        // Collect all nodes with dates converted to strings
+        RETURN {
+            years: collect(DISTINCT {
+                id: y.unique_id,
+                path: y.path,
+                date: toString(y.date),
+                __primarylabel__: 'CalendarYear'
+            }),
+            months: collect(DISTINCT {
+                id: m.unique_id,
+                path: m.path,
+                date: toString(m.date),
+                __primarylabel__: 'CalendarMonth'
+            }),
+            weeks: collect(DISTINCT {
+                id: w.unique_id,
+                path: w.path,
+                date: toString(w.date),
+                __primarylabel__: 'CalendarWeek'
+            }),
+            days: collect(DISTINCT {
+                id: d.unique_id,
+                path: d.path,
+                date: toString(d.date),
+                week_id: w.unique_id,
+                month_id: m.unique_id,
+                __primarylabel__: 'CalendarDay'
+            })
+        } as structure
+        """
+
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query)
+            record = result.single()
+            if not record:
+                raise HTTPException(status_code=404, detail="Calendar structure not found")
+            
+            structure = record["structure"]
+            
+            # Find current day using string comparison
+            today = datetime.now().strftime("%Y-%m-%d")
+            current_day = next(
+                (day["id"] for day in structure["days"] 
+                 if day["date"] == today),
+                structure["days"][0]["id"] if structure["days"] else None
+            )
+
+            return {
+                "status": "success",
+                "structure": {
+                    "years": structure["years"],
+                    "months": structure["months"],
+                    "weeks": structure["weeks"],
+                    "days": structure["days"],
+                    "currentDay": current_day
+                }
+            }
+
+    except Exception as e:
+        logger.error(f"Error getting calendar structure: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/get-calendar-days")
+async def get_calendar_days(db_name: str, start_date: str, end_date: str) -> Dict[str, Any]:
+    """
+    Get all calendar days in a date range.
+    """
+    try:
+        query = """
+        MATCH (d:CalendarDay)
+        WHERE date(d.date) >= date($start_date) AND date(d.date) <= date($end_date)
+        OPTIONAL MATCH (w:CalendarWeek)-[:WEEK_INCLUDES_DAY]->(d)
+        OPTIONAL MATCH (m:CalendarMonth)-[:MONTH_INCLUDES_DAY]->(d)
+        RETURN {
+            id: d.unique_id,
+            path: d.path,
+            date: d.date,
+            week_id: w.unique_id,
+            month_id: m.unique_id,
+            __primarylabel__: 'CalendarDay'
+        } as day
+        ORDER BY d.date
+        """
+
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query, start_date=start_date, end_date=end_date)
+            days = [record["day"] for record in result]
+
+            return {
+                "status": "success",
+                "days": days
+            }
+
+    except Exception as e:
+        logger.error(f"Error getting calendar days: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/get-calendar-weeks")
+async def get_calendar_weeks(db_name: str, start_date: str, end_date: str) -> Dict[str, Any]:
+    """
+    Get all calendar weeks in a date range.
+    """
+    try:
+        query = """
+        MATCH (w:CalendarWeek)-[:WEEK_INCLUDES_DAY]->(d:CalendarDay)
+        WHERE date(w.date) >= date($start_date) AND date(w.date) <= date($end_date)
+        WITH w, collect(d) as days
+        RETURN {
+            id: w.unique_id,
+            path: w.path,
+            date: w.date,
+            day_ids: [day in days | day.unique_id],
+            __primarylabel__: 'CalendarWeek'
+        } as week
+        ORDER BY w.date
+        """
+
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query, start_date=start_date, end_date=end_date)
+            weeks = [record["week"] for record in result]
+
+            return {
+                "status": "success",
+                "weeks": weeks
+            }
+
+    except Exception as e:
+        logger.error(f"Error getting calendar weeks: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/get-calendar-months")
+async def get_calendar_months(db_name: str, start_date: str, end_date: str) -> Dict[str, Any]:
+    """
+    Get all calendar months in a date range.
+    """
+    try:
+        query = """
+        MATCH (m:CalendarMonth)-[:MONTH_INCLUDES_DAY]->(d:CalendarDay)
+        WHERE date(m.date) >= date($start_date) AND date(m.date) <= date($end_date)
+        WITH m, collect(d) as days
+        RETURN {
+            id: m.unique_id,
+            path: m.path,
+            date: m.date,
+            day_ids: [day in days | day.unique_id],
+            __primarylabel__: 'CalendarMonth'
+        } as month
+        ORDER BY m.date
+        """
+
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query, start_date=start_date, end_date=end_date)
+            months = [record["month"] for record in result]
+
+            return {
+                "status": "success",
+                "months": months
+            }
+
+    except Exception as e:
+        logger.error(f"Error getting calendar months: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/get-calendar-years")
+async def get_calendar_years(db_name: str) -> Dict[str, Any]:
+    """
+    Get all calendar years.
+    """
+    try:
+        query = """
+        MATCH (y:CalendarYear)-[:YEAR_INCLUDES_MONTH]->(m:CalendarMonth)
+        WITH y, collect(m) as months
+        RETURN {
+            id: y.unique_id,
+            path: y.path,
+            date: y.date,
+            month_ids: [month in months | month.unique_id],
+            __primarylabel__: 'CalendarYear'
+        } as year
+        ORDER BY y.date
+        """
+
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query)
+            years = [record["year"] for record in result]
+
+            return {
+                "status": "success",
+                "years": years
+            }
+
+    except Exception as e:
+        logger.error(f"Error getting calendar years: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e)) 

routers/database/tools/default_nodes_router.py

@@ -0,0 +1,257 @@
+from fastapi import APIRouter, HTTPException
+from typing import Dict, Any
+from modules.database.tools import neo4j_driver_tools as driver_tools
+from modules.logger_tool import initialise_logger
+from neo4j.time import DateTime, Date
+import os
+from datetime import datetime
+
+logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
+router = APIRouter()
+
+def convert_neo4j_values(value: Any) -> Any:
+    """Convert Neo4j types to JSON-serializable types."""
+    if isinstance(value, DateTime):
+        return value.isoformat()  # Convert to ISO format string
+    elif isinstance(value, Date):
+        return value.isoformat()  # Convert Date to ISO format string
+    elif isinstance(value, dict):
+        return {k: convert_neo4j_values(v) for k, v in value.items()}
+    elif isinstance(value, list):
+        return [convert_neo4j_values(v) for v in value]
+    return value
+
+def get_default_node_week(db_name: str) -> Dict[str, Any]:
+    """Get the current week node."""
+    # Get today's date
+    today = datetime.now()
+    
+    # Find the calendar week node that contains today's date
+    query = """
+    MATCH (w:CalendarWeek)
+    WHERE date(w.start_date) <= date($today) AND date($today) <= date(w.start_date) + duration('P7D')
+    RETURN w
+    """
+    
+    with driver_tools.get_session(database=db_name) as session:
+        result = session.run(query, today=today.strftime('%Y-%m-%d'))
+        week_node = result.single()
+        
+        if not week_node:
+            raise HTTPException(status_code=404, detail="No default node found for context: week")
+            
+        node = week_node["w"]
+        node_data = dict(node)
+        converted_data = convert_neo4j_values(node_data)
+        
+        return {
+            "status": "success",
+            "node": {
+                "id": node["unique_id"],
+                "path": node["path"],
+                "type": "CalendarWeek",
+                "label": node.get("title", "Calendar Week"),
+                "data": converted_data
+            }
+        }
+
+def get_default_node_month(db_name: str) -> Dict[str, Any]:
+    """Get the current month node."""
+    # Get today's date
+    today = datetime.now()
+    
+    # Find the calendar month node for the current month
+    query = """
+    MATCH (m:CalendarMonth)
+    WHERE m.year = $year AND m.month = $month
+    RETURN m
+    """
+    
+    with driver_tools.get_session(database=db_name) as session:
+        result = session.run(query, year=str(today.year), month=str(today.month))
+        month_node = result.single()
+        
+        if not month_node:
+            raise HTTPException(status_code=404, detail="No default node found for context: month")
+            
+        node = month_node["m"]
+        node_data = dict(node)
+        converted_data = convert_neo4j_values(node_data)
+        
+        return {
+            "status": "success",
+            "node": {
+                "id": node["unique_id"],
+                "path": node["path"],
+                "type": "CalendarMonth",
+                "label": node.get("title", "Calendar Month"),
+                "data": converted_data
+            }
+        }
+
+@router.get("/get-default-node/{context}")
+async def get_default_node(context: str, db_name: str, base_context: str | None = None) -> Dict[str, Any]:
+    """Get the default node for a given context."""
+    try:
+        # Handle special cases for week and month
+        if context == 'week':
+            return get_default_node_week(db_name)
+        elif context == 'month':
+            return get_default_node_month(db_name)
+
+        # Map contexts to their default node queries
+        context_queries = {
+            # Base Contexts
+            'profile': """
+                MATCH (n:User)
+                RETURN n LIMIT 1
+            """,
+            'worker': """
+                MATCH (n)
+                WHERE n:SchoolAdmin OR n:Teacher OR n:Student OR n:Developer OR n:SuperAdmin
+                RETURN n LIMIT 1
+            """,
+            'calendar': """
+                MATCH (n:Calendar)
+                RETURN n LIMIT 1
+            """,
+            'teaching': """
+                MATCH (n:Teacher)
+                RETURN n LIMIT 1
+            """,
+            'school': """
+                MATCH (n:School)
+                RETURN n LIMIT 1
+            """,
+            'department': """
+                MATCH (n:Department)
+                RETURN n LIMIT 1
+            """,
+            'class': """
+                MATCH (n:Class)
+                RETURN n LIMIT 1
+            """,
+
+            # Extended Contexts - Overview queries for each base context
+            'overview': """
+                MATCH (n)
+                WHERE CASE $base_context
+                    WHEN 'profile' THEN n:User
+                    WHEN 'calendar' THEN n:Calendar
+                    WHEN 'teaching' THEN n:Teacher
+                    WHEN 'school' THEN n:School
+                    WHEN 'department' THEN n:Department
+                    WHEN 'class' THEN n:Class
+                    ELSE false
+                END
+                RETURN n LIMIT 1
+            """,
+
+            # Extended Contexts - User
+            'settings': """
+                MATCH (n:User)
+                RETURN n LIMIT 1
+            """,
+            'history': """
+                MATCH (n:User)
+                RETURN n LIMIT 1
+            """,
+            'journal': """
+                MATCH (n:Journal)
+                RETURN n LIMIT 1
+            """,
+            'planner': """
+                MATCH (n:Planner)
+                RETURN n LIMIT 1
+            """,
+
+            # Extended Contexts - Calendar
+            'day': """
+                MATCH (n:CalendarDay)
+                WHERE date(n.date) = date()
+                RETURN n LIMIT 1
+            """,
+            'year': """
+                MATCH (n:CalendarYear)
+                WHERE n.year = toString(date().year)
+                RETURN n LIMIT 1
+            """,
+
+            # Extended Contexts - Teaching
+            'timetable': """
+                MATCH (n:UserTeacherTimetable)
+                RETURN n LIMIT 1
+            """,
+            'classes': """
+                MATCH (n:Class)
+                RETURN n LIMIT 1
+            """,
+            'lessons': """
+                MATCH (n:TimetableLesson)
+                RETURN n LIMIT 1
+            """,
+
+            # Extended Contexts - School
+            'departments': """
+                MATCH (n:Department)
+                RETURN n LIMIT 1
+            """,
+            'staff': """
+                MATCH (n:Teacher)
+                RETURN n LIMIT 1
+            """,
+
+            # Extended Contexts - Department
+            'teachers': """
+                MATCH (n:Teacher)
+                RETURN n LIMIT 1
+            """,
+            'subjects': """
+                MATCH (n:Subject)
+                RETURN n LIMIT 1
+            """,
+
+            # Extended Contexts - Class
+            'students': """
+                MATCH (n:Student)
+                RETURN n LIMIT 1
+            """
+        }
+
+        if context not in context_queries:
+            raise HTTPException(status_code=400, detail=f"Invalid context: {context}")
+
+        query = context_queries[context]
+        
+        with driver_tools.get_session(database=db_name) as session:
+            # For overview context, we need to pass the database name as a parameter
+            params = {'db_name': db_name, 'base_context': base_context} if context == 'overview' else {}
+            result = session.run(query, params)
+            record = result.single()
+            
+            if not record:
+                raise HTTPException(
+                    status_code=404, 
+                    detail=f"No default node found for context: {context}"
+                )
+            
+            node = record["n"]
+            node_data = dict(node)
+            
+            # Convert Neo4j types to JSON-serializable types
+            converted_data = convert_neo4j_values(node_data)
+            
+            return {
+                "status": "success",
+                "node": {
+                    "id": node["unique_id"],
+                    "path": node["path"],
+                    "type": list(node.labels)[0],
+                    "label": node.get("title", ""),
+                    "data": converted_data
+                }
+            }
+
+    except Exception as e:
+        logger.error(f"Error getting default node: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))

routers/database/tools/get_events.py

@@ -0,0 +1,101 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'api_routers_calendar_get_events'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+import modules.database.tools.neo4j_driver_tools as driver
+from fastapi import APIRouter, HTTPException
+import colorsys
+import random
+
+# Predefined vibrant color palette
+BASE_COLORS = [
+    "#FF4136", "#FF851B", "#FFDC00", "#2ECC40", "#0074D9", "#B10DC9",
+    "#F012BE", "#FF6F61", "#7FDBFF", "#01FF70", "#001f3f", "#85144b",
+    "#39CCCC", "#3D9970", "#e74c3c", "#e67e22", "#f1c40f", "#2ecc71",
+    "#1abc9c", "#3498db", "#9b59b6", "#34495e", "#16a085", "#27ae60",
+    "#2980b9", "#8e44ad", "#2c3e50", "#d35400", "#c0392b", "#bdc3c7",
+    "#7f8c8d", "#00a86b", "#8B4513", "#4B0082", "#800000", "#1E90FF"
+]
+
+def generate_vibrant_color():
+    h = random.random()
+    s = 0.5 + random.random() * 0.5  # 0.5 to 1.0
+    v = 0.5 + random.random() * 0.5  # 0.5 to 1.0
+    r, g, b = [int(x * 255) for x in colorsys.hsv_to_rgb(h, s, v)]
+    return f"#{r:02x}{g:02x}{b:02x}"
+
+# Extend the color palette
+EXTENDED_COLOR_PALETTE = BASE_COLORS + [generate_vibrant_color() for _ in range(100)]
+
+def get_subject_class_color(subject_class):
+    # Use a hash function to generate a unique number for each subject class
+    hash_value = hash(subject_class)
+    
+    # Use the hash to select a color from the extended palette
+    color_index = hash_value % len(EXTENDED_COLOR_PALETTE)
+    color = EXTENDED_COLOR_PALETTE[color_index]
+    
+    return color
+
+router = APIRouter()
+
+@router.get("/get_teacher_timetable_events")
+async def get_teacher_timetable_events(
+    unique_id: str,
+    worker_db_name: str
+):
+    logging.info(f"Getting timetable events for teacher {unique_id} from database {worker_db_name}")
+    neo_driver = driver.get_driver(db_name=worker_db_name)
+    if neo_driver is None:
+        return {"status": "error", "message": "Failed to connect to the database"}
+    
+    try:
+        with neo_driver.session(database=worker_db_name) as neo_session:
+            query = """
+            MATCH (t:Teacher {unique_id: $unique_id})-[:TEACHER_HAS_TIMETABLE]->(tt:TeacherTimetable)
+            -[:TIMETABLE_HAS_CLASS]->(sc:SubjectClass)-[:CLASS_HAS_LESSON]->(tl:TimetableLesson)
+            RETURN tl.unique_id as id, 
+                   tl.period_code as period_code,
+                   COALESCE(sc.subject_class_code, 'Untitled Class') as subject_class, 
+                   tl.date as date, 
+                   tl.start_time as start_time, 
+                   tl.end_time as end_time,
+                   tl.path as path
+            """
+            result = neo_session.run(query, unique_id=unique_id)
+            
+            events = []
+            for record in result:
+                start = f"{record['date']}T{record['start_time']}"
+                end = f"{record['date']}T{record['end_time']}"
+                title = f"{record['subject_class']}"
+                events.append({
+                    "id": record["id"],
+                    "title": title,
+                    "start": start,
+                    "end": end,
+                    "groupId": f"subject-class-{record['subject_class']}",
+                    "extendedProps": {
+                        "subjectClass": record['subject_class'],
+                        "color": get_subject_class_color(record['subject_class']),
+                        "periodCode": record['period_code'],
+                        "path": record['path']
+                    }
+                })
+            logging.info(f"Found {len(events)} events for teacher {unique_id}")
+            return {"status": "success", "events": events}
+    except Exception as e:
+        logging.error(f"Error fetching events: {str(e)}")
+        raise HTTPException(status_code=500, detail="Internal server error")
+    finally:
+        driver.close_driver(neo_driver)

routers/database/tools/get_nodes.py

@@ -0,0 +1,563 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'api_routers_database_tools_get_nodes'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+import modules.database.tools.neo4j_driver_tools as driver
+import modules.database.tools.neo4j_session_tools as session
+from modules.database.schemas.nodes.calendars import CalendarNode
+from modules.database.schemas.nodes.schools.timetable import SchoolTimetableNode, AcademicYearNode, AcademicTermNode, AcademicWeekNode, AcademicDayNode, AcademicPeriodNode, RegistrationPeriodNode
+from modules.database.schemas.nodes.users import UserNode
+from modules.database.schemas.nodes.workers.workers import TeacherNode, StudentNode, DeveloperNode, SchoolAdminNode
+from modules.database.schemas.nodes.schools.schools import SchoolNode, DepartmentNode, SubjectClassNode, RoomNode
+from modules.database.schemas.nodes.workers.timetable import TeacherTimetableNode, TimetableLessonNode, PlannedLessonNode, UserTeacherTimetableNode
+from fastapi import APIRouter, HTTPException, Query
+
+router = APIRouter()
+
+@router.get("/get-node")
+async def get_node(unique_id: str = Query(...), db_name: str = Query(...)):
+    logging.info(f"Getting node for {unique_id} from database {db_name}")
+    neo_driver = driver.get_driver(db_name=db_name)
+    if neo_driver is None:
+        return {"status": "error", "message": "Failed to connect to the database"}
+    try:
+        with neo_driver.session(database=db_name) as neo_session:
+            query = """
+            MATCH (n {unique_id: $unique_id})
+            RETURN n
+            """
+            result = neo_session.run(query, unique_id=unique_id)
+            record = result.single()
+            
+            if record:
+                node = record['n']
+                node_labels = list(node.labels)
+                node_data = dict(node)
+                
+                try:
+                    # Convert node based on its type
+                    node_type = node_labels[0] if node_labels else "Unknown"
+                    if node_type in globals():
+                        node_class = globals()[f"{node_type}Node"]
+                        node_object = node_class(**node_data)
+                        node_dict = node_object.to_dict()
+                    else:
+                        node_dict = node_data
+                    
+                    return {
+                        "status": "success",
+                        "node": {
+                            "node_type": node_type,
+                            "node_data": node_dict
+                        }
+                    }
+                except Exception as e:
+                    logging.error(f"Error converting node to dict: {str(e)}")
+                    return {
+                        "status": "error",
+                        "message": "Error processing node data",
+                        "details": str(e)
+                    }
+            else:
+                return {"status": "not_found", "message": "Node not found"}
+    except Exception as e:
+        logging.error(f"Error retrieving node: {str(e)}")
+        return {"status": "error", "message": "Internal server error"}
+    finally:
+        driver.close_driver(neo_driver)
+
+@router.get("/get-user-node")
+async def get_user_node(user_id: str = Query(...)):
+    db_name = f"cc.users.{user_id}"
+    logging.info(f"Getting user node for user {user_id} from database {db_name}")
+    neo_driver = driver.get_driver(db_name=db_name)
+    if neo_driver is None:
+        return {"status": "error", "message": "Failed to connect to the database"}
+    
+    try:
+        with neo_driver.session(database=db_name) as neo_session:
+            nodes = session.find_nodes_by_label_and_properties(neo_session, "User", {"user_id": user_id})
+            if nodes:
+                user_node = nodes[0]
+                data = UserNode(**user_node)
+                user_node_data = data.to_dict()
+                return {"status": "success", "user_node": user_node_data, "user_node_raw": nodes}
+            else:
+                return {"status": "not_found", "message": "User node not found"}
+    except Exception as e:
+        logging.error(f"Error retrieving user node: {str(e)}")
+        return {"status": "error", "message": "Internal server error"}
+    finally:
+        driver.close_driver(neo_driver)
+
+@router.get("/get-connected-nodes")
+async def get_connected_nodes(unique_id: str = Query(...), db_name: str = Query(...)):
+    logging.info(f"Getting connected nodes for {unique_id} from database {db_name}")
+    neo_driver = driver.get_driver(db_name=db_name)
+    if neo_driver is None:
+        return {"status": "error", "message": "Failed to connect to the database"}
+    
+    try:
+        with neo_driver.session(database=db_name) as neo_session:
+            query = """
+            MATCH (n {unique_id: $unique_id})
+            OPTIONAL MATCH (n)-[]-(connected)
+            RETURN n, collect(connected) as connected_nodes
+            """
+            result = neo_session.run(query, unique_id=unique_id)
+            record = result.single()
+            if record:
+                main_node = record['n']
+                connected_nodes = record['connected_nodes']
+                
+                main_node_labels = list(main_node.labels)
+                main_node_type = main_node_labels[0] if main_node_labels else "Unknown"
+                main_node_data = dict(main_node)
+                
+                try:
+                    main_node_class = globals()[f"{main_node_type}Node"]
+                    main_node_object = main_node_class(**main_node_data)
+                    main_node_dict = main_node_object.to_dict()
+                except Exception as e:
+                    logging.error(f"Error converting main node to dict: {str(e)}")
+                    main_node_dict = main_node_data
+                
+                connected_nodes_list = []
+                
+                for node in connected_nodes:
+                    node_labels = list(node.labels)
+                    node_type = node_labels[0] if node_labels else "Unknown"
+                    node_data = dict(node)
+                    try:
+                        node_class = globals()[f"{node_type}Node"]
+                        node_object = node_class(**node_data)
+                        connected_node_dict = node_object.to_dict()
+                    except Exception as e:
+                        logging.error(f"Error converting connected node to dict: {str(e)}")
+                        connected_node_dict = node_data
+                    
+                    connected_node_info = {
+                        "node_type": node_type,
+                        "node_data": connected_node_dict
+                    }
+                    connected_nodes_list.append(connected_node_info)
+                
+                logging.debug(f"connected_nodes_list: {connected_nodes_list}")
+                
+                return {
+                    "status": "success",
+                    "main_node": {
+                        "node_type": main_node_type,
+                        "node_data": main_node_dict
+                    },
+                    "connected_nodes": connected_nodes_list
+                }
+            else:
+                return {"status": "not_found", "message": "Node not found"}
+    except Exception as e:
+        logging.error(f"Error retrieving connected nodes: {str(e)}")
+        raise HTTPException(status_code=500, detail="Internal server error")
+    finally:
+        driver.close_driver(neo_driver)
+
+@router.get("/get-user-connected-nodes")
+async def get_user_connected_nodes(unique_id: str = Query(...)):
+    logging.info(f"Getting user adjacent nodes for node {unique_id}")
+    db_name = os.getenv("NEO4J_DB_NAME", "cc.institutes.kevlarai") # TODO: This function needs to be able to take a db_name as a parameter
+    neo_driver = driver.get_driver(db_name=db_name)
+    if neo_driver is None:
+        raise HTTPException(status_code=500, detail="Failed to connect to the database")
+    try:
+        with neo_driver.session(database=db_name) as neo_session:
+            user_node_and_connected_nodes = session.get_node_by_unique_id_and_adjacent_nodes(neo_session, unique_id)
+            user_node = user_node_and_connected_nodes['node']
+            connected_nodes = user_node_and_connected_nodes['connected_nodes']
+            try:
+                data = UserNode(**user_node)
+                user_node_dict = data.to_dict()
+            except Exception as e:
+                logging.error(f"Error converting user node to dict: {str(e)}")
+            connected_nodes_list = []
+            for connected_node in connected_nodes:
+                node_data = connected_node['node']
+                node_labels = list(node_data.labels)
+                logging.debug(f"node_labels: {node_labels}")
+                for label in node_labels:
+                    logging.debug(f"label: {label}")
+                    try:
+                        if 'Developer' == label:
+                            logging.debug(f"Developer node found")
+                            node_object = DeveloperNode(**node_data)
+                        elif 'SchoolAdmin' == label:
+                            logging.debug(f"SchoolAdmin node found")
+                            node_object = SchoolAdminNode(**node_data)
+                        elif 'Teacher' == label:
+                            logging.debug(f"Teacher node found")
+                            node_object = TeacherNode(**node_data)
+                        elif 'Student' == label:
+                            logging.debug(f"Student node found")
+                            node_object = StudentNode(**node_data)
+                        elif 'Calendar' == label:
+                            logging.debug(f"Calendar node found")
+                            node_object = CalendarNode(**node_data)
+                        elif 'TeacherTimetable' == label:
+                            logging.debug(f"TeacherTimetable node found")
+                            node_object = TeacherTimetableNode(**node_data)
+                        elif 'UserTeacherTimetable' == label:
+                            logging.debug(f"UserTeacherTimetable node found")
+                            node_object = UserTeacherTimetableNode(**node_data)
+                        elif 'School' == label:
+                            logging.debug(f"School node found")
+                            node_object = SchoolNode(**node_data)
+                        elif 'Department' == label:
+                            logging.debug(f"Department node found")
+                            node_object = DepartmentNode(**node_data)
+                        elif 'Student' == label:
+                            logging.debug(f"Student node found")
+                            node_object = StudentNode(**node_data)
+                        elif 'Class' == label:
+                            logging.debug(f"Class node found")
+                            node_object = SubjectClassNode(**node_data)
+                        elif 'Room' == label:
+                            logging.debug(f"Room node found")
+                            node_object = RoomNode(**node_data)
+                        else:
+                            logging.error(f"Unknown node label: {node_labels}")
+                            continue
+                        connected_node_dict = node_object.to_dict()
+                        logging.debug(f"connected_node_dict: {connected_node_dict}")
+                        connected_node_info = {
+                            "node_type": label,
+                            "node_data": connected_node_dict
+                        }
+                        connected_nodes_list.append(connected_node_info)
+                    except Exception as e:
+                        logging.error(f"Error converting node to dict: {str(e)}")
+            return {"status": "success", "user_node": user_node_dict, "user_connected_nodes": connected_nodes_list}
+    except Exception as e:
+        logging.error(f"Error retrieving adjacent nodes: {str(e)}")
+        raise HTTPException(status_code=500, detail="Internal server error")
+    finally:
+        driver.close_driver(neo_driver)
+
+@router.get("/get-worker-connected-nodes")
+async def get_worker_connected_nodes(unique_id: str = Query(...)):
+    logging.info(f"Getting worker adjacent nodes for node {unique_id}")
+    db_name = os.getenv("NEO4J_DB_NAME", "cc.institutes.kevlarai") # TODO: This function needs to be able to take a db_name as a parameter
+    neo_driver = driver.get_driver(db_name=db_name)
+    if neo_driver is None:
+        raise HTTPException(status_code=500, detail="Failed to connect to the database")
+    try:
+        with neo_driver.session(database=db_name) as neo_session:
+            node_and_connected_nodes = session.get_node_by_unique_id_and_adjacent_nodes(neo_session, unique_id)
+            worker_node = node_and_connected_nodes['node']
+            connected_nodes = node_and_connected_nodes['connected_nodes']
+            try:
+                data = TeacherNode(**worker_node)
+                worker_node_dict = data.to_dict()
+            except Exception as e:
+                logging.error(f"Error converting user node to dict: {str(e)}")
+            connected_nodes_list = []
+            for connected_node in connected_nodes:
+                node_data = connected_node['node']
+                node_labels = list(node_data.labels)
+                logging.debug(f"node_labels: {node_labels}")
+                for label in node_labels:
+                    logging.debug(f"label: {label}")
+                    try:
+                        if 'Calendar' == label:
+                            logging.debug(f"Calendar node found")
+                            node_object = CalendarNode(**node_data)
+                        elif 'TeacherTimetable' == label:
+                            logging.debug(f"TeacherTimetable node found")
+                            node_object = TeacherTimetableNode(**node_data)
+                        elif 'UserTeacherTimetable' == label:
+                            logging.debug(f"UserTeacherTimetable node found")
+                            node_object = UserTeacherTimetableNode(**node_data)
+                        elif 'School' == label:
+                            logging.debug(f"School node found")
+                            node_object = SchoolNode(**node_data)
+                        elif 'Department' == label:
+                            logging.debug(f"Department node found")
+                            node_object = DepartmentNode(**node_data)
+                        elif 'Student' == label:
+                            logging.debug(f"Student node found")
+                            node_object = StudentNode(**node_data)
+                        elif 'Class' == label:
+                            logging.debug(f"Class node found")
+                            node_object = SubjectClassNode(**node_data)
+                        elif 'Room' == label:
+                            logging.debug(f"Room node found")
+                            node_object = RoomNode(**node_data)
+                        else:
+                            logging.error(f"Unknown node label: {node_labels}")
+                            continue
+                        connected_node_dict = node_object.to_dict()
+                        logging.debug(f"connected_node_dict: {connected_node_dict}")
+                        connected_node_info = {
+                            "node_type": label,
+                            "node_data": connected_node_dict
+                        }
+                        connected_nodes_list.append(connected_node_info)
+                    except Exception as e:
+                        logging.error(f"Error converting node to dict: {str(e)}")
+            return {"status": "success", "user_node": worker_node_dict, "worker_connected_nodes": connected_nodes_list}
+    except Exception as e:
+        logging.error(f"Error retrieving worker adjacent nodes: {str(e)}")
+        raise HTTPException(status_code=500, detail="Internal server error")
+    finally:
+        driver.close_driver(neo_driver)
+
+@router.get("/get-calendar-connected-nodes")
+async def get_calendar_connected_nodes(unique_id: str = Query(...)):
+    db_name = os.getenv("NEO4J_DB_NAME", "cc.institutes.kevlarai")
+    logging.info(f"Getting connected nodes for calendar {unique_id} from database {db_name}")
+    neo_driver = driver.get_driver(db_name=db_name)
+    if neo_driver is None:
+        return {"status": "error", "message": "Failed to connect to the database"}
+    
+    try:
+        with neo_driver.session(database=db_name) as neo_session:
+            query = """
+            MATCH (n)
+            WHERE n.unique_id = $unique_id AND (n:Calendar OR n:CalendarYear OR n:CalendarMonth OR n:CalendarWeek OR n:CalendarDay OR n:CalendarTimeChunk)
+            OPTIONAL MATCH (n)-[]-(connected)
+            RETURN n, collect(connected) as connected_nodes
+            """
+            result = neo_session.run(query, unique_id=unique_id)
+            record = result.single()
+            if record:
+                calendar_node = record['n']
+                connected_nodes = record['connected_nodes']
+                
+                node_type = list(calendar_node.labels)[0]
+                calendar_dict = globals()[f"{node_type}Node"](**calendar_node).to_dict()
+                connected_nodes_list = []
+                
+                for node in connected_nodes:
+                    node_labels = list(node.labels)
+                    node_data = dict(node)
+                    try:
+                        node_class = globals()[f"{node_labels[0]}Node"]
+                        node_object = node_class(**node_data)
+                        connected_node_dict = node_object.to_dict()
+                        connected_node_info = {
+                            "node_type": node_labels[0],
+                            "node_data": connected_node_dict
+                        }
+                        connected_nodes_list.append(connected_node_info)
+                    except Exception as e:
+                        logging.error(f"Error converting node to dict: {str(e)}")
+                
+                return {"status": "success", "calendar_node": calendar_dict, "connected_nodes": connected_nodes_list}
+            else:
+                return {"status": "not_found", "message": "Calendar node not found"}
+    except Exception as e:
+        logging.error(f"Error retrieving connected nodes: {str(e)}")
+        raise HTTPException(status_code=500, detail="Internal server error")
+    finally:
+        driver.close_driver(neo_driver)
+    
+@router.get("/get-teacher-timetable-connected-nodes")
+async def get_teacher_timetable_connected_nodes(unique_id: str = Query(...)):
+    db_name = os.getenv("NEO4J_DB_NAME", "cc.institutes.kevlarai")
+    logging.info(f"Getting connected nodes for teacher timetable {unique_id} from database {db_name}")
+    neo_driver = driver.get_driver(db_name=db_name)
+    if neo_driver is None:
+        return {"status": "error", "message": "Failed to connect to the database"}
+    
+    try:
+        with neo_driver.session(database=db_name) as neo_session:
+            query = """
+            MATCH (n:TeacherTimetable {unique_id: $unique_id})
+            OPTIONAL MATCH (n)-[]-(connected)
+            RETURN n, collect(connected) as connected_nodes
+            """
+            result = neo_session.run(query, unique_id=unique_id)
+            record = result.single()
+            if record:
+                teacher_timetable_node = record['n']
+                connected_nodes = record['connected_nodes']
+                
+                teacher_timetable_dict = TeacherTimetableNode(**teacher_timetable_node).to_dict()
+                connected_nodes_list = []
+                
+                for node in connected_nodes:
+                    node_labels = list(node.labels)
+                    node_data = dict(node)
+                    try:
+                        if 'TimetableLesson' in node_labels:
+                            node_object = TimetableLessonNode(**node_data)
+                        elif 'PlannedLesson' in node_labels:
+                            node_object = PlannedLessonNode(**node_data)
+                        else:
+                            logging.error(f"Unknown node label: {node_labels}")
+                            continue
+                        connected_node_dict = node_object.to_dict()
+                        connected_node_info = {
+                            "node_type": node_labels[0],
+                            "node_data": connected_node_dict
+                        }
+                        connected_nodes_list.append(connected_node_info)
+                    except Exception as e:
+                        logging.error(f"Error converting node to dict: {str(e)}")
+                
+                return {"status": "success", "teacher_timetable_node": teacher_timetable_dict, "connected_nodes": connected_nodes_list}
+            else:
+                return {"status": "not_found", "message": "Teacher timetable node not found"}
+    except Exception as e:
+        logging.error(f"Error retrieving connected nodes: {str(e)}")
+        raise HTTPException(status_code=500, detail="Internal server error")
+    finally:
+        driver.close_driver(neo_driver)
+
+@router.get("/get-school-timetable-connected-nodes")
+async def get_school_timetable_connected_nodes(unique_id: str = Query(...)):
+    db_name = os.getenv("NEO4J_DB_NAME", "cc.institutes.kevlarai")
+    logging.info(f"Getting connected nodes for school timetable {unique_id} from database {db_name}")
+    neo_driver = driver.get_driver(db_name=db_name)
+    if neo_driver is None:
+        return {"status": "error", "message": "Failed to connect to the database"}
+    
+    try:
+        with neo_driver.session(database=db_name) as neo_session:
+            query = """
+            MATCH (n:SchoolTimetable {unique_id: $unique_id})
+            OPTIONAL MATCH (n)-[]-(connected)
+            RETURN n, collect(connected) as connected_nodes
+            """
+            result = neo_session.run(query, unique_id=unique_id)
+            record = result.single()
+            if record:
+                school_timetable_node = record['n']
+                connected_nodes = record['connected_nodes']
+                
+                school_timetable_dict = SchoolTimetableNode(**school_timetable_node).to_dict()
+                connected_nodes_list = []
+                
+                for node in connected_nodes:
+                    node_labels = list(node.labels)
+                    node_data = dict(node)
+                    try:
+                        if 'AcademicYear' in node_labels:
+                            node_object = AcademicYearNode(**node_data)
+                        elif 'AcademicTerm' in node_labels:
+                            node_object = AcademicTermNode(**node_data)
+                        elif 'AcademicWeek' in node_labels:
+                            node_object = AcademicWeekNode(**node_data)
+                        elif 'AcademicDay' in node_labels:
+                            node_object = AcademicDayNode(**node_data)
+                        elif 'AcademicPeriod' in node_labels:
+                            node_object = AcademicPeriodNode(**node_data)
+                        elif 'RegistrationPeriod' in node_labels:
+                            node_object = RegistrationPeriodNode(**node_data)
+                        else:
+                            logging.error(f"Unknown node label: {node_labels}")
+                            continue
+                        connected_node_dict = node_object.to_dict()
+                        connected_node_info = {
+                            "node_type": node_labels[0],
+                            "node_data": connected_node_dict
+                        }
+                        connected_nodes_list.append(connected_node_info)
+                    except Exception as e:
+                        logging.error(f"Error converting node to dict: {str(e)}")
+                
+                return {"status": "success", "school_timetable_node": school_timetable_dict, "connected_nodes": connected_nodes_list}
+            else:
+                return {"status": "not_found", "message": "School timetable node not found"}
+    except Exception as e:
+        logging.error(f"Error retrieving connected nodes: {str(e)}")
+        raise HTTPException(status_code=500, detail="Internal server error")
+    finally:
+        driver.close_driver(neo_driver)
+
+@router.get("/get-curriculum-connected-nodes")
+async def get_curriculum_connected_nodes(unique_id: str = Query(...)):
+    db_name = os.getenv("NEO4J_DB_NAME", "cc.institutes.kevlarai")
+    logging.info(f"Getting connected nodes for curriculum {unique_id} from database {db_name}")
+    neo_driver = driver.get_driver(db_name=db_name)
+    if neo_driver is None:
+        return {"status": "error", "message": "Failed to connect to the database"}
+    
+    try:
+        with neo_driver.session(database=db_name) as neo_session:
+            query = """
+            MATCH (n)
+            WHERE n.unique_id = $unique_id AND (n:PastoralStructure OR n:YearGroup OR n:CurriculumStructure OR n:KeyStage OR n:KeyStageSyllabus OR n:YearGroupSyllabus OR n:Subject OR n:Topic OR n:TopicLesson OR n:LearningStatement OR n:ScienceLab)
+            OPTIONAL MATCH (n)-[]-(connected)
+            RETURN n, collect(connected) as connected_nodes
+            """
+            result = neo_session.run(query, unique_id=unique_id)
+            record = result.single()
+            if record:
+                curriculum_node = record['n']
+                connected_nodes = record['connected_nodes']
+                
+                node_type = list(curriculum_node.labels)[0]
+                curriculum_dict = globals()[f"{node_type}Node"](**curriculum_node).to_dict()
+                connected_nodes_list = []
+                
+                for node in connected_nodes:
+                    node_labels = list(node.labels)
+                    node_data = dict(node)
+                    try:
+                        node_class = globals()[f"{node_labels[0]}Node"]
+                        node_object = node_class(**node_data)
+                        connected_node_dict = node_object.to_dict()
+                        connected_node_info = {
+                            "node_type": node_labels[0],
+                            "node_data": connected_node_dict
+                        }
+                        connected_nodes_list.append(connected_node_info)
+                    except Exception as e:
+                        logging.error(f"Error converting node to dict: {str(e)}")
+                
+                return {"status": "success", "curriculum_node": curriculum_dict, "connected_nodes": connected_nodes_list}
+            else:
+                return {"status": "not_found", "message": "Curriculum node not found"}
+    except Exception as e:
+        logging.error(f"Error retrieving connected nodes: {str(e)}")
+        raise HTTPException(status_code=500, detail="Internal server error")
+    finally:
+        driver.close_driver(neo_driver)
+
+@router.get("/get-school-node")
+async def get_school_node(school_uuid: str = Query(...)):
+    logging.info(f"Getting school node for school {school_uuid}...")
+    db_name = f"cc.institutes.{school_uuid}"
+    neo_driver = driver.get_driver(db_name=db_name)
+    if neo_driver is None:
+        return {"status": "error", "message": "Failed to connect to the database"}
+    
+    try:
+        with neo_driver.session(database=db_name) as neo_session:
+            nodes = session.find_nodes_by_label_and_properties(neo_session, "School", {"school_uuid": school_uuid})
+            if nodes:
+                school_node = nodes[0]
+                data = SchoolNode(
+                    unique_id=school_node["unique_id"],
+                    school_uuid=school_node["school_uuid"],
+                    school_name=school_node["school_name"],
+                    school_website=school_node["school_website"],
+                    path=school_node["path"]
+                )
+                school_node_data = data.to_dict()
+                return {"status": "success", "school_node": school_node_data, "school_node_raw": nodes}
+            else:
+                return {"status": "not_found", "message": "School node not found"}
+    except Exception as e:
+        logging.error(f"Error retrieving school node: {str(e)}")
+        return {"status": "error", "message": "Internal server error"}
+    finally:
+        driver.close_driver(neo_driver)

routers/database/tools/get_nodes_and_edges.py

@@ -0,0 +1,174 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'api_routers_database_tools_get_nodes'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+import modules.database.tools.neo4j_driver_tools as driver
+import modules.database.tools.neo4j_session_tools as session
+from modules.database.schemas.nodes.calendars import CalendarNode, CalendarYearNode, CalendarMonthNode, CalendarWeekNode, CalendarDayNode, CalendarTimeChunkNode
+from modules.database.schemas.nodes.users import UserNode
+from modules.database.schemas.nodes.workers.workers import TeacherNode, StudentNode, DeveloperNode, SchoolAdminNode
+from modules.database.schemas.nodes.structures.schools import PastoralStructureNode, CurriculumStructureNode
+from modules.database.schemas.nodes.schools.pastoral import YearGroupNode, YearGroupSyllabusNode
+from modules.database.schemas.nodes.schools.curriculum import SubjectNode, TopicNode, TopicLessonNode, LearningStatementNode, ScienceLabNode
+from modules.database.schemas.nodes.schools.timetable import SchoolTimetableNode, AcademicYearNode, AcademicTermNode, AcademicWeekNode, AcademicDayNode, OffTimetableDayNode, StaffDayNode, AcademicPeriodNode, RegistrationPeriodNode, OffTimetablePeriodNode, AcademicTermBreakNode, BreakPeriodNode, HolidayDayNode, HolidayWeekNode
+from modules.database.schemas.nodes.workers.timetable import TeacherTimetableNode, TimetableLessonNode, PlannedLessonNode, UserTeacherTimetableNode, StudentTimetableNode, SchoolAdminTimetableNode, DeveloperTimetableNode, SuperAdminTimetableNode
+from modules.database.schemas.nodes.schools.schools import SchoolNode, DepartmentNode, SubjectClassNode, RoomNode
+from fastapi import APIRouter, HTTPException, Query
+
+router = APIRouter()
+
+@router.get("/get-all-nodes-and-edges")
+async def get_all_nodes_and_edges():
+    db_name = os.getenv("NEO4J_DB_NAME", "cc.institutes.kevlarai")
+    logging.info(f"Getting all nodes and edges from database {db_name}")
+    neo_driver = driver.get_driver(db_name=db_name)
+    if neo_driver is None:
+        return {"status": "error", "message": "Failed to connect to the database"}
+    
+    try:
+        with neo_driver.session(database=db_name) as neo_session:
+            query = """
+            MATCH (n)-[r]->(m)
+            RETURN n, r, m
+            """
+            result = neo_session.run(query)
+            nodes = {}
+            relationships = []
+            
+            for record in result:
+                source = record['n']
+                target = record['m']
+                relationship = record['r']
+                
+                for node in [source, target]:
+                    if node.id not in nodes:
+                        node_labels = list(node.labels)
+                        node_type = node_labels[0] if node_labels else "Unknown"
+                        node_data = dict(node)
+                        try:
+                            node_class = globals()[f"{node_type}Node"]
+                            node_object = node_class(**node_data)
+                            node_dict = node_object.to_dict()
+                        except Exception as e:
+                            logging.error(f"Error converting node to dict: {str(e)}")
+                            node_dict = node_data
+                        
+                        nodes[node.id] = {
+                            "node_type": node_type,
+                            "node_data": node_dict
+                        }
+                
+                relationship_info = {
+                    "start_node": source.id,
+                    "end_node": target.id,
+                    "relationship_type": relationship.type,
+                    "relationship_properties": dict(relationship)
+                }
+                relationships.append(relationship_info)
+            
+            return {
+                "status": "success",
+                "nodes": list(nodes.values()),
+                "relationships": relationships
+            }
+    except Exception as e:
+        logging.error(f"Error retrieving all nodes and edges: {str(e)}")
+        return {"status": "error", "message": "Internal server error"}
+    finally:
+        driver.close_driver(neo_driver)
+
+
+@router.get("/get-connected-nodes-and-edges")
+async def get_connected_nodes_and_edges(unique_id: str = Query(...), db_name: str = Query(...)):
+    logging.info(f"Getting connected nodes and edges for {unique_id} from database {db_name}")
+    neo_driver = driver.get_driver(db_name=db_name)
+    if neo_driver is None:
+        return {"status": "error", "message": "Failed to connect to the database"}
+    
+    try:
+        with neo_driver.session(database=db_name) as neo_session:
+            query = """
+            MATCH (n {unique_id: $unique_id})
+            OPTIONAL MATCH (n)-[r]-(connected)
+            RETURN n, collect(connected) as connected_nodes, collect(r) as relationships
+            """
+            result = neo_session.run(query, unique_id=unique_id)
+            record = result.single()
+            if record:
+                main_node = record['n']
+                connected_nodes = record['connected_nodes']
+                relationships = record['relationships']
+                
+                main_node_labels = list(main_node.labels)
+                main_node_type = main_node_labels[0] if main_node_labels else "Unknown"
+                main_node_data = dict(main_node)
+                
+                try:
+                    main_node_class = globals()[f"{main_node_type}Node"]
+                    main_node_object = main_node_class(**main_node_data)
+                    main_node_dict = main_node_object.to_dict()
+                except Exception as e:
+                    logging.error(f"Error converting main node to dict: {str(e)}")
+                    main_node_dict = main_node_data
+                
+                connected_nodes_list = []
+                relationship_list = []
+                
+                for node, relationship in zip(connected_nodes, relationships):
+                    node_labels = list(node.labels)
+                    node_type = node_labels[0] if node_labels else "Unknown"
+                    node_data = dict(node)
+                    try:
+                        node_class = globals()[f"{node_type}Node"]
+                        node_object = node_class(**node_data)
+                        connected_node_dict = node_object.to_dict()
+                    except Exception as e:
+                        logging.error(f"Error converting connected node to dict: {str(e)}")
+                        connected_node_dict = node_data
+                    
+                    connected_node_info = {
+                        "node_type": node_type,
+                        "node_data": connected_node_dict,
+                        "relationship_type": relationship.type,  # Get relationship type
+                        "relationship_properties": dict(relationship)  # Relationship properties, if any
+                    }
+                    connected_nodes_list.append(connected_node_info)
+
+                    relationship_info = {
+                        "start_node": dict(relationship.start_node),
+                        "end_node": dict(relationship.end_node),
+                        "relationship_type": relationship.type,
+                        "relationship_properties": dict(relationship)
+                    }
+                    relationship_list.append(relationship_info)
+                
+                logging.info(f"Main node: {main_node_dict}")
+                logging.info(f"Connected nodes: {connected_nodes_list}")
+                logging.info(f"Relationships: {relationship_list}")
+                
+                return {
+                    "status": "success",
+                    "main_node": {
+                        "node_type": main_node_type,
+                        "node_data": main_node_dict
+                    },
+                    "connected_nodes": connected_nodes_list,
+                    "relationships": relationship_list
+                }
+            else:
+                return {"status": "not_found", "message": "Node not found"}
+    except Exception as e:
+        logging.error(f"Error retrieving connected nodes: {str(e)}")
+        raise HTTPException(status_code=500, detail="Internal server error")
+    finally:
+        driver.close_driver(neo_driver)

routers/database/tools/get_school_curriculum_context.py

@@ -0,0 +1,3 @@
+import os
+from modules.logger_tool import initialise_logger
+logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)

routers/database/tools/tldraw_filesystem.py

@@ -0,0 +1,196 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'api_routers_database_tools_tldraw_filesystem'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+from fastapi import APIRouter, HTTPException, Query
+from typing import Dict
+import json
+
+from modules.database.tools.filesystem_tools import ClassroomCopilotFilesystem
+from modules.database.schemas.nodes.users import UserNode
+from modules.database.tools.neo4j_db_formatter import format_user_email_for_neo_db
+
+router = APIRouter()
+
+@router.post("/get_tldraw_user_node_file")
+async def read_tldraw_user_node_file(user_node: UserNode):
+    logging.debug(f"Reading tldraw file for user node: {user_node.user_email}")
+    
+    # Format the database name using the email
+    formatted_email = format_user_email_for_neo_db(user_node.user_email)
+    db_name = f"cc.users.{formatted_email}"
+    
+    fs = ClassroomCopilotFilesystem(db_name=db_name, init_run_type="user")
+    
+    logging.debug(f"Filesystem root path: {fs.root_path}")
+    
+    # Handle path based on environment
+    if os.getenv("DEV_MODE") == "true":
+        # In dev mode, use the full system path from the node
+        if not user_node.path:
+            raise HTTPException(status_code=400, detail="Node path not found")
+        logging.debug(f"Using DEV_MODE path: {user_node.path}")
+        base_path = os.path.normpath(user_node.path)
+    else:
+        # In prod mode, construct path using formatted email
+        logging.warning(f"Using db_name as base path not ready in prod: {db_name}")
+        base_path = formatted_email
+    
+    # Construct final path including tldraw file
+    logging.debug(f"Base path: {base_path}")
+    file_path = os.path.join(base_path, "tldraw_file.json")
+    logging.debug(f"File path: {file_path}")
+    file_location = os.path.normpath(os.path.join(fs.root_path, file_path))
+    logging.debug(f"File location: {file_location}")
+    
+    logging.debug(f"Attempting to read file at: {file_location}")
+    
+    if os.path.exists(file_location):
+        logging.debug(f"File exists: {file_location}")
+        try:
+            with open(file_location, "r") as file:
+                data = json.load(file)
+            return data
+        except json.JSONDecodeError as e:
+            logging.error(f"Failed to parse JSON from file: {e}")
+            raise HTTPException(status_code=500, detail="Invalid JSON in file")
+        except Exception as e:
+            logging.error(f"Error reading file: {e}")
+            raise HTTPException(status_code=500, detail="Error reading file")
+    else:
+        logging.debug(f"File does not exist: {file_location}")
+        raise HTTPException(status_code=404, detail="File not found")
+
+@router.post("/set_tldraw_user_node_file")
+async def set_tldraw_user_node_file(user_node: UserNode, data: Dict):
+    logging.debug(f"Setting tldraw file for user node: {user_node.user_email}")
+    
+    # Format the database name using the email
+    formatted_email = format_user_email_for_neo_db(user_node.user_email)
+    db_name = f"cc.users.{formatted_email}"
+    
+    fs = ClassroomCopilotFilesystem(db_name=db_name, init_run_type="user")
+    
+    # Handle path based on environment
+    if os.getenv("ENVIRONMENT") == "dev":
+        # In dev mode, use the full system path from the node
+        if not user_node.path:
+            raise HTTPException(status_code=400, detail="Node path not found")
+        base_path = os.path.normpath(user_node.path)
+    else:
+        # In prod mode, construct path using formatted email
+        base_path = formatted_email
+    
+    # Construct final path including tldraw file
+    file_path = os.path.join(base_path, "tldraw_file.json")
+    file_location = os.path.normpath(os.path.join(fs.root_path, file_path))
+    
+    logging.debug(f"Attempting to write file at: {file_location}")
+    
+    try:
+        # Ensure directory exists
+        os.makedirs(os.path.dirname(file_location), exist_ok=True)
+        
+        # Write the file
+        with open(file_location, "w") as file:
+            json.dump(data, file)
+        return {"status": "success"}
+    except Exception as e:
+        logging.error(f"Error writing file: {e}")
+        raise HTTPException(status_code=500, detail="Error writing file")
+
+@router.get("/get_tldraw_node_file")
+async def read_tldraw_node_file(path: str, db_name: str):
+    logging.debug(f"Reading tldraw file for path: {path}")
+    
+    fs = ClassroomCopilotFilesystem(db_name=db_name, init_run_type="user")
+    
+    logging.debug(f"Filesystem root path: {fs.root_path}")
+    
+    # Handle path based on environment
+    if os.getenv("DEV_MODE") == "true":
+        # In dev mode, use the full system path from the node
+        if not path:
+            raise HTTPException(status_code=400, detail="Path not provided")
+        logging.debug(f"Using DEV_MODEpath: {path}")
+        base_path = os.path.normpath(path)
+    else:
+        # In prod mode, construct path
+        logging.warning(f"Using db_name as base path not ready in prod: {db_name}")
+        base_path = db_name
+    
+    # Construct final path including tldraw file
+    logging.debug(f"Base path: {base_path}")
+    file_path = os.path.join(base_path, "tldraw_file.json")
+    logging.debug(f"File path: {file_path}")
+    file_location = os.path.normpath(os.path.join(fs.root_path, file_path))
+    logging.debug(f"File location: {file_location}")
+    
+    logging.debug(f"Attempting to read file at: {file_location}")
+    
+    if os.path.exists(file_location):
+        logging.debug(f"File exists: {file_location}")
+        try:
+            with open(file_location, "r") as file:
+                data = json.load(file)
+            return data
+        except json.JSONDecodeError as e:
+            logging.error(f"Failed to parse JSON from file: {e}")
+            raise HTTPException(status_code=500, detail="Invalid JSON in file")
+        except Exception as e:
+            logging.error(f"Error reading file: {e}")
+            raise HTTPException(status_code=500, detail="Error reading file")
+    else:
+        logging.debug(f"File does not exist: {file_location}")
+        raise HTTPException(status_code=404, detail="File not found")
+
+@router.post("/set_tldraw_node_file")
+async def set_tldraw_node_file(path: str, db_name: str, data: Dict):
+    logging.debug(f"Setting tldraw file for path: {path}")
+    
+    fs = ClassroomCopilotFilesystem(db_name=db_name, init_run_type="user")
+    
+    logging.debug(f"Filesystem root path: {fs.root_path}")
+    
+    # Handle path based on environment
+    if os.getenv("DEV_MODE") == "true":
+        # In dev mode, use the full system path from the node
+        if not path:
+            raise HTTPException(status_code=400, detail="Path not provided")
+        logging.debug(f"Using DEV_MODEpath: {path}")
+        base_path = os.path.normpath(path)
+    else:
+        # In prod mode, construct path
+        logging.warning(f"Using db_name as base path not ready in prod: {db_name}")
+        base_path = db_name
+    
+    # Construct final path including tldraw file
+    logging.debug(f"Base path: {base_path}")
+    file_path = os.path.join(base_path, "tldraw_file.json")
+    logging.debug(f"File path: {file_path}")
+    file_location = os.path.normpath(os.path.join(fs.root_path, file_path))
+    logging.debug(f"File location: {file_location}")
+    
+    logging.debug(f"Attempting to set file at: {file_location}")
+    
+    try:
+        # Ensure directory exists
+        os.makedirs(os.path.dirname(file_location), exist_ok=True)
+        
+        # Write the file
+        with open(file_location, "w") as file:
+            json.dump(data, file)
+        return {"status": "success"}
+    except Exception as e:
+        logging.error(f"Error writing file: {e}")
+        raise HTTPException(status_code=500, detail="Error writing file")

routers/database/tools/worker_structure_router.py

@@ -0,0 +1,190 @@
+import os
+from fastapi import APIRouter, HTTPException
+from typing import List, Dict, Any, Optional
+from datetime import datetime, timedelta
+from modules.logger_tool import initialise_logger
+from modules.database.tools import neo4j_driver_tools as driver_tools
+
+logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
+router = APIRouter()
+
+@router.get("/get-worker-structure")
+async def get_worker_structure(db_name: str) -> Dict[str, Any]:
+    """
+    Get the complete worker structure including timetables, classes, lessons, journals, and planners.
+    """
+    try:
+        # Get all worker-related nodes in a single query
+        query = """
+        // Match all worker-related nodes
+        MATCH (t:Teacher)
+        OPTIONAL MATCH (t)-[:TEACHER_HAS_TIMETABLE]->(tt:UserTeacherTimetable)
+        OPTIONAL MATCH (t)-[:TEACHER_HAS_CLASS]->(c:Class)
+        OPTIONAL MATCH (t)-[:TEACHER_HAS_LESSON]->(l:TimetableLesson)
+        OPTIONAL MATCH (t)-[:TEACHER_HAS_JOURNAL]->(j:Journal)
+        OPTIONAL MATCH (t)-[:TEACHER_HAS_PLANNER]->(p:Planner)
+        WITH t, tt, c, l, j, p
+        ORDER BY tt.start_date, c.created, l.created, j.created, p.created
+
+        // Collect all nodes
+        RETURN {
+            timetables: collect(DISTINCT {
+                id: tt.unique_id,
+                path: tt.path,
+                title: tt.title,
+                type: tt.__primarylabel__,
+                startTime: toString(tt.start_date),
+                endTime: toString(tt.end_date)
+            }),
+            classes: collect(DISTINCT {
+                id: c.unique_id,
+                path: c.path,
+                title: c.title,
+                type: c.__primarylabel__
+            }),
+            lessons: collect(DISTINCT {
+                id: l.unique_id,
+                path: l.path,
+                title: l.title,
+                type: l.__primarylabel__
+            }),
+            journals: collect(DISTINCT {
+                id: j.unique_id,
+                path: j.path,
+                title: j.title,
+                type: j.__primarylabel__
+            }),
+            planners: collect(DISTINCT {
+                id: p.unique_id,
+                path: p.path,
+                title: p.title,
+                type: p.__primarylabel__
+            })
+        } as structure
+        """
+
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query)
+            record = result.single()
+            if not record:
+                raise HTTPException(status_code=404, detail="Worker structure not found")
+            
+            structure = record["structure"]
+            
+            return {
+                "status": "success",
+                "data": {
+                    "timetables": {
+                        "default": structure["timetables"]
+                    },
+                    "classes": {
+                        "default": structure["classes"]
+                    },
+                    "lessons": {
+                        "default": structure["lessons"]
+                    },
+                    "journals": {
+                        "default": structure["journals"]
+                    },
+                    "planners": {
+                        "default": structure["planners"]
+                    }
+                }
+            }
+
+    except Exception as e:
+        logger.error(f"Error getting worker structure: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/get-timetables")
+async def get_timetables(db_name: str, start_date: str, end_date: str) -> Dict[str, Any]:
+    """
+    Get all timetables in a date range.
+    """
+    try:
+        query = """
+        MATCH (tt:UserTeacherTimetable)
+        WHERE date(tt.start_date) >= date($start_date) AND date(tt.end_date) <= date($end_date)
+        RETURN {
+            id: tt.unique_id,
+            path: tt.path,
+            title: tt.title,
+            type: tt.__primarylabel__,
+            startTime: toString(tt.start_date),
+            endTime: toString(tt.end_date)
+        } as timetable
+        ORDER BY tt.start_date
+        """
+
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query, start_date=start_date, end_date=end_date)
+            timetables = [record["timetable"] for record in result]
+
+            return {
+                "status": "success",
+                "timetables": timetables
+            }
+
+    except Exception as e:
+        logger.error(f"Error getting timetables: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/get-journals")
+async def get_journals(db_name: str) -> Dict[str, Any]:
+    """
+    Get all journals.
+    """
+    try:
+        query = """
+        MATCH (j:Journal)
+        RETURN {
+            id: j.unique_id,
+            path: j.path,
+            title: j.title,
+            type: j.__primarylabel__
+        } as journal
+        ORDER BY j.created
+        """
+
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query)
+            journals = [record["journal"] for record in result]
+
+            return {
+                "status": "success",
+                "journals": journals
+            }
+
+    except Exception as e:
+        logger.error(f"Error getting journals: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.get("/get-planners")
+async def get_planners(db_name: str) -> Dict[str, Any]:
+    """
+    Get all planners.
+    """
+    try:
+        query = """
+        MATCH (p:Planner)
+        RETURN {
+            id: p.unique_id,
+            path: p.path,
+            title: p.title,
+            type: p.__primarylabel__
+        } as planner
+        ORDER BY p.created
+        """
+
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query)
+            planners = [record["planner"] for record in result]
+
+            return {
+                "status": "success",
+                "planners": planners
+            }
+
+    except Exception as e:
+        logger.error(f"Error getting planners: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))

routers/dev/document_conversion.py

@@ -0,0 +1,139 @@
+from fastapi import APIRouter, UploadFile, File, HTTPException
+from typing import List, Optional, Dict
+from pathlib import Path
+import shutil
+import tempfile
+from pydantic import BaseModel
+from modules.document_processor import DocumentProcessor
+import os
+
+class BatchConvertRequest(BaseModel):
+    directory: str
+    output_dir: Optional[str] = None
+
+router = APIRouter()
+doc_processor = DocumentProcessor()
+
+@router.post("/convert-to-pdf")
+async def convert_to_pdf(
+    files: List[UploadFile] = File(...),
+    output_format: str = "pdf"
+):
+    """
+    Convert uploaded documents to PDF format
+    """
+    results = []
+    with tempfile.TemporaryDirectory() as temp_dir:
+        for file in files:
+            # Save uploaded file to temp directory
+            temp_file = Path(temp_dir) / file.filename
+            with temp_file.open("wb") as buffer:
+                shutil.copyfileobj(file.file, buffer)
+            
+            try:
+                # Process the document
+                pdf_content = doc_processor.convert_to_pdf(temp_file)
+                results.append({
+                    "filename": file.filename,
+                    "converted_content": pdf_content,
+                    "status": "success"
+                })
+            except Exception as e:
+                results.append({
+                    "filename": file.filename,
+                    "error": str(e),
+                    "status": "error"
+                })
+    
+    return results
+
+@router.post("/batch-convert")
+async def batch_convert(
+    directory: str,
+    output_format: str = "pdf"
+):
+    """
+    Convert all documents in a directory to PDF format
+    """
+    try:
+        results = doc_processor.batch_convert_directory(directory)
+        return results
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.post("/batch-convert-recursive")
+async def batch_convert_recursive(request_data: BatchConvertRequest):
+    """
+    Convert all documents in a directory and its subdirectories to PDF
+    """
+    try:
+        directory_path = Path(request_data.directory)
+        if not directory_path.exists():
+            raise HTTPException(status_code=404, detail=f"Directory not found: {request_data.directory}")
+        
+        output_path = None
+        if request_data.output_dir:
+            output_path = Path(request_data.output_dir)
+            output_path.mkdir(parents=True, exist_ok=True)
+        
+        results = []
+        supported_extensions = doc_processor.supported_extensions.keys()
+        
+        # Debug: Print processing info
+        print(f"Processing directory: {directory_path}")
+        print(f"Output directory: {output_path}")
+        print(f"Supported extensions: {list(supported_extensions)}")
+        
+        # Count files before processing
+        all_files = []
+        for ext in supported_extensions:
+            all_files.extend(list(directory_path.rglob(f"*.{ext}")))
+        print(f"Found {len(all_files)} files to process")
+        
+        # Recursively find all documents
+        for file_path in all_files:
+            try:
+                print(f"Processing: {file_path}")
+                # Convert the document
+                pdf_content = doc_processor.convert_to_pdf(file_path)
+                
+                # Determine output path
+                if output_path:
+                    # Preserve directory structure in output_dir
+                    rel_path = file_path.relative_to(directory_path)
+                    out_path = output_path / rel_path.with_suffix('.pdf')
+                    out_path.parent.mkdir(parents=True, exist_ok=True)
+                else:
+                    out_path = file_path.with_suffix('.pdf')
+                
+                # Save the PDF
+                with open(out_path, 'wb') as f:
+                    f.write(pdf_content)
+                
+                results.append({
+                    "source_file": str(file_path),
+                    "output_file": str(out_path),
+                    "status": "success"
+                })
+                print(f"Successfully converted: {file_path} -> {out_path}")
+                
+            except Exception as e:
+                print(f"Error converting {file_path}: {str(e)}")
+                results.append({
+                    "source_file": str(file_path),
+                    "status": "error",
+                    "error": str(e)
+                })
+        
+        response_data = {
+            "total_files": len(results),
+            "successful": sum(1 for r in results if r["status"] == "success"),
+            "failed": sum(1 for r in results if r["status"] == "error"),
+            "results": results
+        }
+        print(f"Conversion complete: {response_data['successful']} successful, {response_data['failed']} failed")
+        return response_data
+        
+    except Exception as e:
+        print(f"Error in batch conversion: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e)) 

routers/dev/pdf_utils.py

@@ -0,0 +1,46 @@
+from fastapi import APIRouter, UploadFile, File, HTTPException
+from typing import Dict
+from pathlib import Path
+import shutil
+import tempfile
+from modules.pdf_utils import PDFUtils
+
+router = APIRouter()
+
+@router.post("/extract-text")
+async def extract_text(
+    pdf_file: UploadFile = File(...)
+):
+    """
+    Extract text content from a PDF file
+    """
+    try:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            temp_file = Path(temp_dir) / pdf_file.filename
+            with temp_file.open("wb") as buffer:
+                shutil.copyfileobj(pdf_file.file, buffer)
+            
+            text = PDFUtils.extract_text_from_pdf(temp_file)
+            return {"text": text}
+            
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+@router.post("/metadata")
+async def get_metadata(
+    pdf_file: UploadFile = File(...)
+):
+    """
+    Get metadata from a PDF file
+    """
+    try:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            temp_file = Path(temp_dir) / pdf_file.filename
+            with temp_file.open("wb") as buffer:
+                shutil.copyfileobj(pdf_file.file, buffer)
+            
+            metadata = PDFUtils.get_pdf_metadata(temp_file)
+            return metadata
+            
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e)) 

routers/dev/test_analysis.py

@@ -0,0 +1,49 @@
+from fastapi import APIRouter, UploadFile, File, HTTPException, Form
+from typing import Dict
+import json
+from pathlib import Path
+import shutil
+import tempfile
+from modules.test_analyzer import TestAnalyzer, TestAnalysis
+from modules.pdf_utils import PDFUtils
+
+router = APIRouter()
+
+@router.post("/analyze", response_model=TestAnalysis)
+async def analyze_test(
+    test_file: UploadFile = File(...),
+    marks_data: str = Form(...),
+    api_key: str = Form(...),
+    mode: str = Form('detailed')
+):
+    """
+    Analyze a test PDF and generate feedback based on marks data
+    """
+    try:
+        print(f"Received request - Mode: {mode}")
+        marks_data_dict = json.loads(marks_data)
+        
+        with tempfile.TemporaryDirectory() as temp_dir:
+            temp_file = Path(temp_dir) / test_file.filename
+            with temp_file.open("wb") as buffer:
+                shutil.copyfileobj(test_file.file, buffer)
+            
+            print("File saved, initializing analyzer...")
+            analyzer = TestAnalyzer(api_key=api_key)
+            
+            print("Extracting PDF content...")
+            pdf_utils = PDFUtils()
+            pdf_content = pdf_utils.extract_text_from_pdf(temp_file)
+            
+            print("Analyzing content...")
+            analysis = analyzer.analyze_test(pdf_content, marks_data_dict, mode)
+            
+            print("Analysis complete")
+            return analysis
+            
+    except json.JSONDecodeError as e:
+        print(f"JSON decode error: {str(e)}")
+        raise HTTPException(status_code=422, detail=f"Invalid marks_data JSON format: {str(e)}")
+    except Exception as e:
+        print(f"Error in analyze_test: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e)) 

routers/dev/tests/timetable_test.py

@@ -0,0 +1,37 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+from modules.logger_tool import initialise_logger
+logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
+from fastapi import APIRouter
+
+router = APIRouter()
+
+@router.post("/run-pytest-timetable")
+async def run_pytest_timetable():
+    import subprocess    
+    
+    home_dir = os.environ['HOME_DIR']
+    backend_test_dir = os.environ['BACKEND_TEST_DIR']
+    logger.debug(f"original home_dir: {home_dir}")
+    logger.debug(f"original backend_test_dir: {backend_test_dir}")
+    
+    if backend_test_dir[0] != '/':
+        backend_test_dir = '/' + backend_test_dir
+    
+    # Convert backslashes to forward slashes for Windows compatibility
+    home_dir = home_dir.replace('\\', '/')
+    backend_test_dir = backend_test_dir.replace('\\', '/')
+    logger.debug(f"new home_dir: {home_dir}")
+    logger.debug(f"new backend_test_dir: {backend_test_dir}")
+    
+    # Join and normalize the path
+    pytest_dir = os.path.normpath(os.path.join(home_dir, backend_test_dir.lstrip('/'), "pytest_timetable.py"))
+    pytest_dir = pytest_dir.replace('\\', '/')  # Ensure forward slashes
+    f_string = f"pytest {pytest_dir} --maxfail=1 --disable-warnings -q"
+    logger.debug(f"f_string: {f_string}")
+    
+    result = subprocess.run(f_string, capture_output=True, text=True, shell=True)
+    logger.debug(f"result: {result}")
+    
+    return {"stdout": result.stdout, "stderr": result.stderr}

routers/external/youtube.py

@@ -0,0 +1,74 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+
+log_name = 'api_routers_external_youtube'
+log_dir = os.getenv("LOG_PATH", "/logs")
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+
+from fastapi import APIRouter, HTTPException
+from youtube_transcript_api import YouTubeTranscriptApi
+from youtube_transcript_api._errors import TranscriptsDisabled, NoTranscriptFound, VideoUnavailable
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+
+router = APIRouter()
+
+# Initialize the YouTube API client with API key
+youtube = build('youtube', 'v3', developerKey=os.getenv('YOUTUBE_API_KEY'))
+
+@router.get("/youtube-proxy")
+async def youtube_proxy(videoId: str):
+    try:
+        # Fetch transcript using youtube-transcript-api
+        transcript = YouTubeTranscriptApi.get_transcript(videoId, languages=['en'])
+        transcript_lines = [{"start": entry["start"], "duration": entry["duration"], "text": entry["text"]} for entry in transcript]
+
+        # Fetch video details using YouTube Data API
+        video_response = youtube.videos().list(
+            part='snippet,contentDetails,statistics',
+            id=videoId
+        ).execute()
+
+        if 'items' in video_response:
+            video_data = video_response['items'][0]
+            video_info = {
+                'title': video_data['snippet']['title'],
+                'author': video_data['snippet']['channelTitle'],
+                'publishedAt': video_data['snippet']['publishedAt'],
+                'description': video_data['snippet']['description'],
+                'viewCount': video_data['statistics']['viewCount'],
+                'likeCount': video_data['statistics']['likeCount'],
+                'duration': video_data['contentDetails']['duration'],
+            }
+        else:
+            video_info = {}
+
+        return {
+            "transcript": transcript_lines,
+            "video_info": video_info
+        }
+
+    except HttpError as e:
+        logging.error(f"An HTTP error occurred: {str(e)}")
+        raise HTTPException(status_code=500, detail="YouTube API error")
+    except TranscriptsDisabled:
+        logging.error(f"Transcripts are disabled for video {videoId}")
+        raise HTTPException(status_code=404, detail="Transcripts are disabled for this video")
+    except NoTranscriptFound:
+        logging.error(f"No transcript found for video {videoId}")
+        raise HTTPException(status_code=404, detail="Transcript not available for this video")
+    except VideoUnavailable:
+        logging.error(f"Video {videoId} is unavailable")
+        raise HTTPException(status_code=404, detail="Video unavailable")
+    except Exception as e:
+        logging.error(f"Unexpected error: {str(e)}")
+        raise HTTPException(status_code=500, detail="Internal server error")

routers/health.py

@@ -0,0 +1,23 @@
+from fastapi import APIRouter, status
+from pydantic import BaseModel
+
+router = APIRouter()
+
+class HealthCheck(BaseModel):
+    """Response model for health check endpoint"""
+    status: str = "healthy"
+
+@router.get(
+    "/health",
+    tags=["Health"],
+    summary="Perform a Health Check",
+    response_description="Return health status",
+    status_code=status.HTTP_200_OK,
+    response_model=HealthCheck
+)
+async def health_check() -> HealthCheck:
+    """
+    Endpoint to perform a healthcheck. Used by container orchestration systems
+    to determine if the service is healthy and ready to receive traffic.
+    """
+    return HealthCheck()

routers/langchain/interactive_langgraph_query.py

@@ -0,0 +1,80 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'api_routers_interactive_langgraph_query'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+from typing import List
+from modules.langchain.interactive_langgraph_query import perplexity_clone_graph
+from modules.redis_config import get_cached_results, set_cached_results
+from langchain_core.messages import HumanMessage
+
+router = APIRouter()
+
+class QueryRequest(BaseModel):
+    query: str
+    use_cache: bool = False
+
+class QueryResponse(BaseModel):
+    response: str
+    needs_more_info: bool
+
+@router.post("/query", response_model=QueryResponse)
+async def interactive_query(request: QueryRequest):
+    logging.info(f"Received query: {request.query}")
+    try:
+        query_id = generate_random_alphanumeric()
+        config = {"configurable": {"thread_id": f'{query_id}'}, "recursion_limit": 20}
+        
+        inputs = {
+            "messages": [HumanMessage(content=request.query)],
+        }
+        
+        # Check cache for existing results only if DEV_MODE is false
+        use_cache = os.getenv("DEV_MODE", "true").lower() == "false"
+        if use_cache:
+            cache_key = f"langgraph_query:{request.query}"
+            cached_result = get_cached_results(cache_key)
+            if cached_result:
+                logging.info(f"Found cached result for query: {request.query}")
+                return cached_result
+        
+        logging.debug("Updating state with initial message")
+        perplexity_clone_graph.update_state(config, inputs)
+        
+        logging.debug("Invoking perplexity_clone_graph")
+        outputs = await perplexity_clone_graph.ainvoke(inputs, config)
+        
+        final_response = outputs['messages'][-1].content
+        needs_more_info = outputs.get('needs_more_info', False)
+        
+        logging.info(f"Final response: {final_response}")
+        logging.info(f"Needs more info: {needs_more_info}")
+        
+        response = QueryResponse(response=final_response, needs_more_info=needs_more_info)
+        
+        # Cache the result only if DEV_MODE is false
+        if use_cache:
+            set_cached_results(cache_key, response.dict())
+        
+        return response
+    except Exception as e:
+        logging.error(f"Error in interactive query: {str(e)}", exc_info=True)
+        raise HTTPException(status_code=500, detail=f"An error occurred during the query process: {str(e)}")
+
+def generate_random_alphanumeric(length=4):
+    import random
+    import string
+    characters = string.ascii_letters + string.digits
+    return ''.join(random.choice(characters) for i in range(length))

routers/langchain/neo4j_graph_qa.py

@@ -0,0 +1,153 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'api_routers_langchain_graph_qa'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+from fastapi import APIRouter, HTTPException
+from langchain.chains import GraphCypherQAChain
+from langchain_community.graphs import Neo4jGraph
+from langchain_community.chat_models import ChatOpenAI
+from langchain.prompts.prompt import PromptTemplate
+from routers.llm.private.ollama.ollama_wrapper import OllamaWrapper
+
+router = APIRouter()
+
+# Define the schema for nodes and relationships
+node_types = {
+    "KeyStage": ["merged", "key_stage_name", "unique_id", "created"],
+    "KeyStageSyllabus": ["ks_syllabus_name", "unique_id", "created", "merged", "ks_syllabus_key_stage", "ks_syllabus_subject"],
+    "YearGroup": ["created", "merged", "unique_id", "year_group_name"],
+    "YearGroupSyllabus": ["created", "merged", "yr_syllabus_name", "yr_syllabus_year_group", "yr_syllabus_id", "yr_syllabus_subject"],
+    "Topic": ["topic_type", "topic_assessment_type", "created", "merged", "unique_id", "topic_id", "total_number_of_lessons_for_topic", "topic_title"],
+    "Lesson": ["topic_lesson_id", "topic_lesson_type", "created", "merged", "topic_lesson_title", "topic_lesson_length", "topic_lesson_suggested_activities", "topic_lesson_weblinks", "topic_lesson_skills_learned"],
+    "LearningStatement": ["created", "merged", "lesson_learning_statement", "lesson_learning_statement_id", "lesson_learning_statement_type"]
+}
+
+relationship_types = {
+    "KEY_STAGE_INCLUDES_KEY_STAGE_SYLLABUS": ["created", "merged"],
+    "KEY_STAGE_SYLLABUS_INCLUDES_YEAR_GROUP_SYLLABUS": ["created", "merged"],
+    "YEAR_GROUP_FOLLOWS_YEAR_GROUP": ["created", "merged"],
+    "KEY_STAGE_FOLLOWS_KEY_STAGE": ["created", "merged"],
+    "YEAR_SYLLABUS_INCLUDES_TOPIC": ["created", "merged"],
+    "TOPIC_INCLUDES_LESSON": ["created", "merged"],
+    "LESSON_INCLUDES_LEARNING_STATEMENT": ["created", "merged"],
+    "LESSON_FOLLOWS_LESSON": ["created", "merged"]
+}
+
+@router.get("/prompt")
+async def query_graph(
+    database: str, prompt: str, top_k: int = 30, model: str = "gpt-4o", temperature: float = 0,
+    verbose: bool = False, return_intermediate_steps: bool = False, exclude_types: list = None, include_types: list = None,
+    return_direct: bool = False, validate_cypher: bool = False, model_type: str = "openai"
+):
+    logging.info(f"Received request with prompt: {prompt}")
+    if exclude_types is None:
+        logging.info("No exclude_types provided, using default.")
+        exclude_types = []
+    if include_types is None:
+        logging.info("No include_types provided, using default.")
+        include_types = []
+
+    # Validate include_types and exclude_types
+    logging.info(f"Validating include_types and exclude_types...")
+    valid_types = set(node_types.keys()).union(set(relationship_types.keys()))
+    logging.info(f"Valid types: {valid_types}")
+    exclude_types = [t for t in exclude_types if t in valid_types]
+    logging.info(f"Validated exclude_types: {exclude_types}")
+    include_types = [t for t in include_types if t in valid_types]
+    logging.info(f"Validated include_types: {include_types}")
+
+    graph = Neo4jGraph(
+        url=os.environ['APP_BOLT_URL'],
+        username=os.environ['USER_NEO4J'],
+        password=os.environ['PASSWORD_NEO4J'],
+        database=database
+    )
+
+    logging.info("Refreshing schema...")
+    graph.refresh_schema()
+    logging.info("Schema refreshed.")
+    schema = graph.schema
+    logging.info(f"Schema: {schema}")
+
+    CYPHER_GENERATION_TEMPLATE = """Task: Generate a Cypher statement to query a graph database for timetable information.
+    Role:
+    You are an assistant in a school for teachers, specializing in querying graph databases to find answers to questions.
+    The teacher will ask you questions about their timetable.
+
+    Instructions:
+    1. Use only the provided relationship types and properties in the schema.
+    2. Do not use any other relationship types or properties that are not provided.
+
+    Schema:
+    {schema}
+
+    Note:
+    1. Do not include any explanations or apologies in your responses.
+    2. Do not respond to any questions that might ask anything else than for you to construct a Cypher statement.
+    3. Do not include any text except the generated Cypher statement.
+
+    The question is:
+    {question}"""
+
+    CYPHER_GENERATION_PROMPT = PromptTemplate(
+        input_variables=["schema", "question"],
+        template=CYPHER_GENERATION_TEMPLATE
+    )
+
+    if model_type == "ollama":
+        ollama_host = os.getenv("OLLAMA_URL")
+        ollama_port = os.getenv("OLLAMA_PORT")
+        if not ollama_host or not ollama_port:
+            raise HTTPException(status_code=500, detail="Ollama host or port not set")
+        client = OllamaWrapper(host=f'http://{ollama_host}:{ollama_port}')
+        cypher_llm = client
+        qa_llm = client
+    else:
+        cypher_llm = ChatOpenAI(temperature=temperature, model=model)
+        qa_llm = ChatOpenAI(temperature=temperature, model=model)
+
+    chain = GraphCypherQAChain.from_llm(
+        graph=graph,
+        cypher_llm=cypher_llm,
+        qa_llm=qa_llm,
+        top_k=top_k,
+        verbose=verbose,
+        cypher_prompt=CYPHER_GENERATION_PROMPT,
+        return_intermediate_steps=return_intermediate_steps,
+        exclude_types=exclude_types,
+        include_types=include_types,
+        return_direct=return_direct,
+        validate_cypher=validate_cypher
+    )
+    
+    formatted_prompt = CYPHER_GENERATION_PROMPT.format(schema=schema, question=prompt)
+
+    logging.info("\n\n")
+
+    logging.info("==================================================")
+    logging.info("=            graph_qa.py                         =")
+    logging.info("==================================================")
+    logging.info(f"Prompt: {prompt}")
+    logging.info("--------------------------------------------------")
+    logging.info(f"Schema: \n{schema}\n")
+    logging.info("--------------------------------------------------")
+    logging.info(f"Formatted Prompt: \n{formatted_prompt}\n")
+    logging.info("--------------------------------------------------")
+    logging.info(f"Cypher prompt: \n{CYPHER_GENERATION_PROMPT}\n")
+    logging.info("--------------------------------------------------")
+    logging.info(f"Cypher template: \n{CYPHER_GENERATION_TEMPLATE}\n")
+    logging.info("--------------------------------------------------")
+    logging.info(f"Cypher chain: \n{chain}\n")
+    logging.info("==================================================")
+
+    return chain(prompt)

routers/langchain/test.ipynb

@@ -0,0 +1,151 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Running simple query tests with OpenAI:\n",
+      "\n",
+      "Testing simple queries using openai model:\n",
+      "\n",
+      "Query: What is the history of Maidstone, England?\n",
+      "Sending query to http://localhost:8000/api/langchain/interactive_langgraph_query/query with payload: {'query': 'What is the history of Maidstone, England?', 'model': 'openai'}\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "ERROR:root:Error sending query to http://localhost:8000/api/langchain/interactive_langgraph_query/query: 500 Server Error: Internal Server Error for url: http://localhost:8000/api/langchain/interactive_langgraph_query/query\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Response:\n",
+      "{\n",
+      "  \"error\": \"500 Server Error: Internal Server Error for url: http://localhost:8000/api/langchain/interactive_langgraph_query/query\"\n",
+      "}\n",
+      "==================================================\n"
+     ]
+    }
+   ],
+   "source": [
+    "from dotenv import load_dotenv, find_dotenv\n",
+    "load_dotenv(find_dotenv())\n",
+    "import os\n",
+    "import logging\n",
+    "# Function to send a query and get the response\n",
+    "import requests\n",
+    "import json\n",
+    "\n",
+    "# Define the URL of your FastAPI server\n",
+    "BASE_URL = \"http://localhost:8000\"  # Adjust this if your server is running on a different port or host\n",
+    "\n",
+    "# Define the endpoint\n",
+    "ENDPOINT = f\"{BASE_URL}/api/langchain/interactive_langgraph_query/query\"\n",
+    "\n",
+    "def send_query(query, model=\"ollama\"):\n",
+    "    payload = {\"query\": query, \"model\": model}\n",
+    "    headers = {\"Content-Type\": \"application/json\"}\n",
+    "    print(f\"Sending query to {ENDPOINT} with payload: {payload}\")\n",
+    "    \n",
+    "    try:\n",
+    "        response = requests.post(ENDPOINT, json=payload, headers=headers)\n",
+    "        response.raise_for_status()\n",
+    "        print(f\"Received response from {ENDPOINT}: {response.json()}\")\n",
+    "        return response.json()\n",
+    "    except requests.exceptions.RequestException as e:\n",
+    "        logging.error(f\"Error sending query to {ENDPOINT}: {str(e)}\")\n",
+    "        return {\"error\": str(e)}\n",
+    "\n",
+    "def test_simple_queries(model=\"openai\"):\n",
+    "    queries = [\n",
+    "        \"What is the history of Maidstone, England?\"\n",
+    "    ]\n",
+    "    \n",
+    "    print(f\"Testing simple queries using {model} model:\")\n",
+    "    for query in queries:\n",
+    "        print(f\"\\nQuery: {query}\")\n",
+    "        result = send_query(query, model)\n",
+    "        print(\"Response:\")\n",
+    "        print(json.dumps(result, indent=2))\n",
+    "        print(\"=\" * 50)\n",
+    "\n",
+    "def test_followup_queries(model=\"openai\"):\n",
+    "    queries = [\n",
+    "        \"What is the latest local news from a particular town?\"\n",
+    "    ]\n",
+    "    \n",
+    "    print(f\"Testing queries requiring follow-up using {model} model:\")\n",
+    "    for query in queries:\n",
+    "        print(f\"\\nInitial Query: {query}\")\n",
+    "        result = send_query(query, model)\n",
+    "        print(\"Initial Response:\")\n",
+    "        print(json.dumps(result, indent=2))\n",
+    "        \n",
+    "        follow_up_count = 0\n",
+    "        max_follow_ups = 3\n",
+    "        \n",
+    "        while result.get(\"needs_more_info\", False) and follow_up_count < max_follow_ups:\n",
+    "            follow_up = input(\"Please provide more information: \")\n",
+    "            follow_up_query = f\"{query} {follow_up}\"\n",
+    "            follow_up_result = send_query(follow_up_query, model)\n",
+    "            print(f\"\\nFollow-up Response {follow_up_count + 1}:\")\n",
+    "            print(json.dumps(follow_up_result, indent=2))\n",
+    "            \n",
+    "            result = follow_up_result\n",
+    "            follow_up_count += 1\n",
+    "        \n",
+    "        if follow_up_count == max_follow_ups:\n",
+    "            print(\"\\nMaximum number of follow-ups reached. Moving to next query.\")\n",
+    "        elif not result.get(\"needs_more_info\", False):\n",
+    "            print(\"\\nFinal Response:\")\n",
+    "            print(json.dumps(result, indent=2))\n",
+    "        \n",
+    "        print(\"=\" * 50)\n",
+    "\n",
+    "# Run the tests\n",
+    "#print(\"Running simple query tests with Ollama:\\n\")\n",
+    "#test_simple_queries(\"ollama\")\n",
+    "\n",
+    "print(\"\\nRunning simple query tests with OpenAI:\\n\")\n",
+    "test_simple_queries(\"openai\")\n",
+    "\n",
+    "#print(\"\\nRunning follow-up query tests with Ollama:\\n\")\n",
+    "#test_followup_queries(\"ollama\")\n",
+    "\n",
+    "#print(\"\\nRunning follow-up query tests with OpenAI:\\n\")\n",
+    "#test_followup_queries(\"openai\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

routers/llm/private/ollama/ollama.py

@@ -0,0 +1,108 @@
+# Import necessary libraries
+import os
+from dotenv import load_dotenv, find_dotenv
+from fastapi import APIRouter, FastAPI, HTTPException
+from pydantic import BaseModel
+from typing import List, Dict, Optional
+import ollama
+from ollama import Client
+
+load_dotenv(find_dotenv())
+
+router = APIRouter()
+## client = Client(host='http://localhost:11434')
+
+ollama_host = os.getenv("HOST_OLLAMA")
+ollama_port = os.getenv("PORT_OLLAMA")
+
+if not ollama_host or not ollama_port:
+    raise ValueError("Environment variables HOST_OLLAMA or PORT_OLLAMA are not set")
+
+client = Client(host=f'http://{ollama_host}:{ollama_port}')
+
+class UserRequest(BaseModel):
+    question: str
+    model: str = "llama3"
+    temperature: Optional[float] = None
+    top_p: Optional[float] = None
+    max_tokens: Optional[int] = None
+
+@router.post("/ollama_text_prompt")
+async def ollama_text_prompt(user_request: UserRequest):
+    model_name = user_request.model
+    question = user_request.question
+    options = {
+        "temperature": user_request.temperature,
+        "top_p": user_request.top_p,
+        "max_tokens": user_request.max_tokens,
+    }
+
+    supported_models = ["llama2", "llama3", "mistral", "llama3"]
+    if model_name not in supported_models:
+        raise HTTPException(status_code=400, detail="Model not supported")
+
+    messages = [{"role": "user", "content": question}]
+    try:
+        response = client.chat(model=model_name, messages=messages, options=options)
+        if "message" in response and "content" in response["message"]:
+            return {"model": model_name, "response": response["message"]["content"]}
+        else:
+            raise HTTPException(status_code=500, detail="Invalid response structure from model")
+    except Exception as e:
+        print(f"Error: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+class GenerateRequest(BaseModel):
+    model: str
+    prompt: str
+
+@router.post("/ollama_generate")
+async def ollama_generate(request: GenerateRequest):
+    try:
+        response = client.generate(model=request.model, prompt=request.prompt)
+        return {"model": request.model, "response": response}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+class VisionRequest(BaseModel):
+    model: str
+    image_path: str
+    prompt: str
+
+@router.post("/ollama_vision_prompt")
+async def ollama_vision_prompt(request: VisionRequest):
+    try:
+        response = client.vision(model=request.model, image_path=request.image_path, prompt=request.prompt)
+        return {"model": request.model, "response": response}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+    
+class Message(BaseModel):
+    role: str
+    content: str
+
+class CopilotRequest(BaseModel):
+    model: str
+    messages: List[Message]
+    options: Optional[Dict[str, float]] = None
+    
+@router.post("/ollama_copilot_prompt")
+async def ollama_copilot_prompt(request: CopilotRequest):
+    model_name = request.model
+    messages = request.messages
+    options = request.options or {}
+    print(f"Model: {model_name}, Messages: {messages}, Options: {options}")
+
+    try:
+        print("Generating response...")
+        response = ollama.chat(model=model_name, messages=messages, **options)
+        print(f"Response: {response}")
+        if "message" in response and "content" in response["message"]:
+            print(f"Response: {response['message']['content']}")
+            return {"model": model_name, "response": response["message"]["content"]}
+        else:
+            print(f"Invalid response structure from model: {response}")
+            raise HTTPException(status_code=500, detail="Invalid response structure from model")
+    except Exception as e:
+        print(f"Error: {e}")
+        raise HTTPException(status_code=500, detail=str(e))

routers/llm/private/ollama/ollama_wrapper.py

@@ -0,0 +1,28 @@
+from typing import Any, Dict
+from ollama import Client
+from langchain_core.runnables.base import Runnable
+from langchain.prompts.base import StringPromptValue
+
+class OllamaWrapper(Runnable):
+    def __init__(self, host: str):
+        self.client = Client(host=host)
+
+    def invoke(self, prompt: Any, config: Dict[str, Any] = None, **kwargs: Any) -> str:
+        if isinstance(prompt, StringPromptValue):
+            prompt = prompt.to_string()
+
+        model_name = kwargs.get("model", "llama3")
+        options = {
+            "temperature": kwargs.get("temperature"),
+            "top_p": kwargs.get("top_p"),
+            "max_tokens": kwargs.get("max_tokens"),
+        }
+        messages = [{"role": "user", "content": prompt}]
+        response = self.client.chat(model=model_name, messages=messages, options=options)
+        if response and "message" in response and "content" in response["message"]:
+            return response["message"]["content"]
+        else:
+            raise ValueError("Invalid response structure from model")
+
+    async def ainvoke(self, prompt: Any, config: Dict[str, Any] = None, **kwargs: Any) -> str:
+        return self.invoke(prompt, config, **kwargs)