latest

run/initialization/__init__.py

@@ -1,26 +1,75 @@
-from .manager import InitializationManager
-from .initialization import InitializationSystem
+from .infrastructure import initialize_infrastructure
+from .demo_school import initialize_demo_school
+from .demo_users import initialize_demo_users
+from .gais_data import import_gais_data
 from modules.logger_tool import initialise_logger
 import os
 
 logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
 
-def initialize_system() -> None:
-    """Initialize the system if needed"""
-    init_manager = InitializationManager()
+def initialize_infrastructure_mode() -> None:
+    """Initialize infrastructure: Neo4j schema, calendar, and Supabase buckets"""
+    logger.info("Starting infrastructure initialization...")
     
-    if not init_manager.check_initialization_needed():
-        logger.info("No initialization needed")
+    # 1. Initialize Neo4j database, schema, and calendar structure
+    logger.info("Step 1: Initializing Neo4j infrastructure...")
+    from .neo4j import initialize_neo4j
+    neo4j_result = initialize_neo4j()
+    
+    if not neo4j_result["success"]:
+        logger.error(f"Neo4j infrastructure initialization failed: {neo4j_result['message']}")
         return
-        
-    logger.info("Starting system initialization...")
     
-    init_system = InitializationSystem(init_manager)
-    success = init_system.run()
+    # 2. Initialize Supabase storage buckets
+    logger.info("Step 2: Initializing Supabase storage buckets...")
+    from .buckets import initialize_buckets
+    buckets_result = initialize_buckets()
     
-    if success:
-        logger.info("System initialization completed successfully")
-    else:
-        logger.error("System initialization failed")
+    if not buckets_result["success"]:
+        logger.error(f"Storage buckets initialization failed: {buckets_result['message']}")
+        return
+    
+    logger.info("Infrastructure initialization completed successfully!")
+    logger.info(f"Neo4j: {neo4j_result['message']}")
+    logger.info(f"Buckets: {buckets_result['message']}")
 
-__all__ = ['initialize_system', 'InitializationManager', 'InitializationSystem'] 
+def initialize_demo_school_mode() -> None:
+    """Initialize demo school (KevlarAI)"""
+    logger.info("Starting demo school initialization...")
+    result = initialize_demo_school()
+    
+    if result["success"]:
+        logger.info("Demo school initialization completed successfully")
+    else:
+        logger.error(f"Demo school initialization failed: {result['message']}")
+
+def initialize_demo_users_mode() -> None:
+    """Initialize demo users"""
+    logger.info("Starting demo users initialization...")
+    result = initialize_demo_users()
+    
+    if result["success"]:
+        logger.info("Demo users initialization completed successfully")
+    else:
+        logger.error(f"Demo users initialization failed: {result['message']}")
+
+def initialize_gais_data_mode() -> None:
+    """Initialize GAIS data import (Edubase, etc.)"""
+    logger.info("Starting GAIS data import...")
+    result = import_gais_data()
+    
+    if result["success"]:
+        logger.info("GAIS data import completed successfully")
+    else:
+        logger.error(f"GAIS data import failed: {result['message']}")
+
+__all__ = [
+    'initialize_infrastructure_mode',
+    'initialize_demo_school_mode',
+    'initialize_demo_users_mode',
+    'initialize_gais_data_mode',
+    'initialize_infrastructure',
+    'initialize_demo_school',
+    'initialize_demo_users',
+    'import_gais_data'
+] 

run/initialization/buckets.py

@@ -0,0 +1,109 @@
+import os
+from modules.logger_tool import initialise_logger
+from modules.database.supabase.utils.client import SupabaseServiceRoleClient, CreateBucketOptions
+
+logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
+
+def initialize_buckets() -> dict:
+    """
+    Initialize storage buckets for ClassroomCopilot documents and files.
+    
+    Creates buckets for:
+    - TLDraw snapshot JSON files
+    - Office document files (PDF, DOCX, etc.)
+    - Docling document JSON files
+    - Document page images (PNG/base64)
+    - Document page fragment images (smaller PNG/base64)
+    
+    Returns:
+        dict: Result status and message
+    """
+    logger.info("Starting storage bucket initialization...")
+    
+    try:
+        storage_client = SupabaseServiceRoleClient()
+        
+        # Define the buckets to create
+        buckets = [
+            # TLDraw snapshot files
+            {
+                "id": "cc.public.snapshots",
+                "options": CreateBucketOptions(
+                    name="ClassroomCopilot Public TLDraw Snapshots",
+                    public=False,
+                    file_size_limit=1000 * 1024 * 1024,  # 1GB
+                    allowed_mime_types=[
+                        'application/json'
+                    ]
+                )
+            },
+            # User cabinet files
+            {
+                "id": "cc.users",
+                "options": CreateBucketOptions(
+                    name="Classroom Copilot Users - Private",
+                    public=False,
+                    file_size_limit=1000 * 1024 * 1024,  # 1GB
+                )
+            },
+        ]
+        
+        results = {}
+        success_count = 0
+        total_count = len(buckets)
+        
+        for bucket in buckets:
+            try:
+                logger.info(f"Creating bucket: {bucket['id']}")
+                result = storage_client.create_bucket(bucket["id"], bucket["options"])
+                
+                if result:
+                    results[bucket["id"]] = {
+                        "status": "success",
+                        "result": result
+                    }
+                    success_count += 1
+                    logger.info(f"Successfully created bucket: {bucket['id']}")
+                else:
+                    results[bucket["id"]] = {
+                        "status": "error",
+                        "error": "Failed to create bucket"
+                    }
+                    logger.error(f"Failed to create bucket: {bucket['id']}")
+                    
+            except Exception as e:
+                results[bucket["id"]] = {
+                    "status": "error",
+                    "error": str(e)
+                }
+                logger.error(f"Error creating bucket {bucket['id']}: {str(e)}")
+        
+        # Determine overall success
+        if success_count == total_count:
+            message = f"All {total_count} storage buckets created successfully"
+            success = True
+        elif success_count > 0:
+            message = f"Created {success_count}/{total_count} storage buckets. Some failed."
+            success = False
+        else:
+            message = f"Failed to create any storage buckets ({total_count} attempted)"
+            success = False
+        
+        logger.info(f"Bucket initialization completed: {message}")
+        
+        return {
+            "success": success,
+            "message": message,
+            "results": results,
+            "success_count": success_count,
+            "total_count": total_count
+        }
+        
+    except Exception as e:
+        error_msg = f"Failed to initialize storage buckets: {str(e)}"
+        logger.error(error_msg)
+        return {
+            "success": False,
+            "message": error_msg,
+            "error": str(e)
+        }

run/initialization/demo_school.py

@@ -0,0 +1,181 @@
+"""
+Demo school initialization module for ClassroomCopilot
+Creates the KevlarAI demo school
+"""
+import os
+import json
+import requests
+from typing import Dict, Any
+from modules.logger_tool import initialise_logger
+from modules.database.services.provisioning_service import ProvisioningService
+import time
+
+logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
+
+class DemoSchoolInitializer:
+    """Handles demo school creation"""
+    
+    def __init__(self, supabase_url: str, service_role_key: str):
+        self.supabase_url = supabase_url
+        self.service_role_key = service_role_key
+        self.supabase_headers = {
+            "apikey": service_role_key,
+            "Authorization": f"Bearer {service_role_key}",
+            "Content-Type": "application/json"
+        }
+        self.provisioning_service = ProvisioningService()
+    
+    def create_kevlarai_school(self) -> Dict[str, Any]:
+        """Create the KevlarAI demo school"""
+        logger.info("Creating KevlarAI demo school...")
+        
+        try:
+            # Check if KevlarAI school already exists
+            response = self._supabase_request_with_retry(
+                'get',
+                f"{self.supabase_url}/rest/v1/institutes",
+                headers=self.supabase_headers,
+                params={
+                    "select": "*",
+                    "name": "eq.KevlarAI"
+                }
+            )
+            
+            if response.status_code == 200:
+                existing_schools = response.json()
+                if existing_schools and len(existing_schools) > 0:
+                    logger.info("KevlarAI school already exists")
+                    school = existing_schools[0]
+                    try:
+                        self.provisioning_service.ensure_school(school["id"])
+                    except Exception as provisioning_error:
+                        logger.warning(f"Provisioning KevlarAI school failed: {provisioning_error}")
+                    return {
+                        "success": True,
+                        "message": "KevlarAI school already exists",
+                        "school": school
+                    }
+            
+            # Create KevlarAI school
+            school_data = {
+                "name": "KevlarAI",
+                "urn": "KEVLARAI001",
+                "status": "active",
+                "address": {
+                    "street": "123 Innovation Drive",
+                    "town": "Tech City",
+                    "county": "Digital County",
+                    "postcode": "TC1 2AI",
+                    "country": "United Kingdom"
+                },
+                "website": "https://kevlar.ai",
+                "metadata": {
+                    "school_type": "AI and Technology",
+                    "phase_of_education": "Secondary and Further Education",
+                    "establishment_status": "Open",
+                    "specialization": "Artificial Intelligence, Machine Learning, Robotics"
+                }
+            }
+            
+            # Insert the school
+            response = self._supabase_request_with_retry('post', f"{self.supabase_url}/rest/v1/institutes", headers={**self.supabase_headers, "Prefer": "return=representation"}, json=school_data, params={"select": "*"})
+            
+            logger.info(f"Supabase response status: {response.status_code}")
+            logger.info(f"Supabase response headers: {dict(response.headers)}")
+            logger.info(f"Supabase response text: {response.text}")
+            
+            if response.status_code in (200, 201):
+                try:
+                    data = response.json()
+                    school = data[0] if isinstance(data, list) and data else data
+                    logger.info("Successfully created KevlarAI school")
+                    # Ensure Neo4j provisioning is in place
+                    try:
+                        self.provisioning_service.ensure_school(school["id"])
+                    except Exception as provisioning_error:
+                        logger.warning(f"Provisioning KevlarAI school failed: {provisioning_error}")
+                    return {
+                        "success": True,
+                        "message": "Successfully created KevlarAI school",
+                        "school": school
+                    }
+                except json.JSONDecodeError as e:
+                    logger.error(f"Failed to parse JSON response: {str(e)}")
+                    logger.error(f"Response text: {response.text}")
+                    # If the status code is successful but we can't parse JSON, 
+                    # the school was likely created successfully
+                    return {
+                        "success": True,
+                        "message": "Successfully created KevlarAI school (response not JSON)",
+                        "school": None
+                    }
+            else:
+                logger.error(f"Failed to create KevlarAI school: {response.text}")
+                return {
+                    "success": False,
+                    "message": f"Failed to create KevlarAI school: {response.text}"
+                }
+                
+        except Exception as e:
+            logger.error(f"Error creating KevlarAI school: {str(e)}")
+            return {
+                "success": False,
+                "message": f"Error creating KevlarAI school: {str(e)}"
+            }
+    
+    def _supabase_request_with_retry(self, method, url, **kwargs):
+        """Make a request to Supabase with retry logic"""
+        max_retries = 3
+        retry_delay = 2  # seconds
+        
+        for attempt in range(max_retries):
+            try:
+                if method.lower() == 'get':
+                    response = requests.get(url, **kwargs)
+                elif method.lower() == 'post':
+                    response = requests.post(url, **kwargs)
+                elif method.lower() == 'put':
+                    response = requests.put(url, **kwargs)
+                elif method.lower() == 'delete':
+                    response = requests.delete(url, **kwargs)
+                else:
+                    raise ValueError(f"Unsupported HTTP method: {method}")
+                
+                # If successful or client error (4xx), don't retry
+                if response.status_code < 500:
+                    return response
+                
+                # Server error (5xx), retry after delay
+                logger.warning(f"Supabase server error (attempt {attempt+1}/{max_retries}): {response.status_code} - {response.text}")
+                time.sleep(retry_delay * (attempt + 1))  # Exponential backoff
+                
+            except requests.RequestException as e:
+                logger.warning(f"Supabase request exception (attempt {attempt+1}/{max_retries}): {str(e)}")
+                if attempt == max_retries - 1:
+                    raise
+                time.sleep(retry_delay * (attempt + 1))
+        
+        # If we get here, all retries failed with server errors
+        raise requests.RequestException(f"Failed after {max_retries} attempts to {method} {url}")
+
+def initialize_demo_school() -> Dict[str, Any]:
+    """Initialize demo school (KevlarAI)"""
+    logger.info("Starting demo school initialization...")
+    
+    supabase_url = os.getenv("SUPABASE_URL")
+    service_role_key = os.getenv("SERVICE_ROLE_KEY")
+    
+    if not supabase_url or not service_role_key:
+        return {"success": False, "message": "Missing SUPABASE_URL or SERVICE_ROLE_KEY environment variables"}
+    
+    initializer = DemoSchoolInitializer(supabase_url, service_role_key)
+    
+    # Create KevlarAI school
+    result = initializer.create_kevlarai_school()
+    
+    if result["success"]:
+        logger.info("Demo school initialization completed successfully")
+    else:
+        logger.error(f"Demo school initialization failed: {result['message']}")
+    
+    return result

run/initialization/demo_users.py

@@ -0,0 +1,395 @@
+"""
+Demo users initialization module for ClassroomCopilot
+Creates demo teachers and students
+"""
+import os
+import json
+import requests
+import time
+from typing import Dict, Any
+from modules.logger_tool import initialise_logger
+from modules.database.services.provisioning_service import ProvisioningService
+
+logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
+
+class DemoUsersInitializer:
+    """Handles demo users creation"""
+    
+    def __init__(self, supabase_url: str, service_role_key: str):
+        self.supabase_url = supabase_url
+        self.service_role_key = service_role_key
+        self.supabase_headers = {
+            "apikey": service_role_key,
+            "Authorization": f"Bearer {service_role_key}",
+            "Content-Type": "application/json"
+        }
+        self.provisioning_service = ProvisioningService()
+    
+    def create_demo_users(self) -> Dict[str, Any]:
+        """Create demo teachers and students"""
+        logger.info("Creating demo users...")
+        
+        try:
+            # Define demo users
+            demo_users = [
+                # Demo Teachers
+                {
+                    "email": "[email protected]",
+                    "password": "DemoTeacher123!",
+                    "email_confirm": True,
+                    "user_metadata": {
+                        "name": "Dr. Sarah Chen",
+                        "username": "sarah.chen",
+                        "full_name": "Dr. Sarah Chen",
+                        "display_name": "Dr. Chen",
+                        "user_type": "teacher"
+                    },
+                    "app_metadata": {
+                        "provider": "email",
+                        "providers": ["email"]
+                    }
+                },
+                {
+                    "email": "[email protected]",
+                    "password": "DemoTeacher123!",
+                    "email_confirm": True,
+                    "user_metadata": {
+                        "name": "Prof. Marcus Rodriguez",
+                        "username": "marcus.rodriguez",
+                        "full_name": "Professor Marcus Rodriguez",
+                        "display_name": "Prof. Rodriguez",
+                        "user_type": "teacher"
+                    },
+                    "app_metadata": {
+                        "provider": "email",
+                        "providers": ["email"]
+                    }
+                },
+                # Demo Students
+                {
+                    "email": "[email protected]",
+                    "password": "DemoStudent123!",
+                    "email_confirm": True,
+                    "user_metadata": {
+                        "name": "Alex Thompson",
+                        "username": "alex.thompson",
+                        "full_name": "Alex Thompson",
+                        "display_name": "Alex",
+                        "user_type": "student"
+                    },
+                    "app_metadata": {
+                        "provider": "email",
+                        "providers": ["email"]
+                    }
+                },
+                {
+                    "email": "[email protected]",
+                    "password": "DemoStudent123!",
+                    "email_confirm": True,
+                    "user_metadata": {
+                        "name": "Jordan Lee",
+                        "username": "jordan.lee",
+                        "full_name": "Jordan Lee",
+                        "display_name": "Jordan",
+                        "user_type": "student"
+                    },
+                    "app_metadata": {
+                        "provider": "email",
+                        "providers": ["email"]
+                    }
+                }
+            ]
+            
+            created_users = []
+            failed_users = []
+            
+            for user_data in demo_users:
+                try:
+                    # Create user via Auth API
+                    response = self._supabase_request_with_retry(
+                        'post',
+                        f"{self.supabase_url}/auth/v1/admin/users",
+                        headers=self.supabase_headers,
+                        json=user_data
+                    )
+                    
+                    if response.status_code in (200, 201):
+                        user = response.json()
+                        user_id = user.get("id")
+                        
+                        # Wait a moment for user to be created
+                        time.sleep(1)
+                        
+                        # Create profile
+                        profile_data = {
+                            "id": user_id,
+                            "email": user_data["email"],
+                            "user_type": user_data["user_metadata"]["user_type"],
+                            "username": user_data["user_metadata"]["username"],
+                            "full_name": user_data["user_metadata"]["full_name"],
+                            "display_name": user_data["user_metadata"]["display_name"]
+                        }
+                        
+                        profile_response = self._supabase_request_with_retry(
+                            'post',
+                            f"{self.supabase_url}/rest/v1/profiles",
+                            headers=self.supabase_headers,
+                            json=profile_data
+                        )
+                        
+                        if profile_response.status_code in (200, 201):
+                            created_users.append({
+                                "id": user_id,
+                                "email": user_data["email"],
+                                "user_type": user_data["user_metadata"]["user_type"],
+                                "username": user_data["user_metadata"]["username"]
+                            })
+                            logger.info(f"Successfully created user: {user_data['email']}")
+                        else:
+                            logger.warning(f"Failed to create profile for {user_data['email']}: {profile_response.text}")
+                            failed_users.append({
+                                "email": user_data["email"],
+                                "error": f"Profile creation failed: {profile_response.text}"
+                            })
+                    else:
+                        logger.warning(f"Failed to create user {user_data['email']}: {response.text}")
+                        failed_users.append({
+                            "email": user_data["email"],
+                            "error": f"User creation failed: {response.text}"
+                        })
+                        
+                except Exception as e:
+                    logger.error(f"Error creating user {user_data['email']}: {str(e)}")
+                    failed_users.append({
+                        "email": user_data["email"],
+                        "error": str(e)
+                    })
+            
+            # Create institute memberships for KevlarAI and provision users
+            all_users_to_provision = []
+            
+            # Add newly created users
+            if created_users:
+                all_users_to_provision.extend(created_users)
+                self._create_institute_memberships(created_users)
+            
+            # Also provision existing users that failed due to email_exists
+            existing_users = []
+            for failed_user in failed_users:
+                if "email_exists" in failed_user.get("error", ""):
+                    # Get the existing user ID from Supabase
+                    existing_user_id = self._get_existing_user_id(failed_user["email"])
+                    if existing_user_id:
+                        existing_users.append({
+                            "id": existing_user_id,
+                            "email": failed_user["email"],
+                            "user_type": self._get_user_type_from_email(failed_user["email"]),
+                            "username": self._get_username_from_email(failed_user["email"])
+                        })
+            
+            if existing_users:
+                logger.info(f"Found {len(existing_users)} existing users to provision")
+                all_users_to_provision.extend(existing_users)
+                self._create_institute_memberships(existing_users)
+            
+            # Provision all users (new and existing)
+            if all_users_to_provision:
+                self._provision_users(all_users_to_provision)
+            
+            logger.info(f"Demo users creation completed: {len(created_users)} created, {len(failed_users)} failed")
+            
+            return {
+                "success": True,
+                "message": f"Successfully created {len(created_users)} demo users",
+                "created_users": created_users,
+                "failed_users": failed_users
+            }
+            
+        except Exception as e:
+            logger.error(f"Error creating demo users: {str(e)}")
+            return {
+                "success": False,
+                "message": f"Error creating demo users: {str(e)}"
+            }
+    
+    def _create_institute_memberships(self, users: list) -> None:
+        """Create institute memberships for users in KevlarAI"""
+        logger.info("Creating institute memberships for demo users...")
+        
+        try:
+            # Get KevlarAI institute ID
+            response = self._supabase_request_with_retry(
+                'get',
+                f"{self.supabase_url}/rest/v1/institutes",
+                headers=self.supabase_headers,
+                params={
+                    "select": "id",
+                    "name": "eq.KevlarAI"
+                }
+            )
+            
+            if response.status_code != 200:
+                logger.warning("Could not get KevlarAI institute ID for memberships")
+                return
+            
+            institutes = response.json()
+            if not institutes:
+                logger.warning("KevlarAI institute not found for memberships")
+                return
+            
+            institute_id = institutes[0]["id"]
+            
+            # Get user profile IDs
+            for user in users:
+                try:
+                    profile_response = self._supabase_request_with_retry(
+                        'get',
+                        f"{self.supabase_url}/rest/v1/profiles",
+                        headers=self.supabase_headers,
+                        params={
+                            "select": "id",
+                            "email": f"eq.{user['email']}"
+                        }
+                    )
+                    
+                    if profile_response.status_code == 200:
+                        profiles = profile_response.json()
+                        if profiles:
+                            profile_id = profiles[0]["id"]
+                            
+                            # Create membership
+                            membership_data = {
+                                "profile_id": profile_id,
+                                "institute_id": institute_id,
+                                "role": user["user_type"]
+                            }
+                            
+                            membership_response = self._supabase_request_with_retry(
+                                'post',
+                                f"{self.supabase_url}/rest/v1/institute_memberships",
+                                headers=self.supabase_headers,
+                                json=membership_data
+                            )
+                            
+                            if membership_response.status_code in (200, 201):
+                                logger.info(f"Created membership for {user['email']} in KevlarAI")
+                            else:
+                                logger.warning(f"Failed to create membership for {user['email']}: {membership_response.text}")
+                                
+                except Exception as e:
+                    logger.warning(f"Error creating membership for {user['email']}: {str(e)}")
+                    
+        except Exception as e:
+            logger.warning(f"Error creating institute memberships: {str(e)}")
+
+    def _get_existing_user_id(self, email: str) -> str:
+        """Get the user ID for an existing user by email"""
+        try:
+            response = self._supabase_request_with_retry(
+                'get',
+                f"{self.supabase_url}/rest/v1/profiles",
+                headers=self.supabase_headers,
+                params={
+                    "select": "id",
+                    "email": f"eq.{email}"
+                }
+            )
+            
+            if response.status_code == 200:
+                profiles = response.json()
+                if profiles and len(profiles) > 0:
+                    return profiles[0].get("id")
+            
+            logger.warning(f"Could not find existing user ID for {email}")
+            return None
+            
+        except Exception as e:
+            logger.warning(f"Error getting existing user ID for {email}: {str(e)}")
+            return None
+    
+    def _get_user_type_from_email(self, email: str) -> str:
+        """Get user type from email based on demo user definitions"""
+        if "teacher" in email:
+            return "teacher"
+        elif "student" in email:
+            return "student"
+        return "teacher"  # default
+    
+    def _get_username_from_email(self, email: str) -> str:
+        """Get username from email based on demo user definitions"""
+        username_map = {
+            "[email protected]": "sarah.chen",
+            "[email protected]": "marcus.rodriguez", 
+            "[email protected]": "alex.thompson",
+            "[email protected]": "jordan.lee"
+        }
+        return username_map.get(email, email.split("@")[0])
+
+    def _provision_users(self, users: list) -> None:
+        """Provision Neo4j databases for the created demo users."""
+        for user in users:
+            user_id = user.get("id")
+            if not user_id:
+                continue
+            try:
+                self.provisioning_service.ensure_user(user_id)
+                logger.info(f"Provisioned Neo4j resources for {user.get('email')}")
+            except Exception as exc:
+                logger.warning(f"Failed to provision Neo4j resources for {user.get('email')}: {exc}")
+    
+    def _supabase_request_with_retry(self, method, url, **kwargs):
+        """Make a request to Supabase with retry logic"""
+        max_retries = 3
+        retry_delay = 2  # seconds
+        
+        for attempt in range(max_retries):
+            try:
+                if method.lower() == 'get':
+                    response = requests.get(url, **kwargs)
+                elif method.lower() == 'post':
+                    response = requests.post(url, **kwargs)
+                elif method.lower() == 'put':
+                    response = requests.put(url, **kwargs)
+                elif method.lower() == 'delete':
+                    response = requests.delete(url, **kwargs)
+                else:
+                    raise ValueError(f"Unsupported HTTP method: {method}")
+                
+                # If successful or client error (4xx), don't retry
+                if response.status_code < 500:
+                    return response
+                
+                # Server error (5xx), retry after delay
+                logger.warning(f"Supabase server error (attempt {attempt+1}/{max_retries}): {response.status_code} - {response.text}")
+                time.sleep(retry_delay * (attempt + 1))  # Exponential backoff
+                
+            except requests.RequestException as e:
+                logger.warning(f"Supabase request exception (attempt {attempt+1}/{max_retries}): {str(e)}")
+                if attempt == max_retries - 1:
+                    raise
+                time.sleep(retry_delay * (attempt + 1))
+        
+        # If we get here, all retries failed with server errors
+        raise requests.RequestException(f"Failed after {max_retries} attempts to {method} {url}")
+
+def initialize_demo_users() -> Dict[str, Any]:
+    """Initialize demo users"""
+    logger.info("Starting demo users initialization...")
+    
+    supabase_url = os.getenv("SUPABASE_URL")
+    service_role_key = os.getenv("SERVICE_ROLE_KEY")
+    
+    if not supabase_url or not service_role_key:
+        return {"success": False, "message": "Missing SUPABASE_URL or SERVICE_ROLE_KEY environment variables"}
+    
+    initializer = DemoUsersInitializer(supabase_url, service_role_key)
+    
+    # Create demo users
+    result = initializer.create_demo_users()
+    
+    if result["success"]:
+        logger.info("Demo users initialization completed successfully")
+    else:
+        logger.error(f"Demo users initialization failed: {result['message']}")
+    
+    return result

run/initialization/infrastructure.py

@@ -0,0 +1,58 @@
+"""
+Infrastructure initialization module for ClassroomCopilot
+Handles Neo4j database, schema, calendar structure, and Supabase storage buckets
+"""
+import os
+from typing import Dict, Any
+from modules.logger_tool import initialise_logger
+
+logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
+
+def initialize_infrastructure() -> Dict[str, Any]:
+    """
+    Initialize infrastructure: Neo4j database, schema, calendar structure, and Supabase buckets
+    
+    Returns:
+        dict: Result status and message
+    """
+    logger.info("Starting infrastructure initialization...")
+    
+    try:
+        # 1. Initialize Neo4j database, schema, and calendar structure
+        logger.info("Step 1: Initializing Neo4j infrastructure...")
+        from .neo4j import initialize_neo4j
+        neo4j_result = initialize_neo4j()
+        
+        if not neo4j_result["success"]:
+            return {
+                "success": False,
+                "message": f"Neo4j infrastructure initialization failed: {neo4j_result['message']}"
+            }
+        
+        # 2. Initialize Supabase storage buckets
+        logger.info("Step 2: Initializing Supabase storage buckets...")
+        from .buckets import initialize_buckets
+        buckets_result = initialize_buckets()
+        
+        if not buckets_result["success"]:
+            return {
+                "success": False,
+                "message": f"Storage buckets initialization failed: {buckets_result['message']}"
+            }
+        
+        logger.info("Infrastructure initialization completed successfully!")
+        return {
+            "success": True,
+            "message": "Infrastructure initialization completed successfully",
+            "neo4j": neo4j_result,
+            "buckets": buckets_result
+        }
+        
+    except Exception as e:
+        error_msg = f"Failed to initialize infrastructure: {str(e)}"
+        logger.error(error_msg)
+        return {
+            "success": False,
+            "message": error_msg,
+            "error": str(e)
+        }

run/initialization/manager.py

@@ -1,146 +0,0 @@
-"""
-Initialization manager for ClassroomCopilot
-"""
-import os
-import json
-from typing import Dict
-import requests
-
-from modules.logger_tool import initialise_logger
-
-logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
-
-class InitializationManager:
-    def __init__(self):
-        self.init_dir = os.getenv("BACKEND_INIT_PATH")
-        self.status_file = os.path.join(self.init_dir, "status.json")
-        self.data_dir = os.path.join(self.init_dir, "data")
-        self.supabase_url = os.getenv("SUPABASE_URL")
-        
-        # Ensure directories exist
-        os.makedirs(self.init_dir, exist_ok=True)
-        os.makedirs(self.data_dir, exist_ok=True)
-        
-        self.supabase_headers = {
-            "apikey": os.getenv("SERVICE_ROLE_KEY"),
-            "Authorization": f"Bearer {os.getenv('SERVICE_ROLE_KEY')}",
-            "Content-Type": "application/json"
-        }
-        
-        # Define default status structure
-        self.default_status = {
-            "super_admin_created": False,
-            "admin_token_obtained": False,
-            "storage": {
-                "initialized": False,
-                "buckets": {
-                    "cc.users": False,
-                    "cc.institutes": False
-                }
-            },
-            "neo4j": {
-                "initialized": False,
-                "database_created": False,
-                "schema_initialized": False,
-                "schools_imported": False
-            },
-            "completed": False,
-            "timestamp": None,
-            "steps": []
-        }
-        
-        self.status = self._load_status()
-
-    def _load_status(self) -> Dict:
-        """Load or create initialization status"""
-        try:
-            with open(self.status_file, "r") as f:
-                status = json.load(f)
-            
-            # Update with any missing keys
-            def update_dict(current: Dict, default: Dict) -> Dict:
-                for key, value in default.items():
-                    if key not in current:
-                        current[key] = value
-                    elif isinstance(value, dict) and isinstance(current[key], dict):
-                        current[key] = update_dict(current[key], value)
-                return current
-            
-            status = update_dict(status, self.default_status)
-            self._save_status(status)
-            return status
-            
-        except (FileNotFoundError, json.JSONDecodeError):
-            self._save_status(self.default_status)
-            return self.default_status.copy()
-
-    def _save_status(self, status: Dict) -> None:
-        """Save status to file"""
-        os.makedirs(os.path.dirname(self.status_file), exist_ok=True)
-        with open(self.status_file, "w") as f:
-            json.dump(status, f, indent=2)
-
-    def check_admin_exists(self) -> bool:
-        """Check if super admin already exists"""
-        try:
-            responseURL = f"{self.supabase_url}/auth/v1/admin/users"
-            logger.info(f"Checking admin existence at: {responseURL}")
-            response = requests.get(
-                responseURL,
-                headers=self.supabase_headers
-            )
-            
-            if response.status_code != 200:
-                logger.error(f"Error checking admin existence: {response.status_code}")
-                return False
-            
-            data = response.json()
-            # Fix: response format is {'users': [...], 'aud': 'authenticated'}
-            users = data.get('users', [])
-            if not isinstance(users, list):
-                logger.error(f"Unexpected users format: {users}")
-                return False
-                
-            admin_email = os.getenv('ADMIN_EMAIL')
-            
-            # Check for admin in users
-            admin_user = next(
-                (user for user in users 
-                 if user.get("email") == admin_email 
-                 and user.get("app_metadata", {}).get("role") == "supabase_admin"),
-                None
-            )
-            
-            if admin_user:
-                logger.info(f"Super admin {admin_email} already exists")
-                return True
-                
-            return False
-            
-        except Exception as e:
-            logger.error(f"Error checking admin existence: {str(e)}")
-            return False
-
-    def check_initialization_needed(self) -> bool:
-        """Check if initialization is needed"""
-        # First check if admin exists
-        if self.check_admin_exists():
-            logger.info("Super admin exists, skipping initialization")
-            return False
-            
-        # Then check status file
-        if self.status.get("completed"):
-            logger.info("Initialization already completed")
-            return False
-            
-        # Check if any step needs completion
-        incomplete = not all(
-            v for k, v in self.status.items() 
-            if k not in ("timestamp", "steps")
-        )
-        
-        if incomplete:
-            logger.info("Incomplete initialization detected")
-            return True
-            
-        return False 

run/initialization/neo4j.py

@@ -0,0 +1,169 @@
+"""
+Neo4j database initialization module for ClassroomCopilot
+Handles database creation, schema setup, and calendar structure
+"""
+import os
+import json
+import time
+from typing import Dict, Any, Optional
+from datetime import datetime, timedelta
+from modules.database.services.neo4j_service import Neo4jService
+from modules.database.init.init_calendar import create_calendar
+from modules.logger_tool import initialise_logger
+
+logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
+
+class Neo4jInitializer:
+    """Handles Neo4j database initialization including database, schema, and calendar structure"""
+    
+    def __init__(self):
+        self.neo4j_service = Neo4jService()
+        # Use lowercase to avoid case sensitivity issues
+        self.db_name = os.getenv("NEO4J_CC_DB", "classroomcopilot")
+    
+    def initialize_database(self) -> Dict[str, Any]:
+        """Initialize the single ClassroomCopilot database"""
+        logger.info(f"Initializing Neo4j database: {self.db_name}")
+        
+        try:
+            # Create the main database
+            result = self.neo4j_service.create_database(self.db_name)
+            if result["status"] != "success":
+                return {
+                    "success": False,
+                    "message": f"Failed to create {self.db_name} database: {result['message']}"
+                }
+            
+            logger.info(f"Successfully created {self.db_name} database")
+            
+            # Wait for database to be fully available
+            logger.info("Waiting for database to be fully available...")
+            time.sleep(5)  # Wait 5 seconds for database to be ready
+            
+            # Verify database exists and is accessible
+            max_retries = 10
+            retry_delay = 2
+            for attempt in range(max_retries):
+                try:
+                    # Try to check if the database exists to verify it's ready
+                    test_result = self.neo4j_service.check_database_exists(self.db_name)
+                    if test_result["status"] == "success" and test_result["exists"]:
+                        logger.info(f"Database {self.db_name} is ready and accessible")
+                        break
+                    else:
+                        logger.info(f"Database not ready yet (attempt {attempt + 1}/{max_retries}), waiting {retry_delay}s...")
+                        time.sleep(retry_delay)
+                except Exception as e:
+                    if attempt < max_retries - 1:
+                        logger.info(f"Database existence check failed (attempt {attempt + 1}/{max_retries}), waiting {retry_delay}s...")
+                        time.sleep(retry_delay)
+                    else:
+                        logger.warning(f"Database existence check failed after {max_retries} attempts: {str(e)}")
+                        # Continue anyway as the database was created successfully
+            
+            return {
+                "success": True,
+                "message": f"Successfully created {self.db_name} database",
+                "result": result
+            }
+            
+        except Exception as e:
+            logger.error(f"Error creating database: {str(e)}")
+            return {
+                "success": False,
+                "message": f"Error creating database: {str(e)}"
+            }
+    
+    def initialize_schema(self) -> Dict[str, Any]:
+        """Initialize Neo4j schema on the ClassroomCopilot database"""
+        logger.info(f"Initializing Neo4j schema on {self.db_name}...")
+        
+        try:
+            result = self.neo4j_service.initialize_schema(self.db_name)
+            if result["status"] != "success":
+                return {
+                    "success": False,
+                    "message": f"Failed to initialize schema on {self.db_name}: {result['message']}"
+                }
+            
+            logger.info(f"Successfully initialized schema on {self.db_name}")
+            return {
+                "success": True,
+                "message": f"Successfully initialized schema on {self.db_name}",
+                "result": result
+            }
+            
+        except Exception as e:
+            logger.error(f"Error initializing schema: {str(e)}")
+            return {
+                "success": False,
+                "message": f"Error initializing schema: {str(e)}"
+            }
+    
+    def initialize_calendar_structure(self) -> Dict[str, Any]:
+        """Initialize the calendar structure with days, weeks, months, and years"""
+        logger.info("Initializing calendar structure...")
+        
+        try:
+            # Create calendar structure for the next 1 year
+            start_date = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
+            end_date = start_date + timedelta(days=365)  # 1 year
+            
+            # Create calendar structure directly (no calendar node needed)
+            calendar_nodes = create_calendar(
+                db_name=self.db_name,
+                start_date=start_date,
+                end_date=end_date,
+                time_chunk_node_length=0,
+                storage_tools=None
+            )
+            
+            if calendar_nodes:
+                logger.info("Calendar structure created successfully")
+                return {
+                    "success": True,
+                    "message": "Calendar structure created successfully",
+                    "calendar_nodes": calendar_nodes
+                }
+            else:
+                return {
+                    "success": False,
+                    "message": "Failed to create calendar structure"
+                }
+            
+        except Exception as e:
+            logger.error(f"Error creating calendar structure: {str(e)}")
+            return {
+                "success": False,
+                "message": f"Error creating calendar structure: {str(e)}"
+            }
+
+def initialize_neo4j() -> Dict[str, Any]:
+    """Initialize Neo4j database setup"""
+    logger.info("Starting Neo4j database initialization...")
+    
+    initializer = Neo4jInitializer()
+    
+    # 1. Create the single database
+    db_result = initializer.initialize_database()
+    if not db_result["success"]:
+        return db_result
+    
+    # 2. Initialize schema
+    schema_result = initializer.initialize_schema()
+    if not schema_result["success"]:
+        return schema_result
+    
+    # 3. Initialize calendar structure
+    calendar_result = initializer.initialize_calendar_structure()
+    if not calendar_result["success"]:
+        return calendar_result
+    
+    logger.info("Neo4j database initialization completed successfully")
+    return {
+        "success": True,
+        "message": "Neo4j database initialization completed successfully",
+        "database": db_result,
+        "schema": schema_result,
+        "calendar": calendar_result
+    } 

run/routers.py

@@ -7,20 +7,25 @@ from fastapi import FastAPI
 from routers.health import router as health_router
 from routers.msgraph import router_onenote
 from routers.dev.tests import timetable_test
-from routers.admin_routes import router as admin_routes_router
 from routers.database.init import entity_init, calendar, timetables, curriculum, get_data, schools
-from routers.database.tools import get_nodes, get_nodes_and_edges, tldraw_filesystem, get_events, calendar_structure_router, default_nodes_router, worker_structure_router
+from routers.database.tools import get_nodes, get_nodes_and_edges, tldraw_filesystem, tldraw_supabase_storage, get_events, calendar_structure_router, default_nodes_router, worker_structure_router
+from routers.database.files import cabinets as cabinets_router
+from routers.database.files import files as files_router
+from routers.simple_upload import router as simple_upload_router
 from routers.assets import powerpoint, word, pdf
 from routers.llm.private.ollama import ollama
-from routers.llm.public.openai import openai
+#from routers.llm.public.openai import openai
 from routers.connections.arbor_router import router as arbor_router
 from routers.langchain.neo4j_graph_qa import router as graph_qa_router
-from routers.langchain.interactive_langgraph_query import router as interactive_langgraph_query_router
+#from routers.langchain.interactive_langgraph_query import router as interactive_langgraph_query_router
 from routers.rpi import rpi_whisperlive_client
 from routers.external import youtube
 from routers.solid.pod_provisioner import router as solid_pod_router
 from routers.dev.document_conversion import router as document_conversion_router
 from routers.dev.test_analysis import router as test_analysis_router
+from routers.queue_management import router as queue_management_router
+from routers.maintenance.redis_admin import router as redis_admin_router
+from routers import provisioning as provisioning_router
 
 def register_routes(app: FastAPI):
     logger.info("Starting to register routes...")
@@ -28,9 +33,6 @@ def register_routes(app: FastAPI):
     # Health check route
     app.include_router(health_router, prefix="/health", tags=["Health"])
 
-    # Admin Routes
-    app.include_router(admin_routes_router, prefix="/admin", tags=["Admin"])
-
     # Microsoft Graph Routes
     app.include_router(router_onenote.router, prefix="/msgraph", tags=["Microsoft Graph"])
 
@@ -52,6 +54,12 @@ def register_routes(app: FastAPI):
 
     # Database Filesystem Routes
     app.include_router(tldraw_filesystem.router, prefix="/database/tldraw_fs", tags=["TLDraw Filesystem"])
+    app.include_router(tldraw_supabase_storage.router, prefix="/database/tldraw_supabase", tags=["TLDraw Supabase Storage"])
+    app.include_router(cabinets_router.router, prefix="/database", tags=["Cabinets"])
+    app.include_router(files_router.router, prefix="/database", tags=["Files"])
+    
+    # Simple Upload Routes (no auto-processing)
+    app.include_router(simple_upload_router, prefix="/simple-upload", tags=["Simple Upload"])
     
     # Assets Routes
     app.include_router(powerpoint.router, prefix="/assets/powerpoint", tags=["PowerPoint"])
@@ -60,11 +68,11 @@ def register_routes(app: FastAPI):
 
     # LLM Routes
     app.include_router(ollama.router, prefix="/llm/private/ollama", tags=["LLM"])
-    app.include_router(openai.router, prefix="/llm/public/openai", tags=["LLM"])
+    #app.include_router(openai.router, prefix="/llm/public/openai", tags=["LLM"])
 
     # Langchain Routes
     app.include_router(graph_qa_router, prefix="/langchain/graph_qa", tags=["Langchain"])
-    app.include_router(interactive_langgraph_query_router, prefix="/langchain/interactive_langgraph_query", tags=["Langchain"])
+    #app.include_router(interactive_langgraph_query_router, prefix="/langchain/interactive_langgraph_query", tags=["Langchain"])
 
     # External Routes
     app.include_router(youtube.router, prefix="/external", tags=["External"])
@@ -84,5 +92,14 @@ def register_routes(app: FastAPI):
     # Test Analysis Routes
     app.include_router(test_analysis_router, prefix="/dev/tests", tags=["Test Analysis"])
 
+    # Queue Management Routes
+    app.include_router(queue_management_router, prefix="/queue", tags=["Queue Management"])
+
+    # Maintenance Routes
+    app.include_router(redis_admin_router, prefix="/maintenance/redis", tags=["Maintenance","Redis"])
+
+    # Provisioning Routes
+    app.include_router(provisioning_router.router)
+
     # Test Routes
     app.include_router(timetable_test.router, prefix="/tests", tags=["Tests"])

run/setup.py

@@ -12,12 +12,7 @@ from fastapi import FastAPI
 def setup_cors(app: FastAPI) -> None:
     """Configure CORS middleware for the FastAPI application"""
     from fastapi.middleware.cors import CORSMiddleware
-    origins = [
-        os.getenv('SITE_URL'),
-        os.getenv('APP_SITE_URL'),
-        os.getenv('APP_API_URL'),
-        os.getenv('APP_ADMIN_URL'),
-    ]
+    origins = [o.strip() for o in os.getenv('CORS_SITE_URL','').split(',') if o.strip()]
     logger.info(f"Setting up CORS with origins: {origins}")
 
     app.add_middleware(

run/tests/.archive/pytest_init_calendar.py

@@ -0,0 +1,64 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'pytest_calendar'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+import modules.database.tools.neo4j_driver_tools as driver_tools
+import modules.database.tools.neontology_tools as neon
+import pytest
+from fastapi.testclient import TestClient
+from routers.database.init.calendar import router
+from fastapi import FastAPI
+from datetime import datetime, timedelta
+
+app = FastAPI()
+app.include_router(router)
+
+client = TestClient(app)
+
+# Define a list of date ranges for testing
+date_ranges = [
+    (datetime.now(), datetime.now() + timedelta(days=1)),  # 1 day
+    (datetime.now(), datetime.now() + timedelta(days=7)),  # 1 week
+    (datetime.now(), datetime.now() + timedelta(days=30)), # 1 month
+    (datetime.now(), datetime.now() + timedelta(days=183)),# 6 months
+    (datetime.now(), datetime.now() + timedelta(days=365)) # 1 year
+]
+
+# Fixture to manage database name increment
+@pytest.fixture(scope="function", autouse=True)
+def increment_db_name_counter(request):
+    if not hasattr(request.module, "db_name_counter"):
+        request.module.db_name_counter = 0
+    request.module.db_name_counter += 1
+    return request.module.db_name_counter
+
+@pytest.mark.parametrize("start_date, end_date", date_ranges)
+def test_create_calendar(start_date, end_date, increment_db_name_counter):
+    db_name = f"test_create_calendar_db_{increment_db_name_counter}"
+    neo_safe_db_name = db_name.replace("_", "")
+    logging.info(f"Creating calendar for {db_name} from {start_date} to {end_date}")
+    logging.info(f"Creating calendar for {db_name} from {start_date} to {end_date}")
+    response = client.post(
+        "/create-calendar",
+        params={
+            "db_name": neo_safe_db_name,
+            "start_date": start_date.strftime('%Y-%m-%d'),
+            "end_date": end_date.strftime('%Y-%m-%d')
+        }
+    )
+    assert response.status_code == 200
+    response_json = response.json()
+    assert "calendar_year_nodes" in response_json and response_json["calendar_year_nodes"] != 0
+    assert "calendar_month_nodes" in response_json and response_json["calendar_month_nodes"] != 0
+    assert "calendar_week_nodes" in response_json and response_json["calendar_week_nodes"] != 0
+    assert "calendar_day_nodes" in response_json and response_json["calendar_day_nodes"] != 0

run/tests/.archive/pytest_init_curriculum.py

@@ -0,0 +1,61 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'pytest_init_curriculum'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+import modules.database.tools.neo4j_driver_tools as driver_tools
+import modules.database.tools.neontology_tools as neon
+import pytest
+from fastapi.testclient import TestClient
+from routers.database.init.curriculum import router
+from fastapi import FastAPI
+
+app = FastAPI()
+app.include_router(router)
+
+client = TestClient(app)
+
+db_name = log_name.replace('_', '')
+excel_file = os.environ['EXCEL_CURRICULUM_FILE']
+
+driver = driver_tools.get_driver(database=db_name)
+neon.init_neontology_connection()
+    
+@pytest.fixture
+def sample_file():
+    # Use the existing Excel file to upload
+    file_path = excel_file
+    logging.info(f"Using sample file at {file_path}")
+    yield file_path
+
+def test_upload_curriculum(sample_file):
+    db_name = "test_curriculum_db"
+    with open(sample_file, "rb") as f:
+        response = client.post(
+            "/upload-curriculum",
+            files={"file": (excel_file, f, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")},
+            data={"db_name": db_name.replace('_', '')}
+        )
+    logging.info(f"Response status code: {response.status_code}")
+    logging.info(f"Response JSON: {response.json()}")
+    
+    assert response.status_code == 200
+    response_json = response.json()
+    logging.info(f"Response JSON keys: {response_json.keys()}")
+    
+    # Adjust the assertions based on the actual response structure
+    assert "status" in response_json or "12" in response_json
+    if "status" in response_json:
+        assert response_json["status"] == "Success"
+    else:
+        assert "created" in response_json["12"]
+        assert "merged" in response_json["12"]

run/tests/.archive/pytest_init_school_timetable.py

@@ -0,0 +1,54 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'pytest_timetable'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+import modules.database.tools.neo4j_driver_tools as driver_tools
+import modules.database.tools.neontology_tools as neon
+import pytest
+from fastapi.testclient import TestClient
+from routers.database.init.timetable import router
+from fastapi import FastAPI
+import pandas as pd
+
+app = FastAPI()
+app.include_router(router)
+
+client = TestClient(app)
+
+db_name = log_name.replace('_', '')
+excel_file = os.environ['EXCEL_TIMETABLE_FILE']
+    
+@pytest.fixture
+def sample_file():
+    # Use the existing Excel file to upload
+    file_path = excel_file
+    logging.info(f"Using sample file at {file_path}")
+    yield file_path
+
+def test_upload_school_timetable(sample_file):
+    db_name = "pytest_school_timetable_db"
+    with open(sample_file, "rb") as f:
+        response = client.post(
+            "/upload-school-timetable",
+            data={"db_name": db_name.replace('_', '')},
+            files={"file": (excel_file, f, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")}
+        )
+    logging.info(f"Response status code: {response.status_code}")
+    logging.info(f"Response JSON: {response.json()}")
+    
+    assert response.status_code == 200
+    response_json = response.json()
+    assert "calendar_nodes" in response_json
+    assert "school_timetable_nodes" in response_json
+    assert response_json["calendar_nodes"] is not None
+    assert response_json["school_timetable_nodes"] is not None

run/tests/.archive/pytest_init_user.py

@@ -0,0 +1,44 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'pytest_timetable'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+import modules.database.tools.neo4j_driver_tools as driver_tools
+import modules.database.tools.neontology_tools as neon
+import pytest
+from fastapi.testclient import TestClient
+from fastapi import FastAPI
+import pandas as pd
+
+# Import the router from entity_init.py
+from routers.database.init.entity_init import router
+
+app = FastAPI()
+app.include_router(router)
+
+client = TestClient(app)
+
+@pytest.mark.parametrize("username, email, user_id", [
+    ("user1", "[email protected]", "uuid1"),
+    ("user2", "[email protected]", "uuid2"),
+    ("user3", "[email protected]", "uuid3")
+])
+def test_create_user(username, email, user_id):
+    response = client.post(
+        "/create-user",
+        data={"username": username, "email": email, "user_id": user_id}
+    )
+    logging.info(f"Tested creating user {username}. Response status code: {response.status_code}")
+    response_json = response.json()
+    logging.info(f"Response JSON: {response_json}")
+
+    assert response.status_code == 200

run/tests/.archive/pytest_transcribe.py

@@ -0,0 +1,35 @@
+import os
+import pytest
+from fastapi.testclient import TestClient
+from main import app  # Adjust the import based on your project structure
+
+client = TestClient(app)
+
+@pytest.fixture(autouse=True)
+def setup_env():
+    os.environ["WHISPERLIVE_HOST"] = "localhost"
+    os.environ["WHISPERLIVE_PORT"] = "9090"
+
+def test_start_transcription():
+    user_id = "test_user"
+    response = client.post(f"/transcribe/live/start_transcription/{user_id}")
+    assert response.status_code == 200
+    assert response.json() == {"message": "Transcription started", "user_id": user_id}
+
+def test_handle_whisper_live_eos_utterance():
+    user_id = "test_user"
+    data = {
+        "utterance": "Hello, world!",
+        "start": 0,
+        "end": 1,
+        "eos": True
+    }
+    response = client.post(f"/transcribe/utterance/handle_whisper_live_eos_utterance/{user_id}", json=data)
+    assert response.status_code == 200
+    assert response.json() == {"message": "Utterance logged successfully"}
+
+def test_get_utterances():
+    user_id = "test_user"
+    response = client.get(f"/transcribe/utterance/get_utterances/{user_id}")
+    assert response.status_code == 200
+    assert "utterances" in response.json()

run/tests/_transcribe.py

@@ -0,0 +1,34 @@
+from modules.whisper_live.client import TranscriptionClient
+import os
+import time
+
+def setup_directories(user_dir, user_id):
+    user_transcript_dir = f"{user_dir}/{user_id}/transcripts"
+    if not os.path.exists(user_transcript_dir):
+        os.makedirs(user_transcript_dir)
+    return user_transcript_dir
+
+def timestamped_callback(text, is_final):
+    if is_final:
+        print(f"Timestamp: {time.strftime('%H:%M:%S')}, Transcription: {text}")
+
+def main():
+    user_dir = "../../data/users"
+    user_id = "kcar"
+    user_transcript_dir = setup_directories(user_dir, user_id)
+
+    client = TranscriptionClient(
+        "localhost",
+        9090,
+        lang="en",
+        translate=False,
+        use_vad=True,
+        save_output_recording=True,
+        output_recording_filename=f"{user_transcript_dir}/output_recording.wav",
+        output_transcription_path=f"{user_transcript_dir}/output.srt",
+    )
+
+    client()
+
+if __name__ == "__main__":
+    main()

run/tests/conftest.py

@@ -0,0 +1,3 @@
+from modules.logger_tool import PytestFormatter
+
+pytest_formatter = PytestFormatter()

run/tests/formatting.py

@@ -0,0 +1,53 @@
+def ascii_header():
+    return r"""
+==================================================
+=                                                =
+=        _______                                 =
+=       |       |        _                       =
+=       |       |____   | |                      =
+=       |       /    |  | |   ___  __ _ _ __     =
+=       |      |     |  | |  / _ \/ _` | '_ \    =
+=       |       \____|  | | |  __/ (_| | | | |   =
+=       |_______|       |_|  \___|\__,_|_| |_|   =
+=                                                =
+=                                                =
+=        _________                               =
+=       |         |                              =
+=       |  BOOK   |                              =
+=       |_________|                              =
+=                                                =
+=                                                =
+=          /\                                    =
+=         /  \                                   =
+=        /____\                                  =
+=       /      \                                 =
+=      /        \                                =
+=     /__________\                               =
+=                                                =
+=                                                =
+=    _____________                               =
+=   |             |                              =
+=   |  COMPUTER   |                              =
+=   |_____________|                              =
+=                                                =
+=                                                =
+=                                                =
+=      _________                                 =
+=     |         |                                =
+=     |  TEACH  |                                =
+=     |_________|                                =
+=                                                =
+=                                                =
+=        ____                                    =
+=       /    \                                   =
+=      /      \                                  =
+=     /        \                                 =
+=    /__________\                                =
+=                                                =
+=                                                =
+==================================================
+=                                                =
+=             classroom-copilot.ai               =
+=                                                =
+==================================================
+"""

run/tests/pytest_arbor.py

@@ -0,0 +1,33 @@
+import os
+import requests
+import pytest
+import json
+
+# Define the base URL and the tokens
+base_url = f"{os.environ.get('APP_API_URL')}/arbor/data"
+tokens = {
+    1: os.getenv("KS3_COURSE_CLASS_MEMBERSHIP_AUTH"),
+    2: os.getenv("TEACHING_GROUP_MEMBERSHIPS_2023_2024_AUTH"),
+    3: os.getenv("SCHEDULED_TIMETABLE_SLOTS_AUTH"),
+    4: os.getenv("BEHAVIOURAL_INCIDENTS_REPORTING_AUTH"),
+    5: os.getenv("Y7_LESSON_TIMETABLE_AUTH")
+}
+
+@pytest.mark.parametrize("id", [1, 2, 3, 4, 5])
+def test_fetch_arbor_data(id):
+    token = tokens.get(id)
+    if not token:
+        pytest.fail(f"Token for ID {id} is not set")
+    
+    endpoint = f"{base_url}/{id}"
+    headers = {"Authorization": f"Basic {token}"}
+    params = {"token": token}
+    
+    response = requests.get(endpoint, headers=headers, params=params)
+    
+    if response.status_code == 200:
+        print(json.dumps(response.json()))
+        assert response.status_code == 200
+    else:
+        pytest.fail(f"Failed for ID {id}: {response.status_code} {response.text}")
+

run/tests/pytest_init_curriculum_graph_qa.py

@@ -0,0 +1,123 @@
+import os
+import json
+import requests
+import pytest
+from dotenv import load_dotenv, find_dotenv
+from .formatting import ascii_header
+import modules.logger_tool as logger
+
+load_dotenv(find_dotenv())
+
+log_name = 'api_router_graph_qa_test'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+
+@pytest.fixture(scope="module")
+def config():
+    return {
+        "database": "cc.institutes.kevlarai",
+        "top_k": 40,
+        "model": "gpt-4o",
+        "temperature": 0,
+        "verbose": False,
+        "return_intermediate_steps": True,
+        "return_direct": False,
+        "validate_cypher": True,
+        "model_type": "openai"  # Default model_type
+    }
+
+def load_test_cases():
+    with open('backend/app/tests/test_inputs/init_curriculum_db_cases.json', 'r') as f:
+        return json.load(f)
+
+test_cases = load_test_cases()
+
+@pytest.mark.parametrize("case", test_cases["curriculum_cases"])
+def test_curriculum_cases(case, config):
+    assert run_test_case(case, config)
+
+@pytest.mark.parametrize("case", test_cases["include_exclude_cases"]["includes"])
+def test_include_cases(case, config):
+    assert run_test_case(case, config)
+
+@pytest.mark.parametrize("case", test_cases["include_exclude_cases"]["excludes"])
+def test_exclude_cases(case, config):
+    assert run_test_case(case, config)
+
+@pytest.mark.parametrize("case", test_cases["include_exclude_cases"]["includes_excludes"])
+def test_include_exclude_cases(case, config):
+    assert run_test_case(case, config)
+
+def run_test_case(case, config):
+    logging.info(f"Starting test case with prompt: {case['prompt']}")
+    url = f"{os.environ['APP_API_URL']}/langchain/graph_qa/prompt"
+    params = {
+        "database": config["database"],
+        "prompt": case["prompt"],
+        "top_k": config["top_k"],
+        "model": config["model"],
+        "temperature": config["temperature"],
+        "verbose": config["verbose"],
+        "return_intermediate_steps": config["return_intermediate_steps"],
+        "exclude_types": case["exclude_types"],
+        "include_types": case["include_types"],
+        "return_direct": config["return_direct"],
+        "validate_cypher": config["validate_cypher"],
+        "model_type": config["model_type"]
+    }
+    logging.info(f"Constructed URL: {url}")
+    logging.info(f"Parameters: {params}")
+
+    try:
+        logging.info("Sending request to API...")
+        response = requests.get(url, params=params)
+        logging.info(f"HTTP Response Status: {response.status_code}")
+        response.raise_for_status()
+        data = response.json()
+        logging.info(f"Response Data: {data}")
+
+        # Log detailed test execution information
+        logging.info("==================================================")
+        logging.info("=                Test Execution                  =")
+        logging.info("==================================================")
+        logging.info(f"= Prompt: {data.get('query', 'N/A')}")
+        logging.info("=                                                =")
+        logging.info(f"= Query: \n{data.get('intermediate_steps', [{'query': 'N/A'}])[0].get('query', 'N/A')}")
+        logging.info("=                                                =")
+        logging.info("==================================================")
+
+        # Determine if the test passed or failed
+        response_text = data.get('result', 'N/A')
+        context = data.get('intermediate_steps', [{'context': 'N/A'}])[1].get('context', 'N/A')
+        if "I don't know" in response_text or not context:
+            logging.error("==================================================")
+            logging.error("=                  XX Test Failed XX             =")
+            logging.error("==================================================")
+            logging.error(f"= Prompt: {case['prompt']}")
+            logging.error(f"= Context: {context}")
+            logging.error(f"= Response: {response_text}")
+            logging.error("==================================================")
+            return False
+        else:
+            logging.info("==================================================")
+            logging.info("=                  ** Test Passed **             =")
+            logging.info("==================================================")
+            logging.info(f"= Prompt: {case['prompt']}")
+            logging.info(f"= Context: {context}")
+            logging.info(f"= Response: {response_text}")
+            logging.info("==================================================")
+            return True
+    except requests.exceptions.RequestException as e:
+        logging.error("==================================================")
+        logging.error("=                  ERROR                         =")
+        logging.error("==================================================")
+        logging.error(f"Error: {e}")
+        logging.error("==================================================")
+        return False

run/tests/pytest_init_x.py

@@ -0,0 +1,294 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import pytest
+from fastapi.testclient import TestClient
+from fastapi import FastAPI
+import json
+import modules.logger_tool as logger
+from routers.database.init.entity_init import router as entity_init_router
+from routers.database.init.timetables import router as timetables_router
+from routers.database.init.curriculum import router as curriculum_router
+from backend.modules.database.schemas.entities import SchoolNode, UserNode
+from modules.database.schemas.nodes.calendars import CalendarNode
+
+# Pytest configuration
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "school: mark test as part of school creation"
+    )
+    config.addinivalue_line(
+        "markers", "users: mark test as part of user creation"
+    )
+    config.addinivalue_line(
+        "markers", "timetable: mark test as part of timetable upload"
+    )
+    config.addinivalue_line(
+        "markers", "curriculum: mark test as part of curriculum upload"
+    )
+
+# Setup logging
+log_name = 'pytest_init_x'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+
+# Setup FastAPI app and test client
+app = FastAPI()
+app.include_router(entity_init_router)
+app.include_router(timetables_router)
+app.include_router(curriculum_router)
+client = TestClient(app)
+
+school_timetable_file = os.environ['EXCEL_TIMETABLE_FILE']
+school_curriculum_file = os.environ['EXCEL_CURRICULUM_FILE']
+
+@pytest.fixture(scope="module")
+def school_info():
+    db_name = "cc.institutes.devschool"
+    school_data = {
+        "db_name": db_name,
+        "uuid_string": "uuid1",
+        "name": "school1",
+        "website": "www.school1.com"
+    }
+    return school_data
+
+@pytest.fixture(scope="module")
+def created_school(school_info):
+    school_data = school_info
+    response = client.post("/create-school", data=school_data)
+    logging.info(f"Create school response: {response.json()}")
+    assert response.status_code == 200
+    logging.success("School created successfully")
+    
+    response_json = response.json()
+    school_node = SchoolNode(**response_json["school_node"])
+    
+    logging.success(f"School node created: {school_node}")
+    return school_node
+
+@pytest.mark.school
+def test_create_school(created_school):
+    school_node = created_school
+    assert school_node is not None
+
+@pytest.mark.users
+@pytest.mark.parametrize("user_type, expected_status", [
+    ("standard", 200),
+    ("developer", 200)
+])
+def test_create_non_school_user(user_type, expected_status):
+    db_name = "cc.users.devusers"
+    user_data = {
+        "user_type": user_type,
+        "user_name": f"test_{user_type}",
+        "user_email": f"test_{user_type}@example.com",
+        "user_id": f"{user_type}_uuid"
+    }
+    response = client.post("/create-user", data=user_data)
+    assert response.status_code == expected_status
+    logging.success(f"{user_type.capitalize()} user created successfully")
+
+@pytest.mark.users
+@pytest.mark.parametrize("user_type, expected_status", [
+    ("cc_email_school_admin", 200),
+    ("cc_email_teacher", 200),
+    ("cc_email_student", 200)
+])
+def test_create_school_user(created_school, user_type, expected_status):
+    school_node = created_school
+    worker_data = {
+        "cc_email_school_admin": {
+            "admin_code": "ADM001",
+            "admin_name_formal": "Mr. Admin",
+            "admin_email": "[email protected]"
+        },
+        "cc_email_teacher": {
+            "teacher_code": "TCH001",
+            "teacher_name_formal": "Ms. Teacher",
+            "teacher_email": "[email protected]"
+        },
+        "cc_email_student": {
+            "student_code": "STU001",
+            "student_name_formal": "Student Name",
+            "student_email": "[email protected]"
+        }
+    }
+    user_data = {
+        "user_type": user_type,
+        "user_name": f"test_{user_type}",
+        "user_email": f"test_{user_type}@example.com",
+        "user_id": f"{user_type}_uuid",
+        "school_uuid_string": school_node.uuid_string,
+        "school_name": school_node.name,
+        "school_website": school_node.website,
+        "school_node_storage_path": school_node.node_storage_path,
+        "worker_data": json.dumps(worker_data[user_type])
+    }
+    logging.info(f"Sending user data: {user_data}")
+    response = client.post("/create-user", data=user_data)
+    assert response.status_code == expected_status
+    logging.success(f"{user_type.capitalize()} user created successfully")
+
+def test_create_user_invalid_data():
+    invalid_user_data = {
+        "user_type": "invalid_type",
+        "user_name": "test_invalid",
+        "user_email": "[email protected]",
+        "user_id": "invalid_uuid"
+    }
+    response = client.post("/create-user", data=invalid_user_data)
+    assert response.status_code == 400
+    logging.success("Invalid user data handled correctly")
+
+@pytest.mark.users
+def test_create_school_user_without_school_node():
+    user_data = {
+        "user_type": "cc_email_teacher",
+        "user_name": "test_teacher_no_school",
+        "user_email": "[email protected]",
+        "user_id": "teacher_no_school_uuid"
+    }
+    response = client.post("/create-user", data=user_data)
+    assert response.status_code == 400
+    logging.success("School-related user without school_node handled correctly")
+
+@pytest.fixture
+def sample_file():
+    logging.info(f"Using sample file: {school_timetable_file}")
+    return school_timetable_file
+
+@pytest.mark.timetable
+def test_upload_school_timetable(created_school, sample_file):
+    school_node = created_school
+    with open(sample_file, "rb") as f:
+        response = client.post(
+            "/upload-school-timetable",
+            data={
+                "db_name": "cc.institutes.devschool",
+                "uuid_string": school_node.uuid_string,
+                "school_uuid": school_node.school_uuid,
+                "school_name": school_node.school_name,
+                "school_db_name": school_node.school_db_name,
+                "school_website": school_node.school_website,
+                "path": school_node.path
+            },
+            files={"file": (os.path.basename(sample_file), f, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")}
+        )
+    logging.info(f"Timetable upload response: {response.json()}")
+    assert response.status_code == 200
+    logging.success("Timetable uploaded successfully")
+    
+    response_json = response.json()
+    
+    for key in ["school_node", "school_calendar_nodes", "school_timetable_nodes"]:
+        assert key in response_json
+        logging.success(f"{key} present in response")
+    
+    school_node = SchoolNode(**response_json["school_node"])
+    calendar_node = CalendarNode(**response_json['school_calendar_nodes']['calendar_node'])
+    
+    logging.success(f"School node validated: {school_node}")
+    logging.success(f"Calendar node validated: {calendar_node}")
+    
+    for key in ["school_node", "school_calendar_nodes", "school_timetable_nodes"]:
+        assert response_json[key] is not None
+        logging.success(f"{key} is not None")
+    
+    logging.success("All assertions passed in test_upload_school_timetable")
+
+@pytest.fixture
+def curriculum_sample_file():
+    logging.info(f"Using curriculum sample file: {school_curriculum_file}")
+    return school_curriculum_file
+
+
+@pytest.mark.curriculum
+def test_upload_school_curriculum(created_school, curriculum_sample_file):
+    school_node = created_school
+    with open(curriculum_sample_file, "rb") as f:
+        response = client.post(
+            "/upload-school-curriculum",
+            data={
+                "db_name": "cc.institutes.devschool",
+                "school_uuid": school_node.school_uuid,
+                "school_name": school_node.school_name,
+                "school_website": school_node.school_website,
+                "school_path": school_node.path
+            },
+            files={"file": (os.path.basename(curriculum_sample_file), f, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")}
+        )
+    
+    assert response.status_code == 200
+    logging.success("Curriculum uploaded successfully")
+    
+    response_json = response.json()
+    
+    assert "curriculum_node" in response_json
+    assert "pastoral_node" in response_json
+    assert "key_stage_nodes" in response_json
+    assert "year_group_syllabus_nodes" in response_json
+    assert "topic_nodes" in response_json
+    assert "topic_lesson_nodes" in response_json
+    assert "statement_nodes" in response_json
+    
+    logging.success("All assertions passed in test_upload_school_curriculum")
+
+@pytest.mark.users
+@pytest.mark.timetable
+def test_create_kcar_user_and_upload_timetable(created_school):
+    school_node = created_school
+    user_data = {
+        "user_type": "cc_email_teacher",
+        "user_name": "K Car",
+        "user_email": "[email protected]",
+        "user_id": "kcar_uuid",
+        "school_uuid": school_node.school_uuid,
+        "school_name": school_node.school_name,
+        "school_website": school_node.school_website,
+        "school_path": school_node.path,
+        "worker_data": json.dumps({
+            "teacher_code": "KCAR",
+            "teacher_name_formal": "Mr. K Car",
+            "teacher_email": "[email protected]"
+        })
+    }
+    logging.info(f"Creating KCar user with data: {user_data}")
+    response = client.post("/create-user", data=user_data)
+    logging.info(f"KCar user creation response: {response.json()}")
+    assert response.status_code == 200
+    logging.success("KCar user created successfully")
+    kcar_user = UserNode(**response.json()["data"]["user_node"])
+
+    user_timetable_file = os.environ['KCAR_TIMETABLE_URL']
+    logging.info(f"User timetable file: {user_timetable_file}")
+    with open(user_timetable_file, "rb") as f:
+        logging.info(f"Uploading teacher timetable for K Car: {user_timetable_file}")
+        response = client.post(
+            "/upload-worker-timetable",
+            data={
+                "user_id": kcar_user.user_id,
+                "db_name": "cc.institutes.devschool"
+            },
+            files={"file": (os.path.basename(user_timetable_file), f, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")}
+        )
+        logging.info(f"Teacher timetable upload response: {response.json()}")
+    assert response.status_code == 200
+    logging.success("K Car teacher timetable uploaded successfully")
+    
+    response_json = response.json()
+    
+    assert response_json["message"] == "Teacher timetable initialized successfully"
+    
+    logging.success("All assertions passed in test_create_kcar_user_and_upload_timetable")
+
+def pytest_runtest_makereport(item, call):
+    if call.when == "call" and call.excinfo is None:
+        logging.success(f"Test passed: {item.name}")

run/tests/pytest_langgraph.py

@@ -0,0 +1,85 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'api_modules_interactive_langgraph_query'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+import pytest
+import requests
+
+# Define the URL of your FastAPI server
+BASE_URL = "http://localhost:8000"
+ENDPOINT = f"{BASE_URL}/api/langchain/interactive_langgraph_query/query"
+
+def send_query(query):
+    payload = {"query": query}
+    headers = {"Content-Type": "application/json"}
+    logging.info(f"Sending query to {ENDPOINT} with payload: {payload}")
+    
+    try:
+        response = requests.post(ENDPOINT, json=payload, headers=headers)
+        response.raise_for_status()
+        result = response.json()
+        logging.info(f"Received response from {ENDPOINT}: {result}")
+        return result
+    except requests.exceptions.RequestException as e:
+        logging.error(f"Error sending query to {ENDPOINT}: {str(e)}")
+        return {"error": str(e)}
+
+@pytest.mark.simple
+def test_simple_queries():
+    query = "Describe the relevance of Maidstone, England during the English Civil War."
+    logging.info(f"Running simple query test with query: {query}")
+    result = send_query(query)
+    
+    logging.info(f"Assertion 1: Checking for absence of error")
+    assert "error" not in result, f"Error in response: {result.get('error')}"
+    
+    logging.info(f"Assertion 2: Checking for presence of response")
+    assert "response" in result, "Response does not contain an answer"
+    
+    logging.info(f"Assertion 3: Checking for non-empty answer")
+    assert len(result["response"]) > 0, "Answer is empty"
+    
+    logging.info(f"All assertions passed. Response: {result['response'][:100]}...")
+
+@pytest.mark.followup
+def test_followup_queries():
+    initial_query = "What is the latest local news from a particular town?"
+    logging.info(f"Running followup query test with initial query: {initial_query}")
+    result = send_query(initial_query)
+    
+    logging.info(f"Assertion 1: Checking for absence of error")
+    assert "error" not in result, f"Error in response: {result.get('error')}"
+    
+    if result.get("needs_more_info", False):
+        logging.info("Follow-up required. Sending follow-up query.")
+        follow_up_query = f"{initial_query} The town is Maidstone."
+        follow_up_result = send_query(follow_up_query)
+        
+        logging.info(f"Assertion 2: Checking for absence of error in follow-up")
+        assert "error" not in follow_up_result, f"Error in follow-up response: {follow_up_result.get('error')}"
+        
+        logging.info(f"Assertion 3: Checking for presence of response in follow-up")
+        assert "response" in follow_up_result, "Follow-up response does not contain an answer"
+        
+        logging.info(f"Assertion 4: Checking for non-empty answer in follow-up")
+        assert len(follow_up_result["response"]) > 0, "Follow-up answer is empty"
+        
+        logging.info(f"All follow-up assertions passed. Response: {follow_up_result['response'][:100]}...")
+    else:
+        logging.info(f"Assertion 2: Checking for presence of response")
+        assert "response" in result, "Response does not contain an answer"
+        
+        logging.info(f"Assertion 3: Checking for non-empty answer")
+        assert len(result["response"]) > 0, "Answer is empty"
+        
+        logging.info(f"All assertions passed. Response: {result['response'][:100]}...")

run/tests/run_tests.py

@@ -0,0 +1,179 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import sys
+import subprocess
+from datetime import datetime
+import webbrowser
+import threading
+import shutil
+import time
+
+# Add the parent directory to the Python path
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import modules.logger_tool as logger
+
+# Setup logging
+log_name = 'pytest_run_tests'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+
+def find_project_root():
+    # Start from the current file location
+    root = os.path.dirname(os.path.abspath(__file__))
+    # Traverse up until you find the .env file
+    while not os.path.exists(os.path.join(root, '.env')):
+        new_root = os.path.dirname(root)
+        if root == new_root:  # root directory reached without finding .env
+            raise Exception("Project root not found.")
+        root = new_root
+    return root
+
+def load_env():
+    project_root = find_project_root()
+    dotenv_path = find_dotenv(os.path.join(project_root, '.env'))
+    load_dotenv(dotenv_path)
+    required_vars = ["FIXME"]
+    for var in required_vars:
+        if var not in os.environ:
+            print(f"Error: {var} is not set in the environment.")
+            sys.exit(1)
+
+def select_test_file():
+    project_root = find_project_root()
+    test_categories = {
+        "A": {
+            "name": "X Copilot Initialization",
+            "tests": {
+                "1": os.path.join(project_root, "backend", "app", "tests", "pytest_init_x.py")
+            }
+        },
+        "B": {
+            "name": "Graph QA",
+            "tests": {
+                "1": os.path.join(project_root, "backend", "app", "tests", "pytest_init_school_timetable_graph_qa.py"),
+                "2": os.path.join(project_root, "backend", "app", "tests", "pytest_init_curriculum_graph_qa.py"),
+                "3": os.path.join(project_root, "backend", "app", "tests", "pytest_init_calendar_graph_qa.py")
+            }
+        },
+        "C": {
+            "name": "Connections",
+            "tests": {
+                "1": os.path.join(project_root, "backend", "app", "tests", "pytest_arbor.py")
+            }
+        },
+        "D": {
+            "name": "Transcription",
+            "tests": {
+                "1": os.path.join(project_root, "tests", "pytest_transcribe.py")
+            }
+        },
+        "E": {
+            "name": "LangGraph",
+            "tests": {
+                "1": os.path.join(project_root, "backend", "app", "tests", "pytest_langgraph.py")
+            }
+        }
+    }
+
+    print("Select a test file to run:")
+    for category_key, category in test_categories.items():
+        print(f"\n{category_key}: {category['name']}")
+        for test_key, test_file in category["tests"].items():
+            print(f"  {category_key}{test_key}: {os.path.basename(test_file)}")
+
+    choice = input("\nEnter your choice (e.g., A1): ").upper()
+    if len(choice) == 2 and choice[0] in test_categories and choice[1] in test_categories[choice[0]]["tests"]:
+        category_key, test_key = choice[0], choice[1]
+        return test_categories[category_key]["tests"][test_key], choice
+
+    print("Invalid choice.")
+    sys.exit(1)
+
+def create_log_dir(choice, project_root):
+    log_dir = os.path.join(project_root, "logs", "pytests")
+    if choice[0] == "A":
+        log_dir = os.path.join(log_dir, "database", "init")
+    elif choice[0] == "B":
+        log_dir = os.path.join(log_dir, "database", "langchain", "graph_qa")
+    elif choice[0] == "C":
+        log_dir = os.path.join(log_dir, "database", "connections", "arbor")
+    elif choice[0] == "D":
+        log_dir = os.path.join(log_dir, "transcribe")
+    elif choice[0] == "E":
+        log_dir = os.path.join(log_dir, "langgraph")
+    else:
+        print("Invalid choice.")
+        sys.exit(1)
+    
+    os.makedirs(log_dir, exist_ok=True)
+    return log_dir
+
+def open_html_report_in_browser(html_path):
+    """Function to open the HTML report in the default web browser."""
+    # Check for the existence of the file every 2 seconds, up to a maximum of 10 checks
+    for _ in range(10):
+        if os.path.exists(html_path):
+            webbrowser.open(html_path)
+            break
+        time.sleep(2)
+    else:
+        print("HTML report was not generated in time.")
+
+def run_tests(test_file, log_dir, choice):
+    timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+    base_filename = os.path.basename(test_file).replace('.py', '')
+    html_report = os.path.join(log_dir, f"{base_filename}_pytest_report_{timestamp}.html")
+    xml_report = os.path.join(log_dir, f"{base_filename}_pytest_report_{timestamp}.xml")
+    
+    pytest_command = [
+        "pytest",
+        "-v",
+        test_file,
+        f"--junitxml={xml_report}",
+        f"--html={html_report}",
+        "--self-contained-html",
+        "--capture=tee-sys",
+        "--show-capture=all"
+    ]
+
+    if choice[0] == "A":
+        test_components = input("Enter test components to run (school,users,timetable), comma-separated, or 'all': ").lower()
+        if test_components != 'all':
+            components = test_components.split(',')
+            for component in components:
+                pytest_command.append(f"-m {component}")
+
+    print("Running command:", ' '.join(pytest_command))
+
+    # Start a thread to open the HTML report, checking for its existence
+    threading.Thread(target=open_html_report_in_browser, args=(html_report,)).start()
+
+    result = subprocess.run(pytest_command, check=True)
+    return result
+
+def main():
+    project_root = find_project_root()
+    load_env()
+    data_dir = os.path.join(project_root, "APP_DATA")
+    # TODO: Modify this after initial testing
+    if os.path.exists(data_dir):
+        shutil.rmtree(data_dir)
+    test_file, choice = select_test_file()
+    if not test_file:
+        print("Invalid choice.")
+        sys.exit(1)
+
+    log_dir = create_log_dir(choice, project_root)
+    run_tests(test_file, log_dir, choice)
+
+if __name__ == "__main__":
+    main()

run/tests/test_inputs/init_curriculum_db_cases.json

@@ -0,0 +1,150 @@
+{
+    "curriculum_cases": [
+        {
+            "description": "Retrieve Information About Lessons in a Topic",
+            "prompt": "What are the lessons in the topic 'Maths Skills For Scientists'?",
+            "exclude_types": ["KeyStage", "KeyStageSyllabus", "YearGroup", "YearGroupSyllabus"],
+            "include_types": ["Topic", "Lesson", "LESSON_INCLUDES_LEARNING_STATEMENT"]
+        },
+        {
+            "description": "Retrieve Information About a Specific Year Group Syllabus",
+            "prompt": "What is the syllabus for Year 8?",
+            "exclude_types": ["KeyStage", "KeyStageSyllabus", "Topic", "Lesson", "LearningStatement"],
+            "include_types": ["YearGroup", "YearGroupSyllabus", "YEAR_SYLLABUS_INCLUDES_TOPIC"]
+        },
+        {
+            "description": "Retrieve Key Stages and Their Syllabuses",
+            "prompt": "What are the key stages and their syllabuses?",
+            "exclude_types": ["YearGroup", "YearGroupSyllabus", "Topic", "Lesson", "LearningStatement"],
+            "include_types": ["KeyStage", "KeyStageSyllabus", "KEY_STAGE_INCLUDES_KEY_STAGE_SYLLABUS"]
+        },
+        {
+            "description": "Retrieve Topics Within a Specific Year Group Syllabus",
+            "prompt": "What are the topics in the Year 8 Science syllabus?",
+            "exclude_types": ["KeyStage", "KeyStageSyllabus", "Lesson", "LearningStatement"],
+            "include_types": ["YearGroup", "YearGroupSyllabus", "Topic", "YEAR_SYLLABUS_INCLUDES_TOPIC"]
+        },
+        {
+            "description": "Retrieve All Learning Statements for a Specific Lesson",
+            "prompt": "What are the learning statements for the lesson '8P6.R'?",
+            "exclude_types": ["KeyStage", "KeyStageSyllabus", "YearGroup", "YearGroupSyllabus", "Topic"],
+            "include_types": ["Lesson", "LearningStatement", "LESSON_INCLUDES_LEARNING_STATEMENT"]
+        },
+        {
+            "description": "General Information Retrieval Without Exclusions",
+            "prompt": "Give me an overview of the school curriculum.",
+            "exclude_types": [],
+            "include_types": []
+        },
+        {
+            "description": "Retrieve Detailed Information About a Specific Node Type",
+            "prompt": "Give me detailed information about all topics.",
+            "exclude_types": [],
+            "include_types": ["Topic"]
+        },
+        {
+            "description": "Retrieve Relationships Between Specific Node Types",
+            "prompt": "What are the relationships between Year Groups and their syllabuses?",
+            "exclude_types": [],
+            "include_types": ["YearGroup", "YearGroupSyllabus", "KEY_STAGE_SYLLABUS_INCLUDES_YEAR_GROUP_SYLLABUS"]
+        }
+    ],
+    "include_exclude_cases": {
+        "includes": [
+            {
+                "description": "Include only Lessons",
+                "prompt": "What are the lessons in the topic 'Maths Skills For Scientists'?",
+                "exclude_types": [],
+                "include_types": ["Lesson"]
+            },
+            {
+                "description": "Include only Topics",
+                "prompt": "What are the topics in the Year 8 Science syllabus?",
+                "exclude_types": [],
+                "include_types": ["Topic"]
+            },
+            {
+                "description": "Include only Year Groups",
+                "prompt": "What are the year groups in the school curriculum?",
+                "exclude_types": [],
+                "include_types": ["YearGroup"]
+            },
+            {
+                "description": "Include only Learning Statements",
+                "prompt": "What are the learning statements for the lesson '8P6.R'?",
+                "exclude_types": [],
+                "include_types": ["LearningStatement"]
+            },
+            {
+                "description": "Include only Key Stages",
+                "prompt": "What are the key stages in the school curriculum?",
+                "exclude_types": [],
+                "include_types": ["KeyStage"]
+            }
+        ],
+        "excludes": [
+            {
+                "description": "Exclude Lessons",
+                "prompt": "What are the lessons in the topic 'Maths Skills For Scientists'?",
+                "exclude_types": ["Lesson"],
+                "include_types": []
+            },
+            {
+                "description": "Exclude Topics",
+                "prompt": "What are the topics in the Year 8 Science syllabus?",
+                "exclude_types": ["Topic"],
+                "include_types": []
+            },
+            {
+                "description": "Exclude Year Groups",
+                "prompt": "What are the year groups in the school curriculum?",
+                "exclude_types": ["YearGroup"],
+                "include_types": []
+            },
+            {
+                "description": "Exclude Learning Statements",
+                "prompt": "What are the learning statements for the lesson '8P6.R'?",
+                "exclude_types": ["LearningStatement"],
+                "include_types": []
+            },
+            {
+                "description": "Exclude Key Stages",
+                "prompt": "What are the key stages in the school curriculum?",
+                "exclude_types": ["KeyStage"],
+                "include_types": []
+            }
+        ],
+        "includes_excludes": [
+            {
+                "description": "Include Lessons, Exclude Topics",
+                "prompt": "What are the lessons in the topic 'Maths Skills For Scientists'?",
+                "exclude_types": ["Topic"],
+                "include_types": ["Lesson"]
+            },
+            {
+                "description": "Include Topics, Exclude Lessons",
+                "prompt": "What are the topics in the Year 8 Science syllabus?",
+                "exclude_types": ["Lesson"],
+                "include_types": ["Topic"]
+            },
+            {
+                "description": "Include Year Groups, Exclude Key Stages",
+                "prompt": "What are the year groups in the school curriculum?",
+                "exclude_types": ["KeyStage"],
+                "include_types": ["YearGroup"]
+            },
+            {
+                "description": "Include Learning Statements, Exclude Lessons",
+                "prompt": "What are the learning statements for the lesson '8P6.R'?",
+                "exclude_types": ["Lesson"],
+                "include_types": ["LearningStatement"]
+            },
+            {
+                "description": "Include Key Stages, Exclude Year Groups",
+                "prompt": "What are the key stages in the school curriculum?",
+                "exclude_types": ["YearGroup"],
+                "include_types": ["KeyStage"]
+            }
+        ]
+    }
+}