Initial commit

modules/database/tools/db_operations.py

@@ -0,0 +1,74 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'api_modules_database_tools_db_operations'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+from fastapi.testclient import TestClient
+from fastapi import HTTPException
+import time
+from neo4j import GraphDatabase
+
+class DatabaseNotFoundError(Exception):
+    """Exception raised when the specified database cannot be found."""
+    def __init__(self, db_name):
+        super().__init__(f"Database '{db_name}' not found.")
+
+# Dev ??
+def get_client():
+    from main import app  # Delayed import to avoid circular dependency
+    return TestClient(app)
+
+# Ops ??
+def stop_database(db_name):
+    client = get_client()
+    try:
+        logging.debug(f"Stopping database {db_name}")
+        response = client.post("/database/admin/stop-database", json={"db_name": db_name})
+    except DatabaseNotFoundError:
+        logging.info(f"Database {db_name} not found when attempting to stop. Skipping.")
+    else:
+        logging.info(response.text)
+    return response
+
+def drop_database(db_name):
+    client = get_client()
+    try:
+        response = client.post("/database/admin/drop-database", json={"db_name": db_name})
+    except DatabaseNotFoundError:
+        logging.info(f"Database {db_name} not found when attempting to drop. Skipping.")
+    else:
+        logging.info(response.text)
+    return response
+
+def create_database(db_name):
+    client = get_client()
+    response = client.post("/database/admin/create-database", params={"db_name": db_name})
+    logging.info(response.text)
+    return response
+
+def check_database_availability(db_name, retries=5, delay=5):  # Increased delay
+    client = get_client()
+    attempt = 0
+    while attempt < retries:
+        try:
+            logging.info(f"Attempt {attempt + 1}: Checking availability for database {db_name}")
+            response = client.get(f"/check-database-availability?db_name={db_name}")
+            if response.status_code == 200 and response.json().get('status') == "ready":
+                logging.info(f"Database {db_name} is ready.")
+                return response.json()
+            else:
+                logging.error(f"Database {db_name} is not available: {response.text}")
+        except Exception as e:
+            logging.error(f"Error checking database availability for {db_name} on attempt {attempt + 1}: {e}")
+        time.sleep(delay)  # Increased delay before the next retry
+        attempt += 1
+    raise HTTPException(status_code=503, detail="Database availability check failed after retries")

modules/database/tools/filesystem_tools.py

@@ -0,0 +1,560 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'api_modules_database_tools_filesystem_tools'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+from datetime import timedelta
+import json
+import re
+
+class ClassroomCopilotFilesystem:
+    def __init__(self, db_name: str, init_run_type: str = None):
+        logging.info(f"Initializing ClassroomCopilotFilesystem with db_name: {db_name} and init_run_type: {init_run_type}")
+        
+        self.db_name = db_name
+        
+        # Get base path from environment
+        self.base_path = os.getenv("NODE_FILESYSTEM_PATH")
+        if not self.base_path:
+            raise ValueError("NODE_FILESYSTEM_PATH environment variable not set")
+        
+        # Set root path based on init type
+        if init_run_type == "school":
+            self.root_path = os.path.join(self.base_path, "schools", self.db_name)
+            logging.debug(f"School root path: {self.root_path}")
+        elif init_run_type == "user":
+            self.root_path = os.path.join(self.base_path, "users", self.db_name)
+            logging.debug(f"User root path: {self.root_path}")
+        elif init_run_type == "multiplayer":
+            self.root_path = os.path.join(self.base_path, "multiplayer")
+            logging.debug(f"Multiplayer root path: {self.root_path}")
+        else:
+            self.root_path = os.path.join(self.base_path, self.db_name)
+            logging.debug(f"Default root path: {self.root_path}")
+        
+        # Ensure root directory exists
+        os.makedirs(self.root_path, exist_ok=True)
+        
+        logging.debug(f"Filesystem initialized with run type: {init_run_type} and root path: {self.root_path}")
+
+    def log_directory_structure(self, start_path):
+        for root, dirs, files in os.walk(start_path):
+            level = root.replace(start_path, '').count(os.sep)
+            indent = ' ' * 4 * (level)
+            logging.info(f"{indent}{os.path.basename(root)}/")
+            subindent = ' ' * 4 * (level + 1)
+            for f in files:
+                logging.info(f"{subindent}{f}")
+                
+    def create_directory(self, path):
+        """Utility method to create a directory if it doesn't exist."""
+        if not os.path.exists(path):
+            os.makedirs(path)
+            logging.info(f"Directory {path} created.")
+            return True
+        return False
+
+    def sanitize_username(self, username):
+        return re.sub(r'[^\w\-_\.]', '_', username)
+
+    def create_user_directory(self, username, user_type=None, school_path=None):
+        """Create a directory for a specific user."""
+        sanitized_username = self.sanitize_username(username)
+        
+        if school_path:
+            # For school database: /schools/[school_db]/users/[user_type]/[username]
+            user_path = os.path.join(self.root_path, "users", user_type, sanitized_username)
+        else:
+            # For user database: /users/[user_db]/[username]
+            user_path = os.path.join(self.root_path, sanitized_username)
+            
+        logging.info(f"Creating user directory at {user_path}")
+        return self.create_directory(user_path), user_path
+    
+    def create_user_worker_directory(self, user_path, worker_code):
+        """Create a worker directory under the user directory."""
+        # Create worker directory: [user_path]/[worker_code]
+        worker_path = os.path.join(user_path, worker_code)
+        logging.info(f"Creating worker directory at {worker_path}")
+        return self.create_directory(worker_path), worker_path
+    
+    def create_school_worker_directory(self, school_path, worker_type):
+        """Create a worker directory under the school directory."""
+        worker_path = os.path.join(school_path, "workers", worker_type)
+        logging.info(f"Creating school worker directory at {worker_path}")
+        return self.create_directory(worker_path), worker_path
+    
+    def create_school_directory(self, school_uuid=None):
+        """Create a directory for a specific school."""
+        logging.info(f"Creating school directory with school_uuid: {school_uuid}")
+        if school_uuid is None:
+            logging.debug(f"School UUID is None, creating school directory at {self.root_path}")
+            school_path = self.root_path
+        else:
+            logging.debug(f"School UUID is not None, creating school directory at {os.path.join(self.root_path, school_uuid)}")
+            school_path = os.path.join(self.root_path, school_uuid)
+        return self.create_directory(school_path), school_path
+    
+    def create_year_directory(self, year, calendar_path=None):
+        """Create a directory for a specific year."""
+        if calendar_path is None:
+            year_path = os.path.join(self.root_path, "calendar", str(year))
+        else:
+            year_path = os.path.join(calendar_path, "years", str(year))
+
+        return self.create_directory(year_path), year_path
+
+    def create_month_directory(self, year, month, calendar_path=None):
+        """Create a directory for a specific month."""
+        if calendar_path is None:
+            month_path = os.path.join(self.root_path, "calendar", str(year), "months", f"{month:02}")
+        else:
+            month_path = os.path.join(calendar_path, "years", str(year), "months", f"{month:02}")
+
+        return self.create_directory(month_path), month_path
+
+    def create_week_directory(self, year, week, calendar_path=None):
+        """Create a directory for a specific week."""
+        if calendar_path is None:
+            week_path = os.path.join(self.root_path, "calendar", str(year), "weeks", f"{week}")
+        else:
+            week_path = os.path.join(calendar_path, "years", str(year), "weeks", f"{week}")
+
+        return self.create_directory(week_path), week_path
+
+    def create_day_directory(self, year, month, day, calendar_path=None):
+        """Create a directory for a specific day."""
+        if calendar_path is None:
+            day_path = os.path.join(self.root_path, "calendar", str(year), "months", f"{month:02}", f"{day:02}")
+        else:
+            day_path = os.path.join(calendar_path, "years", str(year), "months", f"{month:02}", f"{day:02}")
+
+        return self.create_directory(day_path), day_path
+
+    def setup_calendar_directories(self, start_date, end_date, calendar_path=None):
+        """Setup directories for the range from start_date to end_date."""
+        current_date = start_date
+        while current_date <= end_date:
+            year, month, day = current_date.year, current_date.month, current_date.day
+            if calendar_path is None:
+                _, year_path = self.create_year_directory(year)
+                _, month_path = self.create_month_directory(year, month)
+                _, week_path = self.create_week_directory(year, current_date.isocalendar()[1])
+                _, day_path = self.create_day_directory(year, month, day)
+            else:
+                _, year_path = self.create_year_directory(year, calendar_path)
+                _, month_path = self.create_month_directory(year, month, calendar_path)
+                _, week_path = self.create_week_directory(year, current_date.isocalendar()[1], calendar_path)
+                _, day_path = self.create_day_directory(year, month, day, calendar_path)
+            current_date += timedelta(days=1)
+        return year_path, month_path, week_path, day_path
+
+    def create_school_timetable_directory(self, school_path=None):
+        """Create a directory for the timetable."""
+        if school_path is None:
+            timetable_path = os.path.join(self.root_path, "timetable")
+        else:
+            timetable_path = os.path.join(school_path, "timetable")
+
+        return self.create_directory(timetable_path), timetable_path
+
+    def create_school_timetable_year_directory(self, timetable_path, year):
+        """Create a directory for a specific academic year within the timetable."""
+        year_path = os.path.join(timetable_path, "years", str(year))
+        return self.create_directory(year_path), year_path
+
+    def create_school_timetable_academic_term_directory(self, timetable_path, term_name, term_number):
+        """Create a directory for a specific term within an academic year."""
+        term_path = os.path.join(timetable_path, "terms", f"{term_number}_{term_name.replace(' ', '_')}")
+        return self.create_directory(term_path), term_path
+    
+    def create_school_timetable_academic_term_break_directory(self, timetable_path, term_name):
+        """Create a directory for a specific term within an academic year."""
+        term_path = os.path.join(timetable_path, "terms", "term_breaks", f"{term_name.replace(' ', '_')}")
+        return self.create_directory(term_path), term_path
+
+    def create_school_timetable_academic_week_directory(self, timetable_path, week_number):
+        """Create a directory for a specific week within a term of a specific year."""
+        week_path = os.path.join(timetable_path, "weeks", f"{week_number}")
+        return self.create_directory(week_path), week_path
+
+    def create_school_timetable_academic_day_directory(self, timetable_path, academic_day):
+        """Create a directory for a specific day within a week of a term."""
+        day_path = os.path.join(timetable_path, "days",f"{academic_day:02}")
+        return self.create_directory(day_path), day_path
+    
+    def create_school_timetable_period_directory(self, timetable_path, academic_day, period_dir):
+        """Create a directory for a specific period within a day."""
+        period_path = os.path.join(timetable_path, "days",f"{academic_day:02}", f"{period_dir}")
+        return self.create_directory(period_path), period_path
+    
+    def create_school_curriculum_directory(self, school_path=None):
+        """Create a directory for the curriculum."""
+        if school_path is None:
+            curriculum_path = os.path.join(self.root_path, "curriculum")
+        else:
+            curriculum_path = os.path.join(school_path, "curriculum")
+
+        return self.create_directory(curriculum_path), curriculum_path
+    
+    def create_school_pastoral_directory(self, school_path=None):
+        """Create a directory for the pastoral."""
+        if school_path is None:
+            pastoral_path = os.path.join(self.root_path, "pastoral")
+        else:
+            pastoral_path = os.path.join(school_path, "pastoral")
+
+        return self.create_directory(pastoral_path), pastoral_path
+    
+    def create_school_department_directory(self, school_path, department):
+        """Create a directory for a specific department within the school."""
+        department_path = os.path.join(school_path, "departments", f"{department}")
+        return self.create_directory(department_path), department_path
+    
+    def create_department_subject_directory(self, department_path, subject_name):
+        """Create a directory for a specific subject within a department."""
+        subject_path = os.path.join(department_path, "subjects", f"{subject_name}")
+        return self.create_directory(subject_path), subject_path
+        
+    def create_curriculum_key_stage_syllabus_directory(self, curriculum_path, key_stage, subject_name, syllabus_id):
+        """Create a directory for a specific key stage syllabus under the curriculum structure."""
+        # Replace spaces with underscores and remove any special characters from subject name
+        safe_subject_name = re.sub(r'[^\w\-_\.]', '_', subject_name)
+        syllabus_path = os.path.join(curriculum_path, "subjects", safe_subject_name, "key_stage_syllabuses", f"KS{key_stage}", f"KS{key_stage}.{safe_subject_name}")
+        return self.create_directory(syllabus_path), syllabus_path
+
+    def create_pastoral_year_group_directory(self, pastoral_path, year_group):
+        """Create a directory for a specific year group under the pastoral structure."""
+        year_group_path = os.path.join(pastoral_path, "year_groups", f"Y{year_group}")
+        return self.create_directory(year_group_path), year_group_path
+        
+    def create_curriculum_year_group_syllabus_directory(self, curriculum_path, subject_name, year_group, syllabus_id):
+        """Create a directory for a specific year group syllabus under the curriculum structure."""
+        # Replace spaces with underscores and remove any special characters from subject name
+        safe_subject_name = re.sub(r'[^\w\-_\.]', '_', subject_name)
+        syllabus_path = os.path.join(curriculum_path, "subjects", safe_subject_name, "year_group_syllabuses", f"Y{year_group}", f"Y{year_group}.{safe_subject_name}")
+        return self.create_directory(syllabus_path), syllabus_path
+        
+    def create_curriculum_topic_directory(self, year_group_syllabus_path, topic_id):
+        """Create a directory for a specific topic under a year group syllabus."""
+        topic_path = os.path.join(year_group_syllabus_path, "topics", f"{topic_id}")
+        return self.create_directory(topic_path), topic_path
+        
+    def create_curriculum_lesson_directory(self, topic_path, lesson_id):
+        """Create a directory for a specific lesson under a topic."""
+        lesson_path = os.path.join(topic_path, "lessons", f"{lesson_id}")
+        return self.create_directory(lesson_path), lesson_path
+        
+    def create_curriculum_learning_statement_directory(self, lesson_path, statement_id):
+        """Create a directory for a specific learning statement under a lesson."""
+        statement_path = os.path.join(lesson_path, "learning_statements", f"{statement_id}")
+        return self.create_directory(statement_path), statement_path
+
+    # Remove or mark as deprecated the old methods
+
+
+    def create_teacher_timetable_directory(self, teacher_path):
+        teacher_timetable_path = os.path.join(teacher_path, "timetable")
+        return self.create_directory(teacher_timetable_path), teacher_timetable_path
+
+    def create_teacher_class_directory(self, teacher_timetable_path, class_name):
+        class_path = os.path.join(teacher_timetable_path, "classes", class_name)
+        return self.create_directory(class_path), class_path
+
+    def create_teacher_timetable_lesson_directory(self, class_path, lesson_id):
+        lesson_path = os.path.join(class_path, "timetabled_lessons", lesson_id)
+        return self.create_directory(lesson_path), lesson_path
+
+    def create_teacher_planned_lesson_directory(self, class_path, lesson_id):
+        planned_lesson_path = os.path.join(class_path, "planned_lessons", lesson_id)
+        return self.create_directory(planned_lesson_path), planned_lesson_path
+    
+    # TLDraw File Creation
+    def create_default_tldraw_file(self, node_path, node_data):
+        """Create a tldraw file for a node."""
+        logging.info(f"Creating tldraw file for node at {node_path}")
+        
+        # Ensure the directory exists
+        os.makedirs(node_path, exist_ok=True)
+        
+        tldraw_path = os.path.join(node_path, 'tldraw_file.json')
+        
+        # Create default tldraw content
+        tldraw_content = {
+            "document": {
+                "store": {
+                    "document:document": {
+                            "gridSize": 10,
+                            "name": "",
+                            "meta": {},
+                            "id": "document:document",
+                            "typeName": "document"
+                        },
+                        "page:page": {
+                            "meta": {},
+                            "id": "page:page",
+                            "name": "Page 1",
+                            "index": "a1",
+                            "typeName": "page"
+                        }
+                },
+                "schema":
+                    {"schemaVersion":2,
+                    "sequences": {
+                        "com.tldraw.store":4,
+                        "com.tldraw.asset":1,
+                        "com.tldraw.camera":1,
+                        "com.tldraw.document":2,
+                        "com.tldraw.instance":25,
+                        "com.tldraw.instance_page_state":5,
+                        "com.tldraw.page":1,
+                        "com.tldraw.instance_presence":5,
+                        "com.tldraw.pointer":1,
+                        "com.tldraw.shape":4,
+                        "com.tldraw.asset.bookmark":2,
+                        "com.tldraw.asset.image":5,
+                        "com.tldraw.asset.video":5,
+                        "com.tldraw.shape.arrow":5,
+                        "com.tldraw.shape.bookmark":2,
+                        "com.tldraw.shape.draw":2,
+                        "com.tldraw.shape.embed":4,
+                        "com.tldraw.shape.frame":0,
+                        "com.tldraw.shape.geo":9,
+                        "com.tldraw.shape.group":0,
+                        "com.tldraw.shape.highlight":1,
+                        "com.tldraw.shape.image":4,
+                        "com.tldraw.shape.line":5,
+                        "com.tldraw.shape.note":8,
+                        "com.tldraw.shape.text":2,
+                        "com.tldraw.shape.video":2,
+                        "com.tldraw.shape.youtube-embed":0,
+                        "com.tldraw.shape.calendar":0,
+                        "com.tldraw.shape.microphone":1,
+                        "com.tldraw.shape.transcriptionText":0,
+                        "com.tldraw.shape.slide":0,"com.tldraw.shape.slideshow":0,
+                        "com.tldraw.shape.user_node":1,
+                        "com.tldraw.shape.developer_node":1,
+                        "com.tldraw.shape.student_node":1,
+                        "com.tldraw.shape.teacher_node":1,
+                        "com.tldraw.shape.calendar_node":1,
+                        "com.tldraw.shape.calendar_year_node":1,
+                        "com.tldraw.shape.calendar_month_node":1,
+                        "com.tldraw.shape.calendar_week_node":1,
+                        "com.tldraw.shape.calendar_day_node":1,
+                        "com.tldraw.shape.calendar_time_chunk_node":1,
+                        "com.tldraw.shape.teacher_timetable_node":1,
+                        "com.tldraw.shape.timetable_lesson_node":1,
+                        "com.tldraw.shape.planned_lesson_node":1,
+                        "com.tldraw.shape.pastoral_structure_node":1,
+                        "com.tldraw.shape.year_group_node":1,
+                        "com.tldraw.shape.curriculum_structure_node":1,
+                        "com.tldraw.shape.key_stage_node":1,
+                        "com.tldraw.shape.key_stage_syllabus_node":1,
+                        "com.tldraw.shape.year_group_syllabus_node":1,
+                        "com.tldraw.shape.subject_node":1,
+                        "com.tldraw.shape.topic_node":1,
+                        "com.tldraw.shape.topic_lesson_node":1,
+                        "com.tldraw.shape.learning_statement_node":1,
+                        "com.tldraw.shape.science_lab_node":1,
+                        "com.tldraw.shape.school_timetable_node":1,
+                        "com.tldraw.shape.academic_year_node":1,
+                        "com.tldraw.shape.academic_term_node":1,
+                        "com.tldraw.shape.academic_week_node":1,
+                        "com.tldraw.shape.academic_day_node":1,
+                        "com.tldraw.shape.academic_period_node":1,
+                        "com.tldraw.shape.registration_period_node":1,
+                        "com.tldraw.shape.school_node":1,
+                        "com.tldraw.shape.department_node":1,
+                        "com.tldraw.shape.room_node":1,
+                        "com.tldraw.shape.subject_class_node":1,
+                        "com.tldraw.shape.general_relationship":1,
+                        "com.tldraw.binding.arrow":0,
+                        "com.tldraw.binding.slide-layout":0
+                    }
+                },
+                "recordVersions": {
+                    "asset": { "version": 1, "subTypeKey": "type", "subTypeVersions": {} },
+                    "camera": { "version": 1 },
+                    "document": { "version": 2 },
+                    "instance": { "version": 21 },
+                    "instance_page_state": { "version": 5 },
+                    "page": { "version": 1 },
+                    "shape": { "version": 3, "subTypeKey": "type", "subTypeVersions": {} },
+                    "instance_presence": { "version": 5 },
+                    "pointer": { "version": 1 }
+                },
+                "rootShapeIds":[],
+                "bindings":[],
+                "assets":[]
+                },
+                "session": {
+                    "version": 0,
+                    "currentPageId": "page:page",
+                    "pageStates": [{
+                        "pageId": "page:page",
+                        "camera": {"x": 0, "y": 0, "z": 1},
+                        "selectedShapeIds": []
+                    }]
+                },
+                "node_data": node_data
+        }
+        
+        with open(tldraw_path, 'w') as f:
+            json.dump(tldraw_content, f, indent=4)
+            
+        logging.info(f"tldraw file created at {tldraw_path}")
+        return tldraw_path
+
+    def create_default_tldraw_file_in_storage(self, admin_supabase, bucket_id, file_path, node_data):
+        """Create a tldraw file in Supabase storage."""
+        logging.info(f"Creating tldraw file in storage at {file_path}")
+        
+        # Create default tldraw content
+        tldraw_content = {
+            "document": {
+                "store": {
+                    "document:document": {
+                            "gridSize": 10,
+                            "name": "",
+                            "meta": {},
+                            "id": "document:document",
+                            "typeName": "document"
+                        },
+                        "page:page": {
+                            "meta": {},
+                            "id": "page:page",
+                            "name": "Page 1",
+                            "index": "a1",
+                            "typeName": "page"
+                        }
+                },
+                "schema":
+                    {"schemaVersion":2,
+                    "sequences": {
+                        "com.tldraw.store":4,
+                        "com.tldraw.asset":1,
+                        "com.tldraw.camera":1,
+                        "com.tldraw.document":2,
+                        "com.tldraw.instance":25,
+                        "com.tldraw.instance_page_state":5,
+                        "com.tldraw.page":1,
+                        "com.tldraw.instance_presence":5,
+                        "com.tldraw.pointer":1,
+                        "com.tldraw.shape":4,
+                        "com.tldraw.asset.bookmark":2,
+                        "com.tldraw.asset.image":5,
+                        "com.tldraw.asset.video":5,
+                        "com.tldraw.shape.arrow":5,
+                        "com.tldraw.shape.bookmark":2,
+                        "com.tldraw.shape.draw":2,
+                        "com.tldraw.shape.embed":4,
+                        "com.tldraw.shape.frame":0,
+                        "com.tldraw.shape.geo":9,
+                        "com.tldraw.shape.group":0,
+                        "com.tldraw.shape.highlight":1,
+                        "com.tldraw.shape.image":4,
+                        "com.tldraw.shape.line":5,
+                        "com.tldraw.shape.note":8,
+                        "com.tldraw.shape.text":2,
+                        "com.tldraw.shape.video":2,
+                        "com.tldraw.shape.youtube-embed":0,
+                        "com.tldraw.shape.calendar":0,
+                        "com.tldraw.shape.microphone":1,
+                        "com.tldraw.shape.transcriptionText":0,
+                        "com.tldraw.shape.slide":0,"com.tldraw.shape.slideshow":0,
+                        "com.tldraw.shape.user_node":1,
+                        "com.tldraw.shape.developer_node":1,
+                        "com.tldraw.shape.student_node":1,
+                        "com.tldraw.shape.teacher_node":1,
+                        "com.tldraw.shape.calendar_node":1,
+                        "com.tldraw.shape.calendar_year_node":1,
+                        "com.tldraw.shape.calendar_month_node":1,
+                        "com.tldraw.shape.calendar_week_node":1,
+                        "com.tldraw.shape.calendar_day_node":1,
+                        "com.tldraw.shape.calendar_time_chunk_node":1,
+                        "com.tldraw.shape.teacher_timetable_node":1,
+                        "com.tldraw.shape.timetable_lesson_node":1,
+                        "com.tldraw.shape.planned_lesson_node":1,
+                        "com.tldraw.shape.pastoral_structure_node":1,
+                        "com.tldraw.shape.year_group_node":1,
+                        "com.tldraw.shape.curriculum_structure_node":1,
+                        "com.tldraw.shape.key_stage_node":1,
+                        "com.tldraw.shape.key_stage_syllabus_node":1,
+                        "com.tldraw.shape.year_group_syllabus_node":1,
+                        "com.tldraw.shape.subject_node":1,
+                        "com.tldraw.shape.topic_node":1,
+                        "com.tldraw.shape.topic_lesson_node":1,
+                        "com.tldraw.shape.learning_statement_node":1,
+                        "com.tldraw.shape.science_lab_node":1,
+                        "com.tldraw.shape.school_timetable_node":1,
+                        "com.tldraw.shape.academic_year_node":1,
+                        "com.tldraw.shape.academic_term_node":1,
+                        "com.tldraw.shape.academic_week_node":1,
+                        "com.tldraw.shape.academic_day_node":1,
+                        "com.tldraw.shape.academic_period_node":1,
+                        "com.tldraw.shape.registration_period_node":1,
+                        "com.tldraw.shape.school_node":1,
+                        "com.tldraw.shape.department_node":1,
+                        "com.tldraw.shape.room_node":1,
+                        "com.tldraw.shape.subject_class_node":1,
+                        "com.tldraw.shape.general_relationship":1,
+                        "com.tldraw.binding.arrow":0,
+                        "com.tldraw.binding.slide-layout":0
+                    }
+                },
+                "recordVersions": {
+                    "asset": { "version": 1, "subTypeKey": "type", "subTypeVersions": {} },
+                    "camera": { "version": 1 },
+                    "document": { "version": 2 },
+                    "instance": { "version": 21 },
+                    "instance_page_state": { "version": 5 },
+                    "page": { "version": 1 },
+                    "shape": { "version": 3, "subTypeKey": "type", "subTypeVersions": {} },
+                    "instance_presence": { "version": 5 },
+                    "pointer": { "version": 1 }
+                },
+                "rootShapeIds":[],
+                "bindings":[],
+                "assets":[]
+                },
+                "session": {
+                    "version": 0,
+                    "currentPageId": "page:page",
+                    "pageStates": [{
+                        "pageId": "page:page",
+                        "camera": {"x": 0, "y": 0, "z": 1},
+                        "selectedShapeIds": []
+                    }]
+                },
+                "node_data": node_data
+        }
+        
+        # Convert the content to JSON string
+        tldraw_json = json.dumps(tldraw_content, indent=4)
+        
+        try:
+            # Upload the file to Supabase storage
+            result = admin_supabase.storage.from_(bucket_id).upload(
+                path=file_path,
+                file=tldraw_json,
+                file_options={"content-type": "application/json"}
+            )
+            
+            if result.get('error'):
+                logging.error(f"Error creating tldraw file in storage: {result['error']}")
+                raise Exception(f"Failed to create tldraw file: {result['error']}")
+            
+            logging.info(f"tldraw file created in storage at {file_path}")
+            return True
+        except Exception as e:
+            logging.error(f"Error creating tldraw file in storage: {str(e)}")
+            raise e

modules/database/tools/navigation/user_navigation.py

@@ -0,0 +1,491 @@
+import os
+from modules.logger_tool import initialise_logger
+logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
+import modules.database.tools.neo4j_driver_tools as driver_tools
+import modules.database.tools.neo4j_session_tools as session_tools
+from datetime import datetime, timedelta
+from typing import List, Optional, Dict, Any
+
+def get_static_nodes(context: str, db_name: str) -> List[Dict[str, Any]]:
+    """Get static nodes for a specific context."""
+    if context == 'workers':
+        # For workers context, show teacher node first, then timetables and classes
+        query = """
+        MATCH (t:Teacher)
+        RETURN DISTINCT {
+            id: t.unique_id,
+            path: t.path,
+            label: t.teacher_name_formal,
+            type: 'Teacher',
+            isStatic: true,
+            order: 0,
+            section: 'Root'
+        } as node
+        UNION ALL
+        MATCH (t:UserTeacherTimetable)
+        RETURN DISTINCT {
+            id: t.unique_id,
+            path: t.path,
+            label: t.name,
+            type: 'UserTeacherTimetable',
+            isStatic: true,
+            order: 1,
+            section: 'Timetables'
+        } as node
+        UNION ALL
+        MATCH (t:UserTeacherTimetable)-[:HAS_CLASS]->(c:Class)
+        RETURN DISTINCT {
+            id: c.unique_id,
+            path: c.path,
+            label: c.name,
+            type: 'Class',
+            isStatic: true,
+            order: 2,
+            section: 'Classes'
+        } as node
+        """
+    elif context == 'user':
+        # For user context, show the user node
+        query = """
+        MATCH (u:User)
+        RETURN DISTINCT {
+            id: u.unique_id,
+            path: u.path,
+            label: u.user_name,
+            type: 'User',
+            isStatic: true,
+            order: 0,
+            section: 'Root'
+        } as node
+        """
+    else:
+        # For calendar context, show today's calendar node first, then other calendar nodes
+        today = datetime.now().strftime("%Y-%m-%d")
+        query = """
+        MATCH (n:Calendar)
+        WITH n, 
+        CASE 
+            WHEN date($today) >= date(n.start_date) AND date($today) <= date(n.end_date) 
+            THEN 0 
+            ELSE 1 
+        END as nodeOrder
+        RETURN DISTINCT {
+            id: n.unique_id,
+            path: n.path,
+            label: n.name,
+            type: 'Calendar',
+            isStatic: true,
+            order: nodeOrder,
+            section: CASE nodeOrder 
+                WHEN 0 THEN 'Today'
+                ELSE 'Calendar'
+            END
+        } as node
+        ORDER BY node.order, node.label
+        """ 
+
+    try:
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query, today=datetime.now().strftime("%Y-%m-%d"))
+            return [record["node"] for record in result]
+    except Exception as e:
+        logger.error(f"Error getting static nodes: {str(e)}")
+        return []
+
+def get_today_calendar_node(db_name: str) -> Optional[Dict[str, Any]]:
+    """Get today's calendar node."""
+    today = datetime.now().strftime("%Y-%m-%d")
+    query = """
+    MATCH (n:Calendar)
+    WHERE date($today) >= date(n.start_date) AND date($today) <= date(n.end_date)
+    RETURN n.unique_id as id, n.path as path, n.name as label, 
+           'Calendar' as type
+    LIMIT 1
+    """
+    try:
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query, today=today)
+            record = result.single()
+            return dict(record) if record else None
+    except Exception as e:
+        logger.error(f"Error getting today's calendar node: {str(e)}")
+        return None
+
+def get_relative_calendar_node(day_offset: int, db_name: str) -> Optional[Dict[str, Any]]:
+    """Get calendar node relative to today."""
+    target_date = (datetime.now() + timedelta(days=day_offset)).strftime("%Y-%m-%d")
+    query = """
+    MATCH (n:Calendar)
+    WHERE date($target_date) >= date(n.start_date) AND date($target_date) <= date(n.end_date)
+    RETURN n.unique_id as id, n.path as path, n.name as label, 
+           'Calendar' as type
+    LIMIT 1
+    """
+    try:
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query, target_date=target_date)
+            record = result.single()
+            return dict(record) if record else None
+    except Exception as e:
+        logger.error(f"Error getting relative calendar node: {str(e)}")
+        return None
+
+def get_next_month_node(db_name: str) -> Optional[Dict[str, Any]]:
+    """Get next month's calendar node."""
+    next_month_start = (datetime.now().replace(day=1) + timedelta(days=32)).replace(day=1).strftime("%Y-%m-%d")
+    query = """
+    MATCH (n:Calendar)
+    WHERE date($next_month_start) >= date(n.start_date) AND date($next_month_start) <= date(n.end_date)
+    RETURN n.unique_id as id, n.path as path, n.name as label, 
+           'Calendar' as type
+    LIMIT 1
+    """
+    try:
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query, next_month_start=next_month_start)
+            record = result.single()
+            return dict(record) if record else None
+    except Exception as e:
+        logger.error(f"Error getting next month node: {str(e)}")
+        return None
+
+def get_previous_month_node(db_name: str) -> Optional[Dict[str, Any]]:
+    """Get previous month's calendar node."""
+    prev_month_start = (datetime.now().replace(day=1) - timedelta(days=1)).replace(day=1).strftime("%Y-%m-%d")
+    query = """
+    MATCH (n:Calendar)
+    WHERE date($prev_month_start) >= date(n.start_date) AND date($prev_month_start) <= date(n.end_date)
+    RETURN n.unique_id as id, n.path as path, n.name as label, 
+           'Calendar' as type
+    LIMIT 1
+    """
+    try:
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query, prev_month_start=prev_month_start)
+            record = result.single()
+            return dict(record) if record else None
+    except Exception as e:
+        logger.error(f"Error getting previous month node: {str(e)}")
+        return None
+
+def get_user_timetables(db_name: str) -> List[Dict[str, Any]]:
+    """Get user's timetables."""
+    query = """
+    MATCH (t:UserTeacherTimetable)
+    RETURN t.unique_id as id, t.path as path, t.name as label, 
+           'UserTeacherTimetable' as type
+    """
+    try:
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query)
+            return [dict(record) for record in result]
+    except Exception as e:
+        logger.error(f"Error getting user timetables: {str(e)}")
+        return []
+
+def get_timetable_classes(timetable_id: str, db_name: str) -> List[Dict[str, Any]]:
+    """Get classes for a timetable."""
+    query = """
+    MATCH (t:UserTeacherTimetable {unique_id: $timetable_id})-[:HAS_CLASS]->(c:Class)
+    RETURN c.unique_id as id, c.path as path, c.name as label, 
+           'Class' as type
+    """
+    try:
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query, timetable_id=timetable_id)
+            return [dict(record) for record in result]
+    except Exception as e:
+        logger.error(f"Error getting timetable classes: {str(e)}")
+        return []
+
+def get_next_lesson(class_id: str, db_name: str) -> Optional[Dict[str, Any]]:
+    """Get next lesson for a class."""
+    now = datetime.now().strftime("%Y-%m-%d %H:%M")
+    query = """
+    MATCH (c:Class {unique_id: $class_id})-[:HAS_LESSON]->(l:Lesson)
+    WHERE l.start_time > $now
+    RETURN l.unique_id as id, l.path as path, l.name as label, 
+           'Lesson' as type
+    ORDER BY l.start_time ASC
+    LIMIT 1
+    """
+    try:
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query, class_id=class_id, now=now)
+            record = result.single()
+            return dict(record) if record else None
+    except Exception as e:
+        logger.error(f"Error getting next lesson: {str(e)}")
+        return None
+
+def get_previous_lesson(class_id: str, db_name: str) -> Optional[Dict[str, Any]]:
+    """Get previous lesson for a class."""
+    now = datetime.now().strftime("%Y-%m-%d %H:%M")
+    query = """
+    MATCH (c:Class {unique_id: $class_id})-[:HAS_LESSON]->(l:Lesson)
+    WHERE l.start_time < $now
+    RETURN l.unique_id as id, l.path as path, l.name as label, 
+           'Lesson' as type
+    ORDER BY l.start_time DESC
+    LIMIT 1
+    """
+    try:
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query, class_id=class_id, now=now)
+            record = result.single()
+            return dict(record) if record else None
+    except Exception as e:
+        logger.error(f"Error getting previous lesson: {str(e)}")
+        return None
+
+def save_shared_snapshot(path: str, room_id: str, snapshot: Dict[str, Any]) -> bool:
+    """Save snapshot to a shared room."""
+    try:
+        # Save the snapshot to the shared room's storage
+        session_tools.save_tldraw_node_file(path, room_id, snapshot)
+        return True
+    except Exception as e:
+        logger.error(f"Error saving shared snapshot: {str(e)}")
+        return False
+
+def get_connected_nodes_for_workers(node_id: str, db_name: str) -> List[Dict[str, Any]]:
+    """Get connected nodes specific to the workers context."""
+    query = """
+    MATCH (n {unique_id: $node_id})
+    WITH n
+    CALL {
+        WITH n
+        MATCH (n:UserTeacherTimetable)-[:HAS_CLASS]->(c:Class)
+        RETURN c.unique_id as id, c.path as path, c.name as label, 
+               'Class' as type
+        UNION
+        MATCH (n:Class)<-[:HAS_CLASS]-(t:UserTeacherTimetable)
+        RETURN t.unique_id as id, t.path as path, t.name as label, 
+               'UserTeacherTimetable' as type
+        UNION
+        MATCH (n:Class)-[:HAS_LESSON]->(l:Lesson)
+        RETURN l.unique_id as id, l.path as path, l.name as label, 
+               'Lesson' as type
+    }
+    RETURN DISTINCT id, path, label, type
+    """
+    try:
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query, node_id=node_id)
+            return [dict(record) for record in result]
+    except Exception as e:
+        logger.error(f"Error getting connected nodes for workers: {str(e)}")
+        return []
+
+def get_connected_nodes(node_id: str, db_name: str, context: str = None) -> List[Dict[str, Any]]:
+    """Get connected nodes based on context."""
+    if context == 'workers':
+        return get_connected_nodes_for_workers(node_id, db_name)
+    
+    # Default query for other contexts
+    query = """
+    MATCH (n {unique_id: $node_id})-[r]-(connected)
+    RETURN DISTINCT connected.unique_id as id, connected.path as path, 
+           connected.name as label, labels(connected)[0] as type
+    """
+    try:
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query, node_id=node_id)
+            return [dict(record) for record in result]
+    except Exception as e:
+        logger.error(f"Error getting connected nodes: {str(e)}")
+        return []
+
+## Worker Navigation
+
+def get_worker_structure(db_name: str) -> Dict[str, Any]:
+    """Get the complete worker structure including schools, departments, timetables, classes, and lessons."""
+    try:
+        query = """
+        // Match all worker-related nodes
+        MATCH (s:School)
+        OPTIONAL MATCH (s)-[:HAS_DEPARTMENT]->(d:Department)
+        OPTIONAL MATCH (d)-[:HAS_TIMETABLE]->(t:UserTeacherTimetable)
+        OPTIONAL MATCH (t)-[:HAS_CLASS]->(c:Class)
+        OPTIONAL MATCH (c)-[:HAS_LESSON]->(l:TimetableLesson)
+        WITH s, d, t, c, l
+        ORDER BY s.school_name, d.department_code, t.name, c.class_code, l.start_time
+
+        // Collect all nodes
+        RETURN {
+            schools: collect(DISTINCT {
+                id: s.unique_id,
+                path: s.path,
+                name: s.school_name,
+                __primarylabel__: 'School'
+            }),
+            departments: collect(DISTINCT {
+                id: d.unique_id,
+                path: d.path,
+                code: d.department_code,
+                school_id: s.unique_id,
+                __primarylabel__: 'Department'
+            }),
+            timetables: collect(DISTINCT {
+                id: t.unique_id,
+                path: t.path,
+                name: t.name,
+                department_id: d.unique_id,
+                __primarylabel__: 'UserTeacherTimetable'
+            }),
+            classes: collect(DISTINCT {
+                id: c.unique_id,
+                path: c.path,
+                code: c.class_code,
+                timetable_id: t.unique_id,
+                __primarylabel__: 'Class'
+            }),
+            lessons: collect(DISTINCT {
+                id: l.unique_id,
+                path: l.path,
+                start_time: l.start_time,
+                class_id: c.unique_id,
+                __primarylabel__: 'TimetableLesson'
+            })
+        } as structure
+        """
+
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query)
+            record = result.single()
+            if not record:
+                logger.error('No worker structure found')
+                return None
+            
+            return {
+                "status": "success",
+                "structure": record["structure"]
+            }
+
+    except Exception as e:
+        logger.error(f"Error getting worker structure: {str(e)}")
+        return None
+
+def get_school_node(school_id: str, db_name: str) -> Optional[Dict[str, Any]]:
+    """Get a specific school node."""
+    query = """
+    MATCH (s:School {unique_id: $school_id})
+    RETURN {
+        id: s.unique_id,
+        path: s.path,
+        name: s.school_name,
+        __primarylabel__: 'School'
+    } as node
+    """
+    try:
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query, school_id=school_id)
+            record = result.single()
+            return record["node"] if record else None
+    except Exception as e:
+        logger.error(f"Error getting school node: {str(e)}")
+        return None
+
+def get_department_node(dept_id: str, db_name: str) -> Optional[Dict[str, Any]]:
+    """Get a specific department node."""
+    query = """
+    MATCH (d:Department {unique_id: $dept_id})
+    RETURN {
+        id: d.unique_id,
+        path: d.path,
+        code: d.department_code,
+        __primarylabel__: 'Department'
+    } as node
+    """
+    try:
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query, dept_id=dept_id)
+            record = result.single()
+            return record["node"] if record else None
+    except Exception as e:
+        logger.error(f"Error getting department node: {str(e)}")
+        return None
+
+def get_timetable_node(timetable_id: str, db_name: str) -> Optional[Dict[str, Any]]:
+    """Get a specific timetable node."""
+    query = """
+    MATCH (t:UserTeacherTimetable {unique_id: $timetable_id})
+    RETURN {
+        id: t.unique_id,
+        path: t.path,
+        name: t.name,
+        __primarylabel__: 'UserTeacherTimetable'
+    } as node
+    """
+    try:
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query, timetable_id=timetable_id)
+            record = result.single()
+            return record["node"] if record else None
+    except Exception as e:
+        logger.error(f"Error getting timetable node: {str(e)}")
+        return None
+
+def get_class_node(class_id: str, db_name: str) -> Optional[Dict[str, Any]]:
+    """Get a specific class node."""
+    query = """
+    MATCH (c:Class {unique_id: $class_id})
+    RETURN {
+        id: c.unique_id,
+        path: c.path,
+        code: c.class_code,
+        __primarylabel__: 'Class'
+    } as node
+    """
+    try:
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query, class_id=class_id)
+            record = result.single()
+            return record["node"] if record else None
+    except Exception as e:
+        logger.error(f"Error getting class node: {str(e)}")
+        return None
+
+def get_lesson_node(lesson_id: str, db_name: str) -> Optional[Dict[str, Any]]:
+    """Get a specific lesson node."""
+    query = """
+    MATCH (l:TimetableLesson {unique_id: $lesson_id})
+    RETURN {
+        id: l.unique_id,
+        path: l.path,
+        start_time: l.start_time,
+        __primarylabel__: 'TimetableLesson'
+    } as node
+    """
+    try:
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query, lesson_id=lesson_id)
+            record = result.single()
+            return record["node"] if record else None
+    except Exception as e:
+        logger.error(f"Error getting lesson node: {str(e)}")
+        return None
+
+def get_current_lesson(db_name: str) -> Optional[Dict[str, Any]]:
+    """Get the current or next upcoming lesson."""
+    now = datetime.now().strftime("%Y-%m-%d %H:%M")
+    query = """
+    MATCH (l:TimetableLesson)
+    WHERE l.start_time >= $now
+    RETURN {
+        id: l.unique_id,
+        path: l.path,
+        start_time: l.start_time,
+        __primarylabel__: 'TimetableLesson'
+    } as node
+    ORDER BY l.start_time ASC
+    LIMIT 1
+    """
+    try:
+        with driver_tools.get_session(database=db_name) as session:
+            result = session.run(query, now=now)
+            record = result.single()
+            return record["node"] if record else None
+    except Exception as e:
+        logger.error(f"Error getting current lesson: {str(e)}")
+        return None

modules/database/tools/neo4j_db_formatter.py

@@ -0,0 +1,21 @@
+def format_user_email_for_neo_db(user_email):
+    """Format user email for Neo4j database name.
+    
+    Neo4j database names can only contain letters, numbers, dots, and dashes.
+    We'll convert the email to a valid format:
+    [email protected] -> ccuser-example-at-domain-com
+    
+    Args:
+        user_email: Email address to format
+        
+    Returns:
+        Formatted string suitable for Neo4j database name
+    """
+    # Convert to lowercase and replace special characters
+    sanitized = user_email.lower()
+    sanitized = sanitized.replace('@', 'at')
+    sanitized = sanitized.replace('.', 'dot')
+    sanitized = sanitized.replace('_', 'underscore')
+    sanitized = sanitized.replace('-', 'dash')
+
+    return f"{sanitized}"

modules/database/tools/neo4j_driver_tools.py

@@ -0,0 +1,153 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import time
+from typing import Optional, Tuple, Generator
+from modules.logger_tool import initialise_logger
+from neo4j import GraphDatabase as gd, Driver, Session
+from contextlib import contextmanager
+
+logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
+
+def _retry_with_backoff(
+    func,
+    max_attempts: int = 10,  # Increased from 3 to 10
+    initial_delay: float = 2.0,  # Increased from 1 to 2 seconds
+    max_total_wait: float = 60.0,  # Maximum total time to wait (60 seconds)
+    max_delay: float = 10.0  # Maximum delay between retries
+) -> any:
+    """
+    Helper function to retry operations with exponential backoff.
+    
+    Args:
+        func: Function to retry
+        max_attempts: Maximum number of retry attempts
+        initial_delay: Initial delay between retries in seconds
+        max_total_wait: Maximum total time to wait before giving up
+        max_delay: Maximum delay between retries
+    """
+    attempt = 0
+    delay = initial_delay
+    start_time = time.time()
+    
+    while attempt < max_attempts:
+        try:
+            return func()
+        except Exception as e:
+            attempt += 1
+            elapsed_time = time.time() - start_time
+            
+            # Check if we've exceeded the maximum total wait time
+            if elapsed_time >= max_total_wait:
+                logger.error(f"Exceeded maximum total wait time of {max_total_wait} seconds")
+                raise
+            
+            if attempt == max_attempts:
+                logger.error(f"Final attempt {attempt} failed: {e}")
+                raise
+            
+            # Calculate next delay with exponential backoff, but cap it
+            delay = min(delay * 2, max_delay)
+            
+            # If we're in a container initialization scenario, provide more context
+            if "Connection refused" in str(e):
+                logger.warning(
+                    f"Attempt {attempt} failed: Connection refused. "
+                    f"This might indicate that Neo4j is still starting up. "
+                    f"Retrying in {delay:.1f} seconds... "
+                    f"(Total elapsed: {elapsed_time:.1f}s)"
+                )
+            else:
+                logger.warning(f"Attempt {attempt} failed: {e}. Retrying in {delay:.1f} seconds...")
+            
+            time.sleep(delay)
+
+def get_driver(db_name: Optional[str] = None, url: Optional[str] = None, auth: Optional[Tuple[str, str]] = None) -> Optional[Driver]:
+    if url is None:
+        url = os.getenv("APP_BOLT_URL")
+        username = os.getenv("USER_NEO4J")
+        password = os.getenv("PASSWORD_NEO4J")
+        if not username or not password:
+            logger.error("Neo4j credentials not found in environment")
+            return None
+        auth = (username, password)
+    
+    if auth is None:
+        logger.error("No authentication credentials provided")
+        return None
+
+    def create_driver():
+        logger.info(f"Attempting to connect to Neo4j at {url}")
+        driver = gd.driver(url, auth=auth)
+        driver.verify_connectivity()
+        logger.info(f"Connected to Neo4j at {url}")
+        return driver
+
+    try:
+        # Use more lenient retry parameters for initial connection
+        driver = _retry_with_backoff(
+            create_driver,
+            max_attempts=10,
+            initial_delay=2.0,
+            max_total_wait=60.0,
+            max_delay=10.0
+        )
+    except Exception as e:
+        logger.error(f"Failed to establish Neo4j connection after all retries: {e}")
+        return None
+    
+    # Test the connection with the specific database
+    if db_name and driver:
+        def verify_database():
+            with driver.session(database=db_name) as session:
+                result = session.run("RETURN 'Connection successful' AS message")
+                record = result.single()
+                if not record or not record.get("message"):
+                    raise Exception(f"Failed to verify database {db_name} connection")
+                logger.info(f"Connection to Neo4j at {url} with database {db_name} successful")
+
+        try:
+            # Use more lenient retry parameters for database verification
+            _retry_with_backoff(
+                verify_database,
+                max_attempts=10,
+                initial_delay=2.0,
+                max_total_wait=60.0,
+                max_delay=10.0
+            )
+        except Exception as e:
+            logger.error(f"Failed to connect to database {db_name} after all retries: {e}")
+            driver.close()
+            return None
+
+    return driver
+
+def close_driver(driver: Optional[Driver]) -> None:
+    if driver:
+        logger.info("Closing driver")
+        driver.close()
+
+# Global driver instance
+_driver: Optional[Driver] = None
+
+def get_global_driver() -> Optional[Driver]:
+    """Get or create the global Neo4j driver instance."""
+    global _driver
+    if _driver is None:
+        _driver = get_driver()
+    return _driver
+
+@contextmanager
+def get_session(database: Optional[str] = None) -> Generator[Session, None, None]:
+    """Get a Neo4j session using the global driver."""
+    driver = get_global_driver()
+    if driver is None:
+        raise Exception("Failed to get Neo4j driver")
+    
+    session = None
+    try:
+        session = driver.session(database=database)
+        yield session
+    finally:
+        if session:
+            session.close()

modules/database/tools/neo4j_http_tools.py

@@ -0,0 +1,64 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'api_modules_database_tools_neo4j_http_tools'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+import requests
+import base64
+
+dev_mode = os.getenv('DEV_MODE', 'false')
+
+def send_query(query, encoded_credentials=None, params=None, method='POST', database="system", endpoint="/tx/commit"):
+    if encoded_credentials is None:
+        logging.debug(f"Sending query to Neo4j: {query}")
+        credentials = f"{os.getenv('USER_NEO4J')}:{os.getenv('PASSWORD_NEO4J')}"
+        encoded_credentials = base64.b64encode(credentials.encode()).decode('utf-8')
+        logging.debug(f"Encoded credentials: {encoded_credentials}")
+    
+    # Use HTTPS for production, HTTP for development
+    neo4j_url = f"{os.getenv('APP_GRAPH_URL')}/db/{database}{endpoint}"
+    logging.debug(f"URL: {neo4j_url}")
+    headers = {'Content-Type': 'application/json', 'Authorization': f'Basic {encoded_credentials}'}
+    logging.debug(f"Headers: {headers}")
+    data = {
+        "statements": [{
+            "statement": query,
+            "parameters": params or {}
+        }]
+    }
+    logging.debug(f"Data: {data}")
+    
+    try:
+        logging.debug(f"Sending request to Neo4j...")
+        response = requests.request(method, neo4j_url, json=data, headers=headers)
+        response.raise_for_status()  # Raise an HTTPError for bad responses
+        logging.debug(f"Response status code: {response.status_code}")
+        logging.debug(f"Response content: {response.content}")
+        return response.json()
+    except requests.exceptions.RequestException as e:
+        logging.error(f"Request to Neo4j failed: {e}")
+        raise
+
+def create_node(node_type: str, node_data: dict, db=None):
+    query = f"CREATE (n:{node_type} $props) RETURN id(n)"
+    params = {"props": node_data}
+    response = send_query(query, database=db, params=params)
+    return response['results'][0]['data'][0]['meta'][0]['id']
+
+def create_relationship(relationship_data: dict, db=None):
+    query = """
+    MATCH (a), (b) WHERE id(a) = $start_id AND id(b) = $end_id
+    CREATE (a)-[r:{rel_type}]->(b)
+    RETURN r
+    """
+    params = {"start_id": relationship_data['start_node']['id'], "end_id": relationship_data['end_node']['id'], "rel_type": relationship_data['relationship_type'], "props": relationship_data.get('properties', {})}
+    return send_query("/db/neo4j/tx/commit", query, params, db=db)

modules/database/tools/neo4j_session_tools.py

@@ -0,0 +1,504 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'api_modules_database_tools_neo4j_session_tools'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+import modules.database.tools.queries as query
+from contextlib import suppress
+
+def get_node_by_unique_id_and_adjacent_nodes(session, unique_id):
+    return session.read_transaction(_get_node_by_unique_id_and_adjacent_nodes, unique_id)
+
+def _get_node_by_unique_id_and_adjacent_nodes(tx, unique_id):
+    query = """
+    MATCH (n {unique_id: $unique_id})
+    OPTIONAL MATCH (n)-[r]-(adjacent)
+    RETURN n AS node, COLLECT(DISTINCT {node: adjacent, relationship: r}) AS connected_nodes
+    """
+    result = tx.run(query, unique_id=unique_id)
+    record = result.single()
+    if record:
+        node = record["node"]
+        connected_nodes = record["connected_nodes"]
+        return {"node": node, "connected_nodes": connected_nodes}
+    return None
+
+def delete_all_nodes_and_relationships(session):
+    total_deleted = 0
+    while True:
+        deleted_count = session.write_transaction(_delete_batch)
+        total_deleted += deleted_count
+        if deleted_count == 0:
+            break
+
+def _delete_batch(tx):
+    result_data = tx.run(query.delete_batch, batch_size=10000).single()
+    return 0 if result_data is None else result_data[0]
+
+def delete_all_constraints(session):
+    if show_constraints_result := session.run(query.show_constraints).data():
+        for constraint in show_constraints_result:
+            constraint_name = constraint['name']
+            session.run(query.drop_constraint(constraint_name))
+        
+def reset_all_indexes(session):
+    indexes = session.run(query.show_indexes).data()
+    for index in indexes:
+        index_name = index['name']
+        session.run(query.drop_index(index_name))
+        
+def reset_databases(session):
+    delete_all_nodes_and_relationships(session)
+    delete_all_constraints(session)
+    reset_all_indexes(session)
+    
+def close_session(session):
+    if session:
+        with suppress(Exception):
+            session.close()
+
+def create_node(session, label, properties, returns=False):
+    """
+    Function to create a node in Neo4j database.
+    
+    Args:
+        driver (neo4j.Driver): The Neo4j driver.
+        session (str): The Neo4j session.
+        label (str): The label of the node.
+        properties (dict): A dictionary of properties for the node.
+        
+    Example usage:
+        create_node(neo4j_driver, "Topic", {"TopicID": "AP.PAG10", "Title": "Topic 10"})
+        
+    Returns:
+        None
+    """
+    transaction = session.write_transaction(_create_node, label, properties)
+    if returns:
+        transaction_id = transaction.id
+        # logging.database(f"Created {label} node with transaction ID {transaction_id} and properties {properties}")
+        print(f"Created {label} node with transaction ID {transaction_id} and properties {properties}")
+        return find_node_by_transaction_id(session, transaction_id)
+    else:
+        # logging.warning(f"Failed to create {label} node with properties {properties}")
+        print(f"Failed to create {label} node with properties {properties}")
+        return None
+
+def _create_node(tx, label, properties):
+    query = f"""
+    CREATE (n:{label} $properties)
+    RETURN n
+    """
+    # logging.query(f"Running query: {query}")
+    print(f"Running query: {query}")
+    result = tx.run(query, properties=properties)
+    return result.single()[0] if result.single() is not None else None  # Handle no record found
+
+# Function to find a node by its element ID
+def find_node_by_transaction_id(session, transaction_id):
+    """
+    Function to find a node in Neo4j database by its element ID.
+
+    Args:
+        driver (neo4j.Driver): The Neo4j driver.
+        element_id (str): The element ID of the node to find.
+
+    Returns:
+        The matched node.
+    """
+    return session.read_transaction(_find_node_by_element_id, transaction_id)
+    
+def _find_node_by_element_id(tx, transaction_id):
+    query = """
+    MATCH (n)
+    WHERE id(n) = $transaction_id
+    RETURN n
+    """
+    # logging.query(f"Running query: {query}")
+    result = tx.run(query, transaction_id=transaction_id)
+    record = result.single()  # Get the single result record, if any
+    return record[0] if record is not None else None  # Handle no record found
+
+# Function to create a relationship between two nodes in Neo4j
+def create_relationship(session, start_node, end_node, label, properties=None, returns=False):
+    """
+    Function to create a relationship between two nodes in Neo4j database.
+    
+    Args:
+        driver (neo4j.Driver): The Neo4j driver.
+        session (str): The Neo4j session.
+        start_node (str): The ID of the start node.
+        end_node (str): The ID of the end node.
+        rel_type (str): The type of the relationship.
+        properties (dict): A dictionary of properties for the relationship.
+        
+    Example usage:
+        create_relationship(neo4j_driver, "AP.PAG10", "AP.PAG11", "HAS_NEXT")
+        
+    Returns:
+        None
+    """
+    relationship = session.write_transaction(_create_relationship, start_node, end_node, label, properties)
+    if returns:
+        relationship_id = relationship.id
+        return find_relationship_by_relationship_id(session, relationship_id)
+    else:
+        return None
+        
+def _create_relationship(tx, start_node, end_node, label, properties):
+    query = f"""
+    MATCH (a), (b)
+    WHERE ID(a) = $start_node_id AND ID(b) = $end_node_id
+    CREATE (a)-[r:{label}]->(b)
+    RETURN r
+    """
+    # logging.query(f"Running query: {query}")
+    result = tx.run(query, start_node_id=start_node.id, end_node_id=end_node.id, properties=properties)
+    single_result = result.single()
+    return single_result[0] if single_result is not None else None
+
+def order_list_of_nodes_by_property(session, label, property_name, order="ASC"):
+    """
+    Function to order a list of nodes in Neo4j database by a property.
+
+    Args:
+        driver (neo4j.Driver): The Neo4j driver.
+        label (str): The label of the nodes to find.
+        property_name (str): The name of the property to order by.
+        order (str): The order of the sorting (ASC or DESC).
+
+    Returns:
+        List of matched nodes.
+    """
+    return session.read_transaction(_order_list_of_nodes_by_property, label, property_name, order)
+
+def _order_list_of_nodes_by_property(tx, label, property_name, order):
+    query = f"""
+    MATCH (n:{label})
+    RETURN n
+    ORDER BY n.{property_name} {order}
+    """
+    # logging.query(f"Running query: {query}")
+    result = tx.run(query)
+    return [record["n"] for record in result]
+
+def find_relationship_by_relationship_id(session, relationship_id):
+    """
+    Function to find a relationship in Neo4j database by its relationship ID.
+
+    Args:
+        driver (neo4j.Driver): The Neo4j driver.
+        relationship_id (str): The relationship ID of the relationship to find.
+
+    Returns:
+        The matched relationship.
+    """
+    return session.read_transaction(_find_relationship_by_relationship_id, relationship_id)
+    
+def _find_relationship_by_relationship_id(tx, relationship_id):
+    query = """
+    MATCH ()-[r]->()
+    WHERE id(r) = $relationship_id
+    """
+    # logging.query(f"Running query: {query}")
+    print(f"Running query: {query}")
+    result = tx.run(query, relationship_id=relationship_id)
+    record = result.single()  # Get the single result record, if any
+    return record[0] if record is not None else None  # Handle no record found
+
+# Function to find nodes in Neo4j database by label
+def find_nodes_by_label(session, label):
+    """
+    Function to find nodes in Neo4j database by label.
+
+    Args:
+        driver (neo4j.Driver): The Neo4j driver.
+        label (str): The label of the nodes to find.
+        
+    Example usage:
+        find_nodes_by_label(neo4j_driver, "Topic")
+
+    Returns:
+        List of matched nodes.
+    """
+    return session.read_transaction(_find_nodes_by_label, label)
+
+def _find_nodes_by_label(tx, label):
+    query = f"""
+    MATCH (n:{label})
+    RETURN n
+    """
+    # logging.query(f"Running query: {query}")
+    print(f"Running query: {query}")
+    result = tx.run(query)
+    return [record["n"] for record in result]
+
+def get_node_by_unique_id(session, unique_id):
+    return session.read_transaction(_get_node_by_unique_id, unique_id)
+
+def _get_node_by_unique_id(tx, unique_id):
+    query = f"""
+    MATCH (n)
+    WHERE n.unique_id = $unique_id
+    RETURN n
+    """
+    logging.debug(f"Executing query with unique_id: {unique_id}")
+    result = tx.run(query, unique_id=unique_id)
+    record = result.single()
+    if record is None:
+        logging.warning(f"No node found with unique_id: {unique_id}")
+        return None
+    return record[0]
+
+# Function to find nodes in Neo4j database by label and properties
+def find_nodes_by_label_and_properties(session, label, properties):
+    """
+    Function to find nodes in Neo4j database by label and properties.
+
+    Args:
+        session (neo4j.Session): The Neo4j session.
+        label (str): The label of the nodes to find.
+        properties (dict): A dictionary of properties to match.
+
+    Returns:
+        List of matched nodes.
+    """
+    logging.debug(f"Finding nodes with label: {label} and properties: {properties}")
+    with session:
+        response = session.read_transaction(_find_nodes_by_label_and_properties, label, properties)
+        logging.debug(f"Response: {response}")
+        return response
+
+def _find_nodes_by_label_and_properties(tx, label, properties):
+    query = f"""
+    MATCH (n:{label})
+    WHERE {' AND '.join([f'n.{key} = ${key}' for key in properties.keys()])}
+    RETURN n
+    """
+    logging.debug(f"Running query: {query}")
+    result = tx.run(query, **properties)
+    logging.debug(f"Result: {result}")
+    return [record["n"] for record in result]
+
+# Function to find relationships in Neo4j database by type
+def find_relationships_by_type(session, rel_type):
+    """
+    Function to find relationships in Neo4j database by type.
+
+    Args:
+        driver (neo4j.Driver): The Neo4j driver.
+        rel_type (str): The type of the relationships to find.
+
+    Returns:
+        List of matched relationships.
+    """
+    return session.read_transaction(_find_relationships_by_type, rel_type)
+    
+def _find_relationships_by_type(tx, rel_type):
+    query = f"""
+    MATCH ()-[r:{rel_type}]->()
+    RETURN r
+    """
+    # logging.query(f"Running query: {query}")
+    print(f"Running query: {query}")
+    result = tx.run(query)
+    return [record["r"] for record in result]
+
+# Function to find relationships in Neo4j database by type and properties
+def find_relationships_by_type_and_properties(session, label, properties):
+    """
+    Function to find relationships in Neo4j database by type and properties.
+
+    Args:
+        driver (neo4j.Driver): The Neo4j driver.
+        rel_type (str): The type of the relationships to find.
+        properties (dict): A dictionary of properties to match.
+
+    Returns:
+        List of matched relationships.
+    """
+    return session.read_transaction(_find_relationships_by_type_and_properties, label, properties)
+    
+
+def _find_relationships_by_type_and_properties(tx, label, properties):
+    query = f"""
+    MATCH (a)-[r:{label}]->(b)
+    WHERE {' AND '.join([f'r.{key} = ${key}' for key in properties.keys()])}
+    RETURN r
+    """
+    # logging.query(f"Running query: {query}")
+    print(f"Running query: {query}")
+    result = tx.run(query, **properties)
+    return [record["r"] for record in result]
+
+# Function to find nodes and relationships in Neo4j database by label and properties
+def find_nodes_and_relationships_by_label_and_properties(session, label, properties):
+    """
+    Function to find nodes and relationships in Neo4j database by label and properties.
+
+    Args:
+        driver (neo4j.Driver): The Neo4j driver.
+        label (str): The label of the nodes to find.
+        properties (dict): A dictionary of properties to match.
+
+    Returns:
+        List of matched nodes and relationships.
+    """
+    return session.read_transaction(_find_nodes_and_relationships_by_label_and_properties, label, properties)
+    
+def _find_nodes_and_relationships_by_label_and_properties(tx, label, properties):
+    query = f"""
+    MATCH (n:{label})
+    WHERE {' AND '.join([f'n.{key} = ${key}' for key in properties.keys()])}
+    RETURN n
+    """
+    # logging.query(f"Running query: {query}")
+    result = tx.run(query, **properties)
+    return [record["n"] for record in result]
+
+# Function to delete nodes in Neo4j based on given criteria
+def delete_nodes(session, criteria, delete_related=False):
+    """
+    Function to delete nodes in Neo4j based on given criteria.
+
+    Args:
+        driver (neo4j.Driver): The Neo4j driver.
+        criteria (dict): A dictionary containing the properties to match for deletion.
+        delete_related (bool): If True, deletes related nodes and relationships; otherwise, deletes only the matched nodes.
+
+    Example usage:
+        # Delete only the nodes matching the criteria
+        delete_nodes(neo4j_driver, {'TopicID': 'AP.PAG10'})
+
+        # Delete the nodes and their related relationships
+        delete_nodes(neo4j_driver, {'TopicID': 'AP.PAG10'}, delete_related=True)
+    """
+    session.write_transaction(_delete_nodes, criteria, delete_related)
+
+def _delete_nodes(tx, criteria, delete_related=False):
+    """
+    Internal function to execute a Cypher query to delete nodes based on criteria.
+
+    Args:
+        tx (neo4j.Transaction): The Neo4j transaction.
+        criteria (dict): A dictionary containing the properties to match for deletion.
+        delete_related (bool): Specifies whether to delete related nodes and relationships.
+    """
+    condition_str = " AND ".join([f"n.{key} = ${key}" for key in criteria])
+    if delete_related:
+        query = f"""
+        MATCH (n)-[r]-()
+        WHERE {condition_str}
+        DELETE n, r
+        """
+    else:
+        query = f"""
+        MATCH (n)
+        WHERE {condition_str}
+        DELETE n
+        """
+    # logging.query(f"Running query: {query}")
+    tx.run(query, **criteria)
+
+# Function to delete all nodes and relationships in the Neo4j database in batches
+def delete_lots_of_nodes_and_relationships(session):
+    """
+    Function to delete all nodes and relationships in the Neo4j database in batches.
+
+    Args:
+        driver (neo4j.Driver): The Neo4j driver.
+        session (str): The Neo4j session.
+    """
+    total_deleted = 0
+    while True:
+        deleted_count = session.write_transaction(_delete_batch)
+        total_deleted += deleted_count
+        if deleted_count == 0:
+            break  # Exit the loop if no more nodes are deleted
+    # logging.prod(f"All nodes and relationships have been deleted. Total deleted: {total_deleted}")
+    print(f"Neo4j:  All nodes and relationships have been deleted. Total deleted: {total_deleted}")
+
+def _delete_batch(tx):
+    """
+    Function to execute a Cypher query to delete a batch of nodes and relationships.
+
+    Args:
+        tx (neo4j.Transaction): The Neo4j transaction.
+    """
+    batch_size = 10000  # Adjust the batch size according to your needs
+    query = """
+    MATCH (n)
+    WITH n LIMIT $batch_size
+    DETACH DELETE n
+    RETURN count(*)
+    """
+    # logging.query(f"Running query: {query}")
+    result = tx.run(query, batch_size=batch_size)
+    result_data = result.single()
+    
+    if result_data is None:
+        return 0
+    deleted_count = result_data[0]
+    if deleted_count is None:  # This check might be redundant, but kept for clarity
+        return 0
+    if deleted_count > 0:
+        # logging.database(f"Deleted {deleted_count} nodes.")
+        print(f"Neo4j: Deleted {deleted_count} nodes.")
+    return deleted_count
+
+def delete_all_constraints(session):
+    # Correct command to fetch all constraints for Neo4j 4.x and later
+    constraints_query = "SHOW CONSTRAINTS"
+    # logging.query(f"Running query: {constraints_query}")
+    if constraints_query_result := session.run(constraints_query).data():
+        for constraint in constraints_query_result:
+            # Ensure correct key is used to extract constraint name
+            constraint_name = constraint['name']  # Adjust this if necessary
+            drop_query = f"DROP CONSTRAINT {constraint_name}"
+            # logging.query(f"Running query: {drop_query}")
+            session.run(drop_query)
+            # logging.database(f"Dropped constraint: {constraint_name}")
+            print(f"Neo4j: Dropped constraint: {constraint_name}")
+    else:
+        # logging.warning("No constraints found to delete.")
+        print("Neo4j: No constraints found to delete.")
+        
+def reset_all_indexes(session):
+    indexes = session.run("SHOW INDEXES").data()
+    for index in indexes:
+        index_name = index['name']
+        session.run(f"DROP INDEX {index_name}")
+        # logging.info(f"Deleted index: {index_name}")
+        print(f"Neo4j: Deleted index: {index_name}")
+
+def reset_database_in_session(session):
+    logging.debug("Neo4j: Resetting database")
+    delete_lots_of_nodes_and_relationships(session)
+    delete_all_constraints(session)
+    reset_all_indexes(session)
+    logging.info("Neo4j: Database reset")
+
+def create_database(session, db_name):
+    """
+    Creates a new database in Neo4j if it does not already exist.
+
+    Args:
+        session (neo4j.Session): The Neo4j session.
+        db_name (str): The name of the database to create.
+    """
+    logging.debug(f"Neo4j: Creating database {db_name}")
+    query = f"CREATE DATABASE `{db_name}` IF NOT EXISTS"
+    try:
+        session.run(query)
+        logging.info(f"Neo4j: Database {db_name} created successfully.")
+    except Exception as e:
+        logging.error(f"Neo4j: Failed to create database {db_name}: {str(e)}")

modules/database/tools/neontology/__init__.py

@@ -0,0 +1,18 @@
+# flake8: noqa
+
+from .basenode import BaseNode
+from .baserelationship import BaseRelationship
+from .graphconnection import GraphConnection, init_neontology
+from .utils import auto_constrain
+
+__all__ = [
+    # BaseNode
+    "BaseNode",
+    # BaseRelationship
+    "BaseRelationship",
+    # GraphConnection
+    "init_neontology",
+    "GraphConnection",
+    # utils
+    "auto_constrain",
+]

modules/database/tools/neontology/basenode.py

@@ -0,0 +1,315 @@
+from typing import Any, ClassVar, Dict, List, Optional, Type, TypeVar, Union
+
+import numpy as np
+import pandas as pd
+
+from .commonmodel import CommonModel
+from .graphconnection import GraphConnection
+
+B = TypeVar("B", bound="BaseNode")
+
+class BaseNode(CommonModel):  # pyre-ignore[13]
+    __primaryproperty__: ClassVar[str]
+    __primarylabel__: ClassVar[Optional[str]]
+    __secondarylabels__: ClassVar[Optional[list]] = []
+
+    def __init__(self, **data: dict):
+        super().__init__(**data)
+
+        # we can define 'abstract' nodes which don't have a label
+        # these are to provide common properties to be used by subclassed nodes
+        # but shouldn't be put in the graph or even instantiated
+        if self.__primarylabel__ is None:
+            raise NotImplementedError(
+                "Nodes to be used in the graph must define a primary label."
+            )
+
+    def _get_merge_parameters(self) -> Dict[str, Any]:
+        """
+
+        Returns:
+            Dict[str, Any]: a dictionary of key/value pairs.
+        """
+
+        params = {
+            "pp": self.neo4j_dict()[self.__primaryproperty__],
+            "always_set": self._get_prop_values(self._always_set),
+            "set_on_match": self._get_prop_values(self._set_on_match),
+            "set_on_create": self._get_prop_values(self._set_on_create),
+        }
+
+        return params
+
+    def get_primary_property_value(self) -> Union[str, int]:
+        return self._get_merge_parameters()["pp"]
+
+    def create(self, database: str = 'neo4j') -> None:
+        """Create this node in the graph."""
+
+        params = self.neo4j_dict()
+
+        all_props = self.neo4j_dict()
+
+        pp_value = all_props.pop(self.__primaryproperty__)
+
+        params = {"pp": pp_value, "all_props": all_props}
+
+        all_labels = [self.__primarylabel__] + self.__secondarylabels__
+
+        cypher = f"""
+        CREATE (n:{":".join(all_labels)} {{ {self.__primaryproperty__}: $pp }})
+        SET n += $all_props
+        RETURN n
+        """
+        graph = GraphConnection()
+        with graph.driver.session(database=database) as session:
+            result = session.run(cypher, params).single()
+            if result:
+                return self.__class__(**dict(result["n"]))
+            return None
+
+    def merge(self, database: str = 'neo4j') -> None:
+        """Merge this node into the graph."""
+
+        params = self._get_merge_parameters()
+
+        all_labels = [self.__primarylabel__] + self.__secondarylabels__
+
+        cypher = f"""
+        MERGE (n:{":".join(all_labels)} {{ {self.__primaryproperty__}: $pp }})
+        ON MATCH SET n += $set_on_match
+        ON CREATE SET n += $set_on_create
+        SET n += $always_set
+        RETURN n
+        """
+
+        graph = GraphConnection()
+        with graph.driver.session(database=database) as session:
+            result = session.run(cypher, params).single()
+            if result:
+                return self.__class__(**dict(result["n"]))
+            return None
+
+    @classmethod
+    def create_nodes(cls: Type[B], nodes: List[B]) -> List[Union[str, int]]:
+        """Create the given nodes in the database.
+
+        Args:
+            nodes (List[B]): A list of nodes to create.
+
+        Returns:
+            list: A list of the primary property values
+
+        Raises:
+            TypeError: Raised if one of the nodes isn't of this type.
+        """
+
+        for node in nodes:
+            if isinstance(node, cls) is False:
+                raise TypeError("Node was incorrect type.")
+
+        node_list = [
+            {"props": x.neo4j_dict(), "pp": x.neo4j_dict()[cls.__primaryproperty__]}
+            for x in nodes
+        ]
+
+        all_labels = [cls.__primarylabel__] + cls.__secondarylabels__
+
+        cypher = f"""
+        UNWIND $node_list AS node
+        create (n:{":".join(all_labels)} {{{cls.__primaryproperty__}: node.pp}})
+        SET n = node.props
+        RETURN n
+        """
+
+        graph = GraphConnection()
+        results = graph.cypher_write_many(
+            cypher=cypher, params={"node_list": node_list}
+        )
+
+        matched_nodes = [cls(**dict(x["n"])) for x in results]
+
+        return matched_nodes
+
+    @classmethod
+    def merge_nodes(cls: Type[B], nodes: List[B]) -> List[B]:
+        """Merge multiple nodes into the database.
+
+        Args:
+            nodes (List[B]): A list of nodes to merge.
+
+        Returns:
+            list: A list of the primary property values
+
+        Raises:
+            TypeError: Raised if any of the nodes provided don't match this class.
+        """
+
+        for node in nodes:
+            if isinstance(node, cls) is False:
+                raise TypeError("Node was incorrect type.")
+
+        node_list = [x._get_merge_parameters() for x in nodes]
+
+        all_labels = [cls.__primarylabel__] + cls.__secondarylabels__
+
+        cypher = f"""
+        UNWIND $node_list AS node
+        MERGE (n:{":".join(all_labels)} {{{cls.__primaryproperty__}: node.pp}})
+        ON MATCH SET n += node.set_on_match
+        ON CREATE SET n += node.set_on_create
+        SET n += node.always_set
+        RETURN n
+        """
+
+        graph = GraphConnection()
+        results = graph.cypher_write_many(
+            cypher=cypher, params={"node_list": node_list}
+        )
+
+        matched_nodes = [cls(**dict(x["n"])) for x in results]
+
+        return matched_nodes
+
+    @classmethod
+    def merge_records(cls: Type[B], records: dict) -> List[B]:
+        """Take a list of dictionaries and use them to merge in nodes in the graph.
+
+        Each dictionary will be used to merge a node where dictionary key/value pairs
+            represent properties to be applied.
+
+        Returns:
+            list: A list of the primary property values
+
+        Args:
+            records (List[Dict[str, Any]]): a list of dictionaries of node properties
+        """
+
+        nodes = [cls(**x) for x in records]
+
+        return cls.merge_nodes(nodes)
+
+    @classmethod
+    def merge_df(cls: Type[B], df: pd.DataFrame, deduplicate: bool = True) -> pd.Series:
+        """Merge in new nodes based on data in a dataframe.
+
+        The dataframe columns must correspond to the Node properties.
+
+        Returns:
+            pd.Series: A list of the primary property values
+
+        Args:
+            df (pd.DataFrame): A pandas dataframe of node properties
+
+        """
+
+        if df.empty is True:
+            return pd.Series(dtype=object)
+
+        input_df = df.replace([np.nan], None).copy()
+
+        if deduplicate is True:
+            # we don't wan't to waste time attempting to merge identical records
+            unique_df = input_df.drop_duplicates(ignore_index=True).copy()
+        else:
+            unique_df = input_df
+
+        records = unique_df.to_dict(orient="records")
+
+        unique_df["generated_nodes"] = pd.Series(cls.merge_records(records))
+
+        # now we need to get the mapping from unique id to primary property
+        # so that we can return the data in the same shape it was received
+        input_df.insert(0, "ontolocy_merging_order", range(0, len(input_df)))
+        merge_cols = list(input_df.columns)
+        merge_cols.remove("ontolocy_merging_order")
+        output_df = input_df.merge(
+            unique_df,
+            how="inner",
+            on=merge_cols,
+        ).sort_values("ontolocy_merging_order", ignore_index=True)
+
+        return output_df.generated_nodes
+
+    @classmethod
+    def match(cls: Type[B], pp: str) -> Optional[B]:
+        """MATCH a single node of this type with the given primary property.
+
+        Args:
+            pp (str): The value of the primary property (pp) to match on.
+
+        Returns:
+            Optional[B]: If the node exists, return it as an instance.
+        """
+
+        cypher = f"""
+        MATCH (n:{cls.__primarylabel__})
+        WHERE n.{cls.__primaryproperty__} = $pp
+        RETURN n
+        """
+
+        params = {"pp": pp}
+
+        graph = GraphConnection()
+
+        result = graph.cypher_read(cypher, params)
+
+        if result:
+            return cls(**dict(result["n"]))
+
+        else:
+            return None
+
+    @classmethod
+    def delete(cls, pp: str) -> None:
+        """Delete a node from the graph.
+
+        Match on label and the pp value provided.
+        If the node exists, delete it and any relationships it has.
+
+        Args:
+            pp (str): Primary property value to match on.
+        """
+
+        cypher = f"""
+        MATCH (n:{cls.__primarylabel__})
+        WHERE n.{cls.__primaryproperty__} = $pp
+        DETACH DELETE n
+        """
+
+        params = {"pp": pp}
+
+        graph = GraphConnection()
+
+        graph.cypher_write(cypher, params)
+
+    @classmethod
+    def match_nodes(cls: Type[B], limit: int = 100, skip: int = 0) -> List[B]:
+        """Get nodes of this type from the database.
+
+        Run a MATCH cypher query to retrieve any Nodes with the label of this class.
+
+        Args:
+            limit (int, optional): Maximum number of results to return. Defaults to 100.
+            skip (int, optional): Skip through this many results (for pagination). Defaults to 0.
+
+        Returns:
+            Optional[List[B]]: A list of node instances.
+        """
+
+        cypher = f"""
+        MATCH(n:{cls.__primarylabel__})
+        RETURN n{{.*}}
+        ORDER BY n.created DESC
+        SKIP $skip
+        LIMIT $limit
+        """
+
+        params = {"skip": skip, "limit": limit}
+
+        graph = GraphConnection()
+        records = graph.cypher_read_many(cypher, params)
+
+        nodes = [cls(**dict(x["n"])) for x in records]
+
+        return nodes

modules/database/tools/neontology/baserelationship.py

@@ -0,0 +1,305 @@
+"""Defines the BaseRelationship class.
+
+The BaseRelationship class is used for creating and matching on relationships in the graph.
+
+    Typical usage example:
+
+    class MyRel(BaseRelationship):
+
+        __relationshiptype__: ClassVar[Optional[str]] = "MY_REL"
+
+        source: SourceNode
+        target: TargetNode
+
+    my_rel = MyRel(source=source_node, target=target_node)
+    my_rel.merge()
+
+"""
+
+from typing import Any, ClassVar, Dict, List, Optional, Type, TypeVar
+
+import numpy as np
+import pandas as pd
+from pydantic import PrivateAttr
+
+from modules.database.tools.neontology.graphconnection import GraphConnection
+
+from .basenode import BaseNode
+from .commonmodel import CommonModel
+
+R = TypeVar("R", bound="BaseRelationship")
+
+
+class BaseRelationship(CommonModel):  # pyre-ignore[13]
+    source: BaseNode
+    target: BaseNode
+
+    __relationshiptype__: ClassVar[Optional[str]] = None
+
+    _merge_on: List[
+        str
+    ] = PrivateAttr()  # what relationship properties should we merge on
+
+    def __init__(self, **data: dict):
+        super().__init__(**data)
+
+        self._merge_on = self._get_prop_usage("merge_on")
+
+        # we can define 'abstract' relationships which don't have a label
+        # these are to provide common properties to be used by subclassed relationships
+        # but shouldn't be put in the graph or even instantiated
+        if self.__relationshiptype__ is None:
+            raise NotImplementedError(
+                "Nodes to be used in the graph must define a primary label."
+            )
+
+    @classmethod
+    def get_relationship_type(cls) -> str:
+        """Get the relationship type to use for creating and matching this relationship.
+
+        If __relationship__ has been specified, use that.
+
+        Otherwise use the class name in uppercase
+
+        Returns:
+            str: the string to use for creating and matching this relationship
+        """
+        return cls.__relationshiptype__  # pyre-ignore[7]
+
+    def _get_merge_parameters(
+        self, source_prop: str, target_prop: str
+    ) -> Dict[str, Any]:
+        """
+
+        Returns:
+            Dict[str, Any]: a dictionary of key/value pairs.
+        """
+
+        exclusions = {"source", "target"}
+
+        # these properties will be referenced individually
+        merge_props = self._get_prop_values(self._merge_on, exclude=exclusions)
+
+        params = {
+            "source_prop": self.source.neo4j_dict()[source_prop],
+            "target_prop": self.target.neo4j_dict()[target_prop],
+            "always_set": self._get_prop_values(self._always_set, exclude=exclusions),
+            "set_on_match": self._get_prop_values(
+                self._set_on_match, exclude=exclusions
+            ),
+            "set_on_create": self._get_prop_values(
+                self._set_on_create, exclude=exclusions
+            ),
+            **merge_props,
+        }
+
+        return params
+
+    def merge(
+        self,
+        database: Optional[str] = 'neo4j'  # default to 'neo4j' if not specified
+    ) -> None:
+        """Merge this relationship into the database."""
+        source_label = self.source.__primarylabel__
+        target_label = self.target.__primarylabel__
+
+        source_pp = self.source.__primaryproperty__
+        target_pp = self.target.__primaryproperty__
+
+        params = self._get_merge_parameters(
+            source_prop=source_pp, target_prop=target_pp
+        )
+
+        rel_type = self.get_relationship_type()
+
+        # build a string of properties to merge on "prop_name: $prop_name"
+        merge_props = ", ".join([f"{x}: ${x}" for x in self._merge_on])
+
+        cypher = f"""
+        MATCH (source:{source_label} {{ {source_pp}: $source_prop }}),
+            (target:{target_label} {{ {target_pp}: $target_prop }})
+        MERGE (source)-[r:{rel_type} {{ {merge_props} }}]->(target)
+        ON MATCH SET r += $set_on_match
+        ON CREATE SET r += $set_on_create
+        SET r += $always_set
+        """
+
+        graph = GraphConnection()
+        # Use session with database instead of USE statement
+        with graph.driver.session(database=database) as session:
+            session.run(cypher, params)
+
+    @classmethod
+    def merge_relationships(
+        cls: Type[R],
+        rels: List[R],
+        source_type: Optional[Type[BaseNode]] = None,
+        target_type: Optional[Type[BaseNode]] = None,
+        source_prop: Optional[str] = None,
+        target_prop: Optional[str] = None,
+        database: Optional[str] = 'neo4j'  # Add database parameter
+    ) -> None:
+        """Merge multiple relationships (of this type) into the database.
+
+        Sometimes the source and target label may be ambiguous (e.g. where we have subclassed nodes)
+            In this case you can explicitly pass in the relevant types
+
+        Sometimes we want to match nodes on a property which isn't the primary property,
+        so we can specify what property to use.
+
+        Args:
+            cls (Type[R]): this class
+            rels (List[R]): a list of relationships which are instances of this class
+            database (Optional[str]): database to use for the operation
+
+        Raises:
+            TypeError: If relationships are provided which aren't of this class
+        """
+
+        if source_type is None:
+            source_type = cls.model_fields["source"].annotation
+
+        if target_type is None:
+            target_type = cls.model_fields["target"].annotation
+
+        for rel in rels:
+            if isinstance(rel, cls) is False:
+                raise TypeError("Relationship was incorrect type.")
+            if type(rel.source) is not source_type:
+                raise TypeError("Received an inappropriate kind of source node.")
+            if type(rel.target) is not target_type:
+                raise TypeError("Received an inappropriate kind of target node.")
+
+        if source_prop is None:
+            source_prop = source_type.__primaryproperty__
+
+        if target_prop is None:
+            target_prop = target_type.__primaryproperty__
+
+        source_label = source_type.__primarylabel__
+        target_label = target_type.__primarylabel__
+
+        # build a string of properties to merge on "prop_name: $prop_name"
+        # we need to instantiate the class so that _merge_on is generated as part of __init__
+        merge_props = ", ".join([f"{x}: ${x}" for x in cls._get_prop_usage("merge_on")])
+
+        rel_list: List[Dict[str, Any]] = [
+            x._get_merge_parameters(source_prop, target_prop) for x in rels
+        ]
+
+        rel_type = cls.get_relationship_type()
+
+        cypher = f"""
+        UNWIND $rel_list AS rel
+        MATCH (source:{source_label})
+        WHERE source.{source_prop} = rel.source_prop
+        MATCH (target:{target_label})
+        WHERE target.{target_prop} = rel.target_prop
+        MERGE (source)-[r:{rel_type} {{ {merge_props} }}]->(target)
+        ON MATCH SET r += rel.set_on_match
+        ON CREATE SET r += rel.set_on_create
+        SET r += rel.always_set
+        """
+
+        graph = GraphConnection()
+        # Use session with database instead of USE statement
+        with graph.driver.session(database=database) as session:
+            session.run(cypher=cypher, parameters={"rel_list": rel_list})
+
+    @classmethod
+    def merge_records(
+        cls: Type[R],
+        records: List[Dict[str, Any]],
+        source_type: Optional[Type[BaseNode]] = None,
+        target_type: Optional[Type[BaseNode]] = None,
+        source_prop: Optional[str] = None,
+        target_prop: Optional[str] = None,
+    ) -> None:
+        """Take a list of dictionaries and use them to merge in relationships in the graph.
+
+        Sometimes, a relationship can accept nodes which subclass a particular node type.
+            In these instances, it may be necessary to explicitly state what type of node should be used.
+
+        Each record should have a source and target key where the value is the primary property
+            value of the respective nodes.
+
+        Args:
+            records (List[Dict[str, Any]]): a list of dictionaries used to populate relationships
+            source_type: explicitly state the class to use for source node
+            target_type: explicitly state the class to use for target node
+        """
+
+        hydrated_list = []
+
+        if source_type is None:
+            source_type = cls.model_fields["source"].annotation
+
+        if target_type is None:
+            target_type = cls.model_fields["target"].annotation
+
+        if source_prop is None:
+            source_prop = source_type.__primaryproperty__
+
+        if target_prop is None:
+            target_prop = target_type.__primaryproperty__
+
+        for record in records:
+            hydrated = dict(record)
+
+            hydrated["source"] = source_type.model_construct(
+                **{source_prop: record["source"]}
+            )
+            hydrated["target"] = target_type.model_construct(
+                **{target_prop: record["target"]}
+            )
+
+            hydrated_list.append(hydrated)
+
+        rels = [cls(**x) for x in hydrated_list]
+
+        cls.merge_relationships(
+            rels,
+            source_type=source_type,
+            source_prop=source_prop,
+            target_type=target_type,
+            target_prop=target_prop,
+        )
+
+    @classmethod
+    def merge_df(
+        cls: Type[R],
+        df: pd.DataFrame,
+        source_type: Optional[Type[BaseNode]] = None,
+        target_type: Optional[Type[BaseNode]] = None,
+        source_prop: Optional[str] = None,
+        target_prop: Optional[str] = None,
+    ) -> None:
+        """Merge in relationships based on data in a pandas data frame
+
+        Expects columns named 'source' and 'target' with the primary property value
+            for the source and target nodes.
+
+        Then additional fields should have a corresponding column.
+
+        Args:
+            df (pd.DataFrame): pandas dataframe where each row represents a relationship to merge
+        """
+
+        if df.empty is False:
+            records = df.replace([np.nan], None).to_dict(orient="records")
+            cls.merge_records(
+                records,
+                source_type=source_type,
+                source_prop=source_prop,
+                target_type=target_type,
+                target_prop=target_prop,
+            )
+
+    @classmethod
+    def to_dict(cls):
+        return {
+            "source": cls.source.to_dict(),
+            "target": cls.target.to_dict(),
+            "relationship_type": cls.__relationshiptype__
+        }
+

modules/database/tools/neontology/commonmodel.py

@@ -0,0 +1,196 @@
+from abc import ABC, abstractmethod
+from datetime import date, datetime, time, timedelta
+from typing import Any, ClassVar, Dict, List, Optional, Set
+
+from neo4j.time import Date as Neo4jDate
+from neo4j.time import DateTime as Neo4jDateTime
+from neo4j.time import Time as Neo4jTime
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    PrivateAttr,
+    field_validator,
+    model_validator,
+)
+
+
+class CommonModel(BaseModel, ABC):
+    model_config = ConfigDict(
+        validate_assignment=True,
+        extra="forbid",
+        arbitrary_types_allowed=True,
+    )
+
+    created: datetime = Field(
+        default_factory=datetime.now, json_schema_extra={"set_on_create": True}
+    )
+    merged: Optional[datetime] = Field(default=None, validate_default=True)
+
+    _set_on_match: List[str] = PrivateAttr()
+    _set_on_create: List[str] = PrivateAttr()
+    _always_set: List[str] = PrivateAttr()
+
+    _neo4j_supported_types: ClassVar[Any] = (
+        list,
+        bool,
+        int,
+        bytearray,
+        float,
+        str,
+        bytes,
+        date,
+        time,
+        datetime,
+        timedelta,
+    )
+
+    def __init__(self, **data: dict):
+        super().__init__(**data)
+
+        self._set_on_match = self._get_prop_usage("set_on_match")
+        self._set_on_create = self._get_prop_usage("set_on_create")
+        self._always_set = [
+            x
+            for x in self.model_dump().keys()
+            if x not in self._set_on_match + self._set_on_create + ["source", "target"]
+        ]
+
+    @classmethod
+    def _get_prop_usage(cls, usage_type: str) -> List[str]:
+        all_props = cls.model_json_schema()["properties"]
+
+        selected_props = []
+
+        for prop, entry in all_props.items():
+            if entry.get(usage_type) is True:
+                selected_props.append(prop)
+
+        return selected_props
+
+    def _get_prop_values(
+        self, props: List[str], exclude: Set[str] = set()
+    ) -> Dict[str, Any]:
+        """
+
+        Returns:
+            Dict[str, Any]: a dictionary of key/value pairs.
+        """
+
+        prop_values = {
+            k: v for k, v in self.neo4j_dict(exclude=exclude).items() if k in props
+        }
+
+        return prop_values
+
+    @abstractmethod
+    def _get_merge_parameters(self) -> Dict[str, Any]:
+        raise NotImplementedError
+
+    @classmethod
+    def export_type_converter(cls, value: Any) -> Any:
+        if isinstance(value, dict):
+            raise TypeError("Neo4j doesn't support dict types for properties.")
+
+        elif isinstance(value, (tuple, set)):
+            new_value = list(value)
+            return cls.export_type_converter(new_value)
+
+        elif isinstance(value, list):
+            # items in a list must all be the same type
+            item_type = type(value[0])
+            for item in value:
+                if isinstance(item, item_type) is False:
+                    raise TypeError(
+                        "For neo4j, all items in a list must be of the same type."
+                    )
+
+            return [cls.export_type_converter(x) for x in value]
+
+        elif isinstance(value, cls._neo4j_supported_types) is False:
+            return str(value)
+
+        else:
+            return value
+
+    @classmethod
+    def _export_dict_converter(cls, original_dict: Dict[str, Any]) -> Dict[str, Any]:
+        """_summary_
+
+        Args:
+            export_dict (Dict[str, Any]): _description_
+
+        Returns:
+            Dict[str, Any]: _description_
+        """
+
+        export_dict = original_dict.copy()
+
+        for k, v in export_dict.items():
+            export_dict[k] = cls.export_type_converter(v)
+
+        return export_dict
+
+    def neo4j_dict(self, **kwargs: Dict[str, Any]) -> Dict[str, Any]:
+        """Return a dict made up of only types compatible with neo4j
+
+        Returns:
+            dict: a dictionary export of this model instance
+        """
+
+        export_dict = self.model_dump(exclude_none=True, **kwargs)
+
+        export_dict = self._export_dict_converter(export_dict)
+
+        return export_dict
+
+    #
+    # validators
+    #
+
+    @field_validator("merged")
+    def set_merged_to_created(
+        cls, value: Optional[datetime], values: Dict[str, Any]
+    ) -> datetime:
+        """By default, set the 'merged' time equal to the 'created' time.
+
+        If the 'merged' value has been explicitly set, this is preserved.
+
+        Args:
+            value (Optional[datetime]): the value of the field.
+            values (Dict[str, Any]): a dictionary of field/value pairs set so far.
+
+        Returns:
+            datetime: The merged datetime value.
+        """
+
+        if value is None:
+            return values.data["created"]
+        else:
+            return value
+
+    @model_validator(mode="before")
+    @classmethod
+    def neo4j_datetime_to_native(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        """Datetimes come back from Neo4j as a non standard DateTime type.
+
+        We check for any values where that is the case and convert them to
+            native Python datetimes.
+
+        See https://neo4j.com/docs/api/python-driver/4.4/temporal_types.html for further info.
+
+        Args:
+            values (Dict[str, Any]): Dictionary of field/value pairs from pydantic.
+
+        Returns:
+            Dict[str, Any]: Returns the dictionary, with any Neo4jDateTimes updated.
+        """
+
+        if not isinstance(values, dict):
+            raise ValueError
+
+        for key in values:
+            if isinstance(values[key], (Neo4jDateTime, Neo4jDate, Neo4jTime)):
+                values[key] = values[key].to_native()
+
+        return values

modules/database/tools/neontology/graphconnection.py

@@ -0,0 +1,253 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'api_modules_database_tools_neontology_graphconnection'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+from typing import Any, Dict, List, Optional
+
+from neo4j import GraphDatabase, Neo4jDriver
+from neo4j import Record as Neo4jRecord
+from neo4j import Result as Neo4jResult
+from neo4j import Transaction as Neo4jTransaction
+
+from .result import NeontologyResult, neo4j_records_to_neontology_records
+
+
+class GraphConnection(object):
+    """Class for managing connections to Neo4j."""
+
+    _instance = None
+
+    def __new__(
+        cls,
+        neo4j_uri: Optional[str] = None,
+        neo4j_username: Optional[str] = None,
+        neo4j_password: Optional[str] = None,
+    ) -> "GraphConnection":
+        """Make sure we only have a single connection to the GraphDatabase.
+
+        This connection then gets used by all instances.
+
+        Args:
+            neo4j_uri (Optional[str], optional): Neo4j URI to connect to. Defaults to None.
+            neo4j_username (Optional[str], optional): Neo4j username. Defaults to None.
+            neo4j_password (Optional[str], optional): Neo4j password. Defaults to None.
+
+        Returns:
+            GraphConnection: Instance of the connection
+        """
+
+        if cls._instance is None:
+            cls._instance = object.__new__(cls)
+
+            if GraphConnection._instance:
+                try:
+                    driver = GraphConnection._instance.driver = GraphDatabase.driver(  # type: ignore
+                        neo4j_uri, auth=(neo4j_username, neo4j_password)
+                    )
+                    driver.verify_connectivity()
+
+                    from .utils import get_node_types, get_rels_by_type
+
+                    # capture all possible types of node and relationship
+                    cls.global_nodes = get_node_types()
+                    cls.global_rels = get_rels_by_type()
+
+                except Exception as error:
+                    logging.error(
+                        "Error: connection not established. Have you run init_neontology? {}".format(
+                            error
+                        )
+                    )
+                    GraphConnection._instance = None
+
+            else:
+                GraphConnection._instance = None
+
+        return cls._instance
+
+    def __del__(self) -> None:
+        """Close the driver gracefully when the class gets deleted."""
+
+        self.driver.close()
+
+    def __init__(
+        self,
+        neo4j_uri: Optional[str] = None,
+        neo4j_username: Optional[str] = None,
+        neo4j_password: Optional[str] = None,
+    ) -> None:
+        if self._instance:
+            self.driver: Neo4jDriver = self._instance.driver
+
+    def run_transaction_single(
+        self, tx: Neo4jTransaction, query: str, params: Dict[str, Any]
+    ) -> Optional[Neo4jRecord]:
+        """Run a transaction which is expected to return a single result.
+
+        Args:
+            tx (Neo4jTransaction): Neo4j Transaction object
+            query (str): cypher query to run
+            params (Dict[str, Any]): Parameters to pass to the query
+
+        Returns:
+            Optional[Neo4jRecord]: The result
+        """
+
+        return tx.run(query, **params).single()
+
+    def run_transaction_many(
+        self, tx: Neo4jTransaction, query: str, params: Dict[str, Any]
+    ) -> List[Neo4jRecord]:
+        """Run a transation which is expected to return multiple nodes.
+
+        Args:
+            tx (Neo4jTransaction): Neo4j Transaction object
+            query (str): cypher query to run
+            params (Dict[str, Any]): parameters to pass the query
+
+        Returns:
+            List[Neo4jRecord]: a list of the results
+        """
+
+        return [record for record in tx.run(query, **params)]
+
+    def cypher_write(self, cypher: str, params: Dict[str, Any] = {}) -> None:
+        """Execute a write transaction.
+
+        Args:
+            cypher (str): cypher query
+            params (Dict[str, Any]): parameters to pass to the query
+        """
+
+        with self.driver.session() as session:
+            session.execute_write(self.run_transaction_single, cypher, params)
+
+    def cypher_write_single(self, cypher: str, params: Dict[str, Any] = {}) -> None:
+        """Execute a write transaction.
+
+        Args:
+            cypher (str): cypher query
+            params (Dict[str, Any]): parameters to pass to the query
+        """
+
+        with self.driver.session() as session:
+            return session.execute_write(self.run_transaction_single, cypher, params)
+
+    def cypher_write_many(self, cypher: str, params: Dict[str, Any] = {}) -> None:
+        """Execute a write transaction.
+
+        Args:
+            cypher (str): cypher query
+            params (Dict[str, Any]): parameters to pass to the query
+        """
+
+        with self.driver.session() as session:
+            return session.execute_write(self.run_transaction_many, cypher, params)
+
+    def cypher_read(
+        self, cypher: str, params: Dict[str, Any] = {}
+    ) -> Optional[Neo4jRecord]:
+        """Run a cypher read only query which is expected to return a single result.
+
+        Args:
+            cypher (str): cypher query string
+            params (Dict[str, Any]): parameters to pass to the query
+
+        Returns:
+            Neo4jRecord: the resulting Neo4j 'Record', or None
+        """
+
+        with self.driver.session() as session:
+            return session.execute_read(self.run_transaction_single, cypher, params)
+
+    def cypher_read_many(
+        self, cypher: str, params: Dict[str, Any] = {}
+    ) -> List[Neo4jRecord]:
+        """Run a cypher read query which will return multiple records.
+
+        Args:
+            cypher (str): cypher string to run
+            params (Dict[str, Any]): parameters to pass to the query
+
+        Returns:
+            List[Neo4jRecord]: A list of Neo4j 'Records' returned by the query.
+        """
+
+        with self.driver.session() as session:
+            return session.execute_read(self.run_transaction_many, cypher, params)
+
+    def apply_constraint(self, label: str, property: str) -> None:
+        cypher = f"""
+        CREATE CONSTRAINT IF NOT EXISTS
+        FOR (n:{label})
+        REQUIRE n.{property} IS UNIQUE
+        """
+
+        self.cypher_write(cypher)
+
+    def evaluate_query_single(self, cypher, params={}):
+        result = self.driver.execute_query(
+            cypher, parameters_=params, result_transformer_=Neo4jResult.single
+        )
+
+        if result:
+            return result.value()
+
+        else:
+            return None
+
+    def evaluate_query(self, cypher, params={}):
+        result = self.driver.execute_query(cypher, parameters_=params)
+
+        neo4j_records = result.records
+        neontology_records = neo4j_records_to_neontology_records(
+            neo4j_records, self.global_nodes, self.global_rels
+        )
+
+        return NeontologyResult(
+            records=neo4j_records, neontology_records=neontology_records
+        )
+
+
+def init_neontology(
+    neo4j_uri: Optional[str] = None,
+    neo4j_username: Optional[str] = None,
+    neo4j_password: Optional[str] = None,
+) -> None:
+    """Initialise neontology.
+
+    If connection properties are explicitly passed in, use these.
+    If not, attempt to load from enviornment variables (optionally in a .env file.)
+
+    Args:
+        neo4j_uri (Optional[str], optional): Neo4j URI to connect to. Defaults to None.
+        neo4j_username (Optional[str], optional): Neo4j username. Defaults to None.
+        neo4j_password (Optional[str], optional): Neo4j password. Defaults to None.
+    """
+
+    # try to load environment variables from .env file
+    load_dotenv()
+
+    if neo4j_uri is None:
+        neo4j_uri = os.getenv("NEO4J_URI")
+
+    if neo4j_password is None:
+        neo4j_password = os.getenv("PASSWORD_NEO4J")
+
+    if neo4j_username is None:
+        neo4j_username = os.getenv("USER_NEO4J")
+
+    GraphConnection(neo4j_uri, neo4j_username, neo4j_password)
+
+def close_neontology():
+    GraphConnection().__del__()

modules/database/tools/neontology/result.py

@@ -0,0 +1,114 @@
+import itertools
+import warnings
+from typing import List
+
+from neo4j import Record as Neo4jRecord
+from neo4j.graph import Node as Neo4jNode
+from neo4j.graph import Relationship as Neo4jRelationship
+from pydantic import BaseModel, computed_field
+
+
+def neo4j_records_to_neontology_records(
+    records: List[Neo4jRecord], node_classes: list, rel_classes: list
+) -> list:
+    new_records = []
+
+    for record in records:
+        new_record = {"nodes": {}, "relationships": {}}
+        for key, entry in record.items():
+            if isinstance(entry, Neo4jNode):
+                node_label = list(entry.labels)[0]
+
+                # gracefully handle cases where we don't have a class defined
+                # for the identified label
+                try:
+                    node = node_classes[node_label](**dict(entry))
+                    new_record["nodes"][key] = node
+                except KeyError:
+                    warnings.warn(
+                        (
+                            f"Could not find a class for {node_label} label."
+                            " Did you define the class before initializing Neontology?"
+                        )
+                    )
+                    pass
+
+            elif isinstance(entry, Neo4jRelationship):
+                rel_type = entry.type
+
+                rel_dict = rel_classes[rel_type]
+
+                if not rel_dict:
+                    warnings.warn(
+                        (
+                            f"Could not find a class for {rel_type} relationship type."
+                            " Did you define the class before initializing Neontology?"
+                        )
+                    )
+                    continue
+
+                src_label = list(entry.nodes[0].labels)[0]
+                tgt_label = list(entry.nodes[1].labels)[0]
+
+                src_node = node_classes[src_label](**dict(entry.nodes[0]))
+                tgt_node = node_classes[tgt_label](**dict(entry.nodes[1]))
+
+                rel_props = dict(entry)
+                rel_props["source"] = src_node
+                rel_props["target"] = tgt_node
+
+                rel = rel_dict["rel_class"](**rel_props)
+
+                new_record["relationships"][key] = rel
+
+        new_records.append(new_record)
+
+    return new_records
+
+
+class NeontologyResult(BaseModel):
+    records: list
+    neontology_records: list
+
+    @computed_field
+    @property
+    def nodes(self) -> list:
+        nodes_list_of_lists = [x["nodes"].values() for x in self.neontology_records]
+        return list(itertools.chain.from_iterable(nodes_list_of_lists))
+
+    @computed_field
+    @property
+    def relationships(self) -> list:
+        nodes_list_of_lists = [
+            x["relationships"].values() for x in self.neontology_records
+        ]
+        return list(itertools.chain.from_iterable(nodes_list_of_lists))
+
+    @computed_field
+    @property
+    def node_link_data(self) -> dict:
+        nodes = [
+            {
+                "id": x.get_primary_property_value(),
+                "label": x.__primarylabel__,
+                "name": str(x),
+            }
+            for x in self.nodes
+        ]
+
+        links = [
+            {
+                "source": x.source.get_primary_property_value(),
+                "target": x.target.get_primary_property_value(),
+            }
+            for x in self.relationships
+        ]
+
+        unique_nodes = list({frozenset(item.items()): item for item in nodes}.values())
+        unique_links = list({frozenset(item.items()): item for item in links}.values())
+        data = {
+            "nodes": unique_nodes,
+            "links": unique_links,
+        }
+
+        return data

modules/database/tools/neontology/utils.py

@@ -0,0 +1,116 @@
+from collections import defaultdict
+from typing import Dict, Set, Type
+
+from .basenode import BaseNode
+from .baserelationship import BaseRelationship
+from .graphconnection import GraphConnection
+
+
+def get_node_types(base_type: Type[BaseNode] = BaseNode) -> Dict[str, Type[BaseNode]]:
+    node_types = {}
+
+    for subclass in base_type.__subclasses__():
+        # we can define 'abstract' nodes which don't have a label
+        # these are to provide common properties to be used by subclassed nodes
+        # but shouldn't be put in the graph
+        if (
+            hasattr(subclass, "__primarylabel__")
+            and subclass.__primarylabel__ is not None
+        ):
+            node_types[subclass.__primarylabel__] = subclass
+
+        if subclass.__subclasses__():
+            subclass_node_types = get_node_types(subclass)
+
+            node_types.update(subclass_node_types)
+
+    return node_types
+
+
+def get_rels_by_type(
+    base_type: Type[BaseRelationship] = BaseRelationship,
+) -> Dict[str, dict]:
+    rel_types: dict = defaultdict(dict)
+
+    for rel_subclass in base_type.__subclasses__():
+        # we can define 'abstract' relationships which don't have a label
+        # these are to provide common properties to be used by subclassed relationships
+        # but shouldn't be put in the graph
+        if (
+            hasattr(rel_subclass, "__relationshiptype__")
+            and rel_subclass.__relationshiptype__ is not None
+        ):
+            rel_types[rel_subclass.__relationshiptype__] = {
+                "rel_class": rel_subclass,
+                "source_class": rel_subclass.model_fields["source"].annotation,
+                "target_class": rel_subclass.model_fields["target"].annotation,
+            }
+
+        if rel_subclass.__subclasses__():
+            subclass_rel_types = get_rels_by_type(rel_subclass)
+
+            rel_types.update(subclass_rel_types)
+
+    return rel_types
+
+
+def all_subclasses(cls: type) -> set:
+    return set(cls.__subclasses__()).union(
+        [s for c in cls.__subclasses__() for s in all_subclasses(c)]
+    )
+
+
+def get_rels_by_node(
+    base_type: Type[BaseRelationship] = BaseRelationship, by_source: bool = True
+) -> Dict[str, Set[str]]:
+    if by_source is True:
+        node_dir = "source_class"
+
+    else:
+        node_dir = "target_class"
+
+    all_rels = get_rels_by_type(base_type)
+
+    by_node: Dict[str, Set[str]] = defaultdict(set)
+
+    for rel_type, entry in all_rels.items():
+        try:
+            node_label = entry[node_dir].__primarylabel__
+        except AttributeError:
+            node_label = None
+
+        if node_label is not None:
+            by_node[node_label].add(rel_type)
+
+        for node_subclass in all_subclasses(entry[node_dir]):
+            subclass_label = node_subclass.__primarylabel__
+            if subclass_label is not None:
+                by_node[subclass_label].add(rel_type)
+
+    return by_node
+
+
+def get_rels_by_source(
+    base_type: Type[BaseRelationship] = BaseRelationship,
+) -> Dict[str, Set[str]]:
+    return get_rels_by_node(by_source=True)
+
+
+def get_rels_by_target(
+    base_type: Type[BaseRelationship] = BaseRelationship,
+) -> Dict[str, Set[str]]:
+    return get_rels_by_node(by_source=False)
+
+
+def auto_constrain() -> None:
+    """Automatically apply constraints
+
+    Get information about all the defined nodes in the current environment.
+
+    Apply constraints based on the primary label and primary property for each node.
+    """
+
+    graph = GraphConnection()
+
+    for node_label, node_type in get_node_types().items():
+        graph.apply_constraint(node_label, node_type.__primaryproperty__)

modules/database/tools/neontology_tools.py

@@ -0,0 +1,121 @@
+from dotenv import load_dotenv, find_dotenv
+load_dotenv(find_dotenv())
+import os
+import modules.logger_tool as logger
+log_name = 'api_modules_database_tools_neontology_tools'
+log_dir = os.getenv("LOG_PATH", "/logs")  # Default path as fallback
+logging = logger.get_logger(
+    name=log_name,
+    log_level=os.getenv("LOG_LEVEL", "DEBUG"),
+    log_path=log_dir,
+    log_file=log_name,
+    runtime=True,
+    log_format='default'
+)
+from modules.database.tools.neontology.graphconnection import init_neontology, close_neontology
+from modules.database.tools.neontology.basenode import BaseNode
+from modules.database.tools.neontology.baserelationship import BaseRelationship
+from pydantic import ValidationError
+import os
+import neo4j
+
+# Initialize Neontology with the Neo4j database details
+def init_neontology_connection(uri=None, user=None, password=None):
+    uri = uri or os.getenv("APP_BOLT_URL")
+    user = user or os.getenv("USER_NEO4J")  # Add default value
+    password = password or os.getenv("PASSWORD_NEO4J")
+
+    if not all([uri, user, password]):
+        raise ValueError("Missing required Neo4j connection parameters")
+
+    try:
+        logging.info(f"Initializing Neontology with URI: {uri}")
+        init_neontology(
+            neo4j_uri=uri,
+            neo4j_username=user,
+            neo4j_password=password
+        )
+        logging.info(f"Neontology connection initialized with URI: {uri}, user: {user}")
+    except Exception as e:
+        raise ValueError(f"Failed to initialize Neontology connection: {str(e)}")
+    
+def close_neontology_connection():
+    logging.debug(f"Attempting to terminate Neontology connection")
+    close_neontology()
+    logging.info(f"Neontology connection terminated")
+
+# Terminates the Neo4j connection
+def close_neo4j_connection():
+    logging.debug(f"Attempting to terminate Neo4j connection")
+    neo4j.close()
+    logging.info(f"Neo4j connection terminated")
+
+# Create a Neontology node in the Neo4j database
+def create_or_merge_neontology_node(node: BaseNode, database: str = 'neo4j', operation: str = "merge"):
+    """
+    Create or merge a Neontology node in the Neo4j database.
+
+    Args:
+        node (BaseNode): A Neontology node object.
+        operation (str): The operation to perform ('create' or 'merge'). Defaults to 'merge'.
+    """
+    try:
+        if operation == "create":
+            node.create(database=database)
+        elif operation == "merge":
+            node.merge(database=database)
+        else:
+            logging.error(f"Invalid operation: {operation}")
+    except Exception as e:
+        logging.error(f"Error in processing node: {e}")
+
+# Create or merge a Neontology node in the Neo4j database. If a ValidationError occurs
+# due to a NaN value, replace it with a default value and retry.
+def create_or_merge_neontology_node_with_default(driver, node: BaseNode, database: str = 'neo4j', operation: str = "merge", default_values: dict = {}):
+    """
+    Create or merge a Neontology node in the Neo4j database. If a ValidationError occurs
+    due to a NaN value, replace it with a default value and retry.
+
+    Args:
+        node (BaseNode): A Neontology node object.
+        operation (str): The operation to perform ('create' or 'merge'). Defaults to 'merge'.
+        default_values (dict): A dictionary of default values for fields that might contain NaN.
+    """
+    try:
+        # Attempt to create or merge the node
+        if operation == "create":
+            node.create(database=database)
+        else:  # "merge" by default
+            node.merge(database=database)
+    except ValidationError as e:
+        # Handle ValidationError due to NaN value
+        for field, error in e.errors():
+            if field in default_values and 'type' in error and error['type'] == 'value_error.nan':
+                setattr(node, field, default_values[field])
+                logging.warning(f"Warning: Replacing NaN in {field} with default value '{default_values[field]}' and retrying.")
+                create_or_merge_neontology_node_with_default(driver, node, database, operation, default_values)
+                break
+        else:
+            # If the error is not due to a NaN value or field not in default_values, re-raise the error
+            logging.error(f"Error in processing node: {e}")
+            raise
+    except Exception as e:
+        logging.error(f"Error in processing node: {e}")
+
+def create_or_merge_neontology_relationship(relationship: BaseRelationship, database: str = 'neo4j', operation: str = "merge"):
+    """
+    Create or merge a Neontology relationship in the Neo4j database.
+
+    Args:
+        relationship (BaseRelationship): A Neontology relationship object.
+        operation (str): The operation to perform ('create' or 'merge'). Defaults to 'merge'.
+    """
+    try:
+        if operation == "create":
+            relationship.create(database=database)
+        elif operation == "merge":
+            relationship.merge(database=database)
+        else:
+            logging.error(f"Invalid operation: {operation}")
+    except Exception as e:
+        logging.error(f"Error in processing relationship: {e}")

modules/database/tools/queries.py

@@ -0,0 +1,25 @@
+def create_database(db_name):
+        return f"CREATE DATABASE `{db_name}` IF NOT EXISTS"
+        
+def stop_database(db_name):
+        return f"STOP DATABASE `{db_name}`"
+        
+def drop_database(db_name):
+        return f"DROP DATABASE `{db_name}`"
+
+show_constraints = "SHOW CONSTRAINTS"
+
+show_indexes = "SHOW INDEXES"
+
+def drop_index(index_name):
+        f"DROP INDEX {index_name}"
+
+def drop_constraint(constraint_name):
+        f"DROP CONSTRAINT {constraint_name}"
+
+delete_batch = """
+        MATCH (n)
+        WITH n LIMIT $batch_size
+        DETACH DELETE n
+        RETURN count(*)
+        """