473 lines
22 KiB
Python
473 lines
22 KiB
Python
import os
|
|
from typing import Dict, List, Optional, BinaryIO
|
|
import json
|
|
import pandas as pd
|
|
from backend.modules.database.schemas import entities
|
|
from modules.logger_tool import initialise_logger
|
|
from modules.database.tools.filesystem_tools import ClassroomCopilotFilesystem
|
|
import modules.database.tools.neo4j_driver_tools as driver_tools
|
|
import modules.database.tools.neo4j_session_tools as session_tools
|
|
from modules.database.admin.neontology_provider import NeontologyProvider
|
|
from modules.database.admin.graph_provider import GraphNamingProvider
|
|
from modules.database.schemas import curriculum_neo
|
|
from modules.database.schemas.relationships import curriculum_relationships, entity_relationships, entity_curriculum_rels
|
|
from modules.database.supabase.utils.storage import StorageManager
|
|
|
|
class SchoolService:
|
|
def __init__(self):
|
|
self.logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
|
self.driver = driver_tools.get_driver()
|
|
self.neontology = NeontologyProvider()
|
|
self.graph_naming = GraphNamingProvider()
|
|
self.storage = StorageManager()
|
|
|
|
def create_schools_database(self) -> Dict:
|
|
"""Creates the main cc.institutes database in Neo4j"""
|
|
try:
|
|
db_name = "cc.institutes"
|
|
with self.driver.session() as session:
|
|
session_tools.create_database(session, db_name)
|
|
self.logger.info(f"Created database {db_name}")
|
|
return {
|
|
"status": "success",
|
|
"message": f"Database {db_name} created successfully"
|
|
}
|
|
except Exception as e:
|
|
self.logger.error(f"Error creating schools database: {str(e)}")
|
|
return {"status": "error", "message": str(e)}
|
|
|
|
def create_school_node(self, school_data: Dict) -> Dict:
|
|
"""Creates a school node in cc.institutes database and stores TLDraw file in Supabase"""
|
|
try:
|
|
# Convert school data to SchoolNode
|
|
school_unique_id = self.graph_naming.get_school_unique_id(school_data['urn'])
|
|
school_path = self.graph_naming.get_school_path("cc.institutes", school_data['urn'])
|
|
|
|
school_node = entities.SchoolNode(
|
|
unique_id=school_unique_id,
|
|
path=school_path,
|
|
urn=school_data['urn'],
|
|
establishment_number=school_data['establishment_number'],
|
|
establishment_name=school_data['establishment_name'],
|
|
establishment_type=school_data['establishment_type'],
|
|
establishment_status=school_data['establishment_status'],
|
|
phase_of_education=school_data['phase_of_education'] if school_data['phase_of_education'] not in [None, ''] else None,
|
|
statutory_low_age=int(school_data['statutory_low_age']) if school_data.get('statutory_low_age') is not None else 0,
|
|
statutory_high_age=int(school_data['statutory_high_age']) if school_data.get('statutory_high_age') is not None else 0,
|
|
religious_character=school_data.get('religious_character') if school_data.get('religious_character') not in [None, ''] else None,
|
|
school_capacity=int(school_data['school_capacity']) if school_data.get('school_capacity') is not None else 0,
|
|
school_website=school_data.get('school_website', ''),
|
|
ofsted_rating=school_data.get('ofsted_rating') if school_data.get('ofsted_rating') not in [None, ''] else None
|
|
)
|
|
|
|
# Create default tldraw file data
|
|
tldraw_data = {
|
|
"document": {
|
|
"version": 1,
|
|
"id": school_data['urn'],
|
|
"name": school_data['establishment_name'],
|
|
"meta": {
|
|
"created_at": "",
|
|
"updated_at": "",
|
|
"creator_id": "",
|
|
"is_template": False,
|
|
"is_snapshot": False,
|
|
"is_draft": False,
|
|
"template_id": None,
|
|
"snapshot_id": None,
|
|
"draft_id": None
|
|
}
|
|
},
|
|
"schema": {
|
|
"schemaVersion": 1,
|
|
"storeVersion": 4,
|
|
"recordVersions": {
|
|
"asset": {
|
|
"version": 1,
|
|
"subTypeKey": "type",
|
|
"subTypeVersions": {}
|
|
},
|
|
"camera": {
|
|
"version": 1
|
|
},
|
|
"document": {
|
|
"version": 2
|
|
},
|
|
"instance": {
|
|
"version": 22
|
|
},
|
|
"instance_page_state": {
|
|
"version": 5
|
|
},
|
|
"page": {
|
|
"version": 1
|
|
},
|
|
"shape": {
|
|
"version": 3,
|
|
"subTypeKey": "type",
|
|
"subTypeVersions": {
|
|
"cc-school-node": 1
|
|
}
|
|
},
|
|
"instance_presence": {
|
|
"version": 5
|
|
},
|
|
"pointer": {
|
|
"version": 1
|
|
}
|
|
}
|
|
},
|
|
"store": {
|
|
"document:document": {
|
|
"gridSize": 10,
|
|
"name": school_data['establishment_name'],
|
|
"meta": {},
|
|
"id": school_data['urn'],
|
|
"typeName": "document"
|
|
},
|
|
"page:page": {
|
|
"meta": {},
|
|
"id": "page",
|
|
"name": "Page 1",
|
|
"index": "a1",
|
|
"typeName": "page"
|
|
},
|
|
"shape:school-node": {
|
|
"x": 0,
|
|
"y": 0,
|
|
"rotation": 0,
|
|
"type": "cc-school-node",
|
|
"id": school_unique_id,
|
|
"parentId": "page",
|
|
"index": "a1",
|
|
"props": school_node.to_dict(),
|
|
"typeName": "shape"
|
|
},
|
|
"instance:instance": {
|
|
"id": "instance",
|
|
"currentPageId": "page",
|
|
"typeName": "instance"
|
|
},
|
|
"camera:camera": {
|
|
"x": 0,
|
|
"y": 0,
|
|
"z": 1,
|
|
"id": "camera",
|
|
"typeName": "camera"
|
|
}
|
|
}
|
|
}
|
|
|
|
# Store tldraw file in Supabase storage
|
|
file_path = f"{school_data['urn']}/tldraw.json"
|
|
file_options = {
|
|
"content-type": "application/json",
|
|
"x-upsert": "true",
|
|
"metadata": {
|
|
"establishment_urn": school_data['urn'],
|
|
"establishment_name": school_data['establishment_name']
|
|
}
|
|
}
|
|
|
|
# Upload file
|
|
self.storage.upload_file(
|
|
bucket_id="cc.institutes",
|
|
file_path=file_path,
|
|
file_data=json.dumps(tldraw_data).encode(),
|
|
content_type="application/json",
|
|
upsert=True
|
|
)
|
|
|
|
# Create node in Neo4j
|
|
with self.neontology as neo:
|
|
self.logger.info(f"Creating school node in Neo4j: {school_node.to_dict()}")
|
|
neo.create_or_merge_node(school_node, database="cc.institutes", operation="merge")
|
|
return {"status": "success", "node": school_node}
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Error creating school node: {str(e)}")
|
|
return {"status": "error", "message": str(e)}
|
|
|
|
def create_private_database(self, school_data: Dict) -> Dict:
|
|
"""Creates a private database for a specific school"""
|
|
try:
|
|
private_db_name = f"cc.institutes.{school_data['urn']}"
|
|
with self.driver.session() as session:
|
|
session_tools.create_database(session, private_db_name)
|
|
self.logger.info(f"Created private database {private_db_name}")
|
|
return {
|
|
"status": "success",
|
|
"message": f"Database {private_db_name} created successfully"
|
|
}
|
|
except Exception as e:
|
|
self.logger.error(f"Error creating private database: {str(e)}")
|
|
return {"status": "error", "message": str(e)}
|
|
|
|
def create_basic_structure(self, school_node: entities.SchoolNode, database_name: str) -> Dict:
|
|
"""Creates basic structural nodes in the specified database"""
|
|
try:
|
|
# Create filesystem paths
|
|
fs_handler = ClassroomCopilotFilesystem(database_name, init_run_type="school")
|
|
|
|
# Create Department Structure node
|
|
department_structure_node_unique_id = f"DepartmentStructure_{school_node.unique_id}"
|
|
_, department_path = fs_handler.create_school_department_directory(school_node.path, "departments")
|
|
department_structure_node = entities.DepartmentStructureNode(
|
|
unique_id=department_structure_node_unique_id,
|
|
path=department_path
|
|
)
|
|
|
|
# Create Curriculum Structure node
|
|
_, curriculum_path = fs_handler.create_school_curriculum_directory(school_node.path)
|
|
curriculum_node = curriculum_neo.CurriculumStructureNode(
|
|
unique_id=f"CurriculumStructure_{school_node.unique_id}",
|
|
path=curriculum_path
|
|
)
|
|
|
|
# Create Pastoral Structure node
|
|
_, pastoral_path = fs_handler.create_school_pastoral_directory(school_node.path)
|
|
pastoral_node = curriculum_neo.PastoralStructureNode(
|
|
unique_id=f"PastoralStructure_{school_node.unique_id}",
|
|
path=pastoral_path
|
|
)
|
|
|
|
with self.neontology as neo:
|
|
# Create nodes
|
|
neo.create_or_merge_node(department_structure_node, database=str(database_name), operation='merge')
|
|
fs_handler.create_default_tldraw_file(department_structure_node.path, department_structure_node.to_dict())
|
|
|
|
neo.create_or_merge_node(curriculum_node, database=str(database_name), operation='merge')
|
|
fs_handler.create_default_tldraw_file(curriculum_node.path, curriculum_node.to_dict())
|
|
|
|
neo.create_or_merge_node(pastoral_node, database=database_name, operation='merge')
|
|
fs_handler.create_default_tldraw_file(pastoral_node.path, pastoral_node.to_dict())
|
|
|
|
# Create relationships
|
|
neo.create_or_merge_relationship(
|
|
entity_relationships.SchoolHasDepartmentStructure(source=school_node, target=department_structure_node),
|
|
database=database_name, operation='merge'
|
|
)
|
|
|
|
neo.create_or_merge_relationship(
|
|
entity_curriculum_rels.SchoolHasCurriculumStructure(source=school_node, target=curriculum_node),
|
|
database=database_name, operation='merge'
|
|
)
|
|
|
|
neo.create_or_merge_relationship(
|
|
entity_curriculum_rels.SchoolHasPastoralStructure(source=school_node, target=pastoral_node),
|
|
database=database_name, operation='merge'
|
|
)
|
|
|
|
return {
|
|
"status": "success",
|
|
"message": "Basic structure created successfully",
|
|
"nodes": {
|
|
"department_structure": department_structure_node,
|
|
"curriculum_structure": curriculum_node,
|
|
"pastoral_structure": pastoral_node
|
|
}
|
|
}
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Error creating basic structure: {str(e)}")
|
|
return {"status": "error", "message": str(e)}
|
|
|
|
def create_detailed_structure(self, school_node: entities.SchoolNode, database_name: str, excel_file: BinaryIO) -> Dict:
|
|
"""Creates detailed structural nodes from Excel file"""
|
|
try:
|
|
# Store Excel file in Supabase
|
|
file_path = f"{school_node.urn}/structure.xlsx"
|
|
|
|
# Upload Excel file
|
|
self.storage.upload_file(
|
|
bucket_id="cc.institutes",
|
|
file_path=file_path,
|
|
file_data=excel_file.read(),
|
|
content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
upsert=True
|
|
)
|
|
|
|
# Process Excel file
|
|
dataframes = pd.read_excel(excel_file, sheet_name=None)
|
|
|
|
# Get existing basic structure nodes
|
|
with self.neontology as neo:
|
|
result = neo.cypher_read("""
|
|
MATCH (s:School {unique_id: $school_id})
|
|
OPTIONAL MATCH (s)-[:HAS_DEPARTMENT_STRUCTURE]->(ds:DepartmentStructure)
|
|
OPTIONAL MATCH (s)-[:HAS_CURRICULUM_STRUCTURE]->(cs:CurriculumStructure)
|
|
OPTIONAL MATCH (s)-[:HAS_PASTORAL_STRUCTURE]->(ps:PastoralStructure)
|
|
RETURN ds, cs, ps
|
|
""", {"school_id": school_node.unique_id}, database=database_name)
|
|
|
|
if not result:
|
|
raise Exception("Basic structure not found")
|
|
|
|
department_structure = result['ds']
|
|
curriculum_structure = result['cs']
|
|
pastoral_structure = result['ps']
|
|
|
|
# Create departments and subjects
|
|
unique_departments = dataframes['keystagesyllabuses']['Department'].dropna().unique()
|
|
|
|
fs_handler = ClassroomCopilotFilesystem(database_name, init_run_type="school")
|
|
node_library = {}
|
|
|
|
with self.neontology as neo:
|
|
for department_name in unique_departments:
|
|
_, department_path = fs_handler.create_school_department_directory(school_node.path, department_name)
|
|
|
|
department_node = entities.DepartmentNode(
|
|
unique_id=f"Department_{school_node.unique_id}_{department_name.replace(' ', '_')}",
|
|
department_name=department_name,
|
|
path=department_path
|
|
)
|
|
neo.create_or_merge_node(department_node, database=database_name, operation='merge')
|
|
fs_handler.create_default_tldraw_file(department_node.path, department_node.to_dict())
|
|
node_library[f'department_{department_name}'] = department_node
|
|
|
|
# Link to department structure
|
|
neo.create_or_merge_relationship(
|
|
entity_relationships.DepartmentStructureHasDepartment(
|
|
source=department_structure,
|
|
target=department_node
|
|
),
|
|
database=database_name,
|
|
operation='merge'
|
|
)
|
|
|
|
# Create year groups
|
|
year_groups = self.sort_year_groups(dataframes['yeargroupsyllabuses'])['YearGroup'].unique()
|
|
last_year_group_node = None
|
|
|
|
for year_group in year_groups:
|
|
numeric_year_group = pd.to_numeric(year_group, errors='coerce')
|
|
if pd.notna(numeric_year_group):
|
|
_, year_group_path = fs_handler.create_pastoral_year_group_directory(
|
|
pastoral_structure.path,
|
|
str(int(numeric_year_group))
|
|
)
|
|
|
|
year_group_node = curriculum_neo.YearGroupNode(
|
|
unique_id=f"YearGroup_{school_node.unique_id}_YGrp{int(numeric_year_group)}",
|
|
year_group=str(int(numeric_year_group)),
|
|
year_group_name=f"Year {int(numeric_year_group)}",
|
|
path=year_group_path
|
|
)
|
|
neo.create_or_merge_node(year_group_node, database=database_name, operation='merge')
|
|
fs_handler.create_default_tldraw_file(year_group_node.path, year_group_node.to_dict())
|
|
node_library[f'year_group_{int(numeric_year_group)}'] = year_group_node
|
|
|
|
# Create sequential relationship
|
|
if last_year_group_node:
|
|
neo.create_or_merge_relationship(
|
|
curriculum_relationships.YearGroupFollowsYearGroup(
|
|
source=last_year_group_node,
|
|
target=year_group_node
|
|
),
|
|
database=database_name,
|
|
operation='merge'
|
|
)
|
|
last_year_group_node = year_group_node
|
|
|
|
# Link to pastoral structure
|
|
neo.create_or_merge_relationship(
|
|
curriculum_relationships.PastoralStructureIncludesYearGroup(
|
|
source=pastoral_structure,
|
|
target=year_group_node
|
|
),
|
|
database=database_name,
|
|
operation='merge'
|
|
)
|
|
|
|
# Create key stages
|
|
key_stages = dataframes['keystagesyllabuses']['KeyStage'].unique()
|
|
last_key_stage_node = None
|
|
|
|
for key_stage in sorted(key_stages):
|
|
_, key_stage_path = fs_handler.create_curriculum_key_stage_directory(
|
|
curriculum_structure.path,
|
|
str(key_stage)
|
|
)
|
|
|
|
key_stage_node = curriculum_neo.KeyStageNode(
|
|
unique_id=f"KeyStage_{curriculum_structure.unique_id}_KStg{key_stage}",
|
|
key_stage_name=f"Key Stage {key_stage}",
|
|
key_stage=str(key_stage),
|
|
path=key_stage_path
|
|
)
|
|
neo.create_or_merge_node(key_stage_node, database=database_name, operation='merge')
|
|
fs_handler.create_default_tldraw_file(key_stage_node.path, key_stage_node.to_dict())
|
|
node_library[f'key_stage_{key_stage}'] = key_stage_node
|
|
|
|
# Create sequential relationship
|
|
if last_key_stage_node:
|
|
neo.create_or_merge_relationship(
|
|
curriculum_relationships.KeyStageFollowsKeyStage(
|
|
source=last_key_stage_node,
|
|
target=key_stage_node
|
|
),
|
|
database=database_name,
|
|
operation='merge'
|
|
)
|
|
last_key_stage_node = key_stage_node
|
|
|
|
# Link to curriculum structure
|
|
neo.create_or_merge_relationship(
|
|
curriculum_relationships.CurriculumStructureIncludesKeyStage(
|
|
source=curriculum_structure,
|
|
target=key_stage_node
|
|
),
|
|
database=database_name,
|
|
operation='merge'
|
|
)
|
|
|
|
return {
|
|
"status": "success",
|
|
"message": "Detailed structure created successfully",
|
|
"node_library": node_library
|
|
}
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Error creating detailed structure: {str(e)}")
|
|
return {"status": "error", "message": str(e)}
|
|
|
|
def sort_year_groups(self, df: pd.DataFrame) -> pd.DataFrame:
|
|
"""Helper function to sort year groups numerically"""
|
|
df = df.copy()
|
|
df['YearGroupNumeric'] = pd.to_numeric(df['YearGroup'], errors='coerce')
|
|
return df.sort_values(by='YearGroupNumeric')
|
|
|
|
def check_schools_database(self) -> Dict:
|
|
"""Check if the schools database exists and has been initialized"""
|
|
try:
|
|
db_name = "cc.institutes"
|
|
with self.driver.session() as session:
|
|
# Check if database exists
|
|
databases = session_tools.list_databases(session)
|
|
if db_name not in databases:
|
|
return {
|
|
"status": "error",
|
|
"message": f"Database {db_name} does not exist"
|
|
}
|
|
|
|
# Check if database has any nodes (indicating it's been initialized)
|
|
session.run("USE " + db_name)
|
|
result = session.run("MATCH (n) RETURN count(n) as count").single()
|
|
node_count = result["count"] if result else 0
|
|
|
|
if node_count == 0:
|
|
return {
|
|
"status": "error",
|
|
"message": f"Database {db_name} exists but has no nodes"
|
|
}
|
|
|
|
return {
|
|
"status": "success",
|
|
"message": f"Database {db_name} exists and has {node_count} nodes"
|
|
}
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Error checking schools database: {str(e)}")
|
|
return {"status": "error", "message": str(e)}
|