ClassroomCopilot/api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

latest

Browse Source
This commit is contained in:
kcar
2025-11-14 14:47:19 +00:00
parent 2a85845835
commit 46b2319e2d
199 changed files with 607543 additions and 11147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/database/admin/__init__.py
View File
modules/database/admin/calendar_provider.py
-280
View File
@@ -1,280 +0,0 @@
import os
from modules.logger_tool import initialise_logger
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
import modules.database.schemas.nodes.calendars as calendar_schemas
import modules.database.schemas.entities as entities
import modules.database.schemas.relationships.calendars as cal_rels
import modules.database.tools.neontology_tools as neon
from modules.database.tools.filesystem_tools import ClassroomCopilotFilesystem
from datetime import timedelta, datetime
def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False, entity_node=None, time_chunk_node=None):
logger.info(f"Creating calendar for {start_date} to {end_date}")
logger.info("Initializing Neontology connection")
neon.init_neontology_connection()
filesystem = ClassroomCopilotFilesystem(db_name, init_run_type="school")
def create_tldraw_file_for_node(node, node_path):
node_data = {
"unique_id": node.unique_id,
"type": node.__class__.__name__,
"name": node.name if hasattr(node, 'name') else 'Unnamed Node'
}
logger.debug(f"Creating tldraw file for node: {node_data}")
filesystem.create_default_tldraw_file(node_path, node_data)
created_years = {}
created_months = {}
created_weeks = {}
created_days = {}
last_year_node = None
last_month_node = None
last_week_node = None
last_day_node = None
calendar_nodes = {
'calendar_node': None,
'calendar_year_nodes': [],
'calendar_month_nodes': [],
'calendar_week_nodes': [],
'calendar_day_nodes': []
}
calendar_type = None
if attach_to_calendar_node and entity_node:
calendar_type = "entity_calendar"
logger.info(f"Attaching calendar to entity node: {entity_node.unique_id}")
entity_unique_id = entity_node.unique_id
calendar_unique_id = f"Calendar_{entity_unique_id}"
calendar_name = f"{start_date.strftime('%Y-%m-%d')} to {end_date.strftime('%Y-%m-%d')}"
calendar_path = os.path.join(entity_node.path, "calendar")
calendar_node = calendar_schemas.CalendarNode(
unique_id=calendar_unique_id,
name=calendar_name,
start_date=start_date,
end_date=end_date,
path=calendar_path
)
neon.create_or_merge_neontology_node(calendar_node, database=db_name, operation='merge')
calendar_nodes['calendar_node'] = calendar_node
logger.info(f"Calendar node created: {calendar_node.unique_id}")
# Create a node tldraw file for the calendar node
create_tldraw_file_for_node(calendar_node, calendar_path)
import backend.modules.database.schemas.relationships.owner_relationships as entity_cal_rels
neon.create_or_merge_neontology_relationship(
entity_cal_rels.EntityHasCalendar(source=entity_node, target=calendar_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {entity_node.unique_id} to {calendar_node.unique_id}")
if entity_node and not attach_to_calendar_node:
calendar_type = "time_entity"
else:
logger.error("Invalid combination of parameters for calendar creation.")
raise ValueError("Invalid combination of parameters for calendar creation.")
current_date = start_date
while current_date <= end_date:
year = current_date.year
month = current_date.month
day = current_date.day
iso_year, iso_week, iso_weekday = current_date.isocalendar()
# Create directories for year, month, week, and day
_, year_path = filesystem.create_year_directory(year, calendar_path)
_, month_path = filesystem.create_month_directory(year, month, calendar_path)
_, week_path = filesystem.create_week_directory(year, iso_week, calendar_path)
_, day_path = filesystem.create_day_directory(year, month, day, calendar_path)
calendar_year_unique_id = f"CalendarYear_{year}"
if year not in created_years:
year_node = calendar_schemas.CalendarYearNode(
unique_id=calendar_year_unique_id,
year=str(year),
path=year_path
)
neon.create_or_merge_neontology_node(year_node, database=db_name, operation='merge')
calendar_nodes['calendar_year_nodes'].append(year_node)
created_years[year] = year_node
create_tldraw_file_for_node(year_node, year_path)
logger.info(f"Year node created: {year_node.unique_id}")
if attach_to_calendar_node:
neon.create_or_merge_neontology_relationship(
cal_rels.CalendarIncludesYear(source=calendar_node, target=year_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {calendar_node.unique_id} to {year_node.unique_id}")
if last_year_node:
neon.create_or_merge_neontology_relationship(
cal_rels.YearFollowsYear(source=last_year_node, target=year_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {last_year_node.unique_id} to {year_node.unique_id}")
last_year_node = year_node
calendar_month_unique_id = f"CalendarMonth_{year}_{month}"
month_key = f"{year}-{month}"
if month_key not in created_months:
month_node = calendar_schemas.CalendarMonthNode(
unique_id=calendar_month_unique_id,
year=str(year),
month=str(month),
month_name=datetime(year, month, 1).strftime('%B'),
path=month_path
)
neon.create_or_merge_neontology_node(month_node, database=db_name, operation='merge')
calendar_nodes['calendar_month_nodes'].append(month_node)
created_months[month_key] = month_node
create_tldraw_file_for_node(month_node, month_path)
logger.info(f"Month node created: {month_node.unique_id}")
# Check for the end of year transition for months
if last_month_node:
if int(month) == 1 and int(last_month_node.month) == 12 and int(last_month_node.year) == year - 1:
neon.create_or_merge_neontology_relationship(
cal_rels.MonthFollowsMonth(source=last_month_node, target=month_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {last_month_node.unique_id} to {month_node.unique_id}")
elif int(month) == int(last_month_node.month) + 1:
neon.create_or_merge_neontology_relationship(
cal_rels.MonthFollowsMonth(source=last_month_node, target=month_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {last_month_node.unique_id} to {month_node.unique_id}")
last_month_node = month_node
neon.create_or_merge_neontology_relationship(
cal_rels.YearIncludesMonth(source=year_node, target=month_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {year_node.unique_id} to {month_node.unique_id}")
calendar_week_unique_id = f"CalendarWeek_{iso_year}_{iso_week}"
week_key = f"{iso_year}-W{iso_week}"
if week_key not in created_weeks:
# Get the date of the first monday of the week
week_start_date = current_date - timedelta(days=current_date.weekday())
week_node = calendar_schemas.CalendarWeekNode(
unique_id=calendar_week_unique_id,
start_date=week_start_date,
week_number=str(iso_week),
iso_week=f"{iso_year}-W{iso_week:02}",
path=week_path
)
neon.create_or_merge_neontology_node(week_node, database=db_name, operation='merge')
calendar_nodes['calendar_week_nodes'].append(week_node)
created_weeks[week_key] = week_node
create_tldraw_file_for_node(week_node, week_path)
logger.info(f"Week node created: {week_node.unique_id}")
if last_week_node and ((last_week_node.iso_week.split('-')[0] == str(iso_year) and int(last_week_node.week_number) == int(iso_week) - 1) or
(last_week_node.iso_week.split('-')[0] != str(iso_year) and int(last_week_node.week_number) == 52 and int(iso_week) == 1)):
neon.create_or_merge_neontology_relationship(
cal_rels.WeekFollowsWeek(source=last_week_node, target=week_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {last_week_node.unique_id} to {week_node.unique_id}")
last_week_node = week_node
neon.create_or_merge_neontology_relationship(
cal_rels.YearIncludesWeek(source=year_node, target=week_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {year_node.unique_id} to {week_node.unique_id}")
calendar_day_unique_id = f"CalendarDay_{year}_{month}_{day}"
day_key = f"{year}-{month}-{day}"
day_node = calendar_schemas.CalendarDayNode(
unique_id=calendar_day_unique_id,
date=current_date,
day_of_week=current_date.strftime('%A'),
iso_day=f"{year}-{month:02}-{day:02}",
path=day_path
)
neon.create_or_merge_neontology_node(day_node, database=db_name, operation='merge')
calendar_nodes['calendar_day_nodes'].append(day_node)
created_days[day_key] = day_node
create_tldraw_file_for_node(day_node, day_path)
logger.info(f"Day node created: {day_node.unique_id}")
if last_day_node:
neon.create_or_merge_neontology_relationship(
cal_rels.DayFollowsDay(source=last_day_node, target=day_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {last_day_node.unique_id} to {day_node.unique_id}")
last_day_node = day_node
neon.create_or_merge_neontology_relationship(
cal_rels.MonthIncludesDay(source=month_node, target=day_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {month_node.unique_id} to {day_node.unique_id}")
neon.create_or_merge_neontology_relationship(
cal_rels.WeekIncludesDay(source=week_node, target=day_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {week_node.unique_id} to {day_node.unique_id}")
current_date += timedelta(days=1)
if time_chunk_node:
time_chunk_interval = time_chunk_node
# Get every calendar day node and create time chunks of length time_chunk_node minutes for the whole day
for day_node in calendar_nodes['calendar_day_nodes']:
day_path = day_node.path
total_time_chunks_in_day = (24 * 60) / time_chunk_interval
for i in range(total_time_chunks_in_day):
time_chunk_unique_id = f"CalendarTimeChunk_{day_node.unique_id}_{i}"
time_chunk_start_time = day_node.date.time() + timedelta(minutes=i * time_chunk_interval)
time_chunk_end_time = time_chunk_start_time + timedelta(minutes=time_chunk_interval)
time_chunk_node = calendar_schemas.CalendarTimeChunkNode(
unique_id=time_chunk_unique_id,
start_time=time_chunk_start_time,
end_time=time_chunk_end_time,
path=day_path
)
neon.create_or_merge_neontology_node(time_chunk_node, database=db_name, operation='merge')
calendar_nodes['calendar_time_chunk_nodes'].append(time_chunk_node)
logger.info(f"Time chunk node created: {time_chunk_node.unique_id}")
# Create a relationship between the time chunk node and the day node
neon.create_or_merge_neontology_relationship(
cal_rels.DayIncludesTimeChunk(source=day_node, target=time_chunk_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {day_node.unique_id} to {time_chunk_node.unique_id}")
# Create sequential relationship between the time chunk nodes
if i > 0:
neon.create_or_merge_neontology_relationship(
cal_rels.TimeChunkFollowsTimeChunk(source=calendar_nodes['calendar_time_chunk_nodes'][i-1], target=time_chunk_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {calendar_nodes['calendar_time_chunk_nodes'][i-1].unique_id} to {time_chunk_node.unique_id}")
logger.info(f'Created calendar: {calendar_nodes["calendar_node"].unique_id}')
return calendar_nodes
modules/database/admin/graph_provider.py
-401
View File
@@ -1,401 +0,0 @@
from enum import Enum
from typing import Optional, List, Dict, Any
import logging
from modules.database.admin.neontology_provider import NeontologyProvider
class NodeLabels(Enum):
SCHOOL = "School"
DEPARTMENT_STRUCTURE = "DepartmentStructure"
CURRICULUM_STRUCTURE = "CurriculumStructure"
PASTORAL_STRUCTURE = "PastoralStructure"
DEPARTMENT = "Department"
KEY_STAGE = "KeyStage"
YEAR_GROUP = "YearGroup"
class RelationshipTypes(Enum):
HAS_DEPARTMENT_STRUCTURE = "HAS_DEPARTMENT_STRUCTURE"
HAS_CURRICULUM_STRUCTURE = "HAS_CURRICULUM_STRUCTURE"
HAS_PASTORAL_STRUCTURE = "HAS_PASTORAL_STRUCTURE"
HAS_DEPARTMENT = "HAS_DEPARTMENT"
INCLUDES_KEY_STAGE = "INCLUDES_KEY_STAGE"
INCLUDES_YEAR_GROUP = "INCLUDES_YEAR_GROUP"
class PropertyKeys(Enum):
UNIQUE_ID = "unique_id"
PATH = "path"
URN = "urn"
ESTABLISHMENT_NUMBER = "establishment_number"
ESTABLISHMENT_NAME = "establishment_name"
ESTABLISHMENT_TYPE = "establishment_type"
ESTABLISHMENT_STATUS = "establishment_status"
PHASE_OF_EDUCATION = "phase_of_education"
STATUTORY_LOW_AGE = "statutory_low_age"
STATUTORY_HIGH_AGE = "statutory_high_age"
RELIGIOUS_CHARACTER = "religious_character"
SCHOOL_CAPACITY = "school_capacity"
SCHOOL_WEBSITE = "school_website"
OFSTED_RATING = "ofsted_rating"
DEPARTMENT_NAME = "department_name"
KEY_STAGE = "key_stage"
KEY_STAGE_NAME = "key_stage_name"
YEAR_GROUP = "year_group"
YEAR_GROUP_NAME = "year_group_name"
CREATED = "created"
MERGED = "merged"
class SchemaDefinition:
"""Class to hold schema definition queries and information"""
@staticmethod
def get_schema_info() -> Dict[str, List[Dict]]:
"""Returns a dictionary containing the schema definition for nodes and relationships."""
return {
"nodes": [
{
"label": "School",
"description": "Represents a school entity",
"required_properties": ["unique_id", "urn", "name"],
"optional_properties": ["address", "postcode", "phone", "email", "website"]
},
{
"label": "DepartmentStructure",
"description": "Represents the department structure of a school",
"required_properties": ["unique_id", "name"],
"optional_properties": ["description", "head_of_department"]
},
{
"label": "CurriculumStructure",
"description": "Represents the curriculum structure of a school",
"required_properties": ["unique_id", "name"],
"optional_properties": ["description", "key_stage", "subject"]
},
{
"label": "PastoralStructure",
"description": "Represents the pastoral structure of a school",
"required_properties": ["unique_id", "name"],
"optional_properties": ["description", "year_group", "form_group"]
}
],
"relationships": [
{
"type": "HAS_DEPARTMENT_STRUCTURE",
"description": "Links a school to its department structure",
"source": "School",
"target": "DepartmentStructure",
"properties": ["created_at"]
},
{
"type": "HAS_CURRICULUM_STRUCTURE",
"description": "Links a school to its curriculum structure",
"source": "School",
"target": "CurriculumStructure",
"properties": ["created_at"]
},
{
"type": "HAS_PASTORAL_STRUCTURE",
"description": "Links a school to its pastoral structure",
"source": "School",
"target": "PastoralStructure",
"properties": ["created_at"]
}
]
}
@staticmethod
def get_schema_creation_queries() -> List[str]:
"""Returns a list of Cypher queries to create the schema."""
return [
# Node Uniqueness Constraints
f"CREATE CONSTRAINT school_unique_id IF NOT EXISTS FOR (n:{NodeLabels.SCHOOL.value}) REQUIRE n.{PropertyKeys.UNIQUE_ID.value} IS UNIQUE",
f"CREATE CONSTRAINT department_unique_id IF NOT EXISTS FOR (n:{NodeLabels.DEPARTMENT_STRUCTURE.value}) REQUIRE n.{PropertyKeys.UNIQUE_ID.value} IS UNIQUE",
f"CREATE CONSTRAINT curriculum_unique_id IF NOT EXISTS FOR (n:{NodeLabels.CURRICULUM_STRUCTURE.value}) REQUIRE n.{PropertyKeys.UNIQUE_ID.value} IS UNIQUE",
f"CREATE CONSTRAINT pastoral_unique_id IF NOT EXISTS FOR (n:{NodeLabels.PASTORAL_STRUCTURE.value}) REQUIRE n.{PropertyKeys.UNIQUE_ID.value} IS UNIQUE",
# Indexes for Performance
f"CREATE INDEX school_urn IF NOT EXISTS FOR (n:{NodeLabels.SCHOOL.value}) ON (n.{PropertyKeys.URN.value})",
f"CREATE INDEX school_name IF NOT EXISTS FOR (n:{NodeLabels.SCHOOL.value}) ON (n.{PropertyKeys.ESTABLISHMENT_NAME.value})",
f"CREATE INDEX department_name IF NOT EXISTS FOR (n:{NodeLabels.DEPARTMENT_STRUCTURE.value}) ON (n.{PropertyKeys.DEPARTMENT_NAME.value})",
f"CREATE INDEX curriculum_name IF NOT EXISTS FOR (n:{NodeLabels.CURRICULUM_STRUCTURE.value}) ON (n.name)",
f"CREATE INDEX pastoral_name IF NOT EXISTS FOR (n:{NodeLabels.PASTORAL_STRUCTURE.value}) ON (n.name)",
]
@staticmethod
def get_schema_verification_queries() -> Dict[str, str]:
"""Returns a dictionary of queries to verify the schema state."""
return {
"constraints": "SHOW CONSTRAINTS",
"indexes": "SHOW INDEXES",
"labels": "CALL db.labels()"
}
class GraphNamingProvider:
@staticmethod
def get_school_unique_id(urn: str) -> str:
"""Generate unique ID for a school node."""
return f"School_{urn}"
@staticmethod
def get_department_structure_unique_id(school_unique_id: str) -> str:
"""Generate unique ID for a department structure node."""
return f"DepartmentStructure_{school_unique_id}"
@staticmethod
def get_curriculum_structure_unique_id(school_unique_id: str) -> str:
"""Generate unique ID for a curriculum structure node."""
return f"CurriculumStructure_{school_unique_id}"
@staticmethod
def get_pastoral_structure_unique_id(school_unique_id: str) -> str:
"""Generate unique ID for a pastoral structure node."""
return f"PastoralStructure_{school_unique_id}"
@staticmethod
def get_department_unique_id(school_unique_id: str, department_name: str) -> str:
"""Generate unique ID for a department node."""
return f"Department_{school_unique_id}_{department_name.replace(' ', '_')}"
@staticmethod
def get_key_stage_unique_id(curriculum_structure_unique_id: str, key_stage: str) -> str:
"""Generate unique ID for a key stage node."""
return f"KeyStage_{curriculum_structure_unique_id}_KStg{key_stage}"
@staticmethod
def get_year_group_unique_id(school_unique_id: str, year_group: int) -> str:
"""Generate unique ID for a year group node."""
return f"YearGroup_{school_unique_id}_YGrp{year_group}"
@staticmethod
def get_school_path(database_name: str, urn: str) -> str:
"""Generate path for a school node."""
return f"/schools/{database_name}/{urn}"
@staticmethod
def get_department_path(school_path: str, department_name: str) -> str:
"""Generate path for a department node."""
return f"{school_path}/departments/{department_name}"
@staticmethod
def get_department_structure_path(school_path: str) -> str:
"""Generate path for a department structure node."""
return f"{school_path}/departments"
@staticmethod
def get_curriculum_path(school_path: str) -> str:
"""Generate path for a curriculum structure node."""
return f"{school_path}/curriculum"
@staticmethod
def get_pastoral_path(school_path: str) -> str:
"""Generate path for a pastoral structure node."""
return f"{school_path}/pastoral"
@staticmethod
def get_key_stage_path(curriculum_path: str, key_stage: str) -> str:
"""Generate path for a key stage node."""
return f"{curriculum_path}/key_stage_{key_stage}"
@staticmethod
def get_year_group_path(pastoral_path: str, year_group: int) -> str:
"""Generate path for a year group node."""
return f"{pastoral_path}/year_{year_group}"
@staticmethod
def get_cypher_match_school(unique_id: str) -> str:
"""Generate Cypher MATCH clause for finding a school node."""
return f"MATCH (s:{NodeLabels.SCHOOL.value} {{{PropertyKeys.UNIQUE_ID.value}: $school_id}})"
@staticmethod
def get_cypher_check_basic_structure() -> str:
"""Generate Cypher query for checking basic structure existence and validity."""
return """
// Find the school node
MATCH (s:{school})
// Check for department structure with any relationship
OPTIONAL MATCH (s)-[r1]-(dept_struct:{dept_struct})
// Check for curriculum structure with any relationship
OPTIONAL MATCH (s)-[r2]-(curr_struct:{curr_struct})
// Check for pastoral structure with any relationship
OPTIONAL MATCH (s)-[r3]-(past_struct:{past_struct})
// Return structure information
RETURN {{
has_basic:
dept_struct IS NOT NULL AND r1 IS NOT NULL AND
curr_struct IS NOT NULL AND r2 IS NOT NULL AND
past_struct IS NOT NULL AND r3 IS NOT NULL,
department_structure: {{
exists: dept_struct IS NOT NULL AND r1 IS NOT NULL
}},
curriculum_structure: {{
exists: curr_struct IS NOT NULL AND r2 IS NOT NULL
}},
pastoral_structure: {{
exists: past_struct IS NOT NULL AND r3 IS NOT NULL
}}
}} as status
""".format(
school=NodeLabels.SCHOOL.value,
dept_struct=NodeLabels.DEPARTMENT_STRUCTURE.value,
curr_struct=NodeLabels.CURRICULUM_STRUCTURE.value,
past_struct=NodeLabels.PASTORAL_STRUCTURE.value
)
@staticmethod
def get_cypher_check_detailed_structure() -> str:
"""Generate Cypher query for checking detailed structure existence and validity."""
return """
// Find the school node
MATCH (s:{school} {{unique_id: $school_id}})
// Check for department structure and departments
OPTIONAL MATCH (s)-[r1]-(dept_struct:{dept_struct})
WHERE dept_struct.unique_id = 'DepartmentStructure_' + s.unique_id
WITH s, dept_struct, r1,
CASE WHEN dept_struct IS NOT NULL
THEN [(dept_struct)-[r]-(d:{dept}) | d]
ELSE []
END as departments
// Check for curriculum structure and key stages
OPTIONAL MATCH (s)-[r2]-(curr_struct:{curr_struct})
WHERE curr_struct.unique_id = 'CurriculumStructure_' + s.unique_id
WITH s, dept_struct, r1, departments, curr_struct, r2,
CASE WHEN curr_struct IS NOT NULL
THEN [(curr_struct)-[r]-(k:{key_stage}) | k]
ELSE []
END as key_stages
// Check for pastoral structure and year groups
OPTIONAL MATCH (s)-[r3]-(past_struct:{past_struct})
WHERE past_struct.unique_id = 'PastoralStructure_' + s.unique_id
WITH dept_struct, r1, departments, curr_struct, r2, key_stages, past_struct, r3,
CASE WHEN past_struct IS NOT NULL
THEN [(past_struct)-[r]-(y:{year_group}) | y]
ELSE []
END as year_groups
// Return structure information
RETURN {{
has_detailed:
dept_struct IS NOT NULL AND r1 IS NOT NULL AND size(departments) > 0 AND
curr_struct IS NOT NULL AND r2 IS NOT NULL AND size(key_stages) > 0 AND
past_struct IS NOT NULL AND r3 IS NOT NULL AND size(year_groups) > 0,
department_structure: {{
exists: dept_struct IS NOT NULL AND r1 IS NOT NULL,
has_departments: size(departments) > 0,
department_count: size(departments),
node_id: dept_struct.unique_id
}},
curriculum_structure: {{
exists: curr_struct IS NOT NULL AND r2 IS NOT NULL,
has_key_stages: size(key_stages) > 0,
key_stage_count: size(key_stages),
node_id: curr_struct.unique_id
}},
pastoral_structure: {{
exists: past_struct IS NOT NULL AND r3 IS NOT NULL,
has_year_groups: size(year_groups) > 0,
year_group_count: size(year_groups),
node_id: past_struct.unique_id
}}
}} as status
""".format(
school=NodeLabels.SCHOOL.value,
dept_struct=NodeLabels.DEPARTMENT_STRUCTURE.value,
curr_struct=NodeLabels.CURRICULUM_STRUCTURE.value,
past_struct=NodeLabels.PASTORAL_STRUCTURE.value,
dept=NodeLabels.DEPARTMENT.value,
key_stage=NodeLabels.KEY_STAGE.value,
year_group=NodeLabels.YEAR_GROUP.value
)
@staticmethod
def get_schema_definition() -> SchemaDefinition:
"""Get the schema definition instance"""
return SchemaDefinition()
@staticmethod
def get_schema_creation_queries() -> List[str]:
"""Get queries to create the schema"""
return SchemaDefinition.get_schema_creation_queries()
@staticmethod
def get_schema_verification_queries() -> Dict[str, str]:
"""Get queries to verify schema state"""
return SchemaDefinition.get_schema_verification_queries()
@staticmethod
def get_schema_info() -> Dict[str, List[Dict]]:
"""Get human-readable schema information"""
return SchemaDefinition.get_schema_info()
class GraphProvider:
def __init__(self):
"""Initialize the graph provider with Neo4j connection."""
self.neontology = NeontologyProvider()
self.graph_naming = GraphNamingProvider()
self.logger = logging.getLogger(__name__)
def check_schema_status(self, database_name: str) -> Dict[str, Any]:
"""
Checks the current state of the schema in the specified database.
Returns a dictionary containing information about constraints, indexes, and labels.
"""
try:
verification_queries = SchemaDefinition.get_schema_verification_queries()
expected_schema = SchemaDefinition.get_schema_info()
# Get current schema state
constraints = self.neontology.run_query(verification_queries["constraints"], {}, database_name)
indexes = self.neontology.run_query(verification_queries["indexes"], {}, database_name)
labels = self.neontology.run_query(verification_queries["labels"], {}, database_name)
# Process results
current_constraints = [c["name"] for c in constraints]
current_indexes = [i["name"] for i in indexes]
current_labels = [l["label"] for l in labels]
# Expected values
expected_labels = [node["label"] for node in expected_schema["nodes"]]
return {
"constraints": current_constraints,
"constraints_valid": len(current_constraints) >= 4, # We expect at least 4 unique constraints
"indexes": current_indexes,
"indexes_valid": len(current_indexes) >= 5, # We expect at least 5 indexes
"labels": current_labels,
"labels_valid": all(label in current_labels for label in expected_labels)
}
except Exception as e:
self.logger.error(f"Error checking schema status: {str(e)}")
return {
"constraints": [], "constraints_valid": False,
"indexes": [], "indexes_valid": False,
"labels": [], "labels_valid": False
}
def initialize_schema(self, database_name: str) -> None:
"""
Initializes the schema for the specified database by creating all necessary
constraints and indexes.
"""
try:
creation_queries = SchemaDefinition.get_schema_creation_queries()
for query in creation_queries:
self.neontology.cypher_write(query, {}, database_name)
self.logger.info(f"Schema initialized successfully for database {database_name}")
except Exception as e:
self.logger.error(f"Error initializing schema: {str(e)}")
raise
def get_schema_info(self) -> Dict[str, Any]:
"""
Returns the schema definition information.
"""
return SchemaDefinition.get_schema_info()
modules/database/admin/neontology_provider.py
-212
View File
@@ -1,212 +0,0 @@
from dotenv import load_dotenv, find_dotenv
load_dotenv(find_dotenv())
import os
import modules.logger_tool as logger
from modules.database.tools.neontology.graphconnection import GraphConnection, init_neontology
from modules.database.tools.neontology.basenode import BaseNode
from modules.database.tools.neontology.baserelationship import BaseRelationship
from typing import Optional, Dict, Any, List
from neo4j import Record as Neo4jRecord
import re
log_name = 'api_modules_database_admin_neontology_provider'
log_dir = os.getenv("LOG_PATH", "/logs") # Default path as fallback
logging = logger.get_logger(
name=log_name,
log_level=os.getenv("LOG_LEVEL", "DEBUG"),
log_path=log_dir,
log_file=log_name,
runtime=True,
log_format='default'
)
class NeontologyProvider:
"""Provider class for managing Neontology connections and operations."""
def __init__(self):
"""Initialize the provider with Neo4j connection details from environment."""
self.bolt_url = os.getenv("APP_BOLT_URL")
self.user = os.getenv("USER_NEO4J")
self.password = os.getenv("PASSWORD_NEO4J")
self.connection = None
self.current_database = None
def _validate_database_name(self, database: str) -> str:
"""
Validate and format database name to handle special characters.
Args:
database: The database name to validate
Returns:
str: The validated database name
Raises:
ValueError: If database name is invalid
"""
if not database:
raise ValueError("Database name cannot be empty")
# Check for valid database name pattern
# Allow letters, numbers, underscores, and dots
if not re.match(r'^[a-zA-Z0-9_\.]+$', database):
raise ValueError("Database name contains invalid characters")
# For database names with multiple dots, we need to handle them specially
# Neo4j treats dots as special characters in some contexts
if database.count('.') > 1:
# Replace dots with underscores except for the first one
parts = database.split('.')
if len(parts) > 2:
# Keep the first dot, replace others with underscore
formatted_name = f"{parts[0]}.{'.'.join(parts[1:])}"
logging.info(f"Reformatted database name from {database} to {formatted_name}")
return formatted_name
return database
def connect(self, database: str = 'neo4j') -> None:
"""Establish connection to Neo4j using Neontology."""
try:
# Validate and format database name
formatted_database = self._validate_database_name(database)
# If we're switching databases, ensure we close the old connection
if self.current_database != formatted_database and self.connection is not None:
self.close()
# Initialize Neontology connection if needed
if self.connection is None:
init_neontology(
neo4j_uri=self.bolt_url,
neo4j_username=self.user,
neo4j_password=self.password
)
# Get the GraphConnection instance
self.connection = GraphConnection()
self.current_database = formatted_database
logging.info(f"Neontology connection initialized with host: {self.host}, port: {self.port}, database: {formatted_database}")
except Exception as e:
logging.error(f"Failed to initialize Neontology connection: {str(e)}")
raise
def reset_connection(self) -> None:
"""Reset the connection, forcing a new one to be created on next use."""
if self.connection:
self.close()
def create_or_merge_node(self, node: BaseNode, database: str = 'neo4j', operation: str = "merge") -> None:
"""Create or merge a node in the Neo4j database."""
try:
if not self.connection or self.current_database != database:
self.connect(database)
if operation == "create":
node.create(database=database)
elif operation == "merge":
node.merge(database=database)
else:
logging.error(f"Invalid operation: {operation}")
raise ValueError(f"Invalid operation: {operation}")
except Exception as e:
logging.error(f"Error in processing node: {e}")
raise
def create_or_merge_relationship(self, relationship: BaseRelationship, database: str = 'neo4j', operation: str = "merge") -> None:
"""Create or merge a relationship in the Neo4j database."""
try:
if not self.connection or self.current_database != database:
self.connect(database)
if operation == "create":
relationship.create(database=database)
elif operation == "merge":
relationship.merge(database=database)
else:
logging.error(f"Invalid operation: {operation}")
raise ValueError(f"Invalid operation: {operation}")
except Exception as e:
logging.error(f"Error in processing relationship: {e}")
raise
def cypher_write(self, cypher: str, params: Dict[str, Any] = {}, database: str = 'neo4j') -> None:
"""Execute a write transaction."""
try:
if not self.connection or self.current_database != database:
self.connect(database)
self.connection.cypher_write(cypher, params)
except Exception as e:
logging.error(f"Error in cypher write: {e}")
raise
def cypher_read(self, cypher: str, params: Dict[str, Any] = {}, database: str = 'neo4j') -> Optional[Neo4jRecord]:
"""Execute a read transaction returning a single record."""
try:
if not self.connection or self.current_database != database:
self.connect(database)
return self.connection.cypher_read(cypher, params)
except Exception as e:
logging.error(f"Error in cypher read: {e}")
raise
def cypher_read_many(self, cypher: str, params: Dict[str, Any] = {}, database: str = 'neo4j') -> List[Neo4jRecord]:
"""Execute a read transaction returning multiple records."""
try:
if not self.connection or self.current_database != database:
self.connect(database)
return self.connection.cypher_read_many(cypher, params)
except Exception as e:
logging.error(f"Error in cypher read many: {e}")
raise
def run_query(self, cypher: str, params: Dict[str, Any] = {}, database: str = 'neo4j') -> List[Dict[str, Any]]:
"""
Execute a Cypher query and return results as a list of dictionaries.
This is a convenience method that handles both single and multiple record results.
Args:
cypher: The Cypher query to execute
params: Query parameters
database: Target database name
Returns:
List[Dict[str, Any]]: Query results as a list of dictionaries
"""
try:
if not self.connection or self.current_database != database:
self.connect(database)
# Use cypher_read_many for consistent return type
records = self.connection.cypher_read_many(cypher, params)
# Convert Neo4j records to dictionaries
results = []
for record in records:
# Handle both Record and dict types
if isinstance(record, Neo4jRecord):
results.append(dict(record))
else:
results.append(record)
return results
except Exception as e:
logging.error(f"Error in run_query: {e}")
raise
def close(self) -> None:
"""Close the Neontology connection."""
if self.connection:
# The connection will be closed when the GraphConnection instance is deleted
self.connection = None
self.current_database = None
logging.info("Neontology connection closed")
def __enter__(self):
"""Context manager entry."""
return self
def __exit__(self, exc_type, exc_val, exc_tb):
"""Context manager exit."""
self.close()
modules/database/admin/school_curriculum_provider.py
-797
View File
@@ -1,797 +0,0 @@
import os
from modules.logger_tool import initialise_logger
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
import backend.modules.database.schemas.entities as neo_entity
import modules.database.schemas.curriculum_neo as neo_curriculum
import modules.database.schemas.relationships.curriculum_relationships as curriculum_relationships
import modules.database.schemas.relationships.entity_relationships as ent_rels
import modules.database.schemas.relationships.entity_curriculum_rels as ent_cur_rels
from modules.database.tools.filesystem_tools import ClassroomCopilotFilesystem
import modules.database.tools.neontology_tools as neon
import pandas as pd
# Default values for nodes
default_topic_values = {
'topic_assessment_type': 'Null',
'topic_type': 'Null',
'total_number_of_lessons_for_topic': '1',
'topic_title': 'Null'
}
default_topic_lesson_values = {
'topic_lesson_title': 'Null',
'topic_lesson_type': 'Null',
'topic_lesson_length': '1',
'topic_lesson_suggested_activities': 'Null',
'topic_lesson_skills_learned': 'Null',
'topic_lesson_weblinks': 'Null',
}
default_learning_statement_values = {
'lesson_learning_statement': 'Null',
'lesson_learning_statement_type': 'Student learning outcome'
}
# Helper function to sort year groups numerically where possible
def sort_year_groups(df):
df = df.copy()
df['YearGroupNumeric'] = pd.to_numeric(df['YearGroup'], errors='coerce')
return df.sort_values(by='YearGroupNumeric')
def create_curriculum(dataframes, school_db_name, curriculum_db_name, school_node):
fs_handler = ClassroomCopilotFilesystem(school_db_name, init_run_type="school")
logger.info(f"Initialising neo4j connection...")
neon.init_neontology_connection()
keystagesyllabus_df = dataframes['keystagesyllabuses']
yeargroupsyllabus_df = dataframes['yeargroupsyllabuses']
topic_df = dataframes['topics']
lesson_df = dataframes['lessons']
statement_df = dataframes['statements']
# resource_df = dataframes['resources'] # TODO
node_library = {}
node_library['key_stage_nodes'] = {}
node_library['year_group_nodes'] = {}
node_library['key_stage_syllabus_nodes'] = {}
node_library['year_group_syllabus_nodes'] = {}
node_library['topic_nodes'] = {}
node_library['topic_lesson_nodes'] = {}
node_library['statement_nodes'] = {}
node_library['department_nodes'] = {}
node_library['subject_nodes'] = {}
curriculum_node = None
pastoral_node = None
key_stage_nodes_created = {}
year_group_nodes_created = {}
last_year_group_node = None
last_key_stage_node = None
# Create Curriculum and Pastoral nodes and relationships with School in both databases
_, curriculum_path = fs_handler.create_school_curriculum_directory(school_node.path)
_, pastoral_path = fs_handler.create_school_pastoral_directory(school_node.path)
# Create Department Structure node
department_structure_node_unique_id = f"DepartmentStructure_{school_node.unique_id}"
department_structure_node = neo_entity.DepartmentStructureNode(
unique_id=department_structure_node_unique_id,
path=os.path.join(school_node.path, "departments")
)
# Create in school database only
neon.create_or_merge_neontology_node(department_structure_node, database=school_db_name, operation='merge')
fs_handler.create_default_tldraw_file(department_structure_node.path, department_structure_node.to_dict())
node_library['department_structure_node'] = department_structure_node
# Link Department Structure to School
neon.create_or_merge_neontology_relationship(
ent_rels.SchoolHasDepartmentStructure(source=school_node, target=department_structure_node),
database=school_db_name, operation='merge'
)
logger.info(f"Created department structure node and linked to school")
# Create Curriculum Structure node
curriculum_structure_node_unique_id = f"CurriculumStructure_{school_node.unique_id}"
curriculum_node = neo_curriculum.CurriculumStructureNode(
unique_id=curriculum_structure_node_unique_id,
path=curriculum_path
)
# Create in school database only
neon.create_or_merge_neontology_node(curriculum_node, database=school_db_name, operation='merge')
fs_handler.create_default_tldraw_file(curriculum_node.path, curriculum_node.to_dict())
node_library['curriculum_node'] = curriculum_node
# Create relationship in school database only
neon.create_or_merge_neontology_relationship(
ent_cur_rels.SchoolHasCurriculumStructure(source=school_node, target=curriculum_node),
database=school_db_name, operation='merge'
)
logger.info(f"Created curriculum node and relationship with school")
# Create Pastoral Structure node
pastoral_structure_node_unique_id = f"PastoralStructure_{school_node.unique_id}"
pastoral_node = neo_curriculum.PastoralStructureNode(
unique_id=pastoral_structure_node_unique_id,
path=pastoral_path
)
neon.create_or_merge_neontology_node(pastoral_node, database=school_db_name, operation='merge')
fs_handler.create_default_tldraw_file(pastoral_node.path, pastoral_node.to_dict())
node_library['pastoral_node'] = pastoral_node
neon.create_or_merge_neontology_relationship(
ent_cur_rels.SchoolHasPastoralStructure(source=school_node, target=pastoral_node),
database=school_db_name, operation='merge'
)
logger.info(f"Created pastoral node and relationship with school")
# Create departments and subjects
# First get unique departments
unique_departments = keystagesyllabus_df['Department'].dropna().unique()
for department_name in unique_departments:
department_unique_id = f"Department_{school_node.unique_id}_{department_name.replace(' ', '_')}"
_, department_path = fs_handler.create_school_department_directory(school_node.path, department_name)
department_node = neo_entity.DepartmentNode(
unique_id=department_unique_id,
department_name=department_name,
path=department_path
)
# Create department in school database only
neon.create_or_merge_neontology_node(department_node, database=school_db_name, operation='merge')
fs_handler.create_default_tldraw_file(department_node.path, department_node.to_dict())
node_library['department_nodes'][department_name] = department_node
# Link department to department structure in school database
neon.create_or_merge_neontology_relationship(
ent_rels.DepartmentStructureHasDepartment(source=department_structure_node, target=department_node),
database=school_db_name, operation='merge'
)
logger.info(f"Created department node for {department_name} and linked to department structure")
# Create subjects and link to departments
# First get unique subjects from key stage syllabuses (which have department info)
unique_subjects = keystagesyllabus_df[['Subject', 'SubjectCode', 'Department']].drop_duplicates()
# Then add any additional subjects from year group syllabuses (without department info)
additional_subjects = yeargroupsyllabus_df[['Subject', 'SubjectCode']].drop_duplicates()
additional_subjects = additional_subjects[~additional_subjects['SubjectCode'].isin(unique_subjects['SubjectCode'])]
# Process subjects from key stage syllabuses first (these have department info)
for _, subject_row in unique_subjects.iterrows():
subject_unique_id = f"Subject_{school_node.unique_id}_{subject_row['SubjectCode']}"
department_node = node_library['department_nodes'].get(subject_row['Department'])
if not department_node:
logger.warning(f"No department found for subject {subject_row['Subject']} with code {subject_row['SubjectCode']}")
continue
_, subject_path = fs_handler.create_department_subject_directory(
department_node.path,
subject_row['Subject'] # Use full subject name instead of SubjectCode
)
logger.info(f"Created subject directory for {subject_path}")
subject_node = neo_curriculum.SubjectNode(
unique_id=subject_unique_id,
subject_code=subject_row['SubjectCode'],
subject_name=subject_row['Subject'],
path=subject_path
)
# Create subject in both databases
neon.create_or_merge_neontology_node(subject_node, database=school_db_name, operation='merge')
neon.create_or_merge_neontology_node(subject_node, database=curriculum_db_name, operation='merge')
fs_handler.create_default_tldraw_file(subject_node.path, subject_node.to_dict())
node_library['subject_nodes'][subject_row['Subject']] = subject_node
# Link subject to department in school database only
neon.create_or_merge_neontology_relationship(
ent_rels.DepartmentManagesSubject(source=department_node, target=subject_node),
database=school_db_name, operation='merge'
)
logger.info(f"Created subject node for {subject_row['Subject']} and linked to department {subject_row['Department']}")
# Process any additional subjects from year group syllabuses (these won't have department info)
for _, subject_row in additional_subjects.iterrows():
subject_unique_id = f"Subject_{school_node.unique_id}_{subject_row['SubjectCode']}"
# Create in a special "Unassigned" department
unassigned_dept_name = "Unassigned Department"
if unassigned_dept_name not in node_library['department_nodes']:
_, dept_path = fs_handler.create_school_department_directory(school_node.path, unassigned_dept_name)
department_node = neo_entity.DepartmentNode(
unique_id=f"Department_{school_node.unique_id}_Unassigned",
department_name=unassigned_dept_name,
path=dept_path
)
neon.create_or_merge_neontology_node(department_node, database=school_db_name, operation='merge')
fs_handler.create_default_tldraw_file(department_node.path, department_node.to_dict())
node_library['department_nodes'][unassigned_dept_name] = department_node
# Link unassigned department to department structure
neon.create_or_merge_neontology_relationship(
ent_rels.DepartmentStructureHasDepartment(source=department_structure_node, target=department_node),
database=school_db_name, operation='merge'
)
logger.info(f"Created unassigned department node and linked to department structure")
_, subject_path = fs_handler.create_department_subject_directory(
node_library['department_nodes'][unassigned_dept_name].path,
subject_row['Subject']
)
subject_node = neo_curriculum.SubjectNode(
unique_id=subject_unique_id,
subject_code=subject_row['SubjectCode'],
subject_name=subject_row['Subject'],
path=subject_path
)
# Create subject in both databases
neon.create_or_merge_neontology_node(subject_node, database=school_db_name, operation='merge')
neon.create_or_merge_neontology_node(subject_node, database=curriculum_db_name, operation='merge')
fs_handler.create_default_tldraw_file(subject_node.path, subject_node.to_dict())
node_library['subject_nodes'][subject_row['Subject']] = subject_node
# Link subject to unassigned department in school database only
neon.create_or_merge_neontology_relationship(
ent_rels.DepartmentManagesSubject(
source=node_library['department_nodes'][unassigned_dept_name],
target=subject_node
),
database=school_db_name, operation='merge'
)
logger.warning(f"Created subject node for {subject_row['Subject']} in unassigned department")
# Process key stages and syllabuses
logger.info(f"Processing key stages")
last_key_stage_node = None
# Track last syllabus nodes per subject
last_key_stage_syllabus_nodes = {} # Dictionary to track last key stage syllabus node per subject
last_year_group_syllabus_nodes = {} # Dictionary to track last year group syllabus node per subject
topics_processed = set() # Track which topics have been processed
lessons_processed = set() # Track which lessons have been processed
statements_processed = set() # Track which statements have been processed
# First create all key stage nodes and key stage syllabus nodes
for index, ks_row in keystagesyllabus_df.sort_values('KeyStage').iterrows():
key_stage = str(ks_row['KeyStage'])
logger.debug(f"Processing key stage syllabus row - Subject: {ks_row['Subject']}, Key Stage: {key_stage}")
subject_node = node_library['subject_nodes'].get(ks_row['Subject'])
if not subject_node:
logger.warning(f"No subject node found for subject {ks_row['Subject']}")
continue
if key_stage not in key_stage_nodes_created:
key_stage_node_unique_id = f"KeyStage_{curriculum_node.unique_id}_KStg{key_stage}"
key_stage_node = neo_curriculum.KeyStageNode(
unique_id=key_stage_node_unique_id,
key_stage_name=f"Key Stage {key_stage}",
key_stage=str(key_stage),
path=os.path.join(curriculum_node.path, "key_stages", f"KS{key_stage}")
)
# Create key stage node in both databases
neon.create_or_merge_neontology_node(key_stage_node, database=school_db_name, operation='merge')
neon.create_or_merge_neontology_node(key_stage_node, database=curriculum_db_name, operation='merge')
fs_handler.create_default_tldraw_file(key_stage_node.path, key_stage_node.to_dict())
key_stage_nodes_created[key_stage] = key_stage_node
node_library['key_stage_nodes'][key_stage] = key_stage_node
# Create relationship with curriculum structure in school database only
neon.create_or_merge_neontology_relationship(
curriculum_relationships.CurriculumStructureIncludesKeyStage(source=curriculum_node, target=key_stage_node),
database=school_db_name, operation='merge'
)
logger.info(f"Created key stage node {key_stage_node_unique_id} and relationship with curriculum structure")
# Create sequential relationship between key stages in both databases
if last_key_stage_node:
neon.create_or_merge_neontology_relationship(
curriculum_relationships.KeyStageFollowsKeyStage(source=last_key_stage_node, target=key_stage_node),
database=school_db_name, operation='merge'
)
neon.create_or_merge_neontology_relationship(
curriculum_relationships.KeyStageFollowsKeyStage(source=last_key_stage_node, target=key_stage_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between key stages {last_key_stage_node.unique_id} and {key_stage_node.unique_id}")
last_key_stage_node = key_stage_node
# Create key stage syllabus under the subject's curriculum directory
_, key_stage_syllabus_path = fs_handler.create_curriculum_key_stage_syllabus_directory(
curriculum_node.path,
key_stage,
ks_row['Subject'],
ks_row['ID']
)
logger.debug(f"Creating key stage syllabus node for {ks_row['Subject']} KS{key_stage} with ID {ks_row['ID']}")
key_stage_syllabus_node_unique_id = f"KeyStageSyllabus_{curriculum_node.unique_id}_{ks_row['Title'].replace(' ', '')}"
key_stage_syllabus_node = neo_curriculum.KeyStageSyllabusNode(
unique_id=key_stage_syllabus_node_unique_id,
ks_syllabus_id=ks_row['ID'],
ks_syllabus_name=ks_row['Title'],
ks_syllabus_key_stage=str(ks_row['KeyStage']),
ks_syllabus_subject=ks_row['Subject'],
ks_syllabus_subject_code=ks_row['Subject'],
path=key_stage_syllabus_path
)
# Create key stage syllabus node in both databases
neon.create_or_merge_neontology_node(key_stage_syllabus_node, database=school_db_name, operation='merge')
neon.create_or_merge_neontology_node(key_stage_syllabus_node, database=curriculum_db_name, operation='merge')
fs_handler.create_default_tldraw_file(key_stage_syllabus_node.path, key_stage_syllabus_node.to_dict())
node_library['key_stage_syllabus_nodes'][ks_row['ID']] = key_stage_syllabus_node
logger.debug(f"Created key stage syllabus node {key_stage_syllabus_node_unique_id} for {ks_row['Subject']} KS{key_stage}")
# Link key stage syllabus to its subject in both databases
if subject_node:
neon.create_or_merge_neontology_relationship(
curriculum_relationships.SubjectHasKeyStageSyllabus(source=subject_node, target=key_stage_syllabus_node),
database=school_db_name, operation='merge'
)
neon.create_or_merge_neontology_relationship(
curriculum_relationships.SubjectHasKeyStageSyllabus(source=subject_node, target=key_stage_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between subject {subject_node.unique_id} and key stage syllabus {key_stage_syllabus_node.unique_id}")
# Link key stage syllabus to its key stage in both databases
key_stage_node = key_stage_nodes_created.get(key_stage)
if key_stage_node:
neon.create_or_merge_neontology_relationship(
curriculum_relationships.KeyStageIncludesKeyStageSyllabus(source=key_stage_node, target=key_stage_syllabus_node),
database=school_db_name, operation='merge'
)
neon.create_or_merge_neontology_relationship(
curriculum_relationships.KeyStageIncludesKeyStageSyllabus(source=key_stage_node, target=key_stage_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between key stage {key_stage_node.unique_id} and key stage syllabus {key_stage_syllabus_node.unique_id}")
# Create sequential relationship between key stage syllabuses in both databases
last_key_stage_syllabus_node = last_key_stage_syllabus_nodes.get(ks_row['Subject'])
if last_key_stage_syllabus_node:
neon.create_or_merge_neontology_relationship(
curriculum_relationships.KeyStageSyllabusFollowsKeyStageSyllabus(source=last_key_stage_syllabus_node, target=key_stage_syllabus_node),
database=school_db_name, operation='merge'
)
neon.create_or_merge_neontology_relationship(
curriculum_relationships.KeyStageSyllabusFollowsKeyStageSyllabus(source=last_key_stage_syllabus_node, target=key_stage_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between key stage syllabuses {last_key_stage_syllabus_node.unique_id} and {key_stage_syllabus_node.unique_id}")
last_key_stage_syllabus_nodes[ks_row['Subject']] = key_stage_syllabus_node
# Now process year groups and their syllabuses
for index, ks_row in keystagesyllabus_df.sort_values('KeyStage').iterrows():
key_stage = str(ks_row['KeyStage'])
related_yeargroups = sort_year_groups(yeargroupsyllabus_df[yeargroupsyllabus_df['KeyStage'] == ks_row['KeyStage']])
logger.info(f"Processing year groups for key stage {key_stage}")
for yg_index, yg_row in related_yeargroups.iterrows():
year_group = yg_row['YearGroup']
subject_code = yg_row['SubjectCode']
numeric_year_group = pd.to_numeric(year_group, errors='coerce')
if pd.notna(numeric_year_group):
numeric_year_group = int(numeric_year_group)
if numeric_year_group not in year_group_nodes_created:
# Create year group directory under pastoral structure
_, year_group_path = fs_handler.create_pastoral_year_group_directory(pastoral_node.path, year_group)
logger.info(f"Created year group directory for {year_group_path}")
year_group_node_unique_id = f"YearGroup_{school_node.unique_id}_YGrp{numeric_year_group}"
year_group_node = neo_curriculum.YearGroupNode(
unique_id=year_group_node_unique_id,
year_group=str(numeric_year_group),
year_group_name=f"Year {numeric_year_group}",
path=year_group_path
)
# Create year group node in both databases but use same directory
neon.create_or_merge_neontology_node(year_group_node, database=school_db_name, operation='merge')
neon.create_or_merge_neontology_node(year_group_node, database=curriculum_db_name, operation='merge')
fs_handler.create_default_tldraw_file(year_group_node.path, year_group_node.to_dict())
# Create sequential relationship between year groups in both databases
if last_year_group_node:
neon.create_or_merge_neontology_relationship(
curriculum_relationships.YearGroupFollowsYearGroup(source=last_year_group_node, target=year_group_node),
database=school_db_name, operation='merge'
)
neon.create_or_merge_neontology_relationship(
curriculum_relationships.YearGroupFollowsYearGroup(source=last_year_group_node, target=year_group_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between year groups {last_year_group_node.unique_id} and {year_group_node.unique_id} across key stages")
last_year_group_node = year_group_node
# Create relationship with Pastoral Structure in school database only
neon.create_or_merge_neontology_relationship(
curriculum_relationships.PastoralStructureIncludesYearGroup(source=pastoral_node, target=year_group_node),
database=school_db_name, operation='merge'
)
logger.info(f"Created year group node {year_group_node_unique_id} and relationship with pastoral structure")
year_group_nodes_created[numeric_year_group] = year_group_node
node_library['year_group_nodes'][str(numeric_year_group)] = year_group_node
# Curriculum specific database initialisation begins here
# Create year group syllabus nodes in both databases
year_group_node = year_group_nodes_created.get(numeric_year_group)
if year_group_node:
# Create syllabus directory under curriculum structure
_, year_group_syllabus_path = fs_handler.create_curriculum_year_group_syllabus_directory(
curriculum_node.path,
yg_row['Subject'],
year_group,
yg_row['ID']
)
logger.info(f"Created year group syllabus directory for {year_group_syllabus_path}")
year_group_syllabus_node_unique_id = f"YearGroupSyllabus_{school_node.unique_id}_{yg_row['ID']}"
year_group_syllabus_node = neo_curriculum.YearGroupSyllabusNode(
unique_id=year_group_syllabus_node_unique_id,
yr_syllabus_id=yg_row['ID'],
yr_syllabus_name=yg_row['Title'],
yr_syllabus_year_group=str(yg_row['YearGroup']),
yr_syllabus_subject=yg_row['Subject'],
yr_syllabus_subject_code=yg_row['Subject'],
path=year_group_syllabus_path
)
# Create year group syllabus node in both databases but use same directory
neon.create_or_merge_neontology_node(year_group_syllabus_node, database=school_db_name, operation='merge')
neon.create_or_merge_neontology_node(year_group_syllabus_node, database=curriculum_db_name, operation='merge')
fs_handler.create_default_tldraw_file(year_group_syllabus_node.path, year_group_syllabus_node.to_dict())
node_library['year_group_syllabus_nodes'][yg_row['ID']] = year_group_syllabus_node
# Create sequential relationship between year group syllabuses in both databases
last_year_group_syllabus_node = last_year_group_syllabus_nodes.get(yg_row['Subject'])
# Only create sequential relationship if this year group is higher than the last one
if last_year_group_syllabus_node:
last_year = pd.to_numeric(last_year_group_syllabus_node.yr_syllabus_year_group, errors='coerce')
current_year = pd.to_numeric(year_group_syllabus_node.yr_syllabus_year_group, errors='coerce')
if pd.notna(last_year) and pd.notna(current_year) and current_year > last_year:
neon.create_or_merge_neontology_relationship(
curriculum_relationships.YearGroupSyllabusFollowsYearGroupSyllabus(source=last_year_group_syllabus_node, target=year_group_syllabus_node),
database=school_db_name, operation='merge'
)
neon.create_or_merge_neontology_relationship(
curriculum_relationships.YearGroupSyllabusFollowsYearGroupSyllabus(source=last_year_group_syllabus_node, target=year_group_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between year group syllabuses {last_year_group_syllabus_node.unique_id} and {year_group_syllabus_node.unique_id}")
last_year_group_syllabus_nodes[yg_row['Subject']] = year_group_syllabus_node
# Create relationships in both databases using MATCH to avoid cartesian products
subject_node = node_library['subject_nodes'].get(yg_row['Subject'])
if subject_node:
# Link to subject
neon.create_or_merge_neontology_relationship(
curriculum_relationships.SubjectHasYearGroupSyllabus(source=subject_node, target=year_group_syllabus_node),
database=school_db_name, operation='merge'
)
neon.create_or_merge_neontology_relationship(
curriculum_relationships.SubjectHasYearGroupSyllabus(source=subject_node, target=year_group_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between subject {subject_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
# Link to year group
neon.create_or_merge_neontology_relationship(
curriculum_relationships.YearGroupHasYearGroupSyllabus(source=year_group_node, target=year_group_syllabus_node),
database=school_db_name, operation='merge'
)
neon.create_or_merge_neontology_relationship(
curriculum_relationships.YearGroupHasYearGroupSyllabus(source=year_group_node, target=year_group_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between year group {year_group_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
# Link to key stage syllabus if it exists for the same subject
key_stage_syllabus_node = node_library['key_stage_syllabus_nodes'].get(ks_row['ID'])
if key_stage_syllabus_node and yg_row['Subject'] == ks_row['Subject']:
neon.create_or_merge_neontology_relationship(
curriculum_relationships.KeyStageSyllabusIncludesYearGroupSyllabus(source=key_stage_syllabus_node, target=year_group_syllabus_node),
database=school_db_name, operation='merge'
)
neon.create_or_merge_neontology_relationship(
curriculum_relationships.KeyStageSyllabusIncludesYearGroupSyllabus(source=key_stage_syllabus_node, target=year_group_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between key stage syllabus {key_stage_syllabus_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
# Processing of curriculum topic begins here
# Process topics for this year group syllabus only if not already processed
topics_for_syllabus = topic_df[topic_df['SyllabusYearID'] == yg_row['ID']]
for _, topic_row in topics_for_syllabus.iterrows():
if topic_row['TopicID'] in topics_processed:
continue
topics_processed.add(topic_row['TopicID'])
# Get the correct subject from the topic row
topic_subject = topic_row['SyllabusSubject']
topic_key_stage = topic_row['SyllabusKeyStage']
logger.debug(f"Processing topic {topic_row['TopicID']} for subject {topic_subject} and key stage {topic_key_stage}")
logger.debug(f"Available key stage syllabus nodes: {[node.ks_syllabus_subject + '_KS' + node.ks_syllabus_key_stage for node in node_library['key_stage_syllabus_nodes'].values()]}")
# Find the key stage syllabus node by iterating through all nodes
matching_syllabus_node = None
for syllabus_node in node_library['key_stage_syllabus_nodes'].values():
logger.debug(f"Checking syllabus node - Subject: {syllabus_node.ks_syllabus_subject}, Key Stage: {syllabus_node.ks_syllabus_key_stage}")
logger.debug(f"Comparing with - Subject: {topic_subject}, Key Stage: {str(topic_key_stage)}")
logger.debug(f"Types - Node Subject: {type(syllabus_node.ks_syllabus_subject)}, Topic Subject: {type(topic_subject)}")
logger.debug(f"Types - Node Key Stage: {type(syllabus_node.ks_syllabus_key_stage)}, Topic Key Stage: {type(str(topic_key_stage))}")
if (syllabus_node.ks_syllabus_subject == topic_subject and
syllabus_node.ks_syllabus_key_stage == str(topic_key_stage)):
matching_syllabus_node = syllabus_node
logger.debug(f"Found matching syllabus node: {syllabus_node.unique_id}")
break
if not matching_syllabus_node:
logger.warning(f"No key stage syllabus node found for subject {topic_subject} and key stage {topic_key_stage}, skipping topic creation")
continue
_, topic_path = fs_handler.create_curriculum_topic_directory(matching_syllabus_node.path, topic_row['TopicID'])
logger.info(f"Created topic directory for {topic_path}")
topic_node_unique_id = f"Topic_{matching_syllabus_node.unique_id}_{topic_row['TopicID']}"
topic_node = neo_curriculum.TopicNode(
unique_id=topic_node_unique_id,
topic_id=topic_row['TopicID'],
topic_title=topic_row.get('TopicTitle', default_topic_values['topic_title']),
total_number_of_lessons_for_topic=str(topic_row.get('TotalNumberOfLessonsForTopic', default_topic_values['total_number_of_lessons_for_topic'])),
topic_type=topic_row.get('TopicType', default_topic_values['topic_type']),
topic_assessment_type=topic_row.get('TopicAssessmentType', default_topic_values['topic_assessment_type']),
path=topic_path
)
# Create topic node in curriculum database only
neon.create_or_merge_neontology_node(topic_node, database=curriculum_db_name, operation='merge')
fs_handler.create_default_tldraw_file(topic_node.path, topic_node.to_dict())
node_library['topic_nodes'][topic_row['TopicID']] = topic_node
# Link topic to key stage syllabus as well as year group syllabus
neon.create_or_merge_neontology_relationship(
curriculum_relationships.KeyStageSyllabusIncludesTopic(source=matching_syllabus_node, target=topic_node),
database=curriculum_db_name, operation='merge'
)
neon.create_or_merge_neontology_relationship(
curriculum_relationships.YearGroupSyllabusIncludesTopic(source=year_group_syllabus_node, target=topic_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationships between topic {topic_node_unique_id} and key stage syllabus {matching_syllabus_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
# Process lessons for this topic only if not already processed
lessons_for_topic = lesson_df[
(lesson_df['TopicID'] == topic_row['TopicID']) &
(lesson_df['SyllabusSubject'] == topic_subject)
].copy()
lessons_for_topic.loc[:, 'Lesson'] = lessons_for_topic['Lesson'].astype(str)
lessons_for_topic = lessons_for_topic.sort_values('Lesson')
previous_lesson_node = None
for _, lesson_row in lessons_for_topic.iterrows():
if lesson_row['LessonID'] in lessons_processed:
continue
lessons_processed.add(lesson_row['LessonID'])
_, lesson_path = fs_handler.create_curriculum_lesson_directory(topic_path, lesson_row['LessonID'])
logger.info(f"Created lesson directory for {lesson_path}")
lesson_data = {
'unique_id': f"TopicLesson_{topic_node_unique_id}_{lesson_row['LessonID']}",
'topic_lesson_id': lesson_row['LessonID'],
'topic_lesson_title': lesson_row.get('LessonTitle', default_topic_lesson_values['topic_lesson_title']),
'topic_lesson_type': lesson_row.get('LessonType', default_topic_lesson_values['topic_lesson_type']),
'topic_lesson_length': str(lesson_row.get('SuggestedNumberOfPeriodsForLesson', default_topic_lesson_values['topic_lesson_length'])),
'topic_lesson_suggested_activities': lesson_row.get('SuggestedActivities', default_topic_lesson_values['topic_lesson_suggested_activities']),
'topic_lesson_skills_learned': lesson_row.get('SkillsLearned', default_topic_lesson_values['topic_lesson_skills_learned']),
'topic_lesson_weblinks': lesson_row.get('WebLinks', default_topic_lesson_values['topic_lesson_weblinks']),
'path': lesson_path
}
for key, value in lesson_data.items():
if pd.isna(value):
lesson_data[key] = default_topic_lesson_values.get(key, 'Null')
lesson_node = neo_curriculum.TopicLessonNode(**lesson_data)
# Create lesson node in curriculum database only
neon.create_or_merge_neontology_node(lesson_node, database=curriculum_db_name, operation='merge')
fs_handler.create_default_tldraw_file(lesson_node.path, lesson_node.to_dict())
node_library['topic_lesson_nodes'][lesson_row['LessonID']] = lesson_node
# Link lesson to topic
neon.create_or_merge_neontology_relationship(
curriculum_relationships.TopicIncludesTopicLesson(source=topic_node, target=lesson_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created lesson node {lesson_node.unique_id} and relationship with topic {topic_node.unique_id}")
# Create sequential relationships between lessons
if lesson_row['Lesson'].isdigit() and previous_lesson_node:
neon.create_or_merge_neontology_relationship(
curriculum_relationships.TopicLessonFollowsTopicLesson(source=previous_lesson_node, target=lesson_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.unique_id} and {lesson_node.unique_id}")
previous_lesson_node = lesson_node
# Process learning statements for this lesson only if not already processed
statements_for_lesson = statement_df[
(statement_df['LessonID'] == lesson_row['LessonID']) &
(statement_df['SyllabusSubject'] == topic_subject)
]
for _, statement_row in statements_for_lesson.iterrows():
if statement_row['StatementID'] in statements_processed:
continue
statements_processed.add(statement_row['StatementID'])
_, statement_path = fs_handler.create_curriculum_learning_statement_directory(lesson_path, statement_row['StatementID'])
statement_data = {
'unique_id': f"LearningStatement_{lesson_node.unique_id}_{statement_row['StatementID']}",
'lesson_learning_statement_id': statement_row['StatementID'],
'lesson_learning_statement': statement_row.get('LearningStatement', default_learning_statement_values['lesson_learning_statement']),
'lesson_learning_statement_type': statement_row.get('StatementType', default_learning_statement_values['lesson_learning_statement_type']),
'path': statement_path
}
for key in statement_data:
if pd.isna(statement_data[key]):
statement_data[key] = default_learning_statement_values.get(key, 'Null')
statement_node = neo_curriculum.LearningStatementNode(**statement_data)
# Create statement node in curriculum database only
neon.create_or_merge_neontology_node(statement_node, database=curriculum_db_name, operation='merge')
fs_handler.create_default_tldraw_file(statement_node.path, statement_node.to_dict())
node_library['statement_nodes'][statement_row['StatementID']] = statement_node
# Link learning statement to lesson
neon.create_or_merge_neontology_relationship(
curriculum_relationships.LessonIncludesLearningStatement(source=lesson_node, target=statement_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created learning statement node {statement_node.unique_id} and relationship with lesson {lesson_node.unique_id}")
else:
logger.warning(f"No year group node found for year group {year_group}, skipping syllabus creation")
# After processing all year groups and their syllabuses, process any remaining topics
logger.info("Processing topics without year groups")
for _, topic_row in topic_df.iterrows():
if topic_row['TopicID'] in topics_processed:
continue
topic_subject = topic_row['SyllabusSubject']
topic_key_stage = topic_row['SyllabusKeyStage']
logger.debug(f"Processing topic {topic_row['TopicID']} for subject {topic_subject} and key stage {topic_key_stage} without year group")
# Find the key stage syllabus node
matching_syllabus_node = None
for syllabus_node in node_library['key_stage_syllabus_nodes'].values():
if (syllabus_node.ks_syllabus_subject == topic_subject and
syllabus_node.ks_syllabus_key_stage == str(topic_key_stage)):
matching_syllabus_node = syllabus_node
break
if not matching_syllabus_node:
logger.warning(f"No key stage syllabus node found for subject {topic_subject} and key stage {topic_key_stage}, skipping topic creation")
continue
_, topic_path = fs_handler.create_curriculum_topic_directory(matching_syllabus_node.path, topic_row['TopicID'])
logger.info(f"Created topic directory for {topic_path}")
topic_node_unique_id = f"Topic_{matching_syllabus_node.unique_id}_{topic_row['TopicID']}"
topic_node = neo_curriculum.TopicNode(
unique_id=topic_node_unique_id,
topic_id=topic_row['TopicID'],
topic_title=topic_row.get('TopicTitle', default_topic_values['topic_title']),
total_number_of_lessons_for_topic=str(topic_row.get('TotalNumberOfLessonsForTopic', default_topic_values['total_number_of_lessons_for_topic'])),
topic_type=topic_row.get('TopicType', default_topic_values['topic_type']),
topic_assessment_type=topic_row.get('TopicAssessmentType', default_topic_values['topic_assessment_type']),
path=topic_path
)
# Create topic node in curriculum database only
neon.create_or_merge_neontology_node(topic_node, database=curriculum_db_name, operation='merge')
fs_handler.create_default_tldraw_file(topic_node.path, topic_node.to_dict())
node_library['topic_nodes'][topic_row['TopicID']] = topic_node
topics_processed.add(topic_row['TopicID'])
# Link topic to key stage syllabus
neon.create_or_merge_neontology_relationship(
curriculum_relationships.KeyStageSyllabusIncludesTopic(source=matching_syllabus_node, target=topic_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between topic {topic_node_unique_id} and key stage syllabus {matching_syllabus_node.unique_id}")
# Process lessons for this topic
lessons_for_topic = lesson_df[
(lesson_df['TopicID'] == topic_row['TopicID']) &
(lesson_df['SyllabusSubject'] == topic_subject)
].copy()
lessons_for_topic.loc[:, 'Lesson'] = lessons_for_topic['Lesson'].astype(str)
lessons_for_topic = lessons_for_topic.sort_values('Lesson')
previous_lesson_node = None
for _, lesson_row in lessons_for_topic.iterrows():
if lesson_row['LessonID'] in lessons_processed:
continue
lessons_processed.add(lesson_row['LessonID'])
_, lesson_path = fs_handler.create_curriculum_lesson_directory(topic_path, lesson_row['LessonID'])
logger.info(f"Created lesson directory for {lesson_path}")
lesson_data = {
'unique_id': f"TopicLesson_{topic_node_unique_id}_{lesson_row['LessonID']}",
'topic_lesson_id': lesson_row['LessonID'],
'topic_lesson_title': lesson_row.get('LessonTitle', default_topic_lesson_values['topic_lesson_title']),
'topic_lesson_type': lesson_row.get('LessonType', default_topic_lesson_values['topic_lesson_type']),
'topic_lesson_length': str(lesson_row.get('SuggestedNumberOfPeriodsForLesson', default_topic_lesson_values['topic_lesson_length'])),
'topic_lesson_suggested_activities': lesson_row.get('SuggestedActivities', default_topic_lesson_values['topic_lesson_suggested_activities']),
'topic_lesson_skills_learned': lesson_row.get('SkillsLearned', default_topic_lesson_values['topic_lesson_skills_learned']),
'topic_lesson_weblinks': lesson_row.get('WebLinks', default_topic_lesson_values['topic_lesson_weblinks']),
'path': lesson_path
}
for key, value in lesson_data.items():
if pd.isna(value):
lesson_data[key] = default_topic_lesson_values.get(key, 'Null')
lesson_node = neo_curriculum.TopicLessonNode(**lesson_data)
# Create lesson node in curriculum database only
neon.create_or_merge_neontology_node(lesson_node, database=curriculum_db_name, operation='merge')
fs_handler.create_default_tldraw_file(lesson_node.path, lesson_node.to_dict())
node_library['topic_lesson_nodes'][lesson_row['LessonID']] = lesson_node
# Link lesson to topic
neon.create_or_merge_neontology_relationship(
curriculum_relationships.TopicIncludesTopicLesson(source=topic_node, target=lesson_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created lesson node {lesson_node.unique_id} and relationship with topic {topic_node.unique_id}")
# Create sequential relationships between lessons
if lesson_row['Lesson'].isdigit() and previous_lesson_node:
neon.create_or_merge_neontology_relationship(
curriculum_relationships.TopicLessonFollowsTopicLesson(source=previous_lesson_node, target=lesson_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.unique_id} and {lesson_node.unique_id}")
previous_lesson_node = lesson_node
# Process learning statements for this lesson
statements_for_lesson = statement_df[
(statement_df['LessonID'] == lesson_row['LessonID']) &
(statement_df['SyllabusSubject'] == topic_subject)
]
for _, statement_row in statements_for_lesson.iterrows():
if statement_row['StatementID'] in statements_processed:
continue
statements_processed.add(statement_row['StatementID'])
_, statement_path = fs_handler.create_curriculum_learning_statement_directory(lesson_path, statement_row['StatementID'])
statement_data = {
'unique_id': f"LearningStatement_{lesson_node.unique_id}_{statement_row['StatementID']}",
'lesson_learning_statement_id': statement_row['StatementID'],
'lesson_learning_statement': statement_row.get('LearningStatement', default_learning_statement_values['lesson_learning_statement']),
'lesson_learning_statement_type': statement_row.get('StatementType', default_learning_statement_values['lesson_learning_statement_type']),
'path': statement_path
}
for key in statement_data:
if pd.isna(statement_data[key]):
statement_data[key] = default_learning_statement_values.get(key, 'Null')
statement_node = neo_curriculum.LearningStatementNode(**statement_data)
# Create statement node in curriculum database only
neon.create_or_merge_neontology_node(statement_node, database=curriculum_db_name, operation='merge')
fs_handler.create_default_tldraw_file(statement_node.path, statement_node.to_dict())
node_library['statement_nodes'][statement_row['StatementID']] = statement_node
# Link learning statement to lesson
neon.create_or_merge_neontology_relationship(
curriculum_relationships.LessonIncludesLearningStatement(source=lesson_node, target=statement_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created learning statement node {statement_node.unique_id} and relationship with lesson {lesson_node.unique_id}")
return node_library
modules/database/admin/school_manager.py
-512
View File
@@ -1,512 +0,0 @@
import os
from modules.logger_tool import initialise_logger
from supabase import create_client
import json
import pandas as pd
import modules.database.init.xl_tools as xl
from modules.database.tools.filesystem_tools import ClassroomCopilotFilesystem
import modules.database.tools.neo4j_driver_tools as driver_tools
import modules.database.tools.neo4j_session_tools as session_tools
import modules.database.schemas.nodes.schools.schools as school_schemas
import modules.database.schemas.nodes.schools.curriculum as curriculum_schemas
import modules.database.schemas.nodes.schools.pastoral as pastoral_schemas
import modules.database.schemas.nodes.structures.schools as school_structures
import modules.database.schemas.entities as entities
from modules.database.admin.neontology_provider import NeontologyProvider
from modules.database.admin.graph_provider import GraphNamingProvider
from modules.database.schemas.relationships import curriculum_relationships, entity_relationships, entity_curriculum_rels
class SchoolManager:
def __init__(self):
self.driver = driver_tools.get_driver()
self.logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
self.neontology = NeontologyProvider()
self.graph_naming = GraphNamingProvider()
# Initialize Supabase client with correct URL and service role key
supabase_url = os.getenv("SUPABASE_URL")
service_role_key = os.getenv("SERVICE_ROLE_KEY")
self.logger.info(f"Initializing Supabase client with URL: {supabase_url}")
self.supabase = create_client(supabase_url, service_role_key)
# Set headers for admin operations
self.supabase.headers = {
"apiKey": service_role_key,
"Authorization": f"Bearer {service_role_key}",
"Content-Type": "application/json"
}
# Set storage client headers explicitly
self.supabase.storage._client.headers.update({
"apiKey": service_role_key,
"Authorization": f"Bearer {service_role_key}",
"Content-Type": "application/json"
})
def create_schools_database(self):
"""Creates the main cc.institutes database in Neo4j"""
try:
db_name = "cc.institutes"
with self.driver.session() as session:
return self._extracted_from_create_private_database(
session, db_name, f'Created database {db_name}'
)
except Exception as e:
self.logger.error(f"Error creating schools database: {str(e)}")
return {"status": "error", "message": str(e)}
def create_school_node(self, school_data):
"""Creates a school node in cc.institutes database and stores TLDraw file in Supabase"""
try:
# Convert Supabase school data to SchoolNode using GraphNamingProvider
school_unique_id = self.graph_naming.get_school_unique_id(school_data['urn'])
school_path = self.graph_naming.get_school_path("cc.institutes", school_data['urn'])
school_node = entities.school_schemas.SchoolNode(
unique_id=school_unique_id,
path=school_path,
urn=school_data['urn'],
establishment_number=school_data['establishment_number'],
establishment_name=school_data['establishment_name'],
establishment_type=school_data['establishment_type'],
establishment_status=school_data['establishment_status'],
phase_of_education=school_data['phase_of_education'] if school_data['phase_of_education'] not in [None, ''] else None,
statutory_low_age=int(school_data['statutory_low_age']) if school_data.get('statutory_low_age') is not None else 0,
statutory_high_age=int(school_data['statutory_high_age']) if school_data.get('statutory_high_age') is not None else 0,
religious_character=school_data.get('religious_character') if school_data.get('religious_character') not in [None, ''] else None,
school_capacity=int(school_data['school_capacity']) if school_data.get('school_capacity') is not None else 0,
school_website=school_data.get('school_website', ''),
ofsted_rating=school_data.get('ofsted_rating') if school_data.get('ofsted_rating') not in [None, ''] else None
)
# Create default tldraw file data
tldraw_data = {
"document": {
"version": 1,
"id": school_data['urn'],
"name": school_data['establishment_name'],
"meta": {
"created_at": "",
"updated_at": "",
"creator_id": "",
"is_template": False,
"is_snapshot": False,
"is_draft": False,
"template_id": None,
"snapshot_id": None,
"draft_id": None
}
},
"schema": {
"schemaVersion": 1,
"storeVersion": 4,
"recordVersions": {
"asset": {
"version": 1,
"subTypeKey": "type",
"subTypeVersions": {}
},
"camera": {
"version": 1
},
"document": {
"version": 2
},
"instance": {
"version": 22
},
"instance_page_state": {
"version": 5
},
"page": {
"version": 1
},
"shape": {
"version": 3,
"subTypeKey": "type",
"subTypeVersions": {
"cc-school-node": 1
}
},
"instance_presence": {
"version": 5
},
"pointer": {
"version": 1
}
}
},
"store": {
"document:document": {
"gridSize": 10,
"name": school_data['establishment_name'],
"meta": {},
"id": school_data['urn'],
"typeName": "document"
},
"page:page": {
"meta": {},
"id": "page",
"name": "Page 1",
"index": "a1",
"typeName": "page"
},
"shape:school-node": {
"x": 0,
"y": 0,
"rotation": 0,
"type": "cc-school-node",
"id": school_unique_id,
"parentId": "page",
"index": "a1",
"props": school_node.to_dict(),
"typeName": "shape"
},
"instance:instance": {
"id": "instance",
"currentPageId": "page",
"typeName": "instance"
},
"camera:camera": {
"x": 0,
"y": 0,
"z": 1,
"id": "camera",
"typeName": "camera"
}
}
}
# Store tldraw file in Supabase storage
file_path = f"{school_data['urn']}/tldraw.json"
file_options = {
"content-type": "application/json",
"x-upsert": "true", # Update if exists
"metadata": {
"establishment_urn": school_data['urn'],
"establishment_name": school_data['establishment_name']
}
}
try:
# Create a fresh service role client for storage operations
self.logger.info("Creating fresh service role client for storage operations")
service_client = create_client(
os.getenv("SUPABASE_URL"),
os.getenv("SERVICE_ROLE_KEY")
)
self.logger.debug(f"Service client created with URL: {os.getenv('SUPABASE_URL')}")
service_client.headers = {
"apiKey": os.getenv("SERVICE_ROLE_KEY"),
"Authorization": f"Bearer {os.getenv('SERVICE_ROLE_KEY')}",
"Content-Type": "application/json"
}
service_client.storage._client.headers.update({
"apiKey": os.getenv("SERVICE_ROLE_KEY"),
"Authorization": f"Bearer {os.getenv('SERVICE_ROLE_KEY')}",
"Content-Type": "application/json"
})
self.logger.debug("Headers set for service client and storage client")
# Upload to Supabase storage using service role client
self.logger.info(f"Uploading tldraw file for school {school_data['urn']}")
self.logger.debug(f"File path: {file_path}")
self.logger.debug(f"File options: {file_options}")
# First, ensure the bucket exists
self.logger.info("Checking if bucket cc.institutes exists")
try:
bucket = service_client.storage.get_bucket("cc.institutes")
self.logger.info("Bucket cc.institutes exists")
except Exception as bucket_error:
self.logger.error(f"Error checking bucket: {str(bucket_error)}")
if hasattr(bucket_error, 'response'):
self.logger.error(f"Bucket error response: {bucket_error.response.text if hasattr(bucket_error.response, 'text') else bucket_error.response}")
raise bucket_error
# Attempt the upload
self.logger.info("Attempting file upload")
result = service_client.storage.from_("cc.institutes").upload(
path=file_path,
file=json.dumps(tldraw_data).encode(),
file_options=file_options
)
self.logger.info(f"Upload successful. Result: {result}")
except Exception as upload_error:
self.logger.error(f"Error uploading tldraw file: {str(upload_error)}")
if hasattr(upload_error, 'response'):
self.logger.error(f"Upload error response: {upload_error.response.text if hasattr(upload_error.response, 'text') else upload_error.response}")
raise upload_error
# Create node in Neo4j using Neontology
with self.neontology as neo:
self.logger.info(f"Creating school node in Neo4j: {school_node.to_dict()}")
neo.create_or_merge_node(school_node, database="cc.institutes", operation="merge")
return {"status": "success", "node": school_node}
except Exception as e:
self.logger.error(f"Error creating school node: {str(e)}")
return {"status": "error", "message": str(e)}
def create_private_database(self, school_data):
"""Creates a private database for a specific school"""
try:
private_db_name = f"cc.institutes.{school_data['urn']}"
with self.driver.session() as session:
return self._extracted_from_create_private_database(
session, private_db_name, 'Created private database '
)
except Exception as e:
self.logger.error(f"Error creating private database: {str(e)}")
return {"status": "error", "message": str(e)}
# TODO Rename this here and in `create_schools_database` and `create_private_database`
def _extracted_from_create_private_database(self, session, arg1, arg2):
session_tools.create_database(session, arg1)
self.logger.info(f"{arg2}{arg1}")
return {
"status": "success",
"message": f"Database {arg1} created successfully",
}
def create_basic_structure(self, school_node, database_name):
"""Creates basic structural nodes in the specified database"""
try:
# Create filesystem paths
fs_handler = ClassroomCopilotFilesystem(database_name, init_run_type="school")
# Create Department Structure node
department_structure_node_unique_id = f"DepartmentStructure_{school_node.unique_id}"
_, department_path = fs_handler.create_school_department_directory(school_node.path, "departments")
department_structure_node = entities.school_schemas.DepartmentNode(
unique_id=department_structure_node_unique_id,
path=department_path
)
# Create Curriculum Structure node
_, curriculum_path = fs_handler.create_school_curriculum_directory(school_node.path)
curriculum_node = school_structures.CurriculumStructureNode(
unique_id=f"CurriculumStructure_{school_node.unique_id}",
path=curriculum_path
)
# Create Pastoral Structure node
_, pastoral_path = fs_handler.create_school_pastoral_directory(school_node.path)
pastoral_node = school_structures.PastoralStructureNode(
unique_id=f"PastoralStructure_{school_node.unique_id}",
path=pastoral_path
)
with self.neontology as neo:
# Create nodes
neo.create_or_merge_node(department_structure_node, database=str(database_name), operation='merge')
fs_handler.create_default_tldraw_file(department_structure_node.path, department_structure_node.to_dict())
neo.create_or_merge_node(curriculum_node, database=str(database_name), operation='merge')
fs_handler.create_default_tldraw_file(curriculum_node.path, curriculum_node.to_dict())
neo.create_or_merge_node(pastoral_node, database=database_name, operation='merge')
fs_handler.create_default_tldraw_file(pastoral_node.path, pastoral_node.to_dict())
# Create relationships
neo.create_or_merge_relationship(
entity_relationships.SchoolHasDepartmentStructure(source=school_node, target=department_structure_node),
database=database_name, operation='merge'
)
neo.create_or_merge_relationship(
entity_curriculum_rels.SchoolHasCurriculumStructure(source=school_node, target=curriculum_node),
database=database_name, operation='merge'
)
neo.create_or_merge_relationship(
entity_curriculum_rels.SchoolHasPastoralStructure(source=school_node, target=pastoral_node),
database=database_name, operation='merge'
)
return {
"status": "success",
"message": "Basic structure created successfully",
"nodes": {
"department_structure": department_structure_node,
"curriculum_structure": curriculum_node,
"pastoral_structure": pastoral_node
}
}
except Exception as e:
self.logger.error(f"Error creating basic structure: {str(e)}")
return {"status": "error", "message": str(e)}
def create_detailed_structure(self, school_node, database_name, excel_file):
"""Creates detailed structural nodes from Excel file"""
try:
# First, store the Excel file in Supabase
file_path = f"{school_node.urn}/structure.xlsx"
file_options = {
"content-type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
"x-upsert": "true"
}
# Upload Excel file to storage
self.supabase.storage.from_("cc.institutes").upload(
path=file_path,
file=excel_file,
file_options=file_options
)
# Process Excel file
dataframes = xl.create_dataframes(excel_file)
# Get existing basic structure nodes
with self.neontology as neo:
result = neo.cypher_read("""
MATCH (s:School {unique_id: $school_id})
OPTIONAL MATCH (s)-[:HAS_DEPARTMENT_STRUCTURE]->(ds:DepartmentStructure)
OPTIONAL MATCH (s)-[:HAS_CURRICULUM_STRUCTURE]->(cs:CurriculumStructure)
OPTIONAL MATCH (s)-[:HAS_PASTORAL_STRUCTURE]->(ps:PastoralStructure)
RETURN ds, cs, ps
""", {"school_id": school_node.unique_id}, database=database_name)
if not result:
raise Exception("Basic structure not found")
department_structure = result['ds']
curriculum_structure = result['cs']
pastoral_structure = result['ps']
# Create departments and subjects
unique_departments = dataframes['keystagesyllabuses']['Department'].dropna().unique()
fs_handler = ClassroomCopilotFilesystem(database_name, init_run_type="school")
node_library = {}
with self.neontology as neo:
for department_name in unique_departments:
_, department_path = fs_handler.create_school_department_directory(school_node.path, department_name)
department_node = school_schemas.DepartmentNode(
unique_id=f"Department_{school_node.unique_id}_{department_name.replace(' ', '_')}",
department_name=department_name,
path=department_path
)
neo.create_or_merge_node(department_node, database=database_name, operation='merge')
fs_handler.create_default_tldraw_file(department_node.path, department_node.to_dict())
node_library[f'department_{department_name}'] = department_node
# Link to department structure
neo.create_or_merge_relationship(
entity_relationships.DepartmentStructureHasDepartment(
source=department_structure,
target=department_node
),
database=database_name,
operation='merge'
)
# Create year groups
year_groups = self.sort_year_groups(dataframes['yeargroupsyllabuses'])['YearGroup'].unique()
last_year_group_node = None
for year_group in year_groups:
numeric_year_group = pd.to_numeric(year_group, errors='coerce')
if pd.notna(numeric_year_group):
_, year_group_path = fs_handler.create_pastoral_year_group_directory(
pastoral_structure.path,
str(int(numeric_year_group))
)
year_group_node = pastoral_schemas.YearGroupNode(
unique_id=f"YearGroup_{school_node.unique_id}_YGrp{int(numeric_year_group)}",
year_group=str(int(numeric_year_group)),
year_group_name=f"Year {int(numeric_year_group)}",
path=year_group_path
)
neo.create_or_merge_node(year_group_node, database=database_name, operation='merge')
fs_handler.create_default_tldraw_file(year_group_node.path, year_group_node.to_dict())
node_library[f'year_group_{int(numeric_year_group)}'] = year_group_node
# Create sequential relationship
if last_year_group_node:
neo.create_or_merge_relationship(
curriculum_relationships.YearGroupFollowsYearGroup(
source=last_year_group_node,
target=year_group_node
),
database=database_name,
operation='merge'
)
last_year_group_node = year_group_node
# Link to pastoral structure
neo.create_or_merge_relationship(
curriculum_relationships.PastoralStructureIncludesYearGroup(
source=pastoral_structure,
target=year_group_node
),
database=database_name,
operation='merge'
)
# Create key stages
key_stages = dataframes['keystagesyllabuses']['KeyStage'].unique()
last_key_stage_node = None
for key_stage in sorted(key_stages):
_, key_stage_path = fs_handler.create_curriculum_key_stage_directory(
curriculum_structure.path,
str(key_stage)
)
key_stage_node = curriculum_schemas.KeyStageNode(
unique_id=f"KeyStage_{curriculum_structure.unique_id}_KStg{key_stage}",
key_stage_name=f"Key Stage {key_stage}",
key_stage=str(key_stage),
path=key_stage_path
)
neo.create_or_merge_node(key_stage_node, database=database_name, operation='merge')
fs_handler.create_default_tldraw_file(key_stage_node.path, key_stage_node.to_dict())
node_library[f'key_stage_{key_stage}'] = key_stage_node
# Create sequential relationship
if last_key_stage_node:
neo.create_or_merge_relationship(
curriculum_relationships.KeyStageFollowsKeyStage(
source=last_key_stage_node,
target=key_stage_node
),
database=database_name,
operation='merge'
)
last_key_stage_node = key_stage_node
# Link to curriculum structure
neo.create_or_merge_relationship(
curriculum_relationships.CurriculumStructureIncludesKeyStage(
source=curriculum_structure,
target=key_stage_node
),
database=database_name,
operation='merge'
)
return {
"status": "success",
"message": "Detailed structure created successfully",
"node_library": node_library
}
except Exception as e:
self.logger.error(f"Error creating detailed structure: {str(e)}")
return {"status": "error", "message": str(e)}
def sort_year_groups(self, df):
df = df.copy()
df['YearGroupNumeric'] = pd.to_numeric(df['YearGroup'], errors='coerce')
return df.sort_values(by='YearGroupNumeric')
modules/database/admin/school_syllabus_provider.py
-54
View File
@@ -1,54 +0,0 @@
import os
from modules.logger_tool import initialise_logger
import modules.database.tools.neo4j_driver_tools as driver_tools
import modules.database.tools.neo4j_session_tools as session_tools
import modules.database.tools.neontology_tools as neon
import modules.database.schemas.entities as neo_entity
import modules.database.schemas.nodes.schools.curriculum as curriculum_schemas
import modules.database.schemas.relationships.curriculum_relationships as curriculum_relationships
import modules.database.schemas.relationships.entity_relationships as ent_rels
import modules.database.schemas.relationships.entity_curriculum_rels as ent_cur_rels
import pandas as pd
class SchoolSyllabusProvider:
def __init__(self):
self.driver = driver_tools.get_driver()
self.logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
def process_syllabus_data(self, school_node, database_name, dataframes):
"""Process syllabus data from Excel file and create nodes in the database"""
try:
# This method will contain the syllabus-specific processing code from the
# original SchoolCurriculumProvider, starting from where the comment
# "# Curriculum specific database initialisation begins here" was placed
# We'll implement this in the next iteration after confirming the basic
# structure changes work correctly
return {
"status": "success",
"message": "Syllabus data processed successfully"
}
except Exception as e:
self.logger.error(f"Error processing syllabus data: {str(e)}")
return {"status": "error", "message": str(e)}
def check_syllabus_status(self, school_node, database_name):
"""Check if syllabus data exists in the database"""
try:
with self.driver.session(database=database_name) as session:
result = session.run("""
MATCH (s:School {unique_id: $school_id})
OPTIONAL MATCH (s)-[:HAS_CURRICULUM_STRUCTURE]->(:CurriculumStructure)-[:INCLUDES_KEY_STAGE]->(:KeyStage)-[:INCLUDES_KEY_STAGE_SYLLABUS]->(ks:KeyStageSyllabus)
OPTIONAL MATCH (s)-[:HAS_CURRICULUM_STRUCTURE]->(:CurriculumStructure)-[:INCLUDES_KEY_STAGE]->(:KeyStage)-[:INCLUDES_YEAR_GROUP_SYLLABUS]->(ys:YearGroupSyllabus)
RETURN count(ks) > 0 OR count(ys) > 0 as has_syllabus
""", school_id=school_node.unique_id)
has_syllabus = result.single()["has_syllabus"]
return {"has_syllabus": has_syllabus}
except Exception as e:
self.logger.error(f"Error checking syllabus status: {str(e)}")
raise
modules/database/admin/school_timetable_provider.py
-526
View File
@@ -1,526 +0,0 @@
import os
from modules.logger_tool import initialise_logger
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
import modules.database.init.init_calendar as init_calendar
import modules.database.schemas.nodes.schools.timetable as timetable
import modules.database.schemas.relationships.timetables as tt_rels
import modules.database.schemas.relationships.entity_timetable_rels as entity_tt_rels
import modules.database.schemas.relationships.calendar_timetable_rels as cal_tt_rels
import modules.database.tools.neontology_tools as neon
from modules.database.tools.filesystem_tools import ClassroomCopilotFilesystem
from datetime import timedelta, datetime
import pandas as pd
def create_school_timetable_from_dataframes(dataframes, db_name, school_node=None):
logger.info(f"Creating school timetable for {db_name}")
if dataframes is None:
raise ValueError("Data is required to create the calendar and timetable.")
logger.info("Initialising neo4j connection...")
neon.init_neontology_connection()
# Initialize the filesystem handler
fs_handler = ClassroomCopilotFilesystem(db_name, init_run_type="school")
school_df = dataframes['school']
if school_node is None:
logger.info("School node is None, using school data from dataframe")
school_unique_id = school_df[school_df['Identifier'] == 'SchoolID']['Data'].iloc[0]
else:
logger.info(f"School node is not None, using school data from school node: {school_node}")
school_unique_id = school_node.unique_id
terms_df = dataframes['terms']
weeks_df = dataframes['weeks']
days_df = dataframes['days']
periods_df = dataframes['periods']
school_df_year_start = school_df[school_df['Identifier'] == 'AcademicYearStart']['Data'].iloc[0]
school_df_year_end = school_df[school_df['Identifier'] == 'AcademicYearEnd']['Data'].iloc[0]
if isinstance(school_df_year_start, str):
school_year_start_date = datetime.strptime(school_df_year_start, '%Y-%m-%d')
else:
school_year_start_date = school_df_year_start
if isinstance(school_df_year_end, str):
school_year_end_date = datetime.strptime(school_df_year_end, '%Y-%m-%d')
else:
school_year_end_date = school_df_year_end
# Create a dictionary to store the timetable nodes
timetable_nodes = {
'timetable_node': None,
'academic_year_nodes': [],
'academic_term_nodes': [],
'academic_week_nodes': [],
'academic_day_nodes': [],
'academic_period_nodes': []
}
if school_node:
# Create the root timetable directory
_, timetable_path = fs_handler.create_school_timetable_directory(school_node.path)
else:
# Create the root timetable directory
_, timetable_path = fs_handler.create_school_timetable_directory()
# Create AcademicTimetable Node
school_timetable_unique_id = f"SchoolTimetable_{school_unique_id}_{school_year_start_date.year}_{school_year_end_date.year}"
school_timetable_node = timetable.SchoolTimetableNode(
unique_id=school_timetable_unique_id,
start_date=school_year_start_date,
end_date=school_year_end_date,
path=timetable_path
)
neon.create_or_merge_neontology_node(school_timetable_node, database=db_name, operation='merge')
# Create the tldraw file for the node
fs_handler.create_default_tldraw_file(school_timetable_node.path, school_timetable_node.to_dict())
timetable_nodes['timetable_node'] = school_timetable_node
if school_node:
logger.info(f"Creating calendar for {school_unique_id} from Neo4j SchoolNode: {school_node.unique_id}")
calendar_nodes = init_calendar.create_calendar(db_name, school_year_start_date, school_year_end_date, attach_to_calendar_node=True, entity_node=school_node)
# Link the school node to the timetable node
neon.create_or_merge_neontology_relationship(
entity_tt_rels.SchoolHasTimetable(source=school_node, target=school_timetable_node),
database=db_name, operation='merge'
)
timetable_nodes['calendar_nodes'] = calendar_nodes
else:
logger.info(f"Creating calendar for {school_unique_id} from dataframe SchoolID: {school_unique_id}")
calendar_nodes = init_calendar.create_calendar(db_name, school_year_start_date, school_year_end_date, attach_to_calendar_node=False, entity_node=None)
# Create AcademicYear nodes for each year within the range
for year in range(school_year_start_date.year, school_year_end_date.year + 1):
_, timetable_year_path = fs_handler.create_school_timetable_year_directory(timetable_path, year)
year_str = str(year)
academic_year_unique_id = f"AcademicYear_{school_timetable_unique_id}_{year}"
academic_year_node = timetable.AcademicYearNode(
unique_id=academic_year_unique_id,
year=year_str,
path=timetable_year_path
)
neon.create_or_merge_neontology_node(academic_year_node, database=db_name, operation='merge')
# Create the tldraw file for the node
fs_handler.create_default_tldraw_file(academic_year_node.path, academic_year_node.to_dict())
timetable_nodes['academic_year_nodes'].append(academic_year_node)
logger.info(f'Created academic year node: {academic_year_node.unique_id}')
neon.create_or_merge_neontology_relationship(
tt_rels.AcademicTimetableHasAcademicYear(source=school_timetable_node, target=academic_year_node),
database=db_name, operation='merge'
)
logger.info(f"Created school timetable relationship from {school_timetable_node.unique_id} to {academic_year_node.unique_id}")
# Link the academic year with the corresponding calendar year node
for year_node in calendar_nodes['calendar_year_nodes']:
if year_node.year == year:
neon.create_or_merge_neontology_relationship(
cal_tt_rels.AcademicYearIsCalendarYear(source=academic_year_node, target=year_node),
database=db_name, operation='merge'
)
logger.info(f"Created school timetable relationship from {academic_year_node.unique_id} to {year_node.unique_id}")
break
# Create Term and TermBreak nodes linked to AcademicYear
term_number = 1
academic_term_number = 1
for _, term_row in terms_df.iterrows():
term_node_class = timetable.AcademicTermNode if term_row['TermType'] == 'Term' else timetable.AcademicTermBreakNode
term_name = term_row['TermName']
term_name_no_spaces = term_name.replace(' ', '')
term_start_date = term_row['StartDate']
if isinstance(term_start_date, pd.Timestamp):
term_start_date = term_start_date.strftime('%Y-%m-%d')
term_end_date = term_row['EndDate']
if isinstance(term_end_date, pd.Timestamp):
term_end_date = term_end_date.strftime('%Y-%m-%d')
if term_row['TermType'] == 'Term':
_, timetable_term_path = fs_handler.create_school_timetable_academic_term_directory(
timetable_path=timetable_path,
term_name=term_name,
term_number=academic_term_number
)
term_node_unique_id = f"AcademicTerm_{school_timetable_unique_id}_{academic_term_number}_{term_name_no_spaces}"
academic_term_number_str = str(academic_term_number)
term_node = term_node_class(
unique_id=term_node_unique_id,
term_name=term_name,
term_number=academic_term_number_str,
start_date=datetime.strptime(term_start_date, '%Y-%m-%d'),
end_date=datetime.strptime(term_end_date, '%Y-%m-%d'),
path=timetable_term_path
)
academic_term_number += 1
else:
term_break_node_unique_id = f"AcademicTermBreak_{school_timetable_unique_id}_{term_name_no_spaces}"
term_node = term_node_class(
unique_id=term_break_node_unique_id,
term_break_name=term_name,
start_date=datetime.strptime(term_start_date, '%Y-%m-%d'),
end_date=datetime.strptime(term_end_date, '%Y-%m-%d')
)
neon.create_or_merge_neontology_node(term_node, database=db_name, operation='merge')
if isinstance(term_node, timetable.AcademicTermNode):
# Create the tldraw file for the node
fs_handler.create_default_tldraw_file(term_node.path, term_node.to_dict())
logger.info(f'Created academic term break node: {term_node.unique_id}')
timetable_nodes['academic_term_nodes'].append(term_node)
term_number += 1 # We don't use this but we could
# Link term node to the correct academic year
term_years = set()
term_years.update([term_node.start_date.year, term_node.end_date.year])
for academic_year_node in timetable_nodes['academic_year_nodes']:
if int(academic_year_node.year) in term_years:
relationship_class = tt_rels.AcademicYearHasAcademicTerm if term_row['TermType'] == 'Term' else tt_rels.AcademicYearHasAcademicTermBreak
neon.create_or_merge_neontology_relationship(
relationship_class(source=academic_year_node, target=term_node),
database=db_name, operation='merge'
)
logger.info(f"Created school timetable relationship from {academic_year_node.unique_id} to {term_node.unique_id}")
# Create Week nodes
academic_week_number = 1
for _, week_row in weeks_df.iterrows():
week_node_class = timetable.HolidayWeekNode if week_row['WeekType'] == 'Holiday' else timetable.AcademicWeekNode
week_start_date = week_row['WeekStart']
if isinstance(week_start_date, pd.Timestamp):
week_start_date = week_start_date.strftime('%Y-%m-%d')
if week_row['WeekType'] == 'Holiday':
week_node_unique_id = f"{week_row['WeekType']}Week_{school_timetable_unique_id}_Week_{week_row['WeekNumber']}"
week_node = week_node_class(
unique_id=week_node_unique_id,
start_date=datetime.strptime(week_start_date, '%Y-%m-%d')
)
else:
_, timetable_week_path = fs_handler.create_school_timetable_academic_week_directory(
timetable_path=timetable_path,
week_number=academic_week_number
)
week_node_unique_id = f"AcademicWeek_{school_timetable_unique_id}_Week_{week_row['WeekNumber']}"
academic_week_number_str = str(academic_week_number)
week_type = week_row['WeekType']
week_node = week_node_class(
unique_id=week_node_unique_id,
academic_week_number=academic_week_number_str,
start_date=datetime.strptime(week_start_date, '%Y-%m-%d'),
week_type=week_type,
path=timetable_week_path
)
academic_week_number += 1
neon.create_or_merge_neontology_node(week_node, database=db_name, operation='merge')
timetable_nodes['academic_week_nodes'].append(week_node)
logger.info(f"Created week node: {week_node.unique_id}")
if isinstance(week_node, timetable.AcademicWeekNode):
# Create the tldraw file for the node
fs_handler.create_default_tldraw_file(week_node.path, week_node.to_dict())
for calendar_node in calendar_nodes['calendar_week_nodes']:
if calendar_node.start_date == week_node.start_date:
if isinstance(week_node, timetable.AcademicWeekNode):
neon.create_or_merge_neontology_relationship(
cal_tt_rels.AcademicWeekIsCalendarWeek(source=week_node, target=calendar_node),
database=db_name, operation='merge'
)
logger.info(f"Created school timetable relationship from {calendar_node.unique_id} to {week_node.unique_id}")
elif isinstance(week_node, timetable.HolidayWeekNode):
neon.create_or_merge_neontology_relationship(
cal_tt_rels.HolidayWeekIsCalendarWeek(source=week_node, target=calendar_node),
database=db_name, operation='merge'
)
logger.info(f"Created school timetable relationship from {calendar_node.unique_id} to {week_node.unique_id}")
break
# Link week node to the correct academic term
for term_node in timetable_nodes['academic_term_nodes']:
if term_node.start_date <= week_node.start_date <= term_node.end_date:
relationship_class = tt_rels.AcademicTermHasAcademicWeek if week_row['WeekType'] != 'Holiday' else tt_rels.AcademicTermBreakHasHolidayWeek
neon.create_or_merge_neontology_relationship(
relationship_class(source=term_node, target=week_node),
database=db_name, operation='merge'
)
logger.info(f"Created school timetable relationship from {term_node.unique_id} to {week_node.unique_id}")
break
# Link week node to the correct academic year
for academic_year_node in timetable_nodes['academic_year_nodes']:
if int(academic_year_node.year) == week_node.start_date.year:
relationship_class = tt_rels.AcademicYearHasAcademicWeek if week_row['WeekType'] != 'Holiday' else tt_rels.AcademicYearHasHolidayWeek
neon.create_or_merge_neontology_relationship(
relationship_class(source=academic_year_node, target=week_node),
database=db_name, operation='merge'
)
logger.info(f"Created school timetable relationship from {academic_year_node.unique_id} to {week_node.unique_id}")
break
# Create Day nodes
day_number = 1
academic_day_number = 1
for _, day_row in days_df.iterrows():
date_str = day_row['Date']
if isinstance(date_str, pd.Timestamp):
date_str = date_str.strftime('%Y-%m-%d')
day_node_class = {
'Academic': timetable.AcademicDayNode,
'Holiday': timetable.HolidayDayNode,
'OffTimetable': timetable.OffTimetableDayNode,
'StaffDay': timetable.StaffDayNode
}[day_row['DayType']]
# Format the unique ID as {day_node_class.__name__}Day
day_node_data = {
'unique_id': f"{day_node_class.__name__}Day_{school_timetable_unique_id}_{day_number}",
'date': datetime.strptime(date_str, '%Y-%m-%d'),
'day_of_week': datetime.strptime(date_str, '%Y-%m-%d').strftime('%A')
}
if day_row['DayType'] == 'Academic':
day_node_data['academic_day'] = str(academic_day_number)
day_node_data['day_type'] = day_row['WeekType']
_, timetable_day_path = fs_handler.create_school_timetable_academic_day_directory(
timetable_path=timetable_path,
academic_day=academic_day_number
)
day_node_data['path'] = timetable_day_path
day_node = day_node_class(**day_node_data)
for calendar_node in calendar_nodes['calendar_day_nodes']:
if calendar_node.date == day_node.date:
neon.create_or_merge_neontology_node(day_node, database=db_name, operation='merge')
timetable_nodes['academic_day_nodes'].append(day_node)
logger.info(f"Created day node: {day_node.unique_id}")
if isinstance(day_node, timetable.AcademicDayNode):
fs_handler.create_default_tldraw_file(day_node.path, day_node.to_dict())
relationship_class = cal_tt_rels.AcademicDayIsCalendarDay
elif isinstance(day_node, timetable.HolidayDayNode):
relationship_class = cal_tt_rels.HolidayDayIsCalendarDay
elif isinstance(day_node, timetable.OffTimetableDayNode):
relationship_class = cal_tt_rels.OffTimetableDayIsCalendarDay
elif isinstance(day_node, timetable.StaffDayNode):
relationship_class = cal_tt_rels.StaffDayIsCalendarDay
neon.create_or_merge_neontology_relationship(
relationship_class(source=day_node, target=calendar_node),
database=db_name, operation='merge'
)
logger.info(f'Created relationship from {calendar_node.unique_id} to {day_node.unique_id}')
break
# Link day node to the correct academic week
for academic_week_node in timetable_nodes['academic_week_nodes']:
if academic_week_node.start_date <= day_node.date <= (academic_week_node.start_date + timedelta(days=6)):
if day_row['DayType'] == 'Academic':
relationship_class = tt_rels.AcademicWeekHasAcademicDay
elif day_row['DayType'] == 'Holiday':
if hasattr(academic_week_node, 'week_type') and academic_week_node.week_type in ['A', 'B']:
relationship_class = tt_rels.AcademicWeekHasHolidayDay
else:
relationship_class = tt_rels.HolidayWeekHasHolidayDay
elif day_row['DayType'] == 'OffTimetable':
relationship_class = tt_rels.AcademicWeekHasOffTimetableDay
elif day_row['DayType'] == 'Staff':
relationship_class = tt_rels.AcademicWeekHasStaffDay
else:
continue # Skip linking for other day types
neon.create_or_merge_neontology_relationship(
relationship_class(source=academic_week_node, target=day_node),
database=db_name, operation='merge'
)
logger.info(f"Created relationship from {academic_week_node.unique_id} to {day_node.unique_id}")
break
# Link day node to the correct academic term
for term_node in timetable_nodes['academic_term_nodes']:
if term_node.start_date <= day_node.date <= term_node.end_date:
if day_row['DayType'] == 'Academic':
relationship_class = tt_rels.AcademicTermHasAcademicDay
elif day_row['DayType'] == 'Holiday':
if isinstance(term_node, timetable.AcademicTermNode):
relationship_class = tt_rels.AcademicTermHasHolidayDay
else:
relationship_class = tt_rels.AcademicTermBreakHasHolidayDay
elif day_row['DayType'] == 'OffTimetable':
relationship_class = tt_rels.AcademicTermHasOffTimetableDay
elif day_row['DayType'] == 'Staff':
relationship_class = tt_rels.AcademicTermHasStaffDay
else:
continue # Skip linking for other day types
neon.create_or_merge_neontology_relationship(
relationship_class(source=term_node, target=day_node),
database=db_name, operation='merge'
)
logger.info(f"Created relationship from {term_node.unique_id} to {day_node.unique_id}")
break
# Create Period nodes for each academic day
if day_row['DayType'] == 'Academic':
logger.info(f"Creating periods for {day_node.unique_id}")
period_of_day = 1
academic_or_registration_period_of_day = 1
for _, period_row in periods_df.iterrows():
period_node_class = {
'Academic': timetable.AcademicPeriodNode,
'Registration': timetable.RegistrationPeriodNode,
'Break': timetable.BreakPeriodNode,
'OffTimetable': timetable.OffTimetablePeriodNode
}[period_row['PeriodType']]
logger.info(f"Creating period node for {period_node_class.__name__} Period: {period_of_day}")
period_node_unique_id = f"{period_node_class.__name__}_{school_timetable_unique_id}_Day_{academic_day_number}_Period_{period_of_day}"
logger.debug(f"Period node unique id: {period_node_unique_id}")
period_node_data = {
'unique_id': period_node_unique_id,
'name': period_row['PeriodName'],
'date': day_node.date,
'start_time': datetime.combine(day_node.date, period_row['StartTime']),
'end_time': datetime.combine(day_node.date, period_row['EndTime'])
}
logger.debug(f"Period node data: {period_node_data}")
if period_row['PeriodType'] in ['Academic', 'Registration']:
_, timetable_period_path = fs_handler.create_school_timetable_period_directory(
timetable_path=timetable_path,
academic_day=academic_day_number,
period_dir=f"{academic_or_registration_period_of_day}_{period_row['PeriodName'].replace(' ', '_')}"
)
week_type = day_row['WeekType']
day_name_short = day_node.day_of_week[:3]
period_code = period_row['PeriodCode']
period_code_formatted = f"{week_type}{day_name_short}{period_code}"
period_node_data['period_code'] = period_code_formatted
period_node_data['path'] = timetable_period_path
academic_or_registration_period_of_day += 1
period_node = period_node_class(**period_node_data)
neon.create_or_merge_neontology_node(period_node, database=db_name, operation='merge')
if isinstance(period_node, timetable.AcademicPeriodNode) or isinstance(period_node, timetable.RegistrationPeriodNode):
# Create the tldraw file for the node
fs_handler.create_default_tldraw_file(period_node.path, period_node.to_dict())
timetable_nodes['academic_period_nodes'].append(period_node)
logger.info(f'Created period node: {period_node.unique_id}')
relationship_class = {
'Academic': tt_rels.AcademicDayHasAcademicPeriod,
'Registration': tt_rels.AcademicDayHasRegistrationPeriod,
'Break': tt_rels.AcademicDayHasBreakPeriod,
'OffTimetable': tt_rels.AcademicDayHasOffTimetablePeriod
}[period_row['PeriodType']]
neon.create_or_merge_neontology_relationship(
relationship_class(source=day_node, target=period_node),
database=db_name, operation='merge'
)
logger.info(f"Created relationship from {day_node.unique_id} to {period_node.unique_id}")
period_of_day += 1 # We don't use this but we could
academic_day_number += 1 # This is a bit of a hack but it works to keep the directories aligned (reorganise)
day_number += 1 # We don't use this but we could
def create_school_timetable_node_sequence_rels(timetable_nodes):
def sort_and_create_relationships(nodes, relationship_map, sort_key):
sorted_nodes = sorted(nodes, key=sort_key)
for i in range(len(sorted_nodes) - 1):
source_node = sorted_nodes[i]
target_node = sorted_nodes[i + 1]
node_type_pair = (type(source_node), type(target_node))
relationship_class = relationship_map.get(node_type_pair)
if relationship_class:
# Avoid self-referential relationships
if source_node.unique_id != target_node.unique_id:
neon.create_or_merge_neontology_relationship(
relationship_class(
source=source_node,
target=target_node
),
database=db_name, operation='merge'
)
logger.info(f"Created relationship from {source_node.unique_id} to {target_node.unique_id}")
else:
logger.warning(f"Skipped self-referential relationship for node {source_node.unique_id}")
# Relationship maps for different node types
academic_year_relationship_map = {
(timetable.AcademicYearNode, timetable.AcademicYearNode): tt_rels.AcademicYearFollowsAcademicYear
}
academic_term_relationship_map = {
(timetable.AcademicTermNode, timetable.AcademicTermBreakNode): tt_rels.AcademicTermBreakFollowsAcademicTerm,
(timetable.AcademicTermBreakNode, timetable.AcademicTermNode): tt_rels.AcademicTermFollowsAcademicTermBreak
}
academic_week_relationship_map = {
(timetable.AcademicWeekNode, timetable.AcademicWeekNode): tt_rels.AcademicWeekFollowsAcademicWeek,
(timetable.HolidayWeekNode, timetable.HolidayWeekNode): tt_rels.HolidayWeekFollowsHolidayWeek,
(timetable.AcademicWeekNode, timetable.HolidayWeekNode): tt_rels.HolidayWeekFollowsAcademicWeek,
(timetable.HolidayWeekNode, timetable.AcademicWeekNode): tt_rels.AcademicWeekFollowsHolidayWeek
}
academic_day_relationship_map = {
(timetable.AcademicDayNode, timetable.AcademicDayNode): tt_rels.AcademicDayFollowsAcademicDay,
(timetable.HolidayDayNode, timetable.HolidayDayNode): tt_rels.HolidayDayFollowsHolidayDay,
(timetable.OffTimetableDayNode, timetable.OffTimetableDayNode): tt_rels.OffTimetableDayFollowsOffTimetableDay,
(timetable.StaffDayNode, timetable.StaffDayNode): tt_rels.StaffDayFollowsStaffDay,
(timetable.AcademicDayNode, timetable.HolidayDayNode): tt_rels.HolidayDayFollowsAcademicDay,
(timetable.AcademicDayNode, timetable.OffTimetableDayNode): tt_rels.OffTimetableDayFollowsAcademicDay,
(timetable.AcademicDayNode, timetable.StaffDayNode): tt_rels.StaffDayFollowsAcademicDay,
(timetable.HolidayDayNode, timetable.AcademicDayNode): tt_rels.AcademicDayFollowsHolidayDay,
(timetable.HolidayDayNode, timetable.OffTimetableDayNode): tt_rels.OffTimetableDayFollowsHolidayDay,
(timetable.HolidayDayNode, timetable.StaffDayNode): tt_rels.StaffDayFollowsHolidayDay,
(timetable.OffTimetableDayNode, timetable.AcademicDayNode): tt_rels.AcademicDayFollowsOffTimetableDay,
(timetable.OffTimetableDayNode, timetable.HolidayDayNode): tt_rels.HolidayDayFollowsOffTimetableDay,
(timetable.OffTimetableDayNode, timetable.StaffDayNode): tt_rels.StaffDayFollowsOffTimetableDay,
(timetable.StaffDayNode, timetable.AcademicDayNode): tt_rels.AcademicDayFollowsStaffDay,
(timetable.StaffDayNode, timetable.HolidayDayNode): tt_rels.HolidayDayFollowsStaffDay,
(timetable.StaffDayNode, timetable.OffTimetableDayNode): tt_rels.OffTimetableDayFollowsStaffDay,
}
academic_period_relationship_map = {
(timetable.AcademicPeriodNode, timetable.AcademicPeriodNode): tt_rels.AcademicPeriodFollowsAcademicPeriod,
(timetable.AcademicPeriodNode, timetable.BreakPeriodNode): tt_rels.BreakPeriodFollowsAcademicPeriod,
(timetable.AcademicPeriodNode, timetable.RegistrationPeriodNode): tt_rels.RegistrationPeriodFollowsAcademicPeriod,
(timetable.AcademicPeriodNode, timetable.OffTimetablePeriodNode): tt_rels.OffTimetablePeriodFollowsAcademicPeriod,
(timetable.BreakPeriodNode, timetable.AcademicPeriodNode): tt_rels.AcademicPeriodFollowsBreakPeriod,
(timetable.BreakPeriodNode, timetable.BreakPeriodNode): tt_rels.BreakPeriodFollowsBreakPeriod,
(timetable.BreakPeriodNode, timetable.RegistrationPeriodNode): tt_rels.RegistrationPeriodFollowsBreakPeriod,
(timetable.BreakPeriodNode, timetable.OffTimetablePeriodNode): tt_rels.OffTimetablePeriodFollowsBreakPeriod,
(timetable.RegistrationPeriodNode, timetable.AcademicPeriodNode): tt_rels.AcademicPeriodFollowsRegistrationPeriod,
(timetable.RegistrationPeriodNode, timetable.RegistrationPeriodNode): tt_rels.RegistrationPeriodFollowsRegistrationPeriod,
(timetable.RegistrationPeriodNode, timetable.BreakPeriodNode): tt_rels.BreakPeriodFollowsRegistrationPeriod,
(timetable.RegistrationPeriodNode, timetable.OffTimetablePeriodNode): tt_rels.OffTimetablePeriodFollowsRegistrationPeriod,
(timetable.OffTimetablePeriodNode, timetable.OffTimetablePeriodNode): tt_rels.OffTimetablePeriodFollowsOffTimetablePeriod,
(timetable.OffTimetablePeriodNode, timetable.AcademicPeriodNode): tt_rels.AcademicPeriodFollowsOffTimetablePeriod,
(timetable.OffTimetablePeriodNode, timetable.BreakPeriodNode): tt_rels.BreakPeriodFollowsOffTimetablePeriod,
(timetable.OffTimetablePeriodNode, timetable.RegistrationPeriodNode): tt_rels.RegistrationPeriodFollowsOffTimetablePeriod,
}
# Sort and create relationships
sort_and_create_relationships(timetable_nodes['academic_year_nodes'], academic_year_relationship_map, lambda x: int(x.year))
sort_and_create_relationships(timetable_nodes['academic_term_nodes'], academic_term_relationship_map, lambda x: x.start_date)
sort_and_create_relationships(timetable_nodes['academic_week_nodes'], academic_week_relationship_map, lambda x: x.start_date)
sort_and_create_relationships(timetable_nodes['academic_day_nodes'], academic_day_relationship_map, lambda x: x.date)
sort_and_create_relationships(timetable_nodes['academic_period_nodes'], academic_period_relationship_map, lambda x: (x.start_time, x.end_time))
# Call the function with the created timetable nodes
create_school_timetable_node_sequence_rels(timetable_nodes)
logger.info(f'Created timetable: {timetable_nodes["timetable_node"].unique_id}')
# Log the directory structure after creation
# root_timetable_directory = fs_handler.root_path # Access the root directory of the filesystem handler
# fs_handler.log_directory_structure(root_timetable_directory)
return {
'school_node': school_node,
'school_calendar_nodes': calendar_nodes,
'school_timetable_nodes': timetable_nodes
}
modules/database/init/init_calendar.py
+85 -84
View File
@@ -4,12 +4,25 @@ logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH
import modules.database.schemas.nodes.calendars as calendar_schemas
import modules.database.schemas.relationships.calendars as calendar_relationships
import modules.database.schemas.relationships.calendar_sequence as calendar_sequence_relationships
import modules.database.schemas.relationships.owner_relationships as owner_relationships
import modules.database.tools.supabase_storage_tools as storage_tools
import modules.database.tools.neontology_tools as neon
from datetime import timedelta, datetime
def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False, owner_node=None, time_chunk_node_length: int = None):
logger.info(f"Creating calendar for {start_date} to {end_date}")
def create_calendar(db_name, start_date, end_date, time_chunk_node_length: int = None, storage_tools=None):
"""
Create calendar structure with years, months, weeks, and days
Args:
db_name: Database name to create calendar in
start_date: Start date for calendar
end_date: End date for calendar
time_chunk_node_length: Optional time chunk length in minutes
storage_tools: Optional Supabase storage tools for generating storage paths
Returns:
dict: Dictionary containing created calendar nodes
"""
logger.info(f"Creating calendar structure for {start_date} to {end_date} in database: {db_name}")
logger.info(f"Initializing Neontology connection")
neon.init_neontology_connection()
@@ -25,52 +38,13 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
last_day_node = None
calendar_nodes = {
'calendar_node': None,
'calendar_year_nodes': [],
'calendar_month_nodes': [],
'calendar_week_nodes': [],
'calendar_day_nodes': []
'calendar_day_nodes': [],
'calendar_time_chunk_nodes': []
}
if attach_to_calendar_node and owner_node:
logger.info(f"Attaching calendar to owner's node {owner_node.unique_id} in database: {db_name}")
owner_unique_id = owner_node.unique_id
calendar_unique_id = f"{start_date.strftime('%Y-%m-%d')}_{end_date.strftime('%Y-%m-%d')}"
calendar_node = calendar_schemas.CalendarNode(
unique_id=calendar_unique_id,
name=f"{start_date.strftime('%Y-%m-%d')} to {end_date.strftime('%Y-%m-%d')}",
start_date=start_date,
end_date=end_date,
tldraw_snapshot=""
)
neon.create_or_merge_neontology_node(calendar_node, database=db_name, operation='merge')
calendar_nodes['calendar_node'] = calendar_node
logger.info(f"Calendar node created: {calendar_node.unique_id}")
import modules.database.schemas.relationships.owner_relationships as owner_relationships
neon.create_or_merge_neontology_relationship(
owner_relationships.OwnerHasCalendar(source=owner_node, target=calendar_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {owner_node.unique_id} to {calendar_node.unique_id}")
elif attach_to_calendar_node and not owner_node:
logger.info(f"Creating calendar for {start_date} to {end_date} in database: {db_name}")
calendar_node = calendar_schemas.CalendarNode(
unique_id=f"{start_date.strftime('%Y-%m-%d')}_{end_date.strftime('%Y-%m-%d')}",
name=f"{start_date.strftime('%Y-%m-%d')} to {end_date.strftime('%Y-%m-%d')}",
start_date=start_date,
end_date=end_date,
tldraw_snapshot=""
)
neon.create_or_merge_neontology_node(calendar_node, database=db_name, operation='merge')
calendar_nodes['calendar_node'] = calendar_node
logger.info(f"Calendar node created: {calendar_node.unique_id}")
else:
logger.error("Invalid combination of parameters for calendar creation.")
raise ValueError("Invalid combination of parameters for calendar creation.")
current_date = start_date
while current_date <= end_date:
year = current_date.year
@@ -78,50 +52,55 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
day = current_date.day
iso_year, iso_week, iso_weekday = current_date.isocalendar()
calendar_year_unique_id = f"{year}"
calendar_year_uuid_string = f"{year}"
if year not in created_years:
# Generate storage path for year node using Supabase Storage
if storage_tools:
year_dir_created, node_storage_path = storage_tools.create_calendar_year_storage_path(year)
else:
node_storage_path = ""
year_node = calendar_schemas.CalendarYearNode(
unique_id=calendar_year_unique_id,
uuid_string=calendar_year_uuid_string,
year=str(year),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(year_node, database=db_name, operation='merge')
calendar_nodes['calendar_year_nodes'].append(year_node)
created_years[year] = year_node
logger.info(f"Year node created: {year_node.unique_id}")
logger.info(f"Year node created: {year_node.uuid_string}")
if attach_to_calendar_node:
neon.create_or_merge_neontology_relationship(
calendar_relationships.CalendarIncludesYear(source=calendar_node, target=year_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {calendar_node.unique_id} to {year_node.unique_id}")
if last_year_node:
neon.create_or_merge_neontology_relationship(
calendar_sequence_relationships.YearFollowsYear(source=last_year_node, target=year_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {last_year_node.unique_id} to {year_node.unique_id}")
logger.info(f"Relationship created from {last_year_node.uuid_string} to {year_node.uuid_string}")
last_year_node = year_node
calendar_month_unique_id = f"{year}_{month}"
calendar_month_uuid_string = f"{year}_{month}"
month_key = f"{year}-{month}"
if month_key not in created_months:
# Generate storage path for month node using Supabase Storage
if storage_tools:
month_dir_created, node_storage_path = storage_tools.create_calendar_month_storage_path(year, month)
else:
node_storage_path = ""
month_node = calendar_schemas.CalendarMonthNode(
unique_id=calendar_month_unique_id,
uuid_string=calendar_month_uuid_string,
year=str(year),
month=str(month),
month_name=datetime(year, month, 1).strftime('%B'),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(month_node, database=db_name, operation='merge')
calendar_nodes['calendar_month_nodes'].append(month_node)
created_months[month_key] = month_node
logger.info(f"Month node created: {month_node.unique_id}")
logger.info(f"Month node created: {month_node.uuid_string}")
# Check for the end of year transition for months
if last_month_node:
@@ -131,14 +110,14 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {last_month_node.unique_id} to {month_node.unique_id}")
logger.info(f"Relationship created from {last_month_node.uuid_string} to {month_node.uuid_string}")
elif int(month) == int(last_month_node.month) + 1:
neon.create_or_merge_neontology_relationship(
calendar_sequence_relationships.MonthFollowsMonth(source=last_month_node, target=month_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {last_month_node.unique_id} to {month_node.unique_id}")
logger.info(f"Relationship created from {last_month_node.uuid_string} to {month_node.uuid_string}")
last_month_node = month_node
neon.create_or_merge_neontology_relationship(
@@ -146,25 +125,31 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {year_node.unique_id} to {month_node.unique_id}")
logger.info(f"Relationship created from {year_node.uuid_string} to {month_node.uuid_string}")
calendar_week_unique_id = f"{iso_year}_{iso_week}"
calendar_week_uuid_string = f"{iso_year}_{iso_week}"
week_key = f"{iso_year}-W{iso_week}"
if week_key not in created_weeks:
# Get the date of the first monday of the week
week_start_date = current_date - timedelta(days=current_date.weekday())
# Generate storage path for week node using Supabase Storage
if storage_tools:
week_dir_created, node_storage_path = storage_tools.create_calendar_week_storage_path(iso_year, iso_week)
else:
node_storage_path = ""
week_node = calendar_schemas.CalendarWeekNode(
unique_id=calendar_week_unique_id,
uuid_string=calendar_week_uuid_string,
start_date=week_start_date,
week_number=str(iso_week),
iso_week=f"{iso_year}-W{iso_week:02}",
tldraw_snapshot=""
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(week_node, database=db_name, operation='merge')
calendar_nodes['calendar_week_nodes'].append(week_node)
created_weeks[week_key] = week_node
logger.info(f"Week node created: {week_node.unique_id}")
logger.info(f"Week node created: {week_node.uuid_string}")
if last_week_node and ((last_week_node.iso_week.split('-')[0] == str(iso_year) and int(last_week_node.week_number) == int(iso_week) - 1) or
(last_week_node.iso_week.split('-')[0] != str(iso_year) and int(last_week_node.week_number) == 52 and int(iso_week) == 1)):
@@ -173,7 +158,7 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {last_week_node.unique_id} to {week_node.unique_id}")
logger.info(f"Relationship created from {last_week_node.uuid_string} to {week_node.uuid_string}")
last_week_node = week_node
neon.create_or_merge_neontology_relationship(
@@ -181,23 +166,32 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {year_node.unique_id} to {week_node.unique_id}")
logger.info(f"Relationship created from {year_node.uuid_string} to {week_node.uuid_string}")
# Day node management
calendar_day_unique_id = f"{year}_{month}_{day}"
calendar_day_uuid_string = f"{year}_{month}_{day}"
day_key = f"{year}-{month}-{day}"
# Generate storage path for day node using Supabase Storage
if storage_tools:
day_dir_created, node_storage_path = storage_tools.create_calendar_day_storage_path(year, month, day)
# Store day path for later use in time chunks
created_days[day_key] = {'node': None, 'path': node_storage_path}
else:
node_storage_path = ""
created_days[day_key] = {'node': None, 'path': None}
day_node = calendar_schemas.CalendarDayNode(
unique_id=calendar_day_unique_id,
uuid_string=calendar_day_uuid_string,
date=current_date,
day_of_week=current_date.strftime('%A'),
iso_day=f"{year}-{month:02}-{day:02}",
tldraw_snapshot=""
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(day_node, database=db_name, operation='merge')
calendar_nodes['calendar_day_nodes'].append(day_node)
created_days[day_key] = day_node
logger.info(f"Day node created: {day_node.unique_id}")
created_days[day_key]['node'] = day_node
logger.info(f"Day node created: {day_node.uuid_string}")
if last_day_node:
neon.create_or_merge_neontology_relationship(
@@ -205,7 +199,7 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {last_day_node.unique_id} to {day_node.unique_id}")
logger.info(f"Relationship created from {last_day_node.uuid_string} to {day_node.uuid_string}")
last_day_node = day_node
neon.create_or_merge_neontology_relationship(
@@ -213,13 +207,13 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {month_node.unique_id} to {day_node.unique_id}")
logger.info(f"Relationship created from {month_node.uuid_string} to {day_node.uuid_string}")
neon.create_or_merge_neontology_relationship(
calendar_relationships.WeekIncludesDay(source=week_node, target=day_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {week_node.unique_id} to {day_node.unique_id}")
logger.info(f"Relationship created from {week_node.uuid_string} to {day_node.uuid_string}")
current_date += timedelta(days=1)
if time_chunk_node_length:
@@ -228,25 +222,32 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
for day_node in calendar_nodes['calendar_day_nodes']:
total_time_chunks_in_day = (24 * 60) / time_chunk_interval
for i in range(total_time_chunks_in_day):
time_chunk_unique_id = f"{day_node.unique_id}_{i}"
time_chunk_uuid_string = f"{day_node.uuid_string}_{i}"
time_chunk_start_time = day_node.date.time() + timedelta(minutes=i * time_chunk_interval)
time_chunk_end_time = time_chunk_start_time + timedelta(minutes=time_chunk_interval)
# Generate storage path for time chunk node using Supabase Storage
if storage_tools:
chunk_id = f"{day_node.uuid_string}_{i:02d}"
chunk_dir_created, node_storage_path = storage_tools.create_calendar_time_chunk_storage_path(day_node.uuid_string, i)
else:
node_storage_path = ""
time_chunk_node = calendar_schemas.CalendarTimeChunkNode(
unique_id=time_chunk_unique_id,
uuid_string=time_chunk_uuid_string,
start_time=time_chunk_start_time,
end_time=time_chunk_end_time,
tldraw_snapshot=""
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(time_chunk_node, database=db_name, operation='merge')
calendar_nodes['calendar_time_chunk_nodes'].append(time_chunk_node)
logger.info(f"Time chunk node created: {time_chunk_node.unique_id}")
logger.info(f"Time chunk node created: {time_chunk_node.uuid_string}")
# Create a relationship between the time chunk node and the day node
neon.create_or_merge_neontology_relationship(
calendar_relationships.DayIncludesTimeChunk(source=day_node, target=time_chunk_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {day_node.unique_id} to {time_chunk_node.unique_id}")
logger.info(f"Relationship created from {day_node.uuid_string} to {time_chunk_node.uuid_string}")
# Create sequential relationship between the time chunk nodes
if i > 0:
neon.create_or_merge_neontology_relationship(
@@ -254,7 +255,7 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {calendar_nodes['calendar_time_chunk_nodes'][i-1].unique_id} to {time_chunk_node.unique_id}")
logger.info(f"Relationship created from {calendar_nodes['calendar_time_chunk_nodes'][i-1].uuid_string} to {time_chunk_node.uuid_string}")
logger.info(f'Created calendar: {calendar_nodes["calendar_node"].unique_id}')
logger.info(f'Calendar structure created successfully for {start_date} to {end_date}')
return calendar_nodes
modules/database/init/init_school.py
+24 -105
View File
@@ -2,115 +2,31 @@ import os
from modules.logger_tool import initialise_logger
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
from modules.database.schemas.nodes.schools.schools import SchoolNode
from modules.database.supabase.utils.client import SupabaseServiceRoleClient, CreateBucketOptions
import modules.database.init.init_school_timetable as init_school_timetable
import modules.database.tools.neontology_tools as neon
import modules.database.tools.supabase_storage_tools as storage_tools
def create_school_buckets(school_id: str, school_type: str, school_name: str, admin_access_token: str) -> dict:
"""Create storage buckets for a school
Args:
school_id: The unique identifier for the school
school_type: The type of school (e.g., 'development')
school_name: The display name of the school
admin_access_token: The admin access token for Supabase operations
Returns:
Dictionary containing results of bucket creation operations
"""
logger.info(f"Creating storage buckets for school {school_name} ({school_type}/{school_id})")
def create_school(db_name: str, uuid_string: str, name: str, website: str, school_type: str, is_public: bool = True, school_node: SchoolNode | None = None, dataframes=None):
if not name or not uuid_string or not website or not school_type:
logger.error("School name, uuid_string, website and school_type are required to create a school.")
raise ValueError("School name, uuid_string, website and school_type are required to create a school.")
storage_client = SupabaseServiceRoleClient.for_admin(admin_access_token)
base_path = f"cc.institutes.{school_type}.{school_id}"
buckets = [
# Main school buckets
{
"id": f"{base_path}.public",
"options": CreateBucketOptions(
name=f"{school_type.title()} School Files - {school_name} - Public Files",
public=True,
file_size_limit=50 * 1024 * 1024,
allowed_mime_types=[
'image/*', 'video/*', 'application/pdf',
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
'text/plain', 'text/csv', 'application/json'
]
)
},
{
"id": f"{base_path}.private",
"options": CreateBucketOptions(
name=f"{school_type.title()} School Files - {school_name} - Private Files",
public=False,
file_size_limit=50 * 1024 * 1024,
allowed_mime_types=[
'image/*', 'video/*', 'application/pdf',
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
'text/plain', 'text/csv', 'application/json'
]
)
},
# Curriculum buckets
{
"id": f"{base_path}.curriculum.public",
"options": CreateBucketOptions(
name=f"{school_type.title()} School Files - {school_name} - Curriculum Public Files",
public=True,
file_size_limit=50 * 1024 * 1024,
allowed_mime_types=[
'image/*', 'video/*', 'application/pdf',
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
'text/plain', 'text/csv', 'application/json'
]
)
},
{
"id": f"{base_path}.curriculum.private",
"options": CreateBucketOptions(
name=f"{school_type.title()} School Files - {school_name} - Curriculum Private Files",
public=False,
file_size_limit=50 * 1024 * 1024,
allowed_mime_types=[
'image/*', 'video/*', 'application/pdf',
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
'text/plain', 'text/csv', 'application/json'
]
)
}
]
results = {}
for bucket in buckets:
try:
result = storage_client.create_bucket(bucket["id"], bucket["options"])
results[bucket["id"]] = {
"status": "success",
"result": result
}
logger.info(f"Successfully created bucket {bucket['id']}")
except Exception as e:
logger.error(f"Error creating school bucket {bucket['id']}: {str(e)}")
results[bucket["id"]] = {
"status": "error",
"error": str(e)
}
return results
def create_school(db_name: str, id: str, name: str, website: str, school_type: str, is_public: bool = True, school_node: SchoolNode | None = None, dataframes=None):
if not name or not id or not website or not school_type:
logger.error("School name, id, website and school_type are required to create a school.")
raise ValueError("School name, id, website and school_type are required to create a school.")
logger.info(f"Initialising neo4j connection...")
logger.info(f"Initialising Neontology connection...")
neon.init_neontology_connection()
# Initialize storage tools for school
storage_tools_instance = storage_tools.SupabaseStorageTools(db_name, init_run_type="school")
# Generate the storage path for the school node
school_dir_created, node_storage_path = storage_tools_instance.create_school_storage_path(uuid_string)
logger.info(f"Generated school storage path: {node_storage_path}")
# Create School Node if not provided
if not school_node:
if is_public:
school_node = SchoolNode(
unique_id=f'School_{id}',
tldraw_snapshot="",
id=id,
uuid_string=uuid_string,
node_storage_path=node_storage_path,
name=name,
website=website,
school_type=school_type
@@ -118,9 +34,8 @@ def create_school(db_name: str, id: str, name: str, website: str, school_type: s
else:
# Create private school node with default values
school_node = SchoolNode(
unique_id=f'School_{id}',
tldraw_snapshot="",
id=id,
uuid_string=uuid_string,
node_storage_path=node_storage_path,
name=name,
website=website,
school_type=school_type,
@@ -133,6 +48,9 @@ def create_school(db_name: str, id: str, name: str, website: str, school_type: s
statutory_high_age=18,
school_capacity=1000
)
else:
# Update existing school node with the storage path
school_node.node_storage_path = node_storage_path
# First create/merge the school node in the main cc.institutes database
logger.info(f"Creating school node in main cc.institutes database...")
@@ -144,15 +62,16 @@ def create_school(db_name: str, id: str, name: str, website: str, school_type: s
school_nodes = {
'school_node': school_node,
'db_name': db_name
'db_name': db_name,
'storage_path': node_storage_path
}
if dataframes is not None:
logger.info(f"Creating school timetable for {name} with {len(dataframes)} dataframes...")
school_timetable_nodes = init_school_timetable.create_school_timetable(dataframes, db_name, school_node)
school_timetable_nodes = init_school_timetable.create_school_timetable(dataframes, db_name, school_node, storage_tools_instance)
school_nodes['school_timetable_nodes'] = school_timetable_nodes
else:
logger.warning(f"No dataframes provided for {name}, skipping school timetable...")
logger.info(f"School {name} created successfully...")
logger.info(f"School {name} created successfully with storage path: {node_storage_path}")
return school_nodes
modules/database/init/init_school_curriculum.py
+206 -73
View File
@@ -4,6 +4,7 @@ logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH
import pandas as pd
import modules.database.tools.neontology_tools as neon
import modules.database.tools.supabase_storage_tools as storage_tools
import modules.database.schemas.nodes.schools.schools as school_nodes
import modules.database.schemas.nodes.schools.curriculum as curriculum_nodes
import modules.database.schemas.nodes.schools.pastoral as pastoral_nodes
@@ -40,7 +41,7 @@ def sort_year_groups(df):
df['YearGroupNumeric'] = pd.to_numeric(df['YearGroup'], errors='coerce')
return df.sort_values(by='YearGroupNumeric')
def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_node: school_nodes.SchoolNode):
def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_node: school_nodes.SchoolNode, storage_tools=None):
logger.info(f"Initialising neo4j connection...")
neon.init_neontology_connection()
@@ -70,10 +71,20 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
last_key_stage_node = None
# Create Department Structure node
department_structure_node_unique_id = f"DepartmentStructure_{school_node.unique_id}"
department_structure_node_uuid_string = f"DepartmentStructure_{school_node.uuid_string}"
# For structure nodes, we can use a simple path or leave empty for consistency
# Since this is just an organizational node, we'll use a simple path
if storage_tools:
# Use the school's base department directory as the structure node path
dept_structure_path = f"cc.public.snapshots/DepartmentStructure/{school_node.uuid_string}"
node_storage_path = dept_structure_path
else:
node_storage_path = ""
department_structure_node = school_structures.DepartmentStructureNode(
unique_id=department_structure_node_unique_id,
tldraw_snapshot=""
uuid_string=department_structure_node_uuid_string,
node_storage_path=node_storage_path
)
# Create in school database only
neon.create_or_merge_neontology_node(department_structure_node, database=db_name, operation='merge')
@@ -86,10 +97,17 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
)
logger.info(f"Created department structure node and linked to school")
curriculum_structure_node_unique_id = f"CurriculumStructure_{school_node.unique_id}"
curriculum_structure_node_uuid_string = f"CurriculumStructure_{school_node.uuid_string}"
# Generate storage path for curriculum structure node
if storage_tools:
curriculum_dir_created, node_storage_path = storage_tools.create_curriculum_storage_path(curriculum_structure_node_uuid_string)
else:
node_storage_path = ""
curriculum_node = school_structures.CurriculumStructureNode(
unique_id=curriculum_structure_node_unique_id,
tldraw_snapshot=""
uuid_string=curriculum_structure_node_uuid_string,
node_storage_path=node_storage_path
)
# Create in school database only
neon.create_or_merge_neontology_node(curriculum_node, database=db_name, operation='merge')
@@ -102,10 +120,17 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
)
logger.info(f"Created curriculum node and relationship with school")
pastoral_structure_node_unique_id = f"PastoralStructure_{school_node.unique_id}"
pastoral_structure_node_uuid_string = f"PastoralStructure_{school_node.uuid_string}"
# Generate storage path for pastoral structure node
if storage_tools:
pastoral_dir_created, node_storage_path = storage_tools.create_pastoral_storage_path(pastoral_structure_node_uuid_string)
else:
node_storage_path = ""
pastoral_node = school_structures.PastoralStructureNode(
unique_id=pastoral_structure_node_unique_id,
tldraw_snapshot=""
uuid_string=pastoral_structure_node_uuid_string,
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(pastoral_node, database=db_name, operation='merge')
node_library['pastoral_node'] = pastoral_node
@@ -120,11 +145,20 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
unique_departments = keystagesyllabus_df['Department'].dropna().unique()
for department_name in unique_departments:
department_unique_id = f"Department_{school_node.unique_id}_{department_name.replace(' ', '_')}"
department_uuid_string = f"Department_{school_node.uuid_string}_{department_name.replace(' ', '_')}"
# Generate storage path for department node using the storage tools
if storage_tools:
# Create department directory under the school's department structure
dept_path = f"cc.public.snapshots/Department/{department_uuid_string}"
node_storage_path = dept_path
else:
node_storage_path = ""
department_node = school_nodes.DepartmentNode(
unique_id=department_unique_id,
uuid_string=department_uuid_string,
name=department_name,
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create department in school database only
neon.create_or_merge_neontology_node(department_node, database=db_name, operation='merge')
@@ -147,17 +181,25 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
# Process subjects from key stage syllabuses first (these have department info)
for _, subject_row in unique_subjects.iterrows():
subject_unique_id = f"Subject_{school_node.unique_id}_{subject_row['SubjectCode']}"
subject_uuid_string = f"Subject_{school_node.uuid_string}_{subject_row['SubjectCode']}"
department_node = node_library['department_nodes'].get(subject_row['Department'])
if not department_node:
logger.warning(f"No department found for subject {subject_row['Subject']} with code {subject_row['SubjectCode']}")
continue
# Generate storage path for subject node
if storage_tools:
# Create subject directory under the specific department
subject_path = f"cc.public.snapshots/Subject/{subject_uuid_string}"
node_storage_path = subject_path
else:
node_storage_path = ""
subject_node = curriculum_nodes.SubjectNode(
unique_id=subject_unique_id,
uuid_string=subject_uuid_string,
id=subject_row['SubjectCode'],
name=subject_row['Subject'],
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create subject in both databases
neon.create_or_merge_neontology_node(subject_node, database=db_name, operation='merge')
@@ -173,14 +215,23 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
# Process any additional subjects from year group syllabuses (these won't have department info)
for _, subject_row in additional_subjects.iterrows():
subject_unique_id = f"Subject_{school_node.unique_id}_{subject_row['SubjectCode']}"
subject_uuid_string = f"Subject_{school_node.uuid_string}_{subject_row['SubjectCode']}"
# Create in a special "Unassigned" department
unassigned_dept_name = "Unassigned Department"
if unassigned_dept_name not in node_library['department_nodes']:
# Generate storage path for unassigned department node
if filesystem:
# Create unassigned department directory under the school's department structure
unassigned_dept_path = os.path.join(filesystem.root_path, "departments", "Unassigned")
filesystem.create_directory(unassigned_dept_path)
node_storage_path = os.path.relpath(unassigned_dept_path, filesystem.base_path)
else:
node_storage_path = ""
department_node = school_nodes.DepartmentNode(
unique_id=f"Department_{school_node.unique_id}_Unassigned",
uuid_string=f"Department_{school_node.uuid_string}_Unassigned",
name=unassigned_dept_name,
tldraw_snapshot=""
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(department_node, database=db_name, operation='merge')
node_library['department_nodes'][unassigned_dept_name] = department_node
@@ -192,11 +243,20 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
)
logger.info(f"Created unassigned department node and linked to department structure")
# Generate storage path for subject node
if filesystem:
# Create subject directory under the unassigned department
subject_path = os.path.join(filesystem.root_path, "departments", "Unassigned", subject_row['Subject'].replace(' ', '_'))
filesystem.create_directory(subject_path)
node_storage_path = os.path.relpath(subject_path, filesystem.base_path)
else:
node_storage_path = ""
subject_node = curriculum_nodes.SubjectNode(
unique_id=subject_unique_id,
uuid_string=subject_uuid_string,
id=subject_row['SubjectCode'],
name=subject_row['Subject'],
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create subject in both databases
neon.create_or_merge_neontology_node(subject_node, database=db_name, operation='merge')
@@ -226,20 +286,29 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
# First create all key stage nodes and key stage syllabus nodes
for index, ks_row in keystagesyllabus_df.sort_values('KeyStage').iterrows():
key_stage = str(ks_row['KeyStage'])
logger.debug(f"Processing key stage syllabus row - Subject: {ks_row['Subject']}, Key Stage: {key_stage}")
subject = str(ks_row['Subject'])
syllabus_id = str(ks_row['ID'])
logger.debug(f"Processing key stage syllabus row - Subject: {subject}, Key Stage: {key_stage}, Syllabus ID: {syllabus_id}")
subject_node = node_library['subject_nodes'].get(ks_row['Subject'])
subject_node = node_library['subject_nodes'].get(subject)
if not subject_node:
logger.warning(f"No subject node found for subject {ks_row['Subject']}")
logger.warning(f"No subject node found for subject {subject}")
continue
if key_stage not in key_stage_nodes_created:
key_stage_node_unique_id = f"KeyStage_{curriculum_node.unique_id}_KStg{key_stage}"
key_stage_node_uuid_string = f"KeyStage_{curriculum_node.uuid_string}_KStg{key_stage}"
# Generate storage path for key stage node
if filesystem:
key_stage_dir_created, key_stage_path = filesystem.create_curriculum_key_stage_syllabus_directory(curriculum_path, key_stage, subject, syllabus_id)
node_storage_path = os.path.relpath(key_stage_path, filesystem.base_path)
else:
node_storage_path = ""
key_stage_node = curriculum_nodes.KeyStageNode(
unique_id=key_stage_node_unique_id,
uuid_string=key_stage_node_uuid_string,
name=f"Key Stage {key_stage}",
key_stage=str(key_stage),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create key stage node in both databases
neon.create_or_merge_neontology_node(key_stage_node, database=db_name, operation='merge')
@@ -252,7 +321,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.CurriculumStructureIncludesKeyStage(source=curriculum_node, target=key_stage_node),
database=db_name, operation='merge'
)
logger.info(f"Created key stage node {key_stage_node_unique_id} and relationship with curriculum structure")
logger.info(f"Created key stage node {key_stage_node_uuid_string} and relationship with curriculum structure")
# Create sequential relationship between key stages in both databases
if last_key_stage_node:
@@ -264,27 +333,34 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.KeyStageFollowsKeyStage(source=last_key_stage_node, target=key_stage_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between key stages {last_key_stage_node.unique_id} and {key_stage_node.unique_id}")
logger.info(f"Created sequential relationship between key stages {last_key_stage_node.uuid_string} and {key_stage_node.uuid_string}")
last_key_stage_node = key_stage_node
# Create key stage syllabus under the subject's curriculum directory
key_stage_syllabus_node_unique_id = f"KeyStageSyllabus_{curriculum_node.unique_id}_{ks_row['Title'].replace(' ', '')}"
key_stage_syllabus_node_uuid_string = f"KeyStageSyllabus_{curriculum_node.uuid_string}_{ks_row['Title'].replace(' ', '')}"
logger.debug(f"Creating key stage syllabus node for {ks_row['Subject']} KS{key_stage} with ID {ks_row['ID']}")
key_stage_syllabus_node_unique_id = f"KeyStageSyllabus_{curriculum_node.unique_id}_{ks_row['Title'].replace(' ', '')}"
key_stage_syllabus_node_uuid_string = f"KeyStageSyllabus_{curriculum_node.uuid_string}_{ks_row['Title'].replace(' ', '')}"
# Generate storage path for key stage syllabus node
if filesystem:
syllabus_dir_created, syllabus_path = filesystem.create_curriculum_key_stage_syllabus_directory(curriculum_path, key_stage, ks_row['Subject'], ks_row['ID'])
node_storage_path = os.path.relpath(syllabus_path, filesystem.base_path)
else:
node_storage_path = ""
key_stage_syllabus_node = curriculum_nodes.KeyStageSyllabusNode(
unique_id=key_stage_syllabus_node_unique_id,
uuid_string=key_stage_syllabus_node_uuid_string,
id=ks_row['ID'],
name=ks_row['Title'],
key_stage=str(ks_row['KeyStage']),
subject_name=ks_row['Subject'],
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create key stage syllabus node in both databases
neon.create_or_merge_neontology_node(key_stage_syllabus_node, database=db_name, operation='merge')
neon.create_or_merge_neontology_node(key_stage_syllabus_node, database=curriculum_db_name, operation='merge')
node_library['key_stage_syllabus_nodes'][ks_row['ID']] = key_stage_syllabus_node
logger.debug(f"Created key stage syllabus node {key_stage_syllabus_node_unique_id} for {ks_row['Subject']} KS{key_stage}")
logger.debug(f"Created key stage syllabus node {key_stage_syllabus_node_uuid_string} for {ks_row['Subject']} KS{key_stage}")
# Link key stage syllabus to its subject in both databases
if subject_node:
@@ -296,7 +372,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.SubjectHasKeyStageSyllabus(source=subject_node, target=key_stage_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between subject {subject_node.unique_id} and key stage syllabus {key_stage_syllabus_node.unique_id}")
logger.info(f"Created relationship between subject {subject_node.uuid_string} and key stage syllabus {key_stage_syllabus_node.uuid_string}")
# Link key stage syllabus to its key stage in both databases
key_stage_node = key_stage_nodes_created.get(key_stage)
@@ -309,7 +385,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.KeyStageIncludesKeyStageSyllabus(source=key_stage_node, target=key_stage_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between key stage {key_stage_node.unique_id} and key stage syllabus {key_stage_syllabus_node.unique_id}")
logger.info(f"Created relationship between key stage {key_stage_node.uuid_string} and key stage syllabus {key_stage_syllabus_node.uuid_string}")
# Create sequential relationship between key stage syllabuses in both databases
last_key_stage_syllabus_node = last_key_stage_syllabus_nodes.get(ks_row['Subject'])
@@ -322,7 +398,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.KeyStageSyllabusFollowsKeyStageSyllabus(source=last_key_stage_syllabus_node, target=key_stage_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between key stage syllabuses {last_key_stage_syllabus_node.unique_id} and {key_stage_syllabus_node.unique_id}")
logger.info(f"Created sequential relationship between key stage syllabuses {last_key_stage_syllabus_node.uuid_string} and {key_stage_syllabus_node.uuid_string}")
last_key_stage_syllabus_nodes[ks_row['Subject']] = key_stage_syllabus_node
# Now process year groups and their syllabuses
@@ -339,12 +415,19 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
if pd.notna(numeric_year_group):
numeric_year_group = int(numeric_year_group)
if numeric_year_group not in year_group_nodes_created:
year_group_node_unique_id = f"YearGroup_{school_node.unique_id}_YGrp{numeric_year_group}"
year_group_node_uuid_string = f"YearGroup_{school_node.uuid_string}_YGrp{numeric_year_group}"
# Generate storage path for year group node
if filesystem:
year_group_dir_created, year_group_path = filesystem.create_pastoral_year_group_directory(pastoral_path, numeric_year_group)
node_storage_path = os.path.relpath(year_group_path, filesystem.base_path)
else:
node_storage_path = ""
year_group_node = pastoral_nodes.YearGroupNode(
unique_id=year_group_node_unique_id,
uuid_string=year_group_node_uuid_string,
year_group=str(numeric_year_group),
name=f"Year {numeric_year_group}, {year_group}",
tldraw_snapshot=""
name=f"Year {numeric_year_group}",
node_storage_path=node_storage_path
)
# Create year group node in both databases but use same directory
neon.create_or_merge_neontology_node(year_group_node, database=db_name, operation='merge')
@@ -360,7 +443,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.YearGroupFollowsYearGroup(source=last_year_group_node, target=year_group_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between year groups {last_year_group_node.unique_id} and {year_group_node.unique_id} across key stages")
logger.info(f"Created sequential relationship between year groups {last_year_group_node.uuid_string} and {year_group_node.uuid_string} across key stages")
last_year_group_node = year_group_node
# Create relationship with Pastoral Structure in school database only
@@ -368,7 +451,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.PastoralStructureIncludesYearGroup(source=pastoral_node, target=year_group_node),
database=db_name, operation='merge'
)
logger.info(f"Created year group node {year_group_node_unique_id} and relationship with pastoral structure")
logger.info(f"Created year group node {year_group_node_uuid_string} and relationship with pastoral structure")
year_group_nodes_created[numeric_year_group] = year_group_node
node_library['year_group_nodes'][str(numeric_year_group)] = year_group_node
@@ -376,14 +459,21 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
# Create year group syllabus nodes in both databases
year_group_node = year_group_nodes_created.get(numeric_year_group)
if year_group_node:
year_group_syllabus_node_unique_id = f"YearGroupSyllabus_{school_node.unique_id}_{yg_row['ID']}"
year_group_syllabus_node_uuid_string = f"YearGroupSyllabus_{school_node.uuid_string}_{yg_row['ID']}"
# Generate storage path for year group syllabus node
if filesystem:
yg_syllabus_dir_created, yg_syllabus_path = filesystem.create_curriculum_year_group_syllabus_directory(curriculum_path, yg_row['Subject'], numeric_year_group, yg_row['ID'])
node_storage_path = os.path.relpath(yg_syllabus_path, filesystem.base_path)
else:
node_storage_path = ""
year_group_syllabus_node = pastoral_nodes.YearGroupSyllabusNode(
unique_id=year_group_syllabus_node_unique_id,
uuid_string=year_group_syllabus_node_uuid_string,
id=yg_row['ID'],
name=yg_row['Title'],
year_group=str(yg_row['YearGroup']),
subject_name=yg_row['Subject'],
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create year group syllabus node in both databases but use same directory
@@ -406,7 +496,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.YearGroupSyllabusFollowsYearGroupSyllabus(source=last_year_group_syllabus_node, target=year_group_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between year group syllabuses {last_year_group_syllabus_node.unique_id} and {year_group_syllabus_node.unique_id}")
logger.info(f"Created sequential relationship between year group syllabuses {last_year_group_syllabus_node.uuid_string} and {year_group_syllabus_node.uuid_string}")
last_year_group_syllabus_nodes[yg_row['Subject']] = year_group_syllabus_node
# Create relationships in both databases using MATCH to avoid cartesian products
@@ -421,7 +511,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.SubjectHasYearGroupSyllabus(source=subject_node, target=year_group_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between subject {subject_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
logger.info(f"Created relationship between subject {subject_node.uuid_string} and year group syllabus {year_group_syllabus_node_uuid_string}")
# Link to year group
neon.create_or_merge_neontology_relationship(
@@ -432,7 +522,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.YearGroupHasYearGroupSyllabus(source=year_group_node, target=year_group_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between year group {year_group_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
logger.info(f"Created relationship between year group {year_group_node.uuid_string} and year group syllabus {year_group_syllabus_node_uuid_string}")
# Link to key stage syllabus if it exists for the same subject
key_stage_syllabus_node = node_library['key_stage_syllabus_nodes'].get(ks_row['ID'])
@@ -445,7 +535,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.KeyStageSyllabusIncludesYearGroupSyllabus(source=key_stage_syllabus_node, target=year_group_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between key stage syllabus {key_stage_syllabus_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
logger.info(f"Created relationship between key stage syllabus {key_stage_syllabus_node.uuid_string} and year group syllabus {year_group_syllabus_node_uuid_string}")
# Process topics for this year group syllabus only if not already processed
topics_for_syllabus = topic_df[topic_df['SyllabusYearID'] == yg_row['ID']]
@@ -472,22 +562,29 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
if (syllabus_node.subject_name == topic_subject and
syllabus_node.key_stage == str(topic_key_stage)):
matching_syllabus_node = syllabus_node
logger.debug(f"Found matching syllabus node: {syllabus_node.unique_id}")
logger.debug(f"Found matching syllabus node: {syllabus_node.uuid_string}")
break
if not matching_syllabus_node:
logger.warning(f"No key stage syllabus node found for subject {topic_subject} and key stage {topic_key_stage}, skipping topic creation")
continue
topic_node_unique_id = f"Topic_{matching_syllabus_node.unique_id}_{topic_row['TopicID']}"
topic_node_uuid_string = f"Topic_{matching_syllabus_node.uuid_string}_{topic_row['TopicID']}"
# Generate storage path for topic node
if filesystem:
topic_dir_created, topic_path = filesystem.create_curriculum_topic_directory(yg_syllabus_path, topic_row['TopicID'])
node_storage_path = os.path.relpath(topic_path, filesystem.base_path)
else:
node_storage_path = ""
topic_node = curriculum_nodes.TopicNode(
unique_id=topic_node_unique_id,
uuid_string=topic_node_uuid_string,
id=topic_row['TopicID'],
name=topic_row.get('TopicTitle', default_topic_values['topic_title']),
total_number_of_lessons_for_topic=str(topic_row.get('TotalNumberOfLessonsForTopic', default_topic_values['total_number_of_lessons_for_topic'])),
type=topic_row.get('TopicType', default_topic_values['topic_type']),
assessment_type=topic_row.get('TopicAssessmentType', default_topic_values['topic_assessment_type']),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create topic node in curriculum database only
neon.create_or_merge_neontology_node(topic_node, database=curriculum_db_name, operation='merge')
@@ -502,7 +599,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.YearGroupSyllabusIncludesTopic(source=year_group_syllabus_node, target=topic_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationships between topic {topic_node_unique_id} and key stage syllabus {matching_syllabus_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
logger.info(f"Created relationships between topic {topic_node_uuid_string} and key stage syllabus {matching_syllabus_node.uuid_string} and year group syllabus {year_group_syllabus_node_uuid_string}")
# Process lessons for this topic only if not already processed
lessons_for_topic = lesson_df[
@@ -518,8 +615,15 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
continue
lessons_processed.add(lesson_row['LessonID'])
# Generate storage path for lesson node
if filesystem:
lesson_dir_created, lesson_path = filesystem.create_curriculum_lesson_directory(topic_path, lesson_row['LessonID'])
node_storage_path = os.path.relpath(lesson_path, filesystem.base_path)
else:
node_storage_path = ""
lesson_node = curriculum_nodes.TopicLessonNode(
unique_id=f"TopicLesson_{topic_node_unique_id}_{lesson_row['LessonID']}",
uuid_string=f"TopicLesson_{topic_node_uuid_string}_{lesson_row['LessonID']}",
id=lesson_row['LessonID'],
name=lesson_row.get('LessonTitle', default_topic_lesson_values['topic_lesson_title']),
type=lesson_row.get('LessonType', default_topic_lesson_values['topic_lesson_type']),
@@ -527,7 +631,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
suggested_activities=str(lesson_row.get('SuggestedActivities', default_topic_lesson_values['topic_lesson_suggested_activities'])),
skills_learned=str(lesson_row.get('SkillsLearned', default_topic_lesson_values['topic_lesson_skills_learned'])),
weblinks=str(lesson_row.get('WebLinks', default_topic_lesson_values['topic_lesson_weblinks'])),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create lesson node in curriculum database only
neon.create_or_merge_neontology_node(lesson_node, database=curriculum_db_name, operation='merge')
@@ -538,7 +642,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.TopicIncludesTopicLesson(source=topic_node, target=lesson_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created lesson node {lesson_node.unique_id} and relationship with topic {topic_node.unique_id}")
logger.info(f"Created lesson node {lesson_node.uuid_string} and relationship with topic {topic_node.uuid_string}")
# Create sequential relationships between lessons
if lesson_row['Lesson'].isdigit() and previous_lesson_node:
@@ -546,7 +650,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.TopicLessonFollowsTopicLesson(source=previous_lesson_node, target=lesson_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.unique_id} and {lesson_node.unique_id}")
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.uuid_string} and {lesson_node.uuid_string}")
previous_lesson_node = lesson_node
# Process learning statements for this lesson only if not already processed
@@ -558,12 +662,19 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
if statement_row['StatementID'] in statements_processed:
continue
statements_processed.add(statement_row['StatementID'])
# Generate storage path for learning statement node
if filesystem:
statement_dir_created, statement_path = filesystem.create_curriculum_learning_statement_directory(lesson_path, statement_row['StatementID'])
node_storage_path = os.path.relpath(statement_path, filesystem.base_path)
else:
node_storage_path = ""
statement_node = curriculum_nodes.LearningStatementNode(
unique_id=f"LearningStatement_{lesson_node.unique_id}_{statement_row['StatementID']}",
uuid_string=f"LearningStatement_{lesson_node.uuid_string}_{statement_row['StatementID']}",
id=statement_row['StatementID'],
name=statement_row.get('LearningStatement', default_learning_statement_values['lesson_learning_statement']),
type=statement_row.get('StatementType', default_learning_statement_values['lesson_learning_statement_type']),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create statement node in curriculum database only
neon.create_or_merge_neontology_node(statement_node, database=curriculum_db_name, operation='merge')
@@ -574,7 +685,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.LessonIncludesLearningStatement(source=lesson_node, target=statement_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created learning statement node {statement_node.unique_id} and relationship with lesson {lesson_node.unique_id}")
logger.info(f"Created learning statement node {statement_node.uuid_string} and relationship with lesson {lesson_node.uuid_string}")
else:
logger.warning(f"No year group node found for year group {year_group}, skipping syllabus creation")
@@ -601,15 +712,23 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
logger.warning(f"No key stage syllabus node found for subject {topic_subject} and key stage {topic_key_stage}, skipping topic creation")
continue
topic_node_unique_id = f"Topic_{matching_syllabus_node.unique_id}_{topic_row['TopicID']}"
topic_node_uuid_string = f"Topic_{matching_syllabus_node.uuid_string}_{topic_row['TopicID']}"
# Generate storage path for topic node
if filesystem:
syllabus_path = os.path.join(curriculum_path, "subjects", topic_subject, "key_stage_syllabuses", f"KS{topic_key_stage}", f"KS{topic_key_stage}.{topic_subject}")
topic_dir_created, keystage_topic_path = filesystem.create_curriculum_keystage_topic_directory(syllabus_path, topic_row['TopicID'])
node_storage_path = os.path.relpath(keystage_topic_path, filesystem.base_path)
else:
node_storage_path = ""
topic_node = curriculum_nodes.TopicNode(
unique_id=topic_node_unique_id,
uuid_string=topic_node_uuid_string,
id=topic_row['TopicID'],
name=topic_row.get('TopicTitle', default_topic_values['topic_title']),
total_number_of_lessons_for_topic=str(topic_row.get('TotalNumberOfLessonsForTopic', default_topic_values['total_number_of_lessons_for_topic'])),
type=topic_row.get('TopicType', default_topic_values['topic_type']),
assessment_type=topic_row.get('TopicAssessmentType', default_topic_values['topic_assessment_type']),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create topic node in curriculum database only
neon.create_or_merge_neontology_node(topic_node, database=curriculum_db_name, operation='merge')
@@ -621,7 +740,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.KeyStageSyllabusIncludesTopic(source=matching_syllabus_node, target=topic_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between topic {topic_node_unique_id} and key stage syllabus {matching_syllabus_node.unique_id}")
logger.info(f"Created relationship between topic {topic_node_uuid_string} and key stage syllabus {matching_syllabus_node.uuid_string}")
# Process lessons for this topic
lessons_for_topic = lesson_df[
@@ -636,8 +755,15 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
if lesson_row['LessonID'] in lessons_processed:
continue
lessons_processed.add(lesson_row['LessonID'])
# Generate storage path for lesson node
if filesystem:
lesson_dir_created, lesson_path = filesystem.create_curriculum_lesson_directory(topic_path, lesson_row['LessonID'])
node_storage_path = os.path.relpath(lesson_path, filesystem.base_path)
else:
node_storage_path = ""
lesson_node = curriculum_nodes.TopicLessonNode(
unique_id=f"TopicLesson_{topic_node_unique_id}_{lesson_row['LessonID']}",
uuid_string=f"TopicLesson_{topic_node_uuid_string}_{lesson_row['LessonID']}",
id=lesson_row['LessonID'],
name=lesson_row.get('LessonTitle', default_topic_lesson_values['topic_lesson_title']),
type=lesson_row.get('LessonType', default_topic_lesson_values['topic_lesson_type']),
@@ -645,7 +771,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
suggested_activities=str(lesson_row.get('SuggestedActivities', default_topic_lesson_values['topic_lesson_suggested_activities'])),
skills_learned=str(lesson_row.get('SkillsLearned', default_topic_lesson_values['topic_lesson_skills_learned'])),
weblinks=str(lesson_row.get('WebLinks', default_topic_lesson_values['topic_lesson_weblinks'])),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create lesson node in curriculum database only
neon.create_or_merge_neontology_node(lesson_node, database=curriculum_db_name, operation='merge')
@@ -656,7 +782,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.TopicIncludesTopicLesson(source=topic_node, target=lesson_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created lesson node {lesson_node.unique_id} and relationship with topic {topic_node.unique_id}")
logger.info(f"Created lesson node {lesson_node.uuid_string} and relationship with topic {topic_node.uuid_string}")
# Create sequential relationships between lessons
if lesson_row['Lesson'].isdigit() and previous_lesson_node:
@@ -664,7 +790,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.TopicLessonFollowsTopicLesson(source=previous_lesson_node, target=lesson_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.unique_id} and {lesson_node.unique_id}")
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.uuid_string} and {lesson_node.uuid_string}")
previous_lesson_node = lesson_node
# Process learning statements for this lesson
@@ -676,12 +802,19 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
if statement_row['StatementID'] in statements_processed:
continue
statements_processed.add(statement_row['StatementID'])
# Generate storage path for learning statement node
if filesystem:
statement_dir_created, statement_path = filesystem.create_curriculum_learning_statement_directory(lesson_path, statement_row['StatementID'])
node_storage_path = os.path.relpath(statement_path, filesystem.base_path)
else:
node_storage_path = ""
statement_node = curriculum_nodes.LearningStatementNode(
unique_id=f"LearningStatement_{lesson_node.unique_id}_{statement_row['StatementID']}",
uuid_string=f"LearningStatement_{lesson_node.uuid_string}_{statement_row['StatementID']}",
id=statement_row['StatementID'],
name=statement_row.get('LearningStatement', default_learning_statement_values['lesson_learning_statement']),
type=statement_row.get('StatementType', default_learning_statement_values['lesson_learning_statement_type']),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create statement node in curriculum database only
neon.create_or_merge_neontology_node(statement_node, database=curriculum_db_name, operation='merge')
@@ -692,6 +825,6 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.LessonIncludesLearningStatement(source=lesson_node, target=statement_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created learning statement node {statement_node.unique_id} and relationship with lesson {lesson_node.unique_id}")
logger.info(f"Created learning statement node {statement_node.uuid_string} and relationship with lesson {lesson_node.uuid_string}")
return node_library
modules/database/init/init_school_timetable.py
+101 -54
View File
@@ -11,7 +11,7 @@ import modules.database.schemas.relationships.calendar_timetable_rels as cal_tt_
import modules.database.init.init_calendar as init_calendar
import modules.database.tools.neontology_tools as neon
def create_school_timetable(dataframes, db_name, school_node=None):
def create_school_timetable(dataframes, db_name, school_node=None, filesystem=None):
logger.info(f"Creating school timetable for {db_name}")
if dataframes is None:
raise ValueError("Data is required to create the calendar and timetable.")
@@ -22,10 +22,10 @@ def create_school_timetable(dataframes, db_name, school_node=None):
school_df = dataframes['school']
if school_node is None:
logger.info(f"School node is None, using school data from dataframe")
school_unique_id = school_df[school_df['Identifier'] == 'SchoolID']['Data'].iloc[0]
school_uuid_string = school_df[school_df['Identifier'] == 'SchoolID']['Data'].iloc[0]
else:
logger.info(f"School node is not None, using school data from school node: {school_node}")
school_unique_id = school_node.unique_id
school_uuid_string = school_node.uuid_string
terms_df = dataframes['terms']
weeks_df = dataframes['weeks']
@@ -54,20 +54,30 @@ def create_school_timetable(dataframes, db_name, school_node=None):
}
# Create AcademicTimetable Node
school_timetable_unique_id = f"{school_unique_id}_{school_year_start_date.year}_{school_year_end_date.year}"
school_timetable_uuid_string = f"{school_uuid_string}_{school_year_start_date.year}_{school_year_end_date.year}"
# Generate storage path for timetable node
if filesystem:
timetable_dir_created, timetable_path = filesystem.create_school_timetable_directory()
node_storage_path = os.path.relpath(timetable_path, filesystem.base_path)
logger.info(f"Generated timetable node_storage_path: {node_storage_path}")
else:
node_storage_path = ""
logger.warning("No filesystem provided, using empty storage path")
school_timetable_node = timetable.SchoolTimetableNode(
school_timetable_id=school_timetable_unique_id,
unique_id=school_timetable_unique_id,
school_timetable_id=school_timetable_uuid_string,
uuid_string=school_timetable_uuid_string,
start_date=school_year_start_date,
end_date=school_year_end_date,
tldraw_snapshot=""
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(school_timetable_node, database=db_name, operation='merge')
timetable_nodes['timetable_node'] = school_timetable_node
if school_node:
logger.info(f"Creating calendar for {school_unique_id} from Neo4j SchoolNode: {school_node.unique_id}")
calendar_nodes = init_calendar.create_calendar(db_name, school_year_start_date, school_year_end_date, attach_to_calendar_node=True, owner_node=school_node)
logger.info(f"Creating calendar for {school_uuid_string} from Neo4j SchoolNode: {school_node.uuid_string}")
calendar_nodes = init_calendar.create_calendar(db_name, school_year_start_date, school_year_end_date, attach_to_calendar_node=True, owner_node=school_node, filesystem=filesystem)
# Link the school node to the timetable node
neon.create_or_merge_neontology_relationship(
entity_tt_rels.SchoolHasTimetable(source=school_node, target=school_timetable_node),
@@ -75,26 +85,34 @@ def create_school_timetable(dataframes, db_name, school_node=None):
)
timetable_nodes['calendar_nodes'] = calendar_nodes
else:
logger.info(f"Creating calendar for {school_unique_id} from dataframe SchoolID: {school_unique_id}")
calendar_nodes = init_calendar.create_calendar(db_name, school_year_start_date, school_year_end_date, attach_to_calendar_node=False, owner_node=None)
logger.info(f"Creating calendar for {school_uuid_string} from dataframe SchoolID: {school_uuid_string}")
calendar_nodes = init_calendar.create_calendar(db_name, school_year_start_date, school_year_end_date, attach_to_calendar_node=False, owner_node=None, filesystem=filesystem)
# Create AcademicYear nodes for each year within the range
for year in range(school_year_start_date.year, school_year_end_date.year + 1):
year_str = str(year)
academic_year_unique_id = f"{school_timetable_unique_id}_{year}"
academic_year_uuid_string = f"{school_timetable_uuid_string}_{year}"
# Generate storage path for academic year node
if filesystem:
year_dir_created, year_path = filesystem.create_school_timetable_year_directory(timetable_path, year)
node_storage_path = os.path.relpath(year_path, filesystem.base_path)
else:
node_storage_path = ""
academic_year_node = timetable.AcademicYearNode(
unique_id=academic_year_unique_id,
uuid_string=academic_year_uuid_string,
year=year_str,
tldraw_snapshot=""
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(academic_year_node, database=db_name, operation='merge')
timetable_nodes['academic_year_nodes'].append(academic_year_node)
logger.info(f'Created academic year node: {academic_year_node.unique_id}')
logger.info(f'Created academic year node: {academic_year_node.uuid_string}')
neon.create_or_merge_neontology_relationship(
tt_rels.AcademicTimetableHasAcademicYear(source=school_timetable_node, target=academic_year_node),
database=db_name, operation='merge'
)
logger.info(f"Created school timetable relationship from {school_timetable_node.unique_id} to {academic_year_node.unique_id}")
logger.info(f"Created school timetable relationship from {school_timetable_node.uuid_string} to {academic_year_node.uuid_string}")
# Link the academic year with the corresponding calendar year node
for year_node in calendar_nodes['calendar_year_nodes']:
@@ -103,7 +121,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
cal_tt_rels.AcademicYearIsCalendarYear(source=academic_year_node, target=year_node),
database=db_name, operation='merge'
)
logger.info(f"Created school timetable relationship from {academic_year_node.unique_id} to {year_node.unique_id}")
logger.info(f"Created school timetable relationship from {academic_year_node.uuid_string} to {year_node.uuid_string}")
break
# Create Term and TermBreak nodes linked to AcademicYear
@@ -121,29 +139,39 @@ def create_school_timetable(dataframes, db_name, school_node=None):
if isinstance(term_end_date, pd.Timestamp):
term_end_date = term_end_date.strftime('%Y-%m-%d')
# Generate storage path for term node
if filesystem:
if term_row['TermType'] == 'Term':
term_dir_created, term_path = filesystem.create_school_timetable_academic_term_directory(timetable_path, term_name, academic_term_number)
else:
term_dir_created, term_path = filesystem.create_school_timetable_academic_term_break_directory(timetable_path, term_name)
node_storage_path = os.path.relpath(term_path, filesystem.base_path)
else:
node_storage_path = ""
if term_row['TermType'] == 'Term':
term_node_unique_id = f"{school_timetable_unique_id}_{academic_term_number}_{term_name_no_spaces}"
term_node_uuid_string = f"{school_timetable_uuid_string}_{academic_term_number}_{term_name_no_spaces}"
academic_term_number_str = str(academic_term_number)
term_node = term_node_class(
unique_id=term_node_unique_id,
uuid_string=term_node_uuid_string,
term_name=term_name,
term_number=academic_term_number_str,
start_date=datetime.strptime(term_start_date, '%Y-%m-%d'),
end_date=datetime.strptime(term_end_date, '%Y-%m-%d'),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
academic_term_number += 1
else:
term_break_node_unique_id = f"{school_timetable_unique_id}_{term_name_no_spaces}"
term_break_node_uuid_string = f"{school_timetable_uuid_string}_{term_name_no_spaces}"
term_node = term_node_class(
unique_id=term_break_node_unique_id,
uuid_string=term_break_node_uuid_string,
term_break_name=term_name,
start_date=datetime.strptime(term_start_date, '%Y-%m-%d'),
end_date=datetime.strptime(term_end_date, '%Y-%m-%d'),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(term_node, database=db_name, operation='merge')
logger.info(f'Created academic term break node: {term_node.unique_id}')
logger.info(f'Created academic term break node: {term_node.uuid_string}')
timetable_nodes['academic_term_nodes'].append(term_node)
term_number += 1 # We don't use this but we could
@@ -158,7 +186,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
relationship_class(source=academic_year_node, target=term_node),
database=db_name, operation='merge'
)
logger.info(f"Created school timetable relationship from {academic_year_node.unique_id} to {term_node.unique_id}")
logger.info(f"Created school timetable relationship from {academic_year_node.uuid_string} to {term_node.uuid_string}")
# Create Week nodes
academic_week_number = 1
@@ -168,28 +196,35 @@ def create_school_timetable(dataframes, db_name, school_node=None):
if isinstance(week_start_date, pd.Timestamp):
week_start_date = week_start_date.strftime('%Y-%m-%d')
week_node_unique_id = f"{school_timetable_unique_id}_{week_row['WeekNumber']}_{week_row['WeekType']}Week"
week_node_uuid_string = f"{school_timetable_uuid_string}_{week_row['WeekNumber']}_{week_row['WeekType']}Week"
# Generate storage path for week node
if filesystem:
week_dir_created, week_path = filesystem.create_school_timetable_academic_week_directory(timetable_path, week_row['WeekNumber'])
node_storage_path = os.path.relpath(week_path, filesystem.base_path)
else:
node_storage_path = ""
if week_row['WeekType'] == 'Holiday':
week_node = week_node_class(
unique_id=week_node_unique_id,
uuid_string=week_node_uuid_string,
start_date=datetime.strptime(week_start_date, '%Y-%m-%d'),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
else:
academic_week_number_str = str(academic_week_number)
week_type = week_row['WeekType']
week_node = week_node_class(
unique_id=week_node_unique_id,
uuid_string=week_node_uuid_string,
academic_week_number=academic_week_number_str,
start_date=datetime.strptime(week_start_date, '%Y-%m-%d'),
week_type=week_type,
tldraw_snapshot=""
node_storage_path=node_storage_path
)
academic_week_number += 1
neon.create_or_merge_neontology_node(week_node, database=db_name, operation='merge')
timetable_nodes['academic_week_nodes'].append(week_node)
logger.info(f"Created week node: {week_node.unique_id}")
logger.info(f"Created week node: {week_node.uuid_string}")
for calendar_node in calendar_nodes['calendar_week_nodes']:
if calendar_node.start_date == week_node.start_date:
if isinstance(week_node, timetable.AcademicWeekNode):
@@ -197,13 +232,13 @@ def create_school_timetable(dataframes, db_name, school_node=None):
cal_tt_rels.AcademicWeekIsCalendarWeek(source=week_node, target=calendar_node),
database=db_name, operation='merge'
)
logger.info(f"Created school timetable relationship from {calendar_node.unique_id} to {week_node.unique_id}")
logger.info(f"Created school timetable relationship from {calendar_node.uuid_string} to {week_node.uuid_string}")
elif isinstance(week_node, timetable.HolidayWeekNode):
neon.create_or_merge_neontology_relationship(
cal_tt_rels.HolidayWeekIsCalendarWeek(source=week_node, target=calendar_node),
database=db_name, operation='merge'
)
logger.info(f"Created school timetable relationship from {calendar_node.unique_id} to {week_node.unique_id}")
logger.info(f"Created school timetable relationship from {calendar_node.uuid_string} to {week_node.uuid_string}")
break
# Link week node to the correct academic term
@@ -214,7 +249,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
relationship_class(source=term_node, target=week_node),
database=db_name, operation='merge'
)
logger.info(f"Created school timetable relationship from {term_node.unique_id} to {week_node.unique_id}")
logger.info(f"Created school timetable relationship from {term_node.uuid_string} to {week_node.uuid_string}")
break
# Link week node to the correct academic year
@@ -225,7 +260,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
relationship_class(source=academic_year_node, target=week_node),
database=db_name, operation='merge'
)
logger.info(f"Created school timetable relationship from {academic_year_node.unique_id} to {week_node.unique_id}")
logger.info(f"Created school timetable relationship from {academic_year_node.uuid_string} to {week_node.uuid_string}")
break
# Create Day nodes
@@ -243,18 +278,24 @@ def create_school_timetable(dataframes, db_name, school_node=None):
'StaffDay': timetable.StaffDayNode
}[day_row['DayType']]
# Generate storage path for day node
if filesystem:
day_dir_created, day_path = filesystem.create_school_timetable_academic_day_directory(timetable_path, academic_day_number)
node_storage_path = os.path.relpath(day_path, filesystem.base_path)
else:
node_storage_path = ""
# Format the unique ID as {day_node_class.__name__}Day
day_node_data = {
'unique_id': f"{school_timetable_unique_id}_{day_number}_{day_node_class.__name__}Day",
'uuid_string': f"{school_timetable_uuid_string}_{day_number}_{day_node_class.__name__}Day",
'date': datetime.strptime(date_str, '%Y-%m-%d'),
'day_of_week': datetime.strptime(date_str, '%Y-%m-%d').strftime('%A'),
'tldraw_snapshot': ""
'node_storage_path': node_storage_path
}
if day_row['DayType'] == 'Academic':
day_node_data['academic_day'] = str(academic_day_number)
day_node_data['day_type'] = day_row['WeekType']
day_node_data['tldraw_snapshot'] = ""
day_node = day_node_class(**day_node_data)
@@ -262,7 +303,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
if calendar_node.date == day_node.date:
neon.create_or_merge_neontology_node(day_node, database=db_name, operation='merge')
timetable_nodes['academic_day_nodes'].append(day_node)
logger.info(f"Created day node: {day_node.unique_id}")
logger.info(f"Created day node: {day_node.uuid_string}")
if isinstance(day_node, timetable.AcademicDayNode):
relationship_class = cal_tt_rels.AcademicDayIsCalendarDay
@@ -277,7 +318,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
relationship_class(source=day_node, target=calendar_node),
database=db_name, operation='merge'
)
logger.info(f'Created relationship from {calendar_node.unique_id} to {day_node.unique_id}')
logger.info(f'Created relationship from {calendar_node.uuid_string} to {day_node.uuid_string}')
break
# Link day node to the correct academic week
@@ -300,7 +341,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
relationship_class(source=academic_week_node, target=day_node),
database=db_name, operation='merge'
)
logger.info(f"Created relationship from {academic_week_node.unique_id} to {day_node.unique_id}")
logger.info(f"Created relationship from {academic_week_node.uuid_string} to {day_node.uuid_string}")
break
# Link day node to the correct academic term
@@ -323,12 +364,12 @@ def create_school_timetable(dataframes, db_name, school_node=None):
relationship_class(source=term_node, target=day_node),
database=db_name, operation='merge'
)
logger.info(f"Created relationship from {term_node.unique_id} to {day_node.unique_id}")
logger.info(f"Created relationship from {term_node.uuid_string} to {day_node.uuid_string}")
break
# Create Period nodes for each academic day
if day_row['DayType'] == 'Academic':
logger.info(f"Creating periods for {day_node.unique_id}")
logger.info(f"Creating periods for {day_node.uuid_string}")
period_of_day = 1
academic_or_registration_period_of_day = 1
for _, period_row in periods_df.iterrows():
@@ -340,15 +381,22 @@ def create_school_timetable(dataframes, db_name, school_node=None):
}[period_row['PeriodType']]
logger.info(f"Creating period node for {period_node_class.__name__} Period: {period_of_day}")
period_node_unique_id = f"{school_timetable_unique_id}_{academic_day_number}_{period_of_day}_{period_node_class.__name__}Period"
logger.debug(f"Period node unique id: {period_node_unique_id}")
period_node_uuid_string = f"{school_timetable_uuid_string}_{academic_day_number}_{period_of_day}_{period_node_class.__name__}Period"
logger.debug(f"Period node unique id: {period_node_uuid_string}")
# Generate storage path for period node
if filesystem:
period_dir_created, period_path = filesystem.create_school_timetable_period_directory(timetable_path, academic_day_number, period_row['PeriodCode'])
node_storage_path = os.path.relpath(period_path, filesystem.base_path)
else:
node_storage_path = ""
period_node_data = {
'unique_id': period_node_unique_id,
'uuid_string': period_node_uuid_string,
'name': period_row['PeriodName'],
'date': day_node.date,
'start_time': datetime.combine(day_node.date, period_row['StartTime']),
'end_time': datetime.combine(day_node.date, period_row['EndTime']),
'tldraw_snapshot': ""
'node_storage_path': node_storage_path
}
logger.debug(f"Period node data: {period_node_data}")
if period_row['PeriodType'] in ['Academic', 'Registration']:
@@ -357,14 +405,13 @@ def create_school_timetable(dataframes, db_name, school_node=None):
period_code = period_row['PeriodCode']
period_code_formatted = f"{week_type}{day_name_short}{period_code}"
period_node_data['period_code'] = period_code_formatted
period_node_data['tldraw_snapshot'] = ""
academic_or_registration_period_of_day += 1
period_node = period_node_class(**period_node_data)
neon.create_or_merge_neontology_node(period_node, database=db_name, operation='merge')
timetable_nodes['academic_period_nodes'].append(period_node)
logger.info(f'Created period node: {period_node.unique_id}')
logger.info(f'Created period node: {period_node.uuid_string}')
relationship_class = {
'Academic': tt_rels.AcademicDayHasAcademicPeriod,
@@ -377,7 +424,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
relationship_class(source=day_node, target=period_node),
database=db_name, operation='merge'
)
logger.info(f"Created relationship from {day_node.unique_id} to {period_node.unique_id}")
logger.info(f"Created relationship from {day_node.uuid_string} to {period_node.uuid_string}")
period_of_day += 1 # We don't use this but we could
academic_day_number += 1 # This is a bit of a hack but it works to keep the directories aligned (reorganise)
day_number += 1 # We don't use this but we could
@@ -392,7 +439,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
relationship_class = relationship_map.get(node_type_pair)
if relationship_class:
# Avoid self-referential relationships
if source_node.unique_id != target_node.unique_id:
if source_node.uuid_string != target_node.uuid_string:
neon.create_or_merge_neontology_relationship(
relationship_class(
source=source_node,
@@ -400,9 +447,9 @@ def create_school_timetable(dataframes, db_name, school_node=None):
),
database=db_name, operation='merge'
)
logger.info(f"Created relationship from {source_node.unique_id} to {target_node.unique_id}")
logger.info(f"Created relationship from {source_node.uuid_string} to {target_node.uuid_string}")
else:
logger.warning(f"Skipped self-referential relationship for node {source_node.unique_id}")
logger.warning(f"Skipped self-referential relationship for node {source_node.uuid_string}")
# Relationship maps for different node types
academic_year_relationship_map = {
@@ -474,7 +521,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
# Call the function with the created timetable nodes
create_school_timetable_node_sequence_rels(timetable_nodes)
logger.info(f'Created timetable: {timetable_nodes["timetable_node"].unique_id}')
logger.info(f'Created timetable: {timetable_nodes["timetable_node"].uuid_string}')
# Log the directory structure after creation
# root_timetable_directory = fs_handler.root_path # Access the root directory of the filesystem handler
modules/database/init/init_user.py
+297 -92
View File
@@ -9,6 +9,7 @@ import modules.database.schemas.nodes.workers.workers as worker_nodes
import modules.database.init.init_calendar as init_calendar
import modules.database.schemas.relationships.entity_relationships as entity_relationships
import modules.database.tools.neontology_tools as neon
import modules.database.tools.supabase_storage_tools as storage_tools
from modules.logger_tool import initialise_logger
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
@@ -21,11 +22,27 @@ def create_and_check_db(db_name):
return database_status
class UserCreator(ABC):
def __init__(self, user_id, cc_users_db_name, user_type, worker_type, user_email, worker_email, cc_username, user_name, worker_name, calendar_start_date, calendar_end_date):
cc_schools_db_name = "cc.institutes" # Fix the TODO
def __init__(
self,
user_id,
cc_users_db_name,
user_type,
worker_type,
user_email,
worker_email,
cc_username,
user_name,
worker_name,
calendar_start_date,
calendar_end_date,
storage_tools=None,
user_db_name: Optional[str] = None,
worker_db_name: Optional[str] = None,
cc_schools_db_name: str = "cc.institutes",
):
self.cc_users_db_name = cc_users_db_name
self.user_db_name = f"{cc_users_db_name}.{user_type}.{cc_username}"
self.worker_db_name = f"{cc_schools_db_name}.{user_type}.{cc_username}"
self.user_db_name = user_db_name or f"{cc_users_db_name}.{user_type}.{cc_username}"
self.worker_db_name = worker_db_name or f"{cc_schools_db_name}.{worker_type}.{cc_username}"
self.user_type = user_type
self.worker_type = worker_type
self.cc_username = cc_username
@@ -34,6 +51,7 @@ class UserCreator(ABC):
self.user_name = user_name
self.worker_name = worker_name
self.user_id = user_id
self.storage_tools = storage_tools # Store the storage tools instance
self.user_nodes: Dict[str, Optional[Any]] = {
'default_user_node': None,
'private_user_node': None,
@@ -48,6 +66,25 @@ class UserCreator(ABC):
self.calendar_start_date = datetime.now().date()
self.calendar_end_date = (datetime.now() + timedelta(days=5)).date()
def _derive_user_storage_path(self) -> str:
return os.path.join(
"users",
self.user_id,
"databases",
self.user_db_name,
self.user_id,
).replace('\\', '/')
def _derive_internal_worker_path(self, worker_kind: str) -> str:
return os.path.join(
"users",
self.user_id,
"databases",
self.user_db_name,
worker_kind,
self.user_id,
).replace('\\', '/')
@abstractmethod
def create_user(self):
pass
@@ -66,9 +103,17 @@ class UserCreator(ABC):
# Ensure Neontology is initialized
neon.init_neontology_connection()
# Generate storage path for user node using Supabase Storage
if self.storage_tools:
user_dir_created, node_storage_path = self.storage_tools.create_user_storage_path(self.user_id)
self.user_path = node_storage_path # Store for later use
else:
node_storage_path = self._derive_user_storage_path()
self.user_path = None
user_node = user_nodes.UserNode(
unique_id=f"{self.user_id}",
tldraw_snapshot="",
uuid_string=f"{self.user_id}",
node_storage_path=node_storage_path,
cc_username=f"{self.cc_username}",
user_email=f"{self.user_email}",
user_name=f"{self.user_name}",
@@ -76,81 +121,79 @@ class UserCreator(ABC):
user_type=f"{self.user_type}",
)
logger.debug(f"User node template created: {user_node.to_dict()}. Writing to database {db_name}")
neon.create_or_merge_neontology_node(node=user_node, database=db_name, operation='merge')
logger.info(f"User node created: {user_node.to_dict()}")
logger.debug(f"About to call create_or_merge_neontology_node with node class: {user_node.__class__.__name__}")
logger.debug(f"Node primary label: {user_node.__primarylabel__}")
logger.debug(f"Node primary property: {user_node.__primaryproperty__}")
try:
neon.create_or_merge_neontology_node(node=user_node, database=db_name, operation='merge')
logger.info(f"User node created successfully: {user_node.to_dict()}")
except Exception as e:
logger.error(f"Failed to create user node: {e}")
raise
return user_node
def create_storage_bucket(self, bucket_id: str, bucket_name: str, access_token: Optional[str] = None) -> bool:
"""Create public and private storage buckets for the user using their access token or service role during initialization"""
logger.info(f"Creating storage buckets for user {self.cc_username}")
try:
from modules.database.supabase.utils.client import SupabaseServiceRoleClient, SupabaseAnonClient, CreateBucketOptions
# During initialization (no access token provided), use service role
if not access_token:
logger.info("Using service role client for bucket creation during initialization")
supabase = SupabaseServiceRoleClient()
else:
# For regular operations, use the user's access token
logger.info("Using user token for bucket creation")
supabase = SupabaseAnonClient.for_user(access_token)
# Create both public and private buckets
buckets = [
{
"id": f"{bucket_id}.public",
"options": CreateBucketOptions(
name=f"{bucket_name} - Public Files",
public=True,
file_size_limit=50 * 1024 * 1024, # 50MB
allowed_mime_types=[
'image/*', 'video/*', 'application/pdf',
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
'text/plain', 'text/csv', 'application/json'
]
)
},
{
"id": f"{bucket_id}.private",
"options": CreateBucketOptions(
name=f"{bucket_name} - Private Files",
public=False,
file_size_limit=50 * 1024 * 1024, # 50MB
allowed_mime_types=[
'image/*', 'video/*', 'application/pdf',
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
'text/plain', 'text/csv', 'application/json'
]
)
}
]
success = True
for bucket in buckets:
try:
result = supabase.create_bucket(bucket["id"], bucket["options"])
if not result:
logger.error(f"Failed to create bucket {bucket['id']}")
success = False
else:
logger.info(f"Successfully created bucket {bucket['id']}")
except Exception as e:
logger.error(f"Error creating bucket {bucket['id']}: {str(e)}")
success = False
return success
except Exception as e:
logger.error(f"Error creating storage buckets: {str(e)}")
return False
"""Create storage buckets for the user - DEPRECATED: Use centralized bucket initialization instead"""
logger.warning(f"Individual user bucket creation is deprecated. Use centralized bucket initialization instead.")
logger.info(f"User {self.cc_username} will use centralized storage buckets.")
return True # Return success to avoid breaking existing code
class SchoolUserCreator(UserCreator):
def __init__(self, user_id, cc_users_db_name, user_type, worker_type, user_email, worker_email, cc_username, user_name, worker_name, calendar_start_date, calendar_end_date, school_node, worker_node=None):
super().__init__(user_id, cc_users_db_name, user_type, worker_type, user_email, worker_email, cc_username, user_name, worker_name, calendar_start_date, calendar_end_date)
def __init__(
self,
user_id,
cc_users_db_name,
user_type,
worker_type,
user_email,
worker_email,
cc_username,
user_name,
worker_name,
calendar_start_date,
calendar_end_date,
school_node,
worker_node=None,
storage_tools=None,
user_db_name: Optional[str] = None,
worker_db_name: Optional[str] = None,
):
super().__init__(
user_id,
cc_users_db_name,
user_type,
worker_type,
user_email,
worker_email,
cc_username,
user_name,
worker_name,
calendar_start_date,
calendar_end_date,
storage_tools,
user_db_name=user_db_name,
worker_db_name=worker_db_name or (school_node.private_database_name if hasattr(school_node, "private_database_name") else None),
cc_schools_db_name="cc.institutes",
)
self.school_node = school_node
self.worker_node = worker_node
def _derive_school_worker_path(self, worker_kind: str) -> str:
school_identifier = getattr(self.school_node, 'uuid_string', None) if self.school_node else None
if not school_identifier and self.worker_db_name:
school_identifier = self.worker_db_name.split('.')[-1]
school_identifier = school_identifier or self.user_id
db_name_segment = self.worker_db_name or f"cc.institutes.{school_identifier}"
return os.path.join(
"schools",
school_identifier,
"databases",
db_name_segment,
worker_kind,
self.user_id,
).replace('\\', '/')
def create_user(self):
# Ensure Neontology is initialized
logger.debug(f"Initializing Neontology connection. Closing any existing connection")
@@ -167,7 +210,7 @@ class SchoolUserCreator(UserCreator):
self.user_nodes[f'worker_node'] = worker_node
user_node = self.create_user_node(self.cc_users_db_name)
user_node = self.create_user_node(self.user_db_name)
logger.info(f"User node created: {user_node}")
@@ -190,16 +233,25 @@ class SchoolUserCreator(UserCreator):
raise ValueError(f"Error creating teacher node: {e}") from e
def _create_teacher_node(self):
# Generate storage path for teacher node using Supabase Storage
if self.storage_tools:
teacher_dir_created, node_storage_path = self.storage_tools.create_teacher_storage_path(self.user_id)
else:
node_storage_path = self._derive_school_worker_path(self.worker_type)
teacher_node = worker_nodes.TeacherNode(
unique_id=f"{self.user_id}",
tldraw_snapshot="",
uuid_string=f"{self.user_id}",
node_storage_path=node_storage_path,
worker_name=self.worker_name,
worker_email=self.worker_email,
worker_db_name=self.worker_db_name,
worker_type=self.worker_type
)
# Use the school's private database name if available
school_db = self.school_node.private_database_name if hasattr(self.school_node, 'private_database_name') else f"cc.institutes.{self.school_node.school_type}.{self.school_node.id}"
school_db = self.worker_db_name or (
self.school_node.private_database_name if hasattr(self.school_node, 'private_database_name')
else f"cc.institutes.{self.school_node.school_type}.{getattr(self.school_node, 'id', self.school_node.uuid_string)}"
)
logger.info(f"Teacher node template created: {teacher_node}... setting school db to {school_db}")
neon.create_or_merge_neontology_node(node=teacher_node, database=school_db, operation='merge')
@@ -208,16 +260,25 @@ class SchoolUserCreator(UserCreator):
return teacher_node
def create_student_node(self):
# Generate storage path for student node using Supabase Storage
if self.storage_tools:
student_dir_created, node_storage_path = self.storage_tools.create_student_storage_path(f"Student_{self.user_id}")
else:
node_storage_path = self._derive_school_worker_path(self.worker_type)
student_node = worker_nodes.StudentNode(
unique_id=f"Student_{self.user_id}",
uuid_string=f"Student_{self.user_id}",
worker_name=self.worker_name,
worker_email=self.worker_email,
worker_db_name=self.worker_db_name,
worker_type=self.worker_type,
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Use the school's private database name if available
school_db = self.school_node.private_database_name if hasattr(self.school_node, 'private_database_name') else f"cc.institutes.{self.school_node.school_type}.{self.school_node.id}"
school_db = self.worker_db_name or (
self.school_node.private_database_name if hasattr(self.school_node, 'private_database_name')
else f"cc.institutes.{self.school_node.school_type}.{getattr(self.school_node, 'id', self.school_node.uuid_string)}"
)
logger.info(f"Student node template created: {student_node}... setting school db to {school_db}")
neon.create_or_merge_neontology_node(node=student_node, database=school_db, operation='merge')
@@ -228,20 +289,58 @@ class SchoolUserCreator(UserCreator):
def create_user_worker_relationship(self, user_node, worker_node):
user_role_rel = entity_relationships.UserIsSchoolWorker(source=user_node, target=worker_node)
# Use the school's private database name if available
school_db = self.school_node.private_database_name if hasattr(self.school_node, 'private_database_name') else f"cc.institutes.{self.school_node.school_type}.{self.school_node.id}"
school_db = self.worker_db_name or (
self.school_node.private_database_name if hasattr(self.school_node, 'private_database_name')
else f"cc.institutes.{self.school_node.school_type}.{getattr(self.school_node, 'id', self.school_node.uuid_string)}"
)
neon.create_or_merge_neontology_relationship(user_role_rel, database=school_db, operation='merge')
logger.info(f"Relationship created between user and worker in database {school_db}")
def create_worker_school_relationship(self, worker_node, school_node):
worker_school_rel = entity_relationships.EntityBelongsToSchool(source=worker_node, target=school_node)
# Use the school's private database name if available
school_db = school_node.private_database_name if hasattr(school_node, 'private_database_name') else f"cc.institutes.{school_node.school_type}.{school_node.id}"
school_db = self.worker_db_name or (
school_node.private_database_name if hasattr(school_node, 'private_database_name')
else f"cc.institutes.{school_node.school_type}.{getattr(school_node, 'id', school_node.uuid_string)}"
)
neon.create_or_merge_neontology_relationship(worker_school_rel, database=school_db, operation='merge')
logger.info(f"Relationship created between worker and school in database {school_db}")
class NonSchoolUserCreator(UserCreator):
def __init__(self, user_id, cc_users_db_name, user_type, worker_type, user_email, worker_email, cc_username, user_name, worker_name, calendar_start_date, calendar_end_date, developer_role: str = "developer"):
super().__init__(user_id, cc_users_db_name, user_type, worker_type, user_email, worker_email, cc_username, user_name, worker_name, calendar_start_date, calendar_end_date)
def __init__(
self,
user_id,
cc_users_db_name,
user_type,
worker_type,
user_email,
worker_email,
cc_username,
user_name,
worker_name,
calendar_start_date,
calendar_end_date,
developer_role: str = "developer",
storage_tools=None,
user_db_name: Optional[str] = None,
worker_db_name: Optional[str] = None,
):
super().__init__(
user_id,
cc_users_db_name,
user_type,
worker_type,
user_email,
worker_email,
cc_username,
user_name,
worker_name,
calendar_start_date,
calendar_end_date,
storage_tools,
user_db_name=user_db_name,
worker_db_name=worker_db_name,
)
self.developer_role = developer_role
def create_user(self, access_token: Optional[str] = None):
@@ -273,7 +372,8 @@ class NonSchoolUserCreator(UserCreator):
logger.debug(f"Creating developer db for {self.user_type} user {self.cc_username} in database {self.user_db_name}")
self.create_developer_db()
else:
raise ValueError(f"User type {self.user_type} not supported")
logger.warning(f"User type {self.user_type} not explicitly supported; defaulting to developer workspace")
self.create_developer_db()
logger.debug(f"User nodes after creation: {self.user_nodes}")
return self.user_nodes
@@ -290,10 +390,16 @@ class NonSchoolUserCreator(UserCreator):
logger.debug(f"Creating super admin user node for {self.user_type} user {self.cc_username} in database {self.user_db_name}")
private_user_node = self.create_user_node(self.user_db_name)
# Generate storage path for super admin node using Supabase Storage
if self.storage_tools:
admin_dir_created, node_storage_path = self.storage_tools.create_super_admin_storage_path(self.user_id)
else:
node_storage_path = self._derive_internal_worker_path(self.worker_type)
super_admin_node = worker_nodes.SuperAdminNode(
unique_id=f"SuperAdmin_{self.user_id}",
uuid_string=self.user_id,
worker_email=self.worker_email,
tldraw_snapshot="",
node_storage_path=node_storage_path,
worker_name=self.worker_name,
worker_db_name=self.worker_db_name,
worker_type=self.worker_type
@@ -329,11 +435,17 @@ class NonSchoolUserCreator(UserCreator):
# Create the user node again for the user db
private_user_node = self.create_user_node(self.user_db_name)
# Generate storage path for developer node using Supabase Storage
if self.storage_tools:
dev_dir_created, node_storage_path = self.storage_tools.create_developer_storage_path(self.user_id)
else:
node_storage_path = self._derive_internal_worker_path(self.worker_type)
developer_node = worker_nodes.DeveloperNode(
unique_id=f"Developer_{self.user_id}",
uuid_string=self.user_id,
worker_name=self.worker_name,
worker_email=self.worker_email,
tldraw_snapshot="",
node_storage_path=node_storage_path,
worker_db_name=self.worker_db_name,
worker_type=self.worker_type,
developer_role=self.developer_role
@@ -372,8 +484,101 @@ class NonSchoolUserCreator(UserCreator):
neon.create_or_merge_neontology_relationship(specific_user_rel, database=self.user_db_name, operation='merge')
logger.info("Relationship created between user and specific node")
def create_calendar(self, user_node: user_nodes.UserNode):
calendar_nodes = init_calendar.create_calendar(self.user_db_name, self.calendar_start_date, self.calendar_end_date, attach_to_calendar_node=True, owner_node=user_node)
def create_calendar(self, user_node: user_nodes.UserNode):
calendar_nodes = init_calendar.create_calendar(self.user_db_name, self.calendar_start_date, self.calendar_end_date, attach_to_calendar_node=True, owner_node=user_node, filesystem=self.filesystem)
logger.info(f"Calendar nodes created.")
return calendar_nodes
def _default_date_range():
today = datetime.now().date()
return today, (datetime.now() + timedelta(days=365)).date()
def create_user(
*,
user_id: str,
user_type: str,
username: str,
user_email: str,
user_name: Optional[str] = None,
worker_name: Optional[str] = None,
worker_type: Optional[str] = None,
worker_email: Optional[str] = None,
cc_users_db_name: str = "cc.users",
user_db_name: Optional[str] = None,
worker_db_name: Optional[str] = None,
calendar_start_date: Optional[datetime.date] = None,
calendar_end_date: Optional[datetime.date] = None,
school_node: Optional[Any] = None,
storage_tools=None,
) -> Dict[str, Optional[Any]]:
"""Create a user graph structure in Neo4j.
Args:
user_id: Identifier used as UUID for graph nodes.
user_type: Application-level user type (e.g. email_teacher, developer).
username: Canonical username/slug.
user_email: Contact email for the user node.
user_name: Friendly display name (defaults to username).
worker_name: Friendly name for worker node (defaults to user_name).
worker_type: Worker role (teacher, student, developer, etc.).
worker_email: Email for worker node (defaults to user_email).
cc_users_db_name: Root namespace for user databases (defaults to cc.users).
user_db_name: Fully-qualified target database for the user graph.
worker_db_name: Database for worker entities (usually school private DB).
calendar_start_date/calendar_end_date: Date range for initial calendar seeding.
school_node: Optional school context; if provided a SchoolUserCreator is used.
storage_tools: Optional Supabase storage tools for generating storage paths.
Returns:
Dict describing created nodes keyed by semantic role.
"""
start_date, end_date = calendar_start_date, calendar_end_date
if not start_date or not end_date:
start_date, end_date = _default_date_range()
worker_email = worker_email or user_email
user_name = user_name or username
worker_name = worker_name or user_name
if school_node is not None:
creator = SchoolUserCreator(
user_id=user_id,
cc_users_db_name=cc_users_db_name,
user_type=user_type,
worker_type=worker_type or "teacher",
user_email=user_email,
worker_email=worker_email,
cc_username=username,
user_name=user_name,
worker_name=worker_name,
calendar_start_date=start_date,
calendar_end_date=end_date,
school_node=school_node,
worker_node=None,
storage_tools=storage_tools,
user_db_name=user_db_name,
worker_db_name=worker_db_name,
)
else:
creator = NonSchoolUserCreator(
user_id=user_id,
cc_users_db_name=cc_users_db_name,
user_type=user_type,
worker_type=worker_type or user_type,
user_email=user_email,
worker_email=worker_email,
cc_username=username,
user_name=user_name,
worker_name=worker_name,
calendar_start_date=start_date,
calendar_end_date=end_date,
storage_tools=storage_tools,
user_db_name=user_db_name,
worker_db_name=worker_db_name,
)
return creator.create_user()
modules/database/init/init_user_timetable.py
+49 -49
View File
@@ -7,10 +7,10 @@ import modules.database.tools.neontology_tools as neon
from modules.database.tools.filesystem_tools import ClassroomCopilotFilesystem
from modules.database.schemas.nodes.users import UserNode
from modules.database.schemas.nodes.schools.schools import SubjectClassNode
from modules.database.schemas.nodes.workers.workers import TeacherNode
from modules.database.schemas.nodes.workers.workers import TeacherNode,
from modules.database.schemas.nodes.calendars import CalendarDayNode
from modules.database.schemas.nodes.workers.timetable import (
UserTeacherTimetableNode
UserTeacherTimetableNode, TimetableLessonNode
)
from modules.database.schemas.relationships.entity_timetable_rels import (
EntityHasTimetable
@@ -23,35 +23,35 @@ from modules.database.schemas.relationships.calendar_timetable_rels import (
CalendarDayHasPlannedLesson, PlannedLessonBelongsToCalendarDay
)
def get_school_worker_classes(school_db_name: str, user_unique_id: str, worker_unique_id: str) -> list:
def get_school_worker_classes(school_db_name: str, user_uuid_string: str, worker_uuid_string: str) -> list:
"""
Retrieve all classes for a worker from the school database.
"""
query = """
MATCH (w:Teacher {unique_id: $worker_id})-[:TEACHER_HAS_TIMETABLE]->(tt:TeacherTimetable)
MATCH (w:Teacher {uuid_string: $worker_id})-[:TEACHER_HAS_TIMETABLE]->(tt:TeacherTimetable)
-[:TIMETABLE_HAS_CLASS]->(c:SubjectClass)
RETURN c
"""
with driver.get_driver(db_name=school_db_name).session(database=school_db_name) as session:
result = session.run(query, worker_id=worker_unique_id)
result = session.run(query, worker_id=worker_uuid_string)
classes = [record['c'] for record in result]
if not classes:
logger.warning(f"No classes found for teacher {worker_unique_id} in school database")
logger.warning(f"No classes found for teacher {worker_uuid_string} in school database")
return classes
def get_school_class_periods(school_db_name: str, class_unique_id: str) -> list:
def get_school_class_periods(school_db_name: str, class_uuid_string: str) -> list:
"""
Retrieve all periods for a class from the school database.
"""
query = """
MATCH (c:SubjectClass {unique_id: $class_id})-[:CLASS_HAS_LESSON]->(l:TimetableLesson)
MATCH (c:SubjectClass {uuid_string: $class_id})-[:CLASS_HAS_LESSON]->(l:TimetableLesson)
RETURN l
"""
with driver.get_driver(db_name=school_db_name).session(database=school_db_name) as session:
result = session.run(query, class_id=class_unique_id)
result = session.run(query, class_id=class_uuid_string)
periods = [record['l'] for record in result]
if not periods:
logger.warning(f"No periods found for class {class_unique_id} in school database")
logger.warning(f"No periods found for class {class_uuid_string} in school database")
return periods
def get_user_calendar_nodes(user_db_name: str, user_node: UserNode) -> list:
@@ -60,12 +60,12 @@ def get_user_calendar_nodes(user_db_name: str, user_node: UserNode) -> list:
"""
# First try to find any calendar days to verify the structure
verify_query = """
MATCH (w:User {unique_id: $user_id})
MATCH (w:User {uuid_string: $user_id})
OPTIONAL MATCH (w)-[:HAS_CALENDAR]->(c:Calendar)
OPTIONAL MATCH (c)-[:CALENDAR_INCLUDES_YEAR]->(y:CalendarYear)
OPTIONAL MATCH (y)-[:YEAR_INCLUDES_MONTH]->(m:CalendarMonth)
OPTIONAL MATCH (m)-[:MONTH_INCLUDES_DAY]->(d:CalendarDay)
RETURN w.unique_id as user_id,
RETURN w.uuid_string as user_id,
count(c) as calendar_count,
count(y) as year_count,
count(m) as month_count,
@@ -76,7 +76,7 @@ def get_user_calendar_nodes(user_db_name: str, user_node: UserNode) -> list:
with driver.get_driver(db_name=user_db_name).session(database=user_db_name) as session:
# First check the calendar structure
result = session.run(verify_query, user_id=user_node.unique_id)
result = session.run(verify_query, user_id=user_node.uuid_string)
if stats := result.single():
logger.info(f"Calendar structure for user {stats['user_id']}: "
f"calendars={stats['calendar_count']}, "
@@ -86,50 +86,50 @@ def get_user_calendar_nodes(user_db_name: str, user_node: UserNode) -> list:
f"available years={stats['years']}")
if stats['calendar_count'] == 0:
logger.error(f"No calendar found for user {user_node.unique_id}")
logger.error(f"No calendar found for user {user_node.uuid_string}")
return []
if stats['year_count'] == 0:
logger.error(f"No calendar years found for user {user_node.unique_id}")
logger.error(f"No calendar years found for user {user_node.uuid_string}")
return []
if stats['month_count'] == 0:
logger.error(f"No calendar months found for user {user_node.unique_id}")
logger.error(f"No calendar months found for user {user_node.uuid_string}")
return []
if stats['day_count'] == 0:
logger.error(f"No calendar days found for user {user_node.unique_id}")
logger.error(f"No calendar days found for user {user_node.uuid_string}")
return []
# Get all calendar days without year filter
query = """
MATCH (w:User {unique_id: $user_id})-[:HAS_CALENDAR]->(c:Calendar)
MATCH (w:User {uuid_string: $user_id})-[:HAS_CALENDAR]->(c:Calendar)
-[:CALENDAR_INCLUDES_YEAR]->(y:CalendarYear)
-[:YEAR_INCLUDES_MONTH]->(m:CalendarMonth)
-[:MONTH_INCLUDES_DAY]->(d:CalendarDay)
RETURN d.unique_id as unique_id,
RETURN d.uuid_string as uuid_string,
d.date as date,
d.day_of_week as day_of_week,
d.iso_day as iso_day,
d.path as path
d.node_storage_path as path
ORDER BY d.date
"""
result = session.run(query, user_id=user_node.unique_id)
result = session.run(query, user_id=user_node.uuid_string)
calendar_days = []
for record in result:
calendar_day = CalendarDayNode(
unique_id=record['unique_id'],
uuid_string=record['uuid_string'],
date=record['date'],
day_of_week=record['day_of_week'],
iso_day=record['iso_day'],
path=record['path']
node_storage_path=record['path']
)
calendar_days.append(calendar_day)
if not calendar_days:
logger.error(f"No calendar days found for user {user_node.unique_id}")
logger.error(f"No calendar days found for user {user_node.uuid_string}")
else:
# Log the date range we have
dates = sorted([day.date for day in calendar_days])
logger.info(f"Found {len(calendar_days)} calendar days for user {user_node.unique_id}")
logger.info(f"Found {len(calendar_days)} calendar days for user {user_node.uuid_string}")
logger.info(f"Calendar days range from {dates[0]} to {dates[-1]}")
return calendar_days
@@ -149,7 +149,7 @@ def create_user_worker_timetable(
fs_handler = ClassroomCopilotFilesystem(db_name=user_db_name, init_run_type="user")
# Create teacher timetable directory under the worker's directory
_, worker_timetable_path = fs_handler.create_teacher_timetable_directory(user_worker_node.path)
_, worker_timetable_path = fs_handler.create_teacher_timetable_directory(user_worker_node.node_storage_path)
# Initialize neontology connection
neon.init_neontology_connection()
@@ -157,7 +157,7 @@ def create_user_worker_timetable(
# Get user's calendar nodes
calendar_nodes = get_user_calendar_nodes(user_db_name, user_node)
if not calendar_nodes:
logger.warning(f"No calendar nodes found for user {user_node.unique_id}")
logger.warning(f"No calendar nodes found for user {user_node.uuid_string}")
return {
"status": "error",
"message": "No calendar nodes found for user"
@@ -165,17 +165,17 @@ def create_user_worker_timetable(
try:
# Create UserTeacherTimetableNode
timetable_unique_id = f"UserTeacherTimetable_{user_worker_node.teacher_code}"
timetable_uuid_string = f"UserTeacherTimetable_{user_worker_node.teacher_code}"
worker_timetable = UserTeacherTimetableNode(
unique_id=timetable_unique_id,
uuid_string=timetable_uuid_string,
school_db_name=school_db_name,
school_timetable_id=f"TeacherTimetable_{user_worker_node.teacher_code}",
path=worker_timetable_path
node_storage_path=worker_timetable_path
)
# Create the timetable node and its tldraw file
neon.create_or_merge_neontology_node(worker_timetable, database=user_db_name, operation='merge')
fs_handler.create_default_tldraw_file(worker_timetable.path, worker_timetable.to_dict())
fs_handler.create_default_tldraw_file(worker_timetable.node_storage_path, worker_timetable.to_dict())
# Link timetable to teacher using the correct relationship structure
neon.create_or_merge_neontology_relationship(
@@ -185,9 +185,9 @@ def create_user_worker_timetable(
)
# Get classes from school database
school_classes = get_school_worker_classes(school_db_name, user_node.unique_id, user_worker_node.unique_id)
school_classes = get_school_worker_classes(school_db_name, user_node.uuid_string, user_worker_node.uuid_string)
if not school_classes:
logger.warning(f"No classes found for teacher {user_worker_node.unique_id} in school database")
logger.warning(f"No classes found for teacher {user_worker_node.uuid_string} in school database")
return {
"status": "warning",
"message": "No classes found in school database"
@@ -202,15 +202,15 @@ def create_user_worker_timetable(
# Create SubjectClassNode
subject_class_node = SubjectClassNode(
unique_id=class_data['unique_id'],
uuid_string=class_data['uuid_string'],
subject_class_code=class_data['subject_class_code'],
year_group=class_data['year_group'],
subject=class_data['subject'],
subject_code=class_data['subject_code'],
path=class_path
node_storage_path=class_path
)
neon.create_or_merge_neontology_node(subject_class_node, database=user_db_name, operation='merge')
fs_handler.create_default_tldraw_file(subject_class_node.path, subject_class_node.to_dict())
fs_handler.create_default_tldraw_file(subject_class_node.node_storage_path, subject_class_node.to_dict())
# Link class to timetable
neon.create_or_merge_neontology_relationship(
@@ -220,27 +220,27 @@ def create_user_worker_timetable(
)
# Initialize empty list for this class's lessons
class_lessons[class_data['unique_id']] = []
class_lessons[class_data['uuid_string']] = []
# Get periods from school database
periods = get_school_class_periods(school_db_name, class_data['unique_id'])
periods = get_school_class_periods(school_db_name, class_data['uuid_string'])
if not periods:
logger.warning(f"No periods found for class {class_data['unique_id']} in school database")
logger.warning(f"No periods found for class {class_data['uuid_string']} in school database")
continue
for period_data in periods:
# Create UserTimetableLessonNode
lesson_unique_id = f"UserTimetableLesson_{timetable_unique_id}_{class_name_safe}_{period_data['date']}_{period_data['period_code']}"
timetable_lesson_node = UserTimetableLessonNode(
unique_id=lesson_unique_id,
# Create TimetableLessonNode
lesson_uuid_string = f"UserTimetableLesson_{timetable_uuid_string}_{class_name_safe}_{period_data['date']}_{period_data['period_code']}"
timetable_lesson_node = TimetableLessonNode(
uuid_string=lesson_uuid_string,
subject_class=class_data['subject_class_code'],
date=period_data['date'],
start_time=period_data['start_time'],
end_time=period_data['end_time'],
period_code=period_data['period_code'],
school_db_name=school_db_name,
school_period_id=period_data['unique_id'],
path="Not set" # Will be set after creating directories
school_period_id=period_data['uuid_string'],
node_storage_path="Not set" # Will be set after creating directories
)
if calendar_day := next(
@@ -256,11 +256,11 @@ def create_user_worker_timetable(
class_path,
f"{calendar_day.date}_{period_data['period_code']}"
)
timetable_lesson_node.path = lesson_path
timetable_lesson_node.node_storage_path = lesson_path
# Create and link nodes
neon.create_or_merge_neontology_node(timetable_lesson_node, database=user_db_name, operation='merge')
fs_handler.create_default_tldraw_file(timetable_lesson_node.path, timetable_lesson_node.to_dict())
fs_handler.create_default_tldraw_file(timetable_lesson_node.node_storage_path, timetable_lesson_node.to_dict())
# Link lesson to class
neon.create_or_merge_neontology_relationship(
@@ -280,7 +280,7 @@ def create_user_worker_timetable(
)
# Store the lesson node
class_lessons[class_data['unique_id']].append({
class_lessons[class_data['uuid_string']].append({
'node': timetable_lesson_node,
'date': period_data['date'],
'start_time': period_data['start_time']
@@ -299,7 +299,7 @@ def create_user_worker_timetable(
next_lesson = sorted_lessons[i + 1]['node']
# Skip if current and next lesson are the same node
if current_lesson.unique_id != next_lesson.unique_id:
if current_lesson.uuid_string != next_lesson.uuid_string:
neon.create_or_merge_neontology_relationship(
TimetableLessonFollowsTimetableLesson(
source=current_lesson,
modules/database/init/init_worker_timetable.py
+27 -27
View File
@@ -33,19 +33,19 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
logging.info(f"Initialising filesystem handler...")
fs_handler = ClassroomCopilotFilesystem(db_name=worker_db_name, init_run_type="user")
_, worker_timetable_path = fs_handler.create_teacher_timetable_directory(worker_node.path)
_, worker_timetable_path = fs_handler.create_teacher_timetable_directory(worker_node.node_storage_path)
logging.info(f"Initialising neo4j connection...")
neon.init_neontology_connection()
try:
timetable_unique_id = f"TeacherTimetable_{worker_node.teacher_code}"
timetable_uuid_string = f"TeacherTimetable_{worker_node.teacher_code}"
worker_timetable = TeacherTimetableNode(
unique_id=timetable_unique_id,
path=worker_timetable_path
uuid_string=timetable_uuid_string,
node_storage_path=worker_timetable_path
)
neon.create_or_merge_neontology_node(worker_timetable, database=worker_db_name, operation='merge')
fs_handler.create_default_tldraw_file(worker_timetable.path, worker_timetable.to_dict())
fs_handler.create_default_tldraw_file(worker_timetable.node_storage_path, worker_timetable.to_dict())
neon.create_or_merge_neontology_relationship(
TeacherHasTimetable(source=worker_node, target=worker_timetable),
database=worker_db_name, operation='merge'
@@ -57,28 +57,28 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
for class_name, class_df in class_groups:
if pd.notna(class_name):
class_name_safe = re.sub(r'[^A-Za-z0-9_ ]+', '', class_name)
_, class_path = fs_handler.create_teacher_class_directory(worker_timetable.path, class_name_safe)
_, class_path = fs_handler.create_teacher_class_directory(worker_timetable.node_storage_path, class_name_safe)
subject_class_node_unique_id = f"SubjectClass_{class_name}"
subject_class_node_uuid_string = f"SubjectClass_{class_name}"
subject_class_node = SubjectClassNode(
unique_id=subject_class_node_unique_id,
uuid_string=subject_class_node_uuid_string,
subject_class_code=class_name,
year_group=str(int(class_df['YearGroup'].iloc[0])), # TODO: Hacky fix for the year group being a float
subject=str(class_df['Subject'].iloc[0]),
subject_code=str(class_df['SubjectCode'].iloc[0]),
path=class_path
node_storage_path=class_path
)
neon.create_or_merge_neontology_node(subject_class_node, database=worker_db_name, operation='merge')
logging.info(f"Class node created: {subject_class_node}")
# Create the tldraw file for the node
fs_handler.create_default_tldraw_file(subject_class_node.path, subject_class_node.to_dict())
fs_handler.create_default_tldraw_file(subject_class_node.node_storage_path, subject_class_node.to_dict())
# Link ClassNode to TeacherTimetableNode
neon.create_or_merge_neontology_relationship(
TimetableHasClass(source=worker_timetable, target=subject_class_node),
database=worker_db_name, operation='merge'
)
logging.info(f"Relationship created from {worker_timetable.unique_id} to {subject_class_node.unique_id}")
logging.info(f"Relationship created from {worker_timetable.uuid_string} to {subject_class_node.uuid_string}")
# Link class to corresponding YearGoupSyllabus
@@ -92,7 +92,7 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
SubjectClassBelongsToYearGroupSyllabus(source=subject_class_node, target=year_group_syllabus_node),
database=worker_db_name, operation='merge'
)
logging.info(f"Relationship created from {subject_class_node.unique_id} to {year_group_syllabus_node.unique_id}")
logging.info(f"Relationship created from {subject_class_node.uuid_string} to {year_group_syllabus_node.uuid_string}")
else:
logging.warning(f"No YearGroupSyllabus found for class {class_name} with year group {subject_class_node.year_group} and subject code {subject_class_node.subject_code}")
@@ -125,16 +125,16 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
date = class_lesson['date']
date_safe = date.strftime("%Y-%m-%d")
# Clean the class_name to make it directory-safe (catch all for invalid characters)
timetable_lesson_unique_id = f"TimetableLesson_{timetable_unique_id}_Class_{class_name}_Lesson_{lesson_number}_{date_safe}_{lesson_period_code}"
timetable_lesson_uuid_string = f"TimetableLesson_{timetable_uuid_string}_Class_{class_name}_Lesson_{lesson_number}_{date_safe}_{lesson_period_code}"
timetable_lesson_node = TimetableLessonNode(
unique_id=timetable_lesson_unique_id,
uuid_string=timetable_lesson_uuid_string,
subject_class=class_name,
date=date,
start_time=class_lesson['start_time'].time(), # TODO: This is probably how we should format the start and end time properties for all such nodes
end_time=class_lesson['end_time'].time(),
period_code=lesson_period_code,
path="Not set"
node_storage_path="Not set"
)
neon.create_or_merge_neontology_node(timetable_lesson_node, database=worker_db_name, operation='merge')
logging.info(f"TimetableLessonNode created: {timetable_lesson_node}")
@@ -144,19 +144,19 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
TimetableLessonBelongsToPeriod(source=timetable_lesson_node, target=period_node),
database=worker_db_name, operation='merge'
)
logging.info(f"Relationship created from {timetable_lesson_node.unique_id} to {period_node.unique_id}")
logging.info(f"Relationship created from {timetable_lesson_node.uuid_string} to {period_node.uuid_string}")
# Link TimetableLessonNode to ClassNode
neon.create_or_merge_neontology_relationship(
ClassHasLesson(source=subject_class_node, target=timetable_lesson_node),
database=worker_db_name, operation='merge'
)
logging.info(f"Relationship created from {subject_class_node.unique_id} to {timetable_lesson_node.unique_id}")
logging.info(f"Relationship created from {subject_class_node.uuid_string} to {timetable_lesson_node.uuid_string}")
# Create PlannedLessonNode
planned_lesson_unique_id = f"PlannedLesson_{timetable_unique_id}_Class_{class_name}_Lesson_{lesson_number}_{date_safe}_{lesson_period_code}"
planned_lesson_uuid_string = f"PlannedLesson_{timetable_uuid_string}_Class_{class_name}_Lesson_{lesson_number}_{date_safe}_{lesson_period_code}"
planned_lesson_node = PlannedLessonNode(
unique_id=planned_lesson_unique_id,
uuid_string=planned_lesson_uuid_string,
date=date,
start_time=class_lesson['start_time'].time(),
end_time=class_lesson['end_time'].time(),
@@ -174,7 +174,7 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
learning_statements=None,
learning_resource_codes=None,
learning_resources=None,
path="Not set"
node_storage_path="Not set"
)
# Create the PlannedLessonNode
neon.create_or_merge_neontology_node(planned_lesson_node, database=worker_db_name, operation='merge')
@@ -186,7 +186,7 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
TimetableLessonHasPlannedLesson(source=timetable_lesson_node, target=planned_lesson_node),
database=worker_db_name, operation='merge'
)
logging.info(f"Relationship created from {timetable_lesson_node.unique_id} to {planned_lesson_node.unique_id}")
logging.info(f"Relationship created from {timetable_lesson_node.uuid_string} to {planned_lesson_node.uuid_string}")
lesson_of_same_period += 1
lesson_number += 1
else:
@@ -201,17 +201,17 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
current_node = class_lesson_nodes[i]
i_safe = f"{i:02d}"
_, class_lesson_path = fs_handler.create_teacher_timetable_lesson_directory(class_path, f"{i_safe}_{current_node.date}_{current_node.period_code}")
current_node.path = class_lesson_path
current_node.node_storage_path = class_lesson_path
neon.create_or_merge_neontology_node(current_node, database=worker_db_name, operation='merge')
logging.info(f"TimetableLessonNode directory created and node merged into database: {current_node}")
# Create the tldraw file for the node
fs_handler.create_default_tldraw_file(current_node.path, current_node.to_dict())
fs_handler.create_default_tldraw_file(current_node.node_storage_path, current_node.to_dict())
if previous_node:
neon.create_or_merge_neontology_relationship(
TimetableLessonFollowsTimetableLesson(source=previous_node, target=current_node),
database=worker_db_name, operation='merge'
)
logging.info(f"Sequential relationship created between {previous_node.unique_id} and {current_node.unique_id}")
logging.info(f"Sequential relationship created between {previous_node.uuid_string} and {current_node.uuid_string}")
# Create sequential relationships for PlannedLessonNodes
for i in range(1, len(planned_lesson_nodes)):
@@ -219,17 +219,17 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
current_node = planned_lesson_nodes[i]
i_safe = f"{i:02d}"
_, planned_lesson_path = fs_handler.create_teacher_planned_lesson_directory(class_path, f"{i_safe}_{current_node.date}_{current_node.period_code}")
current_node.path = planned_lesson_path
current_node.node_storage_path = planned_lesson_path
neon.create_or_merge_neontology_node(current_node, database=worker_db_name, operation='merge')
logging.info(f"PlannedLessonNode directory created and node merged into database: {current_node}")
# Create the tldraw file for the node
fs_handler.create_default_tldraw_file(current_node.path, current_node.to_dict())
fs_handler.create_default_tldraw_file(current_node.node_storage_path, current_node.to_dict())
if previous_node:
neon.create_or_merge_neontology_relationship(
PlannedLessonFollowsPlannedLesson(source=previous_node, target=current_node),
database=worker_db_name, operation='merge'
)
logging.info(f"Sequential relationship created between {previous_node.unique_id} and {current_node.unique_id}")
logging.info(f"Sequential relationship created between {previous_node.uuid_string} and {current_node.uuid_string}")
logging.info(f"Successfully initialized worker timetable for worker {worker_node.teacher_code}")
return {"status": "success", "message": "Worker timetable initialized successfully"}
modules/database/schemas/nodes/base_nodes.py
+10 -10
View File
@@ -3,15 +3,15 @@ from modules.database.tools.neontology.basenode import BaseNode
class CCBaseNode(BaseNode):
__primarylabel__: ClassVar[str] = ''
__primaryproperty__: ClassVar[str] = 'unique_id'
unique_id: str
tldraw_snapshot: str
__primaryproperty__: ClassVar[str] = 'uuid_string'
uuid_string: str
node_storage_path: str
def to_dict(self):
return {
"__primarylabel__": self.__primarylabel__,
"unique_id": self.unique_id,
"tldraw_snapshot": self.tldraw_snapshot,
"uuid_string": self.uuid_string,
"node_storage_path": self.node_storage_path,
}
class UserBaseNode(CCBaseNode):
@@ -25,8 +25,8 @@ class UserBaseNode(CCBaseNode):
def to_dict(self):
return {
"__primarylabel__": self.__primarylabel__,
"unique_id": self.unique_id,
"tldraw_snapshot": self.tldraw_snapshot,
"uuid_string": self.uuid_string,
"node_storage_path": self.node_storage_path,
"cc_username": self.cc_username,
"user_db_name": self.user_db_name,
"user_email": self.user_email,
@@ -43,8 +43,8 @@ class WorkerBaseNode(CCBaseNode):
def to_dict(self):
return {
"__primarylabel__": self.__primarylabel__,
"unique_id": self.unique_id,
"tldraw_snapshot": self.tldraw_snapshot,
"uuid_string": self.uuid_string,
"node_storage_path": self.node_storage_path,
"worker_name": self.worker_name,
"worker_email": self.worker_email,
"worker_db_name": self.worker_db_name,
modules/database/schemas/nodes/schools/schools.py
+1 -1
View File
@@ -5,7 +5,7 @@ class SchoolNode(CCBaseNode):
__primarylabel__: ClassVar[str] = 'School'
# Core identification fields (required for all databases)
id: str # School's unique identifier within its type
uuid_string: str # School's unique identifier within its type
school_type: str # e.g., 'development', 'state', 'private', etc.
name: str
website: str = 'unknown'
modules/database/services/admin_service.py
+10
View File
@@ -2,6 +2,7 @@ import os
from typing import Dict, List, Optional
from supabase import create_client
from modules.logger_tool import initialise_logger
from modules.database.services.provisioning_service import ProvisioningService
from pydantic import BaseModel
@@ -31,6 +32,7 @@ class AdminService:
"Authorization": f"Bearer {service_role_key}",
"Content-Type": "application/json",
}
self.provisioner = ProvisioningService()
def get_admin_profile(self, admin_id: str) -> Optional[Dict]:
"""Get admin profile by ID"""
@@ -88,6 +90,10 @@ class AdminService:
result = (
self.supabase.table("admin_profiles").insert(profile_data).execute()
)
try:
self.provisioner.ensure_user(profile_data["id"])
except Exception as exc:
self.logger.warning(f"Provisioning admin user {profile_data['id']} failed: {exc}")
return result.data[0] if result else None
except Exception as e:
@@ -186,6 +192,10 @@ class AdminService:
result = (
self.supabase.table("admin_profiles").insert(profile_data).execute()
)
try:
self.provisioner.ensure_user(profile_data["id"])
except Exception as exc:
self.logger.warning(f"Provisioning super admin {profile_data['id']} failed: {exc}")
return result.data[0] if result else None
except Exception as e:
modules/database/services/graph_service.py
-67
View File
@@ -1,67 +0,0 @@
import os
from typing import Dict, Any
from modules.logger_tool import initialise_logger
import modules.database.tools.neo4j_driver_tools as driver_tools
from modules.database.admin.neontology_provider import NeontologyProvider
from modules.database.admin.graph_provider import GraphNamingProvider
class GraphService:
def __init__(self):
self.logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
self.driver = driver_tools.get_driver()
self.neontology = NeontologyProvider()
self.graph_naming = GraphNamingProvider()
def check_schema_status(self, database_name: str = "neo4j") -> Dict[str, Any]:
"""Check the status of Neo4j schema including constraints, indexes, and labels"""
try:
with self.driver.session(database=database_name) as session:
# Check constraints
constraints_result = session.run("SHOW CONSTRAINTS")
constraints = list(constraints_result)
# Check indexes
indexes_result = session.run("SHOW INDEXES")
indexes = list(indexes_result)
# Check labels
labels_result = session.run("CALL db.labels()")
labels = list(labels_result)
return {
"constraints_count": len(constraints),
"indexes_count": len(indexes),
"labels_count": len(labels),
"constraints": [dict(record) for record in constraints],
"indexes": [dict(record) for record in indexes],
"labels": [dict(record) for record in labels]
}
except Exception as e:
self.logger.error(f"Error checking schema status: {str(e)}")
return {
"constraints_count": 0,
"indexes_count": 0,
"labels_count": 0,
"error": str(e)
}
def initialize_schema(self, database_name: str = "neo4j") -> Dict[str, Any]:
"""Initialize Neo4j schema with required constraints and indexes"""
try:
schema_queries = self.graph_naming.get_schema_creation_queries()
with self.driver.session(database=database_name) as session:
for query in schema_queries:
session.run(query)
return {
"status": "success",
"message": "Schema initialized successfully",
"details": self.check_schema_status(database_name)
}
except Exception as e:
self.logger.error(f"Error initializing schema: {str(e)}")
return {
"status": "error",
"message": str(e)
}
modules/database/services/graph_storage_service.py
View File
modules/database/services/jwt_service.py
-45
View File
@@ -1,45 +0,0 @@
from datetime import datetime, timedelta
import jwt
from typing import Dict, List
class JWTService:
"""JWT Service for Neo4j authentication
TODO: Security Enhancements Needed
- Implement token refresh mechanism
- Add token revocation capability
- Add token validation checks
- Implement rate limiting
- Add audit logging for token generation/usage
- Consider reducing token expiry time and implementing refresh tokens
"""
def __init__(self, secret_key: str, algorithm: str = "HS256"):
self.secret_key = secret_key
self.algorithm = algorithm
def generate_neo4j_token(self, user_data: Dict) -> str:
"""Generate JWT token for Neo4j database access"""
payload = {
"sub": user_data["email"],
"roles": self._get_neo4j_roles(user_data["user_type"]),
"iss": "supabase",
"aud": "neo4j",
"iat": datetime.utcnow(),
"exp": datetime.utcnow() + timedelta(hours=24)
}
if "school_uuid" in user_data:
payload["worker_db_name"] = f"cc.institutes.{user_data['school_uuid']}"
return jwt.encode(payload, self.secret_key, algorithm=self.algorithm)
def _get_neo4j_roles(self, user_type: str) -> List[str]:
"""Map user types to Neo4j roles"""
role_mapping = {
"cc_admin": ["admin", "reader", "writer"],
"developer": ["developer", "reader", "writer"],
"email_teacher": ["teacher", "reader", "writer"],
"email_student": ["student", "reader"]
}
return role_mapping.get(user_type, ["reader"])
modules/database/services/neo4j_service.py
+12 -12
View File
@@ -86,18 +86,18 @@ class Neo4jService:
with self.driver.session(database=database_name) as session:
# Create constraints
constraints = [
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:School) REQUIRE n.unique_id IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Department) REQUIRE n.unique_id IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Subject) REQUIRE n.unique_id IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:YearGroup) REQUIRE n.unique_id IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Class) REQUIRE n.unique_id IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Teacher) REQUIRE n.unique_id IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Student) REQUIRE n.unique_id IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Calendar) REQUIRE n.unique_id IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Term) REQUIRE n.unique_id IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Week) REQUIRE n.unique_id IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Day) REQUIRE n.unique_id IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Period) REQUIRE n.unique_id IS UNIQUE"
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:School) REQUIRE n.uuid_string IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Department) REQUIRE n.uuid_string IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Subject) REQUIRE n.uuid_string IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:YearGroup) REQUIRE n.uuid_string IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Class) REQUIRE n.uuid_string IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Teacher) REQUIRE n.uuid_string IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Student) REQUIRE n.uuid_string IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Calendar) REQUIRE n.uuid_string IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Term) REQUIRE n.uuid_string IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Week) REQUIRE n.uuid_string IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Day) REQUIRE n.uuid_string IS UNIQUE",
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Period) REQUIRE n.uuid_string IS UNIQUE"
]
# Create indexes
modules/database/services/provisioning_service.py
+306
View File
@@ -0,0 +1,306 @@
import json
import os
from datetime import datetime, timedelta
from typing import Dict, Optional, Tuple, List
from modules.logger_tool import initialise_logger
from modules.database.services.neo4j_service import Neo4jService
from modules.database.init import init_user
from modules.database.tools.supabase_storage_tools import SupabaseStorageTools
from modules.database.schemas.nodes.schools.schools import SchoolNode
import modules.database.tools.neontology_tools as neon
from modules.database.tools.neontology_tools import create_or_merge_neontology_node
from modules.database.supabase.utils.client import SupabaseServiceRoleClient
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
_CC_USERS_DB = "cc.users"
_CC_SCHOOLS_DB = "cc.institutes"
_DEFAULT_INSTITUTE_NAME = os.getenv("DEFAULT_INSTITUTE_NAME", "KevlarAI")
_DEFAULT_INSTITUTE_ID = os.getenv("DEFAULT_INSTITUTE_ID")
class ProvisioningService:
"""Coordinates provisioning of Neo4j resources for schools and users."""
def __init__(self):
self.neo4j_service = Neo4jService()
self.supabase = SupabaseServiceRoleClient().supabase
# ------------------------------------------------------------------
# Naming helpers
# ------------------------------------------------------------------
@staticmethod
def _sanitize_component(value: str) -> str:
return "".join(ch for ch in value.lower() if ch.isalnum())
def _build_user_db_name(self, role: str, user_id: str) -> str:
return f"{_CC_USERS_DB}.{self._sanitize_component(role)}.{self._sanitize_component(user_id)}"
def _build_school_db_name(self, institute_id: str) -> str:
return f"{_CC_SCHOOLS_DB}.{self._sanitize_component(institute_id)}"
# ------------------------------------------------------------------
# Supabase helpers
# ------------------------------------------------------------------
def _get_profile(self, user_id: str) -> Dict:
response = (
self.supabase
.table("profiles")
.select("*")
.eq("id", user_id)
.single()
.execute()
)
if not response.data:
raise ValueError(f"Profile {user_id} not found")
return response.data
def _get_membership(self, profile_id: str) -> Optional[Dict]:
response = (
self.supabase
.table("institute_memberships")
.select("*")
.eq("profile_id", profile_id)
.limit(1)
.execute()
)
data = response.data or []
return data[0] if data else None
def _get_institute(self, institute_id: str) -> Optional[Dict]:
response = (
self.supabase
.table("institutes")
.select("*")
.eq("id", institute_id)
.single()
.execute()
)
return response.data if response and response.data else None
def _get_institute_by_name(self, name: str) -> Optional[Dict]:
response = (
self.supabase
.table("institutes")
.select("*")
.eq("name", name)
.limit(1)
.execute()
)
data = response.data or []
return data[0] if data else None
def _determine_membership_role(self, user_type: str) -> str:
if "teacher" in user_type:
return "teacher"
if "student" in user_type:
return "student"
return "staff"
def _ensure_membership(self, profile: Dict, user_type: str) -> Optional[Dict]:
membership = self._get_membership(profile["id"])
if membership:
return membership
institute_id = _DEFAULT_INSTITUTE_ID
institute = None
if institute_id:
institute = self._get_institute(institute_id)
if not institute:
logger.warning(f"Default institute {_DEFAULT_INSTITUTE_ID} not found; attempting lookup by name")
institute_id = None
if not institute_id:
institute = self._get_institute_by_name(_DEFAULT_INSTITUTE_NAME)
if not institute:
raise ValueError(f"Default institute '{_DEFAULT_INSTITUTE_NAME}' not found; cannot create membership")
institute_id = institute["id"]
role = self._determine_membership_role(user_type)
try:
response = (
self.supabase
.table("institute_memberships")
.insert({
"profile_id": profile["id"],
"institute_id": institute_id,
"role": role
})
.execute()
)
data = response.data or []
membership = data[0] if isinstance(data, list) and data else data
email = profile.get("email") or profile.get("user_email") or profile.get("id")
logger.info(f"Created institute membership for {email} -> {institute_id} as {role}")
return membership
except Exception as exc:
logger.warning(f"Failed to create institute membership for {profile['id']}: {exc}")
# Try to fetch again in case of race condition
return self._get_membership(profile["id"])
# ------------------------------------------------------------------
# Provisioning actions
# ------------------------------------------------------------------
def ensure_school(self, institute_id: str) -> Dict[str, str]:
"""Ensure the Neo4j databases and root nodes exist for a school."""
institute = self._get_institute(institute_id)
if not institute:
raise ValueError(f"Institute {institute_id} not found in Supabase")
school_db = self._build_school_db_name(institute_id)
curriculum_db = f"{school_db}.curriculum"
# Ensure root namespaces exist
self.neo4j_service.create_database(_CC_SCHOOLS_DB)
self.neo4j_service.create_database(school_db)
self.neo4j_service.create_database(curriculum_db)
metadata = institute.get("metadata") or {}
if isinstance(metadata, str):
try:
metadata = json.loads(metadata)
except json.JSONDecodeError:
metadata = {}
school_type = metadata.get("school_type") or institute.get("school_type") or "demo"
school_node = SchoolNode(
uuid_string=self._sanitize_component(institute_id),
node_storage_path=f"schools/{self._sanitize_component(institute_id)}/databases/{school_db}/{self._sanitize_component(institute_id)}",
school_type=self._sanitize_component(school_type) or "demo",
name=institute.get("name", "Unknown School"),
website=institute.get("website", "https://example.com"),
)
neon.init_neontology_connection()
try:
create_or_merge_neontology_node(school_node, database=_CC_SCHOOLS_DB, operation='merge')
create_or_merge_neontology_node(school_node, database=school_db, operation='merge')
finally:
neon.close_neontology_connection()
# Try to persist database references back to Supabase (best effort)
updates = {
"neo4j_private_db_name": school_db,
"neo4j_private_sync_status": "ready",
"neo4j_private_sync_at": datetime.utcnow().isoformat(),
}
try:
(
self.supabase
.table("institutes")
.update(updates)
.eq("id", institute_id)
.execute()
)
except Exception as exc: # pragma: no cover - defensive logging only
logger.warning(f"Failed to update institute {institute_id} with db info: {exc}")
return {
"db_name": school_db,
"curriculum_db_name": curriculum_db,
"school_node": school_node,
}
def ensure_user(self, user_id: str) -> Dict[str, Optional[str]]:
"""Provision Neo4j resources for a specific user profile."""
profile = self._get_profile(user_id)
user_type_raw = (profile.get("user_type") or "").lower()
user_type_map = {
"teacher": ("email_teacher", "teacher"),
"email_teacher": ("email_teacher", "teacher"),
"student": ("email_student", "student"),
"email_student": ("email_student", "student"),
"developer": ("developer", "developer"),
"cc_developer": ("developer", "developer"),
"admin": ("superadmin", "superadmin"),
"super_admin": ("superadmin", "superadmin"),
"superadmin": ("superadmin", "superadmin"),
}
neo_user_type, worker_type = user_type_map.get(user_type_raw, (user_type_raw or "standard", user_type_raw or "standard"))
user_db_name = profile.get("user_db_name")
if not user_db_name:
user_db_name = self._build_user_db_name(worker_type, user_id)
full_name = profile.get("full_name") or profile.get("display_name") or profile.get("username") or "User"
username = profile.get("username") or self._sanitize_component(profile.get("email", "user"))
user_email = profile.get("email") or profile.get("user_email") or ""
school_db_name = profile.get("school_db_name")
school_node = None
membership = None
if worker_type in ("teacher", "student"):
membership = self._ensure_membership(profile, user_type_raw)
if not membership:
raise ValueError("Unable to determine institute membership for school-based user")
if membership:
institute_id = membership.get("institute_id")
if institute_id:
ensure_school_result = self.ensure_school(institute_id)
school_db_name = ensure_school_result["db_name"]
school_meta = ensure_school_result.get("school_node")
if isinstance(school_meta, SchoolNode):
school_node = school_meta
else:
school_node = SchoolNode(
uuid_string=self._sanitize_component(institute_id),
node_storage_path="",
school_type=getattr(school_meta, "school_type", "demo") if school_meta else "demo",
name=(school_meta.get("name") if isinstance(school_meta, dict) else None) or "Unknown School",
website=(school_meta.get("website") if isinstance(school_meta, dict) else None) or "https://example.com",
)
# Ensure base namespaces exist before creating user-specific db
self.neo4j_service.create_database(_CC_USERS_DB)
self.neo4j_service.create_database(user_db_name)
calendar_start = datetime.utcnow().date()
calendar_end = (datetime.utcnow() + timedelta(days=365)).date()
# Initialize storage tools for user provisioning
storage_tools = SupabaseStorageTools(user_db_name, init_run_type="user")
init_user.create_user(
user_id=user_id,
user_type=neo_user_type,
username=username,
user_email=user_email,
user_name=full_name,
worker_name=full_name,
worker_type=worker_type,
worker_email=user_email,
cc_users_db_name=_CC_USERS_DB,
user_db_name=user_db_name,
worker_db_name=school_db_name,
calendar_start_date=calendar_start,
calendar_end_date=calendar_end,
school_node=school_node,
storage_tools=storage_tools,
)
profile_updates = {
"user_db_name": user_db_name,
"school_db_name": school_db_name,
"neo4j_sync_status": "ready",
"neo4j_synced_at": datetime.utcnow().isoformat(),
}
try:
(
self.supabase
.table("profiles")
.update(profile_updates)
.eq("id", user_id)
.execute()
)
except Exception as exc: # pragma: no cover - logging only
logger.warning(f"Failed to update profile {user_id} with provisioning info: {exc}")
return {
"user_db_name": user_db_name,
"worker_db_name": school_db_name,
"worker_type": worker_type,
}
modules/database/services/school_admin_service.py
-412
View File
@@ -1,412 +0,0 @@
import os
from typing import Dict, Any, BinaryIO
import json
import pandas as pd
import modules.database.tools.neo4j_driver_tools as driver_tools
import modules.database.tools.neo4j_session_tools as session_tools
import modules.database.schemas.nodes.schools.schools as school_nodes
import modules.database.schemas.nodes.schools.curriculum as curriculum_nodes
import modules.database.schemas.nodes.schools.pastoral as pastoral_nodes
import modules.database.schemas.nodes.structures.schools as school_structures
from modules.database.schemas.entities import entities
from modules.database.schemas.relationships import curriculum_relationships, entity_relationships, entity_curriculum_rels
from modules.database.admin.neontology_provider import NeontologyProvider
from modules.database.admin.graph_provider import GraphNamingProvider
from modules.database.supabase.utils.client import SupabaseAnonClient
from modules.database.supabase.utils.storage import StorageManager
from modules.database.services.neo4j_service import Neo4jService
from modules.logger_tool import initialise_logger
class SchoolAdminService:
def __init__(self):
self.logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
self.driver = driver_tools.get_driver()
self.neontology = NeontologyProvider()
self.graph_naming = GraphNamingProvider()
self.storage = StorageManager(SupabaseAnonClient)
self.neo4j_service = Neo4jService()
def check_database_exists(self, database_name: str) -> Dict[str, Any]:
"""Check if a Neo4j database exists"""
return self.neo4j_service.check_database_exists(database_name)
def create_database(self, db_name: str) -> Dict:
"""Creates a Neo4j database with the given name"""
return self.neo4j_service.create_database(db_name)
def create_school_node(self, school_data: Dict) -> Dict:
"""Creates a school node in cc.institutes database and stores TLDraw file in Supabase"""
try:
# Convert school data to SchoolNode
school_unique_id = self.graph_naming.get_school_unique_id(school_data['urn'])
school_path = self.graph_naming.get_school_path("cc.institutes", school_data['urn'])
school_node = entities.SchoolNode(
unique_id=school_unique_id,
path=school_path,
urn=school_data['urn'],
establishment_number=school_data['establishment_number'],
establishment_name=school_data['establishment_name'],
establishment_type=school_data['establishment_type'],
establishment_status=school_data['establishment_status'],
phase_of_education=school_data['phase_of_education'] if school_data['phase_of_education'] not in [None, ''] else None,
statutory_low_age=int(school_data['statutory_low_age']) if school_data.get('statutory_low_age') is not None else 0,
statutory_high_age=int(school_data['statutory_high_age']) if school_data.get('statutory_high_age') is not None else 0,
religious_character=school_data.get('religious_character') if school_data.get('religious_character') not in [None, ''] else None,
school_capacity=int(school_data['school_capacity']) if school_data.get('school_capacity') is not None else 0,
school_website=school_data.get('school_website', ''),
ofsted_rating=school_data.get('ofsted_rating') if school_data.get('ofsted_rating') not in [None, ''] else None
)
# Create default tldraw file data
tldraw_data = {
"document": {
"version": 1,
"id": school_data['urn'],
"name": school_data['establishment_name'],
"meta": {
"created_at": "",
"updated_at": "",
"creator_id": "",
"is_template": False,
"is_snapshot": False,
"is_draft": False,
"template_id": None,
"snapshot_id": None,
"draft_id": None
}
},
"schema": {
"schemaVersion": 1,
"storeVersion": 4,
"recordVersions": {
"asset": {
"version": 1,
"subTypeKey": "type",
"subTypeVersions": {}
},
"camera": {
"version": 1
},
"document": {
"version": 2
},
"instance": {
"version": 22
},
"instance_page_state": {
"version": 5
},
"page": {
"version": 1
},
"shape": {
"version": 3,
"subTypeKey": "type",
"subTypeVersions": {
"cc-school-node": 1
}
},
"instance_presence": {
"version": 5
},
"pointer": {
"version": 1
}
}
},
"store": {
"document:document": {
"gridSize": 10,
"name": school_data['establishment_name'],
"meta": {},
"id": school_data['urn'],
"typeName": "document"
},
"page:page": {
"meta": {},
"id": "page",
"name": "Page 1",
"index": "a1",
"typeName": "page"
},
"shape:school-node": {
"x": 0,
"y": 0,
"rotation": 0,
"type": "cc-school-node",
"id": school_unique_id,
"parentId": "page",
"index": "a1",
"props": school_node.to_dict(),
"typeName": "shape"
},
"instance:instance": {
"id": "instance",
"currentPageId": "page",
"typeName": "instance"
},
"camera:camera": {
"x": 0,
"y": 0,
"z": 1,
"id": "camera",
"typeName": "camera"
}
}
}
# Store tldraw file in Supabase storage
file_path = f"{school_data['urn']}/tldraw.json"
file_options = {
"content-type": "application/json",
"x-upsert": "true",
"metadata": {
"establishment_urn": school_data['urn'],
"establishment_name": school_data['establishment_name']
}
}
# Upload file
self.storage.upload_file(
bucket_id="cc.institutes",
file_path=file_path,
file_data=json.dumps(tldraw_data).encode(),
content_type="application/json",
upsert=True
)
# Create node in Neo4j
with self.neontology as neo:
self.logger.info(f"Creating school node in Neo4j: {school_node.to_dict()}")
neo.create_or_merge_node(school_node, database="cc.institutes", operation="merge")
return {"status": "success", "node": school_node}
except Exception as e:
self.logger.error(f"Error creating school node: {str(e)}")
return {"status": "error", "message": str(e)}
def create_private_database(self, school_data: Dict) -> Dict:
"""Creates a private database for a specific school"""
try:
private_db_name = f"cc.institutes.{school_data['urn']}"
with self.driver.session() as session:
session_tools.create_database(session, private_db_name)
self.logger.info(f"Created private database {private_db_name}")
return {
"status": "success",
"message": f"Database {private_db_name} created successfully"
}
except Exception as e:
self.logger.error(f"Error creating private database: {str(e)}")
return {"status": "error", "message": str(e)}
def create_basic_structure(self, school_node: school_nodes.SchoolNode, database_name: str) -> Dict:
"""Creates basic structural nodes in the specified database"""
try:
# Create Department Structure node
department_structure_node_unique_id = f"DepartmentStructure_{school_node.unique_id}"
department_structure_node = entities.DepartmentStructureNode(
unique_id=department_structure_node_unique_id,
tldraw_snapshot=""
)
# Create Curriculum Structure node
curriculum_node = curriculum_nodes.CurriculumStructureNode(
unique_id=f"CurriculumStructure_{school_node.unique_id}",
tldraw_snapshot=""
)
# Create Pastoral Structure node
pastoral_node = school_structures.PastoralStructureNode(
unique_id=f"PastoralStructure_{school_node.unique_id}",
tldraw_snapshot=""
)
with self.neontology as neo:
# Create nodes
neo.create_or_merge_node(department_structure_node, database=str(database_name), operation='merge')
neo.create_or_merge_node(curriculum_node, database=str(database_name), operation='merge')
neo.create_or_merge_node(pastoral_node, database=str(database_name), operation='merge')
# Create relationships
neo.create_or_merge_relationship(
entity_relationships.SchoolHasDepartmentStructure(source=school_node, target=department_structure_node),
database=database_name, operation='merge'
)
neo.create_or_merge_relationship(
entity_curriculum_rels.SchoolHasCurriculumStructure(source=school_node, target=curriculum_node),
database=database_name, operation='merge'
)
neo.create_or_merge_relationship(
entity_curriculum_rels.SchoolHasPastoralStructure(source=school_node, target=pastoral_node),
database=database_name, operation='merge'
)
return {
"status": "success",
"message": "Basic structure created successfully",
"nodes": {
"department_structure": department_structure_node,
"curriculum_structure": curriculum_node,
"pastoral_structure": pastoral_node
}
}
except Exception as e:
self.logger.error(f"Error creating basic structure: {str(e)}")
return {"status": "error", "message": str(e)}
def create_detailed_structure(self, school_node: school_nodes.SchoolNode, database_name: str, excel_file: BinaryIO) -> Dict:
"""Creates detailed structural nodes from Excel file"""
try:
# Store Excel file in Supabase
file_path = f"{school_node.urn}/structure.xlsx"
# Upload Excel file
self.storage.upload_file(
bucket_id="cc.institutes",
file_path=file_path,
file_data=excel_file.read(),
content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
upsert=True
)
# Process Excel file
dataframes = pd.read_excel(excel_file, sheet_name=None)
# Get existing basic structure nodes
with self.neontology as neo:
result = neo.cypher_read("""
MATCH (s:School {unique_id: $school_id})
OPTIONAL MATCH (s)-[:HAS_DEPARTMENT_STRUCTURE]->(ds:DepartmentStructure)
OPTIONAL MATCH (s)-[:HAS_CURRICULUM_STRUCTURE]->(cs:CurriculumStructure)
OPTIONAL MATCH (s)-[:HAS_PASTORAL_STRUCTURE]->(ps:PastoralStructure)
RETURN ds, cs, ps
""", {"school_id": school_node.unique_id}, database=database_name)
if not result:
raise Exception("Basic structure not found")
department_structure = result['ds']
curriculum_structure = result['cs']
pastoral_structure = result['ps']
# Create departments and subjects
unique_departments = dataframes['keystagesyllabuses']['Department'].dropna().unique()
node_library = {}
with self.neontology as neo:
for department_name in unique_departments:
department_node = entities.DepartmentNode(
unique_id=f"Department_{school_node.unique_id}_{department_name.replace(' ', '_')}",
department_name=department_name,
tldraw_snapshot=""
)
neo.create_or_merge_node(department_node, database=database_name, operation='merge')
node_library[f'department_{department_name}'] = department_node
# Link to department structure
neo.create_or_merge_relationship(
entity_relationships.DepartmentStructureHasDepartment(
source=department_structure,
target=department_node
),
database=database_name,
operation='merge'
)
# Create year groups
year_groups = self.sort_year_groups(dataframes['yeargroupsyllabuses'])['YearGroup'].unique()
last_year_group_node = None
for year_group in year_groups:
numeric_year_group = pd.to_numeric(year_group, errors='coerce')
if pd.notna(numeric_year_group):
year_group_node = pastoral_nodes.YearGroupNode(
unique_id=f"YearGroup_{school_node.unique_id}_YGrp{int(numeric_year_group)}",
year_group=str(int(numeric_year_group)),
year_group_name=f"Year {int(numeric_year_group)}",
tldraw_snapshot=""
)
neo.create_or_merge_node(year_group_node, database=database_name, operation='merge')
node_library[f'year_group_{int(numeric_year_group)}'] = year_group_node
# Create sequential relationship
if last_year_group_node:
neo.create_or_merge_relationship(
curriculum_relationships.YearGroupFollowsYearGroup(
source=last_year_group_node,
target=year_group_node
),
database=database_name,
operation='merge'
)
last_year_group_node = year_group_node
# Link to pastoral structure
neo.create_or_merge_relationship(
curriculum_relationships.PastoralStructureIncludesYearGroup(
source=pastoral_structure,
target=year_group_node
),
database=database_name,
operation='merge'
)
# Create key stages
key_stages = dataframes['keystagesyllabuses']['KeyStage'].unique()
last_key_stage_node = None
for key_stage in sorted(key_stages):
key_stage_node = curriculum_nodes.KeyStageNode(
unique_id=f"KeyStage_{curriculum_structure.unique_id}_KStg{key_stage}",
key_stage_name=f"Key Stage {key_stage}",
key_stage=str(key_stage),
tldraw_snapshot=""
)
neo.create_or_merge_node(key_stage_node, database=database_name, operation='merge')
node_library[f'key_stage_{key_stage}'] = key_stage_node
# Create sequential relationship
if last_key_stage_node:
neo.create_or_merge_relationship(
curriculum_relationships.KeyStageFollowsKeyStage(
source=last_key_stage_node,
target=key_stage_node
),
database=database_name,
operation='merge'
)
last_key_stage_node = key_stage_node
# Link to curriculum structure
neo.create_or_merge_relationship(
curriculum_relationships.CurriculumStructureIncludesKeyStage(
source=curriculum_structure,
target=key_stage_node
),
database=database_name,
operation='merge'
)
return {
"status": "success",
"message": "Detailed structure created successfully",
"node_library": node_library
}
except Exception as e:
self.logger.error(f"Error creating detailed structure: {str(e)}")
return {"status": "error", "message": str(e)}
def sort_year_groups(self, df: pd.DataFrame) -> pd.DataFrame:
"""Helper function to sort year groups numerically"""
df = df.copy()
df['YearGroupNumeric'] = pd.to_numeric(df['YearGroup'], errors='coerce')
return df.sort_values(by='YearGroupNumeric')
modules/database/services/school_service.py
-472
View File
@@ -1,472 +0,0 @@
import os
from typing import Dict, List, Optional, BinaryIO
import json
import pandas as pd
from backend.modules.database.schemas import entities
from modules.logger_tool import initialise_logger
from modules.database.tools.filesystem_tools import ClassroomCopilotFilesystem
import modules.database.tools.neo4j_driver_tools as driver_tools
import modules.database.tools.neo4j_session_tools as session_tools
from modules.database.admin.neontology_provider import NeontologyProvider
from modules.database.admin.graph_provider import GraphNamingProvider
from modules.database.schemas import curriculum_neo
from modules.database.schemas.relationships import curriculum_relationships, entity_relationships, entity_curriculum_rels
from modules.database.supabase.utils.storage import StorageManager
class SchoolService:
def __init__(self):
self.logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
self.driver = driver_tools.get_driver()
self.neontology = NeontologyProvider()
self.graph_naming = GraphNamingProvider()
self.storage = StorageManager()
def create_schools_database(self) -> Dict:
"""Creates the main cc.institutes database in Neo4j"""
try:
db_name = "cc.institutes"
with self.driver.session() as session:
session_tools.create_database(session, db_name)
self.logger.info(f"Created database {db_name}")
return {
"status": "success",
"message": f"Database {db_name} created successfully"
}
except Exception as e:
self.logger.error(f"Error creating schools database: {str(e)}")
return {"status": "error", "message": str(e)}
def create_school_node(self, school_data: Dict) -> Dict:
"""Creates a school node in cc.institutes database and stores TLDraw file in Supabase"""
try:
# Convert school data to SchoolNode
school_unique_id = self.graph_naming.get_school_unique_id(school_data['urn'])
school_path = self.graph_naming.get_school_path("cc.institutes", school_data['urn'])
school_node = entities.SchoolNode(
unique_id=school_unique_id,
path=school_path,
urn=school_data['urn'],
establishment_number=school_data['establishment_number'],
establishment_name=school_data['establishment_name'],
establishment_type=school_data['establishment_type'],
establishment_status=school_data['establishment_status'],
phase_of_education=school_data['phase_of_education'] if school_data['phase_of_education'] not in [None, ''] else None,
statutory_low_age=int(school_data['statutory_low_age']) if school_data.get('statutory_low_age') is not None else 0,
statutory_high_age=int(school_data['statutory_high_age']) if school_data.get('statutory_high_age') is not None else 0,
religious_character=school_data.get('religious_character') if school_data.get('religious_character') not in [None, ''] else None,
school_capacity=int(school_data['school_capacity']) if school_data.get('school_capacity') is not None else 0,
school_website=school_data.get('school_website', ''),
ofsted_rating=school_data.get('ofsted_rating') if school_data.get('ofsted_rating') not in [None, ''] else None
)
# Create default tldraw file data
tldraw_data = {
"document": {
"version": 1,
"id": school_data['urn'],
"name": school_data['establishment_name'],
"meta": {
"created_at": "",
"updated_at": "",
"creator_id": "",
"is_template": False,
"is_snapshot": False,
"is_draft": False,
"template_id": None,
"snapshot_id": None,
"draft_id": None
}
},
"schema": {
"schemaVersion": 1,
"storeVersion": 4,
"recordVersions": {
"asset": {
"version": 1,
"subTypeKey": "type",
"subTypeVersions": {}
},
"camera": {
"version": 1
},
"document": {
"version": 2
},
"instance": {
"version": 22
},
"instance_page_state": {
"version": 5
},
"page": {
"version": 1
},
"shape": {
"version": 3,
"subTypeKey": "type",
"subTypeVersions": {
"cc-school-node": 1
}
},
"instance_presence": {
"version": 5
},
"pointer": {
"version": 1
}
}
},
"store": {
"document:document": {
"gridSize": 10,
"name": school_data['establishment_name'],
"meta": {},
"id": school_data['urn'],
"typeName": "document"
},
"page:page": {
"meta": {},
"id": "page",
"name": "Page 1",
"index": "a1",
"typeName": "page"
},
"shape:school-node": {
"x": 0,
"y": 0,
"rotation": 0,
"type": "cc-school-node",
"id": school_unique_id,
"parentId": "page",
"index": "a1",
"props": school_node.to_dict(),
"typeName": "shape"
},
"instance:instance": {
"id": "instance",
"currentPageId": "page",
"typeName": "instance"
},
"camera:camera": {
"x": 0,
"y": 0,
"z": 1,
"id": "camera",
"typeName": "camera"
}
}
}
# Store tldraw file in Supabase storage
file_path = f"{school_data['urn']}/tldraw.json"
file_options = {
"content-type": "application/json",
"x-upsert": "true",
"metadata": {
"establishment_urn": school_data['urn'],
"establishment_name": school_data['establishment_name']
}
}
# Upload file
self.storage.upload_file(
bucket_id="cc.institutes",
file_path=file_path,
file_data=json.dumps(tldraw_data).encode(),
content_type="application/json",
upsert=True
)
# Create node in Neo4j
with self.neontology as neo:
self.logger.info(f"Creating school node in Neo4j: {school_node.to_dict()}")
neo.create_or_merge_node(school_node, database="cc.institutes", operation="merge")
return {"status": "success", "node": school_node}
except Exception as e:
self.logger.error(f"Error creating school node: {str(e)}")
return {"status": "error", "message": str(e)}
def create_private_database(self, school_data: Dict) -> Dict:
"""Creates a private database for a specific school"""
try:
private_db_name = f"cc.institutes.{school_data['urn']}"
with self.driver.session() as session:
session_tools.create_database(session, private_db_name)
self.logger.info(f"Created private database {private_db_name}")
return {
"status": "success",
"message": f"Database {private_db_name} created successfully"
}
except Exception as e:
self.logger.error(f"Error creating private database: {str(e)}")
return {"status": "error", "message": str(e)}
def create_basic_structure(self, school_node: entities.SchoolNode, database_name: str) -> Dict:
"""Creates basic structural nodes in the specified database"""
try:
# Create filesystem paths
fs_handler = ClassroomCopilotFilesystem(database_name, init_run_type="school")
# Create Department Structure node
department_structure_node_unique_id = f"DepartmentStructure_{school_node.unique_id}"
_, department_path = fs_handler.create_school_department_directory(school_node.path, "departments")
department_structure_node = entities.DepartmentStructureNode(
unique_id=department_structure_node_unique_id,
path=department_path
)
# Create Curriculum Structure node
_, curriculum_path = fs_handler.create_school_curriculum_directory(school_node.path)
curriculum_node = curriculum_neo.CurriculumStructureNode(
unique_id=f"CurriculumStructure_{school_node.unique_id}",
path=curriculum_path
)
# Create Pastoral Structure node
_, pastoral_path = fs_handler.create_school_pastoral_directory(school_node.path)
pastoral_node = curriculum_neo.PastoralStructureNode(
unique_id=f"PastoralStructure_{school_node.unique_id}",
path=pastoral_path
)
with self.neontology as neo:
# Create nodes
neo.create_or_merge_node(department_structure_node, database=str(database_name), operation='merge')
fs_handler.create_default_tldraw_file(department_structure_node.path, department_structure_node.to_dict())
neo.create_or_merge_node(curriculum_node, database=str(database_name), operation='merge')
fs_handler.create_default_tldraw_file(curriculum_node.path, curriculum_node.to_dict())
neo.create_or_merge_node(pastoral_node, database=database_name, operation='merge')
fs_handler.create_default_tldraw_file(pastoral_node.path, pastoral_node.to_dict())
# Create relationships
neo.create_or_merge_relationship(
entity_relationships.SchoolHasDepartmentStructure(source=school_node, target=department_structure_node),
database=database_name, operation='merge'
)
neo.create_or_merge_relationship(
entity_curriculum_rels.SchoolHasCurriculumStructure(source=school_node, target=curriculum_node),
database=database_name, operation='merge'
)
neo.create_or_merge_relationship(
entity_curriculum_rels.SchoolHasPastoralStructure(source=school_node, target=pastoral_node),
database=database_name, operation='merge'
)
return {
"status": "success",
"message": "Basic structure created successfully",
"nodes": {
"department_structure": department_structure_node,
"curriculum_structure": curriculum_node,
"pastoral_structure": pastoral_node
}
}
except Exception as e:
self.logger.error(f"Error creating basic structure: {str(e)}")
return {"status": "error", "message": str(e)}
def create_detailed_structure(self, school_node: entities.SchoolNode, database_name: str, excel_file: BinaryIO) -> Dict:
"""Creates detailed structural nodes from Excel file"""
try:
# Store Excel file in Supabase
file_path = f"{school_node.urn}/structure.xlsx"
# Upload Excel file
self.storage.upload_file(
bucket_id="cc.institutes",
file_path=file_path,
file_data=excel_file.read(),
content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
upsert=True
)
# Process Excel file
dataframes = pd.read_excel(excel_file, sheet_name=None)
# Get existing basic structure nodes
with self.neontology as neo:
result = neo.cypher_read("""
MATCH (s:School {unique_id: $school_id})
OPTIONAL MATCH (s)-[:HAS_DEPARTMENT_STRUCTURE]->(ds:DepartmentStructure)
OPTIONAL MATCH (s)-[:HAS_CURRICULUM_STRUCTURE]->(cs:CurriculumStructure)
OPTIONAL MATCH (s)-[:HAS_PASTORAL_STRUCTURE]->(ps:PastoralStructure)
RETURN ds, cs, ps
""", {"school_id": school_node.unique_id}, database=database_name)
if not result:
raise Exception("Basic structure not found")
department_structure = result['ds']
curriculum_structure = result['cs']
pastoral_structure = result['ps']
# Create departments and subjects
unique_departments = dataframes['keystagesyllabuses']['Department'].dropna().unique()
fs_handler = ClassroomCopilotFilesystem(database_name, init_run_type="school")
node_library = {}
with self.neontology as neo:
for department_name in unique_departments:
_, department_path = fs_handler.create_school_department_directory(school_node.path, department_name)
department_node = entities.DepartmentNode(
unique_id=f"Department_{school_node.unique_id}_{department_name.replace(' ', '_')}",
department_name=department_name,
path=department_path
)
neo.create_or_merge_node(department_node, database=database_name, operation='merge')
fs_handler.create_default_tldraw_file(department_node.path, department_node.to_dict())
node_library[f'department_{department_name}'] = department_node
# Link to department structure
neo.create_or_merge_relationship(
entity_relationships.DepartmentStructureHasDepartment(
source=department_structure,
target=department_node
),
database=database_name,
operation='merge'
)
# Create year groups
year_groups = self.sort_year_groups(dataframes['yeargroupsyllabuses'])['YearGroup'].unique()
last_year_group_node = None
for year_group in year_groups:
numeric_year_group = pd.to_numeric(year_group, errors='coerce')
if pd.notna(numeric_year_group):
_, year_group_path = fs_handler.create_pastoral_year_group_directory(
pastoral_structure.path,
str(int(numeric_year_group))
)
year_group_node = curriculum_neo.YearGroupNode(
unique_id=f"YearGroup_{school_node.unique_id}_YGrp{int(numeric_year_group)}",
year_group=str(int(numeric_year_group)),
year_group_name=f"Year {int(numeric_year_group)}",
path=year_group_path
)
neo.create_or_merge_node(year_group_node, database=database_name, operation='merge')
fs_handler.create_default_tldraw_file(year_group_node.path, year_group_node.to_dict())
node_library[f'year_group_{int(numeric_year_group)}'] = year_group_node
# Create sequential relationship
if last_year_group_node:
neo.create_or_merge_relationship(
curriculum_relationships.YearGroupFollowsYearGroup(
source=last_year_group_node,
target=year_group_node
),
database=database_name,
operation='merge'
)
last_year_group_node = year_group_node
# Link to pastoral structure
neo.create_or_merge_relationship(
curriculum_relationships.PastoralStructureIncludesYearGroup(
source=pastoral_structure,
target=year_group_node
),
database=database_name,
operation='merge'
)
# Create key stages
key_stages = dataframes['keystagesyllabuses']['KeyStage'].unique()
last_key_stage_node = None
for key_stage in sorted(key_stages):
_, key_stage_path = fs_handler.create_curriculum_key_stage_directory(
curriculum_structure.path,
str(key_stage)
)
key_stage_node = curriculum_neo.KeyStageNode(
unique_id=f"KeyStage_{curriculum_structure.unique_id}_KStg{key_stage}",
key_stage_name=f"Key Stage {key_stage}",
key_stage=str(key_stage),
path=key_stage_path
)
neo.create_or_merge_node(key_stage_node, database=database_name, operation='merge')
fs_handler.create_default_tldraw_file(key_stage_node.path, key_stage_node.to_dict())
node_library[f'key_stage_{key_stage}'] = key_stage_node
# Create sequential relationship
if last_key_stage_node:
neo.create_or_merge_relationship(
curriculum_relationships.KeyStageFollowsKeyStage(
source=last_key_stage_node,
target=key_stage_node
),
database=database_name,
operation='merge'
)
last_key_stage_node = key_stage_node
# Link to curriculum structure
neo.create_or_merge_relationship(
curriculum_relationships.CurriculumStructureIncludesKeyStage(
source=curriculum_structure,
target=key_stage_node
),
database=database_name,
operation='merge'
)
return {
"status": "success",
"message": "Detailed structure created successfully",
"node_library": node_library
}
except Exception as e:
self.logger.error(f"Error creating detailed structure: {str(e)}")
return {"status": "error", "message": str(e)}
def sort_year_groups(self, df: pd.DataFrame) -> pd.DataFrame:
"""Helper function to sort year groups numerically"""
df = df.copy()
df['YearGroupNumeric'] = pd.to_numeric(df['YearGroup'], errors='coerce')
return df.sort_values(by='YearGroupNumeric')
def check_schools_database(self) -> Dict:
"""Check if the schools database exists and has been initialized"""
try:
db_name = "cc.institutes"
with self.driver.session() as session:
# Check if database exists
databases = session_tools.list_databases(session)
if db_name not in databases:
return {
"status": "error",
"message": f"Database {db_name} does not exist"
}
# Check if database has any nodes (indicating it's been initialized)
session.run("USE " + db_name)
result = session.run("MATCH (n) RETURN count(n) as count").single()
node_count = result["count"] if result else 0
if node_count == 0:
return {
"status": "error",
"message": f"Database {db_name} exists but has no nodes"
}
return {
"status": "success",
"message": f"Database {db_name} exists and has {node_count} nodes"
}
except Exception as e:
self.logger.error(f"Error checking schools database: {str(e)}")
return {"status": "error", "message": str(e)}
modules/database/supabase/utils/client.py
+6 -12
View File
@@ -14,14 +14,13 @@ class CreateBucketOptions(TypedDict, total=False):
allowed_mime_types: List[str]
name: str
def _create_base_client(url: str, key: str, options: Optional[Dict[str, Any]] = None, access_token: Optional[str] = None) -> Client:
def _create_base_client(url: str, key: str, options: Optional[Dict[str, Any]] = None) -> Client:
"""Create a base Supabase client with given configuration."""
client_options = SyncClientOptions(
schema="public",
storage=SyncMemoryStorage(),
headers={
"apikey": key,
"Authorization": f"Bearer {access_token if access_token else key}"
"Authorization": f"Bearer {key}"
}
)
return create_client(url, key, options=client_options)
@@ -29,16 +28,16 @@ def _create_base_client(url: str, key: str, options: Optional[Dict[str, Any]] =
class SupabaseServiceRoleClient:
"""Supabase client for making authenticated requests using the service role key"""
def __init__(self, url: Optional[str] = None, service_role_key: Optional[str] = None, access_token: Optional[str] = None):
def __init__(self, url: Optional[str] = None, service_role_key: Optional[str] = None):
"""Initialize the Supabase client with URL and service role key"""
self.url = url or os.environ.get("SUPABASE_URL", "http://kong:8000")
self.url = url or os.environ.get("SUPABASE_URL", "http://localhost:8000")
self.service_role_key = service_role_key or os.environ.get("SERVICE_ROLE_KEY")
if not self.url or not self.service_role_key:
raise ValueError("SUPABASE_URL and SERVICE_ROLE_KEY must be provided")
# Initialize Supabase client with service role key and optional access token
self.supabase = _create_base_client(self.url, self.service_role_key, access_token=access_token)
self.supabase = _create_base_client(self.url, self.service_role_key)
def create_bucket(self, id: str, options: Optional[CreateBucketOptions] = None) -> Dict[str, Any]:
"""Create a storage bucket with the given ID and options"""
@@ -48,17 +47,12 @@ class SupabaseServiceRoleClient:
options['name'] = id # Use ID as default name if not provided
return self.supabase.storage.create_bucket(id, options=options)
@classmethod
def for_admin(cls, access_token: str) -> 'SupabaseServiceRoleClient':
"""Create a client instance for the super admin using their access token"""
return cls(access_token=access_token)
class SupabaseAnonClient:
"""Supabase client for making authenticated requests using the anon key"""
def __init__(self, url: Optional[str] = None, anon_key: Optional[str] = None, access_token: Optional[str] = None):
"""Initialize the Supabase client with URL and anon key"""
self.url = url or os.environ.get("SUPABASE_URL", "http://kong:8000")
self.url = url or os.environ.get("SUPABASE_URL", "http://localhost:8000")
self.anon_key = anon_key or os.environ.get("ANON_KEY")
if not self.url or not self.anon_key:
modules/database/tools/filesystem_tools.py
+25 -312
View File
@@ -18,7 +18,6 @@ import re
class ClassroomCopilotFilesystem:
def __init__(self, db_name: str, init_run_type: str = None):
logging.info(f"Initializing ClassroomCopilotFilesystem with db_name: {db_name} and init_run_type: {init_run_type}")
self.db_name = db_name
@@ -27,19 +26,21 @@ class ClassroomCopilotFilesystem:
if not self.base_path:
raise ValueError("NODE_FILESYSTEM_PATH environment variable not set")
logging.info(f"Initializing ClassroomCopilotFilesystem with db_name: {db_name} and init_run_type: {init_run_type} with base path: {self.base_path}")
# Set root path based on init type
if init_run_type == "school":
self.root_path = os.path.join(self.base_path, "schools", self.db_name)
self.root_path = os.path.join(self.base_path, "schools")
logging.debug(f"School root path: {self.root_path}")
elif init_run_type == "user":
self.root_path = os.path.join(self.base_path, "users", self.db_name)
self.root_path = os.path.join(self.base_path, "users")
logging.debug(f"User root path: {self.root_path}")
elif init_run_type == "multiplayer":
self.root_path = os.path.join(self.base_path, "multiplayer")
logging.debug(f"Multiplayer root path: {self.root_path}")
else:
self.root_path = os.path.join(self.base_path, self.db_name)
logging.debug(f"Default root path: {self.root_path}")
self.root_path = self.base_path
logging.debug(f"Database root path: {self.root_path}")
# Ensure root directory exists
os.makedirs(self.root_path, exist_ok=True)
@@ -63,45 +64,33 @@ class ClassroomCopilotFilesystem:
return True
return False
def sanitize_username(self, username):
return re.sub(r'[^\w\-_\.]', '_', username)
def create_user_directory(self, username, user_type=None, school_path=None):
def create_private_user_directory(self, user_id):
"""Create a directory for a specific user."""
sanitized_username = self.sanitize_username(username)
if school_path:
# For school database: /schools/[school_db]/users/[user_type]/[username]
user_path = os.path.join(self.root_path, "users", user_type, sanitized_username)
else:
# For user database: /users/[user_db]/[username]
user_path = os.path.join(self.root_path, sanitized_username)
# For user database: /users/[user_db]/[username]
user_path = os.path.join(self.root_path, user_id)
logging.info(f"Creating user directory at {user_path}")
return self.create_directory(user_path), user_path
def create_user_worker_directory(self, user_path, worker_code):
def create_user_worker_directory(self, user_path, worker_id, worker_type):
"""Create a worker directory under the user directory."""
# Create worker directory: [user_path]/[worker_code]
worker_path = os.path.join(user_path, worker_code)
worker_path = os.path.join(user_path, worker_type, worker_id)
logging.info(f"Creating worker directory at {worker_path}")
return self.create_directory(worker_path), worker_path
def create_school_worker_directory(self, school_path, worker_type):
def create_school_worker_directory(self, school_path, worker_type, worker_id):
"""Create a worker directory under the school directory."""
worker_path = os.path.join(school_path, "workers", worker_type)
logging.info(f"Creating school worker directory at {worker_path}")
worker_path = os.path.join(school_path, "workers", worker_type, worker_id)
logging.info(f"Creating school {worker_type} worker directory at {worker_path}")
return self.create_directory(worker_path), worker_path
def create_school_directory(self, school_uuid=None):
def create_school_directory(self, school_uuid_string):
"""Create a directory for a specific school."""
logging.info(f"Creating school directory with school_uuid: {school_uuid}")
if school_uuid is None:
logging.debug(f"School UUID is None, creating school directory at {self.root_path}")
school_path = self.root_path
else:
logging.debug(f"School UUID is not None, creating school directory at {os.path.join(self.root_path, school_uuid)}")
school_path = os.path.join(self.root_path, school_uuid)
logging.info(f"Creating school directory with uuid_string: {school_uuid_string}")
logging.debug(f"School UUID is not None, creating school directory at {os.path.join(self.root_path, school_uuid_string)}")
school_path = os.path.join(self.root_path, school_uuid_string)
return self.create_directory(school_path), school_path
def create_year_directory(self, year, calendar_path=None):
@@ -248,7 +237,12 @@ class ClassroomCopilotFilesystem:
"""Create a directory for a specific topic under a year group syllabus."""
topic_path = os.path.join(year_group_syllabus_path, "topics", f"{topic_id}")
return self.create_directory(topic_path), topic_path
def create_curriculum_keystage_topic_directory(self, keystage_syllabus_path, topic_id):
"""Create a directory for a specific key stage topic under a key stage group syllabus."""
topic_path = os.path.join(keystage_syllabus_path, "core_topics", f"{topic_id}")
return self.create_directory(topic_path), topic_path
def create_curriculum_lesson_directory(self, topic_path, lesson_id):
"""Create a directory for a specific lesson under a topic."""
lesson_path = os.path.join(topic_path, "lessons", f"{lesson_id}")
@@ -276,285 +270,4 @@ class ClassroomCopilotFilesystem:
def create_teacher_planned_lesson_directory(self, class_path, lesson_id):
planned_lesson_path = os.path.join(class_path, "planned_lessons", lesson_id)
return self.create_directory(planned_lesson_path), planned_lesson_path
# TLDraw File Creation
def create_default_tldraw_file(self, node_path, node_data):
"""Create a tldraw file for a node."""
logging.info(f"Creating tldraw file for node at {node_path}")
# Ensure the directory exists
os.makedirs(node_path, exist_ok=True)
tldraw_path = os.path.join(node_path, 'tldraw_file.json')
# Create default tldraw content
tldraw_content = {
"document": {
"store": {
"document:document": {
"gridSize": 10,
"name": "",
"meta": {},
"id": "document:document",
"typeName": "document"
},
"page:page": {
"meta": {},
"id": "page:page",
"name": "Page 1",
"index": "a1",
"typeName": "page"
}
},
"schema":
{"schemaVersion":2,
"sequences": {
"com.tldraw.store":4,
"com.tldraw.asset":1,
"com.tldraw.camera":1,
"com.tldraw.document":2,
"com.tldraw.instance":25,
"com.tldraw.instance_page_state":5,
"com.tldraw.page":1,
"com.tldraw.instance_presence":5,
"com.tldraw.pointer":1,
"com.tldraw.shape":4,
"com.tldraw.asset.bookmark":2,
"com.tldraw.asset.image":5,
"com.tldraw.asset.video":5,
"com.tldraw.shape.arrow":5,
"com.tldraw.shape.bookmark":2,
"com.tldraw.shape.draw":2,
"com.tldraw.shape.embed":4,
"com.tldraw.shape.frame":0,
"com.tldraw.shape.geo":9,
"com.tldraw.shape.group":0,
"com.tldraw.shape.highlight":1,
"com.tldraw.shape.image":4,
"com.tldraw.shape.line":5,
"com.tldraw.shape.note":8,
"com.tldraw.shape.text":2,
"com.tldraw.shape.video":2,
"com.tldraw.shape.youtube-embed":0,
"com.tldraw.shape.calendar":0,
"com.tldraw.shape.microphone":1,
"com.tldraw.shape.transcriptionText":0,
"com.tldraw.shape.slide":0,"com.tldraw.shape.slideshow":0,
"com.tldraw.shape.user_node":1,
"com.tldraw.shape.developer_node":1,
"com.tldraw.shape.student_node":1,
"com.tldraw.shape.teacher_node":1,
"com.tldraw.shape.calendar_node":1,
"com.tldraw.shape.calendar_year_node":1,
"com.tldraw.shape.calendar_month_node":1,
"com.tldraw.shape.calendar_week_node":1,
"com.tldraw.shape.calendar_day_node":1,
"com.tldraw.shape.calendar_time_chunk_node":1,
"com.tldraw.shape.teacher_timetable_node":1,
"com.tldraw.shape.timetable_lesson_node":1,
"com.tldraw.shape.planned_lesson_node":1,
"com.tldraw.shape.pastoral_structure_node":1,
"com.tldraw.shape.year_group_node":1,
"com.tldraw.shape.curriculum_structure_node":1,
"com.tldraw.shape.key_stage_node":1,
"com.tldraw.shape.key_stage_syllabus_node":1,
"com.tldraw.shape.year_group_syllabus_node":1,
"com.tldraw.shape.subject_node":1,
"com.tldraw.shape.topic_node":1,
"com.tldraw.shape.topic_lesson_node":1,
"com.tldraw.shape.learning_statement_node":1,
"com.tldraw.shape.science_lab_node":1,
"com.tldraw.shape.school_timetable_node":1,
"com.tldraw.shape.academic_year_node":1,
"com.tldraw.shape.academic_term_node":1,
"com.tldraw.shape.academic_week_node":1,
"com.tldraw.shape.academic_day_node":1,
"com.tldraw.shape.academic_period_node":1,
"com.tldraw.shape.registration_period_node":1,
"com.tldraw.shape.school_node":1,
"com.tldraw.shape.department_node":1,
"com.tldraw.shape.room_node":1,
"com.tldraw.shape.subject_class_node":1,
"com.tldraw.shape.general_relationship":1,
"com.tldraw.binding.arrow":0,
"com.tldraw.binding.slide-layout":0
}
},
"recordVersions": {
"asset": { "version": 1, "subTypeKey": "type", "subTypeVersions": {} },
"camera": { "version": 1 },
"document": { "version": 2 },
"instance": { "version": 21 },
"instance_page_state": { "version": 5 },
"page": { "version": 1 },
"shape": { "version": 3, "subTypeKey": "type", "subTypeVersions": {} },
"instance_presence": { "version": 5 },
"pointer": { "version": 1 }
},
"rootShapeIds":[],
"bindings":[],
"assets":[]
},
"session": {
"version": 0,
"currentPageId": "page:page",
"pageStates": [{
"pageId": "page:page",
"camera": {"x": 0, "y": 0, "z": 1},
"selectedShapeIds": []
}]
},
"node_data": node_data
}
with open(tldraw_path, 'w') as f:
json.dump(tldraw_content, f, indent=4)
logging.info(f"tldraw file created at {tldraw_path}")
return tldraw_path
def create_default_tldraw_file_in_storage(self, admin_supabase, bucket_id, file_path, node_data):
"""Create a tldraw file in Supabase storage."""
logging.info(f"Creating tldraw file in storage at {file_path}")
# Create default tldraw content
tldraw_content = {
"document": {
"store": {
"document:document": {
"gridSize": 10,
"name": "",
"meta": {},
"id": "document:document",
"typeName": "document"
},
"page:page": {
"meta": {},
"id": "page:page",
"name": "Page 1",
"index": "a1",
"typeName": "page"
}
},
"schema":
{"schemaVersion":2,
"sequences": {
"com.tldraw.store":4,
"com.tldraw.asset":1,
"com.tldraw.camera":1,
"com.tldraw.document":2,
"com.tldraw.instance":25,
"com.tldraw.instance_page_state":5,
"com.tldraw.page":1,
"com.tldraw.instance_presence":5,
"com.tldraw.pointer":1,
"com.tldraw.shape":4,
"com.tldraw.asset.bookmark":2,
"com.tldraw.asset.image":5,
"com.tldraw.asset.video":5,
"com.tldraw.shape.arrow":5,
"com.tldraw.shape.bookmark":2,
"com.tldraw.shape.draw":2,
"com.tldraw.shape.embed":4,
"com.tldraw.shape.frame":0,
"com.tldraw.shape.geo":9,
"com.tldraw.shape.group":0,
"com.tldraw.shape.highlight":1,
"com.tldraw.shape.image":4,
"com.tldraw.shape.line":5,
"com.tldraw.shape.note":8,
"com.tldraw.shape.text":2,
"com.tldraw.shape.video":2,
"com.tldraw.shape.youtube-embed":0,
"com.tldraw.shape.calendar":0,
"com.tldraw.shape.microphone":1,
"com.tldraw.shape.transcriptionText":0,
"com.tldraw.shape.slide":0,"com.tldraw.shape.slideshow":0,
"com.tldraw.shape.user_node":1,
"com.tldraw.shape.developer_node":1,
"com.tldraw.shape.student_node":1,
"com.tldraw.shape.teacher_node":1,
"com.tldraw.shape.calendar_node":1,
"com.tldraw.shape.calendar_year_node":1,
"com.tldraw.shape.calendar_month_node":1,
"com.tldraw.shape.calendar_week_node":1,
"com.tldraw.shape.calendar_day_node":1,
"com.tldraw.shape.calendar_time_chunk_node":1,
"com.tldraw.shape.teacher_timetable_node":1,
"com.tldraw.shape.timetable_lesson_node":1,
"com.tldraw.shape.planned_lesson_node":1,
"com.tldraw.shape.pastoral_structure_node":1,
"com.tldraw.shape.year_group_node":1,
"com.tldraw.shape.curriculum_structure_node":1,
"com.tldraw.shape.key_stage_node":1,
"com.tldraw.shape.key_stage_syllabus_node":1,
"com.tldraw.shape.year_group_syllabus_node":1,
"com.tldraw.shape.subject_node":1,
"com.tldraw.shape.topic_node":1,
"com.tldraw.shape.topic_lesson_node":1,
"com.tldraw.shape.learning_statement_node":1,
"com.tldraw.shape.science_lab_node":1,
"com.tldraw.shape.school_timetable_node":1,
"com.tldraw.shape.academic_year_node":1,
"com.tldraw.shape.academic_term_node":1,
"com.tldraw.shape.academic_week_node":1,
"com.tldraw.shape.academic_day_node":1,
"com.tldraw.shape.academic_period_node":1,
"com.tldraw.shape.registration_period_node":1,
"com.tldraw.shape.school_node":1,
"com.tldraw.shape.department_node":1,
"com.tldraw.shape.room_node":1,
"com.tldraw.shape.subject_class_node":1,
"com.tldraw.shape.general_relationship":1,
"com.tldraw.binding.arrow":0,
"com.tldraw.binding.slide-layout":0
}
},
"recordVersions": {
"asset": { "version": 1, "subTypeKey": "type", "subTypeVersions": {} },
"camera": { "version": 1 },
"document": { "version": 2 },
"instance": { "version": 21 },
"instance_page_state": { "version": 5 },
"page": { "version": 1 },
"shape": { "version": 3, "subTypeKey": "type", "subTypeVersions": {} },
"instance_presence": { "version": 5 },
"pointer": { "version": 1 }
},
"rootShapeIds":[],
"bindings":[],
"assets":[]
},
"session": {
"version": 0,
"currentPageId": "page:page",
"pageStates": [{
"pageId": "page:page",
"camera": {"x": 0, "y": 0, "z": 1},
"selectedShapeIds": []
}]
},
"node_data": node_data
}
# Convert the content to JSON string
tldraw_json = json.dumps(tldraw_content, indent=4)
try:
# Upload the file to Supabase storage
result = admin_supabase.storage.from_(bucket_id).upload(
path=file_path,
file=tldraw_json,
file_options={"content-type": "application/json"}
)
if result.get('error'):
logging.error(f"Error creating tldraw file in storage: {result['error']}")
raise Exception(f"Failed to create tldraw file: {result['error']}")
logging.info(f"tldraw file created in storage at {file_path}")
return True
except Exception as e:
logging.error(f"Error creating tldraw file in storage: {str(e)}")
raise e
return self.create_directory(planned_lesson_path), planned_lesson_path
modules/database/tools/navigation/user_navigation.py
+58 -58
View File
@@ -13,8 +13,8 @@ def get_static_nodes(context: str, db_name: str) -> List[Dict[str, Any]]:
query = """
MATCH (t:Teacher)
RETURN DISTINCT {
id: t.unique_id,
path: t.path,
id: t.uuid_string,
path: t.node_storage_path,
label: t.teacher_name_formal,
type: 'Teacher',
isStatic: true,
@@ -24,8 +24,8 @@ def get_static_nodes(context: str, db_name: str) -> List[Dict[str, Any]]:
UNION ALL
MATCH (t:UserTeacherTimetable)
RETURN DISTINCT {
id: t.unique_id,
path: t.path,
id: t.uuid_string,
path: t.node_storage_path,
label: t.name,
type: 'UserTeacherTimetable',
isStatic: true,
@@ -35,8 +35,8 @@ def get_static_nodes(context: str, db_name: str) -> List[Dict[str, Any]]:
UNION ALL
MATCH (t:UserTeacherTimetable)-[:HAS_CLASS]->(c:Class)
RETURN DISTINCT {
id: c.unique_id,
path: c.path,
id: c.uuid_string,
path: c.node_storage_path,
label: c.name,
type: 'Class',
isStatic: true,
@@ -49,8 +49,8 @@ def get_static_nodes(context: str, db_name: str) -> List[Dict[str, Any]]:
query = """
MATCH (u:User)
RETURN DISTINCT {
id: u.unique_id,
path: u.path,
id: u.uuid_string,
path: u.node_storage_path,
label: u.user_name,
type: 'User',
isStatic: true,
@@ -70,8 +70,8 @@ def get_static_nodes(context: str, db_name: str) -> List[Dict[str, Any]]:
ELSE 1
END as nodeOrder
RETURN DISTINCT {
id: n.unique_id,
path: n.path,
id: n.uuid_string,
path: n.node_storage_path,
label: n.name,
type: 'Calendar',
isStatic: true,
@@ -98,7 +98,7 @@ def get_today_calendar_node(db_name: str) -> Optional[Dict[str, Any]]:
query = """
MATCH (n:Calendar)
WHERE date($today) >= date(n.start_date) AND date($today) <= date(n.end_date)
RETURN n.unique_id as id, n.path as path, n.name as label,
RETURN n.uuid_string as id, n.path as path, n.name as label,
'Calendar' as type
LIMIT 1
"""
@@ -117,7 +117,7 @@ def get_relative_calendar_node(day_offset: int, db_name: str) -> Optional[Dict[s
query = """
MATCH (n:Calendar)
WHERE date($target_date) >= date(n.start_date) AND date($target_date) <= date(n.end_date)
RETURN n.unique_id as id, n.path as path, n.name as label,
RETURN n.uuid_string as id, n.node_storage_path as path, n.name as label,
'Calendar' as type
LIMIT 1
"""
@@ -136,7 +136,7 @@ def get_next_month_node(db_name: str) -> Optional[Dict[str, Any]]:
query = """
MATCH (n:Calendar)
WHERE date($next_month_start) >= date(n.start_date) AND date($next_month_start) <= date(n.end_date)
RETURN n.unique_id as id, n.path as path, n.name as label,
RETURN n.uuid_string as id, n.node_storage_path as path, n.name as label,
'Calendar' as type
LIMIT 1
"""
@@ -155,7 +155,7 @@ def get_previous_month_node(db_name: str) -> Optional[Dict[str, Any]]:
query = """
MATCH (n:Calendar)
WHERE date($prev_month_start) >= date(n.start_date) AND date($prev_month_start) <= date(n.end_date)
RETURN n.unique_id as id, n.path as path, n.name as label,
RETURN n.uuid_string as id, n.node_storage_path as path, n.name as label,
'Calendar' as type
LIMIT 1
"""
@@ -172,7 +172,7 @@ def get_user_timetables(db_name: str) -> List[Dict[str, Any]]:
"""Get user's timetables."""
query = """
MATCH (t:UserTeacherTimetable)
RETURN t.unique_id as id, t.path as path, t.name as label,
RETURN t.uuid_string as id, t.node_storage_path as path, t.name as label,
'UserTeacherTimetable' as type
"""
try:
@@ -186,8 +186,8 @@ def get_user_timetables(db_name: str) -> List[Dict[str, Any]]:
def get_timetable_classes(timetable_id: str, db_name: str) -> List[Dict[str, Any]]:
"""Get classes for a timetable."""
query = """
MATCH (t:UserTeacherTimetable {unique_id: $timetable_id})-[:HAS_CLASS]->(c:Class)
RETURN c.unique_id as id, c.path as path, c.name as label,
MATCH (t:UserTeacherTimetable {uuid_string: $timetable_id})-[:HAS_CLASS]->(c:Class)
RETURN c.uuid_string as id, c.node_storage_path as path, c.name as label,
'Class' as type
"""
try:
@@ -202,9 +202,9 @@ def get_next_lesson(class_id: str, db_name: str) -> Optional[Dict[str, Any]]:
"""Get next lesson for a class."""
now = datetime.now().strftime("%Y-%m-%d %H:%M")
query = """
MATCH (c:Class {unique_id: $class_id})-[:HAS_LESSON]->(l:Lesson)
MATCH (c:Class {uuid_string: $class_id})-[:HAS_LESSON]->(l:Lesson)
WHERE l.start_time > $now
RETURN l.unique_id as id, l.path as path, l.name as label,
RETURN l.uuid_string as id, l.node_storage_path as path, l.name as label,
'Lesson' as type
ORDER BY l.start_time ASC
LIMIT 1
@@ -222,9 +222,9 @@ def get_previous_lesson(class_id: str, db_name: str) -> Optional[Dict[str, Any]]
"""Get previous lesson for a class."""
now = datetime.now().strftime("%Y-%m-%d %H:%M")
query = """
MATCH (c:Class {unique_id: $class_id})-[:HAS_LESSON]->(l:Lesson)
MATCH (c:Class {uuid_string: $class_id})-[:HAS_LESSON]->(l:Lesson)
WHERE l.start_time < $now
RETURN l.unique_id as id, l.path as path, l.name as label,
RETURN l.uuid_string as id, l.path as path, l.name as label,
'Lesson' as type
ORDER BY l.start_time DESC
LIMIT 1
@@ -251,20 +251,20 @@ def save_shared_snapshot(path: str, room_id: str, snapshot: Dict[str, Any]) -> b
def get_connected_nodes_for_workers(node_id: str, db_name: str) -> List[Dict[str, Any]]:
"""Get connected nodes specific to the workers context."""
query = """
MATCH (n {unique_id: $node_id})
MATCH (n {uuid_string: $node_id})
WITH n
CALL {
WITH n
MATCH (n:UserTeacherTimetable)-[:HAS_CLASS]->(c:Class)
RETURN c.unique_id as id, c.path as path, c.name as label,
RETURN c.uuid_string as id, c.node_storage_path as path, c.name as label,
'Class' as type
UNION
MATCH (n:Class)<-[:HAS_CLASS]-(t:UserTeacherTimetable)
RETURN t.unique_id as id, t.path as path, t.name as label,
RETURN t.uuid_string as id, t.node_storage_path as path, t.name as label,
'UserTeacherTimetable' as type
UNION
MATCH (n:Class)-[:HAS_LESSON]->(l:Lesson)
RETURN l.unique_id as id, l.path as path, l.name as label,
RETURN l.uuid_string as id, l.node_storage_path as path, l.name as label,
'Lesson' as type
}
RETURN DISTINCT id, path, label, type
@@ -284,8 +284,8 @@ def get_connected_nodes(node_id: str, db_name: str, context: str = None) -> List
# Default query for other contexts
query = """
MATCH (n {unique_id: $node_id})-[r]-(connected)
RETURN DISTINCT connected.unique_id as id, connected.path as path,
MATCH (n {uuid_string: $node_id})-[r]-(connected)
RETURN DISTINCT connected.uuid_string as id, connected.path as path,
connected.name as label, labels(connected)[0] as type
"""
try:
@@ -314,37 +314,37 @@ def get_worker_structure(db_name: str) -> Dict[str, Any]:
// Collect all nodes
RETURN {
schools: collect(DISTINCT {
id: s.unique_id,
path: s.path,
id: s.uuid_string,
path: s.node_storage_path,
name: s.school_name,
__primarylabel__: 'School'
}),
departments: collect(DISTINCT {
id: d.unique_id,
path: d.path,
id: d.uuid_string,
path: d.node_storage_path,
code: d.department_code,
school_id: s.unique_id,
school_id: s.uuid_string,
__primarylabel__: 'Department'
}),
timetables: collect(DISTINCT {
id: t.unique_id,
path: t.path,
id: t.uuid_string,
path: t.node_storage_path,
name: t.name,
department_id: d.unique_id,
department_id: d.uuid_string,
__primarylabel__: 'UserTeacherTimetable'
}),
classes: collect(DISTINCT {
id: c.unique_id,
path: c.path,
id: c.uuid_string,
path: c.node_storage_path,
code: c.class_code,
timetable_id: t.unique_id,
timetable_id: t.uuid_string,
__primarylabel__: 'Class'
}),
lessons: collect(DISTINCT {
id: l.unique_id,
path: l.path,
id: l.uuid_string,
path: l.node_storage_path,
start_time: l.start_time,
class_id: c.unique_id,
class_id: c.uuid_string,
__primarylabel__: 'TimetableLesson'
})
} as structure
@@ -369,10 +369,10 @@ def get_worker_structure(db_name: str) -> Dict[str, Any]:
def get_school_node(school_id: str, db_name: str) -> Optional[Dict[str, Any]]:
"""Get a specific school node."""
query = """
MATCH (s:School {unique_id: $school_id})
MATCH (s:School {uuid_string: $school_id})
RETURN {
id: s.unique_id,
path: s.path,
id: s.uuid_string,
path: s.node_storage_path,
name: s.school_name,
__primarylabel__: 'School'
} as node
@@ -389,10 +389,10 @@ def get_school_node(school_id: str, db_name: str) -> Optional[Dict[str, Any]]:
def get_department_node(dept_id: str, db_name: str) -> Optional[Dict[str, Any]]:
"""Get a specific department node."""
query = """
MATCH (d:Department {unique_id: $dept_id})
MATCH (d:Department {uuid_string: $dept_id})
RETURN {
id: d.unique_id,
path: d.path,
id: d.uuid_string,
path: d.node_storage_path,
code: d.department_code,
__primarylabel__: 'Department'
} as node
@@ -409,10 +409,10 @@ def get_department_node(dept_id: str, db_name: str) -> Optional[Dict[str, Any]]:
def get_timetable_node(timetable_id: str, db_name: str) -> Optional[Dict[str, Any]]:
"""Get a specific timetable node."""
query = """
MATCH (t:UserTeacherTimetable {unique_id: $timetable_id})
MATCH (t:UserTeacherTimetable {uuid_string: $timetable_id})
RETURN {
id: t.unique_id,
path: t.path,
id: t.uuid_string,
path: t.node_storage_path,
name: t.name,
__primarylabel__: 'UserTeacherTimetable'
} as node
@@ -429,10 +429,10 @@ def get_timetable_node(timetable_id: str, db_name: str) -> Optional[Dict[str, An
def get_class_node(class_id: str, db_name: str) -> Optional[Dict[str, Any]]:
"""Get a specific class node."""
query = """
MATCH (c:Class {unique_id: $class_id})
MATCH (c:Class {uuid_string: $class_id})
RETURN {
id: c.unique_id,
path: c.path,
id: c.uuid_string,
path: c.node_storage_path,
code: c.class_code,
__primarylabel__: 'Class'
} as node
@@ -449,10 +449,10 @@ def get_class_node(class_id: str, db_name: str) -> Optional[Dict[str, Any]]:
def get_lesson_node(lesson_id: str, db_name: str) -> Optional[Dict[str, Any]]:
"""Get a specific lesson node."""
query = """
MATCH (l:TimetableLesson {unique_id: $lesson_id})
MATCH (l:TimetableLesson {uuid_string: $lesson_id})
RETURN {
id: l.unique_id,
path: l.path,
id: l.uuid_string,
path: l.node_storage_path,
start_time: l.start_time,
__primarylabel__: 'TimetableLesson'
} as node
@@ -473,8 +473,8 @@ def get_current_lesson(db_name: str) -> Optional[Dict[str, Any]]:
MATCH (l:TimetableLesson)
WHERE l.start_time >= $now
RETURN {
id: l.unique_id,
path: l.path,
id: l.uuid_string,
path: l.node_storage_path,
start_time: l.start_time,
__primarylabel__: 'TimetableLesson'
} as node
modules/database/tools/neo4j_http_tools.py
-2
View File
@@ -15,8 +15,6 @@ logging = logger.get_logger(
import requests
import base64
dev_mode = os.getenv('DEV_MODE', 'false')
def send_query(query, encoded_credentials=None, params=None, method='POST', database="system", endpoint="/tx/commit"):
if encoded_credentials is None:
logging.debug(f"Sending query to Neo4j: {query}")
modules/database/tools/neo4j_session_tools.py
+12 -12
View File
@@ -15,16 +15,16 @@ logging = logger.get_logger(
import modules.database.tools.queries as query
from contextlib import suppress
def get_node_by_unique_id_and_adjacent_nodes(session, unique_id):
return session.read_transaction(_get_node_by_unique_id_and_adjacent_nodes, unique_id)
def get_node_by_uuid_string_and_adjacent_nodes(session, uuid_string):
return session.read_transaction(_get_node_by_uuid_string_and_adjacent_nodes, uuid_string)
def _get_node_by_unique_id_and_adjacent_nodes(tx, unique_id):
def _get_node_by_uuid_string_and_adjacent_nodes(tx, uuid_string):
query = """
MATCH (n {unique_id: $unique_id})
MATCH (n {uuid_string: $uuid_string})
OPTIONAL MATCH (n)-[r]-(adjacent)
RETURN n AS node, COLLECT(DISTINCT {node: adjacent, relationship: r}) AS connected_nodes
"""
result = tx.run(query, unique_id=unique_id)
result = tx.run(query, uuid_string=uuid_string)
record = result.single()
if record:
node = record["node"]
@@ -242,20 +242,20 @@ def _find_nodes_by_label(tx, label):
result = tx.run(query)
return [record["n"] for record in result]
def get_node_by_unique_id(session, unique_id):
return session.read_transaction(_get_node_by_unique_id, unique_id)
def get_node_by_uuid_string(session, uuid_string):
return session.read_transaction(_get_node_by_uuid_string, uuid_string)
def _get_node_by_unique_id(tx, unique_id):
def _get_node_by_uuid_string(tx, uuid_string):
query = f"""
MATCH (n)
WHERE n.unique_id = $unique_id
WHERE n.uuid_string = $uuid_string
RETURN n
"""
logging.debug(f"Executing query with unique_id: {unique_id}")
result = tx.run(query, unique_id=unique_id)
logging.debug(f"Executing query with uuid_string: {uuid_string}")
result = tx.run(query, uuid_string=uuid_string)
record = result.single()
if record is None:
logging.warning(f"No node found with unique_id: {unique_id}")
logging.warning(f"No node found with uuid_string: {uuid_string}")
return None
return record[0]
modules/database/tools/neontology/basenode.py
+8 -4
View File
@@ -70,11 +70,10 @@ class BaseNode(CommonModel): # pyre-ignore[13]
def merge(self, database: str = 'neo4j') -> None:
"""Merge this node into the graph."""
params = self._get_merge_parameters()
all_labels = [self.__primarylabel__] + self.__secondarylabels__
cypher = f"""
MERGE (n:{":".join(all_labels)} {{ {self.__primaryproperty__}: $pp }})
ON MATCH SET n += $set_on_match
@@ -82,10 +81,15 @@ class BaseNode(CommonModel): # pyre-ignore[13]
SET n += $always_set
RETURN n
"""
print(f"DEBUG: Executing merge query: {cypher}")
print(f"DEBUG: With params: {params}")
print(f"DEBUG: Database: {database}")
graph = GraphConnection()
with graph.driver.session(database=database) as session:
result = session.run(cypher, params).single()
print(f"DEBUG: Merge result: {result}")
if result:
return self.__class__(**dict(result["n"]))
return None
modules/database/tools/neontology_tools.py
+8 -2
View File
@@ -60,14 +60,20 @@ def create_or_merge_neontology_node(node: BaseNode, database: str = 'neo4j', ope
operation (str): The operation to perform ('create' or 'merge'). Defaults to 'merge'.
"""
try:
logging.debug(f"Creating/merging node: {node.__class__.__name__} with label '{node.__primarylabel__}' in database '{database}'")
logging.debug(f"Node data: {node.to_dict()}")
if operation == "create":
node.create(database=database)
result = node.create(database=database)
logging.debug(f"Create result: {result}")
elif operation == "merge":
node.merge(database=database)
result = node.merge(database=database)
logging.debug(f"Merge result: {result}")
else:
logging.error(f"Invalid operation: {operation}")
except Exception as e:
logging.error(f"Error in processing node: {e}")
raise # Re-raise to see the actual error
# Create or merge a Neontology node in the Neo4j database. If a ValidationError occurs
# due to a NaN value, replace it with a default value and retry.
modules/database/tools/supabase_storage_tools.py
+451
View File
@@ -0,0 +1,451 @@
"""
Supabase Storage Tools for ClassroomCopilot
Replaces local filesystem paths with Supabase Storage bucket paths
"""
import os
from modules.logger_tool import initialise_logger
from typing import Tuple, Optional
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
class SupabaseStorageTools:
"""
Generates Supabase Storage paths for TLDraw snapshots and other files.
Path format: bucket/nodetype/node_unique_id
Example: cc.snapshots/User/fda8dca7-4d18-43c9-bb74-260777043447
"""
def __init__(self, db_name: str, init_run_type: str = None):
self.db_name = db_name
self.init_run_type = init_run_type
# Define bucket mappings based on node types
self.bucket_mappings = {
'User': 'cc.public.snapshots',
'Teacher': 'cc.public.snapshots',
'Student': 'cc.public.snapshots',
'School': 'cc.public.snapshots',
'Department': 'cc.public.snapshots',
'Subject': 'cc.public.snapshots',
'CalendarYear': 'cc.public.snapshots',
'CalendarMonth': 'cc.public.snapshots',
'CalendarWeek': 'cc.public.snapshots',
'CalendarDay': 'cc.public.snapshots',
'CalendarTimeChunk': 'cc.public.snapshots',
'KeyStage': 'cc.public.snapshots',
'YearGroup': 'cc.public.snapshots',
'KeyStageSyllabus': 'cc.public.snapshots',
'YearGroupSyllabus': 'cc.public.snapshots',
'Topic': 'cc.public.snapshots',
'TopicLesson': 'cc.public.snapshots',
'LearningStatement': 'cc.public.snapshots',
'UserTeacherTimetable': 'cc.public.snapshots',
'Class': 'cc.public.snapshots',
'TimetableLesson': 'cc.public.snapshots',
'SuperAdmin': 'cc.public.snapshots',
'Developer': 'cc.public.snapshots',
'CurriculumStructure': 'cc.public.snapshots',
'PastoralStructure': 'cc.public.snapshots',
'DepartmentStructure': 'cc.public.snapshots',
}
logger.info(f"Initializing SupabaseStorageTools with db_name: {db_name} and init_run_type: {init_run_type}")
def get_storage_path(self, node_type: str, node_id: str) -> str:
"""
Generate Supabase Storage path for a node.
Args:
node_type: The type of node (e.g., 'User', 'Teacher', 'School')
node_id: The unique identifier for the node
Returns:
str: Storage path in format bucket/nodetype/node_id
"""
bucket = self.bucket_mappings.get(node_type, 'cc.public.snapshots')
path = f"{bucket}/{node_type}/{node_id}"
logger.debug(f"Generated storage path for {node_type} {node_id}: {path}")
return path
def create_user_storage_path(self, user_id: str) -> Tuple[bool, str]:
"""
Create storage path for a user node.
Args:
user_id: The user's unique identifier
Returns:
Tuple[bool, str]: (success, storage_path)
"""
path = self.get_storage_path('User', user_id)
logger.info(f"Created user storage path: {path}")
return True, path
def create_teacher_storage_path(self, teacher_id: str) -> Tuple[bool, str]:
"""
Create storage path for a teacher node.
Args:
teacher_id: The teacher's unique identifier
Returns:
Tuple[bool, str]: (success, storage_path)
"""
path = self.get_storage_path('Teacher', teacher_id)
logger.info(f"Created teacher storage path: {path}")
return True, path
def create_student_storage_path(self, student_id: str) -> Tuple[bool, str]:
"""
Create storage path for a student node.
Args:
student_id: The student's unique identifier
Returns:
Tuple[bool, str]: (success, storage_path)
"""
path = self.get_storage_path('Student', student_id)
logger.info(f"Created student storage path: {path}")
return True, path
def create_school_storage_path(self, school_id: str) -> Tuple[bool, str]:
"""
Create storage path for a school node.
Args:
school_id: The school's unique identifier
Returns:
Tuple[bool, str]: (success, storage_path)
"""
path = self.get_storage_path('School', school_id)
logger.info(f"Created school storage path: {path}")
return True, path
def create_calendar_year_storage_path(self, year: int) -> Tuple[bool, str]:
"""
Create storage path for a calendar year node.
Args:
year: The year
Returns:
Tuple[bool, str]: (success, storage_path)
"""
path = self.get_storage_path('CalendarYear', str(year))
logger.info(f"Created calendar year storage path: {path}")
return True, path
def create_calendar_month_storage_path(self, year: int, month: int) -> Tuple[bool, str]:
"""
Create storage path for a calendar month node.
Args:
year: The year
month: The month
Returns:
Tuple[bool, str]: (success, storage_path)
"""
month_id = f"{year}_{month:02d}"
path = self.get_storage_path('CalendarMonth', month_id)
logger.info(f"Created calendar month storage path: {path}")
return True, path
def create_calendar_week_storage_path(self, year: int, week: int) -> Tuple[bool, str]:
"""
Create storage path for a calendar week node.
Args:
year: The ISO year
week: The ISO week number
Returns:
Tuple[bool, str]: (success, storage_path)
"""
week_id = f"{year}_{week:02d}"
path = self.get_storage_path('CalendarWeek', week_id)
logger.info(f"Created calendar week storage path: {path}")
return True, path
def create_calendar_day_storage_path(self, year: int, month: int, day: int) -> Tuple[bool, str]:
"""
Create storage path for a calendar day node.
Args:
year: The year
month: The month
day: The day
Returns:
Tuple[bool, str]: (success, storage_path)
"""
day_id = f"{year}_{month:02d}_{day:02d}"
path = self.get_storage_path('CalendarDay', day_id)
logger.info(f"Created calendar day storage path: {path}")
return True, path
def create_calendar_time_chunk_storage_path(self, day_id: str, chunk_index: int) -> Tuple[bool, str]:
"""
Create storage path for a calendar time chunk node.
Args:
day_id: The day identifier (e.g., "2025_01_15")
chunk_index: The time chunk index within the day
Returns:
Tuple[bool, str]: (success, storage_path)
"""
chunk_id = f"{day_id}_{chunk_index:02d}"
path = self.get_storage_path('CalendarTimeChunk', chunk_id)
logger.info(f"Created calendar time chunk storage path: {path}")
return True, path
def create_department_storage_path(self, department_id: str) -> Tuple[bool, str]:
"""
Create storage path for a department node.
Args:
department_id: The department's unique identifier
Returns:
Tuple[bool, str]: (success, storage_path)
"""
path = self.get_storage_path('Department', department_id)
logger.info(f"Created department storage path: {path}")
return True, path
def create_subject_storage_path(self, subject_id: str) -> Tuple[bool, str]:
"""
Create storage path for a subject node.
Args:
subject_id: The subject's unique identifier
Returns:
Tuple[bool, str]: (success, storage_path)
"""
path = self.get_storage_path('Subject', subject_id)
logger.info(f"Created subject storage path: {path}")
return True, path
def create_key_stage_storage_path(self, key_stage_id: str) -> Tuple[bool, str]:
"""
Create storage path for a key stage node.
Args:
key_stage_id: The key stage's unique identifier
Returns:
Tuple[bool, str]: (success, storage_path)
"""
path = self.get_storage_path('KeyStage', key_stage_id)
logger.info(f"Created key stage storage path: {path}")
return True, path
def create_year_group_storage_path(self, year_group_id: str) -> Tuple[bool, str]:
"""
Create storage path for a year group node.
Args:
year_group_id: The year group's unique identifier
Returns:
Tuple[bool, str]: (success, storage_path)
"""
path = self.get_storage_path('YearGroup', year_group_id)
logger.info(f"Created year group storage path: {path}")
return True, path
def create_key_stage_syllabus_storage_path(self, syllabus_id: str) -> Tuple[bool, str]:
"""
Create storage path for a key stage syllabus node.
Args:
syllabus_id: The syllabus's unique identifier
Returns:
Tuple[bool, str]: (success, storage_path)
"""
path = self.get_storage_path('KeyStageSyllabus', syllabus_id)
logger.info(f"Created key stage syllabus storage path: {path}")
return True, path
def create_year_group_syllabus_storage_path(self, syllabus_id: str) -> Tuple[bool, str]:
"""
Create storage path for a year group syllabus node.
Args:
syllabus_id: The syllabus's unique identifier
Returns:
Tuple[bool, str]: (success, storage_path)
"""
path = self.get_storage_path('YearGroupSyllabus', syllabus_id)
logger.info(f"Created year group syllabus storage path: {path}")
return True, path
def create_topic_storage_path(self, topic_id: str) -> Tuple[bool, str]:
"""
Create storage path for a topic node.
Args:
topic_id: The topic's unique identifier
Returns:
Tuple[bool, str]: (success, storage_path)
"""
path = self.get_storage_path('Topic', topic_id)
logger.info(f"Created topic storage path: {path}")
return True, path
def create_topic_lesson_storage_path(self, lesson_id: str) -> Tuple[bool, str]:
"""
Create storage path for a topic lesson node.
Args:
lesson_id: The lesson's unique identifier
Returns:
Tuple[bool, str]: (success, storage_path)
"""
path = self.get_storage_path('TopicLesson', lesson_id)
logger.info(f"Created topic lesson storage path: {path}")
return True, path
def create_learning_statement_storage_path(self, statement_id: str) -> Tuple[bool, str]:
"""
Create storage path for a learning statement node.
Args:
statement_id: The statement's unique identifier
Returns:
Tuple[bool, str]: (success, storage_path)
"""
path = self.get_storage_path('LearningStatement', statement_id)
logger.info(f"Created learning statement storage path: {path}")
return True, path
def create_timetable_storage_path(self, timetable_id: str) -> Tuple[bool, str]:
"""
Create storage path for a timetable node.
Args:
timetable_id: The timetable's unique identifier
Returns:
Tuple[bool, str]: (success, storage_path)
"""
path = self.get_storage_path('UserTeacherTimetable', timetable_id)
logger.info(f"Created timetable storage path: {path}")
return True, path
def create_class_storage_path(self, class_id: str) -> Tuple[bool, str]:
"""
Create storage path for a class node.
Args:
class_id: The class's unique identifier
Returns:
Tuple[bool, str]: (success, storage_path)
"""
path = self.get_storage_path('Class', class_id)
logger.info(f"Created class storage path: {path}")
return True, path
def create_timetable_lesson_storage_path(self, lesson_id: str) -> Tuple[bool, str]:
"""
Create storage path for a timetable lesson node.
Args:
lesson_id: The lesson's unique identifier
Returns:
Tuple[bool, str]: (success, storage_path)
"""
path = self.get_storage_path('TimetableLesson', lesson_id)
logger.info(f"Created timetable lesson storage path: {path}")
return True, path
def create_super_admin_storage_path(self, admin_id: str) -> Tuple[bool, str]:
"""
Create storage path for a super admin node.
Args:
admin_id: The admin's unique identifier
Returns:
Tuple[bool, str]: (success, storage_path)
"""
path = self.get_storage_path('SuperAdmin', admin_id)
logger.info(f"Created super admin storage path: {path}")
return True, path
def create_curriculum_storage_path(self, curriculum_id: str) -> Tuple[bool, str]:
"""
Create storage path for a curriculum structure node.
Args:
curriculum_id: The curriculum's unique identifier
Returns:
Tuple[bool, str]: (success, storage_path)
"""
path = self.get_storage_path('CurriculumStructure', curriculum_id)
logger.info(f"Created curriculum structure storage path: {path}")
return True, path
def create_pastoral_storage_path(self, pastoral_id: str) -> Tuple[bool, str]:
"""
Create storage path for a pastoral structure node.
Args:
pastoral_id: The pastoral's unique identifier
Returns:
Tuple[bool, str]: (success, storage_path)
"""
path = self.get_storage_path('PastoralStructure', pastoral_id)
logger.info(f"Created pastoral structure storage path: {path}")
return True, path
# Legacy compatibility methods that return the same interface as filesystem tools
def create_private_user_directory(self, user_id: str) -> Tuple[bool, str]:
"""Legacy compatibility method."""
return self.create_user_storage_path(user_id)
def create_user_worker_directory(self, user_path: str, worker_id: str, worker_type: str) -> Tuple[bool, str]:
"""Legacy compatibility method."""
if worker_type in ['teacher', 'email_teacher', 'ms_teacher']:
return self.create_teacher_storage_path(worker_id)
elif worker_type in ['student', 'email_student', 'ms_student']:
return self.create_student_storage_path(worker_id)
elif worker_type == 'superadmin':
return self.create_super_admin_storage_path(worker_id)
elif worker_type == 'developer':
return self.create_developer_storage_path(worker_id)
else:
# Default to generic storage path
path = self.get_storage_path(worker_type.title(), worker_id)
return True, path
def create_school_directory(self, school_uuid_string: str) -> Tuple[bool, str]:
"""Legacy compatibility method."""
return self.create_school_storage_path(school_uuid_string)
def create_school_curriculum_directory(self, school_path: Optional[str] = None) -> Tuple[bool, str]:
"""Legacy compatibility method - returns empty path since curriculum is handled by individual nodes."""
return True, ""
def create_school_pastoral_directory(self, school_path: Optional[str] = None) -> Tuple[bool, str]:
"""Legacy compatibility method - returns empty path since pastoral is handled by individual nodes."""
return True, ""
def create_directory(self, path: str) -> bool:
"""Legacy compatibility method - always returns True since we don't create physical directories."""
return True