ClassroomCopilot/api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

latest

Browse Source
This commit is contained in:
kcar
2025-11-14 14:47:19 +00:00
parent 2a85845835
commit 3758c7572a
137 changed files with 365654 additions and 11147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/database/init/init_calendar.py
+85 -84
View File
@@ -4,12 +4,25 @@ logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH
import modules.database.schemas.nodes.calendars as calendar_schemas
import modules.database.schemas.relationships.calendars as calendar_relationships
import modules.database.schemas.relationships.calendar_sequence as calendar_sequence_relationships
import modules.database.schemas.relationships.owner_relationships as owner_relationships
import modules.database.tools.supabase_storage_tools as storage_tools
import modules.database.tools.neontology_tools as neon
from datetime import timedelta, datetime
def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False, owner_node=None, time_chunk_node_length: int = None):
logger.info(f"Creating calendar for {start_date} to {end_date}")
def create_calendar(db_name, start_date, end_date, time_chunk_node_length: int = None, storage_tools=None):
"""
Create calendar structure with years, months, weeks, and days
Args:
db_name: Database name to create calendar in
start_date: Start date for calendar
end_date: End date for calendar
time_chunk_node_length: Optional time chunk length in minutes
storage_tools: Optional Supabase storage tools for generating storage paths
Returns:
dict: Dictionary containing created calendar nodes
"""
logger.info(f"Creating calendar structure for {start_date} to {end_date} in database: {db_name}")
logger.info(f"Initializing Neontology connection")
neon.init_neontology_connection()
@@ -25,52 +38,13 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
last_day_node = None
calendar_nodes = {
'calendar_node': None,
'calendar_year_nodes': [],
'calendar_month_nodes': [],
'calendar_week_nodes': [],
'calendar_day_nodes': []
'calendar_day_nodes': [],
'calendar_time_chunk_nodes': []
}
if attach_to_calendar_node and owner_node:
logger.info(f"Attaching calendar to owner's node {owner_node.unique_id} in database: {db_name}")
owner_unique_id = owner_node.unique_id
calendar_unique_id = f"{start_date.strftime('%Y-%m-%d')}_{end_date.strftime('%Y-%m-%d')}"
calendar_node = calendar_schemas.CalendarNode(
unique_id=calendar_unique_id,
name=f"{start_date.strftime('%Y-%m-%d')} to {end_date.strftime('%Y-%m-%d')}",
start_date=start_date,
end_date=end_date,
tldraw_snapshot=""
)
neon.create_or_merge_neontology_node(calendar_node, database=db_name, operation='merge')
calendar_nodes['calendar_node'] = calendar_node
logger.info(f"Calendar node created: {calendar_node.unique_id}")
import modules.database.schemas.relationships.owner_relationships as owner_relationships
neon.create_or_merge_neontology_relationship(
owner_relationships.OwnerHasCalendar(source=owner_node, target=calendar_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {owner_node.unique_id} to {calendar_node.unique_id}")
elif attach_to_calendar_node and not owner_node:
logger.info(f"Creating calendar for {start_date} to {end_date} in database: {db_name}")
calendar_node = calendar_schemas.CalendarNode(
unique_id=f"{start_date.strftime('%Y-%m-%d')}_{end_date.strftime('%Y-%m-%d')}",
name=f"{start_date.strftime('%Y-%m-%d')} to {end_date.strftime('%Y-%m-%d')}",
start_date=start_date,
end_date=end_date,
tldraw_snapshot=""
)
neon.create_or_merge_neontology_node(calendar_node, database=db_name, operation='merge')
calendar_nodes['calendar_node'] = calendar_node
logger.info(f"Calendar node created: {calendar_node.unique_id}")
else:
logger.error("Invalid combination of parameters for calendar creation.")
raise ValueError("Invalid combination of parameters for calendar creation.")
current_date = start_date
while current_date <= end_date:
year = current_date.year
@@ -78,50 +52,55 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
day = current_date.day
iso_year, iso_week, iso_weekday = current_date.isocalendar()
calendar_year_unique_id = f"{year}"
calendar_year_uuid_string = f"{year}"
if year not in created_years:
# Generate storage path for year node using Supabase Storage
if storage_tools:
year_dir_created, node_storage_path = storage_tools.create_calendar_year_storage_path(year)
else:
node_storage_path = ""
year_node = calendar_schemas.CalendarYearNode(
unique_id=calendar_year_unique_id,
uuid_string=calendar_year_uuid_string,
year=str(year),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(year_node, database=db_name, operation='merge')
calendar_nodes['calendar_year_nodes'].append(year_node)
created_years[year] = year_node
logger.info(f"Year node created: {year_node.unique_id}")
logger.info(f"Year node created: {year_node.uuid_string}")
if attach_to_calendar_node:
neon.create_or_merge_neontology_relationship(
calendar_relationships.CalendarIncludesYear(source=calendar_node, target=year_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {calendar_node.unique_id} to {year_node.unique_id}")
if last_year_node:
neon.create_or_merge_neontology_relationship(
calendar_sequence_relationships.YearFollowsYear(source=last_year_node, target=year_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {last_year_node.unique_id} to {year_node.unique_id}")
logger.info(f"Relationship created from {last_year_node.uuid_string} to {year_node.uuid_string}")
last_year_node = year_node
calendar_month_unique_id = f"{year}_{month}"
calendar_month_uuid_string = f"{year}_{month}"
month_key = f"{year}-{month}"
if month_key not in created_months:
# Generate storage path for month node using Supabase Storage
if storage_tools:
month_dir_created, node_storage_path = storage_tools.create_calendar_month_storage_path(year, month)
else:
node_storage_path = ""
month_node = calendar_schemas.CalendarMonthNode(
unique_id=calendar_month_unique_id,
uuid_string=calendar_month_uuid_string,
year=str(year),
month=str(month),
month_name=datetime(year, month, 1).strftime('%B'),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(month_node, database=db_name, operation='merge')
calendar_nodes['calendar_month_nodes'].append(month_node)
created_months[month_key] = month_node
logger.info(f"Month node created: {month_node.unique_id}")
logger.info(f"Month node created: {month_node.uuid_string}")
# Check for the end of year transition for months
if last_month_node:
@@ -131,14 +110,14 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {last_month_node.unique_id} to {month_node.unique_id}")
logger.info(f"Relationship created from {last_month_node.uuid_string} to {month_node.uuid_string}")
elif int(month) == int(last_month_node.month) + 1:
neon.create_or_merge_neontology_relationship(
calendar_sequence_relationships.MonthFollowsMonth(source=last_month_node, target=month_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {last_month_node.unique_id} to {month_node.unique_id}")
logger.info(f"Relationship created from {last_month_node.uuid_string} to {month_node.uuid_string}")
last_month_node = month_node
neon.create_or_merge_neontology_relationship(
@@ -146,25 +125,31 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {year_node.unique_id} to {month_node.unique_id}")
logger.info(f"Relationship created from {year_node.uuid_string} to {month_node.uuid_string}")
calendar_week_unique_id = f"{iso_year}_{iso_week}"
calendar_week_uuid_string = f"{iso_year}_{iso_week}"
week_key = f"{iso_year}-W{iso_week}"
if week_key not in created_weeks:
# Get the date of the first monday of the week
week_start_date = current_date - timedelta(days=current_date.weekday())
# Generate storage path for week node using Supabase Storage
if storage_tools:
week_dir_created, node_storage_path = storage_tools.create_calendar_week_storage_path(iso_year, iso_week)
else:
node_storage_path = ""
week_node = calendar_schemas.CalendarWeekNode(
unique_id=calendar_week_unique_id,
uuid_string=calendar_week_uuid_string,
start_date=week_start_date,
week_number=str(iso_week),
iso_week=f"{iso_year}-W{iso_week:02}",
tldraw_snapshot=""
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(week_node, database=db_name, operation='merge')
calendar_nodes['calendar_week_nodes'].append(week_node)
created_weeks[week_key] = week_node
logger.info(f"Week node created: {week_node.unique_id}")
logger.info(f"Week node created: {week_node.uuid_string}")
if last_week_node and ((last_week_node.iso_week.split('-')[0] == str(iso_year) and int(last_week_node.week_number) == int(iso_week) - 1) or
(last_week_node.iso_week.split('-')[0] != str(iso_year) and int(last_week_node.week_number) == 52 and int(iso_week) == 1)):
@@ -173,7 +158,7 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {last_week_node.unique_id} to {week_node.unique_id}")
logger.info(f"Relationship created from {last_week_node.uuid_string} to {week_node.uuid_string}")
last_week_node = week_node
neon.create_or_merge_neontology_relationship(
@@ -181,23 +166,32 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {year_node.unique_id} to {week_node.unique_id}")
logger.info(f"Relationship created from {year_node.uuid_string} to {week_node.uuid_string}")
# Day node management
calendar_day_unique_id = f"{year}_{month}_{day}"
calendar_day_uuid_string = f"{year}_{month}_{day}"
day_key = f"{year}-{month}-{day}"
# Generate storage path for day node using Supabase Storage
if storage_tools:
day_dir_created, node_storage_path = storage_tools.create_calendar_day_storage_path(year, month, day)
# Store day path for later use in time chunks
created_days[day_key] = {'node': None, 'path': node_storage_path}
else:
node_storage_path = ""
created_days[day_key] = {'node': None, 'path': None}
day_node = calendar_schemas.CalendarDayNode(
unique_id=calendar_day_unique_id,
uuid_string=calendar_day_uuid_string,
date=current_date,
day_of_week=current_date.strftime('%A'),
iso_day=f"{year}-{month:02}-{day:02}",
tldraw_snapshot=""
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(day_node, database=db_name, operation='merge')
calendar_nodes['calendar_day_nodes'].append(day_node)
created_days[day_key] = day_node
logger.info(f"Day node created: {day_node.unique_id}")
created_days[day_key]['node'] = day_node
logger.info(f"Day node created: {day_node.uuid_string}")
if last_day_node:
neon.create_or_merge_neontology_relationship(
@@ -205,7 +199,7 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {last_day_node.unique_id} to {day_node.unique_id}")
logger.info(f"Relationship created from {last_day_node.uuid_string} to {day_node.uuid_string}")
last_day_node = day_node
neon.create_or_merge_neontology_relationship(
@@ -213,13 +207,13 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {month_node.unique_id} to {day_node.unique_id}")
logger.info(f"Relationship created from {month_node.uuid_string} to {day_node.uuid_string}")
neon.create_or_merge_neontology_relationship(
calendar_relationships.WeekIncludesDay(source=week_node, target=day_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {week_node.unique_id} to {day_node.unique_id}")
logger.info(f"Relationship created from {week_node.uuid_string} to {day_node.uuid_string}")
current_date += timedelta(days=1)
if time_chunk_node_length:
@@ -228,25 +222,32 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
for day_node in calendar_nodes['calendar_day_nodes']:
total_time_chunks_in_day = (24 * 60) / time_chunk_interval
for i in range(total_time_chunks_in_day):
time_chunk_unique_id = f"{day_node.unique_id}_{i}"
time_chunk_uuid_string = f"{day_node.uuid_string}_{i}"
time_chunk_start_time = day_node.date.time() + timedelta(minutes=i * time_chunk_interval)
time_chunk_end_time = time_chunk_start_time + timedelta(minutes=time_chunk_interval)
# Generate storage path for time chunk node using Supabase Storage
if storage_tools:
chunk_id = f"{day_node.uuid_string}_{i:02d}"
chunk_dir_created, node_storage_path = storage_tools.create_calendar_time_chunk_storage_path(day_node.uuid_string, i)
else:
node_storage_path = ""
time_chunk_node = calendar_schemas.CalendarTimeChunkNode(
unique_id=time_chunk_unique_id,
uuid_string=time_chunk_uuid_string,
start_time=time_chunk_start_time,
end_time=time_chunk_end_time,
tldraw_snapshot=""
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(time_chunk_node, database=db_name, operation='merge')
calendar_nodes['calendar_time_chunk_nodes'].append(time_chunk_node)
logger.info(f"Time chunk node created: {time_chunk_node.unique_id}")
logger.info(f"Time chunk node created: {time_chunk_node.uuid_string}")
# Create a relationship between the time chunk node and the day node
neon.create_or_merge_neontology_relationship(
calendar_relationships.DayIncludesTimeChunk(source=day_node, target=time_chunk_node),
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {day_node.unique_id} to {time_chunk_node.unique_id}")
logger.info(f"Relationship created from {day_node.uuid_string} to {time_chunk_node.uuid_string}")
# Create sequential relationship between the time chunk nodes
if i > 0:
neon.create_or_merge_neontology_relationship(
@@ -254,7 +255,7 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
database=db_name,
operation='merge'
)
logger.info(f"Relationship created from {calendar_nodes['calendar_time_chunk_nodes'][i-1].unique_id} to {time_chunk_node.unique_id}")
logger.info(f"Relationship created from {calendar_nodes['calendar_time_chunk_nodes'][i-1].uuid_string} to {time_chunk_node.uuid_string}")
logger.info(f'Created calendar: {calendar_nodes["calendar_node"].unique_id}')
logger.info(f'Calendar structure created successfully for {start_date} to {end_date}')
return calendar_nodes
modules/database/init/init_school.py
+24 -105
View File
@@ -2,115 +2,31 @@ import os
from modules.logger_tool import initialise_logger
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
from modules.database.schemas.nodes.schools.schools import SchoolNode
from modules.database.supabase.utils.client import SupabaseServiceRoleClient, CreateBucketOptions
import modules.database.init.init_school_timetable as init_school_timetable
import modules.database.tools.neontology_tools as neon
import modules.database.tools.supabase_storage_tools as storage_tools
def create_school_buckets(school_id: str, school_type: str, school_name: str, admin_access_token: str) -> dict:
"""Create storage buckets for a school
Args:
school_id: The unique identifier for the school
school_type: The type of school (e.g., 'development')
school_name: The display name of the school
admin_access_token: The admin access token for Supabase operations
Returns:
Dictionary containing results of bucket creation operations
"""
logger.info(f"Creating storage buckets for school {school_name} ({school_type}/{school_id})")
def create_school(db_name: str, uuid_string: str, name: str, website: str, school_type: str, is_public: bool = True, school_node: SchoolNode | None = None, dataframes=None):
if not name or not uuid_string or not website or not school_type:
logger.error("School name, uuid_string, website and school_type are required to create a school.")
raise ValueError("School name, uuid_string, website and school_type are required to create a school.")
storage_client = SupabaseServiceRoleClient.for_admin(admin_access_token)
base_path = f"cc.institutes.{school_type}.{school_id}"
buckets = [
# Main school buckets
{
"id": f"{base_path}.public",
"options": CreateBucketOptions(
name=f"{school_type.title()} School Files - {school_name} - Public Files",
public=True,
file_size_limit=50 * 1024 * 1024,
allowed_mime_types=[
'image/*', 'video/*', 'application/pdf',
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
'text/plain', 'text/csv', 'application/json'
]
)
},
{
"id": f"{base_path}.private",
"options": CreateBucketOptions(
name=f"{school_type.title()} School Files - {school_name} - Private Files",
public=False,
file_size_limit=50 * 1024 * 1024,
allowed_mime_types=[
'image/*', 'video/*', 'application/pdf',
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
'text/plain', 'text/csv', 'application/json'
]
)
},
# Curriculum buckets
{
"id": f"{base_path}.curriculum.public",
"options": CreateBucketOptions(
name=f"{school_type.title()} School Files - {school_name} - Curriculum Public Files",
public=True,
file_size_limit=50 * 1024 * 1024,
allowed_mime_types=[
'image/*', 'video/*', 'application/pdf',
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
'text/plain', 'text/csv', 'application/json'
]
)
},
{
"id": f"{base_path}.curriculum.private",
"options": CreateBucketOptions(
name=f"{school_type.title()} School Files - {school_name} - Curriculum Private Files",
public=False,
file_size_limit=50 * 1024 * 1024,
allowed_mime_types=[
'image/*', 'video/*', 'application/pdf',
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
'text/plain', 'text/csv', 'application/json'
]
)
}
]
results = {}
for bucket in buckets:
try:
result = storage_client.create_bucket(bucket["id"], bucket["options"])
results[bucket["id"]] = {
"status": "success",
"result": result
}
logger.info(f"Successfully created bucket {bucket['id']}")
except Exception as e:
logger.error(f"Error creating school bucket {bucket['id']}: {str(e)}")
results[bucket["id"]] = {
"status": "error",
"error": str(e)
}
return results
def create_school(db_name: str, id: str, name: str, website: str, school_type: str, is_public: bool = True, school_node: SchoolNode | None = None, dataframes=None):
if not name or not id or not website or not school_type:
logger.error("School name, id, website and school_type are required to create a school.")
raise ValueError("School name, id, website and school_type are required to create a school.")
logger.info(f"Initialising neo4j connection...")
logger.info(f"Initialising Neontology connection...")
neon.init_neontology_connection()
# Initialize storage tools for school
storage_tools_instance = storage_tools.SupabaseStorageTools(db_name, init_run_type="school")
# Generate the storage path for the school node
school_dir_created, node_storage_path = storage_tools_instance.create_school_storage_path(uuid_string)
logger.info(f"Generated school storage path: {node_storage_path}")
# Create School Node if not provided
if not school_node:
if is_public:
school_node = SchoolNode(
unique_id=f'School_{id}',
tldraw_snapshot="",
id=id,
uuid_string=uuid_string,
node_storage_path=node_storage_path,
name=name,
website=website,
school_type=school_type
@@ -118,9 +34,8 @@ def create_school(db_name: str, id: str, name: str, website: str, school_type: s
else:
# Create private school node with default values
school_node = SchoolNode(
unique_id=f'School_{id}',
tldraw_snapshot="",
id=id,
uuid_string=uuid_string,
node_storage_path=node_storage_path,
name=name,
website=website,
school_type=school_type,
@@ -133,6 +48,9 @@ def create_school(db_name: str, id: str, name: str, website: str, school_type: s
statutory_high_age=18,
school_capacity=1000
)
else:
# Update existing school node with the storage path
school_node.node_storage_path = node_storage_path
# First create/merge the school node in the main cc.institutes database
logger.info(f"Creating school node in main cc.institutes database...")
@@ -144,15 +62,16 @@ def create_school(db_name: str, id: str, name: str, website: str, school_type: s
school_nodes = {
'school_node': school_node,
'db_name': db_name
'db_name': db_name,
'storage_path': node_storage_path
}
if dataframes is not None:
logger.info(f"Creating school timetable for {name} with {len(dataframes)} dataframes...")
school_timetable_nodes = init_school_timetable.create_school_timetable(dataframes, db_name, school_node)
school_timetable_nodes = init_school_timetable.create_school_timetable(dataframes, db_name, school_node, storage_tools_instance)
school_nodes['school_timetable_nodes'] = school_timetable_nodes
else:
logger.warning(f"No dataframes provided for {name}, skipping school timetable...")
logger.info(f"School {name} created successfully...")
logger.info(f"School {name} created successfully with storage path: {node_storage_path}")
return school_nodes
modules/database/init/init_school_curriculum.py
+206 -73
View File
@@ -4,6 +4,7 @@ logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH
import pandas as pd
import modules.database.tools.neontology_tools as neon
import modules.database.tools.supabase_storage_tools as storage_tools
import modules.database.schemas.nodes.schools.schools as school_nodes
import modules.database.schemas.nodes.schools.curriculum as curriculum_nodes
import modules.database.schemas.nodes.schools.pastoral as pastoral_nodes
@@ -40,7 +41,7 @@ def sort_year_groups(df):
df['YearGroupNumeric'] = pd.to_numeric(df['YearGroup'], errors='coerce')
return df.sort_values(by='YearGroupNumeric')
def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_node: school_nodes.SchoolNode):
def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_node: school_nodes.SchoolNode, storage_tools=None):
logger.info(f"Initialising neo4j connection...")
neon.init_neontology_connection()
@@ -70,10 +71,20 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
last_key_stage_node = None
# Create Department Structure node
department_structure_node_unique_id = f"DepartmentStructure_{school_node.unique_id}"
department_structure_node_uuid_string = f"DepartmentStructure_{school_node.uuid_string}"
# For structure nodes, we can use a simple path or leave empty for consistency
# Since this is just an organizational node, we'll use a simple path
if storage_tools:
# Use the school's base department directory as the structure node path
dept_structure_path = f"cc.public.snapshots/DepartmentStructure/{school_node.uuid_string}"
node_storage_path = dept_structure_path
else:
node_storage_path = ""
department_structure_node = school_structures.DepartmentStructureNode(
unique_id=department_structure_node_unique_id,
tldraw_snapshot=""
uuid_string=department_structure_node_uuid_string,
node_storage_path=node_storage_path
)
# Create in school database only
neon.create_or_merge_neontology_node(department_structure_node, database=db_name, operation='merge')
@@ -86,10 +97,17 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
)
logger.info(f"Created department structure node and linked to school")
curriculum_structure_node_unique_id = f"CurriculumStructure_{school_node.unique_id}"
curriculum_structure_node_uuid_string = f"CurriculumStructure_{school_node.uuid_string}"
# Generate storage path for curriculum structure node
if storage_tools:
curriculum_dir_created, node_storage_path = storage_tools.create_curriculum_storage_path(curriculum_structure_node_uuid_string)
else:
node_storage_path = ""
curriculum_node = school_structures.CurriculumStructureNode(
unique_id=curriculum_structure_node_unique_id,
tldraw_snapshot=""
uuid_string=curriculum_structure_node_uuid_string,
node_storage_path=node_storage_path
)
# Create in school database only
neon.create_or_merge_neontology_node(curriculum_node, database=db_name, operation='merge')
@@ -102,10 +120,17 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
)
logger.info(f"Created curriculum node and relationship with school")
pastoral_structure_node_unique_id = f"PastoralStructure_{school_node.unique_id}"
pastoral_structure_node_uuid_string = f"PastoralStructure_{school_node.uuid_string}"
# Generate storage path for pastoral structure node
if storage_tools:
pastoral_dir_created, node_storage_path = storage_tools.create_pastoral_storage_path(pastoral_structure_node_uuid_string)
else:
node_storage_path = ""
pastoral_node = school_structures.PastoralStructureNode(
unique_id=pastoral_structure_node_unique_id,
tldraw_snapshot=""
uuid_string=pastoral_structure_node_uuid_string,
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(pastoral_node, database=db_name, operation='merge')
node_library['pastoral_node'] = pastoral_node
@@ -120,11 +145,20 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
unique_departments = keystagesyllabus_df['Department'].dropna().unique()
for department_name in unique_departments:
department_unique_id = f"Department_{school_node.unique_id}_{department_name.replace(' ', '_')}"
department_uuid_string = f"Department_{school_node.uuid_string}_{department_name.replace(' ', '_')}"
# Generate storage path for department node using the storage tools
if storage_tools:
# Create department directory under the school's department structure
dept_path = f"cc.public.snapshots/Department/{department_uuid_string}"
node_storage_path = dept_path
else:
node_storage_path = ""
department_node = school_nodes.DepartmentNode(
unique_id=department_unique_id,
uuid_string=department_uuid_string,
name=department_name,
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create department in school database only
neon.create_or_merge_neontology_node(department_node, database=db_name, operation='merge')
@@ -147,17 +181,25 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
# Process subjects from key stage syllabuses first (these have department info)
for _, subject_row in unique_subjects.iterrows():
subject_unique_id = f"Subject_{school_node.unique_id}_{subject_row['SubjectCode']}"
subject_uuid_string = f"Subject_{school_node.uuid_string}_{subject_row['SubjectCode']}"
department_node = node_library['department_nodes'].get(subject_row['Department'])
if not department_node:
logger.warning(f"No department found for subject {subject_row['Subject']} with code {subject_row['SubjectCode']}")
continue
# Generate storage path for subject node
if storage_tools:
# Create subject directory under the specific department
subject_path = f"cc.public.snapshots/Subject/{subject_uuid_string}"
node_storage_path = subject_path
else:
node_storage_path = ""
subject_node = curriculum_nodes.SubjectNode(
unique_id=subject_unique_id,
uuid_string=subject_uuid_string,
id=subject_row['SubjectCode'],
name=subject_row['Subject'],
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create subject in both databases
neon.create_or_merge_neontology_node(subject_node, database=db_name, operation='merge')
@@ -173,14 +215,23 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
# Process any additional subjects from year group syllabuses (these won't have department info)
for _, subject_row in additional_subjects.iterrows():
subject_unique_id = f"Subject_{school_node.unique_id}_{subject_row['SubjectCode']}"
subject_uuid_string = f"Subject_{school_node.uuid_string}_{subject_row['SubjectCode']}"
# Create in a special "Unassigned" department
unassigned_dept_name = "Unassigned Department"
if unassigned_dept_name not in node_library['department_nodes']:
# Generate storage path for unassigned department node
if filesystem:
# Create unassigned department directory under the school's department structure
unassigned_dept_path = os.path.join(filesystem.root_path, "departments", "Unassigned")
filesystem.create_directory(unassigned_dept_path)
node_storage_path = os.path.relpath(unassigned_dept_path, filesystem.base_path)
else:
node_storage_path = ""
department_node = school_nodes.DepartmentNode(
unique_id=f"Department_{school_node.unique_id}_Unassigned",
uuid_string=f"Department_{school_node.uuid_string}_Unassigned",
name=unassigned_dept_name,
tldraw_snapshot=""
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(department_node, database=db_name, operation='merge')
node_library['department_nodes'][unassigned_dept_name] = department_node
@@ -192,11 +243,20 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
)
logger.info(f"Created unassigned department node and linked to department structure")
# Generate storage path for subject node
if filesystem:
# Create subject directory under the unassigned department
subject_path = os.path.join(filesystem.root_path, "departments", "Unassigned", subject_row['Subject'].replace(' ', '_'))
filesystem.create_directory(subject_path)
node_storage_path = os.path.relpath(subject_path, filesystem.base_path)
else:
node_storage_path = ""
subject_node = curriculum_nodes.SubjectNode(
unique_id=subject_unique_id,
uuid_string=subject_uuid_string,
id=subject_row['SubjectCode'],
name=subject_row['Subject'],
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create subject in both databases
neon.create_or_merge_neontology_node(subject_node, database=db_name, operation='merge')
@@ -226,20 +286,29 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
# First create all key stage nodes and key stage syllabus nodes
for index, ks_row in keystagesyllabus_df.sort_values('KeyStage').iterrows():
key_stage = str(ks_row['KeyStage'])
logger.debug(f"Processing key stage syllabus row - Subject: {ks_row['Subject']}, Key Stage: {key_stage}")
subject = str(ks_row['Subject'])
syllabus_id = str(ks_row['ID'])
logger.debug(f"Processing key stage syllabus row - Subject: {subject}, Key Stage: {key_stage}, Syllabus ID: {syllabus_id}")
subject_node = node_library['subject_nodes'].get(ks_row['Subject'])
subject_node = node_library['subject_nodes'].get(subject)
if not subject_node:
logger.warning(f"No subject node found for subject {ks_row['Subject']}")
logger.warning(f"No subject node found for subject {subject}")
continue
if key_stage not in key_stage_nodes_created:
key_stage_node_unique_id = f"KeyStage_{curriculum_node.unique_id}_KStg{key_stage}"
key_stage_node_uuid_string = f"KeyStage_{curriculum_node.uuid_string}_KStg{key_stage}"
# Generate storage path for key stage node
if filesystem:
key_stage_dir_created, key_stage_path = filesystem.create_curriculum_key_stage_syllabus_directory(curriculum_path, key_stage, subject, syllabus_id)
node_storage_path = os.path.relpath(key_stage_path, filesystem.base_path)
else:
node_storage_path = ""
key_stage_node = curriculum_nodes.KeyStageNode(
unique_id=key_stage_node_unique_id,
uuid_string=key_stage_node_uuid_string,
name=f"Key Stage {key_stage}",
key_stage=str(key_stage),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create key stage node in both databases
neon.create_or_merge_neontology_node(key_stage_node, database=db_name, operation='merge')
@@ -252,7 +321,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.CurriculumStructureIncludesKeyStage(source=curriculum_node, target=key_stage_node),
database=db_name, operation='merge'
)
logger.info(f"Created key stage node {key_stage_node_unique_id} and relationship with curriculum structure")
logger.info(f"Created key stage node {key_stage_node_uuid_string} and relationship with curriculum structure")
# Create sequential relationship between key stages in both databases
if last_key_stage_node:
@@ -264,27 +333,34 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.KeyStageFollowsKeyStage(source=last_key_stage_node, target=key_stage_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between key stages {last_key_stage_node.unique_id} and {key_stage_node.unique_id}")
logger.info(f"Created sequential relationship between key stages {last_key_stage_node.uuid_string} and {key_stage_node.uuid_string}")
last_key_stage_node = key_stage_node
# Create key stage syllabus under the subject's curriculum directory
key_stage_syllabus_node_unique_id = f"KeyStageSyllabus_{curriculum_node.unique_id}_{ks_row['Title'].replace(' ', '')}"
key_stage_syllabus_node_uuid_string = f"KeyStageSyllabus_{curriculum_node.uuid_string}_{ks_row['Title'].replace(' ', '')}"
logger.debug(f"Creating key stage syllabus node for {ks_row['Subject']} KS{key_stage} with ID {ks_row['ID']}")
key_stage_syllabus_node_unique_id = f"KeyStageSyllabus_{curriculum_node.unique_id}_{ks_row['Title'].replace(' ', '')}"
key_stage_syllabus_node_uuid_string = f"KeyStageSyllabus_{curriculum_node.uuid_string}_{ks_row['Title'].replace(' ', '')}"
# Generate storage path for key stage syllabus node
if filesystem:
syllabus_dir_created, syllabus_path = filesystem.create_curriculum_key_stage_syllabus_directory(curriculum_path, key_stage, ks_row['Subject'], ks_row['ID'])
node_storage_path = os.path.relpath(syllabus_path, filesystem.base_path)
else:
node_storage_path = ""
key_stage_syllabus_node = curriculum_nodes.KeyStageSyllabusNode(
unique_id=key_stage_syllabus_node_unique_id,
uuid_string=key_stage_syllabus_node_uuid_string,
id=ks_row['ID'],
name=ks_row['Title'],
key_stage=str(ks_row['KeyStage']),
subject_name=ks_row['Subject'],
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create key stage syllabus node in both databases
neon.create_or_merge_neontology_node(key_stage_syllabus_node, database=db_name, operation='merge')
neon.create_or_merge_neontology_node(key_stage_syllabus_node, database=curriculum_db_name, operation='merge')
node_library['key_stage_syllabus_nodes'][ks_row['ID']] = key_stage_syllabus_node
logger.debug(f"Created key stage syllabus node {key_stage_syllabus_node_unique_id} for {ks_row['Subject']} KS{key_stage}")
logger.debug(f"Created key stage syllabus node {key_stage_syllabus_node_uuid_string} for {ks_row['Subject']} KS{key_stage}")
# Link key stage syllabus to its subject in both databases
if subject_node:
@@ -296,7 +372,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.SubjectHasKeyStageSyllabus(source=subject_node, target=key_stage_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between subject {subject_node.unique_id} and key stage syllabus {key_stage_syllabus_node.unique_id}")
logger.info(f"Created relationship between subject {subject_node.uuid_string} and key stage syllabus {key_stage_syllabus_node.uuid_string}")
# Link key stage syllabus to its key stage in both databases
key_stage_node = key_stage_nodes_created.get(key_stage)
@@ -309,7 +385,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.KeyStageIncludesKeyStageSyllabus(source=key_stage_node, target=key_stage_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between key stage {key_stage_node.unique_id} and key stage syllabus {key_stage_syllabus_node.unique_id}")
logger.info(f"Created relationship between key stage {key_stage_node.uuid_string} and key stage syllabus {key_stage_syllabus_node.uuid_string}")
# Create sequential relationship between key stage syllabuses in both databases
last_key_stage_syllabus_node = last_key_stage_syllabus_nodes.get(ks_row['Subject'])
@@ -322,7 +398,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.KeyStageSyllabusFollowsKeyStageSyllabus(source=last_key_stage_syllabus_node, target=key_stage_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between key stage syllabuses {last_key_stage_syllabus_node.unique_id} and {key_stage_syllabus_node.unique_id}")
logger.info(f"Created sequential relationship between key stage syllabuses {last_key_stage_syllabus_node.uuid_string} and {key_stage_syllabus_node.uuid_string}")
last_key_stage_syllabus_nodes[ks_row['Subject']] = key_stage_syllabus_node
# Now process year groups and their syllabuses
@@ -339,12 +415,19 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
if pd.notna(numeric_year_group):
numeric_year_group = int(numeric_year_group)
if numeric_year_group not in year_group_nodes_created:
year_group_node_unique_id = f"YearGroup_{school_node.unique_id}_YGrp{numeric_year_group}"
year_group_node_uuid_string = f"YearGroup_{school_node.uuid_string}_YGrp{numeric_year_group}"
# Generate storage path for year group node
if filesystem:
year_group_dir_created, year_group_path = filesystem.create_pastoral_year_group_directory(pastoral_path, numeric_year_group)
node_storage_path = os.path.relpath(year_group_path, filesystem.base_path)
else:
node_storage_path = ""
year_group_node = pastoral_nodes.YearGroupNode(
unique_id=year_group_node_unique_id,
uuid_string=year_group_node_uuid_string,
year_group=str(numeric_year_group),
name=f"Year {numeric_year_group}, {year_group}",
tldraw_snapshot=""
name=f"Year {numeric_year_group}",
node_storage_path=node_storage_path
)
# Create year group node in both databases but use same directory
neon.create_or_merge_neontology_node(year_group_node, database=db_name, operation='merge')
@@ -360,7 +443,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.YearGroupFollowsYearGroup(source=last_year_group_node, target=year_group_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between year groups {last_year_group_node.unique_id} and {year_group_node.unique_id} across key stages")
logger.info(f"Created sequential relationship between year groups {last_year_group_node.uuid_string} and {year_group_node.uuid_string} across key stages")
last_year_group_node = year_group_node
# Create relationship with Pastoral Structure in school database only
@@ -368,7 +451,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.PastoralStructureIncludesYearGroup(source=pastoral_node, target=year_group_node),
database=db_name, operation='merge'
)
logger.info(f"Created year group node {year_group_node_unique_id} and relationship with pastoral structure")
logger.info(f"Created year group node {year_group_node_uuid_string} and relationship with pastoral structure")
year_group_nodes_created[numeric_year_group] = year_group_node
node_library['year_group_nodes'][str(numeric_year_group)] = year_group_node
@@ -376,14 +459,21 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
# Create year group syllabus nodes in both databases
year_group_node = year_group_nodes_created.get(numeric_year_group)
if year_group_node:
year_group_syllabus_node_unique_id = f"YearGroupSyllabus_{school_node.unique_id}_{yg_row['ID']}"
year_group_syllabus_node_uuid_string = f"YearGroupSyllabus_{school_node.uuid_string}_{yg_row['ID']}"
# Generate storage path for year group syllabus node
if filesystem:
yg_syllabus_dir_created, yg_syllabus_path = filesystem.create_curriculum_year_group_syllabus_directory(curriculum_path, yg_row['Subject'], numeric_year_group, yg_row['ID'])
node_storage_path = os.path.relpath(yg_syllabus_path, filesystem.base_path)
else:
node_storage_path = ""
year_group_syllabus_node = pastoral_nodes.YearGroupSyllabusNode(
unique_id=year_group_syllabus_node_unique_id,
uuid_string=year_group_syllabus_node_uuid_string,
id=yg_row['ID'],
name=yg_row['Title'],
year_group=str(yg_row['YearGroup']),
subject_name=yg_row['Subject'],
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create year group syllabus node in both databases but use same directory
@@ -406,7 +496,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.YearGroupSyllabusFollowsYearGroupSyllabus(source=last_year_group_syllabus_node, target=year_group_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between year group syllabuses {last_year_group_syllabus_node.unique_id} and {year_group_syllabus_node.unique_id}")
logger.info(f"Created sequential relationship between year group syllabuses {last_year_group_syllabus_node.uuid_string} and {year_group_syllabus_node.uuid_string}")
last_year_group_syllabus_nodes[yg_row['Subject']] = year_group_syllabus_node
# Create relationships in both databases using MATCH to avoid cartesian products
@@ -421,7 +511,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.SubjectHasYearGroupSyllabus(source=subject_node, target=year_group_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between subject {subject_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
logger.info(f"Created relationship between subject {subject_node.uuid_string} and year group syllabus {year_group_syllabus_node_uuid_string}")
# Link to year group
neon.create_or_merge_neontology_relationship(
@@ -432,7 +522,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.YearGroupHasYearGroupSyllabus(source=year_group_node, target=year_group_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between year group {year_group_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
logger.info(f"Created relationship between year group {year_group_node.uuid_string} and year group syllabus {year_group_syllabus_node_uuid_string}")
# Link to key stage syllabus if it exists for the same subject
key_stage_syllabus_node = node_library['key_stage_syllabus_nodes'].get(ks_row['ID'])
@@ -445,7 +535,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.KeyStageSyllabusIncludesYearGroupSyllabus(source=key_stage_syllabus_node, target=year_group_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between key stage syllabus {key_stage_syllabus_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
logger.info(f"Created relationship between key stage syllabus {key_stage_syllabus_node.uuid_string} and year group syllabus {year_group_syllabus_node_uuid_string}")
# Process topics for this year group syllabus only if not already processed
topics_for_syllabus = topic_df[topic_df['SyllabusYearID'] == yg_row['ID']]
@@ -472,22 +562,29 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
if (syllabus_node.subject_name == topic_subject and
syllabus_node.key_stage == str(topic_key_stage)):
matching_syllabus_node = syllabus_node
logger.debug(f"Found matching syllabus node: {syllabus_node.unique_id}")
logger.debug(f"Found matching syllabus node: {syllabus_node.uuid_string}")
break
if not matching_syllabus_node:
logger.warning(f"No key stage syllabus node found for subject {topic_subject} and key stage {topic_key_stage}, skipping topic creation")
continue
topic_node_unique_id = f"Topic_{matching_syllabus_node.unique_id}_{topic_row['TopicID']}"
topic_node_uuid_string = f"Topic_{matching_syllabus_node.uuid_string}_{topic_row['TopicID']}"
# Generate storage path for topic node
if filesystem:
topic_dir_created, topic_path = filesystem.create_curriculum_topic_directory(yg_syllabus_path, topic_row['TopicID'])
node_storage_path = os.path.relpath(topic_path, filesystem.base_path)
else:
node_storage_path = ""
topic_node = curriculum_nodes.TopicNode(
unique_id=topic_node_unique_id,
uuid_string=topic_node_uuid_string,
id=topic_row['TopicID'],
name=topic_row.get('TopicTitle', default_topic_values['topic_title']),
total_number_of_lessons_for_topic=str(topic_row.get('TotalNumberOfLessonsForTopic', default_topic_values['total_number_of_lessons_for_topic'])),
type=topic_row.get('TopicType', default_topic_values['topic_type']),
assessment_type=topic_row.get('TopicAssessmentType', default_topic_values['topic_assessment_type']),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create topic node in curriculum database only
neon.create_or_merge_neontology_node(topic_node, database=curriculum_db_name, operation='merge')
@@ -502,7 +599,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.YearGroupSyllabusIncludesTopic(source=year_group_syllabus_node, target=topic_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationships between topic {topic_node_unique_id} and key stage syllabus {matching_syllabus_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
logger.info(f"Created relationships between topic {topic_node_uuid_string} and key stage syllabus {matching_syllabus_node.uuid_string} and year group syllabus {year_group_syllabus_node_uuid_string}")
# Process lessons for this topic only if not already processed
lessons_for_topic = lesson_df[
@@ -518,8 +615,15 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
continue
lessons_processed.add(lesson_row['LessonID'])
# Generate storage path for lesson node
if filesystem:
lesson_dir_created, lesson_path = filesystem.create_curriculum_lesson_directory(topic_path, lesson_row['LessonID'])
node_storage_path = os.path.relpath(lesson_path, filesystem.base_path)
else:
node_storage_path = ""
lesson_node = curriculum_nodes.TopicLessonNode(
unique_id=f"TopicLesson_{topic_node_unique_id}_{lesson_row['LessonID']}",
uuid_string=f"TopicLesson_{topic_node_uuid_string}_{lesson_row['LessonID']}",
id=lesson_row['LessonID'],
name=lesson_row.get('LessonTitle', default_topic_lesson_values['topic_lesson_title']),
type=lesson_row.get('LessonType', default_topic_lesson_values['topic_lesson_type']),
@@ -527,7 +631,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
suggested_activities=str(lesson_row.get('SuggestedActivities', default_topic_lesson_values['topic_lesson_suggested_activities'])),
skills_learned=str(lesson_row.get('SkillsLearned', default_topic_lesson_values['topic_lesson_skills_learned'])),
weblinks=str(lesson_row.get('WebLinks', default_topic_lesson_values['topic_lesson_weblinks'])),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create lesson node in curriculum database only
neon.create_or_merge_neontology_node(lesson_node, database=curriculum_db_name, operation='merge')
@@ -538,7 +642,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.TopicIncludesTopicLesson(source=topic_node, target=lesson_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created lesson node {lesson_node.unique_id} and relationship with topic {topic_node.unique_id}")
logger.info(f"Created lesson node {lesson_node.uuid_string} and relationship with topic {topic_node.uuid_string}")
# Create sequential relationships between lessons
if lesson_row['Lesson'].isdigit() and previous_lesson_node:
@@ -546,7 +650,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.TopicLessonFollowsTopicLesson(source=previous_lesson_node, target=lesson_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.unique_id} and {lesson_node.unique_id}")
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.uuid_string} and {lesson_node.uuid_string}")
previous_lesson_node = lesson_node
# Process learning statements for this lesson only if not already processed
@@ -558,12 +662,19 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
if statement_row['StatementID'] in statements_processed:
continue
statements_processed.add(statement_row['StatementID'])
# Generate storage path for learning statement node
if filesystem:
statement_dir_created, statement_path = filesystem.create_curriculum_learning_statement_directory(lesson_path, statement_row['StatementID'])
node_storage_path = os.path.relpath(statement_path, filesystem.base_path)
else:
node_storage_path = ""
statement_node = curriculum_nodes.LearningStatementNode(
unique_id=f"LearningStatement_{lesson_node.unique_id}_{statement_row['StatementID']}",
uuid_string=f"LearningStatement_{lesson_node.uuid_string}_{statement_row['StatementID']}",
id=statement_row['StatementID'],
name=statement_row.get('LearningStatement', default_learning_statement_values['lesson_learning_statement']),
type=statement_row.get('StatementType', default_learning_statement_values['lesson_learning_statement_type']),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create statement node in curriculum database only
neon.create_or_merge_neontology_node(statement_node, database=curriculum_db_name, operation='merge')
@@ -574,7 +685,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.LessonIncludesLearningStatement(source=lesson_node, target=statement_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created learning statement node {statement_node.unique_id} and relationship with lesson {lesson_node.unique_id}")
logger.info(f"Created learning statement node {statement_node.uuid_string} and relationship with lesson {lesson_node.uuid_string}")
else:
logger.warning(f"No year group node found for year group {year_group}, skipping syllabus creation")
@@ -601,15 +712,23 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
logger.warning(f"No key stage syllabus node found for subject {topic_subject} and key stage {topic_key_stage}, skipping topic creation")
continue
topic_node_unique_id = f"Topic_{matching_syllabus_node.unique_id}_{topic_row['TopicID']}"
topic_node_uuid_string = f"Topic_{matching_syllabus_node.uuid_string}_{topic_row['TopicID']}"
# Generate storage path for topic node
if filesystem:
syllabus_path = os.path.join(curriculum_path, "subjects", topic_subject, "key_stage_syllabuses", f"KS{topic_key_stage}", f"KS{topic_key_stage}.{topic_subject}")
topic_dir_created, keystage_topic_path = filesystem.create_curriculum_keystage_topic_directory(syllabus_path, topic_row['TopicID'])
node_storage_path = os.path.relpath(keystage_topic_path, filesystem.base_path)
else:
node_storage_path = ""
topic_node = curriculum_nodes.TopicNode(
unique_id=topic_node_unique_id,
uuid_string=topic_node_uuid_string,
id=topic_row['TopicID'],
name=topic_row.get('TopicTitle', default_topic_values['topic_title']),
total_number_of_lessons_for_topic=str(topic_row.get('TotalNumberOfLessonsForTopic', default_topic_values['total_number_of_lessons_for_topic'])),
type=topic_row.get('TopicType', default_topic_values['topic_type']),
assessment_type=topic_row.get('TopicAssessmentType', default_topic_values['topic_assessment_type']),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create topic node in curriculum database only
neon.create_or_merge_neontology_node(topic_node, database=curriculum_db_name, operation='merge')
@@ -621,7 +740,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.KeyStageSyllabusIncludesTopic(source=matching_syllabus_node, target=topic_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between topic {topic_node_unique_id} and key stage syllabus {matching_syllabus_node.unique_id}")
logger.info(f"Created relationship between topic {topic_node_uuid_string} and key stage syllabus {matching_syllabus_node.uuid_string}")
# Process lessons for this topic
lessons_for_topic = lesson_df[
@@ -636,8 +755,15 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
if lesson_row['LessonID'] in lessons_processed:
continue
lessons_processed.add(lesson_row['LessonID'])
# Generate storage path for lesson node
if filesystem:
lesson_dir_created, lesson_path = filesystem.create_curriculum_lesson_directory(topic_path, lesson_row['LessonID'])
node_storage_path = os.path.relpath(lesson_path, filesystem.base_path)
else:
node_storage_path = ""
lesson_node = curriculum_nodes.TopicLessonNode(
unique_id=f"TopicLesson_{topic_node_unique_id}_{lesson_row['LessonID']}",
uuid_string=f"TopicLesson_{topic_node_uuid_string}_{lesson_row['LessonID']}",
id=lesson_row['LessonID'],
name=lesson_row.get('LessonTitle', default_topic_lesson_values['topic_lesson_title']),
type=lesson_row.get('LessonType', default_topic_lesson_values['topic_lesson_type']),
@@ -645,7 +771,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
suggested_activities=str(lesson_row.get('SuggestedActivities', default_topic_lesson_values['topic_lesson_suggested_activities'])),
skills_learned=str(lesson_row.get('SkillsLearned', default_topic_lesson_values['topic_lesson_skills_learned'])),
weblinks=str(lesson_row.get('WebLinks', default_topic_lesson_values['topic_lesson_weblinks'])),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create lesson node in curriculum database only
neon.create_or_merge_neontology_node(lesson_node, database=curriculum_db_name, operation='merge')
@@ -656,7 +782,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.TopicIncludesTopicLesson(source=topic_node, target=lesson_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created lesson node {lesson_node.unique_id} and relationship with topic {topic_node.unique_id}")
logger.info(f"Created lesson node {lesson_node.uuid_string} and relationship with topic {topic_node.uuid_string}")
# Create sequential relationships between lessons
if lesson_row['Lesson'].isdigit() and previous_lesson_node:
@@ -664,7 +790,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.TopicLessonFollowsTopicLesson(source=previous_lesson_node, target=lesson_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.unique_id} and {lesson_node.unique_id}")
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.uuid_string} and {lesson_node.uuid_string}")
previous_lesson_node = lesson_node
# Process learning statements for this lesson
@@ -676,12 +802,19 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
if statement_row['StatementID'] in statements_processed:
continue
statements_processed.add(statement_row['StatementID'])
# Generate storage path for learning statement node
if filesystem:
statement_dir_created, statement_path = filesystem.create_curriculum_learning_statement_directory(lesson_path, statement_row['StatementID'])
node_storage_path = os.path.relpath(statement_path, filesystem.base_path)
else:
node_storage_path = ""
statement_node = curriculum_nodes.LearningStatementNode(
unique_id=f"LearningStatement_{lesson_node.unique_id}_{statement_row['StatementID']}",
uuid_string=f"LearningStatement_{lesson_node.uuid_string}_{statement_row['StatementID']}",
id=statement_row['StatementID'],
name=statement_row.get('LearningStatement', default_learning_statement_values['lesson_learning_statement']),
type=statement_row.get('StatementType', default_learning_statement_values['lesson_learning_statement_type']),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create statement node in curriculum database only
neon.create_or_merge_neontology_node(statement_node, database=curriculum_db_name, operation='merge')
@@ -692,6 +825,6 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.LessonIncludesLearningStatement(source=lesson_node, target=statement_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created learning statement node {statement_node.unique_id} and relationship with lesson {lesson_node.unique_id}")
logger.info(f"Created learning statement node {statement_node.uuid_string} and relationship with lesson {lesson_node.uuid_string}")
return node_library
modules/database/init/init_school_timetable.py
+101 -54
View File
@@ -11,7 +11,7 @@ import modules.database.schemas.relationships.calendar_timetable_rels as cal_tt_
import modules.database.init.init_calendar as init_calendar
import modules.database.tools.neontology_tools as neon
def create_school_timetable(dataframes, db_name, school_node=None):
def create_school_timetable(dataframes, db_name, school_node=None, filesystem=None):
logger.info(f"Creating school timetable for {db_name}")
if dataframes is None:
raise ValueError("Data is required to create the calendar and timetable.")
@@ -22,10 +22,10 @@ def create_school_timetable(dataframes, db_name, school_node=None):
school_df = dataframes['school']
if school_node is None:
logger.info(f"School node is None, using school data from dataframe")
school_unique_id = school_df[school_df['Identifier'] == 'SchoolID']['Data'].iloc[0]
school_uuid_string = school_df[school_df['Identifier'] == 'SchoolID']['Data'].iloc[0]
else:
logger.info(f"School node is not None, using school data from school node: {school_node}")
school_unique_id = school_node.unique_id
school_uuid_string = school_node.uuid_string
terms_df = dataframes['terms']
weeks_df = dataframes['weeks']
@@ -54,20 +54,30 @@ def create_school_timetable(dataframes, db_name, school_node=None):
}
# Create AcademicTimetable Node
school_timetable_unique_id = f"{school_unique_id}_{school_year_start_date.year}_{school_year_end_date.year}"
school_timetable_uuid_string = f"{school_uuid_string}_{school_year_start_date.year}_{school_year_end_date.year}"
# Generate storage path for timetable node
if filesystem:
timetable_dir_created, timetable_path = filesystem.create_school_timetable_directory()
node_storage_path = os.path.relpath(timetable_path, filesystem.base_path)
logger.info(f"Generated timetable node_storage_path: {node_storage_path}")
else:
node_storage_path = ""
logger.warning("No filesystem provided, using empty storage path")
school_timetable_node = timetable.SchoolTimetableNode(
school_timetable_id=school_timetable_unique_id,
unique_id=school_timetable_unique_id,
school_timetable_id=school_timetable_uuid_string,
uuid_string=school_timetable_uuid_string,
start_date=school_year_start_date,
end_date=school_year_end_date,
tldraw_snapshot=""
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(school_timetable_node, database=db_name, operation='merge')
timetable_nodes['timetable_node'] = school_timetable_node
if school_node:
logger.info(f"Creating calendar for {school_unique_id} from Neo4j SchoolNode: {school_node.unique_id}")
calendar_nodes = init_calendar.create_calendar(db_name, school_year_start_date, school_year_end_date, attach_to_calendar_node=True, owner_node=school_node)
logger.info(f"Creating calendar for {school_uuid_string} from Neo4j SchoolNode: {school_node.uuid_string}")
calendar_nodes = init_calendar.create_calendar(db_name, school_year_start_date, school_year_end_date, attach_to_calendar_node=True, owner_node=school_node, filesystem=filesystem)
# Link the school node to the timetable node
neon.create_or_merge_neontology_relationship(
entity_tt_rels.SchoolHasTimetable(source=school_node, target=school_timetable_node),
@@ -75,26 +85,34 @@ def create_school_timetable(dataframes, db_name, school_node=None):
)
timetable_nodes['calendar_nodes'] = calendar_nodes
else:
logger.info(f"Creating calendar for {school_unique_id} from dataframe SchoolID: {school_unique_id}")
calendar_nodes = init_calendar.create_calendar(db_name, school_year_start_date, school_year_end_date, attach_to_calendar_node=False, owner_node=None)
logger.info(f"Creating calendar for {school_uuid_string} from dataframe SchoolID: {school_uuid_string}")
calendar_nodes = init_calendar.create_calendar(db_name, school_year_start_date, school_year_end_date, attach_to_calendar_node=False, owner_node=None, filesystem=filesystem)
# Create AcademicYear nodes for each year within the range
for year in range(school_year_start_date.year, school_year_end_date.year + 1):
year_str = str(year)
academic_year_unique_id = f"{school_timetable_unique_id}_{year}"
academic_year_uuid_string = f"{school_timetable_uuid_string}_{year}"
# Generate storage path for academic year node
if filesystem:
year_dir_created, year_path = filesystem.create_school_timetable_year_directory(timetable_path, year)
node_storage_path = os.path.relpath(year_path, filesystem.base_path)
else:
node_storage_path = ""
academic_year_node = timetable.AcademicYearNode(
unique_id=academic_year_unique_id,
uuid_string=academic_year_uuid_string,
year=year_str,
tldraw_snapshot=""
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(academic_year_node, database=db_name, operation='merge')
timetable_nodes['academic_year_nodes'].append(academic_year_node)
logger.info(f'Created academic year node: {academic_year_node.unique_id}')
logger.info(f'Created academic year node: {academic_year_node.uuid_string}')
neon.create_or_merge_neontology_relationship(
tt_rels.AcademicTimetableHasAcademicYear(source=school_timetable_node, target=academic_year_node),
database=db_name, operation='merge'
)
logger.info(f"Created school timetable relationship from {school_timetable_node.unique_id} to {academic_year_node.unique_id}")
logger.info(f"Created school timetable relationship from {school_timetable_node.uuid_string} to {academic_year_node.uuid_string}")
# Link the academic year with the corresponding calendar year node
for year_node in calendar_nodes['calendar_year_nodes']:
@@ -103,7 +121,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
cal_tt_rels.AcademicYearIsCalendarYear(source=academic_year_node, target=year_node),
database=db_name, operation='merge'
)
logger.info(f"Created school timetable relationship from {academic_year_node.unique_id} to {year_node.unique_id}")
logger.info(f"Created school timetable relationship from {academic_year_node.uuid_string} to {year_node.uuid_string}")
break
# Create Term and TermBreak nodes linked to AcademicYear
@@ -121,29 +139,39 @@ def create_school_timetable(dataframes, db_name, school_node=None):
if isinstance(term_end_date, pd.Timestamp):
term_end_date = term_end_date.strftime('%Y-%m-%d')
# Generate storage path for term node
if filesystem:
if term_row['TermType'] == 'Term':
term_dir_created, term_path = filesystem.create_school_timetable_academic_term_directory(timetable_path, term_name, academic_term_number)
else:
term_dir_created, term_path = filesystem.create_school_timetable_academic_term_break_directory(timetable_path, term_name)
node_storage_path = os.path.relpath(term_path, filesystem.base_path)
else:
node_storage_path = ""
if term_row['TermType'] == 'Term':
term_node_unique_id = f"{school_timetable_unique_id}_{academic_term_number}_{term_name_no_spaces}"
term_node_uuid_string = f"{school_timetable_uuid_string}_{academic_term_number}_{term_name_no_spaces}"
academic_term_number_str = str(academic_term_number)
term_node = term_node_class(
unique_id=term_node_unique_id,
uuid_string=term_node_uuid_string,
term_name=term_name,
term_number=academic_term_number_str,
start_date=datetime.strptime(term_start_date, '%Y-%m-%d'),
end_date=datetime.strptime(term_end_date, '%Y-%m-%d'),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
academic_term_number += 1
else:
term_break_node_unique_id = f"{school_timetable_unique_id}_{term_name_no_spaces}"
term_break_node_uuid_string = f"{school_timetable_uuid_string}_{term_name_no_spaces}"
term_node = term_node_class(
unique_id=term_break_node_unique_id,
uuid_string=term_break_node_uuid_string,
term_break_name=term_name,
start_date=datetime.strptime(term_start_date, '%Y-%m-%d'),
end_date=datetime.strptime(term_end_date, '%Y-%m-%d'),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(term_node, database=db_name, operation='merge')
logger.info(f'Created academic term break node: {term_node.unique_id}')
logger.info(f'Created academic term break node: {term_node.uuid_string}')
timetable_nodes['academic_term_nodes'].append(term_node)
term_number += 1 # We don't use this but we could
@@ -158,7 +186,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
relationship_class(source=academic_year_node, target=term_node),
database=db_name, operation='merge'
)
logger.info(f"Created school timetable relationship from {academic_year_node.unique_id} to {term_node.unique_id}")
logger.info(f"Created school timetable relationship from {academic_year_node.uuid_string} to {term_node.uuid_string}")
# Create Week nodes
academic_week_number = 1
@@ -168,28 +196,35 @@ def create_school_timetable(dataframes, db_name, school_node=None):
if isinstance(week_start_date, pd.Timestamp):
week_start_date = week_start_date.strftime('%Y-%m-%d')
week_node_unique_id = f"{school_timetable_unique_id}_{week_row['WeekNumber']}_{week_row['WeekType']}Week"
week_node_uuid_string = f"{school_timetable_uuid_string}_{week_row['WeekNumber']}_{week_row['WeekType']}Week"
# Generate storage path for week node
if filesystem:
week_dir_created, week_path = filesystem.create_school_timetable_academic_week_directory(timetable_path, week_row['WeekNumber'])
node_storage_path = os.path.relpath(week_path, filesystem.base_path)
else:
node_storage_path = ""
if week_row['WeekType'] == 'Holiday':
week_node = week_node_class(
unique_id=week_node_unique_id,
uuid_string=week_node_uuid_string,
start_date=datetime.strptime(week_start_date, '%Y-%m-%d'),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
else:
academic_week_number_str = str(academic_week_number)
week_type = week_row['WeekType']
week_node = week_node_class(
unique_id=week_node_unique_id,
uuid_string=week_node_uuid_string,
academic_week_number=academic_week_number_str,
start_date=datetime.strptime(week_start_date, '%Y-%m-%d'),
week_type=week_type,
tldraw_snapshot=""
node_storage_path=node_storage_path
)
academic_week_number += 1
neon.create_or_merge_neontology_node(week_node, database=db_name, operation='merge')
timetable_nodes['academic_week_nodes'].append(week_node)
logger.info(f"Created week node: {week_node.unique_id}")
logger.info(f"Created week node: {week_node.uuid_string}")
for calendar_node in calendar_nodes['calendar_week_nodes']:
if calendar_node.start_date == week_node.start_date:
if isinstance(week_node, timetable.AcademicWeekNode):
@@ -197,13 +232,13 @@ def create_school_timetable(dataframes, db_name, school_node=None):
cal_tt_rels.AcademicWeekIsCalendarWeek(source=week_node, target=calendar_node),
database=db_name, operation='merge'
)
logger.info(f"Created school timetable relationship from {calendar_node.unique_id} to {week_node.unique_id}")
logger.info(f"Created school timetable relationship from {calendar_node.uuid_string} to {week_node.uuid_string}")
elif isinstance(week_node, timetable.HolidayWeekNode):
neon.create_or_merge_neontology_relationship(
cal_tt_rels.HolidayWeekIsCalendarWeek(source=week_node, target=calendar_node),
database=db_name, operation='merge'
)
logger.info(f"Created school timetable relationship from {calendar_node.unique_id} to {week_node.unique_id}")
logger.info(f"Created school timetable relationship from {calendar_node.uuid_string} to {week_node.uuid_string}")
break
# Link week node to the correct academic term
@@ -214,7 +249,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
relationship_class(source=term_node, target=week_node),
database=db_name, operation='merge'
)
logger.info(f"Created school timetable relationship from {term_node.unique_id} to {week_node.unique_id}")
logger.info(f"Created school timetable relationship from {term_node.uuid_string} to {week_node.uuid_string}")
break
# Link week node to the correct academic year
@@ -225,7 +260,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
relationship_class(source=academic_year_node, target=week_node),
database=db_name, operation='merge'
)
logger.info(f"Created school timetable relationship from {academic_year_node.unique_id} to {week_node.unique_id}")
logger.info(f"Created school timetable relationship from {academic_year_node.uuid_string} to {week_node.uuid_string}")
break
# Create Day nodes
@@ -243,18 +278,24 @@ def create_school_timetable(dataframes, db_name, school_node=None):
'StaffDay': timetable.StaffDayNode
}[day_row['DayType']]
# Generate storage path for day node
if filesystem:
day_dir_created, day_path = filesystem.create_school_timetable_academic_day_directory(timetable_path, academic_day_number)
node_storage_path = os.path.relpath(day_path, filesystem.base_path)
else:
node_storage_path = ""
# Format the unique ID as {day_node_class.__name__}Day
day_node_data = {
'unique_id': f"{school_timetable_unique_id}_{day_number}_{day_node_class.__name__}Day",
'uuid_string': f"{school_timetable_uuid_string}_{day_number}_{day_node_class.__name__}Day",
'date': datetime.strptime(date_str, '%Y-%m-%d'),
'day_of_week': datetime.strptime(date_str, '%Y-%m-%d').strftime('%A'),
'tldraw_snapshot': ""
'node_storage_path': node_storage_path
}
if day_row['DayType'] == 'Academic':
day_node_data['academic_day'] = str(academic_day_number)
day_node_data['day_type'] = day_row['WeekType']
day_node_data['tldraw_snapshot'] = ""
day_node = day_node_class(**day_node_data)
@@ -262,7 +303,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
if calendar_node.date == day_node.date:
neon.create_or_merge_neontology_node(day_node, database=db_name, operation='merge')
timetable_nodes['academic_day_nodes'].append(day_node)
logger.info(f"Created day node: {day_node.unique_id}")
logger.info(f"Created day node: {day_node.uuid_string}")
if isinstance(day_node, timetable.AcademicDayNode):
relationship_class = cal_tt_rels.AcademicDayIsCalendarDay
@@ -277,7 +318,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
relationship_class(source=day_node, target=calendar_node),
database=db_name, operation='merge'
)
logger.info(f'Created relationship from {calendar_node.unique_id} to {day_node.unique_id}')
logger.info(f'Created relationship from {calendar_node.uuid_string} to {day_node.uuid_string}')
break
# Link day node to the correct academic week
@@ -300,7 +341,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
relationship_class(source=academic_week_node, target=day_node),
database=db_name, operation='merge'
)
logger.info(f"Created relationship from {academic_week_node.unique_id} to {day_node.unique_id}")
logger.info(f"Created relationship from {academic_week_node.uuid_string} to {day_node.uuid_string}")
break
# Link day node to the correct academic term
@@ -323,12 +364,12 @@ def create_school_timetable(dataframes, db_name, school_node=None):
relationship_class(source=term_node, target=day_node),
database=db_name, operation='merge'
)
logger.info(f"Created relationship from {term_node.unique_id} to {day_node.unique_id}")
logger.info(f"Created relationship from {term_node.uuid_string} to {day_node.uuid_string}")
break
# Create Period nodes for each academic day
if day_row['DayType'] == 'Academic':
logger.info(f"Creating periods for {day_node.unique_id}")
logger.info(f"Creating periods for {day_node.uuid_string}")
period_of_day = 1
academic_or_registration_period_of_day = 1
for _, period_row in periods_df.iterrows():
@@ -340,15 +381,22 @@ def create_school_timetable(dataframes, db_name, school_node=None):
}[period_row['PeriodType']]
logger.info(f"Creating period node for {period_node_class.__name__} Period: {period_of_day}")
period_node_unique_id = f"{school_timetable_unique_id}_{academic_day_number}_{period_of_day}_{period_node_class.__name__}Period"
logger.debug(f"Period node unique id: {period_node_unique_id}")
period_node_uuid_string = f"{school_timetable_uuid_string}_{academic_day_number}_{period_of_day}_{period_node_class.__name__}Period"
logger.debug(f"Period node unique id: {period_node_uuid_string}")
# Generate storage path for period node
if filesystem:
period_dir_created, period_path = filesystem.create_school_timetable_period_directory(timetable_path, academic_day_number, period_row['PeriodCode'])
node_storage_path = os.path.relpath(period_path, filesystem.base_path)
else:
node_storage_path = ""
period_node_data = {
'unique_id': period_node_unique_id,
'uuid_string': period_node_uuid_string,
'name': period_row['PeriodName'],
'date': day_node.date,
'start_time': datetime.combine(day_node.date, period_row['StartTime']),
'end_time': datetime.combine(day_node.date, period_row['EndTime']),
'tldraw_snapshot': ""
'node_storage_path': node_storage_path
}
logger.debug(f"Period node data: {period_node_data}")
if period_row['PeriodType'] in ['Academic', 'Registration']:
@@ -357,14 +405,13 @@ def create_school_timetable(dataframes, db_name, school_node=None):
period_code = period_row['PeriodCode']
period_code_formatted = f"{week_type}{day_name_short}{period_code}"
period_node_data['period_code'] = period_code_formatted
period_node_data['tldraw_snapshot'] = ""
academic_or_registration_period_of_day += 1
period_node = period_node_class(**period_node_data)
neon.create_or_merge_neontology_node(period_node, database=db_name, operation='merge')
timetable_nodes['academic_period_nodes'].append(period_node)
logger.info(f'Created period node: {period_node.unique_id}')
logger.info(f'Created period node: {period_node.uuid_string}')
relationship_class = {
'Academic': tt_rels.AcademicDayHasAcademicPeriod,
@@ -377,7 +424,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
relationship_class(source=day_node, target=period_node),
database=db_name, operation='merge'
)
logger.info(f"Created relationship from {day_node.unique_id} to {period_node.unique_id}")
logger.info(f"Created relationship from {day_node.uuid_string} to {period_node.uuid_string}")
period_of_day += 1 # We don't use this but we could
academic_day_number += 1 # This is a bit of a hack but it works to keep the directories aligned (reorganise)
day_number += 1 # We don't use this but we could
@@ -392,7 +439,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
relationship_class = relationship_map.get(node_type_pair)
if relationship_class:
# Avoid self-referential relationships
if source_node.unique_id != target_node.unique_id:
if source_node.uuid_string != target_node.uuid_string:
neon.create_or_merge_neontology_relationship(
relationship_class(
source=source_node,
@@ -400,9 +447,9 @@ def create_school_timetable(dataframes, db_name, school_node=None):
),
database=db_name, operation='merge'
)
logger.info(f"Created relationship from {source_node.unique_id} to {target_node.unique_id}")
logger.info(f"Created relationship from {source_node.uuid_string} to {target_node.uuid_string}")
else:
logger.warning(f"Skipped self-referential relationship for node {source_node.unique_id}")
logger.warning(f"Skipped self-referential relationship for node {source_node.uuid_string}")
# Relationship maps for different node types
academic_year_relationship_map = {
@@ -474,7 +521,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
# Call the function with the created timetable nodes
create_school_timetable_node_sequence_rels(timetable_nodes)
logger.info(f'Created timetable: {timetable_nodes["timetable_node"].unique_id}')
logger.info(f'Created timetable: {timetable_nodes["timetable_node"].uuid_string}')
# Log the directory structure after creation
# root_timetable_directory = fs_handler.root_path # Access the root directory of the filesystem handler
modules/database/init/init_user.py
+297 -92
View File
@@ -9,6 +9,7 @@ import modules.database.schemas.nodes.workers.workers as worker_nodes
import modules.database.init.init_calendar as init_calendar
import modules.database.schemas.relationships.entity_relationships as entity_relationships
import modules.database.tools.neontology_tools as neon
import modules.database.tools.supabase_storage_tools as storage_tools
from modules.logger_tool import initialise_logger
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
@@ -21,11 +22,27 @@ def create_and_check_db(db_name):
return database_status
class UserCreator(ABC):
def __init__(self, user_id, cc_users_db_name, user_type, worker_type, user_email, worker_email, cc_username, user_name, worker_name, calendar_start_date, calendar_end_date):
cc_schools_db_name = "cc.institutes" # Fix the TODO
def __init__(
self,
user_id,
cc_users_db_name,
user_type,
worker_type,
user_email,
worker_email,
cc_username,
user_name,
worker_name,
calendar_start_date,
calendar_end_date,
storage_tools=None,
user_db_name: Optional[str] = None,
worker_db_name: Optional[str] = None,
cc_schools_db_name: str = "cc.institutes",
):
self.cc_users_db_name = cc_users_db_name
self.user_db_name = f"{cc_users_db_name}.{user_type}.{cc_username}"
self.worker_db_name = f"{cc_schools_db_name}.{user_type}.{cc_username}"
self.user_db_name = user_db_name or f"{cc_users_db_name}.{user_type}.{cc_username}"
self.worker_db_name = worker_db_name or f"{cc_schools_db_name}.{worker_type}.{cc_username}"
self.user_type = user_type
self.worker_type = worker_type
self.cc_username = cc_username
@@ -34,6 +51,7 @@ class UserCreator(ABC):
self.user_name = user_name
self.worker_name = worker_name
self.user_id = user_id
self.storage_tools = storage_tools # Store the storage tools instance
self.user_nodes: Dict[str, Optional[Any]] = {
'default_user_node': None,
'private_user_node': None,
@@ -48,6 +66,25 @@ class UserCreator(ABC):
self.calendar_start_date = datetime.now().date()
self.calendar_end_date = (datetime.now() + timedelta(days=5)).date()
def _derive_user_storage_path(self) -> str:
return os.path.join(
"users",
self.user_id,
"databases",
self.user_db_name,
self.user_id,
).replace('\\', '/')
def _derive_internal_worker_path(self, worker_kind: str) -> str:
return os.path.join(
"users",
self.user_id,
"databases",
self.user_db_name,
worker_kind,
self.user_id,
).replace('\\', '/')
@abstractmethod
def create_user(self):
pass
@@ -66,9 +103,17 @@ class UserCreator(ABC):
# Ensure Neontology is initialized
neon.init_neontology_connection()
# Generate storage path for user node using Supabase Storage
if self.storage_tools:
user_dir_created, node_storage_path = self.storage_tools.create_user_storage_path(self.user_id)
self.user_path = node_storage_path # Store for later use
else:
node_storage_path = self._derive_user_storage_path()
self.user_path = None
user_node = user_nodes.UserNode(
unique_id=f"{self.user_id}",
tldraw_snapshot="",
uuid_string=f"{self.user_id}",
node_storage_path=node_storage_path,
cc_username=f"{self.cc_username}",
user_email=f"{self.user_email}",
user_name=f"{self.user_name}",
@@ -76,81 +121,79 @@ class UserCreator(ABC):
user_type=f"{self.user_type}",
)
logger.debug(f"User node template created: {user_node.to_dict()}. Writing to database {db_name}")
neon.create_or_merge_neontology_node(node=user_node, database=db_name, operation='merge')
logger.info(f"User node created: {user_node.to_dict()}")
logger.debug(f"About to call create_or_merge_neontology_node with node class: {user_node.__class__.__name__}")
logger.debug(f"Node primary label: {user_node.__primarylabel__}")
logger.debug(f"Node primary property: {user_node.__primaryproperty__}")
try:
neon.create_or_merge_neontology_node(node=user_node, database=db_name, operation='merge')
logger.info(f"User node created successfully: {user_node.to_dict()}")
except Exception as e:
logger.error(f"Failed to create user node: {e}")
raise
return user_node
def create_storage_bucket(self, bucket_id: str, bucket_name: str, access_token: Optional[str] = None) -> bool:
"""Create public and private storage buckets for the user using their access token or service role during initialization"""
logger.info(f"Creating storage buckets for user {self.cc_username}")
try:
from modules.database.supabase.utils.client import SupabaseServiceRoleClient, SupabaseAnonClient, CreateBucketOptions
# During initialization (no access token provided), use service role
if not access_token:
logger.info("Using service role client for bucket creation during initialization")
supabase = SupabaseServiceRoleClient()
else:
# For regular operations, use the user's access token
logger.info("Using user token for bucket creation")
supabase = SupabaseAnonClient.for_user(access_token)
# Create both public and private buckets
buckets = [
{
"id": f"{bucket_id}.public",
"options": CreateBucketOptions(
name=f"{bucket_name} - Public Files",
public=True,
file_size_limit=50 * 1024 * 1024, # 50MB
allowed_mime_types=[
'image/*', 'video/*', 'application/pdf',
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
'text/plain', 'text/csv', 'application/json'
]
)
},
{
"id": f"{bucket_id}.private",
"options": CreateBucketOptions(
name=f"{bucket_name} - Private Files",
public=False,
file_size_limit=50 * 1024 * 1024, # 50MB
allowed_mime_types=[
'image/*', 'video/*', 'application/pdf',
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
'text/plain', 'text/csv', 'application/json'
]
)
}
]
success = True
for bucket in buckets:
try:
result = supabase.create_bucket(bucket["id"], bucket["options"])
if not result:
logger.error(f"Failed to create bucket {bucket['id']}")
success = False
else:
logger.info(f"Successfully created bucket {bucket['id']}")
except Exception as e:
logger.error(f"Error creating bucket {bucket['id']}: {str(e)}")
success = False
return success
except Exception as e:
logger.error(f"Error creating storage buckets: {str(e)}")
return False
"""Create storage buckets for the user - DEPRECATED: Use centralized bucket initialization instead"""
logger.warning(f"Individual user bucket creation is deprecated. Use centralized bucket initialization instead.")
logger.info(f"User {self.cc_username} will use centralized storage buckets.")
return True # Return success to avoid breaking existing code
class SchoolUserCreator(UserCreator):
def __init__(self, user_id, cc_users_db_name, user_type, worker_type, user_email, worker_email, cc_username, user_name, worker_name, calendar_start_date, calendar_end_date, school_node, worker_node=None):
super().__init__(user_id, cc_users_db_name, user_type, worker_type, user_email, worker_email, cc_username, user_name, worker_name, calendar_start_date, calendar_end_date)
def __init__(
self,
user_id,
cc_users_db_name,
user_type,
worker_type,
user_email,
worker_email,
cc_username,
user_name,
worker_name,
calendar_start_date,
calendar_end_date,
school_node,
worker_node=None,
storage_tools=None,
user_db_name: Optional[str] = None,
worker_db_name: Optional[str] = None,
):
super().__init__(
user_id,
cc_users_db_name,
user_type,
worker_type,
user_email,
worker_email,
cc_username,
user_name,
worker_name,
calendar_start_date,
calendar_end_date,
storage_tools,
user_db_name=user_db_name,
worker_db_name=worker_db_name or (school_node.private_database_name if hasattr(school_node, "private_database_name") else None),
cc_schools_db_name="cc.institutes",
)
self.school_node = school_node
self.worker_node = worker_node
def _derive_school_worker_path(self, worker_kind: str) -> str:
school_identifier = getattr(self.school_node, 'uuid_string', None) if self.school_node else None
if not school_identifier and self.worker_db_name:
school_identifier = self.worker_db_name.split('.')[-1]
school_identifier = school_identifier or self.user_id
db_name_segment = self.worker_db_name or f"cc.institutes.{school_identifier}"
return os.path.join(
"schools",
school_identifier,
"databases",
db_name_segment,
worker_kind,
self.user_id,
).replace('\\', '/')
def create_user(self):
# Ensure Neontology is initialized
logger.debug(f"Initializing Neontology connection. Closing any existing connection")
@@ -167,7 +210,7 @@ class SchoolUserCreator(UserCreator):
self.user_nodes[f'worker_node'] = worker_node
user_node = self.create_user_node(self.cc_users_db_name)
user_node = self.create_user_node(self.user_db_name)
logger.info(f"User node created: {user_node}")
@@ -190,16 +233,25 @@ class SchoolUserCreator(UserCreator):
raise ValueError(f"Error creating teacher node: {e}") from e
def _create_teacher_node(self):
# Generate storage path for teacher node using Supabase Storage
if self.storage_tools:
teacher_dir_created, node_storage_path = self.storage_tools.create_teacher_storage_path(self.user_id)
else:
node_storage_path = self._derive_school_worker_path(self.worker_type)
teacher_node = worker_nodes.TeacherNode(
unique_id=f"{self.user_id}",
tldraw_snapshot="",
uuid_string=f"{self.user_id}",
node_storage_path=node_storage_path,
worker_name=self.worker_name,
worker_email=self.worker_email,
worker_db_name=self.worker_db_name,
worker_type=self.worker_type
)
# Use the school's private database name if available
school_db = self.school_node.private_database_name if hasattr(self.school_node, 'private_database_name') else f"cc.institutes.{self.school_node.school_type}.{self.school_node.id}"
school_db = self.worker_db_name or (
self.school_node.private_database_name if hasattr(self.school_node, 'private_database_name')
else f"cc.institutes.{self.school_node.school_type}.{getattr(self.school_node, 'id', self.school_node.uuid_string)}"
)
logger.info(f"Teacher node template created: {teacher_node}... setting school db to {school_db}")
neon.create_or_merge_neontology_node(node=teacher_node, database=school_db, operation='merge')
@@ -208,16 +260,25 @@ class SchoolUserCreator(UserCreator):
return teacher_node
def create_student_node(self):
# Generate storage path for student node using Supabase Storage
if self.storage_tools:
student_dir_created, node_storage_path = self.storage_tools.create_student_storage_path(f"Student_{self.user_id}")
else:
node_storage_path = self._derive_school_worker_path(self.worker_type)
student_node = worker_nodes.StudentNode(
unique_id=f"Student_{self.user_id}",
uuid_string=f"Student_{self.user_id}",
worker_name=self.worker_name,
worker_email=self.worker_email,
worker_db_name=self.worker_db_name,
worker_type=self.worker_type,
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Use the school's private database name if available
school_db = self.school_node.private_database_name if hasattr(self.school_node, 'private_database_name') else f"cc.institutes.{self.school_node.school_type}.{self.school_node.id}"
school_db = self.worker_db_name or (
self.school_node.private_database_name if hasattr(self.school_node, 'private_database_name')
else f"cc.institutes.{self.school_node.school_type}.{getattr(self.school_node, 'id', self.school_node.uuid_string)}"
)
logger.info(f"Student node template created: {student_node}... setting school db to {school_db}")
neon.create_or_merge_neontology_node(node=student_node, database=school_db, operation='merge')
@@ -228,20 +289,58 @@ class SchoolUserCreator(UserCreator):
def create_user_worker_relationship(self, user_node, worker_node):
user_role_rel = entity_relationships.UserIsSchoolWorker(source=user_node, target=worker_node)
# Use the school's private database name if available
school_db = self.school_node.private_database_name if hasattr(self.school_node, 'private_database_name') else f"cc.institutes.{self.school_node.school_type}.{self.school_node.id}"
school_db = self.worker_db_name or (
self.school_node.private_database_name if hasattr(self.school_node, 'private_database_name')
else f"cc.institutes.{self.school_node.school_type}.{getattr(self.school_node, 'id', self.school_node.uuid_string)}"
)
neon.create_or_merge_neontology_relationship(user_role_rel, database=school_db, operation='merge')
logger.info(f"Relationship created between user and worker in database {school_db}")
def create_worker_school_relationship(self, worker_node, school_node):
worker_school_rel = entity_relationships.EntityBelongsToSchool(source=worker_node, target=school_node)
# Use the school's private database name if available
school_db = school_node.private_database_name if hasattr(school_node, 'private_database_name') else f"cc.institutes.{school_node.school_type}.{school_node.id}"
school_db = self.worker_db_name or (
school_node.private_database_name if hasattr(school_node, 'private_database_name')
else f"cc.institutes.{school_node.school_type}.{getattr(school_node, 'id', school_node.uuid_string)}"
)
neon.create_or_merge_neontology_relationship(worker_school_rel, database=school_db, operation='merge')
logger.info(f"Relationship created between worker and school in database {school_db}")
class NonSchoolUserCreator(UserCreator):
def __init__(self, user_id, cc_users_db_name, user_type, worker_type, user_email, worker_email, cc_username, user_name, worker_name, calendar_start_date, calendar_end_date, developer_role: str = "developer"):
super().__init__(user_id, cc_users_db_name, user_type, worker_type, user_email, worker_email, cc_username, user_name, worker_name, calendar_start_date, calendar_end_date)
def __init__(
self,
user_id,
cc_users_db_name,
user_type,
worker_type,
user_email,
worker_email,
cc_username,
user_name,
worker_name,
calendar_start_date,
calendar_end_date,
developer_role: str = "developer",
storage_tools=None,
user_db_name: Optional[str] = None,
worker_db_name: Optional[str] = None,
):
super().__init__(
user_id,
cc_users_db_name,
user_type,
worker_type,
user_email,
worker_email,
cc_username,
user_name,
worker_name,
calendar_start_date,
calendar_end_date,
storage_tools,
user_db_name=user_db_name,
worker_db_name=worker_db_name,
)
self.developer_role = developer_role
def create_user(self, access_token: Optional[str] = None):
@@ -273,7 +372,8 @@ class NonSchoolUserCreator(UserCreator):
logger.debug(f"Creating developer db for {self.user_type} user {self.cc_username} in database {self.user_db_name}")
self.create_developer_db()
else:
raise ValueError(f"User type {self.user_type} not supported")
logger.warning(f"User type {self.user_type} not explicitly supported; defaulting to developer workspace")
self.create_developer_db()
logger.debug(f"User nodes after creation: {self.user_nodes}")
return self.user_nodes
@@ -290,10 +390,16 @@ class NonSchoolUserCreator(UserCreator):
logger.debug(f"Creating super admin user node for {self.user_type} user {self.cc_username} in database {self.user_db_name}")
private_user_node = self.create_user_node(self.user_db_name)
# Generate storage path for super admin node using Supabase Storage
if self.storage_tools:
admin_dir_created, node_storage_path = self.storage_tools.create_super_admin_storage_path(self.user_id)
else:
node_storage_path = self._derive_internal_worker_path(self.worker_type)
super_admin_node = worker_nodes.SuperAdminNode(
unique_id=f"SuperAdmin_{self.user_id}",
uuid_string=self.user_id,
worker_email=self.worker_email,
tldraw_snapshot="",
node_storage_path=node_storage_path,
worker_name=self.worker_name,
worker_db_name=self.worker_db_name,
worker_type=self.worker_type
@@ -329,11 +435,17 @@ class NonSchoolUserCreator(UserCreator):
# Create the user node again for the user db
private_user_node = self.create_user_node(self.user_db_name)
# Generate storage path for developer node using Supabase Storage
if self.storage_tools:
dev_dir_created, node_storage_path = self.storage_tools.create_developer_storage_path(self.user_id)
else:
node_storage_path = self._derive_internal_worker_path(self.worker_type)
developer_node = worker_nodes.DeveloperNode(
unique_id=f"Developer_{self.user_id}",
uuid_string=self.user_id,
worker_name=self.worker_name,
worker_email=self.worker_email,
tldraw_snapshot="",
node_storage_path=node_storage_path,
worker_db_name=self.worker_db_name,
worker_type=self.worker_type,
developer_role=self.developer_role
@@ -372,8 +484,101 @@ class NonSchoolUserCreator(UserCreator):
neon.create_or_merge_neontology_relationship(specific_user_rel, database=self.user_db_name, operation='merge')
logger.info("Relationship created between user and specific node")
def create_calendar(self, user_node: user_nodes.UserNode):
calendar_nodes = init_calendar.create_calendar(self.user_db_name, self.calendar_start_date, self.calendar_end_date, attach_to_calendar_node=True, owner_node=user_node)
def create_calendar(self, user_node: user_nodes.UserNode):
calendar_nodes = init_calendar.create_calendar(self.user_db_name, self.calendar_start_date, self.calendar_end_date, attach_to_calendar_node=True, owner_node=user_node, filesystem=self.filesystem)
logger.info(f"Calendar nodes created.")
return calendar_nodes
def _default_date_range():
today = datetime.now().date()
return today, (datetime.now() + timedelta(days=365)).date()
def create_user(
*,
user_id: str,
user_type: str,
username: str,
user_email: str,
user_name: Optional[str] = None,
worker_name: Optional[str] = None,
worker_type: Optional[str] = None,
worker_email: Optional[str] = None,
cc_users_db_name: str = "cc.users",
user_db_name: Optional[str] = None,
worker_db_name: Optional[str] = None,
calendar_start_date: Optional[datetime.date] = None,
calendar_end_date: Optional[datetime.date] = None,
school_node: Optional[Any] = None,
storage_tools=None,
) -> Dict[str, Optional[Any]]:
"""Create a user graph structure in Neo4j.
Args:
user_id: Identifier used as UUID for graph nodes.
user_type: Application-level user type (e.g. email_teacher, developer).
username: Canonical username/slug.
user_email: Contact email for the user node.
user_name: Friendly display name (defaults to username).
worker_name: Friendly name for worker node (defaults to user_name).
worker_type: Worker role (teacher, student, developer, etc.).
worker_email: Email for worker node (defaults to user_email).
cc_users_db_name: Root namespace for user databases (defaults to cc.users).
user_db_name: Fully-qualified target database for the user graph.
worker_db_name: Database for worker entities (usually school private DB).
calendar_start_date/calendar_end_date: Date range for initial calendar seeding.
school_node: Optional school context; if provided a SchoolUserCreator is used.
storage_tools: Optional Supabase storage tools for generating storage paths.
Returns:
Dict describing created nodes keyed by semantic role.
"""
start_date, end_date = calendar_start_date, calendar_end_date
if not start_date or not end_date:
start_date, end_date = _default_date_range()
worker_email = worker_email or user_email
user_name = user_name or username
worker_name = worker_name or user_name
if school_node is not None:
creator = SchoolUserCreator(
user_id=user_id,
cc_users_db_name=cc_users_db_name,
user_type=user_type,
worker_type=worker_type or "teacher",
user_email=user_email,
worker_email=worker_email,
cc_username=username,
user_name=user_name,
worker_name=worker_name,
calendar_start_date=start_date,
calendar_end_date=end_date,
school_node=school_node,
worker_node=None,
storage_tools=storage_tools,
user_db_name=user_db_name,
worker_db_name=worker_db_name,
)
else:
creator = NonSchoolUserCreator(
user_id=user_id,
cc_users_db_name=cc_users_db_name,
user_type=user_type,
worker_type=worker_type or user_type,
user_email=user_email,
worker_email=worker_email,
cc_username=username,
user_name=user_name,
worker_name=worker_name,
calendar_start_date=start_date,
calendar_end_date=end_date,
storage_tools=storage_tools,
user_db_name=user_db_name,
worker_db_name=worker_db_name,
)
return creator.create_user()
modules/database/init/init_user_timetable.py
+49 -49
View File
@@ -7,10 +7,10 @@ import modules.database.tools.neontology_tools as neon
from modules.database.tools.filesystem_tools import ClassroomCopilotFilesystem
from modules.database.schemas.nodes.users import UserNode
from modules.database.schemas.nodes.schools.schools import SubjectClassNode
from modules.database.schemas.nodes.workers.workers import TeacherNode
from modules.database.schemas.nodes.workers.workers import TeacherNode,
from modules.database.schemas.nodes.calendars import CalendarDayNode
from modules.database.schemas.nodes.workers.timetable import (
UserTeacherTimetableNode
UserTeacherTimetableNode, TimetableLessonNode
)
from modules.database.schemas.relationships.entity_timetable_rels import (
EntityHasTimetable
@@ -23,35 +23,35 @@ from modules.database.schemas.relationships.calendar_timetable_rels import (
CalendarDayHasPlannedLesson, PlannedLessonBelongsToCalendarDay
)
def get_school_worker_classes(school_db_name: str, user_unique_id: str, worker_unique_id: str) -> list:
def get_school_worker_classes(school_db_name: str, user_uuid_string: str, worker_uuid_string: str) -> list:
"""
Retrieve all classes for a worker from the school database.
"""
query = """
MATCH (w:Teacher {unique_id: $worker_id})-[:TEACHER_HAS_TIMETABLE]->(tt:TeacherTimetable)
MATCH (w:Teacher {uuid_string: $worker_id})-[:TEACHER_HAS_TIMETABLE]->(tt:TeacherTimetable)
-[:TIMETABLE_HAS_CLASS]->(c:SubjectClass)
RETURN c
"""
with driver.get_driver(db_name=school_db_name).session(database=school_db_name) as session:
result = session.run(query, worker_id=worker_unique_id)
result = session.run(query, worker_id=worker_uuid_string)
classes = [record['c'] for record in result]
if not classes:
logger.warning(f"No classes found for teacher {worker_unique_id} in school database")
logger.warning(f"No classes found for teacher {worker_uuid_string} in school database")
return classes
def get_school_class_periods(school_db_name: str, class_unique_id: str) -> list:
def get_school_class_periods(school_db_name: str, class_uuid_string: str) -> list:
"""
Retrieve all periods for a class from the school database.
"""
query = """
MATCH (c:SubjectClass {unique_id: $class_id})-[:CLASS_HAS_LESSON]->(l:TimetableLesson)
MATCH (c:SubjectClass {uuid_string: $class_id})-[:CLASS_HAS_LESSON]->(l:TimetableLesson)
RETURN l
"""
with driver.get_driver(db_name=school_db_name).session(database=school_db_name) as session:
result = session.run(query, class_id=class_unique_id)
result = session.run(query, class_id=class_uuid_string)
periods = [record['l'] for record in result]
if not periods:
logger.warning(f"No periods found for class {class_unique_id} in school database")
logger.warning(f"No periods found for class {class_uuid_string} in school database")
return periods
def get_user_calendar_nodes(user_db_name: str, user_node: UserNode) -> list:
@@ -60,12 +60,12 @@ def get_user_calendar_nodes(user_db_name: str, user_node: UserNode) -> list:
"""
# First try to find any calendar days to verify the structure
verify_query = """
MATCH (w:User {unique_id: $user_id})
MATCH (w:User {uuid_string: $user_id})
OPTIONAL MATCH (w)-[:HAS_CALENDAR]->(c:Calendar)
OPTIONAL MATCH (c)-[:CALENDAR_INCLUDES_YEAR]->(y:CalendarYear)
OPTIONAL MATCH (y)-[:YEAR_INCLUDES_MONTH]->(m:CalendarMonth)
OPTIONAL MATCH (m)-[:MONTH_INCLUDES_DAY]->(d:CalendarDay)
RETURN w.unique_id as user_id,
RETURN w.uuid_string as user_id,
count(c) as calendar_count,
count(y) as year_count,
count(m) as month_count,
@@ -76,7 +76,7 @@ def get_user_calendar_nodes(user_db_name: str, user_node: UserNode) -> list:
with driver.get_driver(db_name=user_db_name).session(database=user_db_name) as session:
# First check the calendar structure
result = session.run(verify_query, user_id=user_node.unique_id)
result = session.run(verify_query, user_id=user_node.uuid_string)
if stats := result.single():
logger.info(f"Calendar structure for user {stats['user_id']}: "
f"calendars={stats['calendar_count']}, "
@@ -86,50 +86,50 @@ def get_user_calendar_nodes(user_db_name: str, user_node: UserNode) -> list:
f"available years={stats['years']}")
if stats['calendar_count'] == 0:
logger.error(f"No calendar found for user {user_node.unique_id}")
logger.error(f"No calendar found for user {user_node.uuid_string}")
return []
if stats['year_count'] == 0:
logger.error(f"No calendar years found for user {user_node.unique_id}")
logger.error(f"No calendar years found for user {user_node.uuid_string}")
return []
if stats['month_count'] == 0:
logger.error(f"No calendar months found for user {user_node.unique_id}")
logger.error(f"No calendar months found for user {user_node.uuid_string}")
return []
if stats['day_count'] == 0:
logger.error(f"No calendar days found for user {user_node.unique_id}")
logger.error(f"No calendar days found for user {user_node.uuid_string}")
return []
# Get all calendar days without year filter
query = """
MATCH (w:User {unique_id: $user_id})-[:HAS_CALENDAR]->(c:Calendar)
MATCH (w:User {uuid_string: $user_id})-[:HAS_CALENDAR]->(c:Calendar)
-[:CALENDAR_INCLUDES_YEAR]->(y:CalendarYear)
-[:YEAR_INCLUDES_MONTH]->(m:CalendarMonth)
-[:MONTH_INCLUDES_DAY]->(d:CalendarDay)
RETURN d.unique_id as unique_id,
RETURN d.uuid_string as uuid_string,
d.date as date,
d.day_of_week as day_of_week,
d.iso_day as iso_day,
d.path as path
d.node_storage_path as path
ORDER BY d.date
"""
result = session.run(query, user_id=user_node.unique_id)
result = session.run(query, user_id=user_node.uuid_string)
calendar_days = []
for record in result:
calendar_day = CalendarDayNode(
unique_id=record['unique_id'],
uuid_string=record['uuid_string'],
date=record['date'],
day_of_week=record['day_of_week'],
iso_day=record['iso_day'],
path=record['path']
node_storage_path=record['path']
)
calendar_days.append(calendar_day)
if not calendar_days:
logger.error(f"No calendar days found for user {user_node.unique_id}")
logger.error(f"No calendar days found for user {user_node.uuid_string}")
else:
# Log the date range we have
dates = sorted([day.date for day in calendar_days])
logger.info(f"Found {len(calendar_days)} calendar days for user {user_node.unique_id}")
logger.info(f"Found {len(calendar_days)} calendar days for user {user_node.uuid_string}")
logger.info(f"Calendar days range from {dates[0]} to {dates[-1]}")
return calendar_days
@@ -149,7 +149,7 @@ def create_user_worker_timetable(
fs_handler = ClassroomCopilotFilesystem(db_name=user_db_name, init_run_type="user")
# Create teacher timetable directory under the worker's directory
_, worker_timetable_path = fs_handler.create_teacher_timetable_directory(user_worker_node.path)
_, worker_timetable_path = fs_handler.create_teacher_timetable_directory(user_worker_node.node_storage_path)
# Initialize neontology connection
neon.init_neontology_connection()
@@ -157,7 +157,7 @@ def create_user_worker_timetable(
# Get user's calendar nodes
calendar_nodes = get_user_calendar_nodes(user_db_name, user_node)
if not calendar_nodes:
logger.warning(f"No calendar nodes found for user {user_node.unique_id}")
logger.warning(f"No calendar nodes found for user {user_node.uuid_string}")
return {
"status": "error",
"message": "No calendar nodes found for user"
@@ -165,17 +165,17 @@ def create_user_worker_timetable(
try:
# Create UserTeacherTimetableNode
timetable_unique_id = f"UserTeacherTimetable_{user_worker_node.teacher_code}"
timetable_uuid_string = f"UserTeacherTimetable_{user_worker_node.teacher_code}"
worker_timetable = UserTeacherTimetableNode(
unique_id=timetable_unique_id,
uuid_string=timetable_uuid_string,
school_db_name=school_db_name,
school_timetable_id=f"TeacherTimetable_{user_worker_node.teacher_code}",
path=worker_timetable_path
node_storage_path=worker_timetable_path
)
# Create the timetable node and its tldraw file
neon.create_or_merge_neontology_node(worker_timetable, database=user_db_name, operation='merge')
fs_handler.create_default_tldraw_file(worker_timetable.path, worker_timetable.to_dict())
fs_handler.create_default_tldraw_file(worker_timetable.node_storage_path, worker_timetable.to_dict())
# Link timetable to teacher using the correct relationship structure
neon.create_or_merge_neontology_relationship(
@@ -185,9 +185,9 @@ def create_user_worker_timetable(
)
# Get classes from school database
school_classes = get_school_worker_classes(school_db_name, user_node.unique_id, user_worker_node.unique_id)
school_classes = get_school_worker_classes(school_db_name, user_node.uuid_string, user_worker_node.uuid_string)
if not school_classes:
logger.warning(f"No classes found for teacher {user_worker_node.unique_id} in school database")
logger.warning(f"No classes found for teacher {user_worker_node.uuid_string} in school database")
return {
"status": "warning",
"message": "No classes found in school database"
@@ -202,15 +202,15 @@ def create_user_worker_timetable(
# Create SubjectClassNode
subject_class_node = SubjectClassNode(
unique_id=class_data['unique_id'],
uuid_string=class_data['uuid_string'],
subject_class_code=class_data['subject_class_code'],
year_group=class_data['year_group'],
subject=class_data['subject'],
subject_code=class_data['subject_code'],
path=class_path
node_storage_path=class_path
)
neon.create_or_merge_neontology_node(subject_class_node, database=user_db_name, operation='merge')
fs_handler.create_default_tldraw_file(subject_class_node.path, subject_class_node.to_dict())
fs_handler.create_default_tldraw_file(subject_class_node.node_storage_path, subject_class_node.to_dict())
# Link class to timetable
neon.create_or_merge_neontology_relationship(
@@ -220,27 +220,27 @@ def create_user_worker_timetable(
)
# Initialize empty list for this class's lessons
class_lessons[class_data['unique_id']] = []
class_lessons[class_data['uuid_string']] = []
# Get periods from school database
periods = get_school_class_periods(school_db_name, class_data['unique_id'])
periods = get_school_class_periods(school_db_name, class_data['uuid_string'])
if not periods:
logger.warning(f"No periods found for class {class_data['unique_id']} in school database")
logger.warning(f"No periods found for class {class_data['uuid_string']} in school database")
continue
for period_data in periods:
# Create UserTimetableLessonNode
lesson_unique_id = f"UserTimetableLesson_{timetable_unique_id}_{class_name_safe}_{period_data['date']}_{period_data['period_code']}"
timetable_lesson_node = UserTimetableLessonNode(
unique_id=lesson_unique_id,
# Create TimetableLessonNode
lesson_uuid_string = f"UserTimetableLesson_{timetable_uuid_string}_{class_name_safe}_{period_data['date']}_{period_data['period_code']}"
timetable_lesson_node = TimetableLessonNode(
uuid_string=lesson_uuid_string,
subject_class=class_data['subject_class_code'],
date=period_data['date'],
start_time=period_data['start_time'],
end_time=period_data['end_time'],
period_code=period_data['period_code'],
school_db_name=school_db_name,
school_period_id=period_data['unique_id'],
path="Not set" # Will be set after creating directories
school_period_id=period_data['uuid_string'],
node_storage_path="Not set" # Will be set after creating directories
)
if calendar_day := next(
@@ -256,11 +256,11 @@ def create_user_worker_timetable(
class_path,
f"{calendar_day.date}_{period_data['period_code']}"
)
timetable_lesson_node.path = lesson_path
timetable_lesson_node.node_storage_path = lesson_path
# Create and link nodes
neon.create_or_merge_neontology_node(timetable_lesson_node, database=user_db_name, operation='merge')
fs_handler.create_default_tldraw_file(timetable_lesson_node.path, timetable_lesson_node.to_dict())
fs_handler.create_default_tldraw_file(timetable_lesson_node.node_storage_path, timetable_lesson_node.to_dict())
# Link lesson to class
neon.create_or_merge_neontology_relationship(
@@ -280,7 +280,7 @@ def create_user_worker_timetable(
)
# Store the lesson node
class_lessons[class_data['unique_id']].append({
class_lessons[class_data['uuid_string']].append({
'node': timetable_lesson_node,
'date': period_data['date'],
'start_time': period_data['start_time']
@@ -299,7 +299,7 @@ def create_user_worker_timetable(
next_lesson = sorted_lessons[i + 1]['node']
# Skip if current and next lesson are the same node
if current_lesson.unique_id != next_lesson.unique_id:
if current_lesson.uuid_string != next_lesson.uuid_string:
neon.create_or_merge_neontology_relationship(
TimetableLessonFollowsTimetableLesson(
source=current_lesson,
modules/database/init/init_worker_timetable.py
+27 -27
View File
@@ -33,19 +33,19 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
logging.info(f"Initialising filesystem handler...")
fs_handler = ClassroomCopilotFilesystem(db_name=worker_db_name, init_run_type="user")
_, worker_timetable_path = fs_handler.create_teacher_timetable_directory(worker_node.path)
_, worker_timetable_path = fs_handler.create_teacher_timetable_directory(worker_node.node_storage_path)
logging.info(f"Initialising neo4j connection...")
neon.init_neontology_connection()
try:
timetable_unique_id = f"TeacherTimetable_{worker_node.teacher_code}"
timetable_uuid_string = f"TeacherTimetable_{worker_node.teacher_code}"
worker_timetable = TeacherTimetableNode(
unique_id=timetable_unique_id,
path=worker_timetable_path
uuid_string=timetable_uuid_string,
node_storage_path=worker_timetable_path
)
neon.create_or_merge_neontology_node(worker_timetable, database=worker_db_name, operation='merge')
fs_handler.create_default_tldraw_file(worker_timetable.path, worker_timetable.to_dict())
fs_handler.create_default_tldraw_file(worker_timetable.node_storage_path, worker_timetable.to_dict())
neon.create_or_merge_neontology_relationship(
TeacherHasTimetable(source=worker_node, target=worker_timetable),
database=worker_db_name, operation='merge'
@@ -57,28 +57,28 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
for class_name, class_df in class_groups:
if pd.notna(class_name):
class_name_safe = re.sub(r'[^A-Za-z0-9_ ]+', '', class_name)
_, class_path = fs_handler.create_teacher_class_directory(worker_timetable.path, class_name_safe)
_, class_path = fs_handler.create_teacher_class_directory(worker_timetable.node_storage_path, class_name_safe)
subject_class_node_unique_id = f"SubjectClass_{class_name}"
subject_class_node_uuid_string = f"SubjectClass_{class_name}"
subject_class_node = SubjectClassNode(
unique_id=subject_class_node_unique_id,
uuid_string=subject_class_node_uuid_string,
subject_class_code=class_name,
year_group=str(int(class_df['YearGroup'].iloc[0])), # TODO: Hacky fix for the year group being a float
subject=str(class_df['Subject'].iloc[0]),
subject_code=str(class_df['SubjectCode'].iloc[0]),
path=class_path
node_storage_path=class_path
)
neon.create_or_merge_neontology_node(subject_class_node, database=worker_db_name, operation='merge')
logging.info(f"Class node created: {subject_class_node}")
# Create the tldraw file for the node
fs_handler.create_default_tldraw_file(subject_class_node.path, subject_class_node.to_dict())
fs_handler.create_default_tldraw_file(subject_class_node.node_storage_path, subject_class_node.to_dict())
# Link ClassNode to TeacherTimetableNode
neon.create_or_merge_neontology_relationship(
TimetableHasClass(source=worker_timetable, target=subject_class_node),
database=worker_db_name, operation='merge'
)
logging.info(f"Relationship created from {worker_timetable.unique_id} to {subject_class_node.unique_id}")
logging.info(f"Relationship created from {worker_timetable.uuid_string} to {subject_class_node.uuid_string}")
# Link class to corresponding YearGoupSyllabus
@@ -92,7 +92,7 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
SubjectClassBelongsToYearGroupSyllabus(source=subject_class_node, target=year_group_syllabus_node),
database=worker_db_name, operation='merge'
)
logging.info(f"Relationship created from {subject_class_node.unique_id} to {year_group_syllabus_node.unique_id}")
logging.info(f"Relationship created from {subject_class_node.uuid_string} to {year_group_syllabus_node.uuid_string}")
else:
logging.warning(f"No YearGroupSyllabus found for class {class_name} with year group {subject_class_node.year_group} and subject code {subject_class_node.subject_code}")
@@ -125,16 +125,16 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
date = class_lesson['date']
date_safe = date.strftime("%Y-%m-%d")
# Clean the class_name to make it directory-safe (catch all for invalid characters)
timetable_lesson_unique_id = f"TimetableLesson_{timetable_unique_id}_Class_{class_name}_Lesson_{lesson_number}_{date_safe}_{lesson_period_code}"
timetable_lesson_uuid_string = f"TimetableLesson_{timetable_uuid_string}_Class_{class_name}_Lesson_{lesson_number}_{date_safe}_{lesson_period_code}"
timetable_lesson_node = TimetableLessonNode(
unique_id=timetable_lesson_unique_id,
uuid_string=timetable_lesson_uuid_string,
subject_class=class_name,
date=date,
start_time=class_lesson['start_time'].time(), # TODO: This is probably how we should format the start and end time properties for all such nodes
end_time=class_lesson['end_time'].time(),
period_code=lesson_period_code,
path="Not set"
node_storage_path="Not set"
)
neon.create_or_merge_neontology_node(timetable_lesson_node, database=worker_db_name, operation='merge')
logging.info(f"TimetableLessonNode created: {timetable_lesson_node}")
@@ -144,19 +144,19 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
TimetableLessonBelongsToPeriod(source=timetable_lesson_node, target=period_node),
database=worker_db_name, operation='merge'
)
logging.info(f"Relationship created from {timetable_lesson_node.unique_id} to {period_node.unique_id}")
logging.info(f"Relationship created from {timetable_lesson_node.uuid_string} to {period_node.uuid_string}")
# Link TimetableLessonNode to ClassNode
neon.create_or_merge_neontology_relationship(
ClassHasLesson(source=subject_class_node, target=timetable_lesson_node),
database=worker_db_name, operation='merge'
)
logging.info(f"Relationship created from {subject_class_node.unique_id} to {timetable_lesson_node.unique_id}")
logging.info(f"Relationship created from {subject_class_node.uuid_string} to {timetable_lesson_node.uuid_string}")
# Create PlannedLessonNode
planned_lesson_unique_id = f"PlannedLesson_{timetable_unique_id}_Class_{class_name}_Lesson_{lesson_number}_{date_safe}_{lesson_period_code}"
planned_lesson_uuid_string = f"PlannedLesson_{timetable_uuid_string}_Class_{class_name}_Lesson_{lesson_number}_{date_safe}_{lesson_period_code}"
planned_lesson_node = PlannedLessonNode(
unique_id=planned_lesson_unique_id,
uuid_string=planned_lesson_uuid_string,
date=date,
start_time=class_lesson['start_time'].time(),
end_time=class_lesson['end_time'].time(),
@@ -174,7 +174,7 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
learning_statements=None,
learning_resource_codes=None,
learning_resources=None,
path="Not set"
node_storage_path="Not set"
)
# Create the PlannedLessonNode
neon.create_or_merge_neontology_node(planned_lesson_node, database=worker_db_name, operation='merge')
@@ -186,7 +186,7 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
TimetableLessonHasPlannedLesson(source=timetable_lesson_node, target=planned_lesson_node),
database=worker_db_name, operation='merge'
)
logging.info(f"Relationship created from {timetable_lesson_node.unique_id} to {planned_lesson_node.unique_id}")
logging.info(f"Relationship created from {timetable_lesson_node.uuid_string} to {planned_lesson_node.uuid_string}")
lesson_of_same_period += 1
lesson_number += 1
else:
@@ -201,17 +201,17 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
current_node = class_lesson_nodes[i]
i_safe = f"{i:02d}"
_, class_lesson_path = fs_handler.create_teacher_timetable_lesson_directory(class_path, f"{i_safe}_{current_node.date}_{current_node.period_code}")
current_node.path = class_lesson_path
current_node.node_storage_path = class_lesson_path
neon.create_or_merge_neontology_node(current_node, database=worker_db_name, operation='merge')
logging.info(f"TimetableLessonNode directory created and node merged into database: {current_node}")
# Create the tldraw file for the node
fs_handler.create_default_tldraw_file(current_node.path, current_node.to_dict())
fs_handler.create_default_tldraw_file(current_node.node_storage_path, current_node.to_dict())
if previous_node:
neon.create_or_merge_neontology_relationship(
TimetableLessonFollowsTimetableLesson(source=previous_node, target=current_node),
database=worker_db_name, operation='merge'
)
logging.info(f"Sequential relationship created between {previous_node.unique_id} and {current_node.unique_id}")
logging.info(f"Sequential relationship created between {previous_node.uuid_string} and {current_node.uuid_string}")
# Create sequential relationships for PlannedLessonNodes
for i in range(1, len(planned_lesson_nodes)):
@@ -219,17 +219,17 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
current_node = planned_lesson_nodes[i]
i_safe = f"{i:02d}"
_, planned_lesson_path = fs_handler.create_teacher_planned_lesson_directory(class_path, f"{i_safe}_{current_node.date}_{current_node.period_code}")
current_node.path = planned_lesson_path
current_node.node_storage_path = planned_lesson_path
neon.create_or_merge_neontology_node(current_node, database=worker_db_name, operation='merge')
logging.info(f"PlannedLessonNode directory created and node merged into database: {current_node}")
# Create the tldraw file for the node
fs_handler.create_default_tldraw_file(current_node.path, current_node.to_dict())
fs_handler.create_default_tldraw_file(current_node.node_storage_path, current_node.to_dict())
if previous_node:
neon.create_or_merge_neontology_relationship(
PlannedLessonFollowsPlannedLesson(source=previous_node, target=current_node),
database=worker_db_name, operation='merge'
)
logging.info(f"Sequential relationship created between {previous_node.unique_id} and {current_node.unique_id}")
logging.info(f"Sequential relationship created between {previous_node.uuid_string} and {current_node.uuid_string}")
logging.info(f"Successfully initialized worker timetable for worker {worker_node.teacher_code}")
return {"status": "success", "message": "Worker timetable initialized successfully"}