Initial commit
This commit is contained in:
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,260 @@
|
||||
import os
|
||||
from modules.logger_tool import initialise_logger
|
||||
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
import modules.database.schemas.nodes.calendars as calendar_schemas
|
||||
import modules.database.schemas.relationships.calendars as calendar_relationships
|
||||
import modules.database.schemas.relationships.calendar_sequence as calendar_sequence_relationships
|
||||
import modules.database.schemas.relationships.owner_relationships as owner_relationships
|
||||
import modules.database.tools.neontology_tools as neon
|
||||
from datetime import timedelta, datetime
|
||||
|
||||
def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False, owner_node=None, time_chunk_node_length: int = None):
|
||||
logger.info(f"Creating calendar for {start_date} to {end_date}")
|
||||
|
||||
logger.info(f"Initializing Neontology connection")
|
||||
neon.init_neontology_connection()
|
||||
|
||||
created_years = {}
|
||||
created_months = {}
|
||||
created_weeks = {}
|
||||
created_days = {}
|
||||
|
||||
last_year_node = None
|
||||
last_month_node = None
|
||||
last_week_node = None
|
||||
last_day_node = None
|
||||
|
||||
calendar_nodes = {
|
||||
'calendar_node': None,
|
||||
'calendar_year_nodes': [],
|
||||
'calendar_month_nodes': [],
|
||||
'calendar_week_nodes': [],
|
||||
'calendar_day_nodes': []
|
||||
}
|
||||
|
||||
if attach_to_calendar_node and owner_node:
|
||||
logger.info(f"Attaching calendar to owner's node {owner_node.unique_id} in database: {db_name}")
|
||||
owner_unique_id = owner_node.unique_id
|
||||
calendar_unique_id = f"{start_date.strftime('%Y-%m-%d')}_{end_date.strftime('%Y-%m-%d')}"
|
||||
calendar_node = calendar_schemas.CalendarNode(
|
||||
unique_id=calendar_unique_id,
|
||||
name=f"{start_date.strftime('%Y-%m-%d')} to {end_date.strftime('%Y-%m-%d')}",
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
neon.create_or_merge_neontology_node(calendar_node, database=db_name, operation='merge')
|
||||
calendar_nodes['calendar_node'] = calendar_node
|
||||
logger.info(f"Calendar node created: {calendar_node.unique_id}")
|
||||
|
||||
import modules.database.schemas.relationships.owner_relationships as owner_relationships
|
||||
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
owner_relationships.OwnerHasCalendar(source=owner_node, target=calendar_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {owner_node.unique_id} to {calendar_node.unique_id}")
|
||||
elif attach_to_calendar_node and not owner_node:
|
||||
logger.info(f"Creating calendar for {start_date} to {end_date} in database: {db_name}")
|
||||
calendar_node = calendar_schemas.CalendarNode(
|
||||
unique_id=f"{start_date.strftime('%Y-%m-%d')}_{end_date.strftime('%Y-%m-%d')}",
|
||||
name=f"{start_date.strftime('%Y-%m-%d')} to {end_date.strftime('%Y-%m-%d')}",
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
neon.create_or_merge_neontology_node(calendar_node, database=db_name, operation='merge')
|
||||
calendar_nodes['calendar_node'] = calendar_node
|
||||
logger.info(f"Calendar node created: {calendar_node.unique_id}")
|
||||
else:
|
||||
logger.error("Invalid combination of parameters for calendar creation.")
|
||||
raise ValueError("Invalid combination of parameters for calendar creation.")
|
||||
|
||||
current_date = start_date
|
||||
while current_date <= end_date:
|
||||
year = current_date.year
|
||||
month = current_date.month
|
||||
day = current_date.day
|
||||
iso_year, iso_week, iso_weekday = current_date.isocalendar()
|
||||
|
||||
calendar_year_unique_id = f"{year}"
|
||||
|
||||
if year not in created_years:
|
||||
year_node = calendar_schemas.CalendarYearNode(
|
||||
unique_id=calendar_year_unique_id,
|
||||
year=str(year),
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
neon.create_or_merge_neontology_node(year_node, database=db_name, operation='merge')
|
||||
calendar_nodes['calendar_year_nodes'].append(year_node)
|
||||
created_years[year] = year_node
|
||||
logger.info(f"Year node created: {year_node.unique_id}")
|
||||
|
||||
if attach_to_calendar_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
calendar_relationships.CalendarIncludesYear(source=calendar_node, target=year_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {calendar_node.unique_id} to {year_node.unique_id}")
|
||||
if last_year_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
calendar_sequence_relationships.YearFollowsYear(source=last_year_node, target=year_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {last_year_node.unique_id} to {year_node.unique_id}")
|
||||
last_year_node = year_node
|
||||
|
||||
calendar_month_unique_id = f"{year}_{month}"
|
||||
|
||||
month_key = f"{year}-{month}"
|
||||
if month_key not in created_months:
|
||||
month_node = calendar_schemas.CalendarMonthNode(
|
||||
unique_id=calendar_month_unique_id,
|
||||
year=str(year),
|
||||
month=str(month),
|
||||
month_name=datetime(year, month, 1).strftime('%B'),
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
neon.create_or_merge_neontology_node(month_node, database=db_name, operation='merge')
|
||||
calendar_nodes['calendar_month_nodes'].append(month_node)
|
||||
created_months[month_key] = month_node
|
||||
logger.info(f"Month node created: {month_node.unique_id}")
|
||||
|
||||
# Check for the end of year transition for months
|
||||
if last_month_node:
|
||||
if int(month) == 1 and int(last_month_node.month) == 12 and int(last_month_node.year) == year - 1:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
calendar_sequence_relationships.MonthFollowsMonth(source=last_month_node, target=month_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {last_month_node.unique_id} to {month_node.unique_id}")
|
||||
elif int(month) == int(last_month_node.month) + 1:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
calendar_sequence_relationships.MonthFollowsMonth(source=last_month_node, target=month_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {last_month_node.unique_id} to {month_node.unique_id}")
|
||||
last_month_node = month_node
|
||||
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
calendar_relationships.YearIncludesMonth(source=year_node, target=month_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {year_node.unique_id} to {month_node.unique_id}")
|
||||
|
||||
calendar_week_unique_id = f"{iso_year}_{iso_week}"
|
||||
|
||||
week_key = f"{iso_year}-W{iso_week}"
|
||||
if week_key not in created_weeks:
|
||||
# Get the date of the first monday of the week
|
||||
week_start_date = current_date - timedelta(days=current_date.weekday())
|
||||
week_node = calendar_schemas.CalendarWeekNode(
|
||||
unique_id=calendar_week_unique_id,
|
||||
start_date=week_start_date,
|
||||
week_number=str(iso_week),
|
||||
iso_week=f"{iso_year}-W{iso_week:02}",
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
neon.create_or_merge_neontology_node(week_node, database=db_name, operation='merge')
|
||||
calendar_nodes['calendar_week_nodes'].append(week_node)
|
||||
created_weeks[week_key] = week_node
|
||||
logger.info(f"Week node created: {week_node.unique_id}")
|
||||
|
||||
if last_week_node and ((last_week_node.iso_week.split('-')[0] == str(iso_year) and int(last_week_node.week_number) == int(iso_week) - 1) or
|
||||
(last_week_node.iso_week.split('-')[0] != str(iso_year) and int(last_week_node.week_number) == 52 and int(iso_week) == 1)):
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
calendar_sequence_relationships.WeekFollowsWeek(source=last_week_node, target=week_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {last_week_node.unique_id} to {week_node.unique_id}")
|
||||
last_week_node = week_node
|
||||
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
calendar_relationships.YearIncludesWeek(source=year_node, target=week_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {year_node.unique_id} to {week_node.unique_id}")
|
||||
|
||||
# Day node management
|
||||
calendar_day_unique_id = f"{year}_{month}_{day}"
|
||||
|
||||
day_key = f"{year}-{month}-{day}"
|
||||
day_node = calendar_schemas.CalendarDayNode(
|
||||
unique_id=calendar_day_unique_id,
|
||||
date=current_date,
|
||||
day_of_week=current_date.strftime('%A'),
|
||||
iso_day=f"{year}-{month:02}-{day:02}",
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
neon.create_or_merge_neontology_node(day_node, database=db_name, operation='merge')
|
||||
calendar_nodes['calendar_day_nodes'].append(day_node)
|
||||
created_days[day_key] = day_node
|
||||
logger.info(f"Day node created: {day_node.unique_id}")
|
||||
|
||||
if last_day_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
calendar_sequence_relationships.DayFollowsDay(source=last_day_node, target=day_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {last_day_node.unique_id} to {day_node.unique_id}")
|
||||
last_day_node = day_node
|
||||
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
calendar_relationships.MonthIncludesDay(source=month_node, target=day_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {month_node.unique_id} to {day_node.unique_id}")
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
calendar_relationships.WeekIncludesDay(source=week_node, target=day_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {week_node.unique_id} to {day_node.unique_id}")
|
||||
current_date += timedelta(days=1)
|
||||
|
||||
if time_chunk_node_length:
|
||||
time_chunk_interval = time_chunk_node_length
|
||||
# Get every calendar day node and create time chunks of length time_chunk_node minutes for the whole day
|
||||
for day_node in calendar_nodes['calendar_day_nodes']:
|
||||
total_time_chunks_in_day = (24 * 60) / time_chunk_interval
|
||||
for i in range(total_time_chunks_in_day):
|
||||
time_chunk_unique_id = f"{day_node.unique_id}_{i}"
|
||||
time_chunk_start_time = day_node.date.time() + timedelta(minutes=i * time_chunk_interval)
|
||||
time_chunk_end_time = time_chunk_start_time + timedelta(minutes=time_chunk_interval)
|
||||
time_chunk_node = calendar_schemas.CalendarTimeChunkNode(
|
||||
unique_id=time_chunk_unique_id,
|
||||
start_time=time_chunk_start_time,
|
||||
end_time=time_chunk_end_time,
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
neon.create_or_merge_neontology_node(time_chunk_node, database=db_name, operation='merge')
|
||||
calendar_nodes['calendar_time_chunk_nodes'].append(time_chunk_node)
|
||||
logger.info(f"Time chunk node created: {time_chunk_node.unique_id}")
|
||||
# Create a relationship between the time chunk node and the day node
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
calendar_relationships.DayIncludesTimeChunk(source=day_node, target=time_chunk_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {day_node.unique_id} to {time_chunk_node.unique_id}")
|
||||
# Create sequential relationship between the time chunk nodes
|
||||
if i > 0:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
calendar_sequence_relationships.TimeChunkFollowsTimeChunk(source=calendar_nodes['calendar_time_chunk_nodes'][i-1], target=time_chunk_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {calendar_nodes['calendar_time_chunk_nodes'][i-1].unique_id} to {time_chunk_node.unique_id}")
|
||||
|
||||
logger.info(f'Created calendar: {calendar_nodes["calendar_node"].unique_id}')
|
||||
return calendar_nodes
|
||||
@@ -0,0 +1,158 @@
|
||||
import os
|
||||
from modules.logger_tool import initialise_logger
|
||||
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
from modules.database.schemas.nodes.schools.schools import SchoolNode
|
||||
from modules.database.supabase.utils.client import SupabaseServiceRoleClient, CreateBucketOptions
|
||||
import modules.database.init.init_school_timetable as init_school_timetable
|
||||
import modules.database.tools.neontology_tools as neon
|
||||
|
||||
def create_school_buckets(school_id: str, school_type: str, school_name: str, admin_access_token: str) -> dict:
|
||||
"""Create storage buckets for a school
|
||||
Args:
|
||||
school_id: The unique identifier for the school
|
||||
school_type: The type of school (e.g., 'development')
|
||||
school_name: The display name of the school
|
||||
admin_access_token: The admin access token for Supabase operations
|
||||
Returns:
|
||||
Dictionary containing results of bucket creation operations
|
||||
"""
|
||||
logger.info(f"Creating storage buckets for school {school_name} ({school_type}/{school_id})")
|
||||
|
||||
storage_client = SupabaseServiceRoleClient.for_admin(admin_access_token)
|
||||
base_path = f"cc.institutes.{school_type}.{school_id}"
|
||||
|
||||
buckets = [
|
||||
# Main school buckets
|
||||
{
|
||||
"id": f"{base_path}.public",
|
||||
"options": CreateBucketOptions(
|
||||
name=f"{school_type.title()} School Files - {school_name} - Public Files",
|
||||
public=True,
|
||||
file_size_limit=50 * 1024 * 1024,
|
||||
allowed_mime_types=[
|
||||
'image/*', 'video/*', 'application/pdf',
|
||||
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
|
||||
'text/plain', 'text/csv', 'application/json'
|
||||
]
|
||||
)
|
||||
},
|
||||
{
|
||||
"id": f"{base_path}.private",
|
||||
"options": CreateBucketOptions(
|
||||
name=f"{school_type.title()} School Files - {school_name} - Private Files",
|
||||
public=False,
|
||||
file_size_limit=50 * 1024 * 1024,
|
||||
allowed_mime_types=[
|
||||
'image/*', 'video/*', 'application/pdf',
|
||||
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
|
||||
'text/plain', 'text/csv', 'application/json'
|
||||
]
|
||||
)
|
||||
},
|
||||
# Curriculum buckets
|
||||
{
|
||||
"id": f"{base_path}.curriculum.public",
|
||||
"options": CreateBucketOptions(
|
||||
name=f"{school_type.title()} School Files - {school_name} - Curriculum Public Files",
|
||||
public=True,
|
||||
file_size_limit=50 * 1024 * 1024,
|
||||
allowed_mime_types=[
|
||||
'image/*', 'video/*', 'application/pdf',
|
||||
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
|
||||
'text/plain', 'text/csv', 'application/json'
|
||||
]
|
||||
)
|
||||
},
|
||||
{
|
||||
"id": f"{base_path}.curriculum.private",
|
||||
"options": CreateBucketOptions(
|
||||
name=f"{school_type.title()} School Files - {school_name} - Curriculum Private Files",
|
||||
public=False,
|
||||
file_size_limit=50 * 1024 * 1024,
|
||||
allowed_mime_types=[
|
||||
'image/*', 'video/*', 'application/pdf',
|
||||
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
|
||||
'text/plain', 'text/csv', 'application/json'
|
||||
]
|
||||
)
|
||||
}
|
||||
]
|
||||
|
||||
results = {}
|
||||
for bucket in buckets:
|
||||
try:
|
||||
result = storage_client.create_bucket(bucket["id"], bucket["options"])
|
||||
results[bucket["id"]] = {
|
||||
"status": "success",
|
||||
"result": result
|
||||
}
|
||||
logger.info(f"Successfully created bucket {bucket['id']}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating school bucket {bucket['id']}: {str(e)}")
|
||||
results[bucket["id"]] = {
|
||||
"status": "error",
|
||||
"error": str(e)
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
def create_school(db_name: str, id: str, name: str, website: str, school_type: str, is_public: bool = True, school_node: SchoolNode | None = None, dataframes=None):
|
||||
if not name or not id or not website or not school_type:
|
||||
logger.error("School name, id, website and school_type are required to create a school.")
|
||||
raise ValueError("School name, id, website and school_type are required to create a school.")
|
||||
|
||||
logger.info(f"Initialising neo4j connection...")
|
||||
neon.init_neontology_connection()
|
||||
|
||||
# Create School Node if not provided
|
||||
if not school_node:
|
||||
if is_public:
|
||||
school_node = SchoolNode(
|
||||
unique_id=f'School_{id}',
|
||||
tldraw_snapshot="",
|
||||
id=id,
|
||||
name=name,
|
||||
website=website,
|
||||
school_type=school_type
|
||||
)
|
||||
else:
|
||||
# Create private school node with default values
|
||||
school_node = SchoolNode(
|
||||
unique_id=f'School_{id}',
|
||||
tldraw_snapshot="",
|
||||
id=id,
|
||||
name=name,
|
||||
website=website,
|
||||
school_type=school_type,
|
||||
establishment_number="0000",
|
||||
establishment_name=name,
|
||||
establishment_type="Default",
|
||||
establishment_status="Open",
|
||||
phase_of_education="All",
|
||||
statutory_low_age=11,
|
||||
statutory_high_age=18,
|
||||
school_capacity=1000
|
||||
)
|
||||
|
||||
# First create/merge the school node in the main cc.institutes database
|
||||
logger.info(f"Creating school node in main cc.institutes database...")
|
||||
neon.create_or_merge_neontology_node(school_node, database="cc.institutes", operation='merge')
|
||||
|
||||
# Then create/merge the school node in the specific school database
|
||||
logger.info(f"Creating school node in specific database {db_name}...")
|
||||
neon.create_or_merge_neontology_node(school_node, database=db_name, operation='merge')
|
||||
|
||||
school_nodes = {
|
||||
'school_node': school_node,
|
||||
'db_name': db_name
|
||||
}
|
||||
|
||||
if dataframes is not None:
|
||||
logger.info(f"Creating school timetable for {name} with {len(dataframes)} dataframes...")
|
||||
school_timetable_nodes = init_school_timetable.create_school_timetable(dataframes, db_name, school_node)
|
||||
school_nodes['school_timetable_nodes'] = school_timetable_nodes
|
||||
else:
|
||||
logger.warning(f"No dataframes provided for {name}, skipping school timetable...")
|
||||
|
||||
logger.info(f"School {name} created successfully...")
|
||||
return school_nodes
|
||||
@@ -0,0 +1,697 @@
|
||||
import os
|
||||
from modules.logger_tool import initialise_logger
|
||||
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
import pandas as pd
|
||||
|
||||
import modules.database.tools.neontology_tools as neon
|
||||
import modules.database.schemas.nodes.schools.schools as school_nodes
|
||||
import modules.database.schemas.nodes.schools.curriculum as curriculum_nodes
|
||||
import modules.database.schemas.nodes.schools.pastoral as pastoral_nodes
|
||||
import modules.database.schemas.nodes.structures.schools as school_structures
|
||||
import modules.database.schemas.relationships.curriculum_relationships as curriculum_relationships
|
||||
import modules.database.schemas.relationships.entity_relationships as ent_rels
|
||||
import modules.database.schemas.relationships.entity_curriculum_rels as ent_cur_rels
|
||||
|
||||
# Default values for nodes
|
||||
default_topic_values = {
|
||||
'topic_assessment_type': 'Null',
|
||||
'topic_type': 'Null',
|
||||
'total_number_of_lessons_for_topic': '1',
|
||||
'topic_title': 'Null'
|
||||
}
|
||||
|
||||
default_topic_lesson_values = {
|
||||
'topic_lesson_title': 'Null',
|
||||
'topic_lesson_type': 'Null',
|
||||
'topic_lesson_length': '1',
|
||||
'topic_lesson_suggested_activities': 'Null',
|
||||
'topic_lesson_skills_learned': 'Null',
|
||||
'topic_lesson_weblinks': 'Null',
|
||||
}
|
||||
|
||||
default_learning_statement_values = {
|
||||
'lesson_learning_statement': 'Null',
|
||||
'lesson_learning_statement_type': 'Student learning outcome'
|
||||
}
|
||||
|
||||
# Helper function to sort year groups numerically where possible
|
||||
def sort_year_groups(df):
|
||||
df = df.copy()
|
||||
df['YearGroupNumeric'] = pd.to_numeric(df['YearGroup'], errors='coerce')
|
||||
return df.sort_values(by='YearGroupNumeric')
|
||||
|
||||
def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_node: school_nodes.SchoolNode):
|
||||
|
||||
logger.info(f"Initialising neo4j connection...")
|
||||
neon.init_neontology_connection()
|
||||
|
||||
keystagesyllabus_df = dataframes['keystagesyllabuses']
|
||||
yeargroupsyllabus_df = dataframes['yeargroupsyllabuses']
|
||||
topic_df = dataframes['topics']
|
||||
lesson_df = dataframes['lessons']
|
||||
statement_df = dataframes['statements']
|
||||
# resource_df = dataframes['resources'] # TODO
|
||||
|
||||
node_library = {}
|
||||
node_library['key_stage_nodes'] = {}
|
||||
node_library['year_group_nodes'] = {}
|
||||
node_library['key_stage_syllabus_nodes'] = {}
|
||||
node_library['year_group_syllabus_nodes'] = {}
|
||||
node_library['topic_nodes'] = {}
|
||||
node_library['topic_lesson_nodes'] = {}
|
||||
node_library['statement_nodes'] = {}
|
||||
node_library['department_nodes'] = {}
|
||||
node_library['subject_nodes'] = {}
|
||||
curriculum_node = None
|
||||
pastoral_node = None
|
||||
key_stage_nodes_created = {}
|
||||
year_group_nodes_created = {}
|
||||
last_year_group_node = None
|
||||
last_key_stage_node = None
|
||||
|
||||
# Create Department Structure node
|
||||
department_structure_node_unique_id = f"DepartmentStructure_{school_node.unique_id}"
|
||||
department_structure_node = school_structures.DepartmentStructureNode(
|
||||
unique_id=department_structure_node_unique_id,
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
# Create in school database only
|
||||
neon.create_or_merge_neontology_node(department_structure_node, database=db_name, operation='merge')
|
||||
node_library['department_structure_node'] = department_structure_node
|
||||
|
||||
# Link Department Structure to School
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
ent_rels.SchoolHasDepartmentStructure(source=school_node, target=department_structure_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created department structure node and linked to school")
|
||||
|
||||
curriculum_structure_node_unique_id = f"CurriculumStructure_{school_node.unique_id}"
|
||||
curriculum_node = school_structures.CurriculumStructureNode(
|
||||
unique_id=curriculum_structure_node_unique_id,
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
# Create in school database only
|
||||
neon.create_or_merge_neontology_node(curriculum_node, database=db_name, operation='merge')
|
||||
node_library['curriculum_node'] = curriculum_node
|
||||
|
||||
# Create relationship in school database only
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
ent_cur_rels.SchoolHasCurriculumStructure(source=school_node, target=curriculum_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created curriculum node and relationship with school")
|
||||
|
||||
pastoral_structure_node_unique_id = f"PastoralStructure_{school_node.unique_id}"
|
||||
pastoral_node = school_structures.PastoralStructureNode(
|
||||
unique_id=pastoral_structure_node_unique_id,
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
neon.create_or_merge_neontology_node(pastoral_node, database=db_name, operation='merge')
|
||||
node_library['pastoral_node'] = pastoral_node
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
ent_cur_rels.SchoolHasPastoralStructure(source=school_node, target=pastoral_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created pastoral node and relationship with school")
|
||||
|
||||
# Create departments and subjects
|
||||
# First get unique departments
|
||||
unique_departments = keystagesyllabus_df['Department'].dropna().unique()
|
||||
|
||||
for department_name in unique_departments:
|
||||
department_unique_id = f"Department_{school_node.unique_id}_{department_name.replace(' ', '_')}"
|
||||
department_node = school_nodes.DepartmentNode(
|
||||
unique_id=department_unique_id,
|
||||
name=department_name,
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
# Create department in school database only
|
||||
neon.create_or_merge_neontology_node(department_node, database=db_name, operation='merge')
|
||||
node_library['department_nodes'][department_name] = department_node
|
||||
|
||||
# Link department to department structure in school database
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
ent_rels.DepartmentStructureHasDepartment(source=department_structure_node, target=department_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created department node for {department_name} and linked to department structure")
|
||||
|
||||
# Create subjects and link to departments
|
||||
# First get unique subjects from key stage syllabuses (which have department info)
|
||||
unique_subjects = keystagesyllabus_df[['Subject', 'SubjectCode', 'Department']].drop_duplicates()
|
||||
|
||||
# Then add any additional subjects from year group syllabuses (without department info)
|
||||
additional_subjects = yeargroupsyllabus_df[['Subject', 'SubjectCode']].drop_duplicates()
|
||||
additional_subjects = additional_subjects[~additional_subjects['SubjectCode'].isin(unique_subjects['SubjectCode'])]
|
||||
|
||||
# Process subjects from key stage syllabuses first (these have department info)
|
||||
for _, subject_row in unique_subjects.iterrows():
|
||||
subject_unique_id = f"Subject_{school_node.unique_id}_{subject_row['SubjectCode']}"
|
||||
department_node = node_library['department_nodes'].get(subject_row['Department'])
|
||||
if not department_node:
|
||||
logger.warning(f"No department found for subject {subject_row['Subject']} with code {subject_row['SubjectCode']}")
|
||||
continue
|
||||
|
||||
subject_node = curriculum_nodes.SubjectNode(
|
||||
unique_id=subject_unique_id,
|
||||
id=subject_row['SubjectCode'],
|
||||
name=subject_row['Subject'],
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
# Create subject in both databases
|
||||
neon.create_or_merge_neontology_node(subject_node, database=db_name, operation='merge')
|
||||
neon.create_or_merge_neontology_node(subject_node, database=curriculum_db_name, operation='merge')
|
||||
node_library['subject_nodes'][subject_row['Subject']] = subject_node
|
||||
|
||||
# Link subject to department in school database only
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
ent_rels.DepartmentManagesSubject(source=department_node, target=subject_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created subject node for {subject_row['Subject']} and linked to department {subject_row['Department']}")
|
||||
|
||||
# Process any additional subjects from year group syllabuses (these won't have department info)
|
||||
for _, subject_row in additional_subjects.iterrows():
|
||||
subject_unique_id = f"Subject_{school_node.unique_id}_{subject_row['SubjectCode']}"
|
||||
# Create in a special "Unassigned" department
|
||||
unassigned_dept_name = "Unassigned Department"
|
||||
if unassigned_dept_name not in node_library['department_nodes']:
|
||||
department_node = school_nodes.DepartmentNode(
|
||||
unique_id=f"Department_{school_node.unique_id}_Unassigned",
|
||||
name=unassigned_dept_name,
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
neon.create_or_merge_neontology_node(department_node, database=db_name, operation='merge')
|
||||
node_library['department_nodes'][unassigned_dept_name] = department_node
|
||||
|
||||
# Link unassigned department to department structure
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
ent_rels.DepartmentStructureHasDepartment(source=department_structure_node, target=department_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created unassigned department node and linked to department structure")
|
||||
|
||||
subject_node = curriculum_nodes.SubjectNode(
|
||||
unique_id=subject_unique_id,
|
||||
id=subject_row['SubjectCode'],
|
||||
name=subject_row['Subject'],
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
# Create subject in both databases
|
||||
neon.create_or_merge_neontology_node(subject_node, database=db_name, operation='merge')
|
||||
neon.create_or_merge_neontology_node(subject_node, database=curriculum_db_name, operation='merge')
|
||||
node_library['subject_nodes'][subject_row['Subject']] = subject_node
|
||||
|
||||
# Link subject to unassigned department in school database only
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
ent_rels.DepartmentManagesSubject(
|
||||
source=node_library['department_nodes'][unassigned_dept_name],
|
||||
target=subject_node
|
||||
),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.warning(f"Created subject node for {subject_row['Subject']} in unassigned department")
|
||||
|
||||
# Process key stages and syllabuses
|
||||
logger.info(f"Processing key stages")
|
||||
last_key_stage_node = None
|
||||
# Track last syllabus nodes per subject
|
||||
last_key_stage_syllabus_nodes = {} # Dictionary to track last key stage syllabus node per subject
|
||||
last_year_group_syllabus_nodes = {} # Dictionary to track last year group syllabus node per subject
|
||||
topics_processed = set() # Track which topics have been processed
|
||||
lessons_processed = set() # Track which lessons have been processed
|
||||
statements_processed = set() # Track which statements have been processed
|
||||
|
||||
# First create all key stage nodes and key stage syllabus nodes
|
||||
for index, ks_row in keystagesyllabus_df.sort_values('KeyStage').iterrows():
|
||||
key_stage = str(ks_row['KeyStage'])
|
||||
logger.debug(f"Processing key stage syllabus row - Subject: {ks_row['Subject']}, Key Stage: {key_stage}")
|
||||
|
||||
subject_node = node_library['subject_nodes'].get(ks_row['Subject'])
|
||||
if not subject_node:
|
||||
logger.warning(f"No subject node found for subject {ks_row['Subject']}")
|
||||
continue
|
||||
|
||||
if key_stage not in key_stage_nodes_created:
|
||||
key_stage_node_unique_id = f"KeyStage_{curriculum_node.unique_id}_KStg{key_stage}"
|
||||
key_stage_node = curriculum_nodes.KeyStageNode(
|
||||
unique_id=key_stage_node_unique_id,
|
||||
name=f"Key Stage {key_stage}",
|
||||
key_stage=str(key_stage),
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
# Create key stage node in both databases
|
||||
neon.create_or_merge_neontology_node(key_stage_node, database=db_name, operation='merge')
|
||||
neon.create_or_merge_neontology_node(key_stage_node, database=curriculum_db_name, operation='merge')
|
||||
key_stage_nodes_created[key_stage] = key_stage_node
|
||||
node_library['key_stage_nodes'][key_stage] = key_stage_node
|
||||
|
||||
# Create relationship with curriculum structure in school database only
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.CurriculumStructureIncludesKeyStage(source=curriculum_node, target=key_stage_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created key stage node {key_stage_node_unique_id} and relationship with curriculum structure")
|
||||
|
||||
# Create sequential relationship between key stages in both databases
|
||||
if last_key_stage_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.KeyStageFollowsKeyStage(source=last_key_stage_node, target=key_stage_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.KeyStageFollowsKeyStage(source=last_key_stage_node, target=key_stage_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created sequential relationship between key stages {last_key_stage_node.unique_id} and {key_stage_node.unique_id}")
|
||||
last_key_stage_node = key_stage_node
|
||||
|
||||
# Create key stage syllabus under the subject's curriculum directory
|
||||
key_stage_syllabus_node_unique_id = f"KeyStageSyllabus_{curriculum_node.unique_id}_{ks_row['Title'].replace(' ', '')}"
|
||||
logger.debug(f"Creating key stage syllabus node for {ks_row['Subject']} KS{key_stage} with ID {ks_row['ID']}")
|
||||
|
||||
key_stage_syllabus_node_unique_id = f"KeyStageSyllabus_{curriculum_node.unique_id}_{ks_row['Title'].replace(' ', '')}"
|
||||
key_stage_syllabus_node = curriculum_nodes.KeyStageSyllabusNode(
|
||||
unique_id=key_stage_syllabus_node_unique_id,
|
||||
id=ks_row['ID'],
|
||||
name=ks_row['Title'],
|
||||
key_stage=str(ks_row['KeyStage']),
|
||||
subject_name=ks_row['Subject'],
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
# Create key stage syllabus node in both databases
|
||||
neon.create_or_merge_neontology_node(key_stage_syllabus_node, database=db_name, operation='merge')
|
||||
neon.create_or_merge_neontology_node(key_stage_syllabus_node, database=curriculum_db_name, operation='merge')
|
||||
node_library['key_stage_syllabus_nodes'][ks_row['ID']] = key_stage_syllabus_node
|
||||
logger.debug(f"Created key stage syllabus node {key_stage_syllabus_node_unique_id} for {ks_row['Subject']} KS{key_stage}")
|
||||
|
||||
# Link key stage syllabus to its subject in both databases
|
||||
if subject_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.SubjectHasKeyStageSyllabus(source=subject_node, target=key_stage_syllabus_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.SubjectHasKeyStageSyllabus(source=subject_node, target=key_stage_syllabus_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship between subject {subject_node.unique_id} and key stage syllabus {key_stage_syllabus_node.unique_id}")
|
||||
|
||||
# Link key stage syllabus to its key stage in both databases
|
||||
key_stage_node = key_stage_nodes_created.get(key_stage)
|
||||
if key_stage_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.KeyStageIncludesKeyStageSyllabus(source=key_stage_node, target=key_stage_syllabus_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.KeyStageIncludesKeyStageSyllabus(source=key_stage_node, target=key_stage_syllabus_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship between key stage {key_stage_node.unique_id} and key stage syllabus {key_stage_syllabus_node.unique_id}")
|
||||
|
||||
# Create sequential relationship between key stage syllabuses in both databases
|
||||
last_key_stage_syllabus_node = last_key_stage_syllabus_nodes.get(ks_row['Subject'])
|
||||
if last_key_stage_syllabus_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.KeyStageSyllabusFollowsKeyStageSyllabus(source=last_key_stage_syllabus_node, target=key_stage_syllabus_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.KeyStageSyllabusFollowsKeyStageSyllabus(source=last_key_stage_syllabus_node, target=key_stage_syllabus_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created sequential relationship between key stage syllabuses {last_key_stage_syllabus_node.unique_id} and {key_stage_syllabus_node.unique_id}")
|
||||
last_key_stage_syllabus_nodes[ks_row['Subject']] = key_stage_syllabus_node
|
||||
|
||||
# Now process year groups and their syllabuses
|
||||
for index, ks_row in keystagesyllabus_df.sort_values('KeyStage').iterrows():
|
||||
key_stage = str(ks_row['KeyStage'])
|
||||
related_yeargroups = sort_year_groups(yeargroupsyllabus_df[yeargroupsyllabus_df['KeyStage'] == ks_row['KeyStage']])
|
||||
|
||||
logger.info(f"Processing year groups for key stage {key_stage}")
|
||||
for yg_index, yg_row in related_yeargroups.iterrows():
|
||||
year_group = yg_row['YearGroup']
|
||||
subject_code = yg_row['SubjectCode']
|
||||
numeric_year_group = pd.to_numeric(year_group, errors='coerce')
|
||||
|
||||
if pd.notna(numeric_year_group):
|
||||
numeric_year_group = int(numeric_year_group)
|
||||
if numeric_year_group not in year_group_nodes_created:
|
||||
year_group_node_unique_id = f"YearGroup_{school_node.unique_id}_YGrp{numeric_year_group}"
|
||||
year_group_node = pastoral_nodes.YearGroupNode(
|
||||
unique_id=year_group_node_unique_id,
|
||||
year_group=str(numeric_year_group),
|
||||
name=f"Year {numeric_year_group}, {year_group}",
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
# Create year group node in both databases but use same directory
|
||||
neon.create_or_merge_neontology_node(year_group_node, database=db_name, operation='merge')
|
||||
neon.create_or_merge_neontology_node(year_group_node, database=curriculum_db_name, operation='merge')
|
||||
|
||||
# Create sequential relationship between year groups in both databases
|
||||
if last_year_group_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.YearGroupFollowsYearGroup(source=last_year_group_node, target=year_group_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.YearGroupFollowsYearGroup(source=last_year_group_node, target=year_group_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created sequential relationship between year groups {last_year_group_node.unique_id} and {year_group_node.unique_id} across key stages")
|
||||
last_year_group_node = year_group_node
|
||||
|
||||
# Create relationship with Pastoral Structure in school database only
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.PastoralStructureIncludesYearGroup(source=pastoral_node, target=year_group_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created year group node {year_group_node_unique_id} and relationship with pastoral structure")
|
||||
|
||||
year_group_nodes_created[numeric_year_group] = year_group_node
|
||||
node_library['year_group_nodes'][str(numeric_year_group)] = year_group_node
|
||||
|
||||
# Create year group syllabus nodes in both databases
|
||||
year_group_node = year_group_nodes_created.get(numeric_year_group)
|
||||
if year_group_node:
|
||||
year_group_syllabus_node_unique_id = f"YearGroupSyllabus_{school_node.unique_id}_{yg_row['ID']}"
|
||||
year_group_syllabus_node = pastoral_nodes.YearGroupSyllabusNode(
|
||||
unique_id=year_group_syllabus_node_unique_id,
|
||||
id=yg_row['ID'],
|
||||
name=yg_row['Title'],
|
||||
year_group=str(yg_row['YearGroup']),
|
||||
subject_name=yg_row['Subject'],
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
|
||||
# Create year group syllabus node in both databases but use same directory
|
||||
neon.create_or_merge_neontology_node(year_group_syllabus_node, database=db_name, operation='merge')
|
||||
neon.create_or_merge_neontology_node(year_group_syllabus_node, database=curriculum_db_name, operation='merge')
|
||||
node_library['year_group_syllabus_nodes'][yg_row['ID']] = year_group_syllabus_node
|
||||
|
||||
# Create sequential relationship between year group syllabuses in both databases
|
||||
last_year_group_syllabus_node = last_year_group_syllabus_nodes.get(yg_row['Subject'])
|
||||
# Only create sequential relationship if this year group is higher than the last one
|
||||
if last_year_group_syllabus_node:
|
||||
last_year = pd.to_numeric(last_year_group_syllabus_node.year_group, errors='coerce')
|
||||
current_year = pd.to_numeric(year_group_syllabus_node.year_group, errors='coerce')
|
||||
if pd.notna(last_year) and pd.notna(current_year) and current_year > last_year:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.YearGroupSyllabusFollowsYearGroupSyllabus(source=last_year_group_syllabus_node, target=year_group_syllabus_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.YearGroupSyllabusFollowsYearGroupSyllabus(source=last_year_group_syllabus_node, target=year_group_syllabus_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created sequential relationship between year group syllabuses {last_year_group_syllabus_node.unique_id} and {year_group_syllabus_node.unique_id}")
|
||||
last_year_group_syllabus_nodes[yg_row['Subject']] = year_group_syllabus_node
|
||||
|
||||
# Create relationships in both databases using MATCH to avoid cartesian products
|
||||
subject_node = node_library['subject_nodes'].get(yg_row['Subject'])
|
||||
if subject_node:
|
||||
# Link to subject
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.SubjectHasYearGroupSyllabus(source=subject_node, target=year_group_syllabus_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.SubjectHasYearGroupSyllabus(source=subject_node, target=year_group_syllabus_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship between subject {subject_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
|
||||
|
||||
# Link to year group
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.YearGroupHasYearGroupSyllabus(source=year_group_node, target=year_group_syllabus_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.YearGroupHasYearGroupSyllabus(source=year_group_node, target=year_group_syllabus_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship between year group {year_group_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
|
||||
|
||||
# Link to key stage syllabus if it exists for the same subject
|
||||
key_stage_syllabus_node = node_library['key_stage_syllabus_nodes'].get(ks_row['ID'])
|
||||
if key_stage_syllabus_node and yg_row['Subject'] == ks_row['Subject']:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.KeyStageSyllabusIncludesYearGroupSyllabus(source=key_stage_syllabus_node, target=year_group_syllabus_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.KeyStageSyllabusIncludesYearGroupSyllabus(source=key_stage_syllabus_node, target=year_group_syllabus_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship between key stage syllabus {key_stage_syllabus_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
|
||||
|
||||
# Process topics for this year group syllabus only if not already processed
|
||||
topics_for_syllabus = topic_df[topic_df['SyllabusYearID'] == yg_row['ID']]
|
||||
for _, topic_row in topics_for_syllabus.iterrows():
|
||||
if topic_row['TopicID'] in topics_processed:
|
||||
continue
|
||||
topics_processed.add(topic_row['TopicID'])
|
||||
|
||||
# Get the correct subject from the topic row
|
||||
topic_subject = topic_row['SyllabusSubject']
|
||||
topic_key_stage = topic_row['SyllabusKeyStage']
|
||||
|
||||
logger.debug(f"Processing topic {topic_row['TopicID']} for subject {topic_subject} and key stage {topic_key_stage}")
|
||||
logger.debug(f"Available key stage syllabus nodes: {[node.subject_name + '_KS' + node.key_stage for node in node_library['key_stage_syllabus_nodes'].values()]}")
|
||||
|
||||
# Find the key stage syllabus node by iterating through all nodes
|
||||
matching_syllabus_node = None
|
||||
for syllabus_node in node_library['key_stage_syllabus_nodes'].values():
|
||||
logger.debug(f"Checking syllabus node - Subject: {syllabus_node.subject_name}, Key Stage: {syllabus_node.key_stage}")
|
||||
logger.debug(f"Comparing with - Subject: {topic_subject}, Key Stage: {str(topic_key_stage)}")
|
||||
logger.debug(f"Types - Node Subject: {type(syllabus_node.subject_name)}, Topic Subject: {type(topic_subject)}")
|
||||
logger.debug(f"Types - Node Key Stage: {type(syllabus_node.key_stage)}, Topic Key Stage: {type(str(topic_key_stage))}")
|
||||
|
||||
if (syllabus_node.subject_name == topic_subject and
|
||||
syllabus_node.key_stage == str(topic_key_stage)):
|
||||
matching_syllabus_node = syllabus_node
|
||||
logger.debug(f"Found matching syllabus node: {syllabus_node.unique_id}")
|
||||
break
|
||||
|
||||
if not matching_syllabus_node:
|
||||
logger.warning(f"No key stage syllabus node found for subject {topic_subject} and key stage {topic_key_stage}, skipping topic creation")
|
||||
continue
|
||||
|
||||
topic_node_unique_id = f"Topic_{matching_syllabus_node.unique_id}_{topic_row['TopicID']}"
|
||||
topic_node = curriculum_nodes.TopicNode(
|
||||
unique_id=topic_node_unique_id,
|
||||
id=topic_row['TopicID'],
|
||||
name=topic_row.get('TopicTitle', default_topic_values['topic_title']),
|
||||
total_number_of_lessons_for_topic=str(topic_row.get('TotalNumberOfLessonsForTopic', default_topic_values['total_number_of_lessons_for_topic'])),
|
||||
type=topic_row.get('TopicType', default_topic_values['topic_type']),
|
||||
assessment_type=topic_row.get('TopicAssessmentType', default_topic_values['topic_assessment_type']),
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
# Create topic node in curriculum database only
|
||||
neon.create_or_merge_neontology_node(topic_node, database=curriculum_db_name, operation='merge')
|
||||
node_library['topic_nodes'][topic_row['TopicID']] = topic_node
|
||||
|
||||
# Link topic to key stage syllabus as well as year group syllabus
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.KeyStageSyllabusIncludesTopic(source=matching_syllabus_node, target=topic_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.YearGroupSyllabusIncludesTopic(source=year_group_syllabus_node, target=topic_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationships between topic {topic_node_unique_id} and key stage syllabus {matching_syllabus_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
|
||||
|
||||
# Process lessons for this topic only if not already processed
|
||||
lessons_for_topic = lesson_df[
|
||||
(lesson_df['TopicID'] == topic_row['TopicID']) &
|
||||
(lesson_df['SyllabusSubject'] == topic_subject)
|
||||
].copy()
|
||||
lessons_for_topic.loc[:, 'Lesson'] = lessons_for_topic['Lesson'].astype(str)
|
||||
lessons_for_topic = lessons_for_topic.sort_values('Lesson')
|
||||
|
||||
previous_lesson_node = None
|
||||
for _, lesson_row in lessons_for_topic.iterrows():
|
||||
if lesson_row['LessonID'] in lessons_processed:
|
||||
continue
|
||||
lessons_processed.add(lesson_row['LessonID'])
|
||||
|
||||
lesson_node = curriculum_nodes.TopicLessonNode(
|
||||
unique_id=f"TopicLesson_{topic_node_unique_id}_{lesson_row['LessonID']}",
|
||||
id=lesson_row['LessonID'],
|
||||
name=lesson_row.get('LessonTitle', default_topic_lesson_values['topic_lesson_title']),
|
||||
type=lesson_row.get('LessonType', default_topic_lesson_values['topic_lesson_type']),
|
||||
length=str(lesson_row.get('SuggestedNumberOfPeriodsForLesson', default_topic_lesson_values['topic_lesson_length'])),
|
||||
suggested_activities=str(lesson_row.get('SuggestedActivities', default_topic_lesson_values['topic_lesson_suggested_activities'])),
|
||||
skills_learned=str(lesson_row.get('SkillsLearned', default_topic_lesson_values['topic_lesson_skills_learned'])),
|
||||
weblinks=str(lesson_row.get('WebLinks', default_topic_lesson_values['topic_lesson_weblinks'])),
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
# Create lesson node in curriculum database only
|
||||
neon.create_or_merge_neontology_node(lesson_node, database=curriculum_db_name, operation='merge')
|
||||
node_library['topic_lesson_nodes'][lesson_row['LessonID']] = lesson_node
|
||||
|
||||
# Link lesson to topic
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.TopicIncludesTopicLesson(source=topic_node, target=lesson_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created lesson node {lesson_node.unique_id} and relationship with topic {topic_node.unique_id}")
|
||||
|
||||
# Create sequential relationships between lessons
|
||||
if lesson_row['Lesson'].isdigit() and previous_lesson_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.TopicLessonFollowsTopicLesson(source=previous_lesson_node, target=lesson_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.unique_id} and {lesson_node.unique_id}")
|
||||
previous_lesson_node = lesson_node
|
||||
|
||||
# Process learning statements for this lesson only if not already processed
|
||||
statements_for_lesson = statement_df[
|
||||
(statement_df['LessonID'] == lesson_row['LessonID']) &
|
||||
(statement_df['SyllabusSubject'] == topic_subject)
|
||||
]
|
||||
for _, statement_row in statements_for_lesson.iterrows():
|
||||
if statement_row['StatementID'] in statements_processed:
|
||||
continue
|
||||
statements_processed.add(statement_row['StatementID'])
|
||||
statement_node = curriculum_nodes.LearningStatementNode(
|
||||
unique_id=f"LearningStatement_{lesson_node.unique_id}_{statement_row['StatementID']}",
|
||||
id=statement_row['StatementID'],
|
||||
name=statement_row.get('LearningStatement', default_learning_statement_values['lesson_learning_statement']),
|
||||
type=statement_row.get('StatementType', default_learning_statement_values['lesson_learning_statement_type']),
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
# Create statement node in curriculum database only
|
||||
neon.create_or_merge_neontology_node(statement_node, database=curriculum_db_name, operation='merge')
|
||||
node_library['statement_nodes'][statement_row['StatementID']] = statement_node
|
||||
|
||||
# Link learning statement to lesson
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.LessonIncludesLearningStatement(source=lesson_node, target=statement_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created learning statement node {statement_node.unique_id} and relationship with lesson {lesson_node.unique_id}")
|
||||
else:
|
||||
logger.warning(f"No year group node found for year group {year_group}, skipping syllabus creation")
|
||||
|
||||
# After processing all year groups and their syllabuses, process any remaining topics
|
||||
logger.info("Processing topics without year groups")
|
||||
for _, topic_row in topic_df.iterrows():
|
||||
if topic_row['TopicID'] in topics_processed:
|
||||
continue
|
||||
|
||||
topic_subject = topic_row['SyllabusSubject']
|
||||
topic_key_stage = topic_row['SyllabusKeyStage']
|
||||
|
||||
logger.debug(f"Processing topic {topic_row['TopicID']} for subject {topic_subject} and key stage {topic_key_stage} without year group")
|
||||
|
||||
# Find the key stage syllabus node
|
||||
matching_syllabus_node = None
|
||||
for syllabus_node in node_library['key_stage_syllabus_nodes'].values():
|
||||
if (syllabus_node.subject_name == topic_subject and
|
||||
syllabus_node.key_stage == str(topic_key_stage)):
|
||||
matching_syllabus_node = syllabus_node
|
||||
break
|
||||
|
||||
if not matching_syllabus_node:
|
||||
logger.warning(f"No key stage syllabus node found for subject {topic_subject} and key stage {topic_key_stage}, skipping topic creation")
|
||||
continue
|
||||
|
||||
topic_node_unique_id = f"Topic_{matching_syllabus_node.unique_id}_{topic_row['TopicID']}"
|
||||
topic_node = curriculum_nodes.TopicNode(
|
||||
unique_id=topic_node_unique_id,
|
||||
id=topic_row['TopicID'],
|
||||
name=topic_row.get('TopicTitle', default_topic_values['topic_title']),
|
||||
total_number_of_lessons_for_topic=str(topic_row.get('TotalNumberOfLessonsForTopic', default_topic_values['total_number_of_lessons_for_topic'])),
|
||||
type=topic_row.get('TopicType', default_topic_values['topic_type']),
|
||||
assessment_type=topic_row.get('TopicAssessmentType', default_topic_values['topic_assessment_type']),
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
# Create topic node in curriculum database only
|
||||
neon.create_or_merge_neontology_node(topic_node, database=curriculum_db_name, operation='merge')
|
||||
node_library['topic_nodes'][topic_row['TopicID']] = topic_node
|
||||
topics_processed.add(topic_row['TopicID'])
|
||||
|
||||
# Link topic to key stage syllabus
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.KeyStageSyllabusIncludesTopic(source=matching_syllabus_node, target=topic_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship between topic {topic_node_unique_id} and key stage syllabus {matching_syllabus_node.unique_id}")
|
||||
|
||||
# Process lessons for this topic
|
||||
lessons_for_topic = lesson_df[
|
||||
(lesson_df['TopicID'] == topic_row['TopicID']) &
|
||||
(lesson_df['SyllabusSubject'] == topic_subject)
|
||||
].copy()
|
||||
lessons_for_topic.loc[:, 'Lesson'] = lessons_for_topic['Lesson'].astype(str)
|
||||
lessons_for_topic = lessons_for_topic.sort_values('Lesson')
|
||||
|
||||
previous_lesson_node = None
|
||||
for _, lesson_row in lessons_for_topic.iterrows():
|
||||
if lesson_row['LessonID'] in lessons_processed:
|
||||
continue
|
||||
lessons_processed.add(lesson_row['LessonID'])
|
||||
lesson_node = curriculum_nodes.TopicLessonNode(
|
||||
unique_id=f"TopicLesson_{topic_node_unique_id}_{lesson_row['LessonID']}",
|
||||
id=lesson_row['LessonID'],
|
||||
name=lesson_row.get('LessonTitle', default_topic_lesson_values['topic_lesson_title']),
|
||||
type=lesson_row.get('LessonType', default_topic_lesson_values['topic_lesson_type']),
|
||||
length=str(lesson_row.get('SuggestedNumberOfPeriodsForLesson', default_topic_lesson_values['topic_lesson_length'])),
|
||||
suggested_activities=str(lesson_row.get('SuggestedActivities', default_topic_lesson_values['topic_lesson_suggested_activities'])),
|
||||
skills_learned=str(lesson_row.get('SkillsLearned', default_topic_lesson_values['topic_lesson_skills_learned'])),
|
||||
weblinks=str(lesson_row.get('WebLinks', default_topic_lesson_values['topic_lesson_weblinks'])),
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
# Create lesson node in curriculum database only
|
||||
neon.create_or_merge_neontology_node(lesson_node, database=curriculum_db_name, operation='merge')
|
||||
node_library['topic_lesson_nodes'][lesson_row['LessonID']] = lesson_node
|
||||
|
||||
# Link lesson to topic
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.TopicIncludesTopicLesson(source=topic_node, target=lesson_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created lesson node {lesson_node.unique_id} and relationship with topic {topic_node.unique_id}")
|
||||
|
||||
# Create sequential relationships between lessons
|
||||
if lesson_row['Lesson'].isdigit() and previous_lesson_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.TopicLessonFollowsTopicLesson(source=previous_lesson_node, target=lesson_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.unique_id} and {lesson_node.unique_id}")
|
||||
previous_lesson_node = lesson_node
|
||||
|
||||
# Process learning statements for this lesson
|
||||
statements_for_lesson = statement_df[
|
||||
(statement_df['LessonID'] == lesson_row['LessonID']) &
|
||||
(statement_df['SyllabusSubject'] == topic_subject)
|
||||
]
|
||||
for _, statement_row in statements_for_lesson.iterrows():
|
||||
if statement_row['StatementID'] in statements_processed:
|
||||
continue
|
||||
statements_processed.add(statement_row['StatementID'])
|
||||
statement_node = curriculum_nodes.LearningStatementNode(
|
||||
unique_id=f"LearningStatement_{lesson_node.unique_id}_{statement_row['StatementID']}",
|
||||
id=statement_row['StatementID'],
|
||||
name=statement_row.get('LearningStatement', default_learning_statement_values['lesson_learning_statement']),
|
||||
type=statement_row.get('StatementType', default_learning_statement_values['lesson_learning_statement_type']),
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
# Create statement node in curriculum database only
|
||||
neon.create_or_merge_neontology_node(statement_node, database=curriculum_db_name, operation='merge')
|
||||
node_library['statement_nodes'][statement_row['StatementID']] = statement_node
|
||||
|
||||
# Link learning statement to lesson
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.LessonIncludesLearningStatement(source=lesson_node, target=statement_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created learning statement node {statement_node.unique_id} and relationship with lesson {lesson_node.unique_id}")
|
||||
|
||||
return node_library
|
||||
@@ -0,0 +1,487 @@
|
||||
import os
|
||||
from modules.logger_tool import initialise_logger
|
||||
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
from datetime import timedelta, datetime
|
||||
import pandas as pd
|
||||
from modules.database.schemas.structures import structures
|
||||
import modules.database.schemas.nodes.schools.timetable as timetable
|
||||
import modules.database.schemas.relationships.timetables as tt_rels
|
||||
import modules.database.schemas.relationships.entity_timetable_rels as entity_tt_rels
|
||||
import modules.database.schemas.relationships.calendar_timetable_rels as cal_tt_rels
|
||||
import modules.database.init.init_calendar as init_calendar
|
||||
import modules.database.tools.neontology_tools as neon
|
||||
|
||||
def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
logger.info(f"Creating school timetable for {db_name}")
|
||||
if dataframes is None:
|
||||
raise ValueError("Data is required to create the calendar and timetable.")
|
||||
|
||||
logger.info(f"Initialising neo4j connection...")
|
||||
neon.init_neontology_connection()
|
||||
|
||||
school_df = dataframes['school']
|
||||
if school_node is None:
|
||||
logger.info(f"School node is None, using school data from dataframe")
|
||||
school_unique_id = school_df[school_df['Identifier'] == 'SchoolID']['Data'].iloc[0]
|
||||
else:
|
||||
logger.info(f"School node is not None, using school data from school node: {school_node}")
|
||||
school_unique_id = school_node.unique_id
|
||||
|
||||
terms_df = dataframes['terms']
|
||||
weeks_df = dataframes['weeks']
|
||||
days_df = dataframes['days']
|
||||
periods_df = dataframes['periods']
|
||||
|
||||
school_df_year_start = school_df[school_df['Identifier'] == 'AcademicYearStart']['Data'].iloc[0]
|
||||
school_df_year_end = school_df[school_df['Identifier'] == 'AcademicYearEnd']['Data'].iloc[0]
|
||||
if isinstance(school_df_year_start, str):
|
||||
school_year_start_date = datetime.strptime(school_df_year_start, '%Y-%m-%d')
|
||||
else:
|
||||
school_year_start_date = school_df_year_start
|
||||
if isinstance(school_df_year_end, str):
|
||||
school_year_end_date = datetime.strptime(school_df_year_end, '%Y-%m-%d')
|
||||
else:
|
||||
school_year_end_date = school_df_year_end
|
||||
|
||||
# Create a dictionary to store the timetable nodes
|
||||
timetable_nodes = {
|
||||
'timetable_node': None,
|
||||
'academic_year_nodes': [],
|
||||
'academic_term_nodes': [],
|
||||
'academic_week_nodes': [],
|
||||
'academic_day_nodes': [],
|
||||
'academic_period_nodes': []
|
||||
}
|
||||
|
||||
# Create AcademicTimetable Node
|
||||
school_timetable_unique_id = f"{school_unique_id}_{school_year_start_date.year}_{school_year_end_date.year}"
|
||||
school_timetable_node = timetable.SchoolTimetableNode(
|
||||
school_timetable_id=school_timetable_unique_id,
|
||||
unique_id=school_timetable_unique_id,
|
||||
start_date=school_year_start_date,
|
||||
end_date=school_year_end_date,
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
neon.create_or_merge_neontology_node(school_timetable_node, database=db_name, operation='merge')
|
||||
timetable_nodes['timetable_node'] = school_timetable_node
|
||||
|
||||
if school_node:
|
||||
logger.info(f"Creating calendar for {school_unique_id} from Neo4j SchoolNode: {school_node.unique_id}")
|
||||
calendar_nodes = init_calendar.create_calendar(db_name, school_year_start_date, school_year_end_date, attach_to_calendar_node=True, owner_node=school_node)
|
||||
# Link the school node to the timetable node
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
entity_tt_rels.SchoolHasTimetable(source=school_node, target=school_timetable_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
timetable_nodes['calendar_nodes'] = calendar_nodes
|
||||
else:
|
||||
logger.info(f"Creating calendar for {school_unique_id} from dataframe SchoolID: {school_unique_id}")
|
||||
calendar_nodes = init_calendar.create_calendar(db_name, school_year_start_date, school_year_end_date, attach_to_calendar_node=False, owner_node=None)
|
||||
|
||||
# Create AcademicYear nodes for each year within the range
|
||||
for year in range(school_year_start_date.year, school_year_end_date.year + 1):
|
||||
year_str = str(year)
|
||||
academic_year_unique_id = f"{school_timetable_unique_id}_{year}"
|
||||
academic_year_node = timetable.AcademicYearNode(
|
||||
unique_id=academic_year_unique_id,
|
||||
year=year_str,
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
neon.create_or_merge_neontology_node(academic_year_node, database=db_name, operation='merge')
|
||||
timetable_nodes['academic_year_nodes'].append(academic_year_node)
|
||||
logger.info(f'Created academic year node: {academic_year_node.unique_id}')
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
tt_rels.AcademicTimetableHasAcademicYear(source=school_timetable_node, target=academic_year_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created school timetable relationship from {school_timetable_node.unique_id} to {academic_year_node.unique_id}")
|
||||
|
||||
# Link the academic year with the corresponding calendar year node
|
||||
for year_node in calendar_nodes['calendar_year_nodes']:
|
||||
if year_node.year == year:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
cal_tt_rels.AcademicYearIsCalendarYear(source=academic_year_node, target=year_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created school timetable relationship from {academic_year_node.unique_id} to {year_node.unique_id}")
|
||||
break
|
||||
|
||||
# Create Term and TermBreak nodes linked to AcademicYear
|
||||
term_number = 1
|
||||
academic_term_number = 1
|
||||
for _, term_row in terms_df.iterrows():
|
||||
term_node_class = timetable.AcademicTermNode if term_row['TermType'] == 'Term' else timetable.AcademicTermBreakNode
|
||||
term_name = term_row['TermName']
|
||||
term_name_no_spaces = term_name.replace(' ', '')
|
||||
term_start_date = term_row['StartDate']
|
||||
if isinstance(term_start_date, pd.Timestamp):
|
||||
term_start_date = term_start_date.strftime('%Y-%m-%d')
|
||||
|
||||
term_end_date = term_row['EndDate']
|
||||
if isinstance(term_end_date, pd.Timestamp):
|
||||
term_end_date = term_end_date.strftime('%Y-%m-%d')
|
||||
|
||||
if term_row['TermType'] == 'Term':
|
||||
term_node_unique_id = f"{school_timetable_unique_id}_{academic_term_number}_{term_name_no_spaces}"
|
||||
academic_term_number_str = str(academic_term_number)
|
||||
term_node = term_node_class(
|
||||
unique_id=term_node_unique_id,
|
||||
term_name=term_name,
|
||||
term_number=academic_term_number_str,
|
||||
start_date=datetime.strptime(term_start_date, '%Y-%m-%d'),
|
||||
end_date=datetime.strptime(term_end_date, '%Y-%m-%d'),
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
academic_term_number += 1
|
||||
else:
|
||||
term_break_node_unique_id = f"{school_timetable_unique_id}_{term_name_no_spaces}"
|
||||
term_node = term_node_class(
|
||||
unique_id=term_break_node_unique_id,
|
||||
term_break_name=term_name,
|
||||
start_date=datetime.strptime(term_start_date, '%Y-%m-%d'),
|
||||
end_date=datetime.strptime(term_end_date, '%Y-%m-%d'),
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
neon.create_or_merge_neontology_node(term_node, database=db_name, operation='merge')
|
||||
logger.info(f'Created academic term break node: {term_node.unique_id}')
|
||||
timetable_nodes['academic_term_nodes'].append(term_node)
|
||||
term_number += 1 # We don't use this but we could
|
||||
|
||||
# Link term node to the correct academic year
|
||||
term_years = set()
|
||||
term_years.update([term_node.start_date.year, term_node.end_date.year])
|
||||
|
||||
for academic_year_node in timetable_nodes['academic_year_nodes']:
|
||||
if int(academic_year_node.year) in term_years:
|
||||
relationship_class = tt_rels.AcademicYearHasAcademicTerm if term_row['TermType'] == 'Term' else tt_rels.AcademicYearHasAcademicTermBreak
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
relationship_class(source=academic_year_node, target=term_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created school timetable relationship from {academic_year_node.unique_id} to {term_node.unique_id}")
|
||||
|
||||
# Create Week nodes
|
||||
academic_week_number = 1
|
||||
for _, week_row in weeks_df.iterrows():
|
||||
week_node_class = timetable.HolidayWeekNode if week_row['WeekType'] == 'Holiday' else timetable.AcademicWeekNode
|
||||
week_start_date = week_row['WeekStart']
|
||||
if isinstance(week_start_date, pd.Timestamp):
|
||||
week_start_date = week_start_date.strftime('%Y-%m-%d')
|
||||
|
||||
week_node_unique_id = f"{school_timetable_unique_id}_{week_row['WeekNumber']}_{week_row['WeekType']}Week"
|
||||
|
||||
if week_row['WeekType'] == 'Holiday':
|
||||
week_node = week_node_class(
|
||||
unique_id=week_node_unique_id,
|
||||
start_date=datetime.strptime(week_start_date, '%Y-%m-%d'),
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
else:
|
||||
academic_week_number_str = str(academic_week_number)
|
||||
week_type = week_row['WeekType']
|
||||
week_node = week_node_class(
|
||||
unique_id=week_node_unique_id,
|
||||
academic_week_number=academic_week_number_str,
|
||||
start_date=datetime.strptime(week_start_date, '%Y-%m-%d'),
|
||||
week_type=week_type,
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
academic_week_number += 1
|
||||
neon.create_or_merge_neontology_node(week_node, database=db_name, operation='merge')
|
||||
timetable_nodes['academic_week_nodes'].append(week_node)
|
||||
logger.info(f"Created week node: {week_node.unique_id}")
|
||||
for calendar_node in calendar_nodes['calendar_week_nodes']:
|
||||
if calendar_node.start_date == week_node.start_date:
|
||||
if isinstance(week_node, timetable.AcademicWeekNode):
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
cal_tt_rels.AcademicWeekIsCalendarWeek(source=week_node, target=calendar_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created school timetable relationship from {calendar_node.unique_id} to {week_node.unique_id}")
|
||||
elif isinstance(week_node, timetable.HolidayWeekNode):
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
cal_tt_rels.HolidayWeekIsCalendarWeek(source=week_node, target=calendar_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created school timetable relationship from {calendar_node.unique_id} to {week_node.unique_id}")
|
||||
break
|
||||
|
||||
# Link week node to the correct academic term
|
||||
for term_node in timetable_nodes['academic_term_nodes']:
|
||||
if term_node.start_date <= week_node.start_date <= term_node.end_date:
|
||||
relationship_class = tt_rels.AcademicTermHasAcademicWeek if week_row['WeekType'] != 'Holiday' else tt_rels.AcademicTermBreakHasHolidayWeek
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
relationship_class(source=term_node, target=week_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created school timetable relationship from {term_node.unique_id} to {week_node.unique_id}")
|
||||
break
|
||||
|
||||
# Link week node to the correct academic year
|
||||
for academic_year_node in timetable_nodes['academic_year_nodes']:
|
||||
if int(academic_year_node.year) == week_node.start_date.year:
|
||||
relationship_class = tt_rels.AcademicYearHasAcademicWeek if week_row['WeekType'] != 'Holiday' else tt_rels.AcademicYearHasHolidayWeek
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
relationship_class(source=academic_year_node, target=week_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created school timetable relationship from {academic_year_node.unique_id} to {week_node.unique_id}")
|
||||
break
|
||||
|
||||
# Create Day nodes
|
||||
day_number = 1
|
||||
academic_day_number = 1
|
||||
for _, day_row in days_df.iterrows():
|
||||
date_str = day_row['Date']
|
||||
if isinstance(date_str, pd.Timestamp):
|
||||
date_str = date_str.strftime('%Y-%m-%d')
|
||||
|
||||
day_node_class = {
|
||||
'Academic': timetable.AcademicDayNode,
|
||||
'Holiday': timetable.HolidayDayNode,
|
||||
'OffTimetable': timetable.OffTimetableDayNode,
|
||||
'StaffDay': timetable.StaffDayNode
|
||||
}[day_row['DayType']]
|
||||
|
||||
# Format the unique ID as {day_node_class.__name__}Day
|
||||
day_node_data = {
|
||||
'unique_id': f"{school_timetable_unique_id}_{day_number}_{day_node_class.__name__}Day",
|
||||
'date': datetime.strptime(date_str, '%Y-%m-%d'),
|
||||
'day_of_week': datetime.strptime(date_str, '%Y-%m-%d').strftime('%A'),
|
||||
'tldraw_snapshot': ""
|
||||
}
|
||||
|
||||
if day_row['DayType'] == 'Academic':
|
||||
day_node_data['academic_day'] = str(academic_day_number)
|
||||
day_node_data['day_type'] = day_row['WeekType']
|
||||
day_node_data['tldraw_snapshot'] = ""
|
||||
|
||||
day_node = day_node_class(**day_node_data)
|
||||
|
||||
for calendar_node in calendar_nodes['calendar_day_nodes']:
|
||||
if calendar_node.date == day_node.date:
|
||||
neon.create_or_merge_neontology_node(day_node, database=db_name, operation='merge')
|
||||
timetable_nodes['academic_day_nodes'].append(day_node)
|
||||
logger.info(f"Created day node: {day_node.unique_id}")
|
||||
|
||||
if isinstance(day_node, timetable.AcademicDayNode):
|
||||
relationship_class = cal_tt_rels.AcademicDayIsCalendarDay
|
||||
elif isinstance(day_node, timetable.HolidayDayNode):
|
||||
relationship_class = cal_tt_rels.HolidayDayIsCalendarDay
|
||||
elif isinstance(day_node, timetable.OffTimetableDayNode):
|
||||
relationship_class = cal_tt_rels.OffTimetableDayIsCalendarDay
|
||||
elif isinstance(day_node, timetable.StaffDayNode):
|
||||
relationship_class = cal_tt_rels.StaffDayIsCalendarDay
|
||||
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
relationship_class(source=day_node, target=calendar_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f'Created relationship from {calendar_node.unique_id} to {day_node.unique_id}')
|
||||
break
|
||||
|
||||
# Link day node to the correct academic week
|
||||
for academic_week_node in timetable_nodes['academic_week_nodes']:
|
||||
if academic_week_node.start_date <= day_node.date <= (academic_week_node.start_date + timedelta(days=6)):
|
||||
if day_row['DayType'] == 'Academic':
|
||||
relationship_class = tt_rels.AcademicWeekHasAcademicDay
|
||||
elif day_row['DayType'] == 'Holiday':
|
||||
if hasattr(academic_week_node, 'week_type') and academic_week_node.week_type in ['A', 'B']:
|
||||
relationship_class = tt_rels.AcademicWeekHasHolidayDay
|
||||
else:
|
||||
relationship_class = tt_rels.HolidayWeekHasHolidayDay
|
||||
elif day_row['DayType'] == 'OffTimetable':
|
||||
relationship_class = tt_rels.AcademicWeekHasOffTimetableDay
|
||||
elif day_row['DayType'] == 'Staff':
|
||||
relationship_class = tt_rels.AcademicWeekHasStaffDay
|
||||
else:
|
||||
continue # Skip linking for other day types
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
relationship_class(source=academic_week_node, target=day_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship from {academic_week_node.unique_id} to {day_node.unique_id}")
|
||||
break
|
||||
|
||||
# Link day node to the correct academic term
|
||||
for term_node in timetable_nodes['academic_term_nodes']:
|
||||
if term_node.start_date <= day_node.date <= term_node.end_date:
|
||||
if day_row['DayType'] == 'Academic':
|
||||
relationship_class = tt_rels.AcademicTermHasAcademicDay
|
||||
elif day_row['DayType'] == 'Holiday':
|
||||
if isinstance(term_node, timetable.AcademicTermNode):
|
||||
relationship_class = tt_rels.AcademicTermHasHolidayDay
|
||||
else:
|
||||
relationship_class = tt_rels.AcademicTermBreakHasHolidayDay
|
||||
elif day_row['DayType'] == 'OffTimetable':
|
||||
relationship_class = tt_rels.AcademicTermHasOffTimetableDay
|
||||
elif day_row['DayType'] == 'Staff':
|
||||
relationship_class = tt_rels.AcademicTermHasStaffDay
|
||||
else:
|
||||
continue # Skip linking for other day types
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
relationship_class(source=term_node, target=day_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship from {term_node.unique_id} to {day_node.unique_id}")
|
||||
break
|
||||
|
||||
# Create Period nodes for each academic day
|
||||
if day_row['DayType'] == 'Academic':
|
||||
logger.info(f"Creating periods for {day_node.unique_id}")
|
||||
period_of_day = 1
|
||||
academic_or_registration_period_of_day = 1
|
||||
for _, period_row in periods_df.iterrows():
|
||||
period_node_class = {
|
||||
'Academic': timetable.AcademicPeriodNode,
|
||||
'Registration': timetable.RegistrationPeriodNode,
|
||||
'Break': timetable.BreakPeriodNode,
|
||||
'OffTimetable': timetable.OffTimetablePeriodNode
|
||||
}[period_row['PeriodType']]
|
||||
|
||||
logger.info(f"Creating period node for {period_node_class.__name__} Period: {period_of_day}")
|
||||
period_node_unique_id = f"{school_timetable_unique_id}_{academic_day_number}_{period_of_day}_{period_node_class.__name__}Period"
|
||||
logger.debug(f"Period node unique id: {period_node_unique_id}")
|
||||
period_node_data = {
|
||||
'unique_id': period_node_unique_id,
|
||||
'name': period_row['PeriodName'],
|
||||
'date': day_node.date,
|
||||
'start_time': datetime.combine(day_node.date, period_row['StartTime']),
|
||||
'end_time': datetime.combine(day_node.date, period_row['EndTime']),
|
||||
'tldraw_snapshot': ""
|
||||
}
|
||||
logger.debug(f"Period node data: {period_node_data}")
|
||||
if period_row['PeriodType'] in ['Academic', 'Registration']:
|
||||
week_type = day_row['WeekType']
|
||||
day_name_short = day_node.day_of_week[:3]
|
||||
period_code = period_row['PeriodCode']
|
||||
period_code_formatted = f"{week_type}{day_name_short}{period_code}"
|
||||
period_node_data['period_code'] = period_code_formatted
|
||||
period_node_data['tldraw_snapshot'] = ""
|
||||
|
||||
academic_or_registration_period_of_day += 1
|
||||
|
||||
period_node = period_node_class(**period_node_data)
|
||||
neon.create_or_merge_neontology_node(period_node, database=db_name, operation='merge')
|
||||
timetable_nodes['academic_period_nodes'].append(period_node)
|
||||
logger.info(f'Created period node: {period_node.unique_id}')
|
||||
|
||||
relationship_class = {
|
||||
'Academic': tt_rels.AcademicDayHasAcademicPeriod,
|
||||
'Registration': tt_rels.AcademicDayHasRegistrationPeriod,
|
||||
'Break': tt_rels.AcademicDayHasBreakPeriod,
|
||||
'OffTimetable': tt_rels.AcademicDayHasOffTimetablePeriod
|
||||
}[period_row['PeriodType']]
|
||||
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
relationship_class(source=day_node, target=period_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship from {day_node.unique_id} to {period_node.unique_id}")
|
||||
period_of_day += 1 # We don't use this but we could
|
||||
academic_day_number += 1 # This is a bit of a hack but it works to keep the directories aligned (reorganise)
|
||||
day_number += 1 # We don't use this but we could
|
||||
|
||||
def create_school_timetable_node_sequence_rels(timetable_nodes):
|
||||
def sort_and_create_relationships(nodes, relationship_map, sort_key):
|
||||
sorted_nodes = sorted(nodes, key=sort_key)
|
||||
for i in range(len(sorted_nodes) - 1):
|
||||
source_node = sorted_nodes[i]
|
||||
target_node = sorted_nodes[i + 1]
|
||||
node_type_pair = (type(source_node), type(target_node))
|
||||
relationship_class = relationship_map.get(node_type_pair)
|
||||
if relationship_class:
|
||||
# Avoid self-referential relationships
|
||||
if source_node.unique_id != target_node.unique_id:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
relationship_class(
|
||||
source=source_node,
|
||||
target=target_node
|
||||
),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship from {source_node.unique_id} to {target_node.unique_id}")
|
||||
else:
|
||||
logger.warning(f"Skipped self-referential relationship for node {source_node.unique_id}")
|
||||
|
||||
# Relationship maps for different node types
|
||||
academic_year_relationship_map = {
|
||||
(timetable.AcademicYearNode, timetable.AcademicYearNode): tt_rels.AcademicYearFollowsAcademicYear
|
||||
}
|
||||
|
||||
academic_term_relationship_map = {
|
||||
(timetable.AcademicTermNode, timetable.AcademicTermBreakNode): tt_rels.AcademicTermBreakFollowsAcademicTerm,
|
||||
(timetable.AcademicTermBreakNode, timetable.AcademicTermNode): tt_rels.AcademicTermFollowsAcademicTermBreak
|
||||
}
|
||||
|
||||
academic_week_relationship_map = {
|
||||
(timetable.AcademicWeekNode, timetable.AcademicWeekNode): tt_rels.AcademicWeekFollowsAcademicWeek,
|
||||
(timetable.HolidayWeekNode, timetable.HolidayWeekNode): tt_rels.HolidayWeekFollowsHolidayWeek,
|
||||
(timetable.AcademicWeekNode, timetable.HolidayWeekNode): tt_rels.HolidayWeekFollowsAcademicWeek,
|
||||
(timetable.HolidayWeekNode, timetable.AcademicWeekNode): tt_rels.AcademicWeekFollowsHolidayWeek
|
||||
}
|
||||
|
||||
academic_day_relationship_map = {
|
||||
(timetable.AcademicDayNode, timetable.AcademicDayNode): tt_rels.AcademicDayFollowsAcademicDay,
|
||||
(timetable.HolidayDayNode, timetable.HolidayDayNode): tt_rels.HolidayDayFollowsHolidayDay,
|
||||
(timetable.OffTimetableDayNode, timetable.OffTimetableDayNode): tt_rels.OffTimetableDayFollowsOffTimetableDay,
|
||||
(timetable.StaffDayNode, timetable.StaffDayNode): tt_rels.StaffDayFollowsStaffDay,
|
||||
|
||||
(timetable.AcademicDayNode, timetable.HolidayDayNode): tt_rels.HolidayDayFollowsAcademicDay,
|
||||
(timetable.AcademicDayNode, timetable.OffTimetableDayNode): tt_rels.OffTimetableDayFollowsAcademicDay,
|
||||
(timetable.AcademicDayNode, timetable.StaffDayNode): tt_rels.StaffDayFollowsAcademicDay,
|
||||
|
||||
(timetable.HolidayDayNode, timetable.AcademicDayNode): tt_rels.AcademicDayFollowsHolidayDay,
|
||||
(timetable.HolidayDayNode, timetable.OffTimetableDayNode): tt_rels.OffTimetableDayFollowsHolidayDay,
|
||||
(timetable.HolidayDayNode, timetable.StaffDayNode): tt_rels.StaffDayFollowsHolidayDay,
|
||||
|
||||
(timetable.OffTimetableDayNode, timetable.AcademicDayNode): tt_rels.AcademicDayFollowsOffTimetableDay,
|
||||
(timetable.OffTimetableDayNode, timetable.HolidayDayNode): tt_rels.HolidayDayFollowsOffTimetableDay,
|
||||
(timetable.OffTimetableDayNode, timetable.StaffDayNode): tt_rels.StaffDayFollowsOffTimetableDay,
|
||||
|
||||
(timetable.StaffDayNode, timetable.AcademicDayNode): tt_rels.AcademicDayFollowsStaffDay,
|
||||
(timetable.StaffDayNode, timetable.HolidayDayNode): tt_rels.HolidayDayFollowsStaffDay,
|
||||
(timetable.StaffDayNode, timetable.OffTimetableDayNode): tt_rels.OffTimetableDayFollowsStaffDay,
|
||||
}
|
||||
|
||||
academic_period_relationship_map = {
|
||||
(timetable.AcademicPeriodNode, timetable.AcademicPeriodNode): tt_rels.AcademicPeriodFollowsAcademicPeriod,
|
||||
(timetable.AcademicPeriodNode, timetable.BreakPeriodNode): tt_rels.BreakPeriodFollowsAcademicPeriod,
|
||||
(timetable.AcademicPeriodNode, timetable.RegistrationPeriodNode): tt_rels.RegistrationPeriodFollowsAcademicPeriod,
|
||||
(timetable.AcademicPeriodNode, timetable.OffTimetablePeriodNode): tt_rels.OffTimetablePeriodFollowsAcademicPeriod,
|
||||
(timetable.BreakPeriodNode, timetable.AcademicPeriodNode): tt_rels.AcademicPeriodFollowsBreakPeriod,
|
||||
(timetable.BreakPeriodNode, timetable.BreakPeriodNode): tt_rels.BreakPeriodFollowsBreakPeriod,
|
||||
(timetable.BreakPeriodNode, timetable.RegistrationPeriodNode): tt_rels.RegistrationPeriodFollowsBreakPeriod,
|
||||
(timetable.BreakPeriodNode, timetable.OffTimetablePeriodNode): tt_rels.OffTimetablePeriodFollowsBreakPeriod,
|
||||
(timetable.RegistrationPeriodNode, timetable.AcademicPeriodNode): tt_rels.AcademicPeriodFollowsRegistrationPeriod,
|
||||
(timetable.RegistrationPeriodNode, timetable.RegistrationPeriodNode): tt_rels.RegistrationPeriodFollowsRegistrationPeriod,
|
||||
(timetable.RegistrationPeriodNode, timetable.BreakPeriodNode): tt_rels.BreakPeriodFollowsRegistrationPeriod,
|
||||
(timetable.RegistrationPeriodNode, timetable.OffTimetablePeriodNode): tt_rels.OffTimetablePeriodFollowsRegistrationPeriod,
|
||||
(timetable.OffTimetablePeriodNode, timetable.OffTimetablePeriodNode): tt_rels.OffTimetablePeriodFollowsOffTimetablePeriod,
|
||||
(timetable.OffTimetablePeriodNode, timetable.AcademicPeriodNode): tt_rels.AcademicPeriodFollowsOffTimetablePeriod,
|
||||
(timetable.OffTimetablePeriodNode, timetable.BreakPeriodNode): tt_rels.BreakPeriodFollowsOffTimetablePeriod,
|
||||
(timetable.OffTimetablePeriodNode, timetable.RegistrationPeriodNode): tt_rels.RegistrationPeriodFollowsOffTimetablePeriod,
|
||||
}
|
||||
|
||||
|
||||
# Sort and create relationships
|
||||
sort_and_create_relationships(timetable_nodes['academic_year_nodes'], academic_year_relationship_map, lambda x: int(x.year))
|
||||
sort_and_create_relationships(timetable_nodes['academic_term_nodes'], academic_term_relationship_map, lambda x: x.start_date)
|
||||
sort_and_create_relationships(timetable_nodes['academic_week_nodes'], academic_week_relationship_map, lambda x: x.start_date)
|
||||
sort_and_create_relationships(timetable_nodes['academic_day_nodes'], academic_day_relationship_map, lambda x: x.date)
|
||||
sort_and_create_relationships(timetable_nodes['academic_period_nodes'], academic_period_relationship_map, lambda x: (x.start_time, x.end_time))
|
||||
|
||||
# Call the function with the created timetable nodes
|
||||
create_school_timetable_node_sequence_rels(timetable_nodes)
|
||||
|
||||
logger.info(f'Created timetable: {timetable_nodes["timetable_node"].unique_id}')
|
||||
|
||||
# Log the directory structure after creation
|
||||
# root_timetable_directory = fs_handler.root_path # Access the root directory of the filesystem handler
|
||||
# fs_handler.log_directory_structure(root_timetable_directory)
|
||||
|
||||
return {
|
||||
'school_node': school_node,
|
||||
'school_calendar_nodes': calendar_nodes,
|
||||
'school_timetable_nodes': timetable_nodes
|
||||
}
|
||||
@@ -0,0 +1,379 @@
|
||||
import os
|
||||
from datetime import timedelta, datetime
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Dict, Optional, Any, Union
|
||||
|
||||
from modules.database.services.neo4j_service import Neo4jService
|
||||
import modules.database.schemas.nodes.users as user_nodes
|
||||
import modules.database.schemas.nodes.workers.workers as worker_nodes
|
||||
import modules.database.init.init_calendar as init_calendar
|
||||
import modules.database.schemas.relationships.entity_relationships as entity_relationships
|
||||
import modules.database.tools.neontology_tools as neon
|
||||
from modules.logger_tool import initialise_logger
|
||||
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
|
||||
def create_and_check_db(db_name):
|
||||
neo4j_service = Neo4jService()
|
||||
neo4j_service.create_database(db_name)
|
||||
database_status = neo4j_service.check_database_exists(db_name)
|
||||
if not database_status['exists']:
|
||||
raise ValueError(f"Database {db_name} not found")
|
||||
return database_status
|
||||
|
||||
class UserCreator(ABC):
|
||||
def __init__(self, user_id, cc_users_db_name, user_type, worker_type, user_email, worker_email, cc_username, user_name, worker_name, calendar_start_date, calendar_end_date):
|
||||
cc_schools_db_name = "cc.institutes" # Fix the TODO
|
||||
self.cc_users_db_name = cc_users_db_name
|
||||
self.user_db_name = f"{cc_users_db_name}.{user_type}.{cc_username}"
|
||||
self.worker_db_name = f"{cc_schools_db_name}.{user_type}.{cc_username}"
|
||||
self.user_type = user_type
|
||||
self.worker_type = worker_type
|
||||
self.cc_username = cc_username
|
||||
self.user_email = user_email
|
||||
self.worker_email = worker_email
|
||||
self.user_name = user_name
|
||||
self.worker_name = worker_name
|
||||
self.user_id = user_id
|
||||
self.user_nodes: Dict[str, Optional[Any]] = {
|
||||
'default_user_node': None,
|
||||
'private_user_node': None,
|
||||
'worker_node': None,
|
||||
'calendar_node': None
|
||||
}
|
||||
if calendar_start_date and calendar_end_date:
|
||||
self.calendar_start_date = calendar_start_date
|
||||
self.calendar_end_date = calendar_end_date
|
||||
else:
|
||||
logger.warning("No calendar start and end date provided, using default values")
|
||||
self.calendar_start_date = datetime.now().date()
|
||||
self.calendar_end_date = (datetime.now() + timedelta(days=5)).date()
|
||||
|
||||
@abstractmethod
|
||||
def create_user(self):
|
||||
pass
|
||||
|
||||
def create_user_node(self, db_name: str):
|
||||
logger.info(f"Module is creating {self.cc_users_db_name} user node for {self.user_type} user {self.cc_username}")
|
||||
try:
|
||||
user_node = self._create_user_node(db_name)
|
||||
logger.debug(f"User node creation completed for {self.cc_users_db_name} user node for {self.user_type} user {self.cc_username}: {user_node.to_dict()}")
|
||||
return user_node
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating user node: {e}")
|
||||
raise
|
||||
|
||||
def _create_user_node(self, db_name: str):
|
||||
# Ensure Neontology is initialized
|
||||
neon.init_neontology_connection()
|
||||
|
||||
user_node = user_nodes.UserNode(
|
||||
unique_id=f"{self.user_id}",
|
||||
tldraw_snapshot="",
|
||||
cc_username=f"{self.cc_username}",
|
||||
user_email=f"{self.user_email}",
|
||||
user_name=f"{self.user_name}",
|
||||
user_db_name=f"{self.user_db_name}",
|
||||
user_type=f"{self.user_type}",
|
||||
)
|
||||
logger.debug(f"User node template created: {user_node.to_dict()}. Writing to database {db_name}")
|
||||
neon.create_or_merge_neontology_node(node=user_node, database=db_name, operation='merge')
|
||||
logger.info(f"User node created: {user_node.to_dict()}")
|
||||
return user_node
|
||||
|
||||
def create_storage_bucket(self, bucket_id: str, bucket_name: str, access_token: Optional[str] = None) -> bool:
|
||||
"""Create public and private storage buckets for the user using their access token or service role during initialization"""
|
||||
logger.info(f"Creating storage buckets for user {self.cc_username}")
|
||||
|
||||
try:
|
||||
from modules.database.supabase.utils.client import SupabaseServiceRoleClient, SupabaseAnonClient, CreateBucketOptions
|
||||
|
||||
# During initialization (no access token provided), use service role
|
||||
if not access_token:
|
||||
logger.info("Using service role client for bucket creation during initialization")
|
||||
supabase = SupabaseServiceRoleClient()
|
||||
else:
|
||||
# For regular operations, use the user's access token
|
||||
logger.info("Using user token for bucket creation")
|
||||
supabase = SupabaseAnonClient.for_user(access_token)
|
||||
|
||||
# Create both public and private buckets
|
||||
buckets = [
|
||||
{
|
||||
"id": f"{bucket_id}.public",
|
||||
"options": CreateBucketOptions(
|
||||
name=f"{bucket_name} - Public Files",
|
||||
public=True,
|
||||
file_size_limit=50 * 1024 * 1024, # 50MB
|
||||
allowed_mime_types=[
|
||||
'image/*', 'video/*', 'application/pdf',
|
||||
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
|
||||
'text/plain', 'text/csv', 'application/json'
|
||||
]
|
||||
)
|
||||
},
|
||||
{
|
||||
"id": f"{bucket_id}.private",
|
||||
"options": CreateBucketOptions(
|
||||
name=f"{bucket_name} - Private Files",
|
||||
public=False,
|
||||
file_size_limit=50 * 1024 * 1024, # 50MB
|
||||
allowed_mime_types=[
|
||||
'image/*', 'video/*', 'application/pdf',
|
||||
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
|
||||
'text/plain', 'text/csv', 'application/json'
|
||||
]
|
||||
)
|
||||
}
|
||||
]
|
||||
|
||||
success = True
|
||||
for bucket in buckets:
|
||||
try:
|
||||
result = supabase.create_bucket(bucket["id"], bucket["options"])
|
||||
if not result:
|
||||
logger.error(f"Failed to create bucket {bucket['id']}")
|
||||
success = False
|
||||
else:
|
||||
logger.info(f"Successfully created bucket {bucket['id']}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating bucket {bucket['id']}: {str(e)}")
|
||||
success = False
|
||||
|
||||
return success
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating storage buckets: {str(e)}")
|
||||
return False
|
||||
|
||||
class SchoolUserCreator(UserCreator):
|
||||
def __init__(self, user_id, cc_users_db_name, user_type, worker_type, user_email, worker_email, cc_username, user_name, worker_name, calendar_start_date, calendar_end_date, school_node, worker_node=None):
|
||||
super().__init__(user_id, cc_users_db_name, user_type, worker_type, user_email, worker_email, cc_username, user_name, worker_name, calendar_start_date, calendar_end_date)
|
||||
self.school_node = school_node
|
||||
self.worker_node = worker_node
|
||||
|
||||
def create_user(self):
|
||||
# Ensure Neontology is initialized
|
||||
logger.debug(f"Initializing Neontology connection. Closing any existing connection")
|
||||
neon.close_neontology_connection()
|
||||
logger.debug(f"Neontology connection closed. Initializing new connection")
|
||||
neon.init_neontology_connection()
|
||||
|
||||
if self.user_type in ['email_teacher', 'ms_teacher']:
|
||||
worker_node = self.create_teacher_node()
|
||||
elif self.user_type in ['email_student', 'ms_student']:
|
||||
worker_node = self.create_student_node()
|
||||
else:
|
||||
raise ValueError(f"User type {self.user_type} not supported")
|
||||
|
||||
self.user_nodes[f'worker_node'] = worker_node
|
||||
|
||||
user_node = self.create_user_node(self.cc_users_db_name)
|
||||
|
||||
logger.info(f"User node created: {user_node}")
|
||||
|
||||
self.user_nodes['default_user_node'] = user_node
|
||||
|
||||
self.create_user_worker_relationship(user_node, worker_node)
|
||||
|
||||
self.create_worker_school_relationship(worker_node, self.school_node)
|
||||
|
||||
logger.info(f"Worker school relationship created between {worker_node} and {self.school_node}")
|
||||
return self.user_nodes
|
||||
|
||||
def create_teacher_node(self):
|
||||
logger.debug(f"Teacher node will be created for school: {self.school_node}")
|
||||
try:
|
||||
return self._create_teacher_node()
|
||||
except KeyError as ke:
|
||||
raise ValueError(f"Missing required key in worker_data: {ke}") from ke
|
||||
except Exception as e:
|
||||
raise ValueError(f"Error creating teacher node: {e}") from e
|
||||
|
||||
def _create_teacher_node(self):
|
||||
teacher_node = worker_nodes.TeacherNode(
|
||||
unique_id=f"{self.user_id}",
|
||||
tldraw_snapshot="",
|
||||
worker_name=self.worker_name,
|
||||
worker_email=self.worker_email,
|
||||
worker_db_name=self.worker_db_name,
|
||||
worker_type=self.worker_type
|
||||
)
|
||||
# Use the school's private database name if available
|
||||
school_db = self.school_node.private_database_name if hasattr(self.school_node, 'private_database_name') else f"cc.institutes.{self.school_node.school_type}.{self.school_node.id}"
|
||||
logger.info(f"Teacher node template created: {teacher_node}... setting school db to {school_db}")
|
||||
|
||||
neon.create_or_merge_neontology_node(node=teacher_node, database=school_db, operation='merge')
|
||||
|
||||
logger.info(f"Teacher node merged into database {school_db}: {teacher_node}")
|
||||
return teacher_node
|
||||
|
||||
def create_student_node(self):
|
||||
student_node = worker_nodes.StudentNode(
|
||||
unique_id=f"Student_{self.user_id}",
|
||||
worker_name=self.worker_name,
|
||||
worker_email=self.worker_email,
|
||||
worker_db_name=self.worker_db_name,
|
||||
worker_type=self.worker_type,
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
# Use the school's private database name if available
|
||||
school_db = self.school_node.private_database_name if hasattr(self.school_node, 'private_database_name') else f"cc.institutes.{self.school_node.school_type}.{self.school_node.id}"
|
||||
logger.info(f"Student node template created: {student_node}... setting school db to {school_db}")
|
||||
|
||||
neon.create_or_merge_neontology_node(node=student_node, database=school_db, operation='merge')
|
||||
|
||||
logger.info(f"Student node merged into database {school_db}: {student_node}")
|
||||
return student_node
|
||||
|
||||
def create_user_worker_relationship(self, user_node, worker_node):
|
||||
user_role_rel = entity_relationships.UserIsSchoolWorker(source=user_node, target=worker_node)
|
||||
# Use the school's private database name if available
|
||||
school_db = self.school_node.private_database_name if hasattr(self.school_node, 'private_database_name') else f"cc.institutes.{self.school_node.school_type}.{self.school_node.id}"
|
||||
neon.create_or_merge_neontology_relationship(user_role_rel, database=school_db, operation='merge')
|
||||
logger.info(f"Relationship created between user and worker in database {school_db}")
|
||||
|
||||
def create_worker_school_relationship(self, worker_node, school_node):
|
||||
worker_school_rel = entity_relationships.EntityBelongsToSchool(source=worker_node, target=school_node)
|
||||
# Use the school's private database name if available
|
||||
school_db = school_node.private_database_name if hasattr(school_node, 'private_database_name') else f"cc.institutes.{school_node.school_type}.{school_node.id}"
|
||||
neon.create_or_merge_neontology_relationship(worker_school_rel, database=school_db, operation='merge')
|
||||
logger.info(f"Relationship created between worker and school in database {school_db}")
|
||||
|
||||
class NonSchoolUserCreator(UserCreator):
|
||||
def __init__(self, user_id, cc_users_db_name, user_type, worker_type, user_email, worker_email, cc_username, user_name, worker_name, calendar_start_date, calendar_end_date, developer_role: str = "developer"):
|
||||
super().__init__(user_id, cc_users_db_name, user_type, worker_type, user_email, worker_email, cc_username, user_name, worker_name, calendar_start_date, calendar_end_date)
|
||||
self.developer_role = developer_role
|
||||
|
||||
def create_user(self, access_token: Optional[str] = None):
|
||||
logger.debug(f"Creating user node for {self.user_type} user {self.cc_username} in database {self.cc_users_db_name}")
|
||||
|
||||
# Create storage buckets for the user
|
||||
user_bucket_id = self.user_db_name
|
||||
user_bucket_name = f"{self.user_type.title()} User Files - {self.user_name}"
|
||||
if not self.create_storage_bucket(user_bucket_id, user_bucket_name, access_token=access_token):
|
||||
logger.error(f"Failed to create storage bucket for user {self.cc_username}")
|
||||
raise ValueError(f"Failed to create storage bucket for user {self.cc_username}")
|
||||
|
||||
# Create default user node first
|
||||
default_user_node = self.create_user_node(self.cc_users_db_name)
|
||||
logger.debug(f"Default user node created: {default_user_node}")
|
||||
|
||||
# Verify the return value of create_user_node
|
||||
if default_user_node is None:
|
||||
logger.error("Failed to create default user node. It is None.")
|
||||
raise ValueError("Failed to create default user node. It is None.")
|
||||
|
||||
self.user_nodes[f'default_user_node'] = default_user_node
|
||||
|
||||
# Create the appropriate user db based on user_type
|
||||
if self.user_type == 'admin':
|
||||
logger.debug(f"Creating super admin db for {self.user_type} user {self.cc_username} in database {self.user_db_name}")
|
||||
self.create_super_admin_db()
|
||||
elif self.user_type == 'developer':
|
||||
logger.debug(f"Creating developer db for {self.user_type} user {self.cc_username} in database {self.user_db_name}")
|
||||
self.create_developer_db()
|
||||
else:
|
||||
raise ValueError(f"User type {self.user_type} not supported")
|
||||
|
||||
logger.debug(f"User nodes after creation: {self.user_nodes}")
|
||||
return self.user_nodes
|
||||
|
||||
def create_super_admin_db(self):
|
||||
logger.debug(f"Creating super admin db for {self.user_type} user {self.cc_username} in database {self.user_db_name}")
|
||||
neon.init_neontology_connection()
|
||||
|
||||
# Create the user db self.user_db_name
|
||||
create_and_check_db(self.user_db_name)
|
||||
|
||||
try:
|
||||
# Create the user node again for the user db
|
||||
logger.debug(f"Creating super admin user node for {self.user_type} user {self.cc_username} in database {self.user_db_name}")
|
||||
private_user_node = self.create_user_node(self.user_db_name)
|
||||
|
||||
super_admin_node = worker_nodes.SuperAdminNode(
|
||||
unique_id=f"SuperAdmin_{self.user_id}",
|
||||
worker_email=self.worker_email,
|
||||
tldraw_snapshot="",
|
||||
worker_name=self.worker_name,
|
||||
worker_db_name=self.worker_db_name,
|
||||
worker_type=self.worker_type
|
||||
)
|
||||
logger.debug(f"Super admin node template created: {super_admin_node}. Writing to database {self.user_db_name}")
|
||||
neon.create_or_merge_neontology_node(node=super_admin_node, database=self.user_db_name, operation='merge')
|
||||
logger.info(f"Super admin node created: {super_admin_node}")
|
||||
|
||||
logger.debug(f"Creating relationship between user node: {private_user_node} and worker node: {super_admin_node}")
|
||||
self.create_user_specific_relationship(user_node=private_user_node, worker_node=super_admin_node)
|
||||
|
||||
logger.debug(f"Creating calendar for {self.user_type} user {self.cc_username} in database {self.user_db_name}")
|
||||
calendar_nodes = self.create_calendar(user_node=private_user_node)
|
||||
logger.info(f"Super admin calendar created.")
|
||||
|
||||
self.user_nodes['private_user_node'] = private_user_node
|
||||
self.user_nodes['worker_node'] = super_admin_node
|
||||
|
||||
logger.info(f"Returning user nodes: {self.user_nodes}")
|
||||
|
||||
return self.user_nodes
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating super admin node: {e}")
|
||||
raise ValueError(f"Error creating super admin node: {e}") from e
|
||||
|
||||
def create_developer_db(self):
|
||||
neon.init_neontology_connection()
|
||||
|
||||
# Create the user db self.user_db_name
|
||||
create_and_check_db(self.user_db_name)
|
||||
|
||||
try:
|
||||
# Create the user node again for the user db
|
||||
private_user_node = self.create_user_node(self.user_db_name)
|
||||
|
||||
developer_node = worker_nodes.DeveloperNode(
|
||||
unique_id=f"Developer_{self.user_id}",
|
||||
worker_name=self.worker_name,
|
||||
worker_email=self.worker_email,
|
||||
tldraw_snapshot="",
|
||||
worker_db_name=self.worker_db_name,
|
||||
worker_type=self.worker_type,
|
||||
developer_role=self.developer_role
|
||||
)
|
||||
|
||||
neon.create_or_merge_neontology_node(developer_node, database=self.user_db_name, operation='merge')
|
||||
logger.info(f"Developer node created: {developer_node}")
|
||||
|
||||
self.user_nodes['private_user_node'] = private_user_node
|
||||
self.user_nodes['worker_node'] = developer_node
|
||||
|
||||
self.create_user_specific_relationship(user_node=private_user_node, worker_node=developer_node)
|
||||
|
||||
return self.user_nodes
|
||||
except Exception as e:
|
||||
raise ValueError(f"Error creating developer node: {e}") from e
|
||||
|
||||
def create_user_specific_relationship(self, user_node: user_nodes.UserNode, worker_node: Union[worker_nodes.SuperAdminNode, worker_nodes.DeveloperNode]):
|
||||
if user_node is None or worker_node is None:
|
||||
logger.error("User node or worker node is None. Cannot create relationship.")
|
||||
raise ValueError("User node or worker node is None. Cannot create relationship.")
|
||||
|
||||
logger.info(f"Creating relationship between user node: {user_node} and worker node: {worker_node}")
|
||||
|
||||
# Log the state of user_node and worker_node
|
||||
logger.debug(f"user_node: {user_node}")
|
||||
logger.debug(f"worker_node: {worker_node}")
|
||||
|
||||
if worker_node.worker_type == 'developer':
|
||||
specific_user_rel = entity_relationships.UserIsSystemWorker(source=user_node, target=worker_node)
|
||||
elif worker_node.worker_type == 'superadmin':
|
||||
specific_user_rel = entity_relationships.UserIsSystemWorker(source=user_node, target=worker_node)
|
||||
else:
|
||||
raise ValueError(f"User type {worker_node.worker_type} not supported")
|
||||
|
||||
neon.create_or_merge_neontology_relationship(specific_user_rel, database=self.user_db_name, operation='merge')
|
||||
logger.info("Relationship created between user and specific node")
|
||||
|
||||
def create_calendar(self, user_node: user_nodes.UserNode):
|
||||
calendar_nodes = init_calendar.create_calendar(self.user_db_name, self.calendar_start_date, self.calendar_end_date, attach_to_calendar_node=True, owner_node=user_node)
|
||||
|
||||
logger.info(f"Calendar nodes created.")
|
||||
return calendar_nodes
|
||||
@@ -0,0 +1,326 @@
|
||||
import os
|
||||
from modules.logger_tool import initialise_logger
|
||||
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
|
||||
import modules.database.tools.neo4j_driver_tools as driver
|
||||
import modules.database.tools.neontology_tools as neon
|
||||
from modules.database.tools.filesystem_tools import ClassroomCopilotFilesystem
|
||||
from modules.database.schemas.nodes.users import UserNode
|
||||
from modules.database.schemas.nodes.schools.schools import SubjectClassNode
|
||||
from modules.database.schemas.nodes.workers.workers import TeacherNode
|
||||
from modules.database.schemas.nodes.calendars import CalendarDayNode
|
||||
from modules.database.schemas.nodes.workers.timetable import (
|
||||
UserTeacherTimetableNode
|
||||
)
|
||||
from modules.database.schemas.relationships.entity_timetable_rels import (
|
||||
EntityHasTimetable
|
||||
)
|
||||
from modules.database.schemas.relationships.planning_relationships import (
|
||||
TeacherHasTimetable, TimetableHasClass, ClassHasLesson,TimetableLessonFollowsTimetableLesson
|
||||
)
|
||||
from modules.database.schemas.relationships.calendar_timetable_rels import (
|
||||
CalendarDayHasTimetableLesson, TimetableLessonBelongsToCalendarDay,
|
||||
CalendarDayHasPlannedLesson, PlannedLessonBelongsToCalendarDay
|
||||
)
|
||||
|
||||
def get_school_worker_classes(school_db_name: str, user_unique_id: str, worker_unique_id: str) -> list:
|
||||
"""
|
||||
Retrieve all classes for a worker from the school database.
|
||||
"""
|
||||
query = """
|
||||
MATCH (w:Teacher {unique_id: $worker_id})-[:TEACHER_HAS_TIMETABLE]->(tt:TeacherTimetable)
|
||||
-[:TIMETABLE_HAS_CLASS]->(c:SubjectClass)
|
||||
RETURN c
|
||||
"""
|
||||
with driver.get_driver(db_name=school_db_name).session(database=school_db_name) as session:
|
||||
result = session.run(query, worker_id=worker_unique_id)
|
||||
classes = [record['c'] for record in result]
|
||||
if not classes:
|
||||
logger.warning(f"No classes found for teacher {worker_unique_id} in school database")
|
||||
return classes
|
||||
|
||||
def get_school_class_periods(school_db_name: str, class_unique_id: str) -> list:
|
||||
"""
|
||||
Retrieve all periods for a class from the school database.
|
||||
"""
|
||||
query = """
|
||||
MATCH (c:SubjectClass {unique_id: $class_id})-[:CLASS_HAS_LESSON]->(l:TimetableLesson)
|
||||
RETURN l
|
||||
"""
|
||||
with driver.get_driver(db_name=school_db_name).session(database=school_db_name) as session:
|
||||
result = session.run(query, class_id=class_unique_id)
|
||||
periods = [record['l'] for record in result]
|
||||
if not periods:
|
||||
logger.warning(f"No periods found for class {class_unique_id} in school database")
|
||||
return periods
|
||||
|
||||
def get_user_calendar_nodes(user_db_name: str, user_node: UserNode) -> list:
|
||||
"""
|
||||
Retrieve all calendar day nodes for a user.
|
||||
"""
|
||||
# First try to find any calendar days to verify the structure
|
||||
verify_query = """
|
||||
MATCH (w:User {unique_id: $user_id})
|
||||
OPTIONAL MATCH (w)-[:HAS_CALENDAR]->(c:Calendar)
|
||||
OPTIONAL MATCH (c)-[:CALENDAR_INCLUDES_YEAR]->(y:CalendarYear)
|
||||
OPTIONAL MATCH (y)-[:YEAR_INCLUDES_MONTH]->(m:CalendarMonth)
|
||||
OPTIONAL MATCH (m)-[:MONTH_INCLUDES_DAY]->(d:CalendarDay)
|
||||
RETURN w.unique_id as user_id,
|
||||
count(c) as calendar_count,
|
||||
count(y) as year_count,
|
||||
count(m) as month_count,
|
||||
count(d) as day_count,
|
||||
collect(DISTINCT y.year) as years
|
||||
LIMIT 1
|
||||
"""
|
||||
|
||||
with driver.get_driver(db_name=user_db_name).session(database=user_db_name) as session:
|
||||
# First check the calendar structure
|
||||
result = session.run(verify_query, user_id=user_node.unique_id)
|
||||
if stats := result.single():
|
||||
logger.info(f"Calendar structure for user {stats['user_id']}: "
|
||||
f"calendars={stats['calendar_count']}, "
|
||||
f"years={stats['year_count']}, "
|
||||
f"months={stats['month_count']}, "
|
||||
f"days={stats['day_count']}, "
|
||||
f"available years={stats['years']}")
|
||||
|
||||
if stats['calendar_count'] == 0:
|
||||
logger.error(f"No calendar found for user {user_node.unique_id}")
|
||||
return []
|
||||
if stats['year_count'] == 0:
|
||||
logger.error(f"No calendar years found for user {user_node.unique_id}")
|
||||
return []
|
||||
if stats['month_count'] == 0:
|
||||
logger.error(f"No calendar months found for user {user_node.unique_id}")
|
||||
return []
|
||||
if stats['day_count'] == 0:
|
||||
logger.error(f"No calendar days found for user {user_node.unique_id}")
|
||||
return []
|
||||
|
||||
# Get all calendar days without year filter
|
||||
query = """
|
||||
MATCH (w:User {unique_id: $user_id})-[:HAS_CALENDAR]->(c:Calendar)
|
||||
-[:CALENDAR_INCLUDES_YEAR]->(y:CalendarYear)
|
||||
-[:YEAR_INCLUDES_MONTH]->(m:CalendarMonth)
|
||||
-[:MONTH_INCLUDES_DAY]->(d:CalendarDay)
|
||||
RETURN d.unique_id as unique_id,
|
||||
d.date as date,
|
||||
d.day_of_week as day_of_week,
|
||||
d.iso_day as iso_day,
|
||||
d.path as path
|
||||
ORDER BY d.date
|
||||
"""
|
||||
|
||||
result = session.run(query, user_id=user_node.unique_id)
|
||||
calendar_days = []
|
||||
for record in result:
|
||||
calendar_day = CalendarDayNode(
|
||||
unique_id=record['unique_id'],
|
||||
date=record['date'],
|
||||
day_of_week=record['day_of_week'],
|
||||
iso_day=record['iso_day'],
|
||||
path=record['path']
|
||||
)
|
||||
calendar_days.append(calendar_day)
|
||||
|
||||
if not calendar_days:
|
||||
logger.error(f"No calendar days found for user {user_node.unique_id}")
|
||||
else:
|
||||
# Log the date range we have
|
||||
dates = sorted([day.date for day in calendar_days])
|
||||
logger.info(f"Found {len(calendar_days)} calendar days for user {user_node.unique_id}")
|
||||
logger.info(f"Calendar days range from {dates[0]} to {dates[-1]}")
|
||||
|
||||
return calendar_days
|
||||
|
||||
def create_user_worker_timetable(
|
||||
user_node: UserNode,
|
||||
user_worker_node: TeacherNode,
|
||||
school_db_name: str
|
||||
):
|
||||
"""
|
||||
Create a worker timetable structure in the user's database that mirrors
|
||||
the school timetable, with lessons linked to the user's calendar structure.
|
||||
"""
|
||||
user_db_name = user_worker_node.user_db_name
|
||||
|
||||
# Initialize filesystem and Neo4j
|
||||
fs_handler = ClassroomCopilotFilesystem(db_name=user_db_name, init_run_type="user")
|
||||
|
||||
# Create teacher timetable directory under the worker's directory
|
||||
_, worker_timetable_path = fs_handler.create_teacher_timetable_directory(user_worker_node.path)
|
||||
|
||||
# Initialize neontology connection
|
||||
neon.init_neontology_connection()
|
||||
|
||||
# Get user's calendar nodes
|
||||
calendar_nodes = get_user_calendar_nodes(user_db_name, user_node)
|
||||
if not calendar_nodes:
|
||||
logger.warning(f"No calendar nodes found for user {user_node.unique_id}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "No calendar nodes found for user"
|
||||
}
|
||||
|
||||
try:
|
||||
# Create UserTeacherTimetableNode
|
||||
timetable_unique_id = f"UserTeacherTimetable_{user_worker_node.teacher_code}"
|
||||
worker_timetable = UserTeacherTimetableNode(
|
||||
unique_id=timetable_unique_id,
|
||||
school_db_name=school_db_name,
|
||||
school_timetable_id=f"TeacherTimetable_{user_worker_node.teacher_code}",
|
||||
path=worker_timetable_path
|
||||
)
|
||||
|
||||
# Create the timetable node and its tldraw file
|
||||
neon.create_or_merge_neontology_node(worker_timetable, database=user_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(worker_timetable.path, worker_timetable.to_dict())
|
||||
|
||||
# Link timetable to teacher using the correct relationship structure
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
TeacherHasTimetable(source=user_worker_node, target=worker_timetable),
|
||||
database=user_db_name,
|
||||
operation='merge'
|
||||
)
|
||||
|
||||
# Get classes from school database
|
||||
school_classes = get_school_worker_classes(school_db_name, user_node.unique_id, user_worker_node.unique_id)
|
||||
if not school_classes:
|
||||
logger.warning(f"No classes found for teacher {user_worker_node.unique_id} in school database")
|
||||
return {
|
||||
"status": "warning",
|
||||
"message": "No classes found in school database"
|
||||
}
|
||||
|
||||
# Dictionary to store lessons by class
|
||||
class_lessons = {}
|
||||
|
||||
for class_data in school_classes:
|
||||
class_name_safe = class_data['subject_class_code'].replace(' ', '_')
|
||||
_, class_path = fs_handler.create_teacher_class_directory(worker_timetable_path, class_name_safe)
|
||||
|
||||
# Create SubjectClassNode
|
||||
subject_class_node = SubjectClassNode(
|
||||
unique_id=class_data['unique_id'],
|
||||
subject_class_code=class_data['subject_class_code'],
|
||||
year_group=class_data['year_group'],
|
||||
subject=class_data['subject'],
|
||||
subject_code=class_data['subject_code'],
|
||||
path=class_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(subject_class_node, database=user_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(subject_class_node.path, subject_class_node.to_dict())
|
||||
|
||||
# Link class to timetable
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
TimetableHasClass(source=worker_timetable, target=subject_class_node),
|
||||
database=user_db_name,
|
||||
operation='merge'
|
||||
)
|
||||
|
||||
# Initialize empty list for this class's lessons
|
||||
class_lessons[class_data['unique_id']] = []
|
||||
|
||||
# Get periods from school database
|
||||
periods = get_school_class_periods(school_db_name, class_data['unique_id'])
|
||||
if not periods:
|
||||
logger.warning(f"No periods found for class {class_data['unique_id']} in school database")
|
||||
continue
|
||||
|
||||
for period_data in periods:
|
||||
# Create UserTimetableLessonNode
|
||||
lesson_unique_id = f"UserTimetableLesson_{timetable_unique_id}_{class_name_safe}_{period_data['date']}_{period_data['period_code']}"
|
||||
timetable_lesson_node = UserTimetableLessonNode(
|
||||
unique_id=lesson_unique_id,
|
||||
subject_class=class_data['subject_class_code'],
|
||||
date=period_data['date'],
|
||||
start_time=period_data['start_time'],
|
||||
end_time=period_data['end_time'],
|
||||
period_code=period_data['period_code'],
|
||||
school_db_name=school_db_name,
|
||||
school_period_id=period_data['unique_id'],
|
||||
path="Not set" # Will be set after creating directories
|
||||
)
|
||||
|
||||
if calendar_day := next(
|
||||
(
|
||||
day
|
||||
for day in calendar_nodes
|
||||
if day.date == period_data['date']
|
||||
),
|
||||
None,
|
||||
):
|
||||
# Create lesson directory using calendar info
|
||||
_, lesson_path = fs_handler.create_teacher_timetable_lesson_directory(
|
||||
class_path,
|
||||
f"{calendar_day.date}_{period_data['period_code']}"
|
||||
)
|
||||
timetable_lesson_node.path = lesson_path
|
||||
|
||||
# Create and link nodes
|
||||
neon.create_or_merge_neontology_node(timetable_lesson_node, database=user_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(timetable_lesson_node.path, timetable_lesson_node.to_dict())
|
||||
|
||||
# Link lesson to class
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
ClassHasLesson(source=subject_class_node, target=timetable_lesson_node),
|
||||
database=user_db_name,
|
||||
operation='merge'
|
||||
)
|
||||
|
||||
# Link lesson to calendar day (keeping only one direction)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
CalendarDayHasTimetableLesson(
|
||||
source=calendar_day,
|
||||
target=timetable_lesson_node
|
||||
),
|
||||
database=user_db_name,
|
||||
operation='merge'
|
||||
)
|
||||
|
||||
# Store the lesson node
|
||||
class_lessons[class_data['unique_id']].append({
|
||||
'node': timetable_lesson_node,
|
||||
'date': period_data['date'],
|
||||
'start_time': period_data['start_time']
|
||||
})
|
||||
else:
|
||||
logger.warning(f"No calendar day found for date {period_data['date']} - this is expected if the date is not in the current calendar year")
|
||||
|
||||
# Create sequential relationships for each class
|
||||
for class_id, lessons in class_lessons.items():
|
||||
# Sort lessons by date and start time
|
||||
sorted_lessons = sorted(lessons, key=lambda x: (x['date'], x['start_time']))
|
||||
|
||||
# Create relationships between consecutive lessons
|
||||
for i in range(len(sorted_lessons) - 1):
|
||||
current_lesson = sorted_lessons[i]['node']
|
||||
next_lesson = sorted_lessons[i + 1]['node']
|
||||
|
||||
# Skip if current and next lesson are the same node
|
||||
if current_lesson.unique_id != next_lesson.unique_id:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
TimetableLessonFollowsTimetableLesson(
|
||||
source=current_lesson,
|
||||
target=next_lesson
|
||||
),
|
||||
database=user_db_name,
|
||||
operation='merge'
|
||||
)
|
||||
|
||||
logger.info(f"Created sequential relationships for class {class_id}")
|
||||
|
||||
logger.info(f"Successfully created user timetable structure for {user_worker_node.teacher_code}")
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "User timetable structure created successfully",
|
||||
"timetable_node": worker_timetable.to_dict()
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating user timetable structure: {str(e)}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Error creating user timetable structure: {str(e)}"
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
from dotenv import load_dotenv, find_dotenv
|
||||
load_dotenv(find_dotenv())
|
||||
import os
|
||||
import modules.logger_tool as logger
|
||||
log_name = 'api_modules_database_init_init_worker_timetable'
|
||||
log_dir = os.getenv("LOG_PATH", "/logs") # Default path as fallback
|
||||
logging = logger.get_logger(
|
||||
name=log_name,
|
||||
log_level=os.getenv("LOG_LEVEL", "DEBUG"),
|
||||
log_path=log_dir,
|
||||
log_file=log_name,
|
||||
runtime=True,
|
||||
log_format='default'
|
||||
)
|
||||
import pandas as pd
|
||||
import re
|
||||
import modules.database.tools.neo4j_driver_tools as driver
|
||||
import modules.database.tools.neontology_tools as neon
|
||||
import modules.database.tools.neo4j_session_tools as session
|
||||
from modules.database.tools.filesystem_tools import ClassroomCopilotFilesystem
|
||||
from modules.database.schemas.nodes.schools.schools import SubjectClassNode
|
||||
from modules.database.schemas.nodes.workers.workers import TeacherNode
|
||||
from modules.database.schemas.nodes.schools.timetable import AcademicPeriodNode, RegistrationPeriodNode
|
||||
from modules.database.schemas.nodes.workers.timetable import TeacherTimetableNode, TimetableLessonNode, PlannedLessonNode
|
||||
from modules.database.schemas.nodes.schools.pastoral import YearGroupSyllabusNode
|
||||
from modules.database.schemas.relationships.planning_relationships import TimetableLessonBelongsToPeriod, TimetableLessonHasPlannedLesson, TeacherHasTimetable, TimetableHasClass, ClassHasLesson, TimetableLessonFollowsTimetableLesson, PlannedLessonFollowsPlannedLesson, SubjectClassBelongsToYearGroupSyllabus
|
||||
|
||||
def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: TeacherNode):
|
||||
logging.info(f"School worker node: {school_worker_node}")
|
||||
worker_node = TeacherNode(**school_worker_node)
|
||||
logging.info(f"Worker node: {worker_node}")
|
||||
worker_db_name = worker_node.worker_db_name
|
||||
|
||||
logging.info(f"Initialising filesystem handler...")
|
||||
fs_handler = ClassroomCopilotFilesystem(db_name=worker_db_name, init_run_type="user")
|
||||
_, worker_timetable_path = fs_handler.create_teacher_timetable_directory(worker_node.path)
|
||||
|
||||
logging.info(f"Initialising neo4j connection...")
|
||||
neon.init_neontology_connection()
|
||||
|
||||
try:
|
||||
timetable_unique_id = f"TeacherTimetable_{worker_node.teacher_code}"
|
||||
worker_timetable = TeacherTimetableNode(
|
||||
unique_id=timetable_unique_id,
|
||||
path=worker_timetable_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(worker_timetable, database=worker_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(worker_timetable.path, worker_timetable.to_dict())
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
TeacherHasTimetable(source=worker_node, target=worker_timetable),
|
||||
database=worker_db_name, operation='merge'
|
||||
)
|
||||
logging.info(f"Teacher timetable node created: {worker_timetable}")
|
||||
|
||||
# Group the timetable by class
|
||||
class_groups = timetable_df.groupby('Class')
|
||||
for class_name, class_df in class_groups:
|
||||
if pd.notna(class_name):
|
||||
class_name_safe = re.sub(r'[^A-Za-z0-9_ ]+', '', class_name)
|
||||
_, class_path = fs_handler.create_teacher_class_directory(worker_timetable.path, class_name_safe)
|
||||
|
||||
subject_class_node_unique_id = f"SubjectClass_{class_name}"
|
||||
subject_class_node = SubjectClassNode(
|
||||
unique_id=subject_class_node_unique_id,
|
||||
subject_class_code=class_name,
|
||||
year_group=str(int(class_df['YearGroup'].iloc[0])), # TODO: Hacky fix for the year group being a float
|
||||
subject=str(class_df['Subject'].iloc[0]),
|
||||
subject_code=str(class_df['SubjectCode'].iloc[0]),
|
||||
path=class_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(subject_class_node, database=worker_db_name, operation='merge')
|
||||
logging.info(f"Class node created: {subject_class_node}")
|
||||
# Create the tldraw file for the node
|
||||
fs_handler.create_default_tldraw_file(subject_class_node.path, subject_class_node.to_dict())
|
||||
|
||||
# Link ClassNode to TeacherTimetableNode
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
TimetableHasClass(source=worker_timetable, target=subject_class_node),
|
||||
database=worker_db_name, operation='merge'
|
||||
)
|
||||
logging.info(f"Relationship created from {worker_timetable.unique_id} to {subject_class_node.unique_id}")
|
||||
|
||||
# Link class to corresponding YearGoupSyllabus
|
||||
|
||||
year_group_syllabus_search_driver = driver.get_driver(worker_db_name)
|
||||
year_group_syllabus_search_session = year_group_syllabus_search_driver.session(database=worker_db_name)
|
||||
year_group_syllabus = session.find_nodes_by_label_and_properties(year_group_syllabus_search_session, "YearGroupSyllabus", {"yr_syllabus_year_group": subject_class_node.year_group, "yr_syllabus_subject_code": subject_class_node.subject_code})
|
||||
if year_group_syllabus:
|
||||
year_group_syllabus_node_data = year_group_syllabus[0]
|
||||
year_group_syllabus_node = YearGroupSyllabusNode(**year_group_syllabus_node_data)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
SubjectClassBelongsToYearGroupSyllabus(source=subject_class_node, target=year_group_syllabus_node),
|
||||
database=worker_db_name, operation='merge'
|
||||
)
|
||||
logging.info(f"Relationship created from {subject_class_node.unique_id} to {year_group_syllabus_node.unique_id}")
|
||||
else:
|
||||
logging.warning(f"No YearGroupSyllabus found for class {class_name} with year group {subject_class_node.year_group} and subject code {subject_class_node.subject_code}")
|
||||
|
||||
class_lesson_nodes = []
|
||||
planned_lesson_nodes = []
|
||||
lesson_number = 0
|
||||
for _, row in class_df.iterrows():
|
||||
properties = {
|
||||
"period_code": row['PeriodCode']
|
||||
}
|
||||
class_lessons_search_driver = driver.get_driver(worker_db_name)
|
||||
class_lessons_search_session = class_lessons_search_driver.session(database=worker_db_name)
|
||||
# If the period code contains "Rg" then we want to find the corresponding registration period and use its unique id
|
||||
if "Rg" in row['PeriodCode']: # TODO: This is hacky and not very flexible. We are assuming that any period code containing "Rg" is a registration period. We should probably find a more robust way to identify registration periods
|
||||
logging.info(f"Registration period found for class {class_name} with period code {row['PeriodCode']}")
|
||||
class_lessons = session.find_nodes_by_label_and_properties(class_lessons_search_session, "RegistrationPeriod", properties)
|
||||
else:
|
||||
logging.info(f"Academic period found for class {class_name} with period code {row['PeriodCode']}")
|
||||
class_lessons = session.find_nodes_by_label_and_properties(class_lessons_search_session, "AcademicPeriod", properties)
|
||||
if class_lessons:
|
||||
lesson_of_same_period = 0
|
||||
number_of_lessons = len(class_lessons)
|
||||
while lesson_of_same_period < number_of_lessons:
|
||||
class_lesson = class_lessons[lesson_of_same_period]
|
||||
if "Rg" in row['PeriodCode']:
|
||||
period_node = RegistrationPeriodNode(**class_lesson)
|
||||
else:
|
||||
period_node = AcademicPeriodNode(**class_lesson)
|
||||
lesson_period_code = row['PeriodCode']
|
||||
date = class_lesson['date']
|
||||
date_safe = date.strftime("%Y-%m-%d")
|
||||
# Clean the class_name to make it directory-safe (catch all for invalid characters)
|
||||
timetable_lesson_unique_id = f"TimetableLesson_{timetable_unique_id}_Class_{class_name}_Lesson_{lesson_number}_{date_safe}_{lesson_period_code}"
|
||||
|
||||
timetable_lesson_node = TimetableLessonNode(
|
||||
unique_id=timetable_lesson_unique_id,
|
||||
subject_class=class_name,
|
||||
date=date,
|
||||
start_time=class_lesson['start_time'].time(), # TODO: This is probably how we should format the start and end time properties for all such nodes
|
||||
end_time=class_lesson['end_time'].time(),
|
||||
period_code=lesson_period_code,
|
||||
path="Not set"
|
||||
)
|
||||
neon.create_or_merge_neontology_node(timetable_lesson_node, database=worker_db_name, operation='merge')
|
||||
logging.info(f"TimetableLessonNode created: {timetable_lesson_node}")
|
||||
class_lesson_nodes.append(timetable_lesson_node)
|
||||
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
TimetableLessonBelongsToPeriod(source=timetable_lesson_node, target=period_node),
|
||||
database=worker_db_name, operation='merge'
|
||||
)
|
||||
logging.info(f"Relationship created from {timetable_lesson_node.unique_id} to {period_node.unique_id}")
|
||||
|
||||
# Link TimetableLessonNode to ClassNode
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
ClassHasLesson(source=subject_class_node, target=timetable_lesson_node),
|
||||
database=worker_db_name, operation='merge'
|
||||
)
|
||||
logging.info(f"Relationship created from {subject_class_node.unique_id} to {timetable_lesson_node.unique_id}")
|
||||
|
||||
# Create PlannedLessonNode
|
||||
planned_lesson_unique_id = f"PlannedLesson_{timetable_unique_id}_Class_{class_name}_Lesson_{lesson_number}_{date_safe}_{lesson_period_code}"
|
||||
planned_lesson_node = PlannedLessonNode(
|
||||
unique_id=planned_lesson_unique_id,
|
||||
date=date,
|
||||
start_time=class_lesson['start_time'].time(),
|
||||
end_time=class_lesson['end_time'].time(),
|
||||
period_code=lesson_period_code,
|
||||
subject_class=class_name,
|
||||
year_group=subject_class_node.year_group,
|
||||
subject=subject_class_node.subject,
|
||||
teacher_code=worker_node.teacher_code,
|
||||
planning_status="Unplanned",
|
||||
topic_code=None,
|
||||
topic_name=None,
|
||||
lesson_code=None,
|
||||
lesson_name=None,
|
||||
learning_statement_codes=None,
|
||||
learning_statements=None,
|
||||
learning_resource_codes=None,
|
||||
learning_resources=None,
|
||||
path="Not set"
|
||||
)
|
||||
# Create the PlannedLessonNode
|
||||
neon.create_or_merge_neontology_node(planned_lesson_node, database=worker_db_name, operation='merge')
|
||||
logging.info(f"PlannedLessonNode created: {planned_lesson_node}")
|
||||
planned_lesson_nodes.append(planned_lesson_node)
|
||||
|
||||
# Link PlannedLessonNode to TimetableLessonNode
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
TimetableLessonHasPlannedLesson(source=timetable_lesson_node, target=planned_lesson_node),
|
||||
database=worker_db_name, operation='merge'
|
||||
)
|
||||
logging.info(f"Relationship created from {timetable_lesson_node.unique_id} to {planned_lesson_node.unique_id}")
|
||||
lesson_of_same_period += 1
|
||||
lesson_number += 1
|
||||
else:
|
||||
logging.warning(f"No class periods found for class {class_name} on day {row['DayOfWeek']}")
|
||||
# Sort the nodes by date and start time
|
||||
class_lesson_nodes.sort(key=lambda x: (x.date, x.start_time))
|
||||
planned_lesson_nodes.sort(key=lambda x: (x.date, x.start_time))
|
||||
|
||||
# Create sequential relationships and directories for TimetableLessonNodes
|
||||
for i in range(1, len(class_lesson_nodes)):
|
||||
previous_node = class_lesson_nodes[i - 1]
|
||||
current_node = class_lesson_nodes[i]
|
||||
i_safe = f"{i:02d}"
|
||||
_, class_lesson_path = fs_handler.create_teacher_timetable_lesson_directory(class_path, f"{i_safe}_{current_node.date}_{current_node.period_code}")
|
||||
current_node.path = class_lesson_path
|
||||
neon.create_or_merge_neontology_node(current_node, database=worker_db_name, operation='merge')
|
||||
logging.info(f"TimetableLessonNode directory created and node merged into database: {current_node}")
|
||||
# Create the tldraw file for the node
|
||||
fs_handler.create_default_tldraw_file(current_node.path, current_node.to_dict())
|
||||
if previous_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
TimetableLessonFollowsTimetableLesson(source=previous_node, target=current_node),
|
||||
database=worker_db_name, operation='merge'
|
||||
)
|
||||
logging.info(f"Sequential relationship created between {previous_node.unique_id} and {current_node.unique_id}")
|
||||
|
||||
# Create sequential relationships for PlannedLessonNodes
|
||||
for i in range(1, len(planned_lesson_nodes)):
|
||||
previous_node = planned_lesson_nodes[i - 1]
|
||||
current_node = planned_lesson_nodes[i]
|
||||
i_safe = f"{i:02d}"
|
||||
_, planned_lesson_path = fs_handler.create_teacher_planned_lesson_directory(class_path, f"{i_safe}_{current_node.date}_{current_node.period_code}")
|
||||
current_node.path = planned_lesson_path
|
||||
neon.create_or_merge_neontology_node(current_node, database=worker_db_name, operation='merge')
|
||||
logging.info(f"PlannedLessonNode directory created and node merged into database: {current_node}")
|
||||
# Create the tldraw file for the node
|
||||
fs_handler.create_default_tldraw_file(current_node.path, current_node.to_dict())
|
||||
if previous_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
PlannedLessonFollowsPlannedLesson(source=previous_node, target=current_node),
|
||||
database=worker_db_name, operation='merge'
|
||||
)
|
||||
logging.info(f"Sequential relationship created between {previous_node.unique_id} and {current_node.unique_id}")
|
||||
logging.info(f"Successfully initialized worker timetable for worker {worker_node.teacher_code}")
|
||||
return {"status": "success", "message": "Worker timetable initialized successfully"}
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error initializing worker timetable: {str(e)}")
|
||||
return {"status": "error", "message": f"Error initializing worker timetable: {str(e)}"}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
from dotenv import load_dotenv, find_dotenv
|
||||
load_dotenv(find_dotenv())
|
||||
import os
|
||||
import modules.logger_tool as logger
|
||||
log_name = 'api_modules_database_tools_xl_tools'
|
||||
log_dir = os.getenv("LOG_PATH", "/logs") # Default path as fallback
|
||||
logging = logger.get_logger(
|
||||
name=log_name,
|
||||
log_level=os.getenv("LOG_LEVEL", "DEBUG"),
|
||||
log_path=log_dir,
|
||||
log_file=log_name,
|
||||
runtime=True,
|
||||
log_format='default'
|
||||
)
|
||||
import pandas as pd
|
||||
from fastapi import UploadFile
|
||||
|
||||
def create_dataframes(excel_file, return_clean=False):
|
||||
excel_sheets = pd.read_excel(excel_file, sheet_name=None)
|
||||
# Log the sheet names
|
||||
logging.info(f"Sheet names: {excel_sheets.keys()}")
|
||||
return {sheet.lower(): data for sheet, data in excel_sheets.items()}
|
||||
|
||||
def create_dataframes_from_fastapiuploadfile(upload_file: UploadFile):
|
||||
from io import BytesIO
|
||||
file_content = upload_file.file.read()
|
||||
file_content_io = BytesIO(file_content)
|
||||
return pd.read_excel(file_content_io, sheet_name=None, engine='openpyxl')
|
||||
|
||||
def replace_nan_with_default(data, default_values):
|
||||
for key in default_values:
|
||||
if pd.isna(data.get(key, None)):
|
||||
# logging.debug(f"Replacing NaN in {key} with default value '{default_values[key]}'")
|
||||
data[key] = default_values[key]
|
||||
return data
|
||||
Reference in New Issue
Block a user