ClassroomCopilot/api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

latest

Browse Source
This commit is contained in:
kcar
2025-11-14 14:47:19 +00:00
parent 2a85845835
commit 3758c7572a
137 changed files with 365654 additions and 11147 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/database/init/init_school_curriculum.py
+206 -73
View File
@@ -4,6 +4,7 @@ logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH
import pandas as pd
import modules.database.tools.neontology_tools as neon
import modules.database.tools.supabase_storage_tools as storage_tools
import modules.database.schemas.nodes.schools.schools as school_nodes
import modules.database.schemas.nodes.schools.curriculum as curriculum_nodes
import modules.database.schemas.nodes.schools.pastoral as pastoral_nodes
@@ -40,7 +41,7 @@ def sort_year_groups(df):
df['YearGroupNumeric'] = pd.to_numeric(df['YearGroup'], errors='coerce')
return df.sort_values(by='YearGroupNumeric')
def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_node: school_nodes.SchoolNode):
def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_node: school_nodes.SchoolNode, storage_tools=None):
logger.info(f"Initialising neo4j connection...")
neon.init_neontology_connection()
@@ -70,10 +71,20 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
last_key_stage_node = None
# Create Department Structure node
department_structure_node_unique_id = f"DepartmentStructure_{school_node.unique_id}"
department_structure_node_uuid_string = f"DepartmentStructure_{school_node.uuid_string}"
# For structure nodes, we can use a simple path or leave empty for consistency
# Since this is just an organizational node, we'll use a simple path
if storage_tools:
# Use the school's base department directory as the structure node path
dept_structure_path = f"cc.public.snapshots/DepartmentStructure/{school_node.uuid_string}"
node_storage_path = dept_structure_path
else:
node_storage_path = ""
department_structure_node = school_structures.DepartmentStructureNode(
unique_id=department_structure_node_unique_id,
tldraw_snapshot=""
uuid_string=department_structure_node_uuid_string,
node_storage_path=node_storage_path
)
# Create in school database only
neon.create_or_merge_neontology_node(department_structure_node, database=db_name, operation='merge')
@@ -86,10 +97,17 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
)
logger.info(f"Created department structure node and linked to school")
curriculum_structure_node_unique_id = f"CurriculumStructure_{school_node.unique_id}"
curriculum_structure_node_uuid_string = f"CurriculumStructure_{school_node.uuid_string}"
# Generate storage path for curriculum structure node
if storage_tools:
curriculum_dir_created, node_storage_path = storage_tools.create_curriculum_storage_path(curriculum_structure_node_uuid_string)
else:
node_storage_path = ""
curriculum_node = school_structures.CurriculumStructureNode(
unique_id=curriculum_structure_node_unique_id,
tldraw_snapshot=""
uuid_string=curriculum_structure_node_uuid_string,
node_storage_path=node_storage_path
)
# Create in school database only
neon.create_or_merge_neontology_node(curriculum_node, database=db_name, operation='merge')
@@ -102,10 +120,17 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
)
logger.info(f"Created curriculum node and relationship with school")
pastoral_structure_node_unique_id = f"PastoralStructure_{school_node.unique_id}"
pastoral_structure_node_uuid_string = f"PastoralStructure_{school_node.uuid_string}"
# Generate storage path for pastoral structure node
if storage_tools:
pastoral_dir_created, node_storage_path = storage_tools.create_pastoral_storage_path(pastoral_structure_node_uuid_string)
else:
node_storage_path = ""
pastoral_node = school_structures.PastoralStructureNode(
unique_id=pastoral_structure_node_unique_id,
tldraw_snapshot=""
uuid_string=pastoral_structure_node_uuid_string,
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(pastoral_node, database=db_name, operation='merge')
node_library['pastoral_node'] = pastoral_node
@@ -120,11 +145,20 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
unique_departments = keystagesyllabus_df['Department'].dropna().unique()
for department_name in unique_departments:
department_unique_id = f"Department_{school_node.unique_id}_{department_name.replace(' ', '_')}"
department_uuid_string = f"Department_{school_node.uuid_string}_{department_name.replace(' ', '_')}"
# Generate storage path for department node using the storage tools
if storage_tools:
# Create department directory under the school's department structure
dept_path = f"cc.public.snapshots/Department/{department_uuid_string}"
node_storage_path = dept_path
else:
node_storage_path = ""
department_node = school_nodes.DepartmentNode(
unique_id=department_unique_id,
uuid_string=department_uuid_string,
name=department_name,
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create department in school database only
neon.create_or_merge_neontology_node(department_node, database=db_name, operation='merge')
@@ -147,17 +181,25 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
# Process subjects from key stage syllabuses first (these have department info)
for _, subject_row in unique_subjects.iterrows():
subject_unique_id = f"Subject_{school_node.unique_id}_{subject_row['SubjectCode']}"
subject_uuid_string = f"Subject_{school_node.uuid_string}_{subject_row['SubjectCode']}"
department_node = node_library['department_nodes'].get(subject_row['Department'])
if not department_node:
logger.warning(f"No department found for subject {subject_row['Subject']} with code {subject_row['SubjectCode']}")
continue
# Generate storage path for subject node
if storage_tools:
# Create subject directory under the specific department
subject_path = f"cc.public.snapshots/Subject/{subject_uuid_string}"
node_storage_path = subject_path
else:
node_storage_path = ""
subject_node = curriculum_nodes.SubjectNode(
unique_id=subject_unique_id,
uuid_string=subject_uuid_string,
id=subject_row['SubjectCode'],
name=subject_row['Subject'],
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create subject in both databases
neon.create_or_merge_neontology_node(subject_node, database=db_name, operation='merge')
@@ -173,14 +215,23 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
# Process any additional subjects from year group syllabuses (these won't have department info)
for _, subject_row in additional_subjects.iterrows():
subject_unique_id = f"Subject_{school_node.unique_id}_{subject_row['SubjectCode']}"
subject_uuid_string = f"Subject_{school_node.uuid_string}_{subject_row['SubjectCode']}"
# Create in a special "Unassigned" department
unassigned_dept_name = "Unassigned Department"
if unassigned_dept_name not in node_library['department_nodes']:
# Generate storage path for unassigned department node
if filesystem:
# Create unassigned department directory under the school's department structure
unassigned_dept_path = os.path.join(filesystem.root_path, "departments", "Unassigned")
filesystem.create_directory(unassigned_dept_path)
node_storage_path = os.path.relpath(unassigned_dept_path, filesystem.base_path)
else:
node_storage_path = ""
department_node = school_nodes.DepartmentNode(
unique_id=f"Department_{school_node.unique_id}_Unassigned",
uuid_string=f"Department_{school_node.uuid_string}_Unassigned",
name=unassigned_dept_name,
tldraw_snapshot=""
node_storage_path=node_storage_path
)
neon.create_or_merge_neontology_node(department_node, database=db_name, operation='merge')
node_library['department_nodes'][unassigned_dept_name] = department_node
@@ -192,11 +243,20 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
)
logger.info(f"Created unassigned department node and linked to department structure")
# Generate storage path for subject node
if filesystem:
# Create subject directory under the unassigned department
subject_path = os.path.join(filesystem.root_path, "departments", "Unassigned", subject_row['Subject'].replace(' ', '_'))
filesystem.create_directory(subject_path)
node_storage_path = os.path.relpath(subject_path, filesystem.base_path)
else:
node_storage_path = ""
subject_node = curriculum_nodes.SubjectNode(
unique_id=subject_unique_id,
uuid_string=subject_uuid_string,
id=subject_row['SubjectCode'],
name=subject_row['Subject'],
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create subject in both databases
neon.create_or_merge_neontology_node(subject_node, database=db_name, operation='merge')
@@ -226,20 +286,29 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
# First create all key stage nodes and key stage syllabus nodes
for index, ks_row in keystagesyllabus_df.sort_values('KeyStage').iterrows():
key_stage = str(ks_row['KeyStage'])
logger.debug(f"Processing key stage syllabus row - Subject: {ks_row['Subject']}, Key Stage: {key_stage}")
subject = str(ks_row['Subject'])
syllabus_id = str(ks_row['ID'])
logger.debug(f"Processing key stage syllabus row - Subject: {subject}, Key Stage: {key_stage}, Syllabus ID: {syllabus_id}")
subject_node = node_library['subject_nodes'].get(ks_row['Subject'])
subject_node = node_library['subject_nodes'].get(subject)
if not subject_node:
logger.warning(f"No subject node found for subject {ks_row['Subject']}")
logger.warning(f"No subject node found for subject {subject}")
continue
if key_stage not in key_stage_nodes_created:
key_stage_node_unique_id = f"KeyStage_{curriculum_node.unique_id}_KStg{key_stage}"
key_stage_node_uuid_string = f"KeyStage_{curriculum_node.uuid_string}_KStg{key_stage}"
# Generate storage path for key stage node
if filesystem:
key_stage_dir_created, key_stage_path = filesystem.create_curriculum_key_stage_syllabus_directory(curriculum_path, key_stage, subject, syllabus_id)
node_storage_path = os.path.relpath(key_stage_path, filesystem.base_path)
else:
node_storage_path = ""
key_stage_node = curriculum_nodes.KeyStageNode(
unique_id=key_stage_node_unique_id,
uuid_string=key_stage_node_uuid_string,
name=f"Key Stage {key_stage}",
key_stage=str(key_stage),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create key stage node in both databases
neon.create_or_merge_neontology_node(key_stage_node, database=db_name, operation='merge')
@@ -252,7 +321,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.CurriculumStructureIncludesKeyStage(source=curriculum_node, target=key_stage_node),
database=db_name, operation='merge'
)
logger.info(f"Created key stage node {key_stage_node_unique_id} and relationship with curriculum structure")
logger.info(f"Created key stage node {key_stage_node_uuid_string} and relationship with curriculum structure")
# Create sequential relationship between key stages in both databases
if last_key_stage_node:
@@ -264,27 +333,34 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.KeyStageFollowsKeyStage(source=last_key_stage_node, target=key_stage_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between key stages {last_key_stage_node.unique_id} and {key_stage_node.unique_id}")
logger.info(f"Created sequential relationship between key stages {last_key_stage_node.uuid_string} and {key_stage_node.uuid_string}")
last_key_stage_node = key_stage_node
# Create key stage syllabus under the subject's curriculum directory
key_stage_syllabus_node_unique_id = f"KeyStageSyllabus_{curriculum_node.unique_id}_{ks_row['Title'].replace(' ', '')}"
key_stage_syllabus_node_uuid_string = f"KeyStageSyllabus_{curriculum_node.uuid_string}_{ks_row['Title'].replace(' ', '')}"
logger.debug(f"Creating key stage syllabus node for {ks_row['Subject']} KS{key_stage} with ID {ks_row['ID']}")
key_stage_syllabus_node_unique_id = f"KeyStageSyllabus_{curriculum_node.unique_id}_{ks_row['Title'].replace(' ', '')}"
key_stage_syllabus_node_uuid_string = f"KeyStageSyllabus_{curriculum_node.uuid_string}_{ks_row['Title'].replace(' ', '')}"
# Generate storage path for key stage syllabus node
if filesystem:
syllabus_dir_created, syllabus_path = filesystem.create_curriculum_key_stage_syllabus_directory(curriculum_path, key_stage, ks_row['Subject'], ks_row['ID'])
node_storage_path = os.path.relpath(syllabus_path, filesystem.base_path)
else:
node_storage_path = ""
key_stage_syllabus_node = curriculum_nodes.KeyStageSyllabusNode(
unique_id=key_stage_syllabus_node_unique_id,
uuid_string=key_stage_syllabus_node_uuid_string,
id=ks_row['ID'],
name=ks_row['Title'],
key_stage=str(ks_row['KeyStage']),
subject_name=ks_row['Subject'],
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create key stage syllabus node in both databases
neon.create_or_merge_neontology_node(key_stage_syllabus_node, database=db_name, operation='merge')
neon.create_or_merge_neontology_node(key_stage_syllabus_node, database=curriculum_db_name, operation='merge')
node_library['key_stage_syllabus_nodes'][ks_row['ID']] = key_stage_syllabus_node
logger.debug(f"Created key stage syllabus node {key_stage_syllabus_node_unique_id} for {ks_row['Subject']} KS{key_stage}")
logger.debug(f"Created key stage syllabus node {key_stage_syllabus_node_uuid_string} for {ks_row['Subject']} KS{key_stage}")
# Link key stage syllabus to its subject in both databases
if subject_node:
@@ -296,7 +372,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.SubjectHasKeyStageSyllabus(source=subject_node, target=key_stage_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between subject {subject_node.unique_id} and key stage syllabus {key_stage_syllabus_node.unique_id}")
logger.info(f"Created relationship between subject {subject_node.uuid_string} and key stage syllabus {key_stage_syllabus_node.uuid_string}")
# Link key stage syllabus to its key stage in both databases
key_stage_node = key_stage_nodes_created.get(key_stage)
@@ -309,7 +385,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.KeyStageIncludesKeyStageSyllabus(source=key_stage_node, target=key_stage_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between key stage {key_stage_node.unique_id} and key stage syllabus {key_stage_syllabus_node.unique_id}")
logger.info(f"Created relationship between key stage {key_stage_node.uuid_string} and key stage syllabus {key_stage_syllabus_node.uuid_string}")
# Create sequential relationship between key stage syllabuses in both databases
last_key_stage_syllabus_node = last_key_stage_syllabus_nodes.get(ks_row['Subject'])
@@ -322,7 +398,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.KeyStageSyllabusFollowsKeyStageSyllabus(source=last_key_stage_syllabus_node, target=key_stage_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between key stage syllabuses {last_key_stage_syllabus_node.unique_id} and {key_stage_syllabus_node.unique_id}")
logger.info(f"Created sequential relationship between key stage syllabuses {last_key_stage_syllabus_node.uuid_string} and {key_stage_syllabus_node.uuid_string}")
last_key_stage_syllabus_nodes[ks_row['Subject']] = key_stage_syllabus_node
# Now process year groups and their syllabuses
@@ -339,12 +415,19 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
if pd.notna(numeric_year_group):
numeric_year_group = int(numeric_year_group)
if numeric_year_group not in year_group_nodes_created:
year_group_node_unique_id = f"YearGroup_{school_node.unique_id}_YGrp{numeric_year_group}"
year_group_node_uuid_string = f"YearGroup_{school_node.uuid_string}_YGrp{numeric_year_group}"
# Generate storage path for year group node
if filesystem:
year_group_dir_created, year_group_path = filesystem.create_pastoral_year_group_directory(pastoral_path, numeric_year_group)
node_storage_path = os.path.relpath(year_group_path, filesystem.base_path)
else:
node_storage_path = ""
year_group_node = pastoral_nodes.YearGroupNode(
unique_id=year_group_node_unique_id,
uuid_string=year_group_node_uuid_string,
year_group=str(numeric_year_group),
name=f"Year {numeric_year_group}, {year_group}",
tldraw_snapshot=""
name=f"Year {numeric_year_group}",
node_storage_path=node_storage_path
)
# Create year group node in both databases but use same directory
neon.create_or_merge_neontology_node(year_group_node, database=db_name, operation='merge')
@@ -360,7 +443,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.YearGroupFollowsYearGroup(source=last_year_group_node, target=year_group_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between year groups {last_year_group_node.unique_id} and {year_group_node.unique_id} across key stages")
logger.info(f"Created sequential relationship between year groups {last_year_group_node.uuid_string} and {year_group_node.uuid_string} across key stages")
last_year_group_node = year_group_node
# Create relationship with Pastoral Structure in school database only
@@ -368,7 +451,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.PastoralStructureIncludesYearGroup(source=pastoral_node, target=year_group_node),
database=db_name, operation='merge'
)
logger.info(f"Created year group node {year_group_node_unique_id} and relationship with pastoral structure")
logger.info(f"Created year group node {year_group_node_uuid_string} and relationship with pastoral structure")
year_group_nodes_created[numeric_year_group] = year_group_node
node_library['year_group_nodes'][str(numeric_year_group)] = year_group_node
@@ -376,14 +459,21 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
# Create year group syllabus nodes in both databases
year_group_node = year_group_nodes_created.get(numeric_year_group)
if year_group_node:
year_group_syllabus_node_unique_id = f"YearGroupSyllabus_{school_node.unique_id}_{yg_row['ID']}"
year_group_syllabus_node_uuid_string = f"YearGroupSyllabus_{school_node.uuid_string}_{yg_row['ID']}"
# Generate storage path for year group syllabus node
if filesystem:
yg_syllabus_dir_created, yg_syllabus_path = filesystem.create_curriculum_year_group_syllabus_directory(curriculum_path, yg_row['Subject'], numeric_year_group, yg_row['ID'])
node_storage_path = os.path.relpath(yg_syllabus_path, filesystem.base_path)
else:
node_storage_path = ""
year_group_syllabus_node = pastoral_nodes.YearGroupSyllabusNode(
unique_id=year_group_syllabus_node_unique_id,
uuid_string=year_group_syllabus_node_uuid_string,
id=yg_row['ID'],
name=yg_row['Title'],
year_group=str(yg_row['YearGroup']),
subject_name=yg_row['Subject'],
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create year group syllabus node in both databases but use same directory
@@ -406,7 +496,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.YearGroupSyllabusFollowsYearGroupSyllabus(source=last_year_group_syllabus_node, target=year_group_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between year group syllabuses {last_year_group_syllabus_node.unique_id} and {year_group_syllabus_node.unique_id}")
logger.info(f"Created sequential relationship between year group syllabuses {last_year_group_syllabus_node.uuid_string} and {year_group_syllabus_node.uuid_string}")
last_year_group_syllabus_nodes[yg_row['Subject']] = year_group_syllabus_node
# Create relationships in both databases using MATCH to avoid cartesian products
@@ -421,7 +511,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.SubjectHasYearGroupSyllabus(source=subject_node, target=year_group_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between subject {subject_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
logger.info(f"Created relationship between subject {subject_node.uuid_string} and year group syllabus {year_group_syllabus_node_uuid_string}")
# Link to year group
neon.create_or_merge_neontology_relationship(
@@ -432,7 +522,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.YearGroupHasYearGroupSyllabus(source=year_group_node, target=year_group_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between year group {year_group_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
logger.info(f"Created relationship between year group {year_group_node.uuid_string} and year group syllabus {year_group_syllabus_node_uuid_string}")
# Link to key stage syllabus if it exists for the same subject
key_stage_syllabus_node = node_library['key_stage_syllabus_nodes'].get(ks_row['ID'])
@@ -445,7 +535,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.KeyStageSyllabusIncludesYearGroupSyllabus(source=key_stage_syllabus_node, target=year_group_syllabus_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between key stage syllabus {key_stage_syllabus_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
logger.info(f"Created relationship between key stage syllabus {key_stage_syllabus_node.uuid_string} and year group syllabus {year_group_syllabus_node_uuid_string}")
# Process topics for this year group syllabus only if not already processed
topics_for_syllabus = topic_df[topic_df['SyllabusYearID'] == yg_row['ID']]
@@ -472,22 +562,29 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
if (syllabus_node.subject_name == topic_subject and
syllabus_node.key_stage == str(topic_key_stage)):
matching_syllabus_node = syllabus_node
logger.debug(f"Found matching syllabus node: {syllabus_node.unique_id}")
logger.debug(f"Found matching syllabus node: {syllabus_node.uuid_string}")
break
if not matching_syllabus_node:
logger.warning(f"No key stage syllabus node found for subject {topic_subject} and key stage {topic_key_stage}, skipping topic creation")
continue
topic_node_unique_id = f"Topic_{matching_syllabus_node.unique_id}_{topic_row['TopicID']}"
topic_node_uuid_string = f"Topic_{matching_syllabus_node.uuid_string}_{topic_row['TopicID']}"
# Generate storage path for topic node
if filesystem:
topic_dir_created, topic_path = filesystem.create_curriculum_topic_directory(yg_syllabus_path, topic_row['TopicID'])
node_storage_path = os.path.relpath(topic_path, filesystem.base_path)
else:
node_storage_path = ""
topic_node = curriculum_nodes.TopicNode(
unique_id=topic_node_unique_id,
uuid_string=topic_node_uuid_string,
id=topic_row['TopicID'],
name=topic_row.get('TopicTitle', default_topic_values['topic_title']),
total_number_of_lessons_for_topic=str(topic_row.get('TotalNumberOfLessonsForTopic', default_topic_values['total_number_of_lessons_for_topic'])),
type=topic_row.get('TopicType', default_topic_values['topic_type']),
assessment_type=topic_row.get('TopicAssessmentType', default_topic_values['topic_assessment_type']),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create topic node in curriculum database only
neon.create_or_merge_neontology_node(topic_node, database=curriculum_db_name, operation='merge')
@@ -502,7 +599,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.YearGroupSyllabusIncludesTopic(source=year_group_syllabus_node, target=topic_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationships between topic {topic_node_unique_id} and key stage syllabus {matching_syllabus_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
logger.info(f"Created relationships between topic {topic_node_uuid_string} and key stage syllabus {matching_syllabus_node.uuid_string} and year group syllabus {year_group_syllabus_node_uuid_string}")
# Process lessons for this topic only if not already processed
lessons_for_topic = lesson_df[
@@ -518,8 +615,15 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
continue
lessons_processed.add(lesson_row['LessonID'])
# Generate storage path for lesson node
if filesystem:
lesson_dir_created, lesson_path = filesystem.create_curriculum_lesson_directory(topic_path, lesson_row['LessonID'])
node_storage_path = os.path.relpath(lesson_path, filesystem.base_path)
else:
node_storage_path = ""
lesson_node = curriculum_nodes.TopicLessonNode(
unique_id=f"TopicLesson_{topic_node_unique_id}_{lesson_row['LessonID']}",
uuid_string=f"TopicLesson_{topic_node_uuid_string}_{lesson_row['LessonID']}",
id=lesson_row['LessonID'],
name=lesson_row.get('LessonTitle', default_topic_lesson_values['topic_lesson_title']),
type=lesson_row.get('LessonType', default_topic_lesson_values['topic_lesson_type']),
@@ -527,7 +631,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
suggested_activities=str(lesson_row.get('SuggestedActivities', default_topic_lesson_values['topic_lesson_suggested_activities'])),
skills_learned=str(lesson_row.get('SkillsLearned', default_topic_lesson_values['topic_lesson_skills_learned'])),
weblinks=str(lesson_row.get('WebLinks', default_topic_lesson_values['topic_lesson_weblinks'])),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create lesson node in curriculum database only
neon.create_or_merge_neontology_node(lesson_node, database=curriculum_db_name, operation='merge')
@@ -538,7 +642,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.TopicIncludesTopicLesson(source=topic_node, target=lesson_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created lesson node {lesson_node.unique_id} and relationship with topic {topic_node.unique_id}")
logger.info(f"Created lesson node {lesson_node.uuid_string} and relationship with topic {topic_node.uuid_string}")
# Create sequential relationships between lessons
if lesson_row['Lesson'].isdigit() and previous_lesson_node:
@@ -546,7 +650,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.TopicLessonFollowsTopicLesson(source=previous_lesson_node, target=lesson_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.unique_id} and {lesson_node.unique_id}")
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.uuid_string} and {lesson_node.uuid_string}")
previous_lesson_node = lesson_node
# Process learning statements for this lesson only if not already processed
@@ -558,12 +662,19 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
if statement_row['StatementID'] in statements_processed:
continue
statements_processed.add(statement_row['StatementID'])
# Generate storage path for learning statement node
if filesystem:
statement_dir_created, statement_path = filesystem.create_curriculum_learning_statement_directory(lesson_path, statement_row['StatementID'])
node_storage_path = os.path.relpath(statement_path, filesystem.base_path)
else:
node_storage_path = ""
statement_node = curriculum_nodes.LearningStatementNode(
unique_id=f"LearningStatement_{lesson_node.unique_id}_{statement_row['StatementID']}",
uuid_string=f"LearningStatement_{lesson_node.uuid_string}_{statement_row['StatementID']}",
id=statement_row['StatementID'],
name=statement_row.get('LearningStatement', default_learning_statement_values['lesson_learning_statement']),
type=statement_row.get('StatementType', default_learning_statement_values['lesson_learning_statement_type']),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create statement node in curriculum database only
neon.create_or_merge_neontology_node(statement_node, database=curriculum_db_name, operation='merge')
@@ -574,7 +685,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.LessonIncludesLearningStatement(source=lesson_node, target=statement_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created learning statement node {statement_node.unique_id} and relationship with lesson {lesson_node.unique_id}")
logger.info(f"Created learning statement node {statement_node.uuid_string} and relationship with lesson {lesson_node.uuid_string}")
else:
logger.warning(f"No year group node found for year group {year_group}, skipping syllabus creation")
@@ -601,15 +712,23 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
logger.warning(f"No key stage syllabus node found for subject {topic_subject} and key stage {topic_key_stage}, skipping topic creation")
continue
topic_node_unique_id = f"Topic_{matching_syllabus_node.unique_id}_{topic_row['TopicID']}"
topic_node_uuid_string = f"Topic_{matching_syllabus_node.uuid_string}_{topic_row['TopicID']}"
# Generate storage path for topic node
if filesystem:
syllabus_path = os.path.join(curriculum_path, "subjects", topic_subject, "key_stage_syllabuses", f"KS{topic_key_stage}", f"KS{topic_key_stage}.{topic_subject}")
topic_dir_created, keystage_topic_path = filesystem.create_curriculum_keystage_topic_directory(syllabus_path, topic_row['TopicID'])
node_storage_path = os.path.relpath(keystage_topic_path, filesystem.base_path)
else:
node_storage_path = ""
topic_node = curriculum_nodes.TopicNode(
unique_id=topic_node_unique_id,
uuid_string=topic_node_uuid_string,
id=topic_row['TopicID'],
name=topic_row.get('TopicTitle', default_topic_values['topic_title']),
total_number_of_lessons_for_topic=str(topic_row.get('TotalNumberOfLessonsForTopic', default_topic_values['total_number_of_lessons_for_topic'])),
type=topic_row.get('TopicType', default_topic_values['topic_type']),
assessment_type=topic_row.get('TopicAssessmentType', default_topic_values['topic_assessment_type']),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create topic node in curriculum database only
neon.create_or_merge_neontology_node(topic_node, database=curriculum_db_name, operation='merge')
@@ -621,7 +740,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.KeyStageSyllabusIncludesTopic(source=matching_syllabus_node, target=topic_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created relationship between topic {topic_node_unique_id} and key stage syllabus {matching_syllabus_node.unique_id}")
logger.info(f"Created relationship between topic {topic_node_uuid_string} and key stage syllabus {matching_syllabus_node.uuid_string}")
# Process lessons for this topic
lessons_for_topic = lesson_df[
@@ -636,8 +755,15 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
if lesson_row['LessonID'] in lessons_processed:
continue
lessons_processed.add(lesson_row['LessonID'])
# Generate storage path for lesson node
if filesystem:
lesson_dir_created, lesson_path = filesystem.create_curriculum_lesson_directory(topic_path, lesson_row['LessonID'])
node_storage_path = os.path.relpath(lesson_path, filesystem.base_path)
else:
node_storage_path = ""
lesson_node = curriculum_nodes.TopicLessonNode(
unique_id=f"TopicLesson_{topic_node_unique_id}_{lesson_row['LessonID']}",
uuid_string=f"TopicLesson_{topic_node_uuid_string}_{lesson_row['LessonID']}",
id=lesson_row['LessonID'],
name=lesson_row.get('LessonTitle', default_topic_lesson_values['topic_lesson_title']),
type=lesson_row.get('LessonType', default_topic_lesson_values['topic_lesson_type']),
@@ -645,7 +771,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
suggested_activities=str(lesson_row.get('SuggestedActivities', default_topic_lesson_values['topic_lesson_suggested_activities'])),
skills_learned=str(lesson_row.get('SkillsLearned', default_topic_lesson_values['topic_lesson_skills_learned'])),
weblinks=str(lesson_row.get('WebLinks', default_topic_lesson_values['topic_lesson_weblinks'])),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create lesson node in curriculum database only
neon.create_or_merge_neontology_node(lesson_node, database=curriculum_db_name, operation='merge')
@@ -656,7 +782,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.TopicIncludesTopicLesson(source=topic_node, target=lesson_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created lesson node {lesson_node.unique_id} and relationship with topic {topic_node.unique_id}")
logger.info(f"Created lesson node {lesson_node.uuid_string} and relationship with topic {topic_node.uuid_string}")
# Create sequential relationships between lessons
if lesson_row['Lesson'].isdigit() and previous_lesson_node:
@@ -664,7 +790,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.TopicLessonFollowsTopicLesson(source=previous_lesson_node, target=lesson_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.unique_id} and {lesson_node.unique_id}")
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.uuid_string} and {lesson_node.uuid_string}")
previous_lesson_node = lesson_node
# Process learning statements for this lesson
@@ -676,12 +802,19 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
if statement_row['StatementID'] in statements_processed:
continue
statements_processed.add(statement_row['StatementID'])
# Generate storage path for learning statement node
if filesystem:
statement_dir_created, statement_path = filesystem.create_curriculum_learning_statement_directory(lesson_path, statement_row['StatementID'])
node_storage_path = os.path.relpath(statement_path, filesystem.base_path)
else:
node_storage_path = ""
statement_node = curriculum_nodes.LearningStatementNode(
unique_id=f"LearningStatement_{lesson_node.unique_id}_{statement_row['StatementID']}",
uuid_string=f"LearningStatement_{lesson_node.uuid_string}_{statement_row['StatementID']}",
id=statement_row['StatementID'],
name=statement_row.get('LearningStatement', default_learning_statement_values['lesson_learning_statement']),
type=statement_row.get('StatementType', default_learning_statement_values['lesson_learning_statement_type']),
tldraw_snapshot=""
node_storage_path=node_storage_path
)
# Create statement node in curriculum database only
neon.create_or_merge_neontology_node(statement_node, database=curriculum_db_name, operation='merge')
@@ -692,6 +825,6 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
curriculum_relationships.LessonIncludesLearningStatement(source=lesson_node, target=statement_node),
database=curriculum_db_name, operation='merge'
)
logger.info(f"Created learning statement node {statement_node.unique_id} and relationship with lesson {lesson_node.unique_id}")
logger.info(f"Created learning statement node {statement_node.uuid_string} and relationship with lesson {lesson_node.uuid_string}")
return node_library