import os
from modules.logger_tool import initialise_logger
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
import pandas as pd

import modules.database.tools.neontology_tools as neon
import modules.database.schemas.nodes.schools.schools as school_nodes
import modules.database.schemas.nodes.schools.curriculum as curriculum_nodes
import modules.database.schemas.nodes.schools.pastoral as pastoral_nodes
import modules.database.schemas.nodes.structures.schools as school_structures
import modules.database.schemas.relationships.curriculum_relationships as curriculum_relationships
import modules.database.schemas.relationships.entity_relationships as ent_rels
import modules.database.schemas.relationships.entity_curriculum_rels as ent_cur_rels

# Default values for nodes
default_topic_values = {
    'topic_assessment_type': 'Null',
    'topic_type': 'Null',
    'total_number_of_lessons_for_topic': '1',
    'topic_title': 'Null'
}

default_topic_lesson_values = {
    'topic_lesson_title': 'Null',
    'topic_lesson_type': 'Null',
    'topic_lesson_length': '1',
    'topic_lesson_suggested_activities': 'Null',
    'topic_lesson_skills_learned': 'Null',
    'topic_lesson_weblinks': 'Null',
}

default_learning_statement_values = {
    'lesson_learning_statement': 'Null',
    'lesson_learning_statement_type': 'Student learning outcome'
}

# Helper function to sort year groups numerically where possible
def sort_year_groups(df):
    df = df.copy()
    df['YearGroupNumeric'] = pd.to_numeric(df['YearGroup'], errors='coerce')
    return df.sort_values(by='YearGroupNumeric')

def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_node: school_nodes.SchoolNode):
    
    logger.info(f"Initialising neo4j connection...")
    neon.init_neontology_connection()
    
    keystagesyllabus_df = dataframes['keystagesyllabuses']
    yeargroupsyllabus_df = dataframes['yeargroupsyllabuses']
    topic_df = dataframes['topics']
    lesson_df = dataframes['lessons']
    statement_df = dataframes['statements']
    # resource_df = dataframes['resources'] # TODO
    
    node_library = {}
    node_library['key_stage_nodes'] = {}
    node_library['year_group_nodes'] = {}
    node_library['key_stage_syllabus_nodes'] = {}
    node_library['year_group_syllabus_nodes'] = {}
    node_library['topic_nodes'] = {}
    node_library['topic_lesson_nodes'] = {}
    node_library['statement_nodes'] = {}
    node_library['department_nodes'] = {}
    node_library['subject_nodes'] = {}
    curriculum_node = None
    pastoral_node = None
    key_stage_nodes_created = {}
    year_group_nodes_created = {}
    last_year_group_node = None
    last_key_stage_node = None
    
    # Create Department Structure node
    department_structure_node_unique_id = f"DepartmentStructure_{school_node.unique_id}"
    department_structure_node = school_structures.DepartmentStructureNode(
        unique_id=department_structure_node_unique_id,
        tldraw_snapshot=""
    )
    # Create in school database only
    neon.create_or_merge_neontology_node(department_structure_node, database=db_name, operation='merge')
    node_library['department_structure_node'] = department_structure_node
    
    # Link Department Structure to School
    neon.create_or_merge_neontology_relationship(
        ent_rels.SchoolHasDepartmentStructure(source=school_node, target=department_structure_node),
        database=db_name, operation='merge'
    )
    logger.info(f"Created department structure node and linked to school")
    
    curriculum_structure_node_unique_id = f"CurriculumStructure_{school_node.unique_id}"
    curriculum_node = school_structures.CurriculumStructureNode(
        unique_id=curriculum_structure_node_unique_id,
        tldraw_snapshot=""
    )
    # Create in school database only
    neon.create_or_merge_neontology_node(curriculum_node, database=db_name, operation='merge')
    node_library['curriculum_node'] = curriculum_node
    
    # Create relationship in school database only
    neon.create_or_merge_neontology_relationship(
        ent_cur_rels.SchoolHasCurriculumStructure(source=school_node, target=curriculum_node),
        database=db_name, operation='merge'
    )
    logger.info(f"Created curriculum node and relationship with school")
    
    pastoral_structure_node_unique_id = f"PastoralStructure_{school_node.unique_id}"
    pastoral_node = school_structures.PastoralStructureNode(
        unique_id=pastoral_structure_node_unique_id,
        tldraw_snapshot=""
    )
    neon.create_or_merge_neontology_node(pastoral_node, database=db_name, operation='merge')
    node_library['pastoral_node'] = pastoral_node
    neon.create_or_merge_neontology_relationship(
        ent_cur_rels.SchoolHasPastoralStructure(source=school_node, target=pastoral_node),
        database=db_name, operation='merge'
    )
    logger.info(f"Created pastoral node and relationship with school")
    
    # Create departments and subjects
    # First get unique departments
    unique_departments = keystagesyllabus_df['Department'].dropna().unique()
    
    for department_name in unique_departments:
        department_unique_id = f"Department_{school_node.unique_id}_{department_name.replace(' ', '_')}"
        department_node = school_nodes.DepartmentNode(
            unique_id=department_unique_id,
            name=department_name,
            tldraw_snapshot=""
        )
        # Create department in school database only
        neon.create_or_merge_neontology_node(department_node, database=db_name, operation='merge')
        node_library['department_nodes'][department_name] = department_node
        
        # Link department to department structure in school database
        neon.create_or_merge_neontology_relationship(
            ent_rels.DepartmentStructureHasDepartment(source=department_structure_node, target=department_node),
            database=db_name, operation='merge'
        )
        logger.info(f"Created department node for {department_name} and linked to department structure")
    
    # Create subjects and link to departments
    # First get unique subjects from key stage syllabuses (which have department info)
    unique_subjects = keystagesyllabus_df[['Subject', 'SubjectCode', 'Department']].drop_duplicates()
    
    # Then add any additional subjects from year group syllabuses (without department info)
    additional_subjects = yeargroupsyllabus_df[['Subject', 'SubjectCode']].drop_duplicates()
    additional_subjects = additional_subjects[~additional_subjects['SubjectCode'].isin(unique_subjects['SubjectCode'])]
    
    # Process subjects from key stage syllabuses first (these have department info)
    for _, subject_row in unique_subjects.iterrows():
        subject_unique_id = f"Subject_{school_node.unique_id}_{subject_row['SubjectCode']}"
        department_node = node_library['department_nodes'].get(subject_row['Department'])
        if not department_node:
            logger.warning(f"No department found for subject {subject_row['Subject']} with code {subject_row['SubjectCode']}")
            continue
        
        subject_node = curriculum_nodes.SubjectNode(
            unique_id=subject_unique_id,
            id=subject_row['SubjectCode'],
            name=subject_row['Subject'],
            tldraw_snapshot=""
        )
        # Create subject in both databases
        neon.create_or_merge_neontology_node(subject_node, database=db_name, operation='merge')
        neon.create_or_merge_neontology_node(subject_node, database=curriculum_db_name, operation='merge')
        node_library['subject_nodes'][subject_row['Subject']] = subject_node
        
        # Link subject to department in school database only
        neon.create_or_merge_neontology_relationship(
            ent_rels.DepartmentManagesSubject(source=department_node, target=subject_node),
            database=db_name, operation='merge'
        )
        logger.info(f"Created subject node for {subject_row['Subject']} and linked to department {subject_row['Department']}")
    
    # Process any additional subjects from year group syllabuses (these won't have department info)
    for _, subject_row in additional_subjects.iterrows():
        subject_unique_id = f"Subject_{school_node.unique_id}_{subject_row['SubjectCode']}"
        # Create in a special "Unassigned" department
        unassigned_dept_name = "Unassigned Department"
        if unassigned_dept_name not in node_library['department_nodes']:
            department_node = school_nodes.DepartmentNode(
                unique_id=f"Department_{school_node.unique_id}_Unassigned",
                name=unassigned_dept_name,
                tldraw_snapshot=""
            )
            neon.create_or_merge_neontology_node(department_node, database=db_name, operation='merge')
            node_library['department_nodes'][unassigned_dept_name] = department_node
            
            # Link unassigned department to department structure
            neon.create_or_merge_neontology_relationship(
                ent_rels.DepartmentStructureHasDepartment(source=department_structure_node, target=department_node),
                database=db_name, operation='merge'
            )
            logger.info(f"Created unassigned department node and linked to department structure")
        
        subject_node = curriculum_nodes.SubjectNode(
            unique_id=subject_unique_id,
            id=subject_row['SubjectCode'],
            name=subject_row['Subject'],
            tldraw_snapshot=""
        )
        # Create subject in both databases
        neon.create_or_merge_neontology_node(subject_node, database=db_name, operation='merge')
        neon.create_or_merge_neontology_node(subject_node, database=curriculum_db_name, operation='merge')
        node_library['subject_nodes'][subject_row['Subject']] = subject_node
        
        # Link subject to unassigned department in school database only
        neon.create_or_merge_neontology_relationship(
            ent_rels.DepartmentManagesSubject(
                source=node_library['department_nodes'][unassigned_dept_name], 
                target=subject_node
            ),
            database=db_name, operation='merge'
        )
        logger.warning(f"Created subject node for {subject_row['Subject']} in unassigned department")
    
    # Process key stages and syllabuses
    logger.info(f"Processing key stages")
    last_key_stage_node = None
    # Track last syllabus nodes per subject
    last_key_stage_syllabus_nodes = {}  # Dictionary to track last key stage syllabus node per subject
    last_year_group_syllabus_nodes = {}  # Dictionary to track last year group syllabus node per subject
    topics_processed = set()  # Track which topics have been processed
    lessons_processed = set()  # Track which lessons have been processed
    statements_processed = set()  # Track which statements have been processed
    
    # First create all key stage nodes and key stage syllabus nodes
    for index, ks_row in keystagesyllabus_df.sort_values('KeyStage').iterrows():
        key_stage = str(ks_row['KeyStage'])
        logger.debug(f"Processing key stage syllabus row - Subject: {ks_row['Subject']}, Key Stage: {key_stage}")
        
        subject_node = node_library['subject_nodes'].get(ks_row['Subject'])
        if not subject_node:
            logger.warning(f"No subject node found for subject {ks_row['Subject']}")
            continue
            
        if key_stage not in key_stage_nodes_created:
            key_stage_node_unique_id = f"KeyStage_{curriculum_node.unique_id}_KStg{key_stage}"
            key_stage_node = curriculum_nodes.KeyStageNode(
                unique_id=key_stage_node_unique_id,
                name=f"Key Stage {key_stage}",
                key_stage=str(key_stage),
                tldraw_snapshot=""
            )
            # Create key stage node in both databases
            neon.create_or_merge_neontology_node(key_stage_node, database=db_name, operation='merge')
            neon.create_or_merge_neontology_node(key_stage_node, database=curriculum_db_name, operation='merge')
            key_stage_nodes_created[key_stage] = key_stage_node
            node_library['key_stage_nodes'][key_stage] = key_stage_node
            
            # Create relationship with curriculum structure in school database only
            neon.create_or_merge_neontology_relationship(
                curriculum_relationships.CurriculumStructureIncludesKeyStage(source=curriculum_node, target=key_stage_node),
                database=db_name, operation='merge'
            )
            logger.info(f"Created key stage node {key_stage_node_unique_id} and relationship with curriculum structure")

            # Create sequential relationship between key stages in both databases
            if last_key_stage_node:
                neon.create_or_merge_neontology_relationship(
                    curriculum_relationships.KeyStageFollowsKeyStage(source=last_key_stage_node, target=key_stage_node),
                    database=db_name, operation='merge'
                )
                neon.create_or_merge_neontology_relationship(
                    curriculum_relationships.KeyStageFollowsKeyStage(source=last_key_stage_node, target=key_stage_node),
                    database=curriculum_db_name, operation='merge'
                )
                logger.info(f"Created sequential relationship between key stages {last_key_stage_node.unique_id} and {key_stage_node.unique_id}")
            last_key_stage_node = key_stage_node

        # Create key stage syllabus under the subject's curriculum directory
        key_stage_syllabus_node_unique_id = f"KeyStageSyllabus_{curriculum_node.unique_id}_{ks_row['Title'].replace(' ', '')}"
        logger.debug(f"Creating key stage syllabus node for {ks_row['Subject']} KS{key_stage} with ID {ks_row['ID']}")
        
        key_stage_syllabus_node_unique_id = f"KeyStageSyllabus_{curriculum_node.unique_id}_{ks_row['Title'].replace(' ', '')}"
        key_stage_syllabus_node = curriculum_nodes.KeyStageSyllabusNode(
            unique_id=key_stage_syllabus_node_unique_id,
            id=ks_row['ID'],
            name=ks_row['Title'],
            key_stage=str(ks_row['KeyStage']),
            subject_name=ks_row['Subject'],
            tldraw_snapshot=""
        )
        # Create key stage syllabus node in both databases
        neon.create_or_merge_neontology_node(key_stage_syllabus_node, database=db_name, operation='merge')
        neon.create_or_merge_neontology_node(key_stage_syllabus_node, database=curriculum_db_name, operation='merge')
        node_library['key_stage_syllabus_nodes'][ks_row['ID']] = key_stage_syllabus_node
        logger.debug(f"Created key stage syllabus node {key_stage_syllabus_node_unique_id} for {ks_row['Subject']} KS{key_stage}")
        
        # Link key stage syllabus to its subject in both databases
        if subject_node:
            neon.create_or_merge_neontology_relationship(
                curriculum_relationships.SubjectHasKeyStageSyllabus(source=subject_node, target=key_stage_syllabus_node),
                database=db_name, operation='merge'
            )
            neon.create_or_merge_neontology_relationship(
                curriculum_relationships.SubjectHasKeyStageSyllabus(source=subject_node, target=key_stage_syllabus_node),
                database=curriculum_db_name, operation='merge'
            )
            logger.info(f"Created relationship between subject {subject_node.unique_id} and key stage syllabus {key_stage_syllabus_node.unique_id}")
        
        # Link key stage syllabus to its key stage in both databases
        key_stage_node = key_stage_nodes_created.get(key_stage)
        if key_stage_node:
            neon.create_or_merge_neontology_relationship(
                curriculum_relationships.KeyStageIncludesKeyStageSyllabus(source=key_stage_node, target=key_stage_syllabus_node),
                database=db_name, operation='merge'
            )
            neon.create_or_merge_neontology_relationship(
                curriculum_relationships.KeyStageIncludesKeyStageSyllabus(source=key_stage_node, target=key_stage_syllabus_node),
                database=curriculum_db_name, operation='merge'
            )
            logger.info(f"Created relationship between key stage {key_stage_node.unique_id} and key stage syllabus {key_stage_syllabus_node.unique_id}")
        
        # Create sequential relationship between key stage syllabuses in both databases
        last_key_stage_syllabus_node = last_key_stage_syllabus_nodes.get(ks_row['Subject'])
        if last_key_stage_syllabus_node:
            neon.create_or_merge_neontology_relationship(
                curriculum_relationships.KeyStageSyllabusFollowsKeyStageSyllabus(source=last_key_stage_syllabus_node, target=key_stage_syllabus_node),
                database=db_name, operation='merge'
            )
            neon.create_or_merge_neontology_relationship(
                curriculum_relationships.KeyStageSyllabusFollowsKeyStageSyllabus(source=last_key_stage_syllabus_node, target=key_stage_syllabus_node),
                database=curriculum_db_name, operation='merge'
            )
            logger.info(f"Created sequential relationship between key stage syllabuses {last_key_stage_syllabus_node.unique_id} and {key_stage_syllabus_node.unique_id}")
        last_key_stage_syllabus_nodes[ks_row['Subject']] = key_stage_syllabus_node
    
    # Now process year groups and their syllabuses
    for index, ks_row in keystagesyllabus_df.sort_values('KeyStage').iterrows():
        key_stage = str(ks_row['KeyStage'])
        related_yeargroups = sort_year_groups(yeargroupsyllabus_df[yeargroupsyllabus_df['KeyStage'] == ks_row['KeyStage']])
        
        logger.info(f"Processing year groups for key stage {key_stage}")
        for yg_index, yg_row in related_yeargroups.iterrows():
            year_group = yg_row['YearGroup']
            subject_code = yg_row['SubjectCode']
            numeric_year_group = pd.to_numeric(year_group, errors='coerce')

            if pd.notna(numeric_year_group):
                numeric_year_group = int(numeric_year_group)
                if numeric_year_group not in year_group_nodes_created:
                    year_group_node_unique_id = f"YearGroup_{school_node.unique_id}_YGrp{numeric_year_group}"
                    year_group_node = pastoral_nodes.YearGroupNode(
                        unique_id=year_group_node_unique_id,
                        year_group=str(numeric_year_group),
                        name=f"Year {numeric_year_group}, {year_group}",
                        tldraw_snapshot=""
                    )
                    # Create year group node in both databases but use same directory
                    neon.create_or_merge_neontology_node(year_group_node, database=db_name, operation='merge')
                    neon.create_or_merge_neontology_node(year_group_node, database=curriculum_db_name, operation='merge')
                    
                    # Create sequential relationship between year groups in both databases
                    if last_year_group_node:
                        neon.create_or_merge_neontology_relationship(
                            curriculum_relationships.YearGroupFollowsYearGroup(source=last_year_group_node, target=year_group_node),
                            database=db_name, operation='merge'
                        )
                        neon.create_or_merge_neontology_relationship(
                            curriculum_relationships.YearGroupFollowsYearGroup(source=last_year_group_node, target=year_group_node),
                            database=curriculum_db_name, operation='merge'
                        )
                        logger.info(f"Created sequential relationship between year groups {last_year_group_node.unique_id} and {year_group_node.unique_id} across key stages")
                    last_year_group_node = year_group_node
                    
                    # Create relationship with Pastoral Structure in school database only
                    neon.create_or_merge_neontology_relationship(
                        curriculum_relationships.PastoralStructureIncludesYearGroup(source=pastoral_node, target=year_group_node),
                        database=db_name, operation='merge'
                    )
                    logger.info(f"Created year group node {year_group_node_unique_id} and relationship with pastoral structure")
                    
                    year_group_nodes_created[numeric_year_group] = year_group_node
                    node_library['year_group_nodes'][str(numeric_year_group)] = year_group_node

            # Create year group syllabus nodes in both databases
            year_group_node = year_group_nodes_created.get(numeric_year_group)
            if year_group_node:
                year_group_syllabus_node_unique_id = f"YearGroupSyllabus_{school_node.unique_id}_{yg_row['ID']}"
                year_group_syllabus_node = pastoral_nodes.YearGroupSyllabusNode(
                    unique_id=year_group_syllabus_node_unique_id,
                    id=yg_row['ID'],
                    name=yg_row['Title'],
                    year_group=str(yg_row['YearGroup']),
                    subject_name=yg_row['Subject'],
                    tldraw_snapshot=""
                )
                
                # Create year group syllabus node in both databases but use same directory
                neon.create_or_merge_neontology_node(year_group_syllabus_node, database=db_name, operation='merge')
                neon.create_or_merge_neontology_node(year_group_syllabus_node, database=curriculum_db_name, operation='merge')
                node_library['year_group_syllabus_nodes'][yg_row['ID']] = year_group_syllabus_node
                
                # Create sequential relationship between year group syllabuses in both databases
                last_year_group_syllabus_node = last_year_group_syllabus_nodes.get(yg_row['Subject'])
                # Only create sequential relationship if this year group is higher than the last one
                if last_year_group_syllabus_node:
                    last_year = pd.to_numeric(last_year_group_syllabus_node.year_group, errors='coerce')
                    current_year = pd.to_numeric(year_group_syllabus_node.year_group, errors='coerce')
                    if pd.notna(last_year) and pd.notna(current_year) and current_year > last_year:
                        neon.create_or_merge_neontology_relationship(
                            curriculum_relationships.YearGroupSyllabusFollowsYearGroupSyllabus(source=last_year_group_syllabus_node, target=year_group_syllabus_node),
                            database=db_name, operation='merge'
                        )
                        neon.create_or_merge_neontology_relationship(
                            curriculum_relationships.YearGroupSyllabusFollowsYearGroupSyllabus(source=last_year_group_syllabus_node, target=year_group_syllabus_node),
                            database=curriculum_db_name, operation='merge'
                        )
                        logger.info(f"Created sequential relationship between year group syllabuses {last_year_group_syllabus_node.unique_id} and {year_group_syllabus_node.unique_id}")
                last_year_group_syllabus_nodes[yg_row['Subject']] = year_group_syllabus_node
                
                # Create relationships in both databases using MATCH to avoid cartesian products
                subject_node = node_library['subject_nodes'].get(yg_row['Subject'])
                if subject_node:
                    # Link to subject
                    neon.create_or_merge_neontology_relationship(
                        curriculum_relationships.SubjectHasYearGroupSyllabus(source=subject_node, target=year_group_syllabus_node),
                        database=db_name, operation='merge'
                    )
                    neon.create_or_merge_neontology_relationship(
                        curriculum_relationships.SubjectHasYearGroupSyllabus(source=subject_node, target=year_group_syllabus_node),
                        database=curriculum_db_name, operation='merge'
                    )
                    logger.info(f"Created relationship between subject {subject_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
                
                # Link to year group
                neon.create_or_merge_neontology_relationship(
                    curriculum_relationships.YearGroupHasYearGroupSyllabus(source=year_group_node, target=year_group_syllabus_node),
                    database=db_name, operation='merge'
                )
                neon.create_or_merge_neontology_relationship(
                    curriculum_relationships.YearGroupHasYearGroupSyllabus(source=year_group_node, target=year_group_syllabus_node),
                    database=curriculum_db_name, operation='merge'
                )
                logger.info(f"Created relationship between year group {year_group_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
                
                # Link to key stage syllabus if it exists for the same subject
                key_stage_syllabus_node = node_library['key_stage_syllabus_nodes'].get(ks_row['ID'])
                if key_stage_syllabus_node and yg_row['Subject'] == ks_row['Subject']:
                    neon.create_or_merge_neontology_relationship(
                        curriculum_relationships.KeyStageSyllabusIncludesYearGroupSyllabus(source=key_stage_syllabus_node, target=year_group_syllabus_node),
                        database=db_name, operation='merge'
                    )
                    neon.create_or_merge_neontology_relationship(
                        curriculum_relationships.KeyStageSyllabusIncludesYearGroupSyllabus(source=key_stage_syllabus_node, target=year_group_syllabus_node),
                        database=curriculum_db_name, operation='merge'
                    )
                    logger.info(f"Created relationship between key stage syllabus {key_stage_syllabus_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")

                # Process topics for this year group syllabus only if not already processed
                topics_for_syllabus = topic_df[topic_df['SyllabusYearID'] == yg_row['ID']]
                for _, topic_row in topics_for_syllabus.iterrows():
                    if topic_row['TopicID'] in topics_processed:
                        continue
                    topics_processed.add(topic_row['TopicID'])
                    
                    # Get the correct subject from the topic row
                    topic_subject = topic_row['SyllabusSubject']
                    topic_key_stage = topic_row['SyllabusKeyStage']
                    
                    logger.debug(f"Processing topic {topic_row['TopicID']} for subject {topic_subject} and key stage {topic_key_stage}")
                    logger.debug(f"Available key stage syllabus nodes: {[node.subject_name + '_KS' + node.key_stage for node in node_library['key_stage_syllabus_nodes'].values()]}")
                    
                    # Find the key stage syllabus node by iterating through all nodes
                    matching_syllabus_node = None
                    for syllabus_node in node_library['key_stage_syllabus_nodes'].values():
                        logger.debug(f"Checking syllabus node - Subject: {syllabus_node.subject_name}, Key Stage: {syllabus_node.key_stage}")
                        logger.debug(f"Comparing with - Subject: {topic_subject}, Key Stage: {str(topic_key_stage)}")
                        logger.debug(f"Types - Node Subject: {type(syllabus_node.subject_name)}, Topic Subject: {type(topic_subject)}")
                        logger.debug(f"Types - Node Key Stage: {type(syllabus_node.key_stage)}, Topic Key Stage: {type(str(topic_key_stage))}")
                        
                        if (syllabus_node.subject_name == topic_subject and 
                            syllabus_node.key_stage == str(topic_key_stage)):
                            matching_syllabus_node = syllabus_node
                            logger.debug(f"Found matching syllabus node: {syllabus_node.unique_id}")
                            break
                    
                    if not matching_syllabus_node:
                        logger.warning(f"No key stage syllabus node found for subject {topic_subject} and key stage {topic_key_stage}, skipping topic creation")
                        continue
                    
                    topic_node_unique_id = f"Topic_{matching_syllabus_node.unique_id}_{topic_row['TopicID']}"
                    topic_node = curriculum_nodes.TopicNode(
                        unique_id=topic_node_unique_id,
                        id=topic_row['TopicID'],
                        name=topic_row.get('TopicTitle', default_topic_values['topic_title']),
                        total_number_of_lessons_for_topic=str(topic_row.get('TotalNumberOfLessonsForTopic', default_topic_values['total_number_of_lessons_for_topic'])),
                        type=topic_row.get('TopicType', default_topic_values['topic_type']),
                        assessment_type=topic_row.get('TopicAssessmentType', default_topic_values['topic_assessment_type']),
                        tldraw_snapshot=""
                    )
                    # Create topic node in curriculum database only
                    neon.create_or_merge_neontology_node(topic_node, database=curriculum_db_name, operation='merge')
                    node_library['topic_nodes'][topic_row['TopicID']] = topic_node
                    
                    # Link topic to key stage syllabus as well as year group syllabus
                    neon.create_or_merge_neontology_relationship(
                        curriculum_relationships.KeyStageSyllabusIncludesTopic(source=matching_syllabus_node, target=topic_node),
                        database=curriculum_db_name, operation='merge'
                    )
                    neon.create_or_merge_neontology_relationship(
                        curriculum_relationships.YearGroupSyllabusIncludesTopic(source=year_group_syllabus_node, target=topic_node),
                        database=curriculum_db_name, operation='merge'
                    )
                    logger.info(f"Created relationships between topic {topic_node_unique_id} and key stage syllabus {matching_syllabus_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")

                    # Process lessons for this topic only if not already processed
                    lessons_for_topic = lesson_df[
                        (lesson_df['TopicID'] == topic_row['TopicID']) & 
                        (lesson_df['SyllabusSubject'] == topic_subject)
                    ].copy()
                    lessons_for_topic.loc[:, 'Lesson'] = lessons_for_topic['Lesson'].astype(str)
                    lessons_for_topic = lessons_for_topic.sort_values('Lesson')

                    previous_lesson_node = None
                    for _, lesson_row in lessons_for_topic.iterrows():
                        if lesson_row['LessonID'] in lessons_processed:
                            continue
                        lessons_processed.add(lesson_row['LessonID'])
                        
                        lesson_node = curriculum_nodes.TopicLessonNode(
                            unique_id=f"TopicLesson_{topic_node_unique_id}_{lesson_row['LessonID']}",
                            id=lesson_row['LessonID'],
                            name=lesson_row.get('LessonTitle', default_topic_lesson_values['topic_lesson_title']),
                            type=lesson_row.get('LessonType', default_topic_lesson_values['topic_lesson_type']),
                            length=str(lesson_row.get('SuggestedNumberOfPeriodsForLesson', default_topic_lesson_values['topic_lesson_length'])),
                            suggested_activities=str(lesson_row.get('SuggestedActivities', default_topic_lesson_values['topic_lesson_suggested_activities'])),
                            skills_learned=str(lesson_row.get('SkillsLearned', default_topic_lesson_values['topic_lesson_skills_learned'])),
                            weblinks=str(lesson_row.get('WebLinks', default_topic_lesson_values['topic_lesson_weblinks'])),
                            tldraw_snapshot=""
                        )
                        # Create lesson node in curriculum database only
                        neon.create_or_merge_neontology_node(lesson_node, database=curriculum_db_name, operation='merge')
                        node_library['topic_lesson_nodes'][lesson_row['LessonID']] = lesson_node
                        
                        # Link lesson to topic
                        neon.create_or_merge_neontology_relationship(
                            curriculum_relationships.TopicIncludesTopicLesson(source=topic_node, target=lesson_node),
                            database=curriculum_db_name, operation='merge'
                        )
                        logger.info(f"Created lesson node {lesson_node.unique_id} and relationship with topic {topic_node.unique_id}")

                        # Create sequential relationships between lessons
                        if lesson_row['Lesson'].isdigit() and previous_lesson_node:
                            neon.create_or_merge_neontology_relationship(
                                curriculum_relationships.TopicLessonFollowsTopicLesson(source=previous_lesson_node, target=lesson_node),
                                database=curriculum_db_name, operation='merge'
                            )
                            logger.info(f"Created sequential relationship between lessons {previous_lesson_node.unique_id} and {lesson_node.unique_id}")
                        previous_lesson_node = lesson_node

                        # Process learning statements for this lesson only if not already processed
                        statements_for_lesson = statement_df[
                            (statement_df['LessonID'] == lesson_row['LessonID']) & 
                            (statement_df['SyllabusSubject'] == topic_subject)
                        ]
                        for _, statement_row in statements_for_lesson.iterrows():
                            if statement_row['StatementID'] in statements_processed:
                                continue
                            statements_processed.add(statement_row['StatementID'])
                            statement_node = curriculum_nodes.LearningStatementNode(
                                unique_id=f"LearningStatement_{lesson_node.unique_id}_{statement_row['StatementID']}",
                                id=statement_row['StatementID'],
                                name=statement_row.get('LearningStatement', default_learning_statement_values['lesson_learning_statement']),
                                type=statement_row.get('StatementType', default_learning_statement_values['lesson_learning_statement_type']),
                                tldraw_snapshot=""
                            )
                            # Create statement node in curriculum database only
                            neon.create_or_merge_neontology_node(statement_node, database=curriculum_db_name, operation='merge')
                            node_library['statement_nodes'][statement_row['StatementID']] = statement_node
                            
                            # Link learning statement to lesson
                            neon.create_or_merge_neontology_relationship(
                                curriculum_relationships.LessonIncludesLearningStatement(source=lesson_node, target=statement_node),
                                database=curriculum_db_name, operation='merge'
                            )
                            logger.info(f"Created learning statement node {statement_node.unique_id} and relationship with lesson {lesson_node.unique_id}")
            else:
                logger.warning(f"No year group node found for year group {year_group}, skipping syllabus creation")
    
    # After processing all year groups and their syllabuses, process any remaining topics
    logger.info("Processing topics without year groups")
    for _, topic_row in topic_df.iterrows():
        if topic_row['TopicID'] in topics_processed:
            continue
            
        topic_subject = topic_row['SyllabusSubject']
        topic_key_stage = topic_row['SyllabusKeyStage']
        
        logger.debug(f"Processing topic {topic_row['TopicID']} for subject {topic_subject} and key stage {topic_key_stage} without year group")
        
        # Find the key stage syllabus node
        matching_syllabus_node = None
        for syllabus_node in node_library['key_stage_syllabus_nodes'].values():
            if (syllabus_node.subject_name == topic_subject and 
                syllabus_node.key_stage == str(topic_key_stage)):
                matching_syllabus_node = syllabus_node
                break
        
        if not matching_syllabus_node:
            logger.warning(f"No key stage syllabus node found for subject {topic_subject} and key stage {topic_key_stage}, skipping topic creation")
            continue
        
        topic_node_unique_id = f"Topic_{matching_syllabus_node.unique_id}_{topic_row['TopicID']}"
        topic_node = curriculum_nodes.TopicNode(
            unique_id=topic_node_unique_id,
            id=topic_row['TopicID'],
            name=topic_row.get('TopicTitle', default_topic_values['topic_title']),
            total_number_of_lessons_for_topic=str(topic_row.get('TotalNumberOfLessonsForTopic', default_topic_values['total_number_of_lessons_for_topic'])),
            type=topic_row.get('TopicType', default_topic_values['topic_type']),
            assessment_type=topic_row.get('TopicAssessmentType', default_topic_values['topic_assessment_type']),
            tldraw_snapshot=""
        )
        # Create topic node in curriculum database only
        neon.create_or_merge_neontology_node(topic_node, database=curriculum_db_name, operation='merge')
        node_library['topic_nodes'][topic_row['TopicID']] = topic_node
        topics_processed.add(topic_row['TopicID'])
        
        # Link topic to key stage syllabus
        neon.create_or_merge_neontology_relationship(
            curriculum_relationships.KeyStageSyllabusIncludesTopic(source=matching_syllabus_node, target=topic_node),
            database=curriculum_db_name, operation='merge'
        )
        logger.info(f"Created relationship between topic {topic_node_unique_id} and key stage syllabus {matching_syllabus_node.unique_id}")
        
        # Process lessons for this topic
        lessons_for_topic = lesson_df[
            (lesson_df['TopicID'] == topic_row['TopicID']) & 
            (lesson_df['SyllabusSubject'] == topic_subject)
        ].copy()
        lessons_for_topic.loc[:, 'Lesson'] = lessons_for_topic['Lesson'].astype(str)
        lessons_for_topic = lessons_for_topic.sort_values('Lesson')
        
        previous_lesson_node = None
        for _, lesson_row in lessons_for_topic.iterrows():
            if lesson_row['LessonID'] in lessons_processed:
                continue
            lessons_processed.add(lesson_row['LessonID'])
            lesson_node = curriculum_nodes.TopicLessonNode(
                unique_id=f"TopicLesson_{topic_node_unique_id}_{lesson_row['LessonID']}",
                id=lesson_row['LessonID'],
                name=lesson_row.get('LessonTitle', default_topic_lesson_values['topic_lesson_title']),
                type=lesson_row.get('LessonType', default_topic_lesson_values['topic_lesson_type']),
                length=str(lesson_row.get('SuggestedNumberOfPeriodsForLesson', default_topic_lesson_values['topic_lesson_length'])),
                suggested_activities=str(lesson_row.get('SuggestedActivities', default_topic_lesson_values['topic_lesson_suggested_activities'])),
                skills_learned=str(lesson_row.get('SkillsLearned', default_topic_lesson_values['topic_lesson_skills_learned'])),
                weblinks=str(lesson_row.get('WebLinks', default_topic_lesson_values['topic_lesson_weblinks'])),
                tldraw_snapshot=""
            )
            # Create lesson node in curriculum database only
            neon.create_or_merge_neontology_node(lesson_node, database=curriculum_db_name, operation='merge')
            node_library['topic_lesson_nodes'][lesson_row['LessonID']] = lesson_node
            
            # Link lesson to topic
            neon.create_or_merge_neontology_relationship(
                curriculum_relationships.TopicIncludesTopicLesson(source=topic_node, target=lesson_node),
                database=curriculum_db_name, operation='merge'
            )
            logger.info(f"Created lesson node {lesson_node.unique_id} and relationship with topic {topic_node.unique_id}")
            
            # Create sequential relationships between lessons
            if lesson_row['Lesson'].isdigit() and previous_lesson_node:
                neon.create_or_merge_neontology_relationship(
                    curriculum_relationships.TopicLessonFollowsTopicLesson(source=previous_lesson_node, target=lesson_node),
                    database=curriculum_db_name, operation='merge'
                )
                logger.info(f"Created sequential relationship between lessons {previous_lesson_node.unique_id} and {lesson_node.unique_id}")
            previous_lesson_node = lesson_node
            
            # Process learning statements for this lesson
            statements_for_lesson = statement_df[
                (statement_df['LessonID'] == lesson_row['LessonID']) & 
                (statement_df['SyllabusSubject'] == topic_subject)
            ]
            for _, statement_row in statements_for_lesson.iterrows():
                if statement_row['StatementID'] in statements_processed:
                    continue
                statements_processed.add(statement_row['StatementID'])
                statement_node = curriculum_nodes.LearningStatementNode(
                    unique_id=f"LearningStatement_{lesson_node.unique_id}_{statement_row['StatementID']}",
                    id=statement_row['StatementID'],
                    name=statement_row.get('LearningStatement', default_learning_statement_values['lesson_learning_statement']),
                    type=statement_row.get('StatementType', default_learning_statement_values['lesson_learning_statement_type']),
                    tldraw_snapshot=""
                )
                # Create statement node in curriculum database only
                neon.create_or_merge_neontology_node(statement_node, database=curriculum_db_name, operation='merge')
                node_library['statement_nodes'][statement_row['StatementID']] = statement_node
                
                # Link learning statement to lesson
                neon.create_or_merge_neontology_relationship(
                    curriculum_relationships.LessonIncludesLearningStatement(source=lesson_node, target=statement_node),
                    database=curriculum_db_name, operation='merge'
                )
                logger.info(f"Created learning statement node {statement_node.unique_id} and relationship with lesson {lesson_node.unique_id}")
    
    return node_library