latest
This commit is contained in:
@@ -18,13 +18,14 @@ class SupabaseBearer(HTTPBearer):
|
||||
|
||||
try:
|
||||
token = credentials.credentials
|
||||
payload = verify_supabase_token(token)
|
||||
# Decode using the string-based verifier to avoid async dependency conflicts
|
||||
payload = verify_supabase_jwt_str(token)
|
||||
return payload
|
||||
except Exception as e:
|
||||
logger.error(f"Token verification failed: {str(e)}")
|
||||
raise HTTPException(status_code=403, detail="Invalid token or expired token.")
|
||||
|
||||
def verify_supabase_token(token: str) -> dict:
|
||||
def verify_supabase_jwt_str(token: str) -> dict:
|
||||
"""Verify a Supabase JWT token and return its payload."""
|
||||
try:
|
||||
jwt_secret = os.getenv("JWT_SECRET")
|
||||
@@ -63,7 +64,7 @@ def decodeSupabaseJWT(token: str) -> dict:
|
||||
# Initialize the security instance
|
||||
security = HTTPBearer()
|
||||
|
||||
async def verify_supabase_token(credentials: HTTPAuthorizationCredentials = Depends(security)):
|
||||
async def verify_supabase_token_dep(credentials: HTTPAuthorizationCredentials = Depends(security)):
|
||||
try:
|
||||
token = credentials.credentials
|
||||
# Verify token using your Supabase JWT secret
|
||||
|
||||
@@ -0,0 +1,571 @@
|
||||
"""
|
||||
Uniform Bundle Metadata Architecture
|
||||
|
||||
This module defines standardized metadata structures for all document processing pipelines
|
||||
to ensure consistency and interoperability across OCR, No-OCR, and VLM bundles.
|
||||
|
||||
Features:
|
||||
- Uniform metadata schema for all pipeline types
|
||||
- Consistent grouping and ordering mechanisms
|
||||
- Pipeline-agnostic bundle identification
|
||||
- Enhanced metadata for frontend display
|
||||
- Backward compatibility with existing bundles
|
||||
"""
|
||||
|
||||
import uuid
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, Optional, List, Union, Literal
|
||||
from dataclasses import dataclass, asdict
|
||||
from enum import Enum
|
||||
|
||||
class PipelineType(Enum):
|
||||
"""Supported pipeline types"""
|
||||
STANDARD = "standard"
|
||||
VLM = "vlm"
|
||||
ASR = "asr"
|
||||
|
||||
class ProcessingMode(Enum):
|
||||
"""How content was processed"""
|
||||
WHOLE_DOCUMENT = "whole_document"
|
||||
SPLIT_SECTIONS = "split_sections"
|
||||
INDIVIDUAL_PAGES = "individual_pages"
|
||||
PAGE_BUNDLE = "page_bundle"
|
||||
|
||||
class BundleType(Enum):
|
||||
"""Type of bundle created"""
|
||||
SINGLE_ARTEFACT = "single_artefact"
|
||||
SPLIT_PACK = "split_pack"
|
||||
PAGE_BUNDLE = "page_bundle"
|
||||
VLM_SECTION_BUNDLE = "vlm_section_bundle"
|
||||
# New unified bundle types
|
||||
DOCLING_BUNDLE = "docling_bundle" # Single coherent processing unit
|
||||
DOCLING_BUNDLE_SPLIT = "docling_bundle_split" # Container for multi-unit processing
|
||||
|
||||
@dataclass
|
||||
class BundleMetadata:
|
||||
"""
|
||||
Standardized metadata structure for all document processing bundles.
|
||||
|
||||
This ensures consistency across OCR, No-OCR, and VLM pipelines while
|
||||
maintaining backward compatibility.
|
||||
"""
|
||||
|
||||
# Core identification
|
||||
bundle_id: str
|
||||
file_id: str
|
||||
pipeline: PipelineType
|
||||
processing_mode: ProcessingMode
|
||||
bundle_type: BundleType
|
||||
|
||||
# Grouping and ordering
|
||||
group_id: Optional[str] = None
|
||||
split_order: Optional[int] = None
|
||||
split_total: Optional[int] = None
|
||||
split_heading: Optional[str] = None
|
||||
|
||||
# Content information
|
||||
page_range: Optional[List[int]] = None # [start_page, end_page]
|
||||
page_count: Optional[int] = None
|
||||
section_title: Optional[str] = None
|
||||
section_level: Optional[int] = None
|
||||
|
||||
# Processing details
|
||||
config: Optional[Dict[str, Any]] = None
|
||||
settings_fingerprint: Optional[str] = None
|
||||
processing_time: Optional[float] = None
|
||||
|
||||
# Pipeline-specific metadata
|
||||
pipeline_metadata: Optional[Dict[str, Any]] = None
|
||||
|
||||
# Producer information
|
||||
producer: str = "manual" # manual, auto_phase2, auto_split, etc.
|
||||
created_at: Optional[str] = None
|
||||
|
||||
# Status and quality
|
||||
status: str = "completed"
|
||||
quality_score: Optional[float] = None
|
||||
|
||||
def __post_init__(self):
|
||||
"""Set defaults and compute derived fields"""
|
||||
if self.created_at is None:
|
||||
self.created_at = datetime.utcnow().isoformat()
|
||||
|
||||
if self.bundle_id is None:
|
||||
self.bundle_id = str(uuid.uuid4())
|
||||
|
||||
# Compute settings fingerprint if config provided
|
||||
if self.config and not self.settings_fingerprint:
|
||||
self.settings_fingerprint = self._compute_settings_fingerprint(self.config)
|
||||
|
||||
def _compute_settings_fingerprint(self, config: Dict[str, Any]) -> str:
|
||||
"""Compute a fingerprint for configuration settings (excluding page_range)"""
|
||||
try:
|
||||
config_for_hash = dict(config)
|
||||
config_for_hash.pop('page_range', None) # Exclude page ranges from fingerprint
|
||||
config_str = json.dumps(config_for_hash, sort_keys=True, ensure_ascii=False)
|
||||
return hashlib.sha1(config_str.encode('utf-8')).hexdigest()[:16]
|
||||
except Exception:
|
||||
return str(uuid.uuid4())[:16]
|
||||
|
||||
def to_artefact_extra(self) -> Dict[str, Any]:
|
||||
"""Convert to format suitable for document_artefacts.extra field"""
|
||||
extra = {}
|
||||
|
||||
# Core fields
|
||||
extra['bundle_metadata_version'] = '1.0'
|
||||
extra['pipeline'] = self.pipeline.value
|
||||
extra['processing_mode'] = self.processing_mode.value
|
||||
extra['bundle_type'] = self.bundle_type.value
|
||||
|
||||
# Store original pipeline type for UI differentiation
|
||||
if hasattr(self, 'original_pipeline_type'):
|
||||
extra['original_pipeline'] = self.original_pipeline_type
|
||||
|
||||
# Grouping fields
|
||||
if self.group_id:
|
||||
extra['group_id'] = self.group_id
|
||||
if self.split_order is not None:
|
||||
extra['split_order'] = self.split_order
|
||||
if self.split_total is not None:
|
||||
extra['split_total'] = self.split_total
|
||||
if self.split_heading:
|
||||
extra['split_heading'] = self.split_heading
|
||||
|
||||
# Content fields
|
||||
if self.page_range:
|
||||
extra['page_range'] = self.page_range
|
||||
if self.page_count is not None:
|
||||
extra['page_count'] = self.page_count
|
||||
if self.section_title:
|
||||
extra['section_title'] = self.section_title
|
||||
if self.section_level is not None:
|
||||
extra['section_level'] = self.section_level
|
||||
|
||||
# Processing fields
|
||||
if self.config:
|
||||
extra['config'] = self.config
|
||||
if self.settings_fingerprint:
|
||||
extra['settings_fingerprint'] = self.settings_fingerprint
|
||||
if self.processing_time is not None:
|
||||
extra['processing_time'] = self.processing_time
|
||||
|
||||
# Pipeline-specific metadata
|
||||
if self.pipeline_metadata:
|
||||
extra['pipeline_metadata'] = self.pipeline_metadata
|
||||
|
||||
# Producer and quality
|
||||
extra['producer'] = self.producer
|
||||
if self.quality_score is not None:
|
||||
extra['quality_score'] = self.quality_score
|
||||
|
||||
return extra
|
||||
|
||||
@classmethod
|
||||
def from_artefact_extra(cls, file_id: str, artefact_id: str, extra: Dict[str, Any]) -> 'BundleMetadata':
|
||||
"""Create BundleMetadata from document_artefacts.extra field"""
|
||||
|
||||
# Extract core fields with fallbacks for backward compatibility
|
||||
pipeline_str = extra.get('pipeline', 'standard')
|
||||
try:
|
||||
pipeline = PipelineType(pipeline_str)
|
||||
except ValueError:
|
||||
pipeline = PipelineType.STANDARD
|
||||
|
||||
processing_mode_str = extra.get('processing_mode', 'whole_document')
|
||||
try:
|
||||
processing_mode = ProcessingMode(processing_mode_str)
|
||||
except ValueError:
|
||||
processing_mode = ProcessingMode.WHOLE_DOCUMENT
|
||||
|
||||
bundle_type_str = extra.get('bundle_type', 'single_artefact')
|
||||
try:
|
||||
bundle_type = BundleType(bundle_type_str)
|
||||
except ValueError:
|
||||
bundle_type = BundleType.SINGLE_ARTEFACT
|
||||
|
||||
return cls(
|
||||
bundle_id=artefact_id,
|
||||
file_id=file_id,
|
||||
pipeline=pipeline,
|
||||
processing_mode=processing_mode,
|
||||
bundle_type=bundle_type,
|
||||
group_id=extra.get('group_id'),
|
||||
split_order=extra.get('split_order'),
|
||||
split_total=extra.get('split_total'),
|
||||
split_heading=extra.get('split_heading'),
|
||||
page_range=extra.get('page_range'),
|
||||
page_count=extra.get('page_count'),
|
||||
section_title=extra.get('section_title'),
|
||||
section_level=extra.get('section_level'),
|
||||
config=extra.get('config'),
|
||||
settings_fingerprint=extra.get('settings_fingerprint'),
|
||||
processing_time=extra.get('processing_time'),
|
||||
pipeline_metadata=extra.get('pipeline_metadata'),
|
||||
producer=extra.get('producer', 'manual'),
|
||||
created_at=extra.get('created_at'),
|
||||
quality_score=extra.get('quality_score')
|
||||
)
|
||||
|
||||
class BundleMetadataBuilder:
|
||||
"""Helper class to build standardized bundle metadata"""
|
||||
|
||||
def __init__(self, file_id: str, pipeline: PipelineType):
|
||||
self.file_id = file_id
|
||||
self.pipeline = pipeline
|
||||
self.metadata = BundleMetadata(
|
||||
bundle_id=str(uuid.uuid4()),
|
||||
file_id=file_id,
|
||||
pipeline=pipeline,
|
||||
processing_mode=ProcessingMode.WHOLE_DOCUMENT,
|
||||
bundle_type=BundleType.SINGLE_ARTEFACT
|
||||
)
|
||||
|
||||
def set_processing_mode(self, mode: ProcessingMode) -> 'BundleMetadataBuilder':
|
||||
"""Set processing mode"""
|
||||
self.metadata.processing_mode = mode
|
||||
return self
|
||||
|
||||
def set_bundle_type(self, bundle_type: BundleType) -> 'BundleMetadataBuilder':
|
||||
"""Set bundle type"""
|
||||
self.metadata.bundle_type = bundle_type
|
||||
return self
|
||||
|
||||
def set_group_info(self, group_id: str, split_order: int = None,
|
||||
split_total: int = None, split_heading: str = None) -> 'BundleMetadataBuilder':
|
||||
"""Set grouping information for split documents"""
|
||||
self.metadata.group_id = group_id
|
||||
self.metadata.split_order = split_order
|
||||
self.metadata.split_total = split_total
|
||||
self.metadata.split_heading = split_heading
|
||||
return self
|
||||
|
||||
def set_page_info(self, page_range: List[int] = None,
|
||||
page_count: int = None) -> 'BundleMetadataBuilder':
|
||||
"""Set page information"""
|
||||
self.metadata.page_range = page_range
|
||||
self.metadata.page_count = page_count
|
||||
return self
|
||||
|
||||
def set_section_info(self, title: str = None, level: int = None) -> 'BundleMetadataBuilder':
|
||||
"""Set section information"""
|
||||
self.metadata.section_title = title
|
||||
self.metadata.section_level = level
|
||||
return self
|
||||
|
||||
def set_config(self, config: Dict[str, Any]) -> 'BundleMetadataBuilder':
|
||||
"""Set processing configuration"""
|
||||
self.metadata.config = config
|
||||
self.metadata.settings_fingerprint = self.metadata._compute_settings_fingerprint(config)
|
||||
return self
|
||||
|
||||
def set_producer(self, producer: str) -> 'BundleMetadataBuilder':
|
||||
"""Set producer information"""
|
||||
self.metadata.producer = producer
|
||||
return self
|
||||
|
||||
def set_pipeline_metadata(self, metadata: Dict[str, Any]) -> 'BundleMetadataBuilder':
|
||||
"""Set pipeline-specific metadata"""
|
||||
self.metadata.pipeline_metadata = metadata
|
||||
return self
|
||||
|
||||
def set_quality_score(self, score: float) -> 'BundleMetadataBuilder':
|
||||
"""Set quality score"""
|
||||
self.metadata.quality_score = score
|
||||
return self
|
||||
|
||||
def build(self) -> BundleMetadata:
|
||||
"""Build the final metadata"""
|
||||
return self.metadata
|
||||
|
||||
def create_standard_metadata(
|
||||
file_id: str,
|
||||
pipeline: Literal["ocr", "no_ocr", "vlm"] = "no_ocr",
|
||||
processing_mode: Literal["whole_document", "split_sections", "individual_pages", "pages", "sections", "chunks"] = "split_sections",
|
||||
config: Dict[str, Any] = None,
|
||||
group_id: str = None,
|
||||
split_order: int = None,
|
||||
split_total: int = None,
|
||||
split_heading: str = None,
|
||||
page_range: List[int] = None,
|
||||
producer: str = "auto_phase2"
|
||||
) -> BundleMetadata:
|
||||
"""
|
||||
Convenience function to create standardized metadata for common use cases.
|
||||
"""
|
||||
|
||||
# Map pipeline strings to enums
|
||||
pipeline_map = {
|
||||
"ocr": PipelineType.STANDARD,
|
||||
"no_ocr": PipelineType.STANDARD,
|
||||
"vlm": PipelineType.VLM
|
||||
}
|
||||
|
||||
# Enhanced processing mode mapping with new bundle architecture
|
||||
processing_mode_map = {
|
||||
"whole_document": ProcessingMode.WHOLE_DOCUMENT,
|
||||
"split_sections": ProcessingMode.SPLIT_SECTIONS,
|
||||
"individual_pages": ProcessingMode.INDIVIDUAL_PAGES,
|
||||
"split_by_pages": ProcessingMode.INDIVIDUAL_PAGES, # Split by pages processing
|
||||
"split_by_sections": ProcessingMode.SPLIT_SECTIONS, # Split by sections processing
|
||||
"split_by_chunks": ProcessingMode.SPLIT_SECTIONS, # Split by chunks processing
|
||||
"pages": ProcessingMode.INDIVIDUAL_PAGES, # Alias for page-based processing
|
||||
"sections": ProcessingMode.SPLIT_SECTIONS, # Alias for section-based processing
|
||||
"chunks": ProcessingMode.SPLIT_SECTIONS, # Chunks treated as sections
|
||||
}
|
||||
|
||||
# Determine bundle type based on processing mode and grouping
|
||||
if processing_mode == "whole_document":
|
||||
bundle_type = BundleType.DOCLING_BUNDLE
|
||||
else:
|
||||
bundle_type = BundleType.DOCLING_BUNDLE_SPLIT
|
||||
|
||||
builder = BundleMetadataBuilder(file_id, pipeline_map[pipeline])
|
||||
builder.set_processing_mode(processing_mode_map[processing_mode])
|
||||
builder.set_bundle_type(bundle_type)
|
||||
builder.set_producer(producer)
|
||||
|
||||
# Store original pipeline type for UI differentiation
|
||||
builder.metadata.original_pipeline_type = pipeline
|
||||
|
||||
if config:
|
||||
# Add pipeline-specific config markers
|
||||
enhanced_config = dict(config)
|
||||
if pipeline == "ocr":
|
||||
enhanced_config["do_ocr"] = True
|
||||
elif pipeline == "no_ocr":
|
||||
enhanced_config["do_ocr"] = False
|
||||
elif pipeline == "vlm":
|
||||
enhanced_config["pipeline"] = "vlm"
|
||||
|
||||
builder.set_config(enhanced_config)
|
||||
|
||||
if group_id:
|
||||
builder.set_group_info(group_id, split_order, split_total, split_heading)
|
||||
|
||||
if page_range:
|
||||
builder.set_page_info(page_range)
|
||||
|
||||
# Set section info if we have a heading
|
||||
if split_heading:
|
||||
builder.set_section_info(split_heading)
|
||||
|
||||
return builder.build()
|
||||
|
||||
def create_bundle_split_metadata(
|
||||
file_id: str,
|
||||
pipeline: Literal["ocr", "no_ocr", "vlm"] = "no_ocr",
|
||||
split_mode: Literal["split_by_pages", "split_by_sections", "split_by_chunks"] = "split_by_sections",
|
||||
config: Dict[str, Any] = None,
|
||||
group_id: str = None,
|
||||
producer: str = "auto_phase2",
|
||||
processing_data: Dict[str, Any] = None
|
||||
) -> BundleMetadata:
|
||||
"""
|
||||
Create metadata specifically for split bundle processing.
|
||||
|
||||
This is used for the new docling_bundle_split task type.
|
||||
"""
|
||||
|
||||
# Map split modes to processing modes
|
||||
mode_map = {
|
||||
"split_by_pages": "pages",
|
||||
"split_by_sections": "sections",
|
||||
"split_by_chunks": "sections" # Chunks treated as sections
|
||||
}
|
||||
|
||||
processing_mode = mode_map[split_mode]
|
||||
|
||||
metadata = create_standard_metadata(
|
||||
file_id=file_id,
|
||||
pipeline=pipeline,
|
||||
processing_mode=processing_mode,
|
||||
config=config,
|
||||
group_id=group_id,
|
||||
producer=producer
|
||||
)
|
||||
|
||||
# Add split-specific metadata
|
||||
if processing_data:
|
||||
split_metadata = {
|
||||
'split_mode': split_mode,
|
||||
'processing_data': processing_data
|
||||
}
|
||||
if metadata.pipeline_metadata:
|
||||
metadata.pipeline_metadata.update(split_metadata)
|
||||
else:
|
||||
metadata.pipeline_metadata = split_metadata
|
||||
|
||||
return metadata
|
||||
|
||||
def get_bundle_display_name(metadata: BundleMetadata) -> str:
|
||||
"""Generate a user-friendly display name for a bundle"""
|
||||
|
||||
# Use explicit display name if available
|
||||
if hasattr(metadata, 'display_name') and metadata.display_name:
|
||||
return metadata.display_name
|
||||
|
||||
# Generate based on bundle type and processing mode
|
||||
if metadata.bundle_type == BundleType.DOCLING_BUNDLE:
|
||||
return "Complete Document"
|
||||
|
||||
elif metadata.bundle_type == BundleType.DOCLING_BUNDLE_SPLIT:
|
||||
if metadata.processing_mode == ProcessingMode.INDIVIDUAL_PAGES:
|
||||
if metadata.page_range:
|
||||
return f"Page {metadata.page_range[0]}"
|
||||
return "Page Bundle"
|
||||
elif metadata.processing_mode == ProcessingMode.SPLIT_SECTIONS:
|
||||
if metadata.section_title:
|
||||
order_prefix = f"{metadata.split_order:02d}. " if metadata.split_order else ""
|
||||
page_suffix = ""
|
||||
if metadata.page_range and len(metadata.page_range) >= 2:
|
||||
page_suffix = f" (p{metadata.page_range[0]}-{metadata.page_range[1]})"
|
||||
return f"{order_prefix}{metadata.section_title}{page_suffix}"
|
||||
return f"Section {metadata.split_order or 1}"
|
||||
else:
|
||||
return "Document Bundle"
|
||||
|
||||
# Fallback
|
||||
return metadata.section_title or metadata.split_heading or f"Bundle {metadata.bundle_id[:8]}"
|
||||
|
||||
def create_organized_bundle_manifest(bundles: list, split_mode: str, pipeline: str) -> dict:
|
||||
"""
|
||||
Create an organized master manifest for split bundles with proper labeling and ordering.
|
||||
|
||||
Args:
|
||||
bundles: List of individual bundle data
|
||||
split_mode: The splitting mode used (pages, sections, chunks)
|
||||
pipeline: The pipeline type (no_ocr, ocr, vlm)
|
||||
|
||||
Returns:
|
||||
Enhanced manifest with organization metadata
|
||||
"""
|
||||
|
||||
# Sort bundles by their ordering key
|
||||
if split_mode == 'split_by_pages':
|
||||
sorted_bundles = sorted(bundles, key=lambda x: x.get('page_number', 0))
|
||||
display_name = f"{pipeline.upper()} Document Pages ({len(bundles)} pages)"
|
||||
organization = {
|
||||
'type': 'pages',
|
||||
'sort_field': 'page_number',
|
||||
'sort_order': 'asc',
|
||||
'grouping': 'individual_pages'
|
||||
}
|
||||
elif split_mode == 'split_by_sections':
|
||||
sorted_bundles = sorted(bundles, key=lambda x: x.get('split_order', 0))
|
||||
display_name = f"{pipeline.upper()} Document Sections ({len(bundles)} sections)"
|
||||
organization = {
|
||||
'type': 'sections',
|
||||
'sort_field': 'split_order',
|
||||
'sort_order': 'asc',
|
||||
'grouping': 'split_map_sections',
|
||||
'has_titles': True,
|
||||
'ordering_preserved': True
|
||||
}
|
||||
elif split_mode == 'split_by_chunks':
|
||||
sorted_bundles = sorted(bundles, key=lambda x: x.get('split_order', 0))
|
||||
display_name = f"{pipeline.upper()} Document Chunks ({len(bundles)} chunks)"
|
||||
organization = {
|
||||
'type': 'chunks',
|
||||
'sort_field': 'split_order',
|
||||
'sort_order': 'asc',
|
||||
'grouping': 'fallback_chunks'
|
||||
}
|
||||
else:
|
||||
sorted_bundles = bundles
|
||||
display_name = f"{pipeline.upper()} Document Bundles"
|
||||
organization = {
|
||||
'type': 'unknown',
|
||||
'sort_field': 'split_order',
|
||||
'sort_order': 'asc'
|
||||
}
|
||||
|
||||
return {
|
||||
'bundles': sorted_bundles,
|
||||
'display_name': display_name,
|
||||
'organization': organization,
|
||||
'total_bundles': len(bundles),
|
||||
'pipeline': pipeline,
|
||||
'split_mode': split_mode
|
||||
}
|
||||
|
||||
# Pipeline display names
|
||||
pipeline_names = {
|
||||
PipelineType.STANDARD: "Standard",
|
||||
PipelineType.VLM: "VLM",
|
||||
PipelineType.ASR: "ASR"
|
||||
}
|
||||
|
||||
pipeline_name = pipeline_names.get(metadata.pipeline, metadata.pipeline.value)
|
||||
|
||||
# OCR indication for standard pipeline
|
||||
if metadata.pipeline == PipelineType.STANDARD and metadata.config:
|
||||
ocr_enabled = metadata.config.get('do_ocr', False)
|
||||
pipeline_name = f"{pipeline_name} ({'OCR' if ocr_enabled else 'No-OCR'})"
|
||||
|
||||
# Processing mode indication
|
||||
if metadata.processing_mode == ProcessingMode.INDIVIDUAL_PAGES:
|
||||
mode = "Page-by-page"
|
||||
elif metadata.processing_mode == ProcessingMode.SPLIT_SECTIONS:
|
||||
mode = "Sections"
|
||||
else:
|
||||
mode = "Whole doc"
|
||||
|
||||
# Section or page info
|
||||
content_info = ""
|
||||
if metadata.split_heading:
|
||||
content_info = f" - {metadata.split_heading}"
|
||||
elif metadata.page_range and len(metadata.page_range) == 2:
|
||||
if metadata.page_range[0] == metadata.page_range[1]:
|
||||
content_info = f" - Page {metadata.page_range[0]}"
|
||||
else:
|
||||
content_info = f" - Pages {metadata.page_range[0]}-{metadata.page_range[1]}"
|
||||
|
||||
# Producer info
|
||||
producer_info = ""
|
||||
if metadata.producer == "auto_phase2":
|
||||
producer_info = " (Auto)"
|
||||
elif metadata.producer.startswith("auto"):
|
||||
producer_info = " (Auto)"
|
||||
|
||||
return f"{pipeline_name} {mode}{content_info}{producer_info}"
|
||||
|
||||
def group_bundles_by_metadata(bundles: List[Dict[str, Any]]) -> Dict[str, List[Dict[str, Any]]]:
|
||||
"""
|
||||
Group bundles by their metadata for display purposes.
|
||||
|
||||
Returns a dictionary mapping group keys to lists of bundles.
|
||||
"""
|
||||
|
||||
groups = {}
|
||||
ungrouped = []
|
||||
|
||||
for bundle in bundles:
|
||||
extra = bundle.get('extra', {})
|
||||
|
||||
# Skip bundles without the new metadata format
|
||||
if not extra.get('bundle_metadata_version'):
|
||||
ungrouped.append(bundle)
|
||||
continue
|
||||
|
||||
metadata = BundleMetadata.from_artefact_extra(
|
||||
bundle['file_id'],
|
||||
bundle['id'],
|
||||
extra
|
||||
)
|
||||
|
||||
if metadata.group_id:
|
||||
group_key = f"group:{metadata.group_id}"
|
||||
if group_key not in groups:
|
||||
groups[group_key] = []
|
||||
groups[group_key].append(bundle)
|
||||
else:
|
||||
ungrouped.append(bundle)
|
||||
|
||||
# Add ungrouped bundles as individual groups
|
||||
for bundle in ungrouped:
|
||||
single_key = f"single:{bundle['id']}"
|
||||
groups[single_key] = [bundle]
|
||||
|
||||
return groups
|
||||
@@ -1,280 +0,0 @@
|
||||
import os
|
||||
from modules.logger_tool import initialise_logger
|
||||
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
|
||||
import modules.database.schemas.nodes.calendars as calendar_schemas
|
||||
import modules.database.schemas.entities as entities
|
||||
import modules.database.schemas.relationships.calendars as cal_rels
|
||||
import modules.database.tools.neontology_tools as neon
|
||||
from modules.database.tools.filesystem_tools import ClassroomCopilotFilesystem
|
||||
from datetime import timedelta, datetime
|
||||
|
||||
def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False, entity_node=None, time_chunk_node=None):
|
||||
logger.info(f"Creating calendar for {start_date} to {end_date}")
|
||||
|
||||
logger.info("Initializing Neontology connection")
|
||||
|
||||
neon.init_neontology_connection()
|
||||
|
||||
filesystem = ClassroomCopilotFilesystem(db_name, init_run_type="school")
|
||||
|
||||
def create_tldraw_file_for_node(node, node_path):
|
||||
node_data = {
|
||||
"unique_id": node.unique_id,
|
||||
"type": node.__class__.__name__,
|
||||
"name": node.name if hasattr(node, 'name') else 'Unnamed Node'
|
||||
}
|
||||
logger.debug(f"Creating tldraw file for node: {node_data}")
|
||||
filesystem.create_default_tldraw_file(node_path, node_data)
|
||||
|
||||
created_years = {}
|
||||
created_months = {}
|
||||
created_weeks = {}
|
||||
created_days = {}
|
||||
|
||||
last_year_node = None
|
||||
last_month_node = None
|
||||
last_week_node = None
|
||||
last_day_node = None
|
||||
|
||||
calendar_nodes = {
|
||||
'calendar_node': None,
|
||||
'calendar_year_nodes': [],
|
||||
'calendar_month_nodes': [],
|
||||
'calendar_week_nodes': [],
|
||||
'calendar_day_nodes': []
|
||||
}
|
||||
|
||||
calendar_type = None
|
||||
if attach_to_calendar_node and entity_node:
|
||||
calendar_type = "entity_calendar"
|
||||
logger.info(f"Attaching calendar to entity node: {entity_node.unique_id}")
|
||||
entity_unique_id = entity_node.unique_id
|
||||
calendar_unique_id = f"Calendar_{entity_unique_id}"
|
||||
calendar_name = f"{start_date.strftime('%Y-%m-%d')} to {end_date.strftime('%Y-%m-%d')}"
|
||||
calendar_path = os.path.join(entity_node.path, "calendar")
|
||||
calendar_node = calendar_schemas.CalendarNode(
|
||||
unique_id=calendar_unique_id,
|
||||
name=calendar_name,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
path=calendar_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(calendar_node, database=db_name, operation='merge')
|
||||
calendar_nodes['calendar_node'] = calendar_node
|
||||
logger.info(f"Calendar node created: {calendar_node.unique_id}")
|
||||
|
||||
# Create a node tldraw file for the calendar node
|
||||
create_tldraw_file_for_node(calendar_node, calendar_path)
|
||||
|
||||
import backend.modules.database.schemas.relationships.owner_relationships as entity_cal_rels
|
||||
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
entity_cal_rels.EntityHasCalendar(source=entity_node, target=calendar_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {entity_node.unique_id} to {calendar_node.unique_id}")
|
||||
if entity_node and not attach_to_calendar_node:
|
||||
calendar_type = "time_entity"
|
||||
else:
|
||||
logger.error("Invalid combination of parameters for calendar creation.")
|
||||
raise ValueError("Invalid combination of parameters for calendar creation.")
|
||||
|
||||
current_date = start_date
|
||||
while current_date <= end_date:
|
||||
year = current_date.year
|
||||
month = current_date.month
|
||||
day = current_date.day
|
||||
iso_year, iso_week, iso_weekday = current_date.isocalendar()
|
||||
|
||||
# Create directories for year, month, week, and day
|
||||
_, year_path = filesystem.create_year_directory(year, calendar_path)
|
||||
_, month_path = filesystem.create_month_directory(year, month, calendar_path)
|
||||
_, week_path = filesystem.create_week_directory(year, iso_week, calendar_path)
|
||||
_, day_path = filesystem.create_day_directory(year, month, day, calendar_path)
|
||||
|
||||
calendar_year_unique_id = f"CalendarYear_{year}"
|
||||
|
||||
if year not in created_years:
|
||||
year_node = calendar_schemas.CalendarYearNode(
|
||||
unique_id=calendar_year_unique_id,
|
||||
year=str(year),
|
||||
path=year_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(year_node, database=db_name, operation='merge')
|
||||
calendar_nodes['calendar_year_nodes'].append(year_node)
|
||||
created_years[year] = year_node
|
||||
create_tldraw_file_for_node(year_node, year_path)
|
||||
logger.info(f"Year node created: {year_node.unique_id}")
|
||||
|
||||
if attach_to_calendar_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
cal_rels.CalendarIncludesYear(source=calendar_node, target=year_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {calendar_node.unique_id} to {year_node.unique_id}")
|
||||
if last_year_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
cal_rels.YearFollowsYear(source=last_year_node, target=year_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {last_year_node.unique_id} to {year_node.unique_id}")
|
||||
last_year_node = year_node
|
||||
|
||||
calendar_month_unique_id = f"CalendarMonth_{year}_{month}"
|
||||
|
||||
month_key = f"{year}-{month}"
|
||||
if month_key not in created_months:
|
||||
month_node = calendar_schemas.CalendarMonthNode(
|
||||
unique_id=calendar_month_unique_id,
|
||||
year=str(year),
|
||||
month=str(month),
|
||||
month_name=datetime(year, month, 1).strftime('%B'),
|
||||
path=month_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(month_node, database=db_name, operation='merge')
|
||||
calendar_nodes['calendar_month_nodes'].append(month_node)
|
||||
created_months[month_key] = month_node
|
||||
create_tldraw_file_for_node(month_node, month_path)
|
||||
logger.info(f"Month node created: {month_node.unique_id}")
|
||||
|
||||
# Check for the end of year transition for months
|
||||
if last_month_node:
|
||||
if int(month) == 1 and int(last_month_node.month) == 12 and int(last_month_node.year) == year - 1:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
cal_rels.MonthFollowsMonth(source=last_month_node, target=month_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {last_month_node.unique_id} to {month_node.unique_id}")
|
||||
elif int(month) == int(last_month_node.month) + 1:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
cal_rels.MonthFollowsMonth(source=last_month_node, target=month_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {last_month_node.unique_id} to {month_node.unique_id}")
|
||||
last_month_node = month_node
|
||||
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
cal_rels.YearIncludesMonth(source=year_node, target=month_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {year_node.unique_id} to {month_node.unique_id}")
|
||||
|
||||
calendar_week_unique_id = f"CalendarWeek_{iso_year}_{iso_week}"
|
||||
|
||||
week_key = f"{iso_year}-W{iso_week}"
|
||||
if week_key not in created_weeks:
|
||||
# Get the date of the first monday of the week
|
||||
week_start_date = current_date - timedelta(days=current_date.weekday())
|
||||
week_node = calendar_schemas.CalendarWeekNode(
|
||||
unique_id=calendar_week_unique_id,
|
||||
start_date=week_start_date,
|
||||
week_number=str(iso_week),
|
||||
iso_week=f"{iso_year}-W{iso_week:02}",
|
||||
path=week_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(week_node, database=db_name, operation='merge')
|
||||
calendar_nodes['calendar_week_nodes'].append(week_node)
|
||||
created_weeks[week_key] = week_node
|
||||
create_tldraw_file_for_node(week_node, week_path)
|
||||
logger.info(f"Week node created: {week_node.unique_id}")
|
||||
|
||||
if last_week_node and ((last_week_node.iso_week.split('-')[0] == str(iso_year) and int(last_week_node.week_number) == int(iso_week) - 1) or
|
||||
(last_week_node.iso_week.split('-')[0] != str(iso_year) and int(last_week_node.week_number) == 52 and int(iso_week) == 1)):
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
cal_rels.WeekFollowsWeek(source=last_week_node, target=week_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {last_week_node.unique_id} to {week_node.unique_id}")
|
||||
last_week_node = week_node
|
||||
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
cal_rels.YearIncludesWeek(source=year_node, target=week_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {year_node.unique_id} to {week_node.unique_id}")
|
||||
|
||||
calendar_day_unique_id = f"CalendarDay_{year}_{month}_{day}"
|
||||
|
||||
day_key = f"{year}-{month}-{day}"
|
||||
day_node = calendar_schemas.CalendarDayNode(
|
||||
unique_id=calendar_day_unique_id,
|
||||
date=current_date,
|
||||
day_of_week=current_date.strftime('%A'),
|
||||
iso_day=f"{year}-{month:02}-{day:02}",
|
||||
path=day_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(day_node, database=db_name, operation='merge')
|
||||
calendar_nodes['calendar_day_nodes'].append(day_node)
|
||||
created_days[day_key] = day_node
|
||||
create_tldraw_file_for_node(day_node, day_path)
|
||||
logger.info(f"Day node created: {day_node.unique_id}")
|
||||
|
||||
if last_day_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
cal_rels.DayFollowsDay(source=last_day_node, target=day_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {last_day_node.unique_id} to {day_node.unique_id}")
|
||||
last_day_node = day_node
|
||||
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
cal_rels.MonthIncludesDay(source=month_node, target=day_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {month_node.unique_id} to {day_node.unique_id}")
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
cal_rels.WeekIncludesDay(source=week_node, target=day_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {week_node.unique_id} to {day_node.unique_id}")
|
||||
current_date += timedelta(days=1)
|
||||
|
||||
if time_chunk_node:
|
||||
time_chunk_interval = time_chunk_node
|
||||
# Get every calendar day node and create time chunks of length time_chunk_node minutes for the whole day
|
||||
for day_node in calendar_nodes['calendar_day_nodes']:
|
||||
day_path = day_node.path
|
||||
total_time_chunks_in_day = (24 * 60) / time_chunk_interval
|
||||
for i in range(total_time_chunks_in_day):
|
||||
time_chunk_unique_id = f"CalendarTimeChunk_{day_node.unique_id}_{i}"
|
||||
time_chunk_start_time = day_node.date.time() + timedelta(minutes=i * time_chunk_interval)
|
||||
time_chunk_end_time = time_chunk_start_time + timedelta(minutes=time_chunk_interval)
|
||||
time_chunk_node = calendar_schemas.CalendarTimeChunkNode(
|
||||
unique_id=time_chunk_unique_id,
|
||||
start_time=time_chunk_start_time,
|
||||
end_time=time_chunk_end_time,
|
||||
path=day_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(time_chunk_node, database=db_name, operation='merge')
|
||||
calendar_nodes['calendar_time_chunk_nodes'].append(time_chunk_node)
|
||||
logger.info(f"Time chunk node created: {time_chunk_node.unique_id}")
|
||||
# Create a relationship between the time chunk node and the day node
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
cal_rels.DayIncludesTimeChunk(source=day_node, target=time_chunk_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {day_node.unique_id} to {time_chunk_node.unique_id}")
|
||||
# Create sequential relationship between the time chunk nodes
|
||||
if i > 0:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
cal_rels.TimeChunkFollowsTimeChunk(source=calendar_nodes['calendar_time_chunk_nodes'][i-1], target=time_chunk_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {calendar_nodes['calendar_time_chunk_nodes'][i-1].unique_id} to {time_chunk_node.unique_id}")
|
||||
|
||||
logger.info(f'Created calendar: {calendar_nodes["calendar_node"].unique_id}')
|
||||
return calendar_nodes
|
||||
@@ -1,401 +0,0 @@
|
||||
from enum import Enum
|
||||
from typing import Optional, List, Dict, Any
|
||||
import logging
|
||||
|
||||
from modules.database.admin.neontology_provider import NeontologyProvider
|
||||
|
||||
class NodeLabels(Enum):
|
||||
SCHOOL = "School"
|
||||
DEPARTMENT_STRUCTURE = "DepartmentStructure"
|
||||
CURRICULUM_STRUCTURE = "CurriculumStructure"
|
||||
PASTORAL_STRUCTURE = "PastoralStructure"
|
||||
DEPARTMENT = "Department"
|
||||
KEY_STAGE = "KeyStage"
|
||||
YEAR_GROUP = "YearGroup"
|
||||
|
||||
class RelationshipTypes(Enum):
|
||||
HAS_DEPARTMENT_STRUCTURE = "HAS_DEPARTMENT_STRUCTURE"
|
||||
HAS_CURRICULUM_STRUCTURE = "HAS_CURRICULUM_STRUCTURE"
|
||||
HAS_PASTORAL_STRUCTURE = "HAS_PASTORAL_STRUCTURE"
|
||||
HAS_DEPARTMENT = "HAS_DEPARTMENT"
|
||||
INCLUDES_KEY_STAGE = "INCLUDES_KEY_STAGE"
|
||||
INCLUDES_YEAR_GROUP = "INCLUDES_YEAR_GROUP"
|
||||
|
||||
class PropertyKeys(Enum):
|
||||
UNIQUE_ID = "unique_id"
|
||||
PATH = "path"
|
||||
URN = "urn"
|
||||
ESTABLISHMENT_NUMBER = "establishment_number"
|
||||
ESTABLISHMENT_NAME = "establishment_name"
|
||||
ESTABLISHMENT_TYPE = "establishment_type"
|
||||
ESTABLISHMENT_STATUS = "establishment_status"
|
||||
PHASE_OF_EDUCATION = "phase_of_education"
|
||||
STATUTORY_LOW_AGE = "statutory_low_age"
|
||||
STATUTORY_HIGH_AGE = "statutory_high_age"
|
||||
RELIGIOUS_CHARACTER = "religious_character"
|
||||
SCHOOL_CAPACITY = "school_capacity"
|
||||
SCHOOL_WEBSITE = "school_website"
|
||||
OFSTED_RATING = "ofsted_rating"
|
||||
DEPARTMENT_NAME = "department_name"
|
||||
KEY_STAGE = "key_stage"
|
||||
KEY_STAGE_NAME = "key_stage_name"
|
||||
YEAR_GROUP = "year_group"
|
||||
YEAR_GROUP_NAME = "year_group_name"
|
||||
CREATED = "created"
|
||||
MERGED = "merged"
|
||||
|
||||
class SchemaDefinition:
|
||||
"""Class to hold schema definition queries and information"""
|
||||
|
||||
@staticmethod
|
||||
def get_schema_info() -> Dict[str, List[Dict]]:
|
||||
"""Returns a dictionary containing the schema definition for nodes and relationships."""
|
||||
return {
|
||||
"nodes": [
|
||||
{
|
||||
"label": "School",
|
||||
"description": "Represents a school entity",
|
||||
"required_properties": ["unique_id", "urn", "name"],
|
||||
"optional_properties": ["address", "postcode", "phone", "email", "website"]
|
||||
},
|
||||
{
|
||||
"label": "DepartmentStructure",
|
||||
"description": "Represents the department structure of a school",
|
||||
"required_properties": ["unique_id", "name"],
|
||||
"optional_properties": ["description", "head_of_department"]
|
||||
},
|
||||
{
|
||||
"label": "CurriculumStructure",
|
||||
"description": "Represents the curriculum structure of a school",
|
||||
"required_properties": ["unique_id", "name"],
|
||||
"optional_properties": ["description", "key_stage", "subject"]
|
||||
},
|
||||
{
|
||||
"label": "PastoralStructure",
|
||||
"description": "Represents the pastoral structure of a school",
|
||||
"required_properties": ["unique_id", "name"],
|
||||
"optional_properties": ["description", "year_group", "form_group"]
|
||||
}
|
||||
],
|
||||
"relationships": [
|
||||
{
|
||||
"type": "HAS_DEPARTMENT_STRUCTURE",
|
||||
"description": "Links a school to its department structure",
|
||||
"source": "School",
|
||||
"target": "DepartmentStructure",
|
||||
"properties": ["created_at"]
|
||||
},
|
||||
{
|
||||
"type": "HAS_CURRICULUM_STRUCTURE",
|
||||
"description": "Links a school to its curriculum structure",
|
||||
"source": "School",
|
||||
"target": "CurriculumStructure",
|
||||
"properties": ["created_at"]
|
||||
},
|
||||
{
|
||||
"type": "HAS_PASTORAL_STRUCTURE",
|
||||
"description": "Links a school to its pastoral structure",
|
||||
"source": "School",
|
||||
"target": "PastoralStructure",
|
||||
"properties": ["created_at"]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def get_schema_creation_queries() -> List[str]:
|
||||
"""Returns a list of Cypher queries to create the schema."""
|
||||
return [
|
||||
# Node Uniqueness Constraints
|
||||
f"CREATE CONSTRAINT school_unique_id IF NOT EXISTS FOR (n:{NodeLabels.SCHOOL.value}) REQUIRE n.{PropertyKeys.UNIQUE_ID.value} IS UNIQUE",
|
||||
f"CREATE CONSTRAINT department_unique_id IF NOT EXISTS FOR (n:{NodeLabels.DEPARTMENT_STRUCTURE.value}) REQUIRE n.{PropertyKeys.UNIQUE_ID.value} IS UNIQUE",
|
||||
f"CREATE CONSTRAINT curriculum_unique_id IF NOT EXISTS FOR (n:{NodeLabels.CURRICULUM_STRUCTURE.value}) REQUIRE n.{PropertyKeys.UNIQUE_ID.value} IS UNIQUE",
|
||||
f"CREATE CONSTRAINT pastoral_unique_id IF NOT EXISTS FOR (n:{NodeLabels.PASTORAL_STRUCTURE.value}) REQUIRE n.{PropertyKeys.UNIQUE_ID.value} IS UNIQUE",
|
||||
|
||||
# Indexes for Performance
|
||||
f"CREATE INDEX school_urn IF NOT EXISTS FOR (n:{NodeLabels.SCHOOL.value}) ON (n.{PropertyKeys.URN.value})",
|
||||
f"CREATE INDEX school_name IF NOT EXISTS FOR (n:{NodeLabels.SCHOOL.value}) ON (n.{PropertyKeys.ESTABLISHMENT_NAME.value})",
|
||||
f"CREATE INDEX department_name IF NOT EXISTS FOR (n:{NodeLabels.DEPARTMENT_STRUCTURE.value}) ON (n.{PropertyKeys.DEPARTMENT_NAME.value})",
|
||||
f"CREATE INDEX curriculum_name IF NOT EXISTS FOR (n:{NodeLabels.CURRICULUM_STRUCTURE.value}) ON (n.name)",
|
||||
f"CREATE INDEX pastoral_name IF NOT EXISTS FOR (n:{NodeLabels.PASTORAL_STRUCTURE.value}) ON (n.name)",
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def get_schema_verification_queries() -> Dict[str, str]:
|
||||
"""Returns a dictionary of queries to verify the schema state."""
|
||||
return {
|
||||
"constraints": "SHOW CONSTRAINTS",
|
||||
"indexes": "SHOW INDEXES",
|
||||
"labels": "CALL db.labels()"
|
||||
}
|
||||
|
||||
class GraphNamingProvider:
|
||||
@staticmethod
|
||||
def get_school_unique_id(urn: str) -> str:
|
||||
"""Generate unique ID for a school node."""
|
||||
return f"School_{urn}"
|
||||
|
||||
@staticmethod
|
||||
def get_department_structure_unique_id(school_unique_id: str) -> str:
|
||||
"""Generate unique ID for a department structure node."""
|
||||
return f"DepartmentStructure_{school_unique_id}"
|
||||
|
||||
@staticmethod
|
||||
def get_curriculum_structure_unique_id(school_unique_id: str) -> str:
|
||||
"""Generate unique ID for a curriculum structure node."""
|
||||
return f"CurriculumStructure_{school_unique_id}"
|
||||
|
||||
@staticmethod
|
||||
def get_pastoral_structure_unique_id(school_unique_id: str) -> str:
|
||||
"""Generate unique ID for a pastoral structure node."""
|
||||
return f"PastoralStructure_{school_unique_id}"
|
||||
|
||||
@staticmethod
|
||||
def get_department_unique_id(school_unique_id: str, department_name: str) -> str:
|
||||
"""Generate unique ID for a department node."""
|
||||
return f"Department_{school_unique_id}_{department_name.replace(' ', '_')}"
|
||||
|
||||
@staticmethod
|
||||
def get_key_stage_unique_id(curriculum_structure_unique_id: str, key_stage: str) -> str:
|
||||
"""Generate unique ID for a key stage node."""
|
||||
return f"KeyStage_{curriculum_structure_unique_id}_KStg{key_stage}"
|
||||
|
||||
@staticmethod
|
||||
def get_year_group_unique_id(school_unique_id: str, year_group: int) -> str:
|
||||
"""Generate unique ID for a year group node."""
|
||||
return f"YearGroup_{school_unique_id}_YGrp{year_group}"
|
||||
|
||||
@staticmethod
|
||||
def get_school_path(database_name: str, urn: str) -> str:
|
||||
"""Generate path for a school node."""
|
||||
return f"/schools/{database_name}/{urn}"
|
||||
|
||||
@staticmethod
|
||||
def get_department_path(school_path: str, department_name: str) -> str:
|
||||
"""Generate path for a department node."""
|
||||
return f"{school_path}/departments/{department_name}"
|
||||
|
||||
@staticmethod
|
||||
def get_department_structure_path(school_path: str) -> str:
|
||||
"""Generate path for a department structure node."""
|
||||
return f"{school_path}/departments"
|
||||
|
||||
@staticmethod
|
||||
def get_curriculum_path(school_path: str) -> str:
|
||||
"""Generate path for a curriculum structure node."""
|
||||
return f"{school_path}/curriculum"
|
||||
|
||||
@staticmethod
|
||||
def get_pastoral_path(school_path: str) -> str:
|
||||
"""Generate path for a pastoral structure node."""
|
||||
return f"{school_path}/pastoral"
|
||||
|
||||
@staticmethod
|
||||
def get_key_stage_path(curriculum_path: str, key_stage: str) -> str:
|
||||
"""Generate path for a key stage node."""
|
||||
return f"{curriculum_path}/key_stage_{key_stage}"
|
||||
|
||||
@staticmethod
|
||||
def get_year_group_path(pastoral_path: str, year_group: int) -> str:
|
||||
"""Generate path for a year group node."""
|
||||
return f"{pastoral_path}/year_{year_group}"
|
||||
|
||||
@staticmethod
|
||||
def get_cypher_match_school(unique_id: str) -> str:
|
||||
"""Generate Cypher MATCH clause for finding a school node."""
|
||||
return f"MATCH (s:{NodeLabels.SCHOOL.value} {{{PropertyKeys.UNIQUE_ID.value}: $school_id}})"
|
||||
|
||||
@staticmethod
|
||||
def get_cypher_check_basic_structure() -> str:
|
||||
"""Generate Cypher query for checking basic structure existence and validity."""
|
||||
return """
|
||||
// Find the school node
|
||||
MATCH (s:{school})
|
||||
|
||||
// Check for department structure with any relationship
|
||||
OPTIONAL MATCH (s)-[r1]-(dept_struct:{dept_struct})
|
||||
|
||||
// Check for curriculum structure with any relationship
|
||||
OPTIONAL MATCH (s)-[r2]-(curr_struct:{curr_struct})
|
||||
|
||||
// Check for pastoral structure with any relationship
|
||||
OPTIONAL MATCH (s)-[r3]-(past_struct:{past_struct})
|
||||
|
||||
// Return structure information
|
||||
RETURN {{
|
||||
has_basic:
|
||||
dept_struct IS NOT NULL AND r1 IS NOT NULL AND
|
||||
curr_struct IS NOT NULL AND r2 IS NOT NULL AND
|
||||
past_struct IS NOT NULL AND r3 IS NOT NULL,
|
||||
department_structure: {{
|
||||
exists: dept_struct IS NOT NULL AND r1 IS NOT NULL
|
||||
}},
|
||||
curriculum_structure: {{
|
||||
exists: curr_struct IS NOT NULL AND r2 IS NOT NULL
|
||||
}},
|
||||
pastoral_structure: {{
|
||||
exists: past_struct IS NOT NULL AND r3 IS NOT NULL
|
||||
}}
|
||||
}} as status
|
||||
""".format(
|
||||
school=NodeLabels.SCHOOL.value,
|
||||
dept_struct=NodeLabels.DEPARTMENT_STRUCTURE.value,
|
||||
curr_struct=NodeLabels.CURRICULUM_STRUCTURE.value,
|
||||
past_struct=NodeLabels.PASTORAL_STRUCTURE.value
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_cypher_check_detailed_structure() -> str:
|
||||
"""Generate Cypher query for checking detailed structure existence and validity."""
|
||||
return """
|
||||
// Find the school node
|
||||
MATCH (s:{school} {{unique_id: $school_id}})
|
||||
|
||||
// Check for department structure and departments
|
||||
OPTIONAL MATCH (s)-[r1]-(dept_struct:{dept_struct})
|
||||
WHERE dept_struct.unique_id = 'DepartmentStructure_' + s.unique_id
|
||||
WITH s, dept_struct, r1,
|
||||
CASE WHEN dept_struct IS NOT NULL
|
||||
THEN [(dept_struct)-[r]-(d:{dept}) | d]
|
||||
ELSE []
|
||||
END as departments
|
||||
|
||||
// Check for curriculum structure and key stages
|
||||
OPTIONAL MATCH (s)-[r2]-(curr_struct:{curr_struct})
|
||||
WHERE curr_struct.unique_id = 'CurriculumStructure_' + s.unique_id
|
||||
WITH s, dept_struct, r1, departments, curr_struct, r2,
|
||||
CASE WHEN curr_struct IS NOT NULL
|
||||
THEN [(curr_struct)-[r]-(k:{key_stage}) | k]
|
||||
ELSE []
|
||||
END as key_stages
|
||||
|
||||
// Check for pastoral structure and year groups
|
||||
OPTIONAL MATCH (s)-[r3]-(past_struct:{past_struct})
|
||||
WHERE past_struct.unique_id = 'PastoralStructure_' + s.unique_id
|
||||
WITH dept_struct, r1, departments, curr_struct, r2, key_stages, past_struct, r3,
|
||||
CASE WHEN past_struct IS NOT NULL
|
||||
THEN [(past_struct)-[r]-(y:{year_group}) | y]
|
||||
ELSE []
|
||||
END as year_groups
|
||||
|
||||
// Return structure information
|
||||
RETURN {{
|
||||
has_detailed:
|
||||
dept_struct IS NOT NULL AND r1 IS NOT NULL AND size(departments) > 0 AND
|
||||
curr_struct IS NOT NULL AND r2 IS NOT NULL AND size(key_stages) > 0 AND
|
||||
past_struct IS NOT NULL AND r3 IS NOT NULL AND size(year_groups) > 0,
|
||||
department_structure: {{
|
||||
exists: dept_struct IS NOT NULL AND r1 IS NOT NULL,
|
||||
has_departments: size(departments) > 0,
|
||||
department_count: size(departments),
|
||||
node_id: dept_struct.unique_id
|
||||
}},
|
||||
curriculum_structure: {{
|
||||
exists: curr_struct IS NOT NULL AND r2 IS NOT NULL,
|
||||
has_key_stages: size(key_stages) > 0,
|
||||
key_stage_count: size(key_stages),
|
||||
node_id: curr_struct.unique_id
|
||||
}},
|
||||
pastoral_structure: {{
|
||||
exists: past_struct IS NOT NULL AND r3 IS NOT NULL,
|
||||
has_year_groups: size(year_groups) > 0,
|
||||
year_group_count: size(year_groups),
|
||||
node_id: past_struct.unique_id
|
||||
}}
|
||||
}} as status
|
||||
""".format(
|
||||
school=NodeLabels.SCHOOL.value,
|
||||
dept_struct=NodeLabels.DEPARTMENT_STRUCTURE.value,
|
||||
curr_struct=NodeLabels.CURRICULUM_STRUCTURE.value,
|
||||
past_struct=NodeLabels.PASTORAL_STRUCTURE.value,
|
||||
dept=NodeLabels.DEPARTMENT.value,
|
||||
key_stage=NodeLabels.KEY_STAGE.value,
|
||||
year_group=NodeLabels.YEAR_GROUP.value
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_schema_definition() -> SchemaDefinition:
|
||||
"""Get the schema definition instance"""
|
||||
return SchemaDefinition()
|
||||
|
||||
@staticmethod
|
||||
def get_schema_creation_queries() -> List[str]:
|
||||
"""Get queries to create the schema"""
|
||||
return SchemaDefinition.get_schema_creation_queries()
|
||||
|
||||
@staticmethod
|
||||
def get_schema_verification_queries() -> Dict[str, str]:
|
||||
"""Get queries to verify schema state"""
|
||||
return SchemaDefinition.get_schema_verification_queries()
|
||||
|
||||
@staticmethod
|
||||
def get_schema_info() -> Dict[str, List[Dict]]:
|
||||
"""Get human-readable schema information"""
|
||||
return SchemaDefinition.get_schema_info()
|
||||
|
||||
class GraphProvider:
|
||||
def __init__(self):
|
||||
"""Initialize the graph provider with Neo4j connection."""
|
||||
self.neontology = NeontologyProvider()
|
||||
self.graph_naming = GraphNamingProvider()
|
||||
self.logger = logging.getLogger(__name__)
|
||||
|
||||
def check_schema_status(self, database_name: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Checks the current state of the schema in the specified database.
|
||||
Returns a dictionary containing information about constraints, indexes, and labels.
|
||||
"""
|
||||
try:
|
||||
verification_queries = SchemaDefinition.get_schema_verification_queries()
|
||||
expected_schema = SchemaDefinition.get_schema_info()
|
||||
|
||||
# Get current schema state
|
||||
constraints = self.neontology.run_query(verification_queries["constraints"], {}, database_name)
|
||||
indexes = self.neontology.run_query(verification_queries["indexes"], {}, database_name)
|
||||
labels = self.neontology.run_query(verification_queries["labels"], {}, database_name)
|
||||
|
||||
# Process results
|
||||
current_constraints = [c["name"] for c in constraints]
|
||||
current_indexes = [i["name"] for i in indexes]
|
||||
current_labels = [l["label"] for l in labels]
|
||||
|
||||
# Expected values
|
||||
expected_labels = [node["label"] for node in expected_schema["nodes"]]
|
||||
|
||||
return {
|
||||
"constraints": current_constraints,
|
||||
"constraints_valid": len(current_constraints) >= 4, # We expect at least 4 unique constraints
|
||||
"indexes": current_indexes,
|
||||
"indexes_valid": len(current_indexes) >= 5, # We expect at least 5 indexes
|
||||
"labels": current_labels,
|
||||
"labels_valid": all(label in current_labels for label in expected_labels)
|
||||
}
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error checking schema status: {str(e)}")
|
||||
return {
|
||||
"constraints": [], "constraints_valid": False,
|
||||
"indexes": [], "indexes_valid": False,
|
||||
"labels": [], "labels_valid": False
|
||||
}
|
||||
|
||||
def initialize_schema(self, database_name: str) -> None:
|
||||
"""
|
||||
Initializes the schema for the specified database by creating all necessary
|
||||
constraints and indexes.
|
||||
"""
|
||||
try:
|
||||
creation_queries = SchemaDefinition.get_schema_creation_queries()
|
||||
|
||||
for query in creation_queries:
|
||||
self.neontology.cypher_write(query, {}, database_name)
|
||||
|
||||
self.logger.info(f"Schema initialized successfully for database {database_name}")
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error initializing schema: {str(e)}")
|
||||
raise
|
||||
|
||||
def get_schema_info(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Returns the schema definition information.
|
||||
"""
|
||||
return SchemaDefinition.get_schema_info()
|
||||
@@ -1,212 +0,0 @@
|
||||
from dotenv import load_dotenv, find_dotenv
|
||||
load_dotenv(find_dotenv())
|
||||
import os
|
||||
import modules.logger_tool as logger
|
||||
from modules.database.tools.neontology.graphconnection import GraphConnection, init_neontology
|
||||
from modules.database.tools.neontology.basenode import BaseNode
|
||||
from modules.database.tools.neontology.baserelationship import BaseRelationship
|
||||
from typing import Optional, Dict, Any, List
|
||||
from neo4j import Record as Neo4jRecord
|
||||
import re
|
||||
|
||||
log_name = 'api_modules_database_admin_neontology_provider'
|
||||
log_dir = os.getenv("LOG_PATH", "/logs") # Default path as fallback
|
||||
logging = logger.get_logger(
|
||||
name=log_name,
|
||||
log_level=os.getenv("LOG_LEVEL", "DEBUG"),
|
||||
log_path=log_dir,
|
||||
log_file=log_name,
|
||||
runtime=True,
|
||||
log_format='default'
|
||||
)
|
||||
|
||||
class NeontologyProvider:
|
||||
"""Provider class for managing Neontology connections and operations."""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize the provider with Neo4j connection details from environment."""
|
||||
self.bolt_url = os.getenv("APP_BOLT_URL")
|
||||
self.user = os.getenv("USER_NEO4J")
|
||||
self.password = os.getenv("PASSWORD_NEO4J")
|
||||
self.connection = None
|
||||
self.current_database = None
|
||||
|
||||
def _validate_database_name(self, database: str) -> str:
|
||||
"""
|
||||
Validate and format database name to handle special characters.
|
||||
|
||||
Args:
|
||||
database: The database name to validate
|
||||
|
||||
Returns:
|
||||
str: The validated database name
|
||||
|
||||
Raises:
|
||||
ValueError: If database name is invalid
|
||||
"""
|
||||
if not database:
|
||||
raise ValueError("Database name cannot be empty")
|
||||
|
||||
# Check for valid database name pattern
|
||||
# Allow letters, numbers, underscores, and dots
|
||||
if not re.match(r'^[a-zA-Z0-9_\.]+$', database):
|
||||
raise ValueError("Database name contains invalid characters")
|
||||
|
||||
# For database names with multiple dots, we need to handle them specially
|
||||
# Neo4j treats dots as special characters in some contexts
|
||||
if database.count('.') > 1:
|
||||
# Replace dots with underscores except for the first one
|
||||
parts = database.split('.')
|
||||
if len(parts) > 2:
|
||||
# Keep the first dot, replace others with underscore
|
||||
formatted_name = f"{parts[0]}.{'.'.join(parts[1:])}"
|
||||
logging.info(f"Reformatted database name from {database} to {formatted_name}")
|
||||
return formatted_name
|
||||
|
||||
return database
|
||||
|
||||
def connect(self, database: str = 'neo4j') -> None:
|
||||
"""Establish connection to Neo4j using Neontology."""
|
||||
try:
|
||||
# Validate and format database name
|
||||
formatted_database = self._validate_database_name(database)
|
||||
|
||||
# If we're switching databases, ensure we close the old connection
|
||||
if self.current_database != formatted_database and self.connection is not None:
|
||||
self.close()
|
||||
|
||||
# Initialize Neontology connection if needed
|
||||
if self.connection is None:
|
||||
init_neontology(
|
||||
neo4j_uri=self.bolt_url,
|
||||
neo4j_username=self.user,
|
||||
neo4j_password=self.password
|
||||
)
|
||||
# Get the GraphConnection instance
|
||||
self.connection = GraphConnection()
|
||||
self.current_database = formatted_database
|
||||
logging.info(f"Neontology connection initialized with host: {self.host}, port: {self.port}, database: {formatted_database}")
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to initialize Neontology connection: {str(e)}")
|
||||
raise
|
||||
|
||||
def reset_connection(self) -> None:
|
||||
"""Reset the connection, forcing a new one to be created on next use."""
|
||||
if self.connection:
|
||||
self.close()
|
||||
|
||||
def create_or_merge_node(self, node: BaseNode, database: str = 'neo4j', operation: str = "merge") -> None:
|
||||
"""Create or merge a node in the Neo4j database."""
|
||||
try:
|
||||
if not self.connection or self.current_database != database:
|
||||
self.connect(database)
|
||||
|
||||
if operation == "create":
|
||||
node.create(database=database)
|
||||
elif operation == "merge":
|
||||
node.merge(database=database)
|
||||
else:
|
||||
logging.error(f"Invalid operation: {operation}")
|
||||
raise ValueError(f"Invalid operation: {operation}")
|
||||
except Exception as e:
|
||||
logging.error(f"Error in processing node: {e}")
|
||||
raise
|
||||
|
||||
def create_or_merge_relationship(self, relationship: BaseRelationship, database: str = 'neo4j', operation: str = "merge") -> None:
|
||||
"""Create or merge a relationship in the Neo4j database."""
|
||||
try:
|
||||
if not self.connection or self.current_database != database:
|
||||
self.connect(database)
|
||||
|
||||
if operation == "create":
|
||||
relationship.create(database=database)
|
||||
elif operation == "merge":
|
||||
relationship.merge(database=database)
|
||||
else:
|
||||
logging.error(f"Invalid operation: {operation}")
|
||||
raise ValueError(f"Invalid operation: {operation}")
|
||||
except Exception as e:
|
||||
logging.error(f"Error in processing relationship: {e}")
|
||||
raise
|
||||
|
||||
def cypher_write(self, cypher: str, params: Dict[str, Any] = {}, database: str = 'neo4j') -> None:
|
||||
"""Execute a write transaction."""
|
||||
try:
|
||||
if not self.connection or self.current_database != database:
|
||||
self.connect(database)
|
||||
self.connection.cypher_write(cypher, params)
|
||||
except Exception as e:
|
||||
logging.error(f"Error in cypher write: {e}")
|
||||
raise
|
||||
|
||||
def cypher_read(self, cypher: str, params: Dict[str, Any] = {}, database: str = 'neo4j') -> Optional[Neo4jRecord]:
|
||||
"""Execute a read transaction returning a single record."""
|
||||
try:
|
||||
if not self.connection or self.current_database != database:
|
||||
self.connect(database)
|
||||
return self.connection.cypher_read(cypher, params)
|
||||
except Exception as e:
|
||||
logging.error(f"Error in cypher read: {e}")
|
||||
raise
|
||||
|
||||
def cypher_read_many(self, cypher: str, params: Dict[str, Any] = {}, database: str = 'neo4j') -> List[Neo4jRecord]:
|
||||
"""Execute a read transaction returning multiple records."""
|
||||
try:
|
||||
if not self.connection or self.current_database != database:
|
||||
self.connect(database)
|
||||
return self.connection.cypher_read_many(cypher, params)
|
||||
except Exception as e:
|
||||
logging.error(f"Error in cypher read many: {e}")
|
||||
raise
|
||||
|
||||
def run_query(self, cypher: str, params: Dict[str, Any] = {}, database: str = 'neo4j') -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Execute a Cypher query and return results as a list of dictionaries.
|
||||
This is a convenience method that handles both single and multiple record results.
|
||||
|
||||
Args:
|
||||
cypher: The Cypher query to execute
|
||||
params: Query parameters
|
||||
database: Target database name
|
||||
|
||||
Returns:
|
||||
List[Dict[str, Any]]: Query results as a list of dictionaries
|
||||
"""
|
||||
try:
|
||||
if not self.connection or self.current_database != database:
|
||||
self.connect(database)
|
||||
|
||||
# Use cypher_read_many for consistent return type
|
||||
records = self.connection.cypher_read_many(cypher, params)
|
||||
|
||||
# Convert Neo4j records to dictionaries
|
||||
results = []
|
||||
for record in records:
|
||||
# Handle both Record and dict types
|
||||
if isinstance(record, Neo4jRecord):
|
||||
results.append(dict(record))
|
||||
else:
|
||||
results.append(record)
|
||||
|
||||
return results
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error in run_query: {e}")
|
||||
raise
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the Neontology connection."""
|
||||
if self.connection:
|
||||
# The connection will be closed when the GraphConnection instance is deleted
|
||||
self.connection = None
|
||||
self.current_database = None
|
||||
logging.info("Neontology connection closed")
|
||||
|
||||
def __enter__(self):
|
||||
"""Context manager entry."""
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
"""Context manager exit."""
|
||||
self.close()
|
||||
@@ -1,797 +0,0 @@
|
||||
import os
|
||||
from modules.logger_tool import initialise_logger
|
||||
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
|
||||
import backend.modules.database.schemas.entities as neo_entity
|
||||
import modules.database.schemas.curriculum_neo as neo_curriculum
|
||||
import modules.database.schemas.relationships.curriculum_relationships as curriculum_relationships
|
||||
import modules.database.schemas.relationships.entity_relationships as ent_rels
|
||||
import modules.database.schemas.relationships.entity_curriculum_rels as ent_cur_rels
|
||||
from modules.database.tools.filesystem_tools import ClassroomCopilotFilesystem
|
||||
import modules.database.tools.neontology_tools as neon
|
||||
import pandas as pd
|
||||
|
||||
# Default values for nodes
|
||||
default_topic_values = {
|
||||
'topic_assessment_type': 'Null',
|
||||
'topic_type': 'Null',
|
||||
'total_number_of_lessons_for_topic': '1',
|
||||
'topic_title': 'Null'
|
||||
}
|
||||
|
||||
default_topic_lesson_values = {
|
||||
'topic_lesson_title': 'Null',
|
||||
'topic_lesson_type': 'Null',
|
||||
'topic_lesson_length': '1',
|
||||
'topic_lesson_suggested_activities': 'Null',
|
||||
'topic_lesson_skills_learned': 'Null',
|
||||
'topic_lesson_weblinks': 'Null',
|
||||
}
|
||||
|
||||
default_learning_statement_values = {
|
||||
'lesson_learning_statement': 'Null',
|
||||
'lesson_learning_statement_type': 'Student learning outcome'
|
||||
}
|
||||
|
||||
# Helper function to sort year groups numerically where possible
|
||||
def sort_year_groups(df):
|
||||
df = df.copy()
|
||||
df['YearGroupNumeric'] = pd.to_numeric(df['YearGroup'], errors='coerce')
|
||||
return df.sort_values(by='YearGroupNumeric')
|
||||
|
||||
def create_curriculum(dataframes, school_db_name, curriculum_db_name, school_node):
|
||||
|
||||
fs_handler = ClassroomCopilotFilesystem(school_db_name, init_run_type="school")
|
||||
|
||||
logger.info(f"Initialising neo4j connection...")
|
||||
neon.init_neontology_connection()
|
||||
|
||||
keystagesyllabus_df = dataframes['keystagesyllabuses']
|
||||
yeargroupsyllabus_df = dataframes['yeargroupsyllabuses']
|
||||
topic_df = dataframes['topics']
|
||||
lesson_df = dataframes['lessons']
|
||||
statement_df = dataframes['statements']
|
||||
# resource_df = dataframes['resources'] # TODO
|
||||
|
||||
node_library = {}
|
||||
node_library['key_stage_nodes'] = {}
|
||||
node_library['year_group_nodes'] = {}
|
||||
node_library['key_stage_syllabus_nodes'] = {}
|
||||
node_library['year_group_syllabus_nodes'] = {}
|
||||
node_library['topic_nodes'] = {}
|
||||
node_library['topic_lesson_nodes'] = {}
|
||||
node_library['statement_nodes'] = {}
|
||||
node_library['department_nodes'] = {}
|
||||
node_library['subject_nodes'] = {}
|
||||
curriculum_node = None
|
||||
pastoral_node = None
|
||||
key_stage_nodes_created = {}
|
||||
year_group_nodes_created = {}
|
||||
last_year_group_node = None
|
||||
last_key_stage_node = None
|
||||
|
||||
# Create Curriculum and Pastoral nodes and relationships with School in both databases
|
||||
_, curriculum_path = fs_handler.create_school_curriculum_directory(school_node.path)
|
||||
_, pastoral_path = fs_handler.create_school_pastoral_directory(school_node.path)
|
||||
|
||||
# Create Department Structure node
|
||||
department_structure_node_unique_id = f"DepartmentStructure_{school_node.unique_id}"
|
||||
department_structure_node = neo_entity.DepartmentStructureNode(
|
||||
unique_id=department_structure_node_unique_id,
|
||||
path=os.path.join(school_node.path, "departments")
|
||||
)
|
||||
# Create in school database only
|
||||
neon.create_or_merge_neontology_node(department_structure_node, database=school_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(department_structure_node.path, department_structure_node.to_dict())
|
||||
node_library['department_structure_node'] = department_structure_node
|
||||
|
||||
# Link Department Structure to School
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
ent_rels.SchoolHasDepartmentStructure(source=school_node, target=department_structure_node),
|
||||
database=school_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created department structure node and linked to school")
|
||||
|
||||
# Create Curriculum Structure node
|
||||
curriculum_structure_node_unique_id = f"CurriculumStructure_{school_node.unique_id}"
|
||||
curriculum_node = neo_curriculum.CurriculumStructureNode(
|
||||
unique_id=curriculum_structure_node_unique_id,
|
||||
path=curriculum_path
|
||||
)
|
||||
# Create in school database only
|
||||
neon.create_or_merge_neontology_node(curriculum_node, database=school_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(curriculum_node.path, curriculum_node.to_dict())
|
||||
node_library['curriculum_node'] = curriculum_node
|
||||
|
||||
# Create relationship in school database only
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
ent_cur_rels.SchoolHasCurriculumStructure(source=school_node, target=curriculum_node),
|
||||
database=school_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created curriculum node and relationship with school")
|
||||
|
||||
# Create Pastoral Structure node
|
||||
pastoral_structure_node_unique_id = f"PastoralStructure_{school_node.unique_id}"
|
||||
pastoral_node = neo_curriculum.PastoralStructureNode(
|
||||
unique_id=pastoral_structure_node_unique_id,
|
||||
path=pastoral_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(pastoral_node, database=school_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(pastoral_node.path, pastoral_node.to_dict())
|
||||
node_library['pastoral_node'] = pastoral_node
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
ent_cur_rels.SchoolHasPastoralStructure(source=school_node, target=pastoral_node),
|
||||
database=school_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created pastoral node and relationship with school")
|
||||
|
||||
# Create departments and subjects
|
||||
# First get unique departments
|
||||
unique_departments = keystagesyllabus_df['Department'].dropna().unique()
|
||||
|
||||
for department_name in unique_departments:
|
||||
department_unique_id = f"Department_{school_node.unique_id}_{department_name.replace(' ', '_')}"
|
||||
_, department_path = fs_handler.create_school_department_directory(school_node.path, department_name)
|
||||
|
||||
department_node = neo_entity.DepartmentNode(
|
||||
unique_id=department_unique_id,
|
||||
department_name=department_name,
|
||||
path=department_path
|
||||
)
|
||||
# Create department in school database only
|
||||
neon.create_or_merge_neontology_node(department_node, database=school_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(department_node.path, department_node.to_dict())
|
||||
node_library['department_nodes'][department_name] = department_node
|
||||
|
||||
# Link department to department structure in school database
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
ent_rels.DepartmentStructureHasDepartment(source=department_structure_node, target=department_node),
|
||||
database=school_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created department node for {department_name} and linked to department structure")
|
||||
|
||||
# Create subjects and link to departments
|
||||
# First get unique subjects from key stage syllabuses (which have department info)
|
||||
unique_subjects = keystagesyllabus_df[['Subject', 'SubjectCode', 'Department']].drop_duplicates()
|
||||
|
||||
# Then add any additional subjects from year group syllabuses (without department info)
|
||||
additional_subjects = yeargroupsyllabus_df[['Subject', 'SubjectCode']].drop_duplicates()
|
||||
additional_subjects = additional_subjects[~additional_subjects['SubjectCode'].isin(unique_subjects['SubjectCode'])]
|
||||
|
||||
# Process subjects from key stage syllabuses first (these have department info)
|
||||
for _, subject_row in unique_subjects.iterrows():
|
||||
subject_unique_id = f"Subject_{school_node.unique_id}_{subject_row['SubjectCode']}"
|
||||
department_node = node_library['department_nodes'].get(subject_row['Department'])
|
||||
if not department_node:
|
||||
logger.warning(f"No department found for subject {subject_row['Subject']} with code {subject_row['SubjectCode']}")
|
||||
continue
|
||||
|
||||
_, subject_path = fs_handler.create_department_subject_directory(
|
||||
department_node.path,
|
||||
subject_row['Subject'] # Use full subject name instead of SubjectCode
|
||||
)
|
||||
logger.info(f"Created subject directory for {subject_path}")
|
||||
|
||||
subject_node = neo_curriculum.SubjectNode(
|
||||
unique_id=subject_unique_id,
|
||||
subject_code=subject_row['SubjectCode'],
|
||||
subject_name=subject_row['Subject'],
|
||||
path=subject_path
|
||||
)
|
||||
# Create subject in both databases
|
||||
neon.create_or_merge_neontology_node(subject_node, database=school_db_name, operation='merge')
|
||||
neon.create_or_merge_neontology_node(subject_node, database=curriculum_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(subject_node.path, subject_node.to_dict())
|
||||
node_library['subject_nodes'][subject_row['Subject']] = subject_node
|
||||
|
||||
# Link subject to department in school database only
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
ent_rels.DepartmentManagesSubject(source=department_node, target=subject_node),
|
||||
database=school_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created subject node for {subject_row['Subject']} and linked to department {subject_row['Department']}")
|
||||
|
||||
# Process any additional subjects from year group syllabuses (these won't have department info)
|
||||
for _, subject_row in additional_subjects.iterrows():
|
||||
subject_unique_id = f"Subject_{school_node.unique_id}_{subject_row['SubjectCode']}"
|
||||
# Create in a special "Unassigned" department
|
||||
unassigned_dept_name = "Unassigned Department"
|
||||
if unassigned_dept_name not in node_library['department_nodes']:
|
||||
_, dept_path = fs_handler.create_school_department_directory(school_node.path, unassigned_dept_name)
|
||||
department_node = neo_entity.DepartmentNode(
|
||||
unique_id=f"Department_{school_node.unique_id}_Unassigned",
|
||||
department_name=unassigned_dept_name,
|
||||
path=dept_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(department_node, database=school_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(department_node.path, department_node.to_dict())
|
||||
node_library['department_nodes'][unassigned_dept_name] = department_node
|
||||
|
||||
# Link unassigned department to department structure
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
ent_rels.DepartmentStructureHasDepartment(source=department_structure_node, target=department_node),
|
||||
database=school_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created unassigned department node and linked to department structure")
|
||||
|
||||
_, subject_path = fs_handler.create_department_subject_directory(
|
||||
node_library['department_nodes'][unassigned_dept_name].path,
|
||||
subject_row['Subject']
|
||||
)
|
||||
|
||||
subject_node = neo_curriculum.SubjectNode(
|
||||
unique_id=subject_unique_id,
|
||||
subject_code=subject_row['SubjectCode'],
|
||||
subject_name=subject_row['Subject'],
|
||||
path=subject_path
|
||||
)
|
||||
# Create subject in both databases
|
||||
neon.create_or_merge_neontology_node(subject_node, database=school_db_name, operation='merge')
|
||||
neon.create_or_merge_neontology_node(subject_node, database=curriculum_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(subject_node.path, subject_node.to_dict())
|
||||
node_library['subject_nodes'][subject_row['Subject']] = subject_node
|
||||
|
||||
# Link subject to unassigned department in school database only
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
ent_rels.DepartmentManagesSubject(
|
||||
source=node_library['department_nodes'][unassigned_dept_name],
|
||||
target=subject_node
|
||||
),
|
||||
database=school_db_name, operation='merge'
|
||||
)
|
||||
logger.warning(f"Created subject node for {subject_row['Subject']} in unassigned department")
|
||||
|
||||
# Process key stages and syllabuses
|
||||
logger.info(f"Processing key stages")
|
||||
last_key_stage_node = None
|
||||
# Track last syllabus nodes per subject
|
||||
last_key_stage_syllabus_nodes = {} # Dictionary to track last key stage syllabus node per subject
|
||||
last_year_group_syllabus_nodes = {} # Dictionary to track last year group syllabus node per subject
|
||||
topics_processed = set() # Track which topics have been processed
|
||||
lessons_processed = set() # Track which lessons have been processed
|
||||
statements_processed = set() # Track which statements have been processed
|
||||
|
||||
# First create all key stage nodes and key stage syllabus nodes
|
||||
for index, ks_row in keystagesyllabus_df.sort_values('KeyStage').iterrows():
|
||||
key_stage = str(ks_row['KeyStage'])
|
||||
logger.debug(f"Processing key stage syllabus row - Subject: {ks_row['Subject']}, Key Stage: {key_stage}")
|
||||
|
||||
subject_node = node_library['subject_nodes'].get(ks_row['Subject'])
|
||||
if not subject_node:
|
||||
logger.warning(f"No subject node found for subject {ks_row['Subject']}")
|
||||
continue
|
||||
|
||||
if key_stage not in key_stage_nodes_created:
|
||||
key_stage_node_unique_id = f"KeyStage_{curriculum_node.unique_id}_KStg{key_stage}"
|
||||
key_stage_node = neo_curriculum.KeyStageNode(
|
||||
unique_id=key_stage_node_unique_id,
|
||||
key_stage_name=f"Key Stage {key_stage}",
|
||||
key_stage=str(key_stage),
|
||||
path=os.path.join(curriculum_node.path, "key_stages", f"KS{key_stage}")
|
||||
)
|
||||
# Create key stage node in both databases
|
||||
neon.create_or_merge_neontology_node(key_stage_node, database=school_db_name, operation='merge')
|
||||
neon.create_or_merge_neontology_node(key_stage_node, database=curriculum_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(key_stage_node.path, key_stage_node.to_dict())
|
||||
key_stage_nodes_created[key_stage] = key_stage_node
|
||||
node_library['key_stage_nodes'][key_stage] = key_stage_node
|
||||
|
||||
# Create relationship with curriculum structure in school database only
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.CurriculumStructureIncludesKeyStage(source=curriculum_node, target=key_stage_node),
|
||||
database=school_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created key stage node {key_stage_node_unique_id} and relationship with curriculum structure")
|
||||
|
||||
# Create sequential relationship between key stages in both databases
|
||||
if last_key_stage_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.KeyStageFollowsKeyStage(source=last_key_stage_node, target=key_stage_node),
|
||||
database=school_db_name, operation='merge'
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.KeyStageFollowsKeyStage(source=last_key_stage_node, target=key_stage_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created sequential relationship between key stages {last_key_stage_node.unique_id} and {key_stage_node.unique_id}")
|
||||
last_key_stage_node = key_stage_node
|
||||
|
||||
# Create key stage syllabus under the subject's curriculum directory
|
||||
_, key_stage_syllabus_path = fs_handler.create_curriculum_key_stage_syllabus_directory(
|
||||
curriculum_node.path,
|
||||
key_stage,
|
||||
ks_row['Subject'],
|
||||
ks_row['ID']
|
||||
)
|
||||
logger.debug(f"Creating key stage syllabus node for {ks_row['Subject']} KS{key_stage} with ID {ks_row['ID']}")
|
||||
|
||||
key_stage_syllabus_node_unique_id = f"KeyStageSyllabus_{curriculum_node.unique_id}_{ks_row['Title'].replace(' ', '')}"
|
||||
key_stage_syllabus_node = neo_curriculum.KeyStageSyllabusNode(
|
||||
unique_id=key_stage_syllabus_node_unique_id,
|
||||
ks_syllabus_id=ks_row['ID'],
|
||||
ks_syllabus_name=ks_row['Title'],
|
||||
ks_syllabus_key_stage=str(ks_row['KeyStage']),
|
||||
ks_syllabus_subject=ks_row['Subject'],
|
||||
ks_syllabus_subject_code=ks_row['Subject'],
|
||||
path=key_stage_syllabus_path
|
||||
)
|
||||
# Create key stage syllabus node in both databases
|
||||
neon.create_or_merge_neontology_node(key_stage_syllabus_node, database=school_db_name, operation='merge')
|
||||
neon.create_or_merge_neontology_node(key_stage_syllabus_node, database=curriculum_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(key_stage_syllabus_node.path, key_stage_syllabus_node.to_dict())
|
||||
node_library['key_stage_syllabus_nodes'][ks_row['ID']] = key_stage_syllabus_node
|
||||
logger.debug(f"Created key stage syllabus node {key_stage_syllabus_node_unique_id} for {ks_row['Subject']} KS{key_stage}")
|
||||
|
||||
# Link key stage syllabus to its subject in both databases
|
||||
if subject_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.SubjectHasKeyStageSyllabus(source=subject_node, target=key_stage_syllabus_node),
|
||||
database=school_db_name, operation='merge'
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.SubjectHasKeyStageSyllabus(source=subject_node, target=key_stage_syllabus_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship between subject {subject_node.unique_id} and key stage syllabus {key_stage_syllabus_node.unique_id}")
|
||||
|
||||
# Link key stage syllabus to its key stage in both databases
|
||||
key_stage_node = key_stage_nodes_created.get(key_stage)
|
||||
if key_stage_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.KeyStageIncludesKeyStageSyllabus(source=key_stage_node, target=key_stage_syllabus_node),
|
||||
database=school_db_name, operation='merge'
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.KeyStageIncludesKeyStageSyllabus(source=key_stage_node, target=key_stage_syllabus_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship between key stage {key_stage_node.unique_id} and key stage syllabus {key_stage_syllabus_node.unique_id}")
|
||||
|
||||
# Create sequential relationship between key stage syllabuses in both databases
|
||||
last_key_stage_syllabus_node = last_key_stage_syllabus_nodes.get(ks_row['Subject'])
|
||||
if last_key_stage_syllabus_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.KeyStageSyllabusFollowsKeyStageSyllabus(source=last_key_stage_syllabus_node, target=key_stage_syllabus_node),
|
||||
database=school_db_name, operation='merge'
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.KeyStageSyllabusFollowsKeyStageSyllabus(source=last_key_stage_syllabus_node, target=key_stage_syllabus_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created sequential relationship between key stage syllabuses {last_key_stage_syllabus_node.unique_id} and {key_stage_syllabus_node.unique_id}")
|
||||
last_key_stage_syllabus_nodes[ks_row['Subject']] = key_stage_syllabus_node
|
||||
|
||||
# Now process year groups and their syllabuses
|
||||
for index, ks_row in keystagesyllabus_df.sort_values('KeyStage').iterrows():
|
||||
key_stage = str(ks_row['KeyStage'])
|
||||
related_yeargroups = sort_year_groups(yeargroupsyllabus_df[yeargroupsyllabus_df['KeyStage'] == ks_row['KeyStage']])
|
||||
|
||||
logger.info(f"Processing year groups for key stage {key_stage}")
|
||||
for yg_index, yg_row in related_yeargroups.iterrows():
|
||||
year_group = yg_row['YearGroup']
|
||||
subject_code = yg_row['SubjectCode']
|
||||
numeric_year_group = pd.to_numeric(year_group, errors='coerce')
|
||||
|
||||
if pd.notna(numeric_year_group):
|
||||
numeric_year_group = int(numeric_year_group)
|
||||
if numeric_year_group not in year_group_nodes_created:
|
||||
# Create year group directory under pastoral structure
|
||||
_, year_group_path = fs_handler.create_pastoral_year_group_directory(pastoral_node.path, year_group)
|
||||
logger.info(f"Created year group directory for {year_group_path}")
|
||||
|
||||
year_group_node_unique_id = f"YearGroup_{school_node.unique_id}_YGrp{numeric_year_group}"
|
||||
year_group_node = neo_curriculum.YearGroupNode(
|
||||
unique_id=year_group_node_unique_id,
|
||||
year_group=str(numeric_year_group),
|
||||
year_group_name=f"Year {numeric_year_group}",
|
||||
path=year_group_path
|
||||
)
|
||||
# Create year group node in both databases but use same directory
|
||||
neon.create_or_merge_neontology_node(year_group_node, database=school_db_name, operation='merge')
|
||||
neon.create_or_merge_neontology_node(year_group_node, database=curriculum_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(year_group_node.path, year_group_node.to_dict())
|
||||
|
||||
# Create sequential relationship between year groups in both databases
|
||||
if last_year_group_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.YearGroupFollowsYearGroup(source=last_year_group_node, target=year_group_node),
|
||||
database=school_db_name, operation='merge'
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.YearGroupFollowsYearGroup(source=last_year_group_node, target=year_group_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created sequential relationship between year groups {last_year_group_node.unique_id} and {year_group_node.unique_id} across key stages")
|
||||
last_year_group_node = year_group_node
|
||||
|
||||
# Create relationship with Pastoral Structure in school database only
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.PastoralStructureIncludesYearGroup(source=pastoral_node, target=year_group_node),
|
||||
database=school_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created year group node {year_group_node_unique_id} and relationship with pastoral structure")
|
||||
|
||||
year_group_nodes_created[numeric_year_group] = year_group_node
|
||||
node_library['year_group_nodes'][str(numeric_year_group)] = year_group_node
|
||||
|
||||
# Curriculum specific database initialisation begins here
|
||||
# Create year group syllabus nodes in both databases
|
||||
year_group_node = year_group_nodes_created.get(numeric_year_group)
|
||||
if year_group_node:
|
||||
# Create syllabus directory under curriculum structure
|
||||
_, year_group_syllabus_path = fs_handler.create_curriculum_year_group_syllabus_directory(
|
||||
curriculum_node.path,
|
||||
yg_row['Subject'],
|
||||
year_group,
|
||||
yg_row['ID']
|
||||
)
|
||||
logger.info(f"Created year group syllabus directory for {year_group_syllabus_path}")
|
||||
|
||||
year_group_syllabus_node_unique_id = f"YearGroupSyllabus_{school_node.unique_id}_{yg_row['ID']}"
|
||||
year_group_syllabus_node = neo_curriculum.YearGroupSyllabusNode(
|
||||
unique_id=year_group_syllabus_node_unique_id,
|
||||
yr_syllabus_id=yg_row['ID'],
|
||||
yr_syllabus_name=yg_row['Title'],
|
||||
yr_syllabus_year_group=str(yg_row['YearGroup']),
|
||||
yr_syllabus_subject=yg_row['Subject'],
|
||||
yr_syllabus_subject_code=yg_row['Subject'],
|
||||
path=year_group_syllabus_path
|
||||
)
|
||||
|
||||
# Create year group syllabus node in both databases but use same directory
|
||||
neon.create_or_merge_neontology_node(year_group_syllabus_node, database=school_db_name, operation='merge')
|
||||
neon.create_or_merge_neontology_node(year_group_syllabus_node, database=curriculum_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(year_group_syllabus_node.path, year_group_syllabus_node.to_dict())
|
||||
node_library['year_group_syllabus_nodes'][yg_row['ID']] = year_group_syllabus_node
|
||||
|
||||
# Create sequential relationship between year group syllabuses in both databases
|
||||
last_year_group_syllabus_node = last_year_group_syllabus_nodes.get(yg_row['Subject'])
|
||||
# Only create sequential relationship if this year group is higher than the last one
|
||||
if last_year_group_syllabus_node:
|
||||
last_year = pd.to_numeric(last_year_group_syllabus_node.yr_syllabus_year_group, errors='coerce')
|
||||
current_year = pd.to_numeric(year_group_syllabus_node.yr_syllabus_year_group, errors='coerce')
|
||||
if pd.notna(last_year) and pd.notna(current_year) and current_year > last_year:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.YearGroupSyllabusFollowsYearGroupSyllabus(source=last_year_group_syllabus_node, target=year_group_syllabus_node),
|
||||
database=school_db_name, operation='merge'
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.YearGroupSyllabusFollowsYearGroupSyllabus(source=last_year_group_syllabus_node, target=year_group_syllabus_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created sequential relationship between year group syllabuses {last_year_group_syllabus_node.unique_id} and {year_group_syllabus_node.unique_id}")
|
||||
last_year_group_syllabus_nodes[yg_row['Subject']] = year_group_syllabus_node
|
||||
|
||||
# Create relationships in both databases using MATCH to avoid cartesian products
|
||||
subject_node = node_library['subject_nodes'].get(yg_row['Subject'])
|
||||
if subject_node:
|
||||
# Link to subject
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.SubjectHasYearGroupSyllabus(source=subject_node, target=year_group_syllabus_node),
|
||||
database=school_db_name, operation='merge'
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.SubjectHasYearGroupSyllabus(source=subject_node, target=year_group_syllabus_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship between subject {subject_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
|
||||
|
||||
# Link to year group
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.YearGroupHasYearGroupSyllabus(source=year_group_node, target=year_group_syllabus_node),
|
||||
database=school_db_name, operation='merge'
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.YearGroupHasYearGroupSyllabus(source=year_group_node, target=year_group_syllabus_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship between year group {year_group_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
|
||||
|
||||
# Link to key stage syllabus if it exists for the same subject
|
||||
key_stage_syllabus_node = node_library['key_stage_syllabus_nodes'].get(ks_row['ID'])
|
||||
if key_stage_syllabus_node and yg_row['Subject'] == ks_row['Subject']:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.KeyStageSyllabusIncludesYearGroupSyllabus(source=key_stage_syllabus_node, target=year_group_syllabus_node),
|
||||
database=school_db_name, operation='merge'
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.KeyStageSyllabusIncludesYearGroupSyllabus(source=key_stage_syllabus_node, target=year_group_syllabus_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship between key stage syllabus {key_stage_syllabus_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
|
||||
|
||||
# Processing of curriculum topic begins here
|
||||
# Process topics for this year group syllabus only if not already processed
|
||||
topics_for_syllabus = topic_df[topic_df['SyllabusYearID'] == yg_row['ID']]
|
||||
for _, topic_row in topics_for_syllabus.iterrows():
|
||||
if topic_row['TopicID'] in topics_processed:
|
||||
continue
|
||||
topics_processed.add(topic_row['TopicID'])
|
||||
|
||||
# Get the correct subject from the topic row
|
||||
topic_subject = topic_row['SyllabusSubject']
|
||||
topic_key_stage = topic_row['SyllabusKeyStage']
|
||||
|
||||
logger.debug(f"Processing topic {topic_row['TopicID']} for subject {topic_subject} and key stage {topic_key_stage}")
|
||||
logger.debug(f"Available key stage syllabus nodes: {[node.ks_syllabus_subject + '_KS' + node.ks_syllabus_key_stage for node in node_library['key_stage_syllabus_nodes'].values()]}")
|
||||
|
||||
# Find the key stage syllabus node by iterating through all nodes
|
||||
matching_syllabus_node = None
|
||||
for syllabus_node in node_library['key_stage_syllabus_nodes'].values():
|
||||
logger.debug(f"Checking syllabus node - Subject: {syllabus_node.ks_syllabus_subject}, Key Stage: {syllabus_node.ks_syllabus_key_stage}")
|
||||
logger.debug(f"Comparing with - Subject: {topic_subject}, Key Stage: {str(topic_key_stage)}")
|
||||
logger.debug(f"Types - Node Subject: {type(syllabus_node.ks_syllabus_subject)}, Topic Subject: {type(topic_subject)}")
|
||||
logger.debug(f"Types - Node Key Stage: {type(syllabus_node.ks_syllabus_key_stage)}, Topic Key Stage: {type(str(topic_key_stage))}")
|
||||
|
||||
if (syllabus_node.ks_syllabus_subject == topic_subject and
|
||||
syllabus_node.ks_syllabus_key_stage == str(topic_key_stage)):
|
||||
matching_syllabus_node = syllabus_node
|
||||
logger.debug(f"Found matching syllabus node: {syllabus_node.unique_id}")
|
||||
break
|
||||
|
||||
if not matching_syllabus_node:
|
||||
logger.warning(f"No key stage syllabus node found for subject {topic_subject} and key stage {topic_key_stage}, skipping topic creation")
|
||||
continue
|
||||
|
||||
|
||||
_, topic_path = fs_handler.create_curriculum_topic_directory(matching_syllabus_node.path, topic_row['TopicID'])
|
||||
logger.info(f"Created topic directory for {topic_path}")
|
||||
|
||||
topic_node_unique_id = f"Topic_{matching_syllabus_node.unique_id}_{topic_row['TopicID']}"
|
||||
topic_node = neo_curriculum.TopicNode(
|
||||
unique_id=topic_node_unique_id,
|
||||
topic_id=topic_row['TopicID'],
|
||||
topic_title=topic_row.get('TopicTitle', default_topic_values['topic_title']),
|
||||
total_number_of_lessons_for_topic=str(topic_row.get('TotalNumberOfLessonsForTopic', default_topic_values['total_number_of_lessons_for_topic'])),
|
||||
topic_type=topic_row.get('TopicType', default_topic_values['topic_type']),
|
||||
topic_assessment_type=topic_row.get('TopicAssessmentType', default_topic_values['topic_assessment_type']),
|
||||
path=topic_path
|
||||
)
|
||||
# Create topic node in curriculum database only
|
||||
neon.create_or_merge_neontology_node(topic_node, database=curriculum_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(topic_node.path, topic_node.to_dict())
|
||||
node_library['topic_nodes'][topic_row['TopicID']] = topic_node
|
||||
|
||||
# Link topic to key stage syllabus as well as year group syllabus
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.KeyStageSyllabusIncludesTopic(source=matching_syllabus_node, target=topic_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.YearGroupSyllabusIncludesTopic(source=year_group_syllabus_node, target=topic_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationships between topic {topic_node_unique_id} and key stage syllabus {matching_syllabus_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
|
||||
|
||||
# Process lessons for this topic only if not already processed
|
||||
lessons_for_topic = lesson_df[
|
||||
(lesson_df['TopicID'] == topic_row['TopicID']) &
|
||||
(lesson_df['SyllabusSubject'] == topic_subject)
|
||||
].copy()
|
||||
lessons_for_topic.loc[:, 'Lesson'] = lessons_for_topic['Lesson'].astype(str)
|
||||
lessons_for_topic = lessons_for_topic.sort_values('Lesson')
|
||||
|
||||
previous_lesson_node = None
|
||||
for _, lesson_row in lessons_for_topic.iterrows():
|
||||
if lesson_row['LessonID'] in lessons_processed:
|
||||
continue
|
||||
lessons_processed.add(lesson_row['LessonID'])
|
||||
|
||||
_, lesson_path = fs_handler.create_curriculum_lesson_directory(topic_path, lesson_row['LessonID'])
|
||||
logger.info(f"Created lesson directory for {lesson_path}")
|
||||
|
||||
lesson_data = {
|
||||
'unique_id': f"TopicLesson_{topic_node_unique_id}_{lesson_row['LessonID']}",
|
||||
'topic_lesson_id': lesson_row['LessonID'],
|
||||
'topic_lesson_title': lesson_row.get('LessonTitle', default_topic_lesson_values['topic_lesson_title']),
|
||||
'topic_lesson_type': lesson_row.get('LessonType', default_topic_lesson_values['topic_lesson_type']),
|
||||
'topic_lesson_length': str(lesson_row.get('SuggestedNumberOfPeriodsForLesson', default_topic_lesson_values['topic_lesson_length'])),
|
||||
'topic_lesson_suggested_activities': lesson_row.get('SuggestedActivities', default_topic_lesson_values['topic_lesson_suggested_activities']),
|
||||
'topic_lesson_skills_learned': lesson_row.get('SkillsLearned', default_topic_lesson_values['topic_lesson_skills_learned']),
|
||||
'topic_lesson_weblinks': lesson_row.get('WebLinks', default_topic_lesson_values['topic_lesson_weblinks']),
|
||||
'path': lesson_path
|
||||
}
|
||||
for key, value in lesson_data.items():
|
||||
if pd.isna(value):
|
||||
lesson_data[key] = default_topic_lesson_values.get(key, 'Null')
|
||||
|
||||
lesson_node = neo_curriculum.TopicLessonNode(**lesson_data)
|
||||
# Create lesson node in curriculum database only
|
||||
neon.create_or_merge_neontology_node(lesson_node, database=curriculum_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(lesson_node.path, lesson_node.to_dict())
|
||||
node_library['topic_lesson_nodes'][lesson_row['LessonID']] = lesson_node
|
||||
|
||||
# Link lesson to topic
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.TopicIncludesTopicLesson(source=topic_node, target=lesson_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created lesson node {lesson_node.unique_id} and relationship with topic {topic_node.unique_id}")
|
||||
|
||||
# Create sequential relationships between lessons
|
||||
if lesson_row['Lesson'].isdigit() and previous_lesson_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.TopicLessonFollowsTopicLesson(source=previous_lesson_node, target=lesson_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.unique_id} and {lesson_node.unique_id}")
|
||||
previous_lesson_node = lesson_node
|
||||
|
||||
# Process learning statements for this lesson only if not already processed
|
||||
statements_for_lesson = statement_df[
|
||||
(statement_df['LessonID'] == lesson_row['LessonID']) &
|
||||
(statement_df['SyllabusSubject'] == topic_subject)
|
||||
]
|
||||
for _, statement_row in statements_for_lesson.iterrows():
|
||||
if statement_row['StatementID'] in statements_processed:
|
||||
continue
|
||||
statements_processed.add(statement_row['StatementID'])
|
||||
|
||||
_, statement_path = fs_handler.create_curriculum_learning_statement_directory(lesson_path, statement_row['StatementID'])
|
||||
|
||||
statement_data = {
|
||||
'unique_id': f"LearningStatement_{lesson_node.unique_id}_{statement_row['StatementID']}",
|
||||
'lesson_learning_statement_id': statement_row['StatementID'],
|
||||
'lesson_learning_statement': statement_row.get('LearningStatement', default_learning_statement_values['lesson_learning_statement']),
|
||||
'lesson_learning_statement_type': statement_row.get('StatementType', default_learning_statement_values['lesson_learning_statement_type']),
|
||||
'path': statement_path
|
||||
}
|
||||
for key in statement_data:
|
||||
if pd.isna(statement_data[key]):
|
||||
statement_data[key] = default_learning_statement_values.get(key, 'Null')
|
||||
|
||||
statement_node = neo_curriculum.LearningStatementNode(**statement_data)
|
||||
# Create statement node in curriculum database only
|
||||
neon.create_or_merge_neontology_node(statement_node, database=curriculum_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(statement_node.path, statement_node.to_dict())
|
||||
node_library['statement_nodes'][statement_row['StatementID']] = statement_node
|
||||
|
||||
# Link learning statement to lesson
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.LessonIncludesLearningStatement(source=lesson_node, target=statement_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created learning statement node {statement_node.unique_id} and relationship with lesson {lesson_node.unique_id}")
|
||||
else:
|
||||
logger.warning(f"No year group node found for year group {year_group}, skipping syllabus creation")
|
||||
|
||||
# After processing all year groups and their syllabuses, process any remaining topics
|
||||
logger.info("Processing topics without year groups")
|
||||
for _, topic_row in topic_df.iterrows():
|
||||
if topic_row['TopicID'] in topics_processed:
|
||||
continue
|
||||
|
||||
topic_subject = topic_row['SyllabusSubject']
|
||||
topic_key_stage = topic_row['SyllabusKeyStage']
|
||||
|
||||
logger.debug(f"Processing topic {topic_row['TopicID']} for subject {topic_subject} and key stage {topic_key_stage} without year group")
|
||||
|
||||
# Find the key stage syllabus node
|
||||
matching_syllabus_node = None
|
||||
for syllabus_node in node_library['key_stage_syllabus_nodes'].values():
|
||||
if (syllabus_node.ks_syllabus_subject == topic_subject and
|
||||
syllabus_node.ks_syllabus_key_stage == str(topic_key_stage)):
|
||||
matching_syllabus_node = syllabus_node
|
||||
break
|
||||
|
||||
if not matching_syllabus_node:
|
||||
logger.warning(f"No key stage syllabus node found for subject {topic_subject} and key stage {topic_key_stage}, skipping topic creation")
|
||||
continue
|
||||
|
||||
_, topic_path = fs_handler.create_curriculum_topic_directory(matching_syllabus_node.path, topic_row['TopicID'])
|
||||
logger.info(f"Created topic directory for {topic_path}")
|
||||
|
||||
topic_node_unique_id = f"Topic_{matching_syllabus_node.unique_id}_{topic_row['TopicID']}"
|
||||
topic_node = neo_curriculum.TopicNode(
|
||||
unique_id=topic_node_unique_id,
|
||||
topic_id=topic_row['TopicID'],
|
||||
topic_title=topic_row.get('TopicTitle', default_topic_values['topic_title']),
|
||||
total_number_of_lessons_for_topic=str(topic_row.get('TotalNumberOfLessonsForTopic', default_topic_values['total_number_of_lessons_for_topic'])),
|
||||
topic_type=topic_row.get('TopicType', default_topic_values['topic_type']),
|
||||
topic_assessment_type=topic_row.get('TopicAssessmentType', default_topic_values['topic_assessment_type']),
|
||||
path=topic_path
|
||||
)
|
||||
# Create topic node in curriculum database only
|
||||
neon.create_or_merge_neontology_node(topic_node, database=curriculum_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(topic_node.path, topic_node.to_dict())
|
||||
node_library['topic_nodes'][topic_row['TopicID']] = topic_node
|
||||
topics_processed.add(topic_row['TopicID'])
|
||||
|
||||
# Link topic to key stage syllabus
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.KeyStageSyllabusIncludesTopic(source=matching_syllabus_node, target=topic_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship between topic {topic_node_unique_id} and key stage syllabus {matching_syllabus_node.unique_id}")
|
||||
|
||||
# Process lessons for this topic
|
||||
lessons_for_topic = lesson_df[
|
||||
(lesson_df['TopicID'] == topic_row['TopicID']) &
|
||||
(lesson_df['SyllabusSubject'] == topic_subject)
|
||||
].copy()
|
||||
lessons_for_topic.loc[:, 'Lesson'] = lessons_for_topic['Lesson'].astype(str)
|
||||
lessons_for_topic = lessons_for_topic.sort_values('Lesson')
|
||||
|
||||
previous_lesson_node = None
|
||||
for _, lesson_row in lessons_for_topic.iterrows():
|
||||
if lesson_row['LessonID'] in lessons_processed:
|
||||
continue
|
||||
lessons_processed.add(lesson_row['LessonID'])
|
||||
|
||||
_, lesson_path = fs_handler.create_curriculum_lesson_directory(topic_path, lesson_row['LessonID'])
|
||||
logger.info(f"Created lesson directory for {lesson_path}")
|
||||
|
||||
lesson_data = {
|
||||
'unique_id': f"TopicLesson_{topic_node_unique_id}_{lesson_row['LessonID']}",
|
||||
'topic_lesson_id': lesson_row['LessonID'],
|
||||
'topic_lesson_title': lesson_row.get('LessonTitle', default_topic_lesson_values['topic_lesson_title']),
|
||||
'topic_lesson_type': lesson_row.get('LessonType', default_topic_lesson_values['topic_lesson_type']),
|
||||
'topic_lesson_length': str(lesson_row.get('SuggestedNumberOfPeriodsForLesson', default_topic_lesson_values['topic_lesson_length'])),
|
||||
'topic_lesson_suggested_activities': lesson_row.get('SuggestedActivities', default_topic_lesson_values['topic_lesson_suggested_activities']),
|
||||
'topic_lesson_skills_learned': lesson_row.get('SkillsLearned', default_topic_lesson_values['topic_lesson_skills_learned']),
|
||||
'topic_lesson_weblinks': lesson_row.get('WebLinks', default_topic_lesson_values['topic_lesson_weblinks']),
|
||||
'path': lesson_path
|
||||
}
|
||||
for key, value in lesson_data.items():
|
||||
if pd.isna(value):
|
||||
lesson_data[key] = default_topic_lesson_values.get(key, 'Null')
|
||||
|
||||
lesson_node = neo_curriculum.TopicLessonNode(**lesson_data)
|
||||
# Create lesson node in curriculum database only
|
||||
neon.create_or_merge_neontology_node(lesson_node, database=curriculum_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(lesson_node.path, lesson_node.to_dict())
|
||||
node_library['topic_lesson_nodes'][lesson_row['LessonID']] = lesson_node
|
||||
|
||||
# Link lesson to topic
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.TopicIncludesTopicLesson(source=topic_node, target=lesson_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created lesson node {lesson_node.unique_id} and relationship with topic {topic_node.unique_id}")
|
||||
|
||||
# Create sequential relationships between lessons
|
||||
if lesson_row['Lesson'].isdigit() and previous_lesson_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.TopicLessonFollowsTopicLesson(source=previous_lesson_node, target=lesson_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.unique_id} and {lesson_node.unique_id}")
|
||||
previous_lesson_node = lesson_node
|
||||
|
||||
# Process learning statements for this lesson
|
||||
statements_for_lesson = statement_df[
|
||||
(statement_df['LessonID'] == lesson_row['LessonID']) &
|
||||
(statement_df['SyllabusSubject'] == topic_subject)
|
||||
]
|
||||
for _, statement_row in statements_for_lesson.iterrows():
|
||||
if statement_row['StatementID'] in statements_processed:
|
||||
continue
|
||||
statements_processed.add(statement_row['StatementID'])
|
||||
|
||||
_, statement_path = fs_handler.create_curriculum_learning_statement_directory(lesson_path, statement_row['StatementID'])
|
||||
|
||||
statement_data = {
|
||||
'unique_id': f"LearningStatement_{lesson_node.unique_id}_{statement_row['StatementID']}",
|
||||
'lesson_learning_statement_id': statement_row['StatementID'],
|
||||
'lesson_learning_statement': statement_row.get('LearningStatement', default_learning_statement_values['lesson_learning_statement']),
|
||||
'lesson_learning_statement_type': statement_row.get('StatementType', default_learning_statement_values['lesson_learning_statement_type']),
|
||||
'path': statement_path
|
||||
}
|
||||
for key in statement_data:
|
||||
if pd.isna(statement_data[key]):
|
||||
statement_data[key] = default_learning_statement_values.get(key, 'Null')
|
||||
|
||||
statement_node = neo_curriculum.LearningStatementNode(**statement_data)
|
||||
# Create statement node in curriculum database only
|
||||
neon.create_or_merge_neontology_node(statement_node, database=curriculum_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(statement_node.path, statement_node.to_dict())
|
||||
node_library['statement_nodes'][statement_row['StatementID']] = statement_node
|
||||
|
||||
# Link learning statement to lesson
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
curriculum_relationships.LessonIncludesLearningStatement(source=lesson_node, target=statement_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created learning statement node {statement_node.unique_id} and relationship with lesson {lesson_node.unique_id}")
|
||||
|
||||
return node_library
|
||||
@@ -1,512 +0,0 @@
|
||||
import os
|
||||
from modules.logger_tool import initialise_logger
|
||||
from supabase import create_client
|
||||
import json
|
||||
import pandas as pd
|
||||
|
||||
import modules.database.init.xl_tools as xl
|
||||
from modules.database.tools.filesystem_tools import ClassroomCopilotFilesystem
|
||||
import modules.database.tools.neo4j_driver_tools as driver_tools
|
||||
import modules.database.tools.neo4j_session_tools as session_tools
|
||||
import modules.database.schemas.nodes.schools.schools as school_schemas
|
||||
import modules.database.schemas.nodes.schools.curriculum as curriculum_schemas
|
||||
import modules.database.schemas.nodes.schools.pastoral as pastoral_schemas
|
||||
import modules.database.schemas.nodes.structures.schools as school_structures
|
||||
import modules.database.schemas.entities as entities
|
||||
from modules.database.admin.neontology_provider import NeontologyProvider
|
||||
from modules.database.admin.graph_provider import GraphNamingProvider
|
||||
from modules.database.schemas.relationships import curriculum_relationships, entity_relationships, entity_curriculum_rels
|
||||
|
||||
class SchoolManager:
|
||||
def __init__(self):
|
||||
self.driver = driver_tools.get_driver()
|
||||
self.logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
self.neontology = NeontologyProvider()
|
||||
self.graph_naming = GraphNamingProvider()
|
||||
|
||||
# Initialize Supabase client with correct URL and service role key
|
||||
supabase_url = os.getenv("SUPABASE_URL")
|
||||
service_role_key = os.getenv("SERVICE_ROLE_KEY")
|
||||
|
||||
self.logger.info(f"Initializing Supabase client with URL: {supabase_url}")
|
||||
self.supabase = create_client(supabase_url, service_role_key)
|
||||
|
||||
# Set headers for admin operations
|
||||
self.supabase.headers = {
|
||||
"apiKey": service_role_key,
|
||||
"Authorization": f"Bearer {service_role_key}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
# Set storage client headers explicitly
|
||||
self.supabase.storage._client.headers.update({
|
||||
"apiKey": service_role_key,
|
||||
"Authorization": f"Bearer {service_role_key}",
|
||||
"Content-Type": "application/json"
|
||||
})
|
||||
|
||||
def create_schools_database(self):
|
||||
"""Creates the main cc.institutes database in Neo4j"""
|
||||
try:
|
||||
db_name = "cc.institutes"
|
||||
with self.driver.session() as session:
|
||||
return self._extracted_from_create_private_database(
|
||||
session, db_name, f'Created database {db_name}'
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error creating schools database: {str(e)}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
def create_school_node(self, school_data):
|
||||
"""Creates a school node in cc.institutes database and stores TLDraw file in Supabase"""
|
||||
try:
|
||||
# Convert Supabase school data to SchoolNode using GraphNamingProvider
|
||||
school_unique_id = self.graph_naming.get_school_unique_id(school_data['urn'])
|
||||
school_path = self.graph_naming.get_school_path("cc.institutes", school_data['urn'])
|
||||
|
||||
school_node = entities.school_schemas.SchoolNode(
|
||||
unique_id=school_unique_id,
|
||||
path=school_path,
|
||||
urn=school_data['urn'],
|
||||
establishment_number=school_data['establishment_number'],
|
||||
establishment_name=school_data['establishment_name'],
|
||||
establishment_type=school_data['establishment_type'],
|
||||
establishment_status=school_data['establishment_status'],
|
||||
phase_of_education=school_data['phase_of_education'] if school_data['phase_of_education'] not in [None, ''] else None,
|
||||
statutory_low_age=int(school_data['statutory_low_age']) if school_data.get('statutory_low_age') is not None else 0,
|
||||
statutory_high_age=int(school_data['statutory_high_age']) if school_data.get('statutory_high_age') is not None else 0,
|
||||
religious_character=school_data.get('religious_character') if school_data.get('religious_character') not in [None, ''] else None,
|
||||
school_capacity=int(school_data['school_capacity']) if school_data.get('school_capacity') is not None else 0,
|
||||
school_website=school_data.get('school_website', ''),
|
||||
ofsted_rating=school_data.get('ofsted_rating') if school_data.get('ofsted_rating') not in [None, ''] else None
|
||||
)
|
||||
|
||||
# Create default tldraw file data
|
||||
tldraw_data = {
|
||||
"document": {
|
||||
"version": 1,
|
||||
"id": school_data['urn'],
|
||||
"name": school_data['establishment_name'],
|
||||
"meta": {
|
||||
"created_at": "",
|
||||
"updated_at": "",
|
||||
"creator_id": "",
|
||||
"is_template": False,
|
||||
"is_snapshot": False,
|
||||
"is_draft": False,
|
||||
"template_id": None,
|
||||
"snapshot_id": None,
|
||||
"draft_id": None
|
||||
}
|
||||
},
|
||||
"schema": {
|
||||
"schemaVersion": 1,
|
||||
"storeVersion": 4,
|
||||
"recordVersions": {
|
||||
"asset": {
|
||||
"version": 1,
|
||||
"subTypeKey": "type",
|
||||
"subTypeVersions": {}
|
||||
},
|
||||
"camera": {
|
||||
"version": 1
|
||||
},
|
||||
"document": {
|
||||
"version": 2
|
||||
},
|
||||
"instance": {
|
||||
"version": 22
|
||||
},
|
||||
"instance_page_state": {
|
||||
"version": 5
|
||||
},
|
||||
"page": {
|
||||
"version": 1
|
||||
},
|
||||
"shape": {
|
||||
"version": 3,
|
||||
"subTypeKey": "type",
|
||||
"subTypeVersions": {
|
||||
"cc-school-node": 1
|
||||
}
|
||||
},
|
||||
"instance_presence": {
|
||||
"version": 5
|
||||
},
|
||||
"pointer": {
|
||||
"version": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"store": {
|
||||
"document:document": {
|
||||
"gridSize": 10,
|
||||
"name": school_data['establishment_name'],
|
||||
"meta": {},
|
||||
"id": school_data['urn'],
|
||||
"typeName": "document"
|
||||
},
|
||||
"page:page": {
|
||||
"meta": {},
|
||||
"id": "page",
|
||||
"name": "Page 1",
|
||||
"index": "a1",
|
||||
"typeName": "page"
|
||||
},
|
||||
"shape:school-node": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"rotation": 0,
|
||||
"type": "cc-school-node",
|
||||
"id": school_unique_id,
|
||||
"parentId": "page",
|
||||
"index": "a1",
|
||||
"props": school_node.to_dict(),
|
||||
"typeName": "shape"
|
||||
},
|
||||
"instance:instance": {
|
||||
"id": "instance",
|
||||
"currentPageId": "page",
|
||||
"typeName": "instance"
|
||||
},
|
||||
"camera:camera": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"z": 1,
|
||||
"id": "camera",
|
||||
"typeName": "camera"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Store tldraw file in Supabase storage
|
||||
file_path = f"{school_data['urn']}/tldraw.json"
|
||||
file_options = {
|
||||
"content-type": "application/json",
|
||||
"x-upsert": "true", # Update if exists
|
||||
"metadata": {
|
||||
"establishment_urn": school_data['urn'],
|
||||
"establishment_name": school_data['establishment_name']
|
||||
}
|
||||
}
|
||||
|
||||
try:
|
||||
# Create a fresh service role client for storage operations
|
||||
self.logger.info("Creating fresh service role client for storage operations")
|
||||
service_client = create_client(
|
||||
os.getenv("SUPABASE_URL"),
|
||||
os.getenv("SERVICE_ROLE_KEY")
|
||||
)
|
||||
|
||||
self.logger.debug(f"Service client created with URL: {os.getenv('SUPABASE_URL')}")
|
||||
|
||||
service_client.headers = {
|
||||
"apiKey": os.getenv("SERVICE_ROLE_KEY"),
|
||||
"Authorization": f"Bearer {os.getenv('SERVICE_ROLE_KEY')}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
service_client.storage._client.headers.update({
|
||||
"apiKey": os.getenv("SERVICE_ROLE_KEY"),
|
||||
"Authorization": f"Bearer {os.getenv('SERVICE_ROLE_KEY')}",
|
||||
"Content-Type": "application/json"
|
||||
})
|
||||
|
||||
self.logger.debug("Headers set for service client and storage client")
|
||||
|
||||
# Upload to Supabase storage using service role client
|
||||
self.logger.info(f"Uploading tldraw file for school {school_data['urn']}")
|
||||
self.logger.debug(f"File path: {file_path}")
|
||||
self.logger.debug(f"File options: {file_options}")
|
||||
|
||||
# First, ensure the bucket exists
|
||||
self.logger.info("Checking if bucket cc.institutes exists")
|
||||
try:
|
||||
bucket = service_client.storage.get_bucket("cc.institutes")
|
||||
self.logger.info("Bucket cc.institutes exists")
|
||||
except Exception as bucket_error:
|
||||
self.logger.error(f"Error checking bucket: {str(bucket_error)}")
|
||||
if hasattr(bucket_error, 'response'):
|
||||
self.logger.error(f"Bucket error response: {bucket_error.response.text if hasattr(bucket_error.response, 'text') else bucket_error.response}")
|
||||
raise bucket_error
|
||||
|
||||
# Attempt the upload
|
||||
self.logger.info("Attempting file upload")
|
||||
result = service_client.storage.from_("cc.institutes").upload(
|
||||
path=file_path,
|
||||
file=json.dumps(tldraw_data).encode(),
|
||||
file_options=file_options
|
||||
)
|
||||
self.logger.info(f"Upload successful. Result: {result}")
|
||||
|
||||
except Exception as upload_error:
|
||||
self.logger.error(f"Error uploading tldraw file: {str(upload_error)}")
|
||||
if hasattr(upload_error, 'response'):
|
||||
self.logger.error(f"Upload error response: {upload_error.response.text if hasattr(upload_error.response, 'text') else upload_error.response}")
|
||||
raise upload_error
|
||||
|
||||
# Create node in Neo4j using Neontology
|
||||
with self.neontology as neo:
|
||||
self.logger.info(f"Creating school node in Neo4j: {school_node.to_dict()}")
|
||||
neo.create_or_merge_node(school_node, database="cc.institutes", operation="merge")
|
||||
return {"status": "success", "node": school_node}
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error creating school node: {str(e)}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
def create_private_database(self, school_data):
|
||||
"""Creates a private database for a specific school"""
|
||||
try:
|
||||
private_db_name = f"cc.institutes.{school_data['urn']}"
|
||||
with self.driver.session() as session:
|
||||
return self._extracted_from_create_private_database(
|
||||
session, private_db_name, 'Created private database '
|
||||
)
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error creating private database: {str(e)}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
# TODO Rename this here and in `create_schools_database` and `create_private_database`
|
||||
def _extracted_from_create_private_database(self, session, arg1, arg2):
|
||||
session_tools.create_database(session, arg1)
|
||||
self.logger.info(f"{arg2}{arg1}")
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"Database {arg1} created successfully",
|
||||
}
|
||||
|
||||
def create_basic_structure(self, school_node, database_name):
|
||||
"""Creates basic structural nodes in the specified database"""
|
||||
try:
|
||||
# Create filesystem paths
|
||||
fs_handler = ClassroomCopilotFilesystem(database_name, init_run_type="school")
|
||||
|
||||
# Create Department Structure node
|
||||
department_structure_node_unique_id = f"DepartmentStructure_{school_node.unique_id}"
|
||||
_, department_path = fs_handler.create_school_department_directory(school_node.path, "departments")
|
||||
department_structure_node = entities.school_schemas.DepartmentNode(
|
||||
unique_id=department_structure_node_unique_id,
|
||||
path=department_path
|
||||
)
|
||||
|
||||
# Create Curriculum Structure node
|
||||
_, curriculum_path = fs_handler.create_school_curriculum_directory(school_node.path)
|
||||
curriculum_node = school_structures.CurriculumStructureNode(
|
||||
unique_id=f"CurriculumStructure_{school_node.unique_id}",
|
||||
path=curriculum_path
|
||||
)
|
||||
|
||||
# Create Pastoral Structure node
|
||||
_, pastoral_path = fs_handler.create_school_pastoral_directory(school_node.path)
|
||||
pastoral_node = school_structures.PastoralStructureNode(
|
||||
unique_id=f"PastoralStructure_{school_node.unique_id}",
|
||||
path=pastoral_path
|
||||
)
|
||||
|
||||
with self.neontology as neo:
|
||||
# Create nodes
|
||||
neo.create_or_merge_node(department_structure_node, database=str(database_name), operation='merge')
|
||||
fs_handler.create_default_tldraw_file(department_structure_node.path, department_structure_node.to_dict())
|
||||
|
||||
neo.create_or_merge_node(curriculum_node, database=str(database_name), operation='merge')
|
||||
fs_handler.create_default_tldraw_file(curriculum_node.path, curriculum_node.to_dict())
|
||||
|
||||
neo.create_or_merge_node(pastoral_node, database=database_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(pastoral_node.path, pastoral_node.to_dict())
|
||||
|
||||
# Create relationships
|
||||
neo.create_or_merge_relationship(
|
||||
entity_relationships.SchoolHasDepartmentStructure(source=school_node, target=department_structure_node),
|
||||
database=database_name, operation='merge'
|
||||
)
|
||||
|
||||
neo.create_or_merge_relationship(
|
||||
entity_curriculum_rels.SchoolHasCurriculumStructure(source=school_node, target=curriculum_node),
|
||||
database=database_name, operation='merge'
|
||||
)
|
||||
|
||||
neo.create_or_merge_relationship(
|
||||
entity_curriculum_rels.SchoolHasPastoralStructure(source=school_node, target=pastoral_node),
|
||||
database=database_name, operation='merge'
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Basic structure created successfully",
|
||||
"nodes": {
|
||||
"department_structure": department_structure_node,
|
||||
"curriculum_structure": curriculum_node,
|
||||
"pastoral_structure": pastoral_node
|
||||
}
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error creating basic structure: {str(e)}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
def create_detailed_structure(self, school_node, database_name, excel_file):
|
||||
"""Creates detailed structural nodes from Excel file"""
|
||||
try:
|
||||
# First, store the Excel file in Supabase
|
||||
file_path = f"{school_node.urn}/structure.xlsx"
|
||||
file_options = {
|
||||
"content-type": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
"x-upsert": "true"
|
||||
}
|
||||
|
||||
# Upload Excel file to storage
|
||||
self.supabase.storage.from_("cc.institutes").upload(
|
||||
path=file_path,
|
||||
file=excel_file,
|
||||
file_options=file_options
|
||||
)
|
||||
|
||||
# Process Excel file
|
||||
dataframes = xl.create_dataframes(excel_file)
|
||||
|
||||
# Get existing basic structure nodes
|
||||
with self.neontology as neo:
|
||||
result = neo.cypher_read("""
|
||||
MATCH (s:School {unique_id: $school_id})
|
||||
OPTIONAL MATCH (s)-[:HAS_DEPARTMENT_STRUCTURE]->(ds:DepartmentStructure)
|
||||
OPTIONAL MATCH (s)-[:HAS_CURRICULUM_STRUCTURE]->(cs:CurriculumStructure)
|
||||
OPTIONAL MATCH (s)-[:HAS_PASTORAL_STRUCTURE]->(ps:PastoralStructure)
|
||||
RETURN ds, cs, ps
|
||||
""", {"school_id": school_node.unique_id}, database=database_name)
|
||||
|
||||
if not result:
|
||||
raise Exception("Basic structure not found")
|
||||
|
||||
department_structure = result['ds']
|
||||
curriculum_structure = result['cs']
|
||||
pastoral_structure = result['ps']
|
||||
|
||||
# Create departments and subjects
|
||||
unique_departments = dataframes['keystagesyllabuses']['Department'].dropna().unique()
|
||||
|
||||
fs_handler = ClassroomCopilotFilesystem(database_name, init_run_type="school")
|
||||
node_library = {}
|
||||
|
||||
with self.neontology as neo:
|
||||
for department_name in unique_departments:
|
||||
_, department_path = fs_handler.create_school_department_directory(school_node.path, department_name)
|
||||
|
||||
department_node = school_schemas.DepartmentNode(
|
||||
unique_id=f"Department_{school_node.unique_id}_{department_name.replace(' ', '_')}",
|
||||
department_name=department_name,
|
||||
path=department_path
|
||||
)
|
||||
neo.create_or_merge_node(department_node, database=database_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(department_node.path, department_node.to_dict())
|
||||
node_library[f'department_{department_name}'] = department_node
|
||||
|
||||
# Link to department structure
|
||||
neo.create_or_merge_relationship(
|
||||
entity_relationships.DepartmentStructureHasDepartment(
|
||||
source=department_structure,
|
||||
target=department_node
|
||||
),
|
||||
database=database_name,
|
||||
operation='merge'
|
||||
)
|
||||
|
||||
# Create year groups
|
||||
year_groups = self.sort_year_groups(dataframes['yeargroupsyllabuses'])['YearGroup'].unique()
|
||||
last_year_group_node = None
|
||||
|
||||
for year_group in year_groups:
|
||||
numeric_year_group = pd.to_numeric(year_group, errors='coerce')
|
||||
if pd.notna(numeric_year_group):
|
||||
_, year_group_path = fs_handler.create_pastoral_year_group_directory(
|
||||
pastoral_structure.path,
|
||||
str(int(numeric_year_group))
|
||||
)
|
||||
|
||||
year_group_node = pastoral_schemas.YearGroupNode(
|
||||
unique_id=f"YearGroup_{school_node.unique_id}_YGrp{int(numeric_year_group)}",
|
||||
year_group=str(int(numeric_year_group)),
|
||||
year_group_name=f"Year {int(numeric_year_group)}",
|
||||
path=year_group_path
|
||||
)
|
||||
neo.create_or_merge_node(year_group_node, database=database_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(year_group_node.path, year_group_node.to_dict())
|
||||
node_library[f'year_group_{int(numeric_year_group)}'] = year_group_node
|
||||
|
||||
# Create sequential relationship
|
||||
if last_year_group_node:
|
||||
neo.create_or_merge_relationship(
|
||||
curriculum_relationships.YearGroupFollowsYearGroup(
|
||||
source=last_year_group_node,
|
||||
target=year_group_node
|
||||
),
|
||||
database=database_name,
|
||||
operation='merge'
|
||||
)
|
||||
last_year_group_node = year_group_node
|
||||
|
||||
# Link to pastoral structure
|
||||
neo.create_or_merge_relationship(
|
||||
curriculum_relationships.PastoralStructureIncludesYearGroup(
|
||||
source=pastoral_structure,
|
||||
target=year_group_node
|
||||
),
|
||||
database=database_name,
|
||||
operation='merge'
|
||||
)
|
||||
|
||||
# Create key stages
|
||||
key_stages = dataframes['keystagesyllabuses']['KeyStage'].unique()
|
||||
last_key_stage_node = None
|
||||
|
||||
for key_stage in sorted(key_stages):
|
||||
_, key_stage_path = fs_handler.create_curriculum_key_stage_directory(
|
||||
curriculum_structure.path,
|
||||
str(key_stage)
|
||||
)
|
||||
|
||||
key_stage_node = curriculum_schemas.KeyStageNode(
|
||||
unique_id=f"KeyStage_{curriculum_structure.unique_id}_KStg{key_stage}",
|
||||
key_stage_name=f"Key Stage {key_stage}",
|
||||
key_stage=str(key_stage),
|
||||
path=key_stage_path
|
||||
)
|
||||
neo.create_or_merge_node(key_stage_node, database=database_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(key_stage_node.path, key_stage_node.to_dict())
|
||||
node_library[f'key_stage_{key_stage}'] = key_stage_node
|
||||
|
||||
# Create sequential relationship
|
||||
if last_key_stage_node:
|
||||
neo.create_or_merge_relationship(
|
||||
curriculum_relationships.KeyStageFollowsKeyStage(
|
||||
source=last_key_stage_node,
|
||||
target=key_stage_node
|
||||
),
|
||||
database=database_name,
|
||||
operation='merge'
|
||||
)
|
||||
last_key_stage_node = key_stage_node
|
||||
|
||||
# Link to curriculum structure
|
||||
neo.create_or_merge_relationship(
|
||||
curriculum_relationships.CurriculumStructureIncludesKeyStage(
|
||||
source=curriculum_structure,
|
||||
target=key_stage_node
|
||||
),
|
||||
database=database_name,
|
||||
operation='merge'
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Detailed structure created successfully",
|
||||
"node_library": node_library
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error creating detailed structure: {str(e)}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
def sort_year_groups(self, df):
|
||||
df = df.copy()
|
||||
df['YearGroupNumeric'] = pd.to_numeric(df['YearGroup'], errors='coerce')
|
||||
return df.sort_values(by='YearGroupNumeric')
|
||||
@@ -1,54 +0,0 @@
|
||||
import os
|
||||
from modules.logger_tool import initialise_logger
|
||||
import modules.database.tools.neo4j_driver_tools as driver_tools
|
||||
import modules.database.tools.neo4j_session_tools as session_tools
|
||||
import modules.database.tools.neontology_tools as neon
|
||||
import modules.database.schemas.entities as neo_entity
|
||||
import modules.database.schemas.nodes.schools.curriculum as curriculum_schemas
|
||||
import modules.database.schemas.relationships.curriculum_relationships as curriculum_relationships
|
||||
import modules.database.schemas.relationships.entity_relationships as ent_rels
|
||||
import modules.database.schemas.relationships.entity_curriculum_rels as ent_cur_rels
|
||||
import pandas as pd
|
||||
|
||||
class SchoolSyllabusProvider:
|
||||
def __init__(self):
|
||||
self.driver = driver_tools.get_driver()
|
||||
self.logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
|
||||
def process_syllabus_data(self, school_node, database_name, dataframes):
|
||||
"""Process syllabus data from Excel file and create nodes in the database"""
|
||||
try:
|
||||
# This method will contain the syllabus-specific processing code from the
|
||||
# original SchoolCurriculumProvider, starting from where the comment
|
||||
# "# Curriculum specific database initialisation begins here" was placed
|
||||
|
||||
# We'll implement this in the next iteration after confirming the basic
|
||||
# structure changes work correctly
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Syllabus data processed successfully"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error processing syllabus data: {str(e)}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
def check_syllabus_status(self, school_node, database_name):
|
||||
"""Check if syllabus data exists in the database"""
|
||||
try:
|
||||
with self.driver.session(database=database_name) as session:
|
||||
result = session.run("""
|
||||
MATCH (s:School {unique_id: $school_id})
|
||||
OPTIONAL MATCH (s)-[:HAS_CURRICULUM_STRUCTURE]->(:CurriculumStructure)-[:INCLUDES_KEY_STAGE]->(:KeyStage)-[:INCLUDES_KEY_STAGE_SYLLABUS]->(ks:KeyStageSyllabus)
|
||||
OPTIONAL MATCH (s)-[:HAS_CURRICULUM_STRUCTURE]->(:CurriculumStructure)-[:INCLUDES_KEY_STAGE]->(:KeyStage)-[:INCLUDES_YEAR_GROUP_SYLLABUS]->(ys:YearGroupSyllabus)
|
||||
RETURN count(ks) > 0 OR count(ys) > 0 as has_syllabus
|
||||
""", school_id=school_node.unique_id)
|
||||
|
||||
has_syllabus = result.single()["has_syllabus"]
|
||||
|
||||
return {"has_syllabus": has_syllabus}
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error checking syllabus status: {str(e)}")
|
||||
raise
|
||||
@@ -1,526 +0,0 @@
|
||||
import os
|
||||
from modules.logger_tool import initialise_logger
|
||||
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
|
||||
import modules.database.init.init_calendar as init_calendar
|
||||
import modules.database.schemas.nodes.schools.timetable as timetable
|
||||
import modules.database.schemas.relationships.timetables as tt_rels
|
||||
import modules.database.schemas.relationships.entity_timetable_rels as entity_tt_rels
|
||||
import modules.database.schemas.relationships.calendar_timetable_rels as cal_tt_rels
|
||||
import modules.database.tools.neontology_tools as neon
|
||||
from modules.database.tools.filesystem_tools import ClassroomCopilotFilesystem
|
||||
from datetime import timedelta, datetime
|
||||
import pandas as pd
|
||||
|
||||
def create_school_timetable_from_dataframes(dataframes, db_name, school_node=None):
|
||||
logger.info(f"Creating school timetable for {db_name}")
|
||||
if dataframes is None:
|
||||
raise ValueError("Data is required to create the calendar and timetable.")
|
||||
|
||||
logger.info("Initialising neo4j connection...")
|
||||
neon.init_neontology_connection()
|
||||
|
||||
# Initialize the filesystem handler
|
||||
fs_handler = ClassroomCopilotFilesystem(db_name, init_run_type="school")
|
||||
|
||||
school_df = dataframes['school']
|
||||
if school_node is None:
|
||||
logger.info("School node is None, using school data from dataframe")
|
||||
school_unique_id = school_df[school_df['Identifier'] == 'SchoolID']['Data'].iloc[0]
|
||||
else:
|
||||
logger.info(f"School node is not None, using school data from school node: {school_node}")
|
||||
school_unique_id = school_node.unique_id
|
||||
|
||||
terms_df = dataframes['terms']
|
||||
weeks_df = dataframes['weeks']
|
||||
days_df = dataframes['days']
|
||||
periods_df = dataframes['periods']
|
||||
|
||||
school_df_year_start = school_df[school_df['Identifier'] == 'AcademicYearStart']['Data'].iloc[0]
|
||||
school_df_year_end = school_df[school_df['Identifier'] == 'AcademicYearEnd']['Data'].iloc[0]
|
||||
if isinstance(school_df_year_start, str):
|
||||
school_year_start_date = datetime.strptime(school_df_year_start, '%Y-%m-%d')
|
||||
else:
|
||||
school_year_start_date = school_df_year_start
|
||||
if isinstance(school_df_year_end, str):
|
||||
school_year_end_date = datetime.strptime(school_df_year_end, '%Y-%m-%d')
|
||||
else:
|
||||
school_year_end_date = school_df_year_end
|
||||
|
||||
# Create a dictionary to store the timetable nodes
|
||||
timetable_nodes = {
|
||||
'timetable_node': None,
|
||||
'academic_year_nodes': [],
|
||||
'academic_term_nodes': [],
|
||||
'academic_week_nodes': [],
|
||||
'academic_day_nodes': [],
|
||||
'academic_period_nodes': []
|
||||
}
|
||||
|
||||
if school_node:
|
||||
# Create the root timetable directory
|
||||
_, timetable_path = fs_handler.create_school_timetable_directory(school_node.path)
|
||||
else:
|
||||
# Create the root timetable directory
|
||||
_, timetable_path = fs_handler.create_school_timetable_directory()
|
||||
|
||||
# Create AcademicTimetable Node
|
||||
school_timetable_unique_id = f"SchoolTimetable_{school_unique_id}_{school_year_start_date.year}_{school_year_end_date.year}"
|
||||
school_timetable_node = timetable.SchoolTimetableNode(
|
||||
unique_id=school_timetable_unique_id,
|
||||
start_date=school_year_start_date,
|
||||
end_date=school_year_end_date,
|
||||
path=timetable_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(school_timetable_node, database=db_name, operation='merge')
|
||||
# Create the tldraw file for the node
|
||||
fs_handler.create_default_tldraw_file(school_timetable_node.path, school_timetable_node.to_dict())
|
||||
timetable_nodes['timetable_node'] = school_timetable_node
|
||||
|
||||
if school_node:
|
||||
logger.info(f"Creating calendar for {school_unique_id} from Neo4j SchoolNode: {school_node.unique_id}")
|
||||
calendar_nodes = init_calendar.create_calendar(db_name, school_year_start_date, school_year_end_date, attach_to_calendar_node=True, entity_node=school_node)
|
||||
# Link the school node to the timetable node
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
entity_tt_rels.SchoolHasTimetable(source=school_node, target=school_timetable_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
timetable_nodes['calendar_nodes'] = calendar_nodes
|
||||
else:
|
||||
logger.info(f"Creating calendar for {school_unique_id} from dataframe SchoolID: {school_unique_id}")
|
||||
calendar_nodes = init_calendar.create_calendar(db_name, school_year_start_date, school_year_end_date, attach_to_calendar_node=False, entity_node=None)
|
||||
|
||||
# Create AcademicYear nodes for each year within the range
|
||||
for year in range(school_year_start_date.year, school_year_end_date.year + 1):
|
||||
_, timetable_year_path = fs_handler.create_school_timetable_year_directory(timetable_path, year)
|
||||
year_str = str(year)
|
||||
academic_year_unique_id = f"AcademicYear_{school_timetable_unique_id}_{year}"
|
||||
academic_year_node = timetable.AcademicYearNode(
|
||||
unique_id=academic_year_unique_id,
|
||||
year=year_str,
|
||||
path=timetable_year_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(academic_year_node, database=db_name, operation='merge')
|
||||
# Create the tldraw file for the node
|
||||
fs_handler.create_default_tldraw_file(academic_year_node.path, academic_year_node.to_dict())
|
||||
timetable_nodes['academic_year_nodes'].append(academic_year_node)
|
||||
logger.info(f'Created academic year node: {academic_year_node.unique_id}')
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
tt_rels.AcademicTimetableHasAcademicYear(source=school_timetable_node, target=academic_year_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created school timetable relationship from {school_timetable_node.unique_id} to {academic_year_node.unique_id}")
|
||||
|
||||
# Link the academic year with the corresponding calendar year node
|
||||
for year_node in calendar_nodes['calendar_year_nodes']:
|
||||
if year_node.year == year:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
cal_tt_rels.AcademicYearIsCalendarYear(source=academic_year_node, target=year_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created school timetable relationship from {academic_year_node.unique_id} to {year_node.unique_id}")
|
||||
break
|
||||
|
||||
# Create Term and TermBreak nodes linked to AcademicYear
|
||||
term_number = 1
|
||||
academic_term_number = 1
|
||||
for _, term_row in terms_df.iterrows():
|
||||
term_node_class = timetable.AcademicTermNode if term_row['TermType'] == 'Term' else timetable.AcademicTermBreakNode
|
||||
term_name = term_row['TermName']
|
||||
term_name_no_spaces = term_name.replace(' ', '')
|
||||
term_start_date = term_row['StartDate']
|
||||
if isinstance(term_start_date, pd.Timestamp):
|
||||
term_start_date = term_start_date.strftime('%Y-%m-%d')
|
||||
|
||||
term_end_date = term_row['EndDate']
|
||||
if isinstance(term_end_date, pd.Timestamp):
|
||||
term_end_date = term_end_date.strftime('%Y-%m-%d')
|
||||
|
||||
if term_row['TermType'] == 'Term':
|
||||
_, timetable_term_path = fs_handler.create_school_timetable_academic_term_directory(
|
||||
timetable_path=timetable_path,
|
||||
term_name=term_name,
|
||||
term_number=academic_term_number
|
||||
)
|
||||
term_node_unique_id = f"AcademicTerm_{school_timetable_unique_id}_{academic_term_number}_{term_name_no_spaces}"
|
||||
academic_term_number_str = str(academic_term_number)
|
||||
term_node = term_node_class(
|
||||
unique_id=term_node_unique_id,
|
||||
term_name=term_name,
|
||||
term_number=academic_term_number_str,
|
||||
start_date=datetime.strptime(term_start_date, '%Y-%m-%d'),
|
||||
end_date=datetime.strptime(term_end_date, '%Y-%m-%d'),
|
||||
path=timetable_term_path
|
||||
)
|
||||
academic_term_number += 1
|
||||
else:
|
||||
term_break_node_unique_id = f"AcademicTermBreak_{school_timetable_unique_id}_{term_name_no_spaces}"
|
||||
term_node = term_node_class(
|
||||
unique_id=term_break_node_unique_id,
|
||||
term_break_name=term_name,
|
||||
start_date=datetime.strptime(term_start_date, '%Y-%m-%d'),
|
||||
end_date=datetime.strptime(term_end_date, '%Y-%m-%d')
|
||||
)
|
||||
neon.create_or_merge_neontology_node(term_node, database=db_name, operation='merge')
|
||||
if isinstance(term_node, timetable.AcademicTermNode):
|
||||
# Create the tldraw file for the node
|
||||
fs_handler.create_default_tldraw_file(term_node.path, term_node.to_dict())
|
||||
logger.info(f'Created academic term break node: {term_node.unique_id}')
|
||||
timetable_nodes['academic_term_nodes'].append(term_node)
|
||||
term_number += 1 # We don't use this but we could
|
||||
|
||||
# Link term node to the correct academic year
|
||||
term_years = set()
|
||||
term_years.update([term_node.start_date.year, term_node.end_date.year])
|
||||
|
||||
for academic_year_node in timetable_nodes['academic_year_nodes']:
|
||||
if int(academic_year_node.year) in term_years:
|
||||
relationship_class = tt_rels.AcademicYearHasAcademicTerm if term_row['TermType'] == 'Term' else tt_rels.AcademicYearHasAcademicTermBreak
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
relationship_class(source=academic_year_node, target=term_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created school timetable relationship from {academic_year_node.unique_id} to {term_node.unique_id}")
|
||||
|
||||
# Create Week nodes
|
||||
academic_week_number = 1
|
||||
for _, week_row in weeks_df.iterrows():
|
||||
week_node_class = timetable.HolidayWeekNode if week_row['WeekType'] == 'Holiday' else timetable.AcademicWeekNode
|
||||
week_start_date = week_row['WeekStart']
|
||||
if isinstance(week_start_date, pd.Timestamp):
|
||||
week_start_date = week_start_date.strftime('%Y-%m-%d')
|
||||
|
||||
if week_row['WeekType'] == 'Holiday':
|
||||
week_node_unique_id = f"{week_row['WeekType']}Week_{school_timetable_unique_id}_Week_{week_row['WeekNumber']}"
|
||||
week_node = week_node_class(
|
||||
unique_id=week_node_unique_id,
|
||||
start_date=datetime.strptime(week_start_date, '%Y-%m-%d')
|
||||
)
|
||||
else:
|
||||
_, timetable_week_path = fs_handler.create_school_timetable_academic_week_directory(
|
||||
timetable_path=timetable_path,
|
||||
week_number=academic_week_number
|
||||
)
|
||||
week_node_unique_id = f"AcademicWeek_{school_timetable_unique_id}_Week_{week_row['WeekNumber']}"
|
||||
academic_week_number_str = str(academic_week_number)
|
||||
week_type = week_row['WeekType']
|
||||
week_node = week_node_class(
|
||||
unique_id=week_node_unique_id,
|
||||
academic_week_number=academic_week_number_str,
|
||||
start_date=datetime.strptime(week_start_date, '%Y-%m-%d'),
|
||||
week_type=week_type,
|
||||
path=timetable_week_path
|
||||
)
|
||||
academic_week_number += 1
|
||||
neon.create_or_merge_neontology_node(week_node, database=db_name, operation='merge')
|
||||
timetable_nodes['academic_week_nodes'].append(week_node)
|
||||
logger.info(f"Created week node: {week_node.unique_id}")
|
||||
if isinstance(week_node, timetable.AcademicWeekNode):
|
||||
# Create the tldraw file for the node
|
||||
fs_handler.create_default_tldraw_file(week_node.path, week_node.to_dict())
|
||||
for calendar_node in calendar_nodes['calendar_week_nodes']:
|
||||
if calendar_node.start_date == week_node.start_date:
|
||||
if isinstance(week_node, timetable.AcademicWeekNode):
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
cal_tt_rels.AcademicWeekIsCalendarWeek(source=week_node, target=calendar_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created school timetable relationship from {calendar_node.unique_id} to {week_node.unique_id}")
|
||||
elif isinstance(week_node, timetable.HolidayWeekNode):
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
cal_tt_rels.HolidayWeekIsCalendarWeek(source=week_node, target=calendar_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created school timetable relationship from {calendar_node.unique_id} to {week_node.unique_id}")
|
||||
break
|
||||
|
||||
# Link week node to the correct academic term
|
||||
for term_node in timetable_nodes['academic_term_nodes']:
|
||||
if term_node.start_date <= week_node.start_date <= term_node.end_date:
|
||||
relationship_class = tt_rels.AcademicTermHasAcademicWeek if week_row['WeekType'] != 'Holiday' else tt_rels.AcademicTermBreakHasHolidayWeek
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
relationship_class(source=term_node, target=week_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created school timetable relationship from {term_node.unique_id} to {week_node.unique_id}")
|
||||
break
|
||||
|
||||
# Link week node to the correct academic year
|
||||
for academic_year_node in timetable_nodes['academic_year_nodes']:
|
||||
if int(academic_year_node.year) == week_node.start_date.year:
|
||||
relationship_class = tt_rels.AcademicYearHasAcademicWeek if week_row['WeekType'] != 'Holiday' else tt_rels.AcademicYearHasHolidayWeek
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
relationship_class(source=academic_year_node, target=week_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created school timetable relationship from {academic_year_node.unique_id} to {week_node.unique_id}")
|
||||
break
|
||||
|
||||
# Create Day nodes
|
||||
day_number = 1
|
||||
academic_day_number = 1
|
||||
for _, day_row in days_df.iterrows():
|
||||
date_str = day_row['Date']
|
||||
if isinstance(date_str, pd.Timestamp):
|
||||
date_str = date_str.strftime('%Y-%m-%d')
|
||||
|
||||
day_node_class = {
|
||||
'Academic': timetable.AcademicDayNode,
|
||||
'Holiday': timetable.HolidayDayNode,
|
||||
'OffTimetable': timetable.OffTimetableDayNode,
|
||||
'StaffDay': timetable.StaffDayNode
|
||||
}[day_row['DayType']]
|
||||
|
||||
# Format the unique ID as {day_node_class.__name__}Day
|
||||
day_node_data = {
|
||||
'unique_id': f"{day_node_class.__name__}Day_{school_timetable_unique_id}_{day_number}",
|
||||
'date': datetime.strptime(date_str, '%Y-%m-%d'),
|
||||
'day_of_week': datetime.strptime(date_str, '%Y-%m-%d').strftime('%A')
|
||||
}
|
||||
|
||||
if day_row['DayType'] == 'Academic':
|
||||
day_node_data['academic_day'] = str(academic_day_number)
|
||||
day_node_data['day_type'] = day_row['WeekType']
|
||||
_, timetable_day_path = fs_handler.create_school_timetable_academic_day_directory(
|
||||
timetable_path=timetable_path,
|
||||
academic_day=academic_day_number
|
||||
)
|
||||
day_node_data['path'] = timetable_day_path
|
||||
|
||||
day_node = day_node_class(**day_node_data)
|
||||
|
||||
for calendar_node in calendar_nodes['calendar_day_nodes']:
|
||||
if calendar_node.date == day_node.date:
|
||||
neon.create_or_merge_neontology_node(day_node, database=db_name, operation='merge')
|
||||
timetable_nodes['academic_day_nodes'].append(day_node)
|
||||
logger.info(f"Created day node: {day_node.unique_id}")
|
||||
|
||||
if isinstance(day_node, timetable.AcademicDayNode):
|
||||
fs_handler.create_default_tldraw_file(day_node.path, day_node.to_dict())
|
||||
relationship_class = cal_tt_rels.AcademicDayIsCalendarDay
|
||||
elif isinstance(day_node, timetable.HolidayDayNode):
|
||||
relationship_class = cal_tt_rels.HolidayDayIsCalendarDay
|
||||
elif isinstance(day_node, timetable.OffTimetableDayNode):
|
||||
relationship_class = cal_tt_rels.OffTimetableDayIsCalendarDay
|
||||
elif isinstance(day_node, timetable.StaffDayNode):
|
||||
relationship_class = cal_tt_rels.StaffDayIsCalendarDay
|
||||
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
relationship_class(source=day_node, target=calendar_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f'Created relationship from {calendar_node.unique_id} to {day_node.unique_id}')
|
||||
break
|
||||
|
||||
# Link day node to the correct academic week
|
||||
for academic_week_node in timetable_nodes['academic_week_nodes']:
|
||||
if academic_week_node.start_date <= day_node.date <= (academic_week_node.start_date + timedelta(days=6)):
|
||||
if day_row['DayType'] == 'Academic':
|
||||
relationship_class = tt_rels.AcademicWeekHasAcademicDay
|
||||
elif day_row['DayType'] == 'Holiday':
|
||||
if hasattr(academic_week_node, 'week_type') and academic_week_node.week_type in ['A', 'B']:
|
||||
relationship_class = tt_rels.AcademicWeekHasHolidayDay
|
||||
else:
|
||||
relationship_class = tt_rels.HolidayWeekHasHolidayDay
|
||||
elif day_row['DayType'] == 'OffTimetable':
|
||||
relationship_class = tt_rels.AcademicWeekHasOffTimetableDay
|
||||
elif day_row['DayType'] == 'Staff':
|
||||
relationship_class = tt_rels.AcademicWeekHasStaffDay
|
||||
else:
|
||||
continue # Skip linking for other day types
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
relationship_class(source=academic_week_node, target=day_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship from {academic_week_node.unique_id} to {day_node.unique_id}")
|
||||
break
|
||||
|
||||
# Link day node to the correct academic term
|
||||
for term_node in timetable_nodes['academic_term_nodes']:
|
||||
if term_node.start_date <= day_node.date <= term_node.end_date:
|
||||
if day_row['DayType'] == 'Academic':
|
||||
relationship_class = tt_rels.AcademicTermHasAcademicDay
|
||||
elif day_row['DayType'] == 'Holiday':
|
||||
if isinstance(term_node, timetable.AcademicTermNode):
|
||||
relationship_class = tt_rels.AcademicTermHasHolidayDay
|
||||
else:
|
||||
relationship_class = tt_rels.AcademicTermBreakHasHolidayDay
|
||||
elif day_row['DayType'] == 'OffTimetable':
|
||||
relationship_class = tt_rels.AcademicTermHasOffTimetableDay
|
||||
elif day_row['DayType'] == 'Staff':
|
||||
relationship_class = tt_rels.AcademicTermHasStaffDay
|
||||
else:
|
||||
continue # Skip linking for other day types
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
relationship_class(source=term_node, target=day_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship from {term_node.unique_id} to {day_node.unique_id}")
|
||||
break
|
||||
|
||||
# Create Period nodes for each academic day
|
||||
if day_row['DayType'] == 'Academic':
|
||||
logger.info(f"Creating periods for {day_node.unique_id}")
|
||||
period_of_day = 1
|
||||
academic_or_registration_period_of_day = 1
|
||||
for _, period_row in periods_df.iterrows():
|
||||
period_node_class = {
|
||||
'Academic': timetable.AcademicPeriodNode,
|
||||
'Registration': timetable.RegistrationPeriodNode,
|
||||
'Break': timetable.BreakPeriodNode,
|
||||
'OffTimetable': timetable.OffTimetablePeriodNode
|
||||
}[period_row['PeriodType']]
|
||||
|
||||
logger.info(f"Creating period node for {period_node_class.__name__} Period: {period_of_day}")
|
||||
period_node_unique_id = f"{period_node_class.__name__}_{school_timetable_unique_id}_Day_{academic_day_number}_Period_{period_of_day}"
|
||||
logger.debug(f"Period node unique id: {period_node_unique_id}")
|
||||
period_node_data = {
|
||||
'unique_id': period_node_unique_id,
|
||||
'name': period_row['PeriodName'],
|
||||
'date': day_node.date,
|
||||
'start_time': datetime.combine(day_node.date, period_row['StartTime']),
|
||||
'end_time': datetime.combine(day_node.date, period_row['EndTime'])
|
||||
}
|
||||
logger.debug(f"Period node data: {period_node_data}")
|
||||
if period_row['PeriodType'] in ['Academic', 'Registration']:
|
||||
_, timetable_period_path = fs_handler.create_school_timetable_period_directory(
|
||||
timetable_path=timetable_path,
|
||||
academic_day=academic_day_number,
|
||||
period_dir=f"{academic_or_registration_period_of_day}_{period_row['PeriodName'].replace(' ', '_')}"
|
||||
)
|
||||
week_type = day_row['WeekType']
|
||||
day_name_short = day_node.day_of_week[:3]
|
||||
period_code = period_row['PeriodCode']
|
||||
period_code_formatted = f"{week_type}{day_name_short}{period_code}"
|
||||
period_node_data['period_code'] = period_code_formatted
|
||||
period_node_data['path'] = timetable_period_path
|
||||
|
||||
academic_or_registration_period_of_day += 1
|
||||
|
||||
period_node = period_node_class(**period_node_data)
|
||||
neon.create_or_merge_neontology_node(period_node, database=db_name, operation='merge')
|
||||
if isinstance(period_node, timetable.AcademicPeriodNode) or isinstance(period_node, timetable.RegistrationPeriodNode):
|
||||
# Create the tldraw file for the node
|
||||
fs_handler.create_default_tldraw_file(period_node.path, period_node.to_dict())
|
||||
timetable_nodes['academic_period_nodes'].append(period_node)
|
||||
logger.info(f'Created period node: {period_node.unique_id}')
|
||||
|
||||
relationship_class = {
|
||||
'Academic': tt_rels.AcademicDayHasAcademicPeriod,
|
||||
'Registration': tt_rels.AcademicDayHasRegistrationPeriod,
|
||||
'Break': tt_rels.AcademicDayHasBreakPeriod,
|
||||
'OffTimetable': tt_rels.AcademicDayHasOffTimetablePeriod
|
||||
}[period_row['PeriodType']]
|
||||
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
relationship_class(source=day_node, target=period_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship from {day_node.unique_id} to {period_node.unique_id}")
|
||||
period_of_day += 1 # We don't use this but we could
|
||||
academic_day_number += 1 # This is a bit of a hack but it works to keep the directories aligned (reorganise)
|
||||
day_number += 1 # We don't use this but we could
|
||||
|
||||
def create_school_timetable_node_sequence_rels(timetable_nodes):
|
||||
def sort_and_create_relationships(nodes, relationship_map, sort_key):
|
||||
sorted_nodes = sorted(nodes, key=sort_key)
|
||||
for i in range(len(sorted_nodes) - 1):
|
||||
source_node = sorted_nodes[i]
|
||||
target_node = sorted_nodes[i + 1]
|
||||
node_type_pair = (type(source_node), type(target_node))
|
||||
relationship_class = relationship_map.get(node_type_pair)
|
||||
if relationship_class:
|
||||
# Avoid self-referential relationships
|
||||
if source_node.unique_id != target_node.unique_id:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
relationship_class(
|
||||
source=source_node,
|
||||
target=target_node
|
||||
),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship from {source_node.unique_id} to {target_node.unique_id}")
|
||||
else:
|
||||
logger.warning(f"Skipped self-referential relationship for node {source_node.unique_id}")
|
||||
|
||||
# Relationship maps for different node types
|
||||
academic_year_relationship_map = {
|
||||
(timetable.AcademicYearNode, timetable.AcademicYearNode): tt_rels.AcademicYearFollowsAcademicYear
|
||||
}
|
||||
|
||||
academic_term_relationship_map = {
|
||||
(timetable.AcademicTermNode, timetable.AcademicTermBreakNode): tt_rels.AcademicTermBreakFollowsAcademicTerm,
|
||||
(timetable.AcademicTermBreakNode, timetable.AcademicTermNode): tt_rels.AcademicTermFollowsAcademicTermBreak
|
||||
}
|
||||
|
||||
academic_week_relationship_map = {
|
||||
(timetable.AcademicWeekNode, timetable.AcademicWeekNode): tt_rels.AcademicWeekFollowsAcademicWeek,
|
||||
(timetable.HolidayWeekNode, timetable.HolidayWeekNode): tt_rels.HolidayWeekFollowsHolidayWeek,
|
||||
(timetable.AcademicWeekNode, timetable.HolidayWeekNode): tt_rels.HolidayWeekFollowsAcademicWeek,
|
||||
(timetable.HolidayWeekNode, timetable.AcademicWeekNode): tt_rels.AcademicWeekFollowsHolidayWeek
|
||||
}
|
||||
|
||||
academic_day_relationship_map = {
|
||||
(timetable.AcademicDayNode, timetable.AcademicDayNode): tt_rels.AcademicDayFollowsAcademicDay,
|
||||
(timetable.HolidayDayNode, timetable.HolidayDayNode): tt_rels.HolidayDayFollowsHolidayDay,
|
||||
(timetable.OffTimetableDayNode, timetable.OffTimetableDayNode): tt_rels.OffTimetableDayFollowsOffTimetableDay,
|
||||
(timetable.StaffDayNode, timetable.StaffDayNode): tt_rels.StaffDayFollowsStaffDay,
|
||||
|
||||
(timetable.AcademicDayNode, timetable.HolidayDayNode): tt_rels.HolidayDayFollowsAcademicDay,
|
||||
(timetable.AcademicDayNode, timetable.OffTimetableDayNode): tt_rels.OffTimetableDayFollowsAcademicDay,
|
||||
(timetable.AcademicDayNode, timetable.StaffDayNode): tt_rels.StaffDayFollowsAcademicDay,
|
||||
|
||||
(timetable.HolidayDayNode, timetable.AcademicDayNode): tt_rels.AcademicDayFollowsHolidayDay,
|
||||
(timetable.HolidayDayNode, timetable.OffTimetableDayNode): tt_rels.OffTimetableDayFollowsHolidayDay,
|
||||
(timetable.HolidayDayNode, timetable.StaffDayNode): tt_rels.StaffDayFollowsHolidayDay,
|
||||
|
||||
(timetable.OffTimetableDayNode, timetable.AcademicDayNode): tt_rels.AcademicDayFollowsOffTimetableDay,
|
||||
(timetable.OffTimetableDayNode, timetable.HolidayDayNode): tt_rels.HolidayDayFollowsOffTimetableDay,
|
||||
(timetable.OffTimetableDayNode, timetable.StaffDayNode): tt_rels.StaffDayFollowsOffTimetableDay,
|
||||
|
||||
(timetable.StaffDayNode, timetable.AcademicDayNode): tt_rels.AcademicDayFollowsStaffDay,
|
||||
(timetable.StaffDayNode, timetable.HolidayDayNode): tt_rels.HolidayDayFollowsStaffDay,
|
||||
(timetable.StaffDayNode, timetable.OffTimetableDayNode): tt_rels.OffTimetableDayFollowsStaffDay,
|
||||
}
|
||||
|
||||
academic_period_relationship_map = {
|
||||
(timetable.AcademicPeriodNode, timetable.AcademicPeriodNode): tt_rels.AcademicPeriodFollowsAcademicPeriod,
|
||||
(timetable.AcademicPeriodNode, timetable.BreakPeriodNode): tt_rels.BreakPeriodFollowsAcademicPeriod,
|
||||
(timetable.AcademicPeriodNode, timetable.RegistrationPeriodNode): tt_rels.RegistrationPeriodFollowsAcademicPeriod,
|
||||
(timetable.AcademicPeriodNode, timetable.OffTimetablePeriodNode): tt_rels.OffTimetablePeriodFollowsAcademicPeriod,
|
||||
(timetable.BreakPeriodNode, timetable.AcademicPeriodNode): tt_rels.AcademicPeriodFollowsBreakPeriod,
|
||||
(timetable.BreakPeriodNode, timetable.BreakPeriodNode): tt_rels.BreakPeriodFollowsBreakPeriod,
|
||||
(timetable.BreakPeriodNode, timetable.RegistrationPeriodNode): tt_rels.RegistrationPeriodFollowsBreakPeriod,
|
||||
(timetable.BreakPeriodNode, timetable.OffTimetablePeriodNode): tt_rels.OffTimetablePeriodFollowsBreakPeriod,
|
||||
(timetable.RegistrationPeriodNode, timetable.AcademicPeriodNode): tt_rels.AcademicPeriodFollowsRegistrationPeriod,
|
||||
(timetable.RegistrationPeriodNode, timetable.RegistrationPeriodNode): tt_rels.RegistrationPeriodFollowsRegistrationPeriod,
|
||||
(timetable.RegistrationPeriodNode, timetable.BreakPeriodNode): tt_rels.BreakPeriodFollowsRegistrationPeriod,
|
||||
(timetable.RegistrationPeriodNode, timetable.OffTimetablePeriodNode): tt_rels.OffTimetablePeriodFollowsRegistrationPeriod,
|
||||
(timetable.OffTimetablePeriodNode, timetable.OffTimetablePeriodNode): tt_rels.OffTimetablePeriodFollowsOffTimetablePeriod,
|
||||
(timetable.OffTimetablePeriodNode, timetable.AcademicPeriodNode): tt_rels.AcademicPeriodFollowsOffTimetablePeriod,
|
||||
(timetable.OffTimetablePeriodNode, timetable.BreakPeriodNode): tt_rels.BreakPeriodFollowsOffTimetablePeriod,
|
||||
(timetable.OffTimetablePeriodNode, timetable.RegistrationPeriodNode): tt_rels.RegistrationPeriodFollowsOffTimetablePeriod,
|
||||
}
|
||||
|
||||
|
||||
# Sort and create relationships
|
||||
sort_and_create_relationships(timetable_nodes['academic_year_nodes'], academic_year_relationship_map, lambda x: int(x.year))
|
||||
sort_and_create_relationships(timetable_nodes['academic_term_nodes'], academic_term_relationship_map, lambda x: x.start_date)
|
||||
sort_and_create_relationships(timetable_nodes['academic_week_nodes'], academic_week_relationship_map, lambda x: x.start_date)
|
||||
sort_and_create_relationships(timetable_nodes['academic_day_nodes'], academic_day_relationship_map, lambda x: x.date)
|
||||
sort_and_create_relationships(timetable_nodes['academic_period_nodes'], academic_period_relationship_map, lambda x: (x.start_time, x.end_time))
|
||||
|
||||
# Call the function with the created timetable nodes
|
||||
create_school_timetable_node_sequence_rels(timetable_nodes)
|
||||
|
||||
logger.info(f'Created timetable: {timetable_nodes["timetable_node"].unique_id}')
|
||||
|
||||
# Log the directory structure after creation
|
||||
# root_timetable_directory = fs_handler.root_path # Access the root directory of the filesystem handler
|
||||
# fs_handler.log_directory_structure(root_timetable_directory)
|
||||
|
||||
return {
|
||||
'school_node': school_node,
|
||||
'school_calendar_nodes': calendar_nodes,
|
||||
'school_timetable_nodes': timetable_nodes
|
||||
}
|
||||
@@ -4,12 +4,25 @@ logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH
|
||||
import modules.database.schemas.nodes.calendars as calendar_schemas
|
||||
import modules.database.schemas.relationships.calendars as calendar_relationships
|
||||
import modules.database.schemas.relationships.calendar_sequence as calendar_sequence_relationships
|
||||
import modules.database.schemas.relationships.owner_relationships as owner_relationships
|
||||
import modules.database.tools.supabase_storage_tools as storage_tools
|
||||
import modules.database.tools.neontology_tools as neon
|
||||
from datetime import timedelta, datetime
|
||||
|
||||
def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False, owner_node=None, time_chunk_node_length: int = None):
|
||||
logger.info(f"Creating calendar for {start_date} to {end_date}")
|
||||
def create_calendar(db_name, start_date, end_date, time_chunk_node_length: int = None, storage_tools=None):
|
||||
"""
|
||||
Create calendar structure with years, months, weeks, and days
|
||||
|
||||
Args:
|
||||
db_name: Database name to create calendar in
|
||||
start_date: Start date for calendar
|
||||
end_date: End date for calendar
|
||||
time_chunk_node_length: Optional time chunk length in minutes
|
||||
storage_tools: Optional Supabase storage tools for generating storage paths
|
||||
|
||||
Returns:
|
||||
dict: Dictionary containing created calendar nodes
|
||||
"""
|
||||
logger.info(f"Creating calendar structure for {start_date} to {end_date} in database: {db_name}")
|
||||
|
||||
logger.info(f"Initializing Neontology connection")
|
||||
neon.init_neontology_connection()
|
||||
@@ -25,52 +38,13 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
|
||||
last_day_node = None
|
||||
|
||||
calendar_nodes = {
|
||||
'calendar_node': None,
|
||||
'calendar_year_nodes': [],
|
||||
'calendar_month_nodes': [],
|
||||
'calendar_week_nodes': [],
|
||||
'calendar_day_nodes': []
|
||||
'calendar_day_nodes': [],
|
||||
'calendar_time_chunk_nodes': []
|
||||
}
|
||||
|
||||
if attach_to_calendar_node and owner_node:
|
||||
logger.info(f"Attaching calendar to owner's node {owner_node.unique_id} in database: {db_name}")
|
||||
owner_unique_id = owner_node.unique_id
|
||||
calendar_unique_id = f"{start_date.strftime('%Y-%m-%d')}_{end_date.strftime('%Y-%m-%d')}"
|
||||
calendar_node = calendar_schemas.CalendarNode(
|
||||
unique_id=calendar_unique_id,
|
||||
name=f"{start_date.strftime('%Y-%m-%d')} to {end_date.strftime('%Y-%m-%d')}",
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
neon.create_or_merge_neontology_node(calendar_node, database=db_name, operation='merge')
|
||||
calendar_nodes['calendar_node'] = calendar_node
|
||||
logger.info(f"Calendar node created: {calendar_node.unique_id}")
|
||||
|
||||
import modules.database.schemas.relationships.owner_relationships as owner_relationships
|
||||
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
owner_relationships.OwnerHasCalendar(source=owner_node, target=calendar_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {owner_node.unique_id} to {calendar_node.unique_id}")
|
||||
elif attach_to_calendar_node and not owner_node:
|
||||
logger.info(f"Creating calendar for {start_date} to {end_date} in database: {db_name}")
|
||||
calendar_node = calendar_schemas.CalendarNode(
|
||||
unique_id=f"{start_date.strftime('%Y-%m-%d')}_{end_date.strftime('%Y-%m-%d')}",
|
||||
name=f"{start_date.strftime('%Y-%m-%d')} to {end_date.strftime('%Y-%m-%d')}",
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
neon.create_or_merge_neontology_node(calendar_node, database=db_name, operation='merge')
|
||||
calendar_nodes['calendar_node'] = calendar_node
|
||||
logger.info(f"Calendar node created: {calendar_node.unique_id}")
|
||||
else:
|
||||
logger.error("Invalid combination of parameters for calendar creation.")
|
||||
raise ValueError("Invalid combination of parameters for calendar creation.")
|
||||
|
||||
current_date = start_date
|
||||
while current_date <= end_date:
|
||||
year = current_date.year
|
||||
@@ -78,50 +52,55 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
|
||||
day = current_date.day
|
||||
iso_year, iso_week, iso_weekday = current_date.isocalendar()
|
||||
|
||||
calendar_year_unique_id = f"{year}"
|
||||
calendar_year_uuid_string = f"{year}"
|
||||
|
||||
if year not in created_years:
|
||||
# Generate storage path for year node using Supabase Storage
|
||||
if storage_tools:
|
||||
year_dir_created, node_storage_path = storage_tools.create_calendar_year_storage_path(year)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
year_node = calendar_schemas.CalendarYearNode(
|
||||
unique_id=calendar_year_unique_id,
|
||||
uuid_string=calendar_year_uuid_string,
|
||||
year=str(year),
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(year_node, database=db_name, operation='merge')
|
||||
calendar_nodes['calendar_year_nodes'].append(year_node)
|
||||
created_years[year] = year_node
|
||||
logger.info(f"Year node created: {year_node.unique_id}")
|
||||
logger.info(f"Year node created: {year_node.uuid_string}")
|
||||
|
||||
if attach_to_calendar_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
calendar_relationships.CalendarIncludesYear(source=calendar_node, target=year_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {calendar_node.unique_id} to {year_node.unique_id}")
|
||||
if last_year_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
calendar_sequence_relationships.YearFollowsYear(source=last_year_node, target=year_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {last_year_node.unique_id} to {year_node.unique_id}")
|
||||
logger.info(f"Relationship created from {last_year_node.uuid_string} to {year_node.uuid_string}")
|
||||
last_year_node = year_node
|
||||
|
||||
calendar_month_unique_id = f"{year}_{month}"
|
||||
calendar_month_uuid_string = f"{year}_{month}"
|
||||
|
||||
month_key = f"{year}-{month}"
|
||||
if month_key not in created_months:
|
||||
# Generate storage path for month node using Supabase Storage
|
||||
if storage_tools:
|
||||
month_dir_created, node_storage_path = storage_tools.create_calendar_month_storage_path(year, month)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
month_node = calendar_schemas.CalendarMonthNode(
|
||||
unique_id=calendar_month_unique_id,
|
||||
uuid_string=calendar_month_uuid_string,
|
||||
year=str(year),
|
||||
month=str(month),
|
||||
month_name=datetime(year, month, 1).strftime('%B'),
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(month_node, database=db_name, operation='merge')
|
||||
calendar_nodes['calendar_month_nodes'].append(month_node)
|
||||
created_months[month_key] = month_node
|
||||
logger.info(f"Month node created: {month_node.unique_id}")
|
||||
logger.info(f"Month node created: {month_node.uuid_string}")
|
||||
|
||||
# Check for the end of year transition for months
|
||||
if last_month_node:
|
||||
@@ -131,14 +110,14 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {last_month_node.unique_id} to {month_node.unique_id}")
|
||||
logger.info(f"Relationship created from {last_month_node.uuid_string} to {month_node.uuid_string}")
|
||||
elif int(month) == int(last_month_node.month) + 1:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
calendar_sequence_relationships.MonthFollowsMonth(source=last_month_node, target=month_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {last_month_node.unique_id} to {month_node.unique_id}")
|
||||
logger.info(f"Relationship created from {last_month_node.uuid_string} to {month_node.uuid_string}")
|
||||
last_month_node = month_node
|
||||
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
@@ -146,25 +125,31 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {year_node.unique_id} to {month_node.unique_id}")
|
||||
logger.info(f"Relationship created from {year_node.uuid_string} to {month_node.uuid_string}")
|
||||
|
||||
calendar_week_unique_id = f"{iso_year}_{iso_week}"
|
||||
calendar_week_uuid_string = f"{iso_year}_{iso_week}"
|
||||
|
||||
week_key = f"{iso_year}-W{iso_week}"
|
||||
if week_key not in created_weeks:
|
||||
# Get the date of the first monday of the week
|
||||
week_start_date = current_date - timedelta(days=current_date.weekday())
|
||||
# Generate storage path for week node using Supabase Storage
|
||||
if storage_tools:
|
||||
week_dir_created, node_storage_path = storage_tools.create_calendar_week_storage_path(iso_year, iso_week)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
week_node = calendar_schemas.CalendarWeekNode(
|
||||
unique_id=calendar_week_unique_id,
|
||||
uuid_string=calendar_week_uuid_string,
|
||||
start_date=week_start_date,
|
||||
week_number=str(iso_week),
|
||||
iso_week=f"{iso_year}-W{iso_week:02}",
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(week_node, database=db_name, operation='merge')
|
||||
calendar_nodes['calendar_week_nodes'].append(week_node)
|
||||
created_weeks[week_key] = week_node
|
||||
logger.info(f"Week node created: {week_node.unique_id}")
|
||||
logger.info(f"Week node created: {week_node.uuid_string}")
|
||||
|
||||
if last_week_node and ((last_week_node.iso_week.split('-')[0] == str(iso_year) and int(last_week_node.week_number) == int(iso_week) - 1) or
|
||||
(last_week_node.iso_week.split('-')[0] != str(iso_year) and int(last_week_node.week_number) == 52 and int(iso_week) == 1)):
|
||||
@@ -173,7 +158,7 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {last_week_node.unique_id} to {week_node.unique_id}")
|
||||
logger.info(f"Relationship created from {last_week_node.uuid_string} to {week_node.uuid_string}")
|
||||
last_week_node = week_node
|
||||
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
@@ -181,23 +166,32 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {year_node.unique_id} to {week_node.unique_id}")
|
||||
logger.info(f"Relationship created from {year_node.uuid_string} to {week_node.uuid_string}")
|
||||
|
||||
# Day node management
|
||||
calendar_day_unique_id = f"{year}_{month}_{day}"
|
||||
calendar_day_uuid_string = f"{year}_{month}_{day}"
|
||||
|
||||
day_key = f"{year}-{month}-{day}"
|
||||
# Generate storage path for day node using Supabase Storage
|
||||
if storage_tools:
|
||||
day_dir_created, node_storage_path = storage_tools.create_calendar_day_storage_path(year, month, day)
|
||||
# Store day path for later use in time chunks
|
||||
created_days[day_key] = {'node': None, 'path': node_storage_path}
|
||||
else:
|
||||
node_storage_path = ""
|
||||
created_days[day_key] = {'node': None, 'path': None}
|
||||
|
||||
day_node = calendar_schemas.CalendarDayNode(
|
||||
unique_id=calendar_day_unique_id,
|
||||
uuid_string=calendar_day_uuid_string,
|
||||
date=current_date,
|
||||
day_of_week=current_date.strftime('%A'),
|
||||
iso_day=f"{year}-{month:02}-{day:02}",
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(day_node, database=db_name, operation='merge')
|
||||
calendar_nodes['calendar_day_nodes'].append(day_node)
|
||||
created_days[day_key] = day_node
|
||||
logger.info(f"Day node created: {day_node.unique_id}")
|
||||
created_days[day_key]['node'] = day_node
|
||||
logger.info(f"Day node created: {day_node.uuid_string}")
|
||||
|
||||
if last_day_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
@@ -205,7 +199,7 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {last_day_node.unique_id} to {day_node.unique_id}")
|
||||
logger.info(f"Relationship created from {last_day_node.uuid_string} to {day_node.uuid_string}")
|
||||
last_day_node = day_node
|
||||
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
@@ -213,13 +207,13 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {month_node.unique_id} to {day_node.unique_id}")
|
||||
logger.info(f"Relationship created from {month_node.uuid_string} to {day_node.uuid_string}")
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
calendar_relationships.WeekIncludesDay(source=week_node, target=day_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {week_node.unique_id} to {day_node.unique_id}")
|
||||
logger.info(f"Relationship created from {week_node.uuid_string} to {day_node.uuid_string}")
|
||||
current_date += timedelta(days=1)
|
||||
|
||||
if time_chunk_node_length:
|
||||
@@ -228,25 +222,32 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
|
||||
for day_node in calendar_nodes['calendar_day_nodes']:
|
||||
total_time_chunks_in_day = (24 * 60) / time_chunk_interval
|
||||
for i in range(total_time_chunks_in_day):
|
||||
time_chunk_unique_id = f"{day_node.unique_id}_{i}"
|
||||
time_chunk_uuid_string = f"{day_node.uuid_string}_{i}"
|
||||
time_chunk_start_time = day_node.date.time() + timedelta(minutes=i * time_chunk_interval)
|
||||
time_chunk_end_time = time_chunk_start_time + timedelta(minutes=time_chunk_interval)
|
||||
# Generate storage path for time chunk node using Supabase Storage
|
||||
if storage_tools:
|
||||
chunk_id = f"{day_node.uuid_string}_{i:02d}"
|
||||
chunk_dir_created, node_storage_path = storage_tools.create_calendar_time_chunk_storage_path(day_node.uuid_string, i)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
time_chunk_node = calendar_schemas.CalendarTimeChunkNode(
|
||||
unique_id=time_chunk_unique_id,
|
||||
uuid_string=time_chunk_uuid_string,
|
||||
start_time=time_chunk_start_time,
|
||||
end_time=time_chunk_end_time,
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(time_chunk_node, database=db_name, operation='merge')
|
||||
calendar_nodes['calendar_time_chunk_nodes'].append(time_chunk_node)
|
||||
logger.info(f"Time chunk node created: {time_chunk_node.unique_id}")
|
||||
logger.info(f"Time chunk node created: {time_chunk_node.uuid_string}")
|
||||
# Create a relationship between the time chunk node and the day node
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
calendar_relationships.DayIncludesTimeChunk(source=day_node, target=time_chunk_node),
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {day_node.unique_id} to {time_chunk_node.unique_id}")
|
||||
logger.info(f"Relationship created from {day_node.uuid_string} to {time_chunk_node.uuid_string}")
|
||||
# Create sequential relationship between the time chunk nodes
|
||||
if i > 0:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
@@ -254,7 +255,7 @@ def create_calendar(db_name, start_date, end_date, attach_to_calendar_node=False
|
||||
database=db_name,
|
||||
operation='merge'
|
||||
)
|
||||
logger.info(f"Relationship created from {calendar_nodes['calendar_time_chunk_nodes'][i-1].unique_id} to {time_chunk_node.unique_id}")
|
||||
logger.info(f"Relationship created from {calendar_nodes['calendar_time_chunk_nodes'][i-1].uuid_string} to {time_chunk_node.uuid_string}")
|
||||
|
||||
logger.info(f'Created calendar: {calendar_nodes["calendar_node"].unique_id}')
|
||||
logger.info(f'Calendar structure created successfully for {start_date} to {end_date}')
|
||||
return calendar_nodes
|
||||
@@ -2,115 +2,31 @@ import os
|
||||
from modules.logger_tool import initialise_logger
|
||||
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
from modules.database.schemas.nodes.schools.schools import SchoolNode
|
||||
from modules.database.supabase.utils.client import SupabaseServiceRoleClient, CreateBucketOptions
|
||||
import modules.database.init.init_school_timetable as init_school_timetable
|
||||
import modules.database.tools.neontology_tools as neon
|
||||
import modules.database.tools.supabase_storage_tools as storage_tools
|
||||
|
||||
def create_school_buckets(school_id: str, school_type: str, school_name: str, admin_access_token: str) -> dict:
|
||||
"""Create storage buckets for a school
|
||||
Args:
|
||||
school_id: The unique identifier for the school
|
||||
school_type: The type of school (e.g., 'development')
|
||||
school_name: The display name of the school
|
||||
admin_access_token: The admin access token for Supabase operations
|
||||
Returns:
|
||||
Dictionary containing results of bucket creation operations
|
||||
"""
|
||||
logger.info(f"Creating storage buckets for school {school_name} ({school_type}/{school_id})")
|
||||
def create_school(db_name: str, uuid_string: str, name: str, website: str, school_type: str, is_public: bool = True, school_node: SchoolNode | None = None, dataframes=None):
|
||||
if not name or not uuid_string or not website or not school_type:
|
||||
logger.error("School name, uuid_string, website and school_type are required to create a school.")
|
||||
raise ValueError("School name, uuid_string, website and school_type are required to create a school.")
|
||||
|
||||
storage_client = SupabaseServiceRoleClient.for_admin(admin_access_token)
|
||||
base_path = f"cc.institutes.{school_type}.{school_id}"
|
||||
|
||||
buckets = [
|
||||
# Main school buckets
|
||||
{
|
||||
"id": f"{base_path}.public",
|
||||
"options": CreateBucketOptions(
|
||||
name=f"{school_type.title()} School Files - {school_name} - Public Files",
|
||||
public=True,
|
||||
file_size_limit=50 * 1024 * 1024,
|
||||
allowed_mime_types=[
|
||||
'image/*', 'video/*', 'application/pdf',
|
||||
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
|
||||
'text/plain', 'text/csv', 'application/json'
|
||||
]
|
||||
)
|
||||
},
|
||||
{
|
||||
"id": f"{base_path}.private",
|
||||
"options": CreateBucketOptions(
|
||||
name=f"{school_type.title()} School Files - {school_name} - Private Files",
|
||||
public=False,
|
||||
file_size_limit=50 * 1024 * 1024,
|
||||
allowed_mime_types=[
|
||||
'image/*', 'video/*', 'application/pdf',
|
||||
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
|
||||
'text/plain', 'text/csv', 'application/json'
|
||||
]
|
||||
)
|
||||
},
|
||||
# Curriculum buckets
|
||||
{
|
||||
"id": f"{base_path}.curriculum.public",
|
||||
"options": CreateBucketOptions(
|
||||
name=f"{school_type.title()} School Files - {school_name} - Curriculum Public Files",
|
||||
public=True,
|
||||
file_size_limit=50 * 1024 * 1024,
|
||||
allowed_mime_types=[
|
||||
'image/*', 'video/*', 'application/pdf',
|
||||
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
|
||||
'text/plain', 'text/csv', 'application/json'
|
||||
]
|
||||
)
|
||||
},
|
||||
{
|
||||
"id": f"{base_path}.curriculum.private",
|
||||
"options": CreateBucketOptions(
|
||||
name=f"{school_type.title()} School Files - {school_name} - Curriculum Private Files",
|
||||
public=False,
|
||||
file_size_limit=50 * 1024 * 1024,
|
||||
allowed_mime_types=[
|
||||
'image/*', 'video/*', 'application/pdf',
|
||||
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
|
||||
'text/plain', 'text/csv', 'application/json'
|
||||
]
|
||||
)
|
||||
}
|
||||
]
|
||||
|
||||
results = {}
|
||||
for bucket in buckets:
|
||||
try:
|
||||
result = storage_client.create_bucket(bucket["id"], bucket["options"])
|
||||
results[bucket["id"]] = {
|
||||
"status": "success",
|
||||
"result": result
|
||||
}
|
||||
logger.info(f"Successfully created bucket {bucket['id']}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating school bucket {bucket['id']}: {str(e)}")
|
||||
results[bucket["id"]] = {
|
||||
"status": "error",
|
||||
"error": str(e)
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
def create_school(db_name: str, id: str, name: str, website: str, school_type: str, is_public: bool = True, school_node: SchoolNode | None = None, dataframes=None):
|
||||
if not name or not id or not website or not school_type:
|
||||
logger.error("School name, id, website and school_type are required to create a school.")
|
||||
raise ValueError("School name, id, website and school_type are required to create a school.")
|
||||
|
||||
logger.info(f"Initialising neo4j connection...")
|
||||
logger.info(f"Initialising Neontology connection...")
|
||||
neon.init_neontology_connection()
|
||||
|
||||
# Initialize storage tools for school
|
||||
storage_tools_instance = storage_tools.SupabaseStorageTools(db_name, init_run_type="school")
|
||||
|
||||
# Generate the storage path for the school node
|
||||
school_dir_created, node_storage_path = storage_tools_instance.create_school_storage_path(uuid_string)
|
||||
logger.info(f"Generated school storage path: {node_storage_path}")
|
||||
|
||||
# Create School Node if not provided
|
||||
if not school_node:
|
||||
if is_public:
|
||||
school_node = SchoolNode(
|
||||
unique_id=f'School_{id}',
|
||||
tldraw_snapshot="",
|
||||
id=id,
|
||||
uuid_string=uuid_string,
|
||||
node_storage_path=node_storage_path,
|
||||
name=name,
|
||||
website=website,
|
||||
school_type=school_type
|
||||
@@ -118,9 +34,8 @@ def create_school(db_name: str, id: str, name: str, website: str, school_type: s
|
||||
else:
|
||||
# Create private school node with default values
|
||||
school_node = SchoolNode(
|
||||
unique_id=f'School_{id}',
|
||||
tldraw_snapshot="",
|
||||
id=id,
|
||||
uuid_string=uuid_string,
|
||||
node_storage_path=node_storage_path,
|
||||
name=name,
|
||||
website=website,
|
||||
school_type=school_type,
|
||||
@@ -133,6 +48,9 @@ def create_school(db_name: str, id: str, name: str, website: str, school_type: s
|
||||
statutory_high_age=18,
|
||||
school_capacity=1000
|
||||
)
|
||||
else:
|
||||
# Update existing school node with the storage path
|
||||
school_node.node_storage_path = node_storage_path
|
||||
|
||||
# First create/merge the school node in the main cc.institutes database
|
||||
logger.info(f"Creating school node in main cc.institutes database...")
|
||||
@@ -144,15 +62,16 @@ def create_school(db_name: str, id: str, name: str, website: str, school_type: s
|
||||
|
||||
school_nodes = {
|
||||
'school_node': school_node,
|
||||
'db_name': db_name
|
||||
'db_name': db_name,
|
||||
'storage_path': node_storage_path
|
||||
}
|
||||
|
||||
if dataframes is not None:
|
||||
logger.info(f"Creating school timetable for {name} with {len(dataframes)} dataframes...")
|
||||
school_timetable_nodes = init_school_timetable.create_school_timetable(dataframes, db_name, school_node)
|
||||
school_timetable_nodes = init_school_timetable.create_school_timetable(dataframes, db_name, school_node, storage_tools_instance)
|
||||
school_nodes['school_timetable_nodes'] = school_timetable_nodes
|
||||
else:
|
||||
logger.warning(f"No dataframes provided for {name}, skipping school timetable...")
|
||||
|
||||
logger.info(f"School {name} created successfully...")
|
||||
logger.info(f"School {name} created successfully with storage path: {node_storage_path}")
|
||||
return school_nodes
|
||||
|
||||
@@ -4,6 +4,7 @@ logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH
|
||||
import pandas as pd
|
||||
|
||||
import modules.database.tools.neontology_tools as neon
|
||||
import modules.database.tools.supabase_storage_tools as storage_tools
|
||||
import modules.database.schemas.nodes.schools.schools as school_nodes
|
||||
import modules.database.schemas.nodes.schools.curriculum as curriculum_nodes
|
||||
import modules.database.schemas.nodes.schools.pastoral as pastoral_nodes
|
||||
@@ -40,7 +41,7 @@ def sort_year_groups(df):
|
||||
df['YearGroupNumeric'] = pd.to_numeric(df['YearGroup'], errors='coerce')
|
||||
return df.sort_values(by='YearGroupNumeric')
|
||||
|
||||
def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_node: school_nodes.SchoolNode):
|
||||
def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_node: school_nodes.SchoolNode, storage_tools=None):
|
||||
|
||||
logger.info(f"Initialising neo4j connection...")
|
||||
neon.init_neontology_connection()
|
||||
@@ -70,10 +71,20 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
last_key_stage_node = None
|
||||
|
||||
# Create Department Structure node
|
||||
department_structure_node_unique_id = f"DepartmentStructure_{school_node.unique_id}"
|
||||
department_structure_node_uuid_string = f"DepartmentStructure_{school_node.uuid_string}"
|
||||
|
||||
# For structure nodes, we can use a simple path or leave empty for consistency
|
||||
# Since this is just an organizational node, we'll use a simple path
|
||||
if storage_tools:
|
||||
# Use the school's base department directory as the structure node path
|
||||
dept_structure_path = f"cc.public.snapshots/DepartmentStructure/{school_node.uuid_string}"
|
||||
node_storage_path = dept_structure_path
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
department_structure_node = school_structures.DepartmentStructureNode(
|
||||
unique_id=department_structure_node_unique_id,
|
||||
tldraw_snapshot=""
|
||||
uuid_string=department_structure_node_uuid_string,
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
# Create in school database only
|
||||
neon.create_or_merge_neontology_node(department_structure_node, database=db_name, operation='merge')
|
||||
@@ -86,10 +97,17 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
)
|
||||
logger.info(f"Created department structure node and linked to school")
|
||||
|
||||
curriculum_structure_node_unique_id = f"CurriculumStructure_{school_node.unique_id}"
|
||||
curriculum_structure_node_uuid_string = f"CurriculumStructure_{school_node.uuid_string}"
|
||||
|
||||
# Generate storage path for curriculum structure node
|
||||
if storage_tools:
|
||||
curriculum_dir_created, node_storage_path = storage_tools.create_curriculum_storage_path(curriculum_structure_node_uuid_string)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
curriculum_node = school_structures.CurriculumStructureNode(
|
||||
unique_id=curriculum_structure_node_unique_id,
|
||||
tldraw_snapshot=""
|
||||
uuid_string=curriculum_structure_node_uuid_string,
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
# Create in school database only
|
||||
neon.create_or_merge_neontology_node(curriculum_node, database=db_name, operation='merge')
|
||||
@@ -102,10 +120,17 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
)
|
||||
logger.info(f"Created curriculum node and relationship with school")
|
||||
|
||||
pastoral_structure_node_unique_id = f"PastoralStructure_{school_node.unique_id}"
|
||||
pastoral_structure_node_uuid_string = f"PastoralStructure_{school_node.uuid_string}"
|
||||
|
||||
# Generate storage path for pastoral structure node
|
||||
if storage_tools:
|
||||
pastoral_dir_created, node_storage_path = storage_tools.create_pastoral_storage_path(pastoral_structure_node_uuid_string)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
pastoral_node = school_structures.PastoralStructureNode(
|
||||
unique_id=pastoral_structure_node_unique_id,
|
||||
tldraw_snapshot=""
|
||||
uuid_string=pastoral_structure_node_uuid_string,
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(pastoral_node, database=db_name, operation='merge')
|
||||
node_library['pastoral_node'] = pastoral_node
|
||||
@@ -120,11 +145,20 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
unique_departments = keystagesyllabus_df['Department'].dropna().unique()
|
||||
|
||||
for department_name in unique_departments:
|
||||
department_unique_id = f"Department_{school_node.unique_id}_{department_name.replace(' ', '_')}"
|
||||
department_uuid_string = f"Department_{school_node.uuid_string}_{department_name.replace(' ', '_')}"
|
||||
|
||||
# Generate storage path for department node using the storage tools
|
||||
if storage_tools:
|
||||
# Create department directory under the school's department structure
|
||||
dept_path = f"cc.public.snapshots/Department/{department_uuid_string}"
|
||||
node_storage_path = dept_path
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
department_node = school_nodes.DepartmentNode(
|
||||
unique_id=department_unique_id,
|
||||
uuid_string=department_uuid_string,
|
||||
name=department_name,
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
# Create department in school database only
|
||||
neon.create_or_merge_neontology_node(department_node, database=db_name, operation='merge')
|
||||
@@ -147,17 +181,25 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
|
||||
# Process subjects from key stage syllabuses first (these have department info)
|
||||
for _, subject_row in unique_subjects.iterrows():
|
||||
subject_unique_id = f"Subject_{school_node.unique_id}_{subject_row['SubjectCode']}"
|
||||
subject_uuid_string = f"Subject_{school_node.uuid_string}_{subject_row['SubjectCode']}"
|
||||
department_node = node_library['department_nodes'].get(subject_row['Department'])
|
||||
if not department_node:
|
||||
logger.warning(f"No department found for subject {subject_row['Subject']} with code {subject_row['SubjectCode']}")
|
||||
continue
|
||||
|
||||
# Generate storage path for subject node
|
||||
if storage_tools:
|
||||
# Create subject directory under the specific department
|
||||
subject_path = f"cc.public.snapshots/Subject/{subject_uuid_string}"
|
||||
node_storage_path = subject_path
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
subject_node = curriculum_nodes.SubjectNode(
|
||||
unique_id=subject_unique_id,
|
||||
uuid_string=subject_uuid_string,
|
||||
id=subject_row['SubjectCode'],
|
||||
name=subject_row['Subject'],
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
# Create subject in both databases
|
||||
neon.create_or_merge_neontology_node(subject_node, database=db_name, operation='merge')
|
||||
@@ -173,14 +215,23 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
|
||||
# Process any additional subjects from year group syllabuses (these won't have department info)
|
||||
for _, subject_row in additional_subjects.iterrows():
|
||||
subject_unique_id = f"Subject_{school_node.unique_id}_{subject_row['SubjectCode']}"
|
||||
subject_uuid_string = f"Subject_{school_node.uuid_string}_{subject_row['SubjectCode']}"
|
||||
# Create in a special "Unassigned" department
|
||||
unassigned_dept_name = "Unassigned Department"
|
||||
if unassigned_dept_name not in node_library['department_nodes']:
|
||||
# Generate storage path for unassigned department node
|
||||
if filesystem:
|
||||
# Create unassigned department directory under the school's department structure
|
||||
unassigned_dept_path = os.path.join(filesystem.root_path, "departments", "Unassigned")
|
||||
filesystem.create_directory(unassigned_dept_path)
|
||||
node_storage_path = os.path.relpath(unassigned_dept_path, filesystem.base_path)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
department_node = school_nodes.DepartmentNode(
|
||||
unique_id=f"Department_{school_node.unique_id}_Unassigned",
|
||||
uuid_string=f"Department_{school_node.uuid_string}_Unassigned",
|
||||
name=unassigned_dept_name,
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(department_node, database=db_name, operation='merge')
|
||||
node_library['department_nodes'][unassigned_dept_name] = department_node
|
||||
@@ -192,11 +243,20 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
)
|
||||
logger.info(f"Created unassigned department node and linked to department structure")
|
||||
|
||||
# Generate storage path for subject node
|
||||
if filesystem:
|
||||
# Create subject directory under the unassigned department
|
||||
subject_path = os.path.join(filesystem.root_path, "departments", "Unassigned", subject_row['Subject'].replace(' ', '_'))
|
||||
filesystem.create_directory(subject_path)
|
||||
node_storage_path = os.path.relpath(subject_path, filesystem.base_path)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
subject_node = curriculum_nodes.SubjectNode(
|
||||
unique_id=subject_unique_id,
|
||||
uuid_string=subject_uuid_string,
|
||||
id=subject_row['SubjectCode'],
|
||||
name=subject_row['Subject'],
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
# Create subject in both databases
|
||||
neon.create_or_merge_neontology_node(subject_node, database=db_name, operation='merge')
|
||||
@@ -226,20 +286,29 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
# First create all key stage nodes and key stage syllabus nodes
|
||||
for index, ks_row in keystagesyllabus_df.sort_values('KeyStage').iterrows():
|
||||
key_stage = str(ks_row['KeyStage'])
|
||||
logger.debug(f"Processing key stage syllabus row - Subject: {ks_row['Subject']}, Key Stage: {key_stage}")
|
||||
subject = str(ks_row['Subject'])
|
||||
syllabus_id = str(ks_row['ID'])
|
||||
logger.debug(f"Processing key stage syllabus row - Subject: {subject}, Key Stage: {key_stage}, Syllabus ID: {syllabus_id}")
|
||||
|
||||
subject_node = node_library['subject_nodes'].get(ks_row['Subject'])
|
||||
subject_node = node_library['subject_nodes'].get(subject)
|
||||
if not subject_node:
|
||||
logger.warning(f"No subject node found for subject {ks_row['Subject']}")
|
||||
logger.warning(f"No subject node found for subject {subject}")
|
||||
continue
|
||||
|
||||
if key_stage not in key_stage_nodes_created:
|
||||
key_stage_node_unique_id = f"KeyStage_{curriculum_node.unique_id}_KStg{key_stage}"
|
||||
key_stage_node_uuid_string = f"KeyStage_{curriculum_node.uuid_string}_KStg{key_stage}"
|
||||
# Generate storage path for key stage node
|
||||
if filesystem:
|
||||
key_stage_dir_created, key_stage_path = filesystem.create_curriculum_key_stage_syllabus_directory(curriculum_path, key_stage, subject, syllabus_id)
|
||||
node_storage_path = os.path.relpath(key_stage_path, filesystem.base_path)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
key_stage_node = curriculum_nodes.KeyStageNode(
|
||||
unique_id=key_stage_node_unique_id,
|
||||
uuid_string=key_stage_node_uuid_string,
|
||||
name=f"Key Stage {key_stage}",
|
||||
key_stage=str(key_stage),
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
# Create key stage node in both databases
|
||||
neon.create_or_merge_neontology_node(key_stage_node, database=db_name, operation='merge')
|
||||
@@ -252,7 +321,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
curriculum_relationships.CurriculumStructureIncludesKeyStage(source=curriculum_node, target=key_stage_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created key stage node {key_stage_node_unique_id} and relationship with curriculum structure")
|
||||
logger.info(f"Created key stage node {key_stage_node_uuid_string} and relationship with curriculum structure")
|
||||
|
||||
# Create sequential relationship between key stages in both databases
|
||||
if last_key_stage_node:
|
||||
@@ -264,27 +333,34 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
curriculum_relationships.KeyStageFollowsKeyStage(source=last_key_stage_node, target=key_stage_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created sequential relationship between key stages {last_key_stage_node.unique_id} and {key_stage_node.unique_id}")
|
||||
logger.info(f"Created sequential relationship between key stages {last_key_stage_node.uuid_string} and {key_stage_node.uuid_string}")
|
||||
last_key_stage_node = key_stage_node
|
||||
|
||||
# Create key stage syllabus under the subject's curriculum directory
|
||||
key_stage_syllabus_node_unique_id = f"KeyStageSyllabus_{curriculum_node.unique_id}_{ks_row['Title'].replace(' ', '')}"
|
||||
key_stage_syllabus_node_uuid_string = f"KeyStageSyllabus_{curriculum_node.uuid_string}_{ks_row['Title'].replace(' ', '')}"
|
||||
logger.debug(f"Creating key stage syllabus node for {ks_row['Subject']} KS{key_stage} with ID {ks_row['ID']}")
|
||||
|
||||
key_stage_syllabus_node_unique_id = f"KeyStageSyllabus_{curriculum_node.unique_id}_{ks_row['Title'].replace(' ', '')}"
|
||||
key_stage_syllabus_node_uuid_string = f"KeyStageSyllabus_{curriculum_node.uuid_string}_{ks_row['Title'].replace(' ', '')}"
|
||||
# Generate storage path for key stage syllabus node
|
||||
if filesystem:
|
||||
syllabus_dir_created, syllabus_path = filesystem.create_curriculum_key_stage_syllabus_directory(curriculum_path, key_stage, ks_row['Subject'], ks_row['ID'])
|
||||
node_storage_path = os.path.relpath(syllabus_path, filesystem.base_path)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
key_stage_syllabus_node = curriculum_nodes.KeyStageSyllabusNode(
|
||||
unique_id=key_stage_syllabus_node_unique_id,
|
||||
uuid_string=key_stage_syllabus_node_uuid_string,
|
||||
id=ks_row['ID'],
|
||||
name=ks_row['Title'],
|
||||
key_stage=str(ks_row['KeyStage']),
|
||||
subject_name=ks_row['Subject'],
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
# Create key stage syllabus node in both databases
|
||||
neon.create_or_merge_neontology_node(key_stage_syllabus_node, database=db_name, operation='merge')
|
||||
neon.create_or_merge_neontology_node(key_stage_syllabus_node, database=curriculum_db_name, operation='merge')
|
||||
node_library['key_stage_syllabus_nodes'][ks_row['ID']] = key_stage_syllabus_node
|
||||
logger.debug(f"Created key stage syllabus node {key_stage_syllabus_node_unique_id} for {ks_row['Subject']} KS{key_stage}")
|
||||
logger.debug(f"Created key stage syllabus node {key_stage_syllabus_node_uuid_string} for {ks_row['Subject']} KS{key_stage}")
|
||||
|
||||
# Link key stage syllabus to its subject in both databases
|
||||
if subject_node:
|
||||
@@ -296,7 +372,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
curriculum_relationships.SubjectHasKeyStageSyllabus(source=subject_node, target=key_stage_syllabus_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship between subject {subject_node.unique_id} and key stage syllabus {key_stage_syllabus_node.unique_id}")
|
||||
logger.info(f"Created relationship between subject {subject_node.uuid_string} and key stage syllabus {key_stage_syllabus_node.uuid_string}")
|
||||
|
||||
# Link key stage syllabus to its key stage in both databases
|
||||
key_stage_node = key_stage_nodes_created.get(key_stage)
|
||||
@@ -309,7 +385,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
curriculum_relationships.KeyStageIncludesKeyStageSyllabus(source=key_stage_node, target=key_stage_syllabus_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship between key stage {key_stage_node.unique_id} and key stage syllabus {key_stage_syllabus_node.unique_id}")
|
||||
logger.info(f"Created relationship between key stage {key_stage_node.uuid_string} and key stage syllabus {key_stage_syllabus_node.uuid_string}")
|
||||
|
||||
# Create sequential relationship between key stage syllabuses in both databases
|
||||
last_key_stage_syllabus_node = last_key_stage_syllabus_nodes.get(ks_row['Subject'])
|
||||
@@ -322,7 +398,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
curriculum_relationships.KeyStageSyllabusFollowsKeyStageSyllabus(source=last_key_stage_syllabus_node, target=key_stage_syllabus_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created sequential relationship between key stage syllabuses {last_key_stage_syllabus_node.unique_id} and {key_stage_syllabus_node.unique_id}")
|
||||
logger.info(f"Created sequential relationship between key stage syllabuses {last_key_stage_syllabus_node.uuid_string} and {key_stage_syllabus_node.uuid_string}")
|
||||
last_key_stage_syllabus_nodes[ks_row['Subject']] = key_stage_syllabus_node
|
||||
|
||||
# Now process year groups and their syllabuses
|
||||
@@ -339,12 +415,19 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
if pd.notna(numeric_year_group):
|
||||
numeric_year_group = int(numeric_year_group)
|
||||
if numeric_year_group not in year_group_nodes_created:
|
||||
year_group_node_unique_id = f"YearGroup_{school_node.unique_id}_YGrp{numeric_year_group}"
|
||||
year_group_node_uuid_string = f"YearGroup_{school_node.uuid_string}_YGrp{numeric_year_group}"
|
||||
# Generate storage path for year group node
|
||||
if filesystem:
|
||||
year_group_dir_created, year_group_path = filesystem.create_pastoral_year_group_directory(pastoral_path, numeric_year_group)
|
||||
node_storage_path = os.path.relpath(year_group_path, filesystem.base_path)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
year_group_node = pastoral_nodes.YearGroupNode(
|
||||
unique_id=year_group_node_unique_id,
|
||||
uuid_string=year_group_node_uuid_string,
|
||||
year_group=str(numeric_year_group),
|
||||
name=f"Year {numeric_year_group}, {year_group}",
|
||||
tldraw_snapshot=""
|
||||
name=f"Year {numeric_year_group}",
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
# Create year group node in both databases but use same directory
|
||||
neon.create_or_merge_neontology_node(year_group_node, database=db_name, operation='merge')
|
||||
@@ -360,7 +443,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
curriculum_relationships.YearGroupFollowsYearGroup(source=last_year_group_node, target=year_group_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created sequential relationship between year groups {last_year_group_node.unique_id} and {year_group_node.unique_id} across key stages")
|
||||
logger.info(f"Created sequential relationship between year groups {last_year_group_node.uuid_string} and {year_group_node.uuid_string} across key stages")
|
||||
last_year_group_node = year_group_node
|
||||
|
||||
# Create relationship with Pastoral Structure in school database only
|
||||
@@ -368,7 +451,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
curriculum_relationships.PastoralStructureIncludesYearGroup(source=pastoral_node, target=year_group_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created year group node {year_group_node_unique_id} and relationship with pastoral structure")
|
||||
logger.info(f"Created year group node {year_group_node_uuid_string} and relationship with pastoral structure")
|
||||
|
||||
year_group_nodes_created[numeric_year_group] = year_group_node
|
||||
node_library['year_group_nodes'][str(numeric_year_group)] = year_group_node
|
||||
@@ -376,14 +459,21 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
# Create year group syllabus nodes in both databases
|
||||
year_group_node = year_group_nodes_created.get(numeric_year_group)
|
||||
if year_group_node:
|
||||
year_group_syllabus_node_unique_id = f"YearGroupSyllabus_{school_node.unique_id}_{yg_row['ID']}"
|
||||
year_group_syllabus_node_uuid_string = f"YearGroupSyllabus_{school_node.uuid_string}_{yg_row['ID']}"
|
||||
# Generate storage path for year group syllabus node
|
||||
if filesystem:
|
||||
yg_syllabus_dir_created, yg_syllabus_path = filesystem.create_curriculum_year_group_syllabus_directory(curriculum_path, yg_row['Subject'], numeric_year_group, yg_row['ID'])
|
||||
node_storage_path = os.path.relpath(yg_syllabus_path, filesystem.base_path)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
year_group_syllabus_node = pastoral_nodes.YearGroupSyllabusNode(
|
||||
unique_id=year_group_syllabus_node_unique_id,
|
||||
uuid_string=year_group_syllabus_node_uuid_string,
|
||||
id=yg_row['ID'],
|
||||
name=yg_row['Title'],
|
||||
year_group=str(yg_row['YearGroup']),
|
||||
subject_name=yg_row['Subject'],
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
|
||||
# Create year group syllabus node in both databases but use same directory
|
||||
@@ -406,7 +496,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
curriculum_relationships.YearGroupSyllabusFollowsYearGroupSyllabus(source=last_year_group_syllabus_node, target=year_group_syllabus_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created sequential relationship between year group syllabuses {last_year_group_syllabus_node.unique_id} and {year_group_syllabus_node.unique_id}")
|
||||
logger.info(f"Created sequential relationship between year group syllabuses {last_year_group_syllabus_node.uuid_string} and {year_group_syllabus_node.uuid_string}")
|
||||
last_year_group_syllabus_nodes[yg_row['Subject']] = year_group_syllabus_node
|
||||
|
||||
# Create relationships in both databases using MATCH to avoid cartesian products
|
||||
@@ -421,7 +511,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
curriculum_relationships.SubjectHasYearGroupSyllabus(source=subject_node, target=year_group_syllabus_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship between subject {subject_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
|
||||
logger.info(f"Created relationship between subject {subject_node.uuid_string} and year group syllabus {year_group_syllabus_node_uuid_string}")
|
||||
|
||||
# Link to year group
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
@@ -432,7 +522,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
curriculum_relationships.YearGroupHasYearGroupSyllabus(source=year_group_node, target=year_group_syllabus_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship between year group {year_group_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
|
||||
logger.info(f"Created relationship between year group {year_group_node.uuid_string} and year group syllabus {year_group_syllabus_node_uuid_string}")
|
||||
|
||||
# Link to key stage syllabus if it exists for the same subject
|
||||
key_stage_syllabus_node = node_library['key_stage_syllabus_nodes'].get(ks_row['ID'])
|
||||
@@ -445,7 +535,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
curriculum_relationships.KeyStageSyllabusIncludesYearGroupSyllabus(source=key_stage_syllabus_node, target=year_group_syllabus_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship between key stage syllabus {key_stage_syllabus_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
|
||||
logger.info(f"Created relationship between key stage syllabus {key_stage_syllabus_node.uuid_string} and year group syllabus {year_group_syllabus_node_uuid_string}")
|
||||
|
||||
# Process topics for this year group syllabus only if not already processed
|
||||
topics_for_syllabus = topic_df[topic_df['SyllabusYearID'] == yg_row['ID']]
|
||||
@@ -472,22 +562,29 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
if (syllabus_node.subject_name == topic_subject and
|
||||
syllabus_node.key_stage == str(topic_key_stage)):
|
||||
matching_syllabus_node = syllabus_node
|
||||
logger.debug(f"Found matching syllabus node: {syllabus_node.unique_id}")
|
||||
logger.debug(f"Found matching syllabus node: {syllabus_node.uuid_string}")
|
||||
break
|
||||
|
||||
if not matching_syllabus_node:
|
||||
logger.warning(f"No key stage syllabus node found for subject {topic_subject} and key stage {topic_key_stage}, skipping topic creation")
|
||||
continue
|
||||
|
||||
topic_node_unique_id = f"Topic_{matching_syllabus_node.unique_id}_{topic_row['TopicID']}"
|
||||
topic_node_uuid_string = f"Topic_{matching_syllabus_node.uuid_string}_{topic_row['TopicID']}"
|
||||
# Generate storage path for topic node
|
||||
if filesystem:
|
||||
topic_dir_created, topic_path = filesystem.create_curriculum_topic_directory(yg_syllabus_path, topic_row['TopicID'])
|
||||
node_storage_path = os.path.relpath(topic_path, filesystem.base_path)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
topic_node = curriculum_nodes.TopicNode(
|
||||
unique_id=topic_node_unique_id,
|
||||
uuid_string=topic_node_uuid_string,
|
||||
id=topic_row['TopicID'],
|
||||
name=topic_row.get('TopicTitle', default_topic_values['topic_title']),
|
||||
total_number_of_lessons_for_topic=str(topic_row.get('TotalNumberOfLessonsForTopic', default_topic_values['total_number_of_lessons_for_topic'])),
|
||||
type=topic_row.get('TopicType', default_topic_values['topic_type']),
|
||||
assessment_type=topic_row.get('TopicAssessmentType', default_topic_values['topic_assessment_type']),
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
# Create topic node in curriculum database only
|
||||
neon.create_or_merge_neontology_node(topic_node, database=curriculum_db_name, operation='merge')
|
||||
@@ -502,7 +599,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
curriculum_relationships.YearGroupSyllabusIncludesTopic(source=year_group_syllabus_node, target=topic_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationships between topic {topic_node_unique_id} and key stage syllabus {matching_syllabus_node.unique_id} and year group syllabus {year_group_syllabus_node_unique_id}")
|
||||
logger.info(f"Created relationships between topic {topic_node_uuid_string} and key stage syllabus {matching_syllabus_node.uuid_string} and year group syllabus {year_group_syllabus_node_uuid_string}")
|
||||
|
||||
# Process lessons for this topic only if not already processed
|
||||
lessons_for_topic = lesson_df[
|
||||
@@ -518,8 +615,15 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
continue
|
||||
lessons_processed.add(lesson_row['LessonID'])
|
||||
|
||||
# Generate storage path for lesson node
|
||||
if filesystem:
|
||||
lesson_dir_created, lesson_path = filesystem.create_curriculum_lesson_directory(topic_path, lesson_row['LessonID'])
|
||||
node_storage_path = os.path.relpath(lesson_path, filesystem.base_path)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
lesson_node = curriculum_nodes.TopicLessonNode(
|
||||
unique_id=f"TopicLesson_{topic_node_unique_id}_{lesson_row['LessonID']}",
|
||||
uuid_string=f"TopicLesson_{topic_node_uuid_string}_{lesson_row['LessonID']}",
|
||||
id=lesson_row['LessonID'],
|
||||
name=lesson_row.get('LessonTitle', default_topic_lesson_values['topic_lesson_title']),
|
||||
type=lesson_row.get('LessonType', default_topic_lesson_values['topic_lesson_type']),
|
||||
@@ -527,7 +631,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
suggested_activities=str(lesson_row.get('SuggestedActivities', default_topic_lesson_values['topic_lesson_suggested_activities'])),
|
||||
skills_learned=str(lesson_row.get('SkillsLearned', default_topic_lesson_values['topic_lesson_skills_learned'])),
|
||||
weblinks=str(lesson_row.get('WebLinks', default_topic_lesson_values['topic_lesson_weblinks'])),
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
# Create lesson node in curriculum database only
|
||||
neon.create_or_merge_neontology_node(lesson_node, database=curriculum_db_name, operation='merge')
|
||||
@@ -538,7 +642,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
curriculum_relationships.TopicIncludesTopicLesson(source=topic_node, target=lesson_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created lesson node {lesson_node.unique_id} and relationship with topic {topic_node.unique_id}")
|
||||
logger.info(f"Created lesson node {lesson_node.uuid_string} and relationship with topic {topic_node.uuid_string}")
|
||||
|
||||
# Create sequential relationships between lessons
|
||||
if lesson_row['Lesson'].isdigit() and previous_lesson_node:
|
||||
@@ -546,7 +650,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
curriculum_relationships.TopicLessonFollowsTopicLesson(source=previous_lesson_node, target=lesson_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.unique_id} and {lesson_node.unique_id}")
|
||||
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.uuid_string} and {lesson_node.uuid_string}")
|
||||
previous_lesson_node = lesson_node
|
||||
|
||||
# Process learning statements for this lesson only if not already processed
|
||||
@@ -558,12 +662,19 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
if statement_row['StatementID'] in statements_processed:
|
||||
continue
|
||||
statements_processed.add(statement_row['StatementID'])
|
||||
# Generate storage path for learning statement node
|
||||
if filesystem:
|
||||
statement_dir_created, statement_path = filesystem.create_curriculum_learning_statement_directory(lesson_path, statement_row['StatementID'])
|
||||
node_storage_path = os.path.relpath(statement_path, filesystem.base_path)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
statement_node = curriculum_nodes.LearningStatementNode(
|
||||
unique_id=f"LearningStatement_{lesson_node.unique_id}_{statement_row['StatementID']}",
|
||||
uuid_string=f"LearningStatement_{lesson_node.uuid_string}_{statement_row['StatementID']}",
|
||||
id=statement_row['StatementID'],
|
||||
name=statement_row.get('LearningStatement', default_learning_statement_values['lesson_learning_statement']),
|
||||
type=statement_row.get('StatementType', default_learning_statement_values['lesson_learning_statement_type']),
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
# Create statement node in curriculum database only
|
||||
neon.create_or_merge_neontology_node(statement_node, database=curriculum_db_name, operation='merge')
|
||||
@@ -574,7 +685,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
curriculum_relationships.LessonIncludesLearningStatement(source=lesson_node, target=statement_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created learning statement node {statement_node.unique_id} and relationship with lesson {lesson_node.unique_id}")
|
||||
logger.info(f"Created learning statement node {statement_node.uuid_string} and relationship with lesson {lesson_node.uuid_string}")
|
||||
else:
|
||||
logger.warning(f"No year group node found for year group {year_group}, skipping syllabus creation")
|
||||
|
||||
@@ -601,15 +712,23 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
logger.warning(f"No key stage syllabus node found for subject {topic_subject} and key stage {topic_key_stage}, skipping topic creation")
|
||||
continue
|
||||
|
||||
topic_node_unique_id = f"Topic_{matching_syllabus_node.unique_id}_{topic_row['TopicID']}"
|
||||
topic_node_uuid_string = f"Topic_{matching_syllabus_node.uuid_string}_{topic_row['TopicID']}"
|
||||
# Generate storage path for topic node
|
||||
if filesystem:
|
||||
syllabus_path = os.path.join(curriculum_path, "subjects", topic_subject, "key_stage_syllabuses", f"KS{topic_key_stage}", f"KS{topic_key_stage}.{topic_subject}")
|
||||
topic_dir_created, keystage_topic_path = filesystem.create_curriculum_keystage_topic_directory(syllabus_path, topic_row['TopicID'])
|
||||
node_storage_path = os.path.relpath(keystage_topic_path, filesystem.base_path)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
topic_node = curriculum_nodes.TopicNode(
|
||||
unique_id=topic_node_unique_id,
|
||||
uuid_string=topic_node_uuid_string,
|
||||
id=topic_row['TopicID'],
|
||||
name=topic_row.get('TopicTitle', default_topic_values['topic_title']),
|
||||
total_number_of_lessons_for_topic=str(topic_row.get('TotalNumberOfLessonsForTopic', default_topic_values['total_number_of_lessons_for_topic'])),
|
||||
type=topic_row.get('TopicType', default_topic_values['topic_type']),
|
||||
assessment_type=topic_row.get('TopicAssessmentType', default_topic_values['topic_assessment_type']),
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
# Create topic node in curriculum database only
|
||||
neon.create_or_merge_neontology_node(topic_node, database=curriculum_db_name, operation='merge')
|
||||
@@ -621,7 +740,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
curriculum_relationships.KeyStageSyllabusIncludesTopic(source=matching_syllabus_node, target=topic_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship between topic {topic_node_unique_id} and key stage syllabus {matching_syllabus_node.unique_id}")
|
||||
logger.info(f"Created relationship between topic {topic_node_uuid_string} and key stage syllabus {matching_syllabus_node.uuid_string}")
|
||||
|
||||
# Process lessons for this topic
|
||||
lessons_for_topic = lesson_df[
|
||||
@@ -636,8 +755,15 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
if lesson_row['LessonID'] in lessons_processed:
|
||||
continue
|
||||
lessons_processed.add(lesson_row['LessonID'])
|
||||
# Generate storage path for lesson node
|
||||
if filesystem:
|
||||
lesson_dir_created, lesson_path = filesystem.create_curriculum_lesson_directory(topic_path, lesson_row['LessonID'])
|
||||
node_storage_path = os.path.relpath(lesson_path, filesystem.base_path)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
lesson_node = curriculum_nodes.TopicLessonNode(
|
||||
unique_id=f"TopicLesson_{topic_node_unique_id}_{lesson_row['LessonID']}",
|
||||
uuid_string=f"TopicLesson_{topic_node_uuid_string}_{lesson_row['LessonID']}",
|
||||
id=lesson_row['LessonID'],
|
||||
name=lesson_row.get('LessonTitle', default_topic_lesson_values['topic_lesson_title']),
|
||||
type=lesson_row.get('LessonType', default_topic_lesson_values['topic_lesson_type']),
|
||||
@@ -645,7 +771,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
suggested_activities=str(lesson_row.get('SuggestedActivities', default_topic_lesson_values['topic_lesson_suggested_activities'])),
|
||||
skills_learned=str(lesson_row.get('SkillsLearned', default_topic_lesson_values['topic_lesson_skills_learned'])),
|
||||
weblinks=str(lesson_row.get('WebLinks', default_topic_lesson_values['topic_lesson_weblinks'])),
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
# Create lesson node in curriculum database only
|
||||
neon.create_or_merge_neontology_node(lesson_node, database=curriculum_db_name, operation='merge')
|
||||
@@ -656,7 +782,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
curriculum_relationships.TopicIncludesTopicLesson(source=topic_node, target=lesson_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created lesson node {lesson_node.unique_id} and relationship with topic {topic_node.unique_id}")
|
||||
logger.info(f"Created lesson node {lesson_node.uuid_string} and relationship with topic {topic_node.uuid_string}")
|
||||
|
||||
# Create sequential relationships between lessons
|
||||
if lesson_row['Lesson'].isdigit() and previous_lesson_node:
|
||||
@@ -664,7 +790,7 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
curriculum_relationships.TopicLessonFollowsTopicLesson(source=previous_lesson_node, target=lesson_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.unique_id} and {lesson_node.unique_id}")
|
||||
logger.info(f"Created sequential relationship between lessons {previous_lesson_node.uuid_string} and {lesson_node.uuid_string}")
|
||||
previous_lesson_node = lesson_node
|
||||
|
||||
# Process learning statements for this lesson
|
||||
@@ -676,12 +802,19 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
if statement_row['StatementID'] in statements_processed:
|
||||
continue
|
||||
statements_processed.add(statement_row['StatementID'])
|
||||
# Generate storage path for learning statement node
|
||||
if filesystem:
|
||||
statement_dir_created, statement_path = filesystem.create_curriculum_learning_statement_directory(lesson_path, statement_row['StatementID'])
|
||||
node_storage_path = os.path.relpath(statement_path, filesystem.base_path)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
statement_node = curriculum_nodes.LearningStatementNode(
|
||||
unique_id=f"LearningStatement_{lesson_node.unique_id}_{statement_row['StatementID']}",
|
||||
uuid_string=f"LearningStatement_{lesson_node.uuid_string}_{statement_row['StatementID']}",
|
||||
id=statement_row['StatementID'],
|
||||
name=statement_row.get('LearningStatement', default_learning_statement_values['lesson_learning_statement']),
|
||||
type=statement_row.get('StatementType', default_learning_statement_values['lesson_learning_statement_type']),
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
# Create statement node in curriculum database only
|
||||
neon.create_or_merge_neontology_node(statement_node, database=curriculum_db_name, operation='merge')
|
||||
@@ -692,6 +825,6 @@ def create_curriculum(dataframes, db_name: str, curriculum_db_name: str, school_
|
||||
curriculum_relationships.LessonIncludesLearningStatement(source=lesson_node, target=statement_node),
|
||||
database=curriculum_db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created learning statement node {statement_node.unique_id} and relationship with lesson {lesson_node.unique_id}")
|
||||
logger.info(f"Created learning statement node {statement_node.uuid_string} and relationship with lesson {lesson_node.uuid_string}")
|
||||
|
||||
return node_library
|
||||
@@ -11,7 +11,7 @@ import modules.database.schemas.relationships.calendar_timetable_rels as cal_tt_
|
||||
import modules.database.init.init_calendar as init_calendar
|
||||
import modules.database.tools.neontology_tools as neon
|
||||
|
||||
def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
def create_school_timetable(dataframes, db_name, school_node=None, filesystem=None):
|
||||
logger.info(f"Creating school timetable for {db_name}")
|
||||
if dataframes is None:
|
||||
raise ValueError("Data is required to create the calendar and timetable.")
|
||||
@@ -22,10 +22,10 @@ def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
school_df = dataframes['school']
|
||||
if school_node is None:
|
||||
logger.info(f"School node is None, using school data from dataframe")
|
||||
school_unique_id = school_df[school_df['Identifier'] == 'SchoolID']['Data'].iloc[0]
|
||||
school_uuid_string = school_df[school_df['Identifier'] == 'SchoolID']['Data'].iloc[0]
|
||||
else:
|
||||
logger.info(f"School node is not None, using school data from school node: {school_node}")
|
||||
school_unique_id = school_node.unique_id
|
||||
school_uuid_string = school_node.uuid_string
|
||||
|
||||
terms_df = dataframes['terms']
|
||||
weeks_df = dataframes['weeks']
|
||||
@@ -54,20 +54,30 @@ def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
}
|
||||
|
||||
# Create AcademicTimetable Node
|
||||
school_timetable_unique_id = f"{school_unique_id}_{school_year_start_date.year}_{school_year_end_date.year}"
|
||||
school_timetable_uuid_string = f"{school_uuid_string}_{school_year_start_date.year}_{school_year_end_date.year}"
|
||||
|
||||
# Generate storage path for timetable node
|
||||
if filesystem:
|
||||
timetable_dir_created, timetable_path = filesystem.create_school_timetable_directory()
|
||||
node_storage_path = os.path.relpath(timetable_path, filesystem.base_path)
|
||||
logger.info(f"Generated timetable node_storage_path: {node_storage_path}")
|
||||
else:
|
||||
node_storage_path = ""
|
||||
logger.warning("No filesystem provided, using empty storage path")
|
||||
|
||||
school_timetable_node = timetable.SchoolTimetableNode(
|
||||
school_timetable_id=school_timetable_unique_id,
|
||||
unique_id=school_timetable_unique_id,
|
||||
school_timetable_id=school_timetable_uuid_string,
|
||||
uuid_string=school_timetable_uuid_string,
|
||||
start_date=school_year_start_date,
|
||||
end_date=school_year_end_date,
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(school_timetable_node, database=db_name, operation='merge')
|
||||
timetable_nodes['timetable_node'] = school_timetable_node
|
||||
|
||||
if school_node:
|
||||
logger.info(f"Creating calendar for {school_unique_id} from Neo4j SchoolNode: {school_node.unique_id}")
|
||||
calendar_nodes = init_calendar.create_calendar(db_name, school_year_start_date, school_year_end_date, attach_to_calendar_node=True, owner_node=school_node)
|
||||
logger.info(f"Creating calendar for {school_uuid_string} from Neo4j SchoolNode: {school_node.uuid_string}")
|
||||
calendar_nodes = init_calendar.create_calendar(db_name, school_year_start_date, school_year_end_date, attach_to_calendar_node=True, owner_node=school_node, filesystem=filesystem)
|
||||
# Link the school node to the timetable node
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
entity_tt_rels.SchoolHasTimetable(source=school_node, target=school_timetable_node),
|
||||
@@ -75,26 +85,34 @@ def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
)
|
||||
timetable_nodes['calendar_nodes'] = calendar_nodes
|
||||
else:
|
||||
logger.info(f"Creating calendar for {school_unique_id} from dataframe SchoolID: {school_unique_id}")
|
||||
calendar_nodes = init_calendar.create_calendar(db_name, school_year_start_date, school_year_end_date, attach_to_calendar_node=False, owner_node=None)
|
||||
logger.info(f"Creating calendar for {school_uuid_string} from dataframe SchoolID: {school_uuid_string}")
|
||||
calendar_nodes = init_calendar.create_calendar(db_name, school_year_start_date, school_year_end_date, attach_to_calendar_node=False, owner_node=None, filesystem=filesystem)
|
||||
|
||||
# Create AcademicYear nodes for each year within the range
|
||||
for year in range(school_year_start_date.year, school_year_end_date.year + 1):
|
||||
year_str = str(year)
|
||||
academic_year_unique_id = f"{school_timetable_unique_id}_{year}"
|
||||
academic_year_uuid_string = f"{school_timetable_uuid_string}_{year}"
|
||||
|
||||
# Generate storage path for academic year node
|
||||
if filesystem:
|
||||
year_dir_created, year_path = filesystem.create_school_timetable_year_directory(timetable_path, year)
|
||||
node_storage_path = os.path.relpath(year_path, filesystem.base_path)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
academic_year_node = timetable.AcademicYearNode(
|
||||
unique_id=academic_year_unique_id,
|
||||
uuid_string=academic_year_uuid_string,
|
||||
year=year_str,
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(academic_year_node, database=db_name, operation='merge')
|
||||
timetable_nodes['academic_year_nodes'].append(academic_year_node)
|
||||
logger.info(f'Created academic year node: {academic_year_node.unique_id}')
|
||||
logger.info(f'Created academic year node: {academic_year_node.uuid_string}')
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
tt_rels.AcademicTimetableHasAcademicYear(source=school_timetable_node, target=academic_year_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created school timetable relationship from {school_timetable_node.unique_id} to {academic_year_node.unique_id}")
|
||||
logger.info(f"Created school timetable relationship from {school_timetable_node.uuid_string} to {academic_year_node.uuid_string}")
|
||||
|
||||
# Link the academic year with the corresponding calendar year node
|
||||
for year_node in calendar_nodes['calendar_year_nodes']:
|
||||
@@ -103,7 +121,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
cal_tt_rels.AcademicYearIsCalendarYear(source=academic_year_node, target=year_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created school timetable relationship from {academic_year_node.unique_id} to {year_node.unique_id}")
|
||||
logger.info(f"Created school timetable relationship from {academic_year_node.uuid_string} to {year_node.uuid_string}")
|
||||
break
|
||||
|
||||
# Create Term and TermBreak nodes linked to AcademicYear
|
||||
@@ -121,29 +139,39 @@ def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
if isinstance(term_end_date, pd.Timestamp):
|
||||
term_end_date = term_end_date.strftime('%Y-%m-%d')
|
||||
|
||||
# Generate storage path for term node
|
||||
if filesystem:
|
||||
if term_row['TermType'] == 'Term':
|
||||
term_dir_created, term_path = filesystem.create_school_timetable_academic_term_directory(timetable_path, term_name, academic_term_number)
|
||||
else:
|
||||
term_dir_created, term_path = filesystem.create_school_timetable_academic_term_break_directory(timetable_path, term_name)
|
||||
node_storage_path = os.path.relpath(term_path, filesystem.base_path)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
if term_row['TermType'] == 'Term':
|
||||
term_node_unique_id = f"{school_timetable_unique_id}_{academic_term_number}_{term_name_no_spaces}"
|
||||
term_node_uuid_string = f"{school_timetable_uuid_string}_{academic_term_number}_{term_name_no_spaces}"
|
||||
academic_term_number_str = str(academic_term_number)
|
||||
term_node = term_node_class(
|
||||
unique_id=term_node_unique_id,
|
||||
uuid_string=term_node_uuid_string,
|
||||
term_name=term_name,
|
||||
term_number=academic_term_number_str,
|
||||
start_date=datetime.strptime(term_start_date, '%Y-%m-%d'),
|
||||
end_date=datetime.strptime(term_end_date, '%Y-%m-%d'),
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
academic_term_number += 1
|
||||
else:
|
||||
term_break_node_unique_id = f"{school_timetable_unique_id}_{term_name_no_spaces}"
|
||||
term_break_node_uuid_string = f"{school_timetable_uuid_string}_{term_name_no_spaces}"
|
||||
term_node = term_node_class(
|
||||
unique_id=term_break_node_unique_id,
|
||||
uuid_string=term_break_node_uuid_string,
|
||||
term_break_name=term_name,
|
||||
start_date=datetime.strptime(term_start_date, '%Y-%m-%d'),
|
||||
end_date=datetime.strptime(term_end_date, '%Y-%m-%d'),
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(term_node, database=db_name, operation='merge')
|
||||
logger.info(f'Created academic term break node: {term_node.unique_id}')
|
||||
logger.info(f'Created academic term break node: {term_node.uuid_string}')
|
||||
timetable_nodes['academic_term_nodes'].append(term_node)
|
||||
term_number += 1 # We don't use this but we could
|
||||
|
||||
@@ -158,7 +186,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
relationship_class(source=academic_year_node, target=term_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created school timetable relationship from {academic_year_node.unique_id} to {term_node.unique_id}")
|
||||
logger.info(f"Created school timetable relationship from {academic_year_node.uuid_string} to {term_node.uuid_string}")
|
||||
|
||||
# Create Week nodes
|
||||
academic_week_number = 1
|
||||
@@ -168,28 +196,35 @@ def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
if isinstance(week_start_date, pd.Timestamp):
|
||||
week_start_date = week_start_date.strftime('%Y-%m-%d')
|
||||
|
||||
week_node_unique_id = f"{school_timetable_unique_id}_{week_row['WeekNumber']}_{week_row['WeekType']}Week"
|
||||
week_node_uuid_string = f"{school_timetable_uuid_string}_{week_row['WeekNumber']}_{week_row['WeekType']}Week"
|
||||
|
||||
# Generate storage path for week node
|
||||
if filesystem:
|
||||
week_dir_created, week_path = filesystem.create_school_timetable_academic_week_directory(timetable_path, week_row['WeekNumber'])
|
||||
node_storage_path = os.path.relpath(week_path, filesystem.base_path)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
if week_row['WeekType'] == 'Holiday':
|
||||
week_node = week_node_class(
|
||||
unique_id=week_node_unique_id,
|
||||
uuid_string=week_node_uuid_string,
|
||||
start_date=datetime.strptime(week_start_date, '%Y-%m-%d'),
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
else:
|
||||
academic_week_number_str = str(academic_week_number)
|
||||
week_type = week_row['WeekType']
|
||||
week_node = week_node_class(
|
||||
unique_id=week_node_unique_id,
|
||||
uuid_string=week_node_uuid_string,
|
||||
academic_week_number=academic_week_number_str,
|
||||
start_date=datetime.strptime(week_start_date, '%Y-%m-%d'),
|
||||
week_type=week_type,
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
academic_week_number += 1
|
||||
neon.create_or_merge_neontology_node(week_node, database=db_name, operation='merge')
|
||||
timetable_nodes['academic_week_nodes'].append(week_node)
|
||||
logger.info(f"Created week node: {week_node.unique_id}")
|
||||
logger.info(f"Created week node: {week_node.uuid_string}")
|
||||
for calendar_node in calendar_nodes['calendar_week_nodes']:
|
||||
if calendar_node.start_date == week_node.start_date:
|
||||
if isinstance(week_node, timetable.AcademicWeekNode):
|
||||
@@ -197,13 +232,13 @@ def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
cal_tt_rels.AcademicWeekIsCalendarWeek(source=week_node, target=calendar_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created school timetable relationship from {calendar_node.unique_id} to {week_node.unique_id}")
|
||||
logger.info(f"Created school timetable relationship from {calendar_node.uuid_string} to {week_node.uuid_string}")
|
||||
elif isinstance(week_node, timetable.HolidayWeekNode):
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
cal_tt_rels.HolidayWeekIsCalendarWeek(source=week_node, target=calendar_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created school timetable relationship from {calendar_node.unique_id} to {week_node.unique_id}")
|
||||
logger.info(f"Created school timetable relationship from {calendar_node.uuid_string} to {week_node.uuid_string}")
|
||||
break
|
||||
|
||||
# Link week node to the correct academic term
|
||||
@@ -214,7 +249,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
relationship_class(source=term_node, target=week_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created school timetable relationship from {term_node.unique_id} to {week_node.unique_id}")
|
||||
logger.info(f"Created school timetable relationship from {term_node.uuid_string} to {week_node.uuid_string}")
|
||||
break
|
||||
|
||||
# Link week node to the correct academic year
|
||||
@@ -225,7 +260,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
relationship_class(source=academic_year_node, target=week_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created school timetable relationship from {academic_year_node.unique_id} to {week_node.unique_id}")
|
||||
logger.info(f"Created school timetable relationship from {academic_year_node.uuid_string} to {week_node.uuid_string}")
|
||||
break
|
||||
|
||||
# Create Day nodes
|
||||
@@ -243,18 +278,24 @@ def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
'StaffDay': timetable.StaffDayNode
|
||||
}[day_row['DayType']]
|
||||
|
||||
# Generate storage path for day node
|
||||
if filesystem:
|
||||
day_dir_created, day_path = filesystem.create_school_timetable_academic_day_directory(timetable_path, academic_day_number)
|
||||
node_storage_path = os.path.relpath(day_path, filesystem.base_path)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
# Format the unique ID as {day_node_class.__name__}Day
|
||||
day_node_data = {
|
||||
'unique_id': f"{school_timetable_unique_id}_{day_number}_{day_node_class.__name__}Day",
|
||||
'uuid_string': f"{school_timetable_uuid_string}_{day_number}_{day_node_class.__name__}Day",
|
||||
'date': datetime.strptime(date_str, '%Y-%m-%d'),
|
||||
'day_of_week': datetime.strptime(date_str, '%Y-%m-%d').strftime('%A'),
|
||||
'tldraw_snapshot': ""
|
||||
'node_storage_path': node_storage_path
|
||||
}
|
||||
|
||||
if day_row['DayType'] == 'Academic':
|
||||
day_node_data['academic_day'] = str(academic_day_number)
|
||||
day_node_data['day_type'] = day_row['WeekType']
|
||||
day_node_data['tldraw_snapshot'] = ""
|
||||
|
||||
day_node = day_node_class(**day_node_data)
|
||||
|
||||
@@ -262,7 +303,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
if calendar_node.date == day_node.date:
|
||||
neon.create_or_merge_neontology_node(day_node, database=db_name, operation='merge')
|
||||
timetable_nodes['academic_day_nodes'].append(day_node)
|
||||
logger.info(f"Created day node: {day_node.unique_id}")
|
||||
logger.info(f"Created day node: {day_node.uuid_string}")
|
||||
|
||||
if isinstance(day_node, timetable.AcademicDayNode):
|
||||
relationship_class = cal_tt_rels.AcademicDayIsCalendarDay
|
||||
@@ -277,7 +318,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
relationship_class(source=day_node, target=calendar_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f'Created relationship from {calendar_node.unique_id} to {day_node.unique_id}')
|
||||
logger.info(f'Created relationship from {calendar_node.uuid_string} to {day_node.uuid_string}')
|
||||
break
|
||||
|
||||
# Link day node to the correct academic week
|
||||
@@ -300,7 +341,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
relationship_class(source=academic_week_node, target=day_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship from {academic_week_node.unique_id} to {day_node.unique_id}")
|
||||
logger.info(f"Created relationship from {academic_week_node.uuid_string} to {day_node.uuid_string}")
|
||||
break
|
||||
|
||||
# Link day node to the correct academic term
|
||||
@@ -323,12 +364,12 @@ def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
relationship_class(source=term_node, target=day_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship from {term_node.unique_id} to {day_node.unique_id}")
|
||||
logger.info(f"Created relationship from {term_node.uuid_string} to {day_node.uuid_string}")
|
||||
break
|
||||
|
||||
# Create Period nodes for each academic day
|
||||
if day_row['DayType'] == 'Academic':
|
||||
logger.info(f"Creating periods for {day_node.unique_id}")
|
||||
logger.info(f"Creating periods for {day_node.uuid_string}")
|
||||
period_of_day = 1
|
||||
academic_or_registration_period_of_day = 1
|
||||
for _, period_row in periods_df.iterrows():
|
||||
@@ -340,15 +381,22 @@ def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
}[period_row['PeriodType']]
|
||||
|
||||
logger.info(f"Creating period node for {period_node_class.__name__} Period: {period_of_day}")
|
||||
period_node_unique_id = f"{school_timetable_unique_id}_{academic_day_number}_{period_of_day}_{period_node_class.__name__}Period"
|
||||
logger.debug(f"Period node unique id: {period_node_unique_id}")
|
||||
period_node_uuid_string = f"{school_timetable_uuid_string}_{academic_day_number}_{period_of_day}_{period_node_class.__name__}Period"
|
||||
logger.debug(f"Period node unique id: {period_node_uuid_string}")
|
||||
# Generate storage path for period node
|
||||
if filesystem:
|
||||
period_dir_created, period_path = filesystem.create_school_timetable_period_directory(timetable_path, academic_day_number, period_row['PeriodCode'])
|
||||
node_storage_path = os.path.relpath(period_path, filesystem.base_path)
|
||||
else:
|
||||
node_storage_path = ""
|
||||
|
||||
period_node_data = {
|
||||
'unique_id': period_node_unique_id,
|
||||
'uuid_string': period_node_uuid_string,
|
||||
'name': period_row['PeriodName'],
|
||||
'date': day_node.date,
|
||||
'start_time': datetime.combine(day_node.date, period_row['StartTime']),
|
||||
'end_time': datetime.combine(day_node.date, period_row['EndTime']),
|
||||
'tldraw_snapshot': ""
|
||||
'node_storage_path': node_storage_path
|
||||
}
|
||||
logger.debug(f"Period node data: {period_node_data}")
|
||||
if period_row['PeriodType'] in ['Academic', 'Registration']:
|
||||
@@ -357,14 +405,13 @@ def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
period_code = period_row['PeriodCode']
|
||||
period_code_formatted = f"{week_type}{day_name_short}{period_code}"
|
||||
period_node_data['period_code'] = period_code_formatted
|
||||
period_node_data['tldraw_snapshot'] = ""
|
||||
|
||||
academic_or_registration_period_of_day += 1
|
||||
|
||||
period_node = period_node_class(**period_node_data)
|
||||
neon.create_or_merge_neontology_node(period_node, database=db_name, operation='merge')
|
||||
timetable_nodes['academic_period_nodes'].append(period_node)
|
||||
logger.info(f'Created period node: {period_node.unique_id}')
|
||||
logger.info(f'Created period node: {period_node.uuid_string}')
|
||||
|
||||
relationship_class = {
|
||||
'Academic': tt_rels.AcademicDayHasAcademicPeriod,
|
||||
@@ -377,7 +424,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
relationship_class(source=day_node, target=period_node),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship from {day_node.unique_id} to {period_node.unique_id}")
|
||||
logger.info(f"Created relationship from {day_node.uuid_string} to {period_node.uuid_string}")
|
||||
period_of_day += 1 # We don't use this but we could
|
||||
academic_day_number += 1 # This is a bit of a hack but it works to keep the directories aligned (reorganise)
|
||||
day_number += 1 # We don't use this but we could
|
||||
@@ -392,7 +439,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
relationship_class = relationship_map.get(node_type_pair)
|
||||
if relationship_class:
|
||||
# Avoid self-referential relationships
|
||||
if source_node.unique_id != target_node.unique_id:
|
||||
if source_node.uuid_string != target_node.uuid_string:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
relationship_class(
|
||||
source=source_node,
|
||||
@@ -400,9 +447,9 @@ def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
),
|
||||
database=db_name, operation='merge'
|
||||
)
|
||||
logger.info(f"Created relationship from {source_node.unique_id} to {target_node.unique_id}")
|
||||
logger.info(f"Created relationship from {source_node.uuid_string} to {target_node.uuid_string}")
|
||||
else:
|
||||
logger.warning(f"Skipped self-referential relationship for node {source_node.unique_id}")
|
||||
logger.warning(f"Skipped self-referential relationship for node {source_node.uuid_string}")
|
||||
|
||||
# Relationship maps for different node types
|
||||
academic_year_relationship_map = {
|
||||
@@ -474,7 +521,7 @@ def create_school_timetable(dataframes, db_name, school_node=None):
|
||||
# Call the function with the created timetable nodes
|
||||
create_school_timetable_node_sequence_rels(timetable_nodes)
|
||||
|
||||
logger.info(f'Created timetable: {timetable_nodes["timetable_node"].unique_id}')
|
||||
logger.info(f'Created timetable: {timetable_nodes["timetable_node"].uuid_string}')
|
||||
|
||||
# Log the directory structure after creation
|
||||
# root_timetable_directory = fs_handler.root_path # Access the root directory of the filesystem handler
|
||||
|
||||
@@ -9,6 +9,7 @@ import modules.database.schemas.nodes.workers.workers as worker_nodes
|
||||
import modules.database.init.init_calendar as init_calendar
|
||||
import modules.database.schemas.relationships.entity_relationships as entity_relationships
|
||||
import modules.database.tools.neontology_tools as neon
|
||||
import modules.database.tools.supabase_storage_tools as storage_tools
|
||||
from modules.logger_tool import initialise_logger
|
||||
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
|
||||
@@ -21,11 +22,27 @@ def create_and_check_db(db_name):
|
||||
return database_status
|
||||
|
||||
class UserCreator(ABC):
|
||||
def __init__(self, user_id, cc_users_db_name, user_type, worker_type, user_email, worker_email, cc_username, user_name, worker_name, calendar_start_date, calendar_end_date):
|
||||
cc_schools_db_name = "cc.institutes" # Fix the TODO
|
||||
def __init__(
|
||||
self,
|
||||
user_id,
|
||||
cc_users_db_name,
|
||||
user_type,
|
||||
worker_type,
|
||||
user_email,
|
||||
worker_email,
|
||||
cc_username,
|
||||
user_name,
|
||||
worker_name,
|
||||
calendar_start_date,
|
||||
calendar_end_date,
|
||||
storage_tools=None,
|
||||
user_db_name: Optional[str] = None,
|
||||
worker_db_name: Optional[str] = None,
|
||||
cc_schools_db_name: str = "cc.institutes",
|
||||
):
|
||||
self.cc_users_db_name = cc_users_db_name
|
||||
self.user_db_name = f"{cc_users_db_name}.{user_type}.{cc_username}"
|
||||
self.worker_db_name = f"{cc_schools_db_name}.{user_type}.{cc_username}"
|
||||
self.user_db_name = user_db_name or f"{cc_users_db_name}.{user_type}.{cc_username}"
|
||||
self.worker_db_name = worker_db_name or f"{cc_schools_db_name}.{worker_type}.{cc_username}"
|
||||
self.user_type = user_type
|
||||
self.worker_type = worker_type
|
||||
self.cc_username = cc_username
|
||||
@@ -34,6 +51,7 @@ class UserCreator(ABC):
|
||||
self.user_name = user_name
|
||||
self.worker_name = worker_name
|
||||
self.user_id = user_id
|
||||
self.storage_tools = storage_tools # Store the storage tools instance
|
||||
self.user_nodes: Dict[str, Optional[Any]] = {
|
||||
'default_user_node': None,
|
||||
'private_user_node': None,
|
||||
@@ -48,6 +66,25 @@ class UserCreator(ABC):
|
||||
self.calendar_start_date = datetime.now().date()
|
||||
self.calendar_end_date = (datetime.now() + timedelta(days=5)).date()
|
||||
|
||||
def _derive_user_storage_path(self) -> str:
|
||||
return os.path.join(
|
||||
"users",
|
||||
self.user_id,
|
||||
"databases",
|
||||
self.user_db_name,
|
||||
self.user_id,
|
||||
).replace('\\', '/')
|
||||
|
||||
def _derive_internal_worker_path(self, worker_kind: str) -> str:
|
||||
return os.path.join(
|
||||
"users",
|
||||
self.user_id,
|
||||
"databases",
|
||||
self.user_db_name,
|
||||
worker_kind,
|
||||
self.user_id,
|
||||
).replace('\\', '/')
|
||||
|
||||
@abstractmethod
|
||||
def create_user(self):
|
||||
pass
|
||||
@@ -66,9 +103,17 @@ class UserCreator(ABC):
|
||||
# Ensure Neontology is initialized
|
||||
neon.init_neontology_connection()
|
||||
|
||||
# Generate storage path for user node using Supabase Storage
|
||||
if self.storage_tools:
|
||||
user_dir_created, node_storage_path = self.storage_tools.create_user_storage_path(self.user_id)
|
||||
self.user_path = node_storage_path # Store for later use
|
||||
else:
|
||||
node_storage_path = self._derive_user_storage_path()
|
||||
self.user_path = None
|
||||
|
||||
user_node = user_nodes.UserNode(
|
||||
unique_id=f"{self.user_id}",
|
||||
tldraw_snapshot="",
|
||||
uuid_string=f"{self.user_id}",
|
||||
node_storage_path=node_storage_path,
|
||||
cc_username=f"{self.cc_username}",
|
||||
user_email=f"{self.user_email}",
|
||||
user_name=f"{self.user_name}",
|
||||
@@ -76,81 +121,79 @@ class UserCreator(ABC):
|
||||
user_type=f"{self.user_type}",
|
||||
)
|
||||
logger.debug(f"User node template created: {user_node.to_dict()}. Writing to database {db_name}")
|
||||
neon.create_or_merge_neontology_node(node=user_node, database=db_name, operation='merge')
|
||||
logger.info(f"User node created: {user_node.to_dict()}")
|
||||
logger.debug(f"About to call create_or_merge_neontology_node with node class: {user_node.__class__.__name__}")
|
||||
logger.debug(f"Node primary label: {user_node.__primarylabel__}")
|
||||
logger.debug(f"Node primary property: {user_node.__primaryproperty__}")
|
||||
|
||||
try:
|
||||
neon.create_or_merge_neontology_node(node=user_node, database=db_name, operation='merge')
|
||||
logger.info(f"User node created successfully: {user_node.to_dict()}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create user node: {e}")
|
||||
raise
|
||||
return user_node
|
||||
|
||||
def create_storage_bucket(self, bucket_id: str, bucket_name: str, access_token: Optional[str] = None) -> bool:
|
||||
"""Create public and private storage buckets for the user using their access token or service role during initialization"""
|
||||
logger.info(f"Creating storage buckets for user {self.cc_username}")
|
||||
|
||||
try:
|
||||
from modules.database.supabase.utils.client import SupabaseServiceRoleClient, SupabaseAnonClient, CreateBucketOptions
|
||||
|
||||
# During initialization (no access token provided), use service role
|
||||
if not access_token:
|
||||
logger.info("Using service role client for bucket creation during initialization")
|
||||
supabase = SupabaseServiceRoleClient()
|
||||
else:
|
||||
# For regular operations, use the user's access token
|
||||
logger.info("Using user token for bucket creation")
|
||||
supabase = SupabaseAnonClient.for_user(access_token)
|
||||
|
||||
# Create both public and private buckets
|
||||
buckets = [
|
||||
{
|
||||
"id": f"{bucket_id}.public",
|
||||
"options": CreateBucketOptions(
|
||||
name=f"{bucket_name} - Public Files",
|
||||
public=True,
|
||||
file_size_limit=50 * 1024 * 1024, # 50MB
|
||||
allowed_mime_types=[
|
||||
'image/*', 'video/*', 'application/pdf',
|
||||
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
|
||||
'text/plain', 'text/csv', 'application/json'
|
||||
]
|
||||
)
|
||||
},
|
||||
{
|
||||
"id": f"{bucket_id}.private",
|
||||
"options": CreateBucketOptions(
|
||||
name=f"{bucket_name} - Private Files",
|
||||
public=False,
|
||||
file_size_limit=50 * 1024 * 1024, # 50MB
|
||||
allowed_mime_types=[
|
||||
'image/*', 'video/*', 'application/pdf',
|
||||
'application/msword', 'application/vnd.openxmlformats-officedocument.*',
|
||||
'text/plain', 'text/csv', 'application/json'
|
||||
]
|
||||
)
|
||||
}
|
||||
]
|
||||
|
||||
success = True
|
||||
for bucket in buckets:
|
||||
try:
|
||||
result = supabase.create_bucket(bucket["id"], bucket["options"])
|
||||
if not result:
|
||||
logger.error(f"Failed to create bucket {bucket['id']}")
|
||||
success = False
|
||||
else:
|
||||
logger.info(f"Successfully created bucket {bucket['id']}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating bucket {bucket['id']}: {str(e)}")
|
||||
success = False
|
||||
|
||||
return success
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error creating storage buckets: {str(e)}")
|
||||
return False
|
||||
"""Create storage buckets for the user - DEPRECATED: Use centralized bucket initialization instead"""
|
||||
logger.warning(f"Individual user bucket creation is deprecated. Use centralized bucket initialization instead.")
|
||||
logger.info(f"User {self.cc_username} will use centralized storage buckets.")
|
||||
return True # Return success to avoid breaking existing code
|
||||
|
||||
class SchoolUserCreator(UserCreator):
|
||||
def __init__(self, user_id, cc_users_db_name, user_type, worker_type, user_email, worker_email, cc_username, user_name, worker_name, calendar_start_date, calendar_end_date, school_node, worker_node=None):
|
||||
super().__init__(user_id, cc_users_db_name, user_type, worker_type, user_email, worker_email, cc_username, user_name, worker_name, calendar_start_date, calendar_end_date)
|
||||
def __init__(
|
||||
self,
|
||||
user_id,
|
||||
cc_users_db_name,
|
||||
user_type,
|
||||
worker_type,
|
||||
user_email,
|
||||
worker_email,
|
||||
cc_username,
|
||||
user_name,
|
||||
worker_name,
|
||||
calendar_start_date,
|
||||
calendar_end_date,
|
||||
school_node,
|
||||
worker_node=None,
|
||||
storage_tools=None,
|
||||
user_db_name: Optional[str] = None,
|
||||
worker_db_name: Optional[str] = None,
|
||||
):
|
||||
super().__init__(
|
||||
user_id,
|
||||
cc_users_db_name,
|
||||
user_type,
|
||||
worker_type,
|
||||
user_email,
|
||||
worker_email,
|
||||
cc_username,
|
||||
user_name,
|
||||
worker_name,
|
||||
calendar_start_date,
|
||||
calendar_end_date,
|
||||
storage_tools,
|
||||
user_db_name=user_db_name,
|
||||
worker_db_name=worker_db_name or (school_node.private_database_name if hasattr(school_node, "private_database_name") else None),
|
||||
cc_schools_db_name="cc.institutes",
|
||||
)
|
||||
self.school_node = school_node
|
||||
self.worker_node = worker_node
|
||||
|
||||
|
||||
def _derive_school_worker_path(self, worker_kind: str) -> str:
|
||||
school_identifier = getattr(self.school_node, 'uuid_string', None) if self.school_node else None
|
||||
if not school_identifier and self.worker_db_name:
|
||||
school_identifier = self.worker_db_name.split('.')[-1]
|
||||
school_identifier = school_identifier or self.user_id
|
||||
db_name_segment = self.worker_db_name or f"cc.institutes.{school_identifier}"
|
||||
return os.path.join(
|
||||
"schools",
|
||||
school_identifier,
|
||||
"databases",
|
||||
db_name_segment,
|
||||
worker_kind,
|
||||
self.user_id,
|
||||
).replace('\\', '/')
|
||||
|
||||
def create_user(self):
|
||||
# Ensure Neontology is initialized
|
||||
logger.debug(f"Initializing Neontology connection. Closing any existing connection")
|
||||
@@ -167,7 +210,7 @@ class SchoolUserCreator(UserCreator):
|
||||
|
||||
self.user_nodes[f'worker_node'] = worker_node
|
||||
|
||||
user_node = self.create_user_node(self.cc_users_db_name)
|
||||
user_node = self.create_user_node(self.user_db_name)
|
||||
|
||||
logger.info(f"User node created: {user_node}")
|
||||
|
||||
@@ -190,16 +233,25 @@ class SchoolUserCreator(UserCreator):
|
||||
raise ValueError(f"Error creating teacher node: {e}") from e
|
||||
|
||||
def _create_teacher_node(self):
|
||||
# Generate storage path for teacher node using Supabase Storage
|
||||
if self.storage_tools:
|
||||
teacher_dir_created, node_storage_path = self.storage_tools.create_teacher_storage_path(self.user_id)
|
||||
else:
|
||||
node_storage_path = self._derive_school_worker_path(self.worker_type)
|
||||
|
||||
teacher_node = worker_nodes.TeacherNode(
|
||||
unique_id=f"{self.user_id}",
|
||||
tldraw_snapshot="",
|
||||
uuid_string=f"{self.user_id}",
|
||||
node_storage_path=node_storage_path,
|
||||
worker_name=self.worker_name,
|
||||
worker_email=self.worker_email,
|
||||
worker_db_name=self.worker_db_name,
|
||||
worker_type=self.worker_type
|
||||
)
|
||||
# Use the school's private database name if available
|
||||
school_db = self.school_node.private_database_name if hasattr(self.school_node, 'private_database_name') else f"cc.institutes.{self.school_node.school_type}.{self.school_node.id}"
|
||||
school_db = self.worker_db_name or (
|
||||
self.school_node.private_database_name if hasattr(self.school_node, 'private_database_name')
|
||||
else f"cc.institutes.{self.school_node.school_type}.{getattr(self.school_node, 'id', self.school_node.uuid_string)}"
|
||||
)
|
||||
logger.info(f"Teacher node template created: {teacher_node}... setting school db to {school_db}")
|
||||
|
||||
neon.create_or_merge_neontology_node(node=teacher_node, database=school_db, operation='merge')
|
||||
@@ -208,16 +260,25 @@ class SchoolUserCreator(UserCreator):
|
||||
return teacher_node
|
||||
|
||||
def create_student_node(self):
|
||||
# Generate storage path for student node using Supabase Storage
|
||||
if self.storage_tools:
|
||||
student_dir_created, node_storage_path = self.storage_tools.create_student_storage_path(f"Student_{self.user_id}")
|
||||
else:
|
||||
node_storage_path = self._derive_school_worker_path(self.worker_type)
|
||||
|
||||
student_node = worker_nodes.StudentNode(
|
||||
unique_id=f"Student_{self.user_id}",
|
||||
uuid_string=f"Student_{self.user_id}",
|
||||
worker_name=self.worker_name,
|
||||
worker_email=self.worker_email,
|
||||
worker_db_name=self.worker_db_name,
|
||||
worker_type=self.worker_type,
|
||||
tldraw_snapshot=""
|
||||
node_storage_path=node_storage_path
|
||||
)
|
||||
# Use the school's private database name if available
|
||||
school_db = self.school_node.private_database_name if hasattr(self.school_node, 'private_database_name') else f"cc.institutes.{self.school_node.school_type}.{self.school_node.id}"
|
||||
school_db = self.worker_db_name or (
|
||||
self.school_node.private_database_name if hasattr(self.school_node, 'private_database_name')
|
||||
else f"cc.institutes.{self.school_node.school_type}.{getattr(self.school_node, 'id', self.school_node.uuid_string)}"
|
||||
)
|
||||
logger.info(f"Student node template created: {student_node}... setting school db to {school_db}")
|
||||
|
||||
neon.create_or_merge_neontology_node(node=student_node, database=school_db, operation='merge')
|
||||
@@ -228,20 +289,58 @@ class SchoolUserCreator(UserCreator):
|
||||
def create_user_worker_relationship(self, user_node, worker_node):
|
||||
user_role_rel = entity_relationships.UserIsSchoolWorker(source=user_node, target=worker_node)
|
||||
# Use the school's private database name if available
|
||||
school_db = self.school_node.private_database_name if hasattr(self.school_node, 'private_database_name') else f"cc.institutes.{self.school_node.school_type}.{self.school_node.id}"
|
||||
school_db = self.worker_db_name or (
|
||||
self.school_node.private_database_name if hasattr(self.school_node, 'private_database_name')
|
||||
else f"cc.institutes.{self.school_node.school_type}.{getattr(self.school_node, 'id', self.school_node.uuid_string)}"
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(user_role_rel, database=school_db, operation='merge')
|
||||
logger.info(f"Relationship created between user and worker in database {school_db}")
|
||||
|
||||
def create_worker_school_relationship(self, worker_node, school_node):
|
||||
worker_school_rel = entity_relationships.EntityBelongsToSchool(source=worker_node, target=school_node)
|
||||
# Use the school's private database name if available
|
||||
school_db = school_node.private_database_name if hasattr(school_node, 'private_database_name') else f"cc.institutes.{school_node.school_type}.{school_node.id}"
|
||||
school_db = self.worker_db_name or (
|
||||
school_node.private_database_name if hasattr(school_node, 'private_database_name')
|
||||
else f"cc.institutes.{school_node.school_type}.{getattr(school_node, 'id', school_node.uuid_string)}"
|
||||
)
|
||||
neon.create_or_merge_neontology_relationship(worker_school_rel, database=school_db, operation='merge')
|
||||
logger.info(f"Relationship created between worker and school in database {school_db}")
|
||||
|
||||
class NonSchoolUserCreator(UserCreator):
|
||||
def __init__(self, user_id, cc_users_db_name, user_type, worker_type, user_email, worker_email, cc_username, user_name, worker_name, calendar_start_date, calendar_end_date, developer_role: str = "developer"):
|
||||
super().__init__(user_id, cc_users_db_name, user_type, worker_type, user_email, worker_email, cc_username, user_name, worker_name, calendar_start_date, calendar_end_date)
|
||||
def __init__(
|
||||
self,
|
||||
user_id,
|
||||
cc_users_db_name,
|
||||
user_type,
|
||||
worker_type,
|
||||
user_email,
|
||||
worker_email,
|
||||
cc_username,
|
||||
user_name,
|
||||
worker_name,
|
||||
calendar_start_date,
|
||||
calendar_end_date,
|
||||
developer_role: str = "developer",
|
||||
storage_tools=None,
|
||||
user_db_name: Optional[str] = None,
|
||||
worker_db_name: Optional[str] = None,
|
||||
):
|
||||
super().__init__(
|
||||
user_id,
|
||||
cc_users_db_name,
|
||||
user_type,
|
||||
worker_type,
|
||||
user_email,
|
||||
worker_email,
|
||||
cc_username,
|
||||
user_name,
|
||||
worker_name,
|
||||
calendar_start_date,
|
||||
calendar_end_date,
|
||||
storage_tools,
|
||||
user_db_name=user_db_name,
|
||||
worker_db_name=worker_db_name,
|
||||
)
|
||||
self.developer_role = developer_role
|
||||
|
||||
def create_user(self, access_token: Optional[str] = None):
|
||||
@@ -273,7 +372,8 @@ class NonSchoolUserCreator(UserCreator):
|
||||
logger.debug(f"Creating developer db for {self.user_type} user {self.cc_username} in database {self.user_db_name}")
|
||||
self.create_developer_db()
|
||||
else:
|
||||
raise ValueError(f"User type {self.user_type} not supported")
|
||||
logger.warning(f"User type {self.user_type} not explicitly supported; defaulting to developer workspace")
|
||||
self.create_developer_db()
|
||||
|
||||
logger.debug(f"User nodes after creation: {self.user_nodes}")
|
||||
return self.user_nodes
|
||||
@@ -290,10 +390,16 @@ class NonSchoolUserCreator(UserCreator):
|
||||
logger.debug(f"Creating super admin user node for {self.user_type} user {self.cc_username} in database {self.user_db_name}")
|
||||
private_user_node = self.create_user_node(self.user_db_name)
|
||||
|
||||
# Generate storage path for super admin node using Supabase Storage
|
||||
if self.storage_tools:
|
||||
admin_dir_created, node_storage_path = self.storage_tools.create_super_admin_storage_path(self.user_id)
|
||||
else:
|
||||
node_storage_path = self._derive_internal_worker_path(self.worker_type)
|
||||
|
||||
super_admin_node = worker_nodes.SuperAdminNode(
|
||||
unique_id=f"SuperAdmin_{self.user_id}",
|
||||
uuid_string=self.user_id,
|
||||
worker_email=self.worker_email,
|
||||
tldraw_snapshot="",
|
||||
node_storage_path=node_storage_path,
|
||||
worker_name=self.worker_name,
|
||||
worker_db_name=self.worker_db_name,
|
||||
worker_type=self.worker_type
|
||||
@@ -329,11 +435,17 @@ class NonSchoolUserCreator(UserCreator):
|
||||
# Create the user node again for the user db
|
||||
private_user_node = self.create_user_node(self.user_db_name)
|
||||
|
||||
# Generate storage path for developer node using Supabase Storage
|
||||
if self.storage_tools:
|
||||
dev_dir_created, node_storage_path = self.storage_tools.create_developer_storage_path(self.user_id)
|
||||
else:
|
||||
node_storage_path = self._derive_internal_worker_path(self.worker_type)
|
||||
|
||||
developer_node = worker_nodes.DeveloperNode(
|
||||
unique_id=f"Developer_{self.user_id}",
|
||||
uuid_string=self.user_id,
|
||||
worker_name=self.worker_name,
|
||||
worker_email=self.worker_email,
|
||||
tldraw_snapshot="",
|
||||
node_storage_path=node_storage_path,
|
||||
worker_db_name=self.worker_db_name,
|
||||
worker_type=self.worker_type,
|
||||
developer_role=self.developer_role
|
||||
@@ -372,8 +484,101 @@ class NonSchoolUserCreator(UserCreator):
|
||||
neon.create_or_merge_neontology_relationship(specific_user_rel, database=self.user_db_name, operation='merge')
|
||||
logger.info("Relationship created between user and specific node")
|
||||
|
||||
def create_calendar(self, user_node: user_nodes.UserNode):
|
||||
calendar_nodes = init_calendar.create_calendar(self.user_db_name, self.calendar_start_date, self.calendar_end_date, attach_to_calendar_node=True, owner_node=user_node)
|
||||
def create_calendar(self, user_node: user_nodes.UserNode):
|
||||
calendar_nodes = init_calendar.create_calendar(self.user_db_name, self.calendar_start_date, self.calendar_end_date, attach_to_calendar_node=True, owner_node=user_node, filesystem=self.filesystem)
|
||||
|
||||
logger.info(f"Calendar nodes created.")
|
||||
return calendar_nodes
|
||||
|
||||
|
||||
def _default_date_range():
|
||||
today = datetime.now().date()
|
||||
return today, (datetime.now() + timedelta(days=365)).date()
|
||||
|
||||
|
||||
def create_user(
|
||||
*,
|
||||
user_id: str,
|
||||
user_type: str,
|
||||
username: str,
|
||||
user_email: str,
|
||||
user_name: Optional[str] = None,
|
||||
worker_name: Optional[str] = None,
|
||||
worker_type: Optional[str] = None,
|
||||
worker_email: Optional[str] = None,
|
||||
cc_users_db_name: str = "cc.users",
|
||||
user_db_name: Optional[str] = None,
|
||||
worker_db_name: Optional[str] = None,
|
||||
calendar_start_date: Optional[datetime.date] = None,
|
||||
calendar_end_date: Optional[datetime.date] = None,
|
||||
school_node: Optional[Any] = None,
|
||||
storage_tools=None,
|
||||
) -> Dict[str, Optional[Any]]:
|
||||
"""Create a user graph structure in Neo4j.
|
||||
|
||||
Args:
|
||||
user_id: Identifier used as UUID for graph nodes.
|
||||
user_type: Application-level user type (e.g. email_teacher, developer).
|
||||
username: Canonical username/slug.
|
||||
user_email: Contact email for the user node.
|
||||
user_name: Friendly display name (defaults to username).
|
||||
worker_name: Friendly name for worker node (defaults to user_name).
|
||||
worker_type: Worker role (teacher, student, developer, etc.).
|
||||
worker_email: Email for worker node (defaults to user_email).
|
||||
cc_users_db_name: Root namespace for user databases (defaults to cc.users).
|
||||
user_db_name: Fully-qualified target database for the user graph.
|
||||
worker_db_name: Database for worker entities (usually school private DB).
|
||||
calendar_start_date/calendar_end_date: Date range for initial calendar seeding.
|
||||
school_node: Optional school context; if provided a SchoolUserCreator is used.
|
||||
storage_tools: Optional Supabase storage tools for generating storage paths.
|
||||
|
||||
Returns:
|
||||
Dict describing created nodes keyed by semantic role.
|
||||
"""
|
||||
|
||||
start_date, end_date = calendar_start_date, calendar_end_date
|
||||
if not start_date or not end_date:
|
||||
start_date, end_date = _default_date_range()
|
||||
|
||||
worker_email = worker_email or user_email
|
||||
user_name = user_name or username
|
||||
worker_name = worker_name or user_name
|
||||
|
||||
if school_node is not None:
|
||||
creator = SchoolUserCreator(
|
||||
user_id=user_id,
|
||||
cc_users_db_name=cc_users_db_name,
|
||||
user_type=user_type,
|
||||
worker_type=worker_type or "teacher",
|
||||
user_email=user_email,
|
||||
worker_email=worker_email,
|
||||
cc_username=username,
|
||||
user_name=user_name,
|
||||
worker_name=worker_name,
|
||||
calendar_start_date=start_date,
|
||||
calendar_end_date=end_date,
|
||||
school_node=school_node,
|
||||
worker_node=None,
|
||||
storage_tools=storage_tools,
|
||||
user_db_name=user_db_name,
|
||||
worker_db_name=worker_db_name,
|
||||
)
|
||||
else:
|
||||
creator = NonSchoolUserCreator(
|
||||
user_id=user_id,
|
||||
cc_users_db_name=cc_users_db_name,
|
||||
user_type=user_type,
|
||||
worker_type=worker_type or user_type,
|
||||
user_email=user_email,
|
||||
worker_email=worker_email,
|
||||
cc_username=username,
|
||||
user_name=user_name,
|
||||
worker_name=worker_name,
|
||||
calendar_start_date=start_date,
|
||||
calendar_end_date=end_date,
|
||||
storage_tools=storage_tools,
|
||||
user_db_name=user_db_name,
|
||||
worker_db_name=worker_db_name,
|
||||
)
|
||||
|
||||
return creator.create_user()
|
||||
|
||||
@@ -7,10 +7,10 @@ import modules.database.tools.neontology_tools as neon
|
||||
from modules.database.tools.filesystem_tools import ClassroomCopilotFilesystem
|
||||
from modules.database.schemas.nodes.users import UserNode
|
||||
from modules.database.schemas.nodes.schools.schools import SubjectClassNode
|
||||
from modules.database.schemas.nodes.workers.workers import TeacherNode
|
||||
from modules.database.schemas.nodes.workers.workers import TeacherNode,
|
||||
from modules.database.schemas.nodes.calendars import CalendarDayNode
|
||||
from modules.database.schemas.nodes.workers.timetable import (
|
||||
UserTeacherTimetableNode
|
||||
UserTeacherTimetableNode, TimetableLessonNode
|
||||
)
|
||||
from modules.database.schemas.relationships.entity_timetable_rels import (
|
||||
EntityHasTimetable
|
||||
@@ -23,35 +23,35 @@ from modules.database.schemas.relationships.calendar_timetable_rels import (
|
||||
CalendarDayHasPlannedLesson, PlannedLessonBelongsToCalendarDay
|
||||
)
|
||||
|
||||
def get_school_worker_classes(school_db_name: str, user_unique_id: str, worker_unique_id: str) -> list:
|
||||
def get_school_worker_classes(school_db_name: str, user_uuid_string: str, worker_uuid_string: str) -> list:
|
||||
"""
|
||||
Retrieve all classes for a worker from the school database.
|
||||
"""
|
||||
query = """
|
||||
MATCH (w:Teacher {unique_id: $worker_id})-[:TEACHER_HAS_TIMETABLE]->(tt:TeacherTimetable)
|
||||
MATCH (w:Teacher {uuid_string: $worker_id})-[:TEACHER_HAS_TIMETABLE]->(tt:TeacherTimetable)
|
||||
-[:TIMETABLE_HAS_CLASS]->(c:SubjectClass)
|
||||
RETURN c
|
||||
"""
|
||||
with driver.get_driver(db_name=school_db_name).session(database=school_db_name) as session:
|
||||
result = session.run(query, worker_id=worker_unique_id)
|
||||
result = session.run(query, worker_id=worker_uuid_string)
|
||||
classes = [record['c'] for record in result]
|
||||
if not classes:
|
||||
logger.warning(f"No classes found for teacher {worker_unique_id} in school database")
|
||||
logger.warning(f"No classes found for teacher {worker_uuid_string} in school database")
|
||||
return classes
|
||||
|
||||
def get_school_class_periods(school_db_name: str, class_unique_id: str) -> list:
|
||||
def get_school_class_periods(school_db_name: str, class_uuid_string: str) -> list:
|
||||
"""
|
||||
Retrieve all periods for a class from the school database.
|
||||
"""
|
||||
query = """
|
||||
MATCH (c:SubjectClass {unique_id: $class_id})-[:CLASS_HAS_LESSON]->(l:TimetableLesson)
|
||||
MATCH (c:SubjectClass {uuid_string: $class_id})-[:CLASS_HAS_LESSON]->(l:TimetableLesson)
|
||||
RETURN l
|
||||
"""
|
||||
with driver.get_driver(db_name=school_db_name).session(database=school_db_name) as session:
|
||||
result = session.run(query, class_id=class_unique_id)
|
||||
result = session.run(query, class_id=class_uuid_string)
|
||||
periods = [record['l'] for record in result]
|
||||
if not periods:
|
||||
logger.warning(f"No periods found for class {class_unique_id} in school database")
|
||||
logger.warning(f"No periods found for class {class_uuid_string} in school database")
|
||||
return periods
|
||||
|
||||
def get_user_calendar_nodes(user_db_name: str, user_node: UserNode) -> list:
|
||||
@@ -60,12 +60,12 @@ def get_user_calendar_nodes(user_db_name: str, user_node: UserNode) -> list:
|
||||
"""
|
||||
# First try to find any calendar days to verify the structure
|
||||
verify_query = """
|
||||
MATCH (w:User {unique_id: $user_id})
|
||||
MATCH (w:User {uuid_string: $user_id})
|
||||
OPTIONAL MATCH (w)-[:HAS_CALENDAR]->(c:Calendar)
|
||||
OPTIONAL MATCH (c)-[:CALENDAR_INCLUDES_YEAR]->(y:CalendarYear)
|
||||
OPTIONAL MATCH (y)-[:YEAR_INCLUDES_MONTH]->(m:CalendarMonth)
|
||||
OPTIONAL MATCH (m)-[:MONTH_INCLUDES_DAY]->(d:CalendarDay)
|
||||
RETURN w.unique_id as user_id,
|
||||
RETURN w.uuid_string as user_id,
|
||||
count(c) as calendar_count,
|
||||
count(y) as year_count,
|
||||
count(m) as month_count,
|
||||
@@ -76,7 +76,7 @@ def get_user_calendar_nodes(user_db_name: str, user_node: UserNode) -> list:
|
||||
|
||||
with driver.get_driver(db_name=user_db_name).session(database=user_db_name) as session:
|
||||
# First check the calendar structure
|
||||
result = session.run(verify_query, user_id=user_node.unique_id)
|
||||
result = session.run(verify_query, user_id=user_node.uuid_string)
|
||||
if stats := result.single():
|
||||
logger.info(f"Calendar structure for user {stats['user_id']}: "
|
||||
f"calendars={stats['calendar_count']}, "
|
||||
@@ -86,50 +86,50 @@ def get_user_calendar_nodes(user_db_name: str, user_node: UserNode) -> list:
|
||||
f"available years={stats['years']}")
|
||||
|
||||
if stats['calendar_count'] == 0:
|
||||
logger.error(f"No calendar found for user {user_node.unique_id}")
|
||||
logger.error(f"No calendar found for user {user_node.uuid_string}")
|
||||
return []
|
||||
if stats['year_count'] == 0:
|
||||
logger.error(f"No calendar years found for user {user_node.unique_id}")
|
||||
logger.error(f"No calendar years found for user {user_node.uuid_string}")
|
||||
return []
|
||||
if stats['month_count'] == 0:
|
||||
logger.error(f"No calendar months found for user {user_node.unique_id}")
|
||||
logger.error(f"No calendar months found for user {user_node.uuid_string}")
|
||||
return []
|
||||
if stats['day_count'] == 0:
|
||||
logger.error(f"No calendar days found for user {user_node.unique_id}")
|
||||
logger.error(f"No calendar days found for user {user_node.uuid_string}")
|
||||
return []
|
||||
|
||||
# Get all calendar days without year filter
|
||||
query = """
|
||||
MATCH (w:User {unique_id: $user_id})-[:HAS_CALENDAR]->(c:Calendar)
|
||||
MATCH (w:User {uuid_string: $user_id})-[:HAS_CALENDAR]->(c:Calendar)
|
||||
-[:CALENDAR_INCLUDES_YEAR]->(y:CalendarYear)
|
||||
-[:YEAR_INCLUDES_MONTH]->(m:CalendarMonth)
|
||||
-[:MONTH_INCLUDES_DAY]->(d:CalendarDay)
|
||||
RETURN d.unique_id as unique_id,
|
||||
RETURN d.uuid_string as uuid_string,
|
||||
d.date as date,
|
||||
d.day_of_week as day_of_week,
|
||||
d.iso_day as iso_day,
|
||||
d.path as path
|
||||
d.node_storage_path as path
|
||||
ORDER BY d.date
|
||||
"""
|
||||
|
||||
result = session.run(query, user_id=user_node.unique_id)
|
||||
result = session.run(query, user_id=user_node.uuid_string)
|
||||
calendar_days = []
|
||||
for record in result:
|
||||
calendar_day = CalendarDayNode(
|
||||
unique_id=record['unique_id'],
|
||||
uuid_string=record['uuid_string'],
|
||||
date=record['date'],
|
||||
day_of_week=record['day_of_week'],
|
||||
iso_day=record['iso_day'],
|
||||
path=record['path']
|
||||
node_storage_path=record['path']
|
||||
)
|
||||
calendar_days.append(calendar_day)
|
||||
|
||||
if not calendar_days:
|
||||
logger.error(f"No calendar days found for user {user_node.unique_id}")
|
||||
logger.error(f"No calendar days found for user {user_node.uuid_string}")
|
||||
else:
|
||||
# Log the date range we have
|
||||
dates = sorted([day.date for day in calendar_days])
|
||||
logger.info(f"Found {len(calendar_days)} calendar days for user {user_node.unique_id}")
|
||||
logger.info(f"Found {len(calendar_days)} calendar days for user {user_node.uuid_string}")
|
||||
logger.info(f"Calendar days range from {dates[0]} to {dates[-1]}")
|
||||
|
||||
return calendar_days
|
||||
@@ -149,7 +149,7 @@ def create_user_worker_timetable(
|
||||
fs_handler = ClassroomCopilotFilesystem(db_name=user_db_name, init_run_type="user")
|
||||
|
||||
# Create teacher timetable directory under the worker's directory
|
||||
_, worker_timetable_path = fs_handler.create_teacher_timetable_directory(user_worker_node.path)
|
||||
_, worker_timetable_path = fs_handler.create_teacher_timetable_directory(user_worker_node.node_storage_path)
|
||||
|
||||
# Initialize neontology connection
|
||||
neon.init_neontology_connection()
|
||||
@@ -157,7 +157,7 @@ def create_user_worker_timetable(
|
||||
# Get user's calendar nodes
|
||||
calendar_nodes = get_user_calendar_nodes(user_db_name, user_node)
|
||||
if not calendar_nodes:
|
||||
logger.warning(f"No calendar nodes found for user {user_node.unique_id}")
|
||||
logger.warning(f"No calendar nodes found for user {user_node.uuid_string}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": "No calendar nodes found for user"
|
||||
@@ -165,17 +165,17 @@ def create_user_worker_timetable(
|
||||
|
||||
try:
|
||||
# Create UserTeacherTimetableNode
|
||||
timetable_unique_id = f"UserTeacherTimetable_{user_worker_node.teacher_code}"
|
||||
timetable_uuid_string = f"UserTeacherTimetable_{user_worker_node.teacher_code}"
|
||||
worker_timetable = UserTeacherTimetableNode(
|
||||
unique_id=timetable_unique_id,
|
||||
uuid_string=timetable_uuid_string,
|
||||
school_db_name=school_db_name,
|
||||
school_timetable_id=f"TeacherTimetable_{user_worker_node.teacher_code}",
|
||||
path=worker_timetable_path
|
||||
node_storage_path=worker_timetable_path
|
||||
)
|
||||
|
||||
# Create the timetable node and its tldraw file
|
||||
neon.create_or_merge_neontology_node(worker_timetable, database=user_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(worker_timetable.path, worker_timetable.to_dict())
|
||||
fs_handler.create_default_tldraw_file(worker_timetable.node_storage_path, worker_timetable.to_dict())
|
||||
|
||||
# Link timetable to teacher using the correct relationship structure
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
@@ -185,9 +185,9 @@ def create_user_worker_timetable(
|
||||
)
|
||||
|
||||
# Get classes from school database
|
||||
school_classes = get_school_worker_classes(school_db_name, user_node.unique_id, user_worker_node.unique_id)
|
||||
school_classes = get_school_worker_classes(school_db_name, user_node.uuid_string, user_worker_node.uuid_string)
|
||||
if not school_classes:
|
||||
logger.warning(f"No classes found for teacher {user_worker_node.unique_id} in school database")
|
||||
logger.warning(f"No classes found for teacher {user_worker_node.uuid_string} in school database")
|
||||
return {
|
||||
"status": "warning",
|
||||
"message": "No classes found in school database"
|
||||
@@ -202,15 +202,15 @@ def create_user_worker_timetable(
|
||||
|
||||
# Create SubjectClassNode
|
||||
subject_class_node = SubjectClassNode(
|
||||
unique_id=class_data['unique_id'],
|
||||
uuid_string=class_data['uuid_string'],
|
||||
subject_class_code=class_data['subject_class_code'],
|
||||
year_group=class_data['year_group'],
|
||||
subject=class_data['subject'],
|
||||
subject_code=class_data['subject_code'],
|
||||
path=class_path
|
||||
node_storage_path=class_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(subject_class_node, database=user_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(subject_class_node.path, subject_class_node.to_dict())
|
||||
fs_handler.create_default_tldraw_file(subject_class_node.node_storage_path, subject_class_node.to_dict())
|
||||
|
||||
# Link class to timetable
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
@@ -220,27 +220,27 @@ def create_user_worker_timetable(
|
||||
)
|
||||
|
||||
# Initialize empty list for this class's lessons
|
||||
class_lessons[class_data['unique_id']] = []
|
||||
class_lessons[class_data['uuid_string']] = []
|
||||
|
||||
# Get periods from school database
|
||||
periods = get_school_class_periods(school_db_name, class_data['unique_id'])
|
||||
periods = get_school_class_periods(school_db_name, class_data['uuid_string'])
|
||||
if not periods:
|
||||
logger.warning(f"No periods found for class {class_data['unique_id']} in school database")
|
||||
logger.warning(f"No periods found for class {class_data['uuid_string']} in school database")
|
||||
continue
|
||||
|
||||
for period_data in periods:
|
||||
# Create UserTimetableLessonNode
|
||||
lesson_unique_id = f"UserTimetableLesson_{timetable_unique_id}_{class_name_safe}_{period_data['date']}_{period_data['period_code']}"
|
||||
timetable_lesson_node = UserTimetableLessonNode(
|
||||
unique_id=lesson_unique_id,
|
||||
# Create TimetableLessonNode
|
||||
lesson_uuid_string = f"UserTimetableLesson_{timetable_uuid_string}_{class_name_safe}_{period_data['date']}_{period_data['period_code']}"
|
||||
timetable_lesson_node = TimetableLessonNode(
|
||||
uuid_string=lesson_uuid_string,
|
||||
subject_class=class_data['subject_class_code'],
|
||||
date=period_data['date'],
|
||||
start_time=period_data['start_time'],
|
||||
end_time=period_data['end_time'],
|
||||
period_code=period_data['period_code'],
|
||||
school_db_name=school_db_name,
|
||||
school_period_id=period_data['unique_id'],
|
||||
path="Not set" # Will be set after creating directories
|
||||
school_period_id=period_data['uuid_string'],
|
||||
node_storage_path="Not set" # Will be set after creating directories
|
||||
)
|
||||
|
||||
if calendar_day := next(
|
||||
@@ -256,11 +256,11 @@ def create_user_worker_timetable(
|
||||
class_path,
|
||||
f"{calendar_day.date}_{period_data['period_code']}"
|
||||
)
|
||||
timetable_lesson_node.path = lesson_path
|
||||
timetable_lesson_node.node_storage_path = lesson_path
|
||||
|
||||
# Create and link nodes
|
||||
neon.create_or_merge_neontology_node(timetable_lesson_node, database=user_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(timetable_lesson_node.path, timetable_lesson_node.to_dict())
|
||||
fs_handler.create_default_tldraw_file(timetable_lesson_node.node_storage_path, timetable_lesson_node.to_dict())
|
||||
|
||||
# Link lesson to class
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
@@ -280,7 +280,7 @@ def create_user_worker_timetable(
|
||||
)
|
||||
|
||||
# Store the lesson node
|
||||
class_lessons[class_data['unique_id']].append({
|
||||
class_lessons[class_data['uuid_string']].append({
|
||||
'node': timetable_lesson_node,
|
||||
'date': period_data['date'],
|
||||
'start_time': period_data['start_time']
|
||||
@@ -299,7 +299,7 @@ def create_user_worker_timetable(
|
||||
next_lesson = sorted_lessons[i + 1]['node']
|
||||
|
||||
# Skip if current and next lesson are the same node
|
||||
if current_lesson.unique_id != next_lesson.unique_id:
|
||||
if current_lesson.uuid_string != next_lesson.uuid_string:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
TimetableLessonFollowsTimetableLesson(
|
||||
source=current_lesson,
|
||||
|
||||
@@ -33,19 +33,19 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
|
||||
|
||||
logging.info(f"Initialising filesystem handler...")
|
||||
fs_handler = ClassroomCopilotFilesystem(db_name=worker_db_name, init_run_type="user")
|
||||
_, worker_timetable_path = fs_handler.create_teacher_timetable_directory(worker_node.path)
|
||||
_, worker_timetable_path = fs_handler.create_teacher_timetable_directory(worker_node.node_storage_path)
|
||||
|
||||
logging.info(f"Initialising neo4j connection...")
|
||||
neon.init_neontology_connection()
|
||||
|
||||
try:
|
||||
timetable_unique_id = f"TeacherTimetable_{worker_node.teacher_code}"
|
||||
timetable_uuid_string = f"TeacherTimetable_{worker_node.teacher_code}"
|
||||
worker_timetable = TeacherTimetableNode(
|
||||
unique_id=timetable_unique_id,
|
||||
path=worker_timetable_path
|
||||
uuid_string=timetable_uuid_string,
|
||||
node_storage_path=worker_timetable_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(worker_timetable, database=worker_db_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(worker_timetable.path, worker_timetable.to_dict())
|
||||
fs_handler.create_default_tldraw_file(worker_timetable.node_storage_path, worker_timetable.to_dict())
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
TeacherHasTimetable(source=worker_node, target=worker_timetable),
|
||||
database=worker_db_name, operation='merge'
|
||||
@@ -57,28 +57,28 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
|
||||
for class_name, class_df in class_groups:
|
||||
if pd.notna(class_name):
|
||||
class_name_safe = re.sub(r'[^A-Za-z0-9_ ]+', '', class_name)
|
||||
_, class_path = fs_handler.create_teacher_class_directory(worker_timetable.path, class_name_safe)
|
||||
_, class_path = fs_handler.create_teacher_class_directory(worker_timetable.node_storage_path, class_name_safe)
|
||||
|
||||
subject_class_node_unique_id = f"SubjectClass_{class_name}"
|
||||
subject_class_node_uuid_string = f"SubjectClass_{class_name}"
|
||||
subject_class_node = SubjectClassNode(
|
||||
unique_id=subject_class_node_unique_id,
|
||||
uuid_string=subject_class_node_uuid_string,
|
||||
subject_class_code=class_name,
|
||||
year_group=str(int(class_df['YearGroup'].iloc[0])), # TODO: Hacky fix for the year group being a float
|
||||
subject=str(class_df['Subject'].iloc[0]),
|
||||
subject_code=str(class_df['SubjectCode'].iloc[0]),
|
||||
path=class_path
|
||||
node_storage_path=class_path
|
||||
)
|
||||
neon.create_or_merge_neontology_node(subject_class_node, database=worker_db_name, operation='merge')
|
||||
logging.info(f"Class node created: {subject_class_node}")
|
||||
# Create the tldraw file for the node
|
||||
fs_handler.create_default_tldraw_file(subject_class_node.path, subject_class_node.to_dict())
|
||||
fs_handler.create_default_tldraw_file(subject_class_node.node_storage_path, subject_class_node.to_dict())
|
||||
|
||||
# Link ClassNode to TeacherTimetableNode
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
TimetableHasClass(source=worker_timetable, target=subject_class_node),
|
||||
database=worker_db_name, operation='merge'
|
||||
)
|
||||
logging.info(f"Relationship created from {worker_timetable.unique_id} to {subject_class_node.unique_id}")
|
||||
logging.info(f"Relationship created from {worker_timetable.uuid_string} to {subject_class_node.uuid_string}")
|
||||
|
||||
# Link class to corresponding YearGoupSyllabus
|
||||
|
||||
@@ -92,7 +92,7 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
|
||||
SubjectClassBelongsToYearGroupSyllabus(source=subject_class_node, target=year_group_syllabus_node),
|
||||
database=worker_db_name, operation='merge'
|
||||
)
|
||||
logging.info(f"Relationship created from {subject_class_node.unique_id} to {year_group_syllabus_node.unique_id}")
|
||||
logging.info(f"Relationship created from {subject_class_node.uuid_string} to {year_group_syllabus_node.uuid_string}")
|
||||
else:
|
||||
logging.warning(f"No YearGroupSyllabus found for class {class_name} with year group {subject_class_node.year_group} and subject code {subject_class_node.subject_code}")
|
||||
|
||||
@@ -125,16 +125,16 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
|
||||
date = class_lesson['date']
|
||||
date_safe = date.strftime("%Y-%m-%d")
|
||||
# Clean the class_name to make it directory-safe (catch all for invalid characters)
|
||||
timetable_lesson_unique_id = f"TimetableLesson_{timetable_unique_id}_Class_{class_name}_Lesson_{lesson_number}_{date_safe}_{lesson_period_code}"
|
||||
timetable_lesson_uuid_string = f"TimetableLesson_{timetable_uuid_string}_Class_{class_name}_Lesson_{lesson_number}_{date_safe}_{lesson_period_code}"
|
||||
|
||||
timetable_lesson_node = TimetableLessonNode(
|
||||
unique_id=timetable_lesson_unique_id,
|
||||
uuid_string=timetable_lesson_uuid_string,
|
||||
subject_class=class_name,
|
||||
date=date,
|
||||
start_time=class_lesson['start_time'].time(), # TODO: This is probably how we should format the start and end time properties for all such nodes
|
||||
end_time=class_lesson['end_time'].time(),
|
||||
period_code=lesson_period_code,
|
||||
path="Not set"
|
||||
node_storage_path="Not set"
|
||||
)
|
||||
neon.create_or_merge_neontology_node(timetable_lesson_node, database=worker_db_name, operation='merge')
|
||||
logging.info(f"TimetableLessonNode created: {timetable_lesson_node}")
|
||||
@@ -144,19 +144,19 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
|
||||
TimetableLessonBelongsToPeriod(source=timetable_lesson_node, target=period_node),
|
||||
database=worker_db_name, operation='merge'
|
||||
)
|
||||
logging.info(f"Relationship created from {timetable_lesson_node.unique_id} to {period_node.unique_id}")
|
||||
logging.info(f"Relationship created from {timetable_lesson_node.uuid_string} to {period_node.uuid_string}")
|
||||
|
||||
# Link TimetableLessonNode to ClassNode
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
ClassHasLesson(source=subject_class_node, target=timetable_lesson_node),
|
||||
database=worker_db_name, operation='merge'
|
||||
)
|
||||
logging.info(f"Relationship created from {subject_class_node.unique_id} to {timetable_lesson_node.unique_id}")
|
||||
logging.info(f"Relationship created from {subject_class_node.uuid_string} to {timetable_lesson_node.uuid_string}")
|
||||
|
||||
# Create PlannedLessonNode
|
||||
planned_lesson_unique_id = f"PlannedLesson_{timetable_unique_id}_Class_{class_name}_Lesson_{lesson_number}_{date_safe}_{lesson_period_code}"
|
||||
planned_lesson_uuid_string = f"PlannedLesson_{timetable_uuid_string}_Class_{class_name}_Lesson_{lesson_number}_{date_safe}_{lesson_period_code}"
|
||||
planned_lesson_node = PlannedLessonNode(
|
||||
unique_id=planned_lesson_unique_id,
|
||||
uuid_string=planned_lesson_uuid_string,
|
||||
date=date,
|
||||
start_time=class_lesson['start_time'].time(),
|
||||
end_time=class_lesson['end_time'].time(),
|
||||
@@ -174,7 +174,7 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
|
||||
learning_statements=None,
|
||||
learning_resource_codes=None,
|
||||
learning_resources=None,
|
||||
path="Not set"
|
||||
node_storage_path="Not set"
|
||||
)
|
||||
# Create the PlannedLessonNode
|
||||
neon.create_or_merge_neontology_node(planned_lesson_node, database=worker_db_name, operation='merge')
|
||||
@@ -186,7 +186,7 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
|
||||
TimetableLessonHasPlannedLesson(source=timetable_lesson_node, target=planned_lesson_node),
|
||||
database=worker_db_name, operation='merge'
|
||||
)
|
||||
logging.info(f"Relationship created from {timetable_lesson_node.unique_id} to {planned_lesson_node.unique_id}")
|
||||
logging.info(f"Relationship created from {timetable_lesson_node.uuid_string} to {planned_lesson_node.uuid_string}")
|
||||
lesson_of_same_period += 1
|
||||
lesson_number += 1
|
||||
else:
|
||||
@@ -201,17 +201,17 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
|
||||
current_node = class_lesson_nodes[i]
|
||||
i_safe = f"{i:02d}"
|
||||
_, class_lesson_path = fs_handler.create_teacher_timetable_lesson_directory(class_path, f"{i_safe}_{current_node.date}_{current_node.period_code}")
|
||||
current_node.path = class_lesson_path
|
||||
current_node.node_storage_path = class_lesson_path
|
||||
neon.create_or_merge_neontology_node(current_node, database=worker_db_name, operation='merge')
|
||||
logging.info(f"TimetableLessonNode directory created and node merged into database: {current_node}")
|
||||
# Create the tldraw file for the node
|
||||
fs_handler.create_default_tldraw_file(current_node.path, current_node.to_dict())
|
||||
fs_handler.create_default_tldraw_file(current_node.node_storage_path, current_node.to_dict())
|
||||
if previous_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
TimetableLessonFollowsTimetableLesson(source=previous_node, target=current_node),
|
||||
database=worker_db_name, operation='merge'
|
||||
)
|
||||
logging.info(f"Sequential relationship created between {previous_node.unique_id} and {current_node.unique_id}")
|
||||
logging.info(f"Sequential relationship created between {previous_node.uuid_string} and {current_node.uuid_string}")
|
||||
|
||||
# Create sequential relationships for PlannedLessonNodes
|
||||
for i in range(1, len(planned_lesson_nodes)):
|
||||
@@ -219,17 +219,17 @@ def init_worker_timetable(timetable_df: pd.DataFrame, school_worker_node: Teache
|
||||
current_node = planned_lesson_nodes[i]
|
||||
i_safe = f"{i:02d}"
|
||||
_, planned_lesson_path = fs_handler.create_teacher_planned_lesson_directory(class_path, f"{i_safe}_{current_node.date}_{current_node.period_code}")
|
||||
current_node.path = planned_lesson_path
|
||||
current_node.node_storage_path = planned_lesson_path
|
||||
neon.create_or_merge_neontology_node(current_node, database=worker_db_name, operation='merge')
|
||||
logging.info(f"PlannedLessonNode directory created and node merged into database: {current_node}")
|
||||
# Create the tldraw file for the node
|
||||
fs_handler.create_default_tldraw_file(current_node.path, current_node.to_dict())
|
||||
fs_handler.create_default_tldraw_file(current_node.node_storage_path, current_node.to_dict())
|
||||
if previous_node:
|
||||
neon.create_or_merge_neontology_relationship(
|
||||
PlannedLessonFollowsPlannedLesson(source=previous_node, target=current_node),
|
||||
database=worker_db_name, operation='merge'
|
||||
)
|
||||
logging.info(f"Sequential relationship created between {previous_node.unique_id} and {current_node.unique_id}")
|
||||
logging.info(f"Sequential relationship created between {previous_node.uuid_string} and {current_node.uuid_string}")
|
||||
logging.info(f"Successfully initialized worker timetable for worker {worker_node.teacher_code}")
|
||||
return {"status": "success", "message": "Worker timetable initialized successfully"}
|
||||
|
||||
|
||||
@@ -3,15 +3,15 @@ from modules.database.tools.neontology.basenode import BaseNode
|
||||
|
||||
class CCBaseNode(BaseNode):
|
||||
__primarylabel__: ClassVar[str] = ''
|
||||
__primaryproperty__: ClassVar[str] = 'unique_id'
|
||||
unique_id: str
|
||||
tldraw_snapshot: str
|
||||
|
||||
__primaryproperty__: ClassVar[str] = 'uuid_string'
|
||||
uuid_string: str
|
||||
node_storage_path: str
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
"__primarylabel__": self.__primarylabel__,
|
||||
"unique_id": self.unique_id,
|
||||
"tldraw_snapshot": self.tldraw_snapshot,
|
||||
"uuid_string": self.uuid_string,
|
||||
"node_storage_path": self.node_storage_path,
|
||||
}
|
||||
|
||||
class UserBaseNode(CCBaseNode):
|
||||
@@ -25,8 +25,8 @@ class UserBaseNode(CCBaseNode):
|
||||
def to_dict(self):
|
||||
return {
|
||||
"__primarylabel__": self.__primarylabel__,
|
||||
"unique_id": self.unique_id,
|
||||
"tldraw_snapshot": self.tldraw_snapshot,
|
||||
"uuid_string": self.uuid_string,
|
||||
"node_storage_path": self.node_storage_path,
|
||||
"cc_username": self.cc_username,
|
||||
"user_db_name": self.user_db_name,
|
||||
"user_email": self.user_email,
|
||||
@@ -43,8 +43,8 @@ class WorkerBaseNode(CCBaseNode):
|
||||
def to_dict(self):
|
||||
return {
|
||||
"__primarylabel__": self.__primarylabel__,
|
||||
"unique_id": self.unique_id,
|
||||
"tldraw_snapshot": self.tldraw_snapshot,
|
||||
"uuid_string": self.uuid_string,
|
||||
"node_storage_path": self.node_storage_path,
|
||||
"worker_name": self.worker_name,
|
||||
"worker_email": self.worker_email,
|
||||
"worker_db_name": self.worker_db_name,
|
||||
|
||||
@@ -5,7 +5,7 @@ class SchoolNode(CCBaseNode):
|
||||
__primarylabel__: ClassVar[str] = 'School'
|
||||
|
||||
# Core identification fields (required for all databases)
|
||||
id: str # School's unique identifier within its type
|
||||
uuid_string: str # School's unique identifier within its type
|
||||
school_type: str # e.g., 'development', 'state', 'private', etc.
|
||||
name: str
|
||||
website: str = 'unknown'
|
||||
|
||||
@@ -2,6 +2,7 @@ import os
|
||||
from typing import Dict, List, Optional
|
||||
from supabase import create_client
|
||||
from modules.logger_tool import initialise_logger
|
||||
from modules.database.services.provisioning_service import ProvisioningService
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
@@ -31,6 +32,7 @@ class AdminService:
|
||||
"Authorization": f"Bearer {service_role_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
self.provisioner = ProvisioningService()
|
||||
|
||||
def get_admin_profile(self, admin_id: str) -> Optional[Dict]:
|
||||
"""Get admin profile by ID"""
|
||||
@@ -88,6 +90,10 @@ class AdminService:
|
||||
result = (
|
||||
self.supabase.table("admin_profiles").insert(profile_data).execute()
|
||||
)
|
||||
try:
|
||||
self.provisioner.ensure_user(profile_data["id"])
|
||||
except Exception as exc:
|
||||
self.logger.warning(f"Provisioning admin user {profile_data['id']} failed: {exc}")
|
||||
return result.data[0] if result else None
|
||||
|
||||
except Exception as e:
|
||||
@@ -186,6 +192,10 @@ class AdminService:
|
||||
result = (
|
||||
self.supabase.table("admin_profiles").insert(profile_data).execute()
|
||||
)
|
||||
try:
|
||||
self.provisioner.ensure_user(profile_data["id"])
|
||||
except Exception as exc:
|
||||
self.logger.warning(f"Provisioning super admin {profile_data['id']} failed: {exc}")
|
||||
return result.data[0] if result else None
|
||||
|
||||
except Exception as e:
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
import os
|
||||
from typing import Dict, Any
|
||||
from modules.logger_tool import initialise_logger
|
||||
import modules.database.tools.neo4j_driver_tools as driver_tools
|
||||
from modules.database.admin.neontology_provider import NeontologyProvider
|
||||
from modules.database.admin.graph_provider import GraphNamingProvider
|
||||
|
||||
class GraphService:
|
||||
def __init__(self):
|
||||
self.logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
self.driver = driver_tools.get_driver()
|
||||
self.neontology = NeontologyProvider()
|
||||
self.graph_naming = GraphNamingProvider()
|
||||
|
||||
def check_schema_status(self, database_name: str = "neo4j") -> Dict[str, Any]:
|
||||
"""Check the status of Neo4j schema including constraints, indexes, and labels"""
|
||||
try:
|
||||
with self.driver.session(database=database_name) as session:
|
||||
# Check constraints
|
||||
constraints_result = session.run("SHOW CONSTRAINTS")
|
||||
constraints = list(constraints_result)
|
||||
|
||||
# Check indexes
|
||||
indexes_result = session.run("SHOW INDEXES")
|
||||
indexes = list(indexes_result)
|
||||
|
||||
# Check labels
|
||||
labels_result = session.run("CALL db.labels()")
|
||||
labels = list(labels_result)
|
||||
|
||||
return {
|
||||
"constraints_count": len(constraints),
|
||||
"indexes_count": len(indexes),
|
||||
"labels_count": len(labels),
|
||||
"constraints": [dict(record) for record in constraints],
|
||||
"indexes": [dict(record) for record in indexes],
|
||||
"labels": [dict(record) for record in labels]
|
||||
}
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error checking schema status: {str(e)}")
|
||||
return {
|
||||
"constraints_count": 0,
|
||||
"indexes_count": 0,
|
||||
"labels_count": 0,
|
||||
"error": str(e)
|
||||
}
|
||||
|
||||
def initialize_schema(self, database_name: str = "neo4j") -> Dict[str, Any]:
|
||||
"""Initialize Neo4j schema with required constraints and indexes"""
|
||||
try:
|
||||
schema_queries = self.graph_naming.get_schema_creation_queries()
|
||||
|
||||
with self.driver.session(database=database_name) as session:
|
||||
for query in schema_queries:
|
||||
session.run(query)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Schema initialized successfully",
|
||||
"details": self.check_schema_status(database_name)
|
||||
}
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error initializing schema: {str(e)}")
|
||||
return {
|
||||
"status": "error",
|
||||
"message": str(e)
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
from datetime import datetime, timedelta
|
||||
import jwt
|
||||
from typing import Dict, List
|
||||
|
||||
class JWTService:
|
||||
"""JWT Service for Neo4j authentication
|
||||
|
||||
TODO: Security Enhancements Needed
|
||||
- Implement token refresh mechanism
|
||||
- Add token revocation capability
|
||||
- Add token validation checks
|
||||
- Implement rate limiting
|
||||
- Add audit logging for token generation/usage
|
||||
- Consider reducing token expiry time and implementing refresh tokens
|
||||
"""
|
||||
|
||||
def __init__(self, secret_key: str, algorithm: str = "HS256"):
|
||||
self.secret_key = secret_key
|
||||
self.algorithm = algorithm
|
||||
|
||||
def generate_neo4j_token(self, user_data: Dict) -> str:
|
||||
"""Generate JWT token for Neo4j database access"""
|
||||
payload = {
|
||||
"sub": user_data["email"],
|
||||
"roles": self._get_neo4j_roles(user_data["user_type"]),
|
||||
"iss": "supabase",
|
||||
"aud": "neo4j",
|
||||
"iat": datetime.utcnow(),
|
||||
"exp": datetime.utcnow() + timedelta(hours=24)
|
||||
}
|
||||
|
||||
if "school_uuid" in user_data:
|
||||
payload["worker_db_name"] = f"cc.institutes.{user_data['school_uuid']}"
|
||||
|
||||
return jwt.encode(payload, self.secret_key, algorithm=self.algorithm)
|
||||
|
||||
def _get_neo4j_roles(self, user_type: str) -> List[str]:
|
||||
"""Map user types to Neo4j roles"""
|
||||
role_mapping = {
|
||||
"cc_admin": ["admin", "reader", "writer"],
|
||||
"developer": ["developer", "reader", "writer"],
|
||||
"email_teacher": ["teacher", "reader", "writer"],
|
||||
"email_student": ["student", "reader"]
|
||||
}
|
||||
return role_mapping.get(user_type, ["reader"])
|
||||
@@ -86,18 +86,18 @@ class Neo4jService:
|
||||
with self.driver.session(database=database_name) as session:
|
||||
# Create constraints
|
||||
constraints = [
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:School) REQUIRE n.unique_id IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Department) REQUIRE n.unique_id IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Subject) REQUIRE n.unique_id IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:YearGroup) REQUIRE n.unique_id IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Class) REQUIRE n.unique_id IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Teacher) REQUIRE n.unique_id IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Student) REQUIRE n.unique_id IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Calendar) REQUIRE n.unique_id IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Term) REQUIRE n.unique_id IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Week) REQUIRE n.unique_id IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Day) REQUIRE n.unique_id IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Period) REQUIRE n.unique_id IS UNIQUE"
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:School) REQUIRE n.uuid_string IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Department) REQUIRE n.uuid_string IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Subject) REQUIRE n.uuid_string IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:YearGroup) REQUIRE n.uuid_string IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Class) REQUIRE n.uuid_string IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Teacher) REQUIRE n.uuid_string IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Student) REQUIRE n.uuid_string IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Calendar) REQUIRE n.uuid_string IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Term) REQUIRE n.uuid_string IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Week) REQUIRE n.uuid_string IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Day) REQUIRE n.uuid_string IS UNIQUE",
|
||||
"CREATE CONSTRAINT IF NOT EXISTS FOR (n:Period) REQUIRE n.uuid_string IS UNIQUE"
|
||||
]
|
||||
|
||||
# Create indexes
|
||||
|
||||
@@ -0,0 +1,306 @@
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, Optional, Tuple, List
|
||||
|
||||
from modules.logger_tool import initialise_logger
|
||||
from modules.database.services.neo4j_service import Neo4jService
|
||||
from modules.database.init import init_user
|
||||
from modules.database.tools.supabase_storage_tools import SupabaseStorageTools
|
||||
from modules.database.schemas.nodes.schools.schools import SchoolNode
|
||||
import modules.database.tools.neontology_tools as neon
|
||||
from modules.database.tools.neontology_tools import create_or_merge_neontology_node
|
||||
from modules.database.supabase.utils.client import SupabaseServiceRoleClient
|
||||
|
||||
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
|
||||
_CC_USERS_DB = "cc.users"
|
||||
_CC_SCHOOLS_DB = "cc.institutes"
|
||||
_DEFAULT_INSTITUTE_NAME = os.getenv("DEFAULT_INSTITUTE_NAME", "KevlarAI")
|
||||
_DEFAULT_INSTITUTE_ID = os.getenv("DEFAULT_INSTITUTE_ID")
|
||||
|
||||
|
||||
class ProvisioningService:
|
||||
"""Coordinates provisioning of Neo4j resources for schools and users."""
|
||||
|
||||
def __init__(self):
|
||||
self.neo4j_service = Neo4jService()
|
||||
self.supabase = SupabaseServiceRoleClient().supabase
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Naming helpers
|
||||
# ------------------------------------------------------------------
|
||||
@staticmethod
|
||||
def _sanitize_component(value: str) -> str:
|
||||
return "".join(ch for ch in value.lower() if ch.isalnum())
|
||||
|
||||
def _build_user_db_name(self, role: str, user_id: str) -> str:
|
||||
return f"{_CC_USERS_DB}.{self._sanitize_component(role)}.{self._sanitize_component(user_id)}"
|
||||
|
||||
def _build_school_db_name(self, institute_id: str) -> str:
|
||||
return f"{_CC_SCHOOLS_DB}.{self._sanitize_component(institute_id)}"
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Supabase helpers
|
||||
# ------------------------------------------------------------------
|
||||
def _get_profile(self, user_id: str) -> Dict:
|
||||
response = (
|
||||
self.supabase
|
||||
.table("profiles")
|
||||
.select("*")
|
||||
.eq("id", user_id)
|
||||
.single()
|
||||
.execute()
|
||||
)
|
||||
if not response.data:
|
||||
raise ValueError(f"Profile {user_id} not found")
|
||||
return response.data
|
||||
|
||||
def _get_membership(self, profile_id: str) -> Optional[Dict]:
|
||||
response = (
|
||||
self.supabase
|
||||
.table("institute_memberships")
|
||||
.select("*")
|
||||
.eq("profile_id", profile_id)
|
||||
.limit(1)
|
||||
.execute()
|
||||
)
|
||||
data = response.data or []
|
||||
return data[0] if data else None
|
||||
|
||||
def _get_institute(self, institute_id: str) -> Optional[Dict]:
|
||||
response = (
|
||||
self.supabase
|
||||
.table("institutes")
|
||||
.select("*")
|
||||
.eq("id", institute_id)
|
||||
.single()
|
||||
.execute()
|
||||
)
|
||||
return response.data if response and response.data else None
|
||||
|
||||
def _get_institute_by_name(self, name: str) -> Optional[Dict]:
|
||||
response = (
|
||||
self.supabase
|
||||
.table("institutes")
|
||||
.select("*")
|
||||
.eq("name", name)
|
||||
.limit(1)
|
||||
.execute()
|
||||
)
|
||||
data = response.data or []
|
||||
return data[0] if data else None
|
||||
|
||||
def _determine_membership_role(self, user_type: str) -> str:
|
||||
if "teacher" in user_type:
|
||||
return "teacher"
|
||||
if "student" in user_type:
|
||||
return "student"
|
||||
return "staff"
|
||||
|
||||
def _ensure_membership(self, profile: Dict, user_type: str) -> Optional[Dict]:
|
||||
membership = self._get_membership(profile["id"])
|
||||
if membership:
|
||||
return membership
|
||||
|
||||
institute_id = _DEFAULT_INSTITUTE_ID
|
||||
institute = None
|
||||
if institute_id:
|
||||
institute = self._get_institute(institute_id)
|
||||
if not institute:
|
||||
logger.warning(f"Default institute {_DEFAULT_INSTITUTE_ID} not found; attempting lookup by name")
|
||||
institute_id = None
|
||||
|
||||
if not institute_id:
|
||||
institute = self._get_institute_by_name(_DEFAULT_INSTITUTE_NAME)
|
||||
if not institute:
|
||||
raise ValueError(f"Default institute '{_DEFAULT_INSTITUTE_NAME}' not found; cannot create membership")
|
||||
institute_id = institute["id"]
|
||||
|
||||
role = self._determine_membership_role(user_type)
|
||||
|
||||
try:
|
||||
response = (
|
||||
self.supabase
|
||||
.table("institute_memberships")
|
||||
.insert({
|
||||
"profile_id": profile["id"],
|
||||
"institute_id": institute_id,
|
||||
"role": role
|
||||
})
|
||||
.execute()
|
||||
)
|
||||
data = response.data or []
|
||||
membership = data[0] if isinstance(data, list) and data else data
|
||||
email = profile.get("email") or profile.get("user_email") or profile.get("id")
|
||||
logger.info(f"Created institute membership for {email} -> {institute_id} as {role}")
|
||||
return membership
|
||||
except Exception as exc:
|
||||
logger.warning(f"Failed to create institute membership for {profile['id']}: {exc}")
|
||||
# Try to fetch again in case of race condition
|
||||
return self._get_membership(profile["id"])
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Provisioning actions
|
||||
# ------------------------------------------------------------------
|
||||
def ensure_school(self, institute_id: str) -> Dict[str, str]:
|
||||
"""Ensure the Neo4j databases and root nodes exist for a school."""
|
||||
institute = self._get_institute(institute_id)
|
||||
if not institute:
|
||||
raise ValueError(f"Institute {institute_id} not found in Supabase")
|
||||
|
||||
school_db = self._build_school_db_name(institute_id)
|
||||
curriculum_db = f"{school_db}.curriculum"
|
||||
|
||||
# Ensure root namespaces exist
|
||||
self.neo4j_service.create_database(_CC_SCHOOLS_DB)
|
||||
self.neo4j_service.create_database(school_db)
|
||||
self.neo4j_service.create_database(curriculum_db)
|
||||
|
||||
metadata = institute.get("metadata") or {}
|
||||
if isinstance(metadata, str):
|
||||
try:
|
||||
metadata = json.loads(metadata)
|
||||
except json.JSONDecodeError:
|
||||
metadata = {}
|
||||
|
||||
school_type = metadata.get("school_type") or institute.get("school_type") or "demo"
|
||||
school_node = SchoolNode(
|
||||
uuid_string=self._sanitize_component(institute_id),
|
||||
node_storage_path=f"schools/{self._sanitize_component(institute_id)}/databases/{school_db}/{self._sanitize_component(institute_id)}",
|
||||
school_type=self._sanitize_component(school_type) or "demo",
|
||||
name=institute.get("name", "Unknown School"),
|
||||
website=institute.get("website", "https://example.com"),
|
||||
)
|
||||
|
||||
neon.init_neontology_connection()
|
||||
try:
|
||||
create_or_merge_neontology_node(school_node, database=_CC_SCHOOLS_DB, operation='merge')
|
||||
create_or_merge_neontology_node(school_node, database=school_db, operation='merge')
|
||||
finally:
|
||||
neon.close_neontology_connection()
|
||||
|
||||
# Try to persist database references back to Supabase (best effort)
|
||||
updates = {
|
||||
"neo4j_private_db_name": school_db,
|
||||
"neo4j_private_sync_status": "ready",
|
||||
"neo4j_private_sync_at": datetime.utcnow().isoformat(),
|
||||
}
|
||||
try:
|
||||
(
|
||||
self.supabase
|
||||
.table("institutes")
|
||||
.update(updates)
|
||||
.eq("id", institute_id)
|
||||
.execute()
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - defensive logging only
|
||||
logger.warning(f"Failed to update institute {institute_id} with db info: {exc}")
|
||||
|
||||
return {
|
||||
"db_name": school_db,
|
||||
"curriculum_db_name": curriculum_db,
|
||||
"school_node": school_node,
|
||||
}
|
||||
|
||||
def ensure_user(self, user_id: str) -> Dict[str, Optional[str]]:
|
||||
"""Provision Neo4j resources for a specific user profile."""
|
||||
profile = self._get_profile(user_id)
|
||||
user_type_raw = (profile.get("user_type") or "").lower()
|
||||
|
||||
user_type_map = {
|
||||
"teacher": ("email_teacher", "teacher"),
|
||||
"email_teacher": ("email_teacher", "teacher"),
|
||||
"student": ("email_student", "student"),
|
||||
"email_student": ("email_student", "student"),
|
||||
"developer": ("developer", "developer"),
|
||||
"cc_developer": ("developer", "developer"),
|
||||
"admin": ("superadmin", "superadmin"),
|
||||
"super_admin": ("superadmin", "superadmin"),
|
||||
"superadmin": ("superadmin", "superadmin"),
|
||||
}
|
||||
neo_user_type, worker_type = user_type_map.get(user_type_raw, (user_type_raw or "standard", user_type_raw or "standard"))
|
||||
|
||||
user_db_name = profile.get("user_db_name")
|
||||
if not user_db_name:
|
||||
user_db_name = self._build_user_db_name(worker_type, user_id)
|
||||
|
||||
full_name = profile.get("full_name") or profile.get("display_name") or profile.get("username") or "User"
|
||||
username = profile.get("username") or self._sanitize_component(profile.get("email", "user"))
|
||||
user_email = profile.get("email") or profile.get("user_email") or ""
|
||||
|
||||
school_db_name = profile.get("school_db_name")
|
||||
school_node = None
|
||||
|
||||
membership = None
|
||||
if worker_type in ("teacher", "student"):
|
||||
membership = self._ensure_membership(profile, user_type_raw)
|
||||
if not membership:
|
||||
raise ValueError("Unable to determine institute membership for school-based user")
|
||||
if membership:
|
||||
institute_id = membership.get("institute_id")
|
||||
if institute_id:
|
||||
ensure_school_result = self.ensure_school(institute_id)
|
||||
school_db_name = ensure_school_result["db_name"]
|
||||
school_meta = ensure_school_result.get("school_node")
|
||||
if isinstance(school_meta, SchoolNode):
|
||||
school_node = school_meta
|
||||
else:
|
||||
school_node = SchoolNode(
|
||||
uuid_string=self._sanitize_component(institute_id),
|
||||
node_storage_path="",
|
||||
school_type=getattr(school_meta, "school_type", "demo") if school_meta else "demo",
|
||||
name=(school_meta.get("name") if isinstance(school_meta, dict) else None) or "Unknown School",
|
||||
website=(school_meta.get("website") if isinstance(school_meta, dict) else None) or "https://example.com",
|
||||
)
|
||||
|
||||
# Ensure base namespaces exist before creating user-specific db
|
||||
self.neo4j_service.create_database(_CC_USERS_DB)
|
||||
self.neo4j_service.create_database(user_db_name)
|
||||
|
||||
calendar_start = datetime.utcnow().date()
|
||||
calendar_end = (datetime.utcnow() + timedelta(days=365)).date()
|
||||
|
||||
# Initialize storage tools for user provisioning
|
||||
storage_tools = SupabaseStorageTools(user_db_name, init_run_type="user")
|
||||
|
||||
init_user.create_user(
|
||||
user_id=user_id,
|
||||
user_type=neo_user_type,
|
||||
username=username,
|
||||
user_email=user_email,
|
||||
user_name=full_name,
|
||||
worker_name=full_name,
|
||||
worker_type=worker_type,
|
||||
worker_email=user_email,
|
||||
cc_users_db_name=_CC_USERS_DB,
|
||||
user_db_name=user_db_name,
|
||||
worker_db_name=school_db_name,
|
||||
calendar_start_date=calendar_start,
|
||||
calendar_end_date=calendar_end,
|
||||
school_node=school_node,
|
||||
storage_tools=storage_tools,
|
||||
)
|
||||
|
||||
profile_updates = {
|
||||
"user_db_name": user_db_name,
|
||||
"school_db_name": school_db_name,
|
||||
"neo4j_sync_status": "ready",
|
||||
"neo4j_synced_at": datetime.utcnow().isoformat(),
|
||||
}
|
||||
try:
|
||||
(
|
||||
self.supabase
|
||||
.table("profiles")
|
||||
.update(profile_updates)
|
||||
.eq("id", user_id)
|
||||
.execute()
|
||||
)
|
||||
except Exception as exc: # pragma: no cover - logging only
|
||||
logger.warning(f"Failed to update profile {user_id} with provisioning info: {exc}")
|
||||
|
||||
return {
|
||||
"user_db_name": user_db_name,
|
||||
"worker_db_name": school_db_name,
|
||||
"worker_type": worker_type,
|
||||
}
|
||||
@@ -1,412 +0,0 @@
|
||||
import os
|
||||
from typing import Dict, Any, BinaryIO
|
||||
import json
|
||||
import pandas as pd
|
||||
|
||||
import modules.database.tools.neo4j_driver_tools as driver_tools
|
||||
import modules.database.tools.neo4j_session_tools as session_tools
|
||||
import modules.database.schemas.nodes.schools.schools as school_nodes
|
||||
import modules.database.schemas.nodes.schools.curriculum as curriculum_nodes
|
||||
import modules.database.schemas.nodes.schools.pastoral as pastoral_nodes
|
||||
import modules.database.schemas.nodes.structures.schools as school_structures
|
||||
from modules.database.schemas.entities import entities
|
||||
from modules.database.schemas.relationships import curriculum_relationships, entity_relationships, entity_curriculum_rels
|
||||
from modules.database.admin.neontology_provider import NeontologyProvider
|
||||
from modules.database.admin.graph_provider import GraphNamingProvider
|
||||
from modules.database.supabase.utils.client import SupabaseAnonClient
|
||||
from modules.database.supabase.utils.storage import StorageManager
|
||||
from modules.database.services.neo4j_service import Neo4jService
|
||||
from modules.logger_tool import initialise_logger
|
||||
|
||||
class SchoolAdminService:
|
||||
def __init__(self):
|
||||
self.logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
self.driver = driver_tools.get_driver()
|
||||
self.neontology = NeontologyProvider()
|
||||
self.graph_naming = GraphNamingProvider()
|
||||
self.storage = StorageManager(SupabaseAnonClient)
|
||||
self.neo4j_service = Neo4jService()
|
||||
|
||||
def check_database_exists(self, database_name: str) -> Dict[str, Any]:
|
||||
"""Check if a Neo4j database exists"""
|
||||
return self.neo4j_service.check_database_exists(database_name)
|
||||
|
||||
def create_database(self, db_name: str) -> Dict:
|
||||
"""Creates a Neo4j database with the given name"""
|
||||
return self.neo4j_service.create_database(db_name)
|
||||
|
||||
def create_school_node(self, school_data: Dict) -> Dict:
|
||||
"""Creates a school node in cc.institutes database and stores TLDraw file in Supabase"""
|
||||
try:
|
||||
# Convert school data to SchoolNode
|
||||
school_unique_id = self.graph_naming.get_school_unique_id(school_data['urn'])
|
||||
school_path = self.graph_naming.get_school_path("cc.institutes", school_data['urn'])
|
||||
|
||||
school_node = entities.SchoolNode(
|
||||
unique_id=school_unique_id,
|
||||
path=school_path,
|
||||
urn=school_data['urn'],
|
||||
establishment_number=school_data['establishment_number'],
|
||||
establishment_name=school_data['establishment_name'],
|
||||
establishment_type=school_data['establishment_type'],
|
||||
establishment_status=school_data['establishment_status'],
|
||||
phase_of_education=school_data['phase_of_education'] if school_data['phase_of_education'] not in [None, ''] else None,
|
||||
statutory_low_age=int(school_data['statutory_low_age']) if school_data.get('statutory_low_age') is not None else 0,
|
||||
statutory_high_age=int(school_data['statutory_high_age']) if school_data.get('statutory_high_age') is not None else 0,
|
||||
religious_character=school_data.get('religious_character') if school_data.get('religious_character') not in [None, ''] else None,
|
||||
school_capacity=int(school_data['school_capacity']) if school_data.get('school_capacity') is not None else 0,
|
||||
school_website=school_data.get('school_website', ''),
|
||||
ofsted_rating=school_data.get('ofsted_rating') if school_data.get('ofsted_rating') not in [None, ''] else None
|
||||
)
|
||||
|
||||
# Create default tldraw file data
|
||||
tldraw_data = {
|
||||
"document": {
|
||||
"version": 1,
|
||||
"id": school_data['urn'],
|
||||
"name": school_data['establishment_name'],
|
||||
"meta": {
|
||||
"created_at": "",
|
||||
"updated_at": "",
|
||||
"creator_id": "",
|
||||
"is_template": False,
|
||||
"is_snapshot": False,
|
||||
"is_draft": False,
|
||||
"template_id": None,
|
||||
"snapshot_id": None,
|
||||
"draft_id": None
|
||||
}
|
||||
},
|
||||
"schema": {
|
||||
"schemaVersion": 1,
|
||||
"storeVersion": 4,
|
||||
"recordVersions": {
|
||||
"asset": {
|
||||
"version": 1,
|
||||
"subTypeKey": "type",
|
||||
"subTypeVersions": {}
|
||||
},
|
||||
"camera": {
|
||||
"version": 1
|
||||
},
|
||||
"document": {
|
||||
"version": 2
|
||||
},
|
||||
"instance": {
|
||||
"version": 22
|
||||
},
|
||||
"instance_page_state": {
|
||||
"version": 5
|
||||
},
|
||||
"page": {
|
||||
"version": 1
|
||||
},
|
||||
"shape": {
|
||||
"version": 3,
|
||||
"subTypeKey": "type",
|
||||
"subTypeVersions": {
|
||||
"cc-school-node": 1
|
||||
}
|
||||
},
|
||||
"instance_presence": {
|
||||
"version": 5
|
||||
},
|
||||
"pointer": {
|
||||
"version": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"store": {
|
||||
"document:document": {
|
||||
"gridSize": 10,
|
||||
"name": school_data['establishment_name'],
|
||||
"meta": {},
|
||||
"id": school_data['urn'],
|
||||
"typeName": "document"
|
||||
},
|
||||
"page:page": {
|
||||
"meta": {},
|
||||
"id": "page",
|
||||
"name": "Page 1",
|
||||
"index": "a1",
|
||||
"typeName": "page"
|
||||
},
|
||||
"shape:school-node": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"rotation": 0,
|
||||
"type": "cc-school-node",
|
||||
"id": school_unique_id,
|
||||
"parentId": "page",
|
||||
"index": "a1",
|
||||
"props": school_node.to_dict(),
|
||||
"typeName": "shape"
|
||||
},
|
||||
"instance:instance": {
|
||||
"id": "instance",
|
||||
"currentPageId": "page",
|
||||
"typeName": "instance"
|
||||
},
|
||||
"camera:camera": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"z": 1,
|
||||
"id": "camera",
|
||||
"typeName": "camera"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Store tldraw file in Supabase storage
|
||||
file_path = f"{school_data['urn']}/tldraw.json"
|
||||
file_options = {
|
||||
"content-type": "application/json",
|
||||
"x-upsert": "true",
|
||||
"metadata": {
|
||||
"establishment_urn": school_data['urn'],
|
||||
"establishment_name": school_data['establishment_name']
|
||||
}
|
||||
}
|
||||
|
||||
# Upload file
|
||||
self.storage.upload_file(
|
||||
bucket_id="cc.institutes",
|
||||
file_path=file_path,
|
||||
file_data=json.dumps(tldraw_data).encode(),
|
||||
content_type="application/json",
|
||||
upsert=True
|
||||
)
|
||||
|
||||
# Create node in Neo4j
|
||||
with self.neontology as neo:
|
||||
self.logger.info(f"Creating school node in Neo4j: {school_node.to_dict()}")
|
||||
neo.create_or_merge_node(school_node, database="cc.institutes", operation="merge")
|
||||
return {"status": "success", "node": school_node}
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error creating school node: {str(e)}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
def create_private_database(self, school_data: Dict) -> Dict:
|
||||
"""Creates a private database for a specific school"""
|
||||
try:
|
||||
private_db_name = f"cc.institutes.{school_data['urn']}"
|
||||
with self.driver.session() as session:
|
||||
session_tools.create_database(session, private_db_name)
|
||||
self.logger.info(f"Created private database {private_db_name}")
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"Database {private_db_name} created successfully"
|
||||
}
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error creating private database: {str(e)}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
def create_basic_structure(self, school_node: school_nodes.SchoolNode, database_name: str) -> Dict:
|
||||
"""Creates basic structural nodes in the specified database"""
|
||||
try:
|
||||
# Create Department Structure node
|
||||
department_structure_node_unique_id = f"DepartmentStructure_{school_node.unique_id}"
|
||||
department_structure_node = entities.DepartmentStructureNode(
|
||||
unique_id=department_structure_node_unique_id,
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
|
||||
# Create Curriculum Structure node
|
||||
curriculum_node = curriculum_nodes.CurriculumStructureNode(
|
||||
unique_id=f"CurriculumStructure_{school_node.unique_id}",
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
|
||||
# Create Pastoral Structure node
|
||||
pastoral_node = school_structures.PastoralStructureNode(
|
||||
unique_id=f"PastoralStructure_{school_node.unique_id}",
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
|
||||
with self.neontology as neo:
|
||||
# Create nodes
|
||||
neo.create_or_merge_node(department_structure_node, database=str(database_name), operation='merge')
|
||||
neo.create_or_merge_node(curriculum_node, database=str(database_name), operation='merge')
|
||||
neo.create_or_merge_node(pastoral_node, database=str(database_name), operation='merge')
|
||||
|
||||
# Create relationships
|
||||
neo.create_or_merge_relationship(
|
||||
entity_relationships.SchoolHasDepartmentStructure(source=school_node, target=department_structure_node),
|
||||
database=database_name, operation='merge'
|
||||
)
|
||||
neo.create_or_merge_relationship(
|
||||
entity_curriculum_rels.SchoolHasCurriculumStructure(source=school_node, target=curriculum_node),
|
||||
database=database_name, operation='merge'
|
||||
)
|
||||
neo.create_or_merge_relationship(
|
||||
entity_curriculum_rels.SchoolHasPastoralStructure(source=school_node, target=pastoral_node),
|
||||
database=database_name, operation='merge'
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Basic structure created successfully",
|
||||
"nodes": {
|
||||
"department_structure": department_structure_node,
|
||||
"curriculum_structure": curriculum_node,
|
||||
"pastoral_structure": pastoral_node
|
||||
}
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error creating basic structure: {str(e)}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
def create_detailed_structure(self, school_node: school_nodes.SchoolNode, database_name: str, excel_file: BinaryIO) -> Dict:
|
||||
"""Creates detailed structural nodes from Excel file"""
|
||||
try:
|
||||
# Store Excel file in Supabase
|
||||
file_path = f"{school_node.urn}/structure.xlsx"
|
||||
|
||||
# Upload Excel file
|
||||
self.storage.upload_file(
|
||||
bucket_id="cc.institutes",
|
||||
file_path=file_path,
|
||||
file_data=excel_file.read(),
|
||||
content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
upsert=True
|
||||
)
|
||||
|
||||
# Process Excel file
|
||||
dataframes = pd.read_excel(excel_file, sheet_name=None)
|
||||
|
||||
# Get existing basic structure nodes
|
||||
with self.neontology as neo:
|
||||
result = neo.cypher_read("""
|
||||
MATCH (s:School {unique_id: $school_id})
|
||||
OPTIONAL MATCH (s)-[:HAS_DEPARTMENT_STRUCTURE]->(ds:DepartmentStructure)
|
||||
OPTIONAL MATCH (s)-[:HAS_CURRICULUM_STRUCTURE]->(cs:CurriculumStructure)
|
||||
OPTIONAL MATCH (s)-[:HAS_PASTORAL_STRUCTURE]->(ps:PastoralStructure)
|
||||
RETURN ds, cs, ps
|
||||
""", {"school_id": school_node.unique_id}, database=database_name)
|
||||
|
||||
if not result:
|
||||
raise Exception("Basic structure not found")
|
||||
|
||||
department_structure = result['ds']
|
||||
curriculum_structure = result['cs']
|
||||
pastoral_structure = result['ps']
|
||||
|
||||
# Create departments and subjects
|
||||
unique_departments = dataframes['keystagesyllabuses']['Department'].dropna().unique()
|
||||
|
||||
node_library = {}
|
||||
|
||||
with self.neontology as neo:
|
||||
for department_name in unique_departments:
|
||||
|
||||
department_node = entities.DepartmentNode(
|
||||
unique_id=f"Department_{school_node.unique_id}_{department_name.replace(' ', '_')}",
|
||||
department_name=department_name,
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
neo.create_or_merge_node(department_node, database=database_name, operation='merge')
|
||||
node_library[f'department_{department_name}'] = department_node
|
||||
|
||||
# Link to department structure
|
||||
neo.create_or_merge_relationship(
|
||||
entity_relationships.DepartmentStructureHasDepartment(
|
||||
source=department_structure,
|
||||
target=department_node
|
||||
),
|
||||
database=database_name,
|
||||
operation='merge'
|
||||
)
|
||||
|
||||
# Create year groups
|
||||
year_groups = self.sort_year_groups(dataframes['yeargroupsyllabuses'])['YearGroup'].unique()
|
||||
last_year_group_node = None
|
||||
|
||||
for year_group in year_groups:
|
||||
numeric_year_group = pd.to_numeric(year_group, errors='coerce')
|
||||
if pd.notna(numeric_year_group):
|
||||
year_group_node = pastoral_nodes.YearGroupNode(
|
||||
unique_id=f"YearGroup_{school_node.unique_id}_YGrp{int(numeric_year_group)}",
|
||||
year_group=str(int(numeric_year_group)),
|
||||
year_group_name=f"Year {int(numeric_year_group)}",
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
neo.create_or_merge_node(year_group_node, database=database_name, operation='merge')
|
||||
node_library[f'year_group_{int(numeric_year_group)}'] = year_group_node
|
||||
|
||||
# Create sequential relationship
|
||||
if last_year_group_node:
|
||||
neo.create_or_merge_relationship(
|
||||
curriculum_relationships.YearGroupFollowsYearGroup(
|
||||
source=last_year_group_node,
|
||||
target=year_group_node
|
||||
),
|
||||
database=database_name,
|
||||
operation='merge'
|
||||
)
|
||||
last_year_group_node = year_group_node
|
||||
|
||||
# Link to pastoral structure
|
||||
neo.create_or_merge_relationship(
|
||||
curriculum_relationships.PastoralStructureIncludesYearGroup(
|
||||
source=pastoral_structure,
|
||||
target=year_group_node
|
||||
),
|
||||
database=database_name,
|
||||
operation='merge'
|
||||
)
|
||||
|
||||
# Create key stages
|
||||
key_stages = dataframes['keystagesyllabuses']['KeyStage'].unique()
|
||||
last_key_stage_node = None
|
||||
|
||||
for key_stage in sorted(key_stages):
|
||||
key_stage_node = curriculum_nodes.KeyStageNode(
|
||||
unique_id=f"KeyStage_{curriculum_structure.unique_id}_KStg{key_stage}",
|
||||
key_stage_name=f"Key Stage {key_stage}",
|
||||
key_stage=str(key_stage),
|
||||
tldraw_snapshot=""
|
||||
)
|
||||
neo.create_or_merge_node(key_stage_node, database=database_name, operation='merge')
|
||||
node_library[f'key_stage_{key_stage}'] = key_stage_node
|
||||
|
||||
# Create sequential relationship
|
||||
if last_key_stage_node:
|
||||
neo.create_or_merge_relationship(
|
||||
curriculum_relationships.KeyStageFollowsKeyStage(
|
||||
source=last_key_stage_node,
|
||||
target=key_stage_node
|
||||
),
|
||||
database=database_name,
|
||||
operation='merge'
|
||||
)
|
||||
last_key_stage_node = key_stage_node
|
||||
|
||||
# Link to curriculum structure
|
||||
neo.create_or_merge_relationship(
|
||||
curriculum_relationships.CurriculumStructureIncludesKeyStage(
|
||||
source=curriculum_structure,
|
||||
target=key_stage_node
|
||||
),
|
||||
database=database_name,
|
||||
operation='merge'
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Detailed structure created successfully",
|
||||
"node_library": node_library
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error creating detailed structure: {str(e)}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
def sort_year_groups(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Helper function to sort year groups numerically"""
|
||||
df = df.copy()
|
||||
df['YearGroupNumeric'] = pd.to_numeric(df['YearGroup'], errors='coerce')
|
||||
return df.sort_values(by='YearGroupNumeric')
|
||||
|
||||
|
||||
@@ -1,472 +0,0 @@
|
||||
import os
|
||||
from typing import Dict, List, Optional, BinaryIO
|
||||
import json
|
||||
import pandas as pd
|
||||
from backend.modules.database.schemas import entities
|
||||
from modules.logger_tool import initialise_logger
|
||||
from modules.database.tools.filesystem_tools import ClassroomCopilotFilesystem
|
||||
import modules.database.tools.neo4j_driver_tools as driver_tools
|
||||
import modules.database.tools.neo4j_session_tools as session_tools
|
||||
from modules.database.admin.neontology_provider import NeontologyProvider
|
||||
from modules.database.admin.graph_provider import GraphNamingProvider
|
||||
from modules.database.schemas import curriculum_neo
|
||||
from modules.database.schemas.relationships import curriculum_relationships, entity_relationships, entity_curriculum_rels
|
||||
from modules.database.supabase.utils.storage import StorageManager
|
||||
|
||||
class SchoolService:
|
||||
def __init__(self):
|
||||
self.logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
self.driver = driver_tools.get_driver()
|
||||
self.neontology = NeontologyProvider()
|
||||
self.graph_naming = GraphNamingProvider()
|
||||
self.storage = StorageManager()
|
||||
|
||||
def create_schools_database(self) -> Dict:
|
||||
"""Creates the main cc.institutes database in Neo4j"""
|
||||
try:
|
||||
db_name = "cc.institutes"
|
||||
with self.driver.session() as session:
|
||||
session_tools.create_database(session, db_name)
|
||||
self.logger.info(f"Created database {db_name}")
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"Database {db_name} created successfully"
|
||||
}
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error creating schools database: {str(e)}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
def create_school_node(self, school_data: Dict) -> Dict:
|
||||
"""Creates a school node in cc.institutes database and stores TLDraw file in Supabase"""
|
||||
try:
|
||||
# Convert school data to SchoolNode
|
||||
school_unique_id = self.graph_naming.get_school_unique_id(school_data['urn'])
|
||||
school_path = self.graph_naming.get_school_path("cc.institutes", school_data['urn'])
|
||||
|
||||
school_node = entities.SchoolNode(
|
||||
unique_id=school_unique_id,
|
||||
path=school_path,
|
||||
urn=school_data['urn'],
|
||||
establishment_number=school_data['establishment_number'],
|
||||
establishment_name=school_data['establishment_name'],
|
||||
establishment_type=school_data['establishment_type'],
|
||||
establishment_status=school_data['establishment_status'],
|
||||
phase_of_education=school_data['phase_of_education'] if school_data['phase_of_education'] not in [None, ''] else None,
|
||||
statutory_low_age=int(school_data['statutory_low_age']) if school_data.get('statutory_low_age') is not None else 0,
|
||||
statutory_high_age=int(school_data['statutory_high_age']) if school_data.get('statutory_high_age') is not None else 0,
|
||||
religious_character=school_data.get('religious_character') if school_data.get('religious_character') not in [None, ''] else None,
|
||||
school_capacity=int(school_data['school_capacity']) if school_data.get('school_capacity') is not None else 0,
|
||||
school_website=school_data.get('school_website', ''),
|
||||
ofsted_rating=school_data.get('ofsted_rating') if school_data.get('ofsted_rating') not in [None, ''] else None
|
||||
)
|
||||
|
||||
# Create default tldraw file data
|
||||
tldraw_data = {
|
||||
"document": {
|
||||
"version": 1,
|
||||
"id": school_data['urn'],
|
||||
"name": school_data['establishment_name'],
|
||||
"meta": {
|
||||
"created_at": "",
|
||||
"updated_at": "",
|
||||
"creator_id": "",
|
||||
"is_template": False,
|
||||
"is_snapshot": False,
|
||||
"is_draft": False,
|
||||
"template_id": None,
|
||||
"snapshot_id": None,
|
||||
"draft_id": None
|
||||
}
|
||||
},
|
||||
"schema": {
|
||||
"schemaVersion": 1,
|
||||
"storeVersion": 4,
|
||||
"recordVersions": {
|
||||
"asset": {
|
||||
"version": 1,
|
||||
"subTypeKey": "type",
|
||||
"subTypeVersions": {}
|
||||
},
|
||||
"camera": {
|
||||
"version": 1
|
||||
},
|
||||
"document": {
|
||||
"version": 2
|
||||
},
|
||||
"instance": {
|
||||
"version": 22
|
||||
},
|
||||
"instance_page_state": {
|
||||
"version": 5
|
||||
},
|
||||
"page": {
|
||||
"version": 1
|
||||
},
|
||||
"shape": {
|
||||
"version": 3,
|
||||
"subTypeKey": "type",
|
||||
"subTypeVersions": {
|
||||
"cc-school-node": 1
|
||||
}
|
||||
},
|
||||
"instance_presence": {
|
||||
"version": 5
|
||||
},
|
||||
"pointer": {
|
||||
"version": 1
|
||||
}
|
||||
}
|
||||
},
|
||||
"store": {
|
||||
"document:document": {
|
||||
"gridSize": 10,
|
||||
"name": school_data['establishment_name'],
|
||||
"meta": {},
|
||||
"id": school_data['urn'],
|
||||
"typeName": "document"
|
||||
},
|
||||
"page:page": {
|
||||
"meta": {},
|
||||
"id": "page",
|
||||
"name": "Page 1",
|
||||
"index": "a1",
|
||||
"typeName": "page"
|
||||
},
|
||||
"shape:school-node": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"rotation": 0,
|
||||
"type": "cc-school-node",
|
||||
"id": school_unique_id,
|
||||
"parentId": "page",
|
||||
"index": "a1",
|
||||
"props": school_node.to_dict(),
|
||||
"typeName": "shape"
|
||||
},
|
||||
"instance:instance": {
|
||||
"id": "instance",
|
||||
"currentPageId": "page",
|
||||
"typeName": "instance"
|
||||
},
|
||||
"camera:camera": {
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"z": 1,
|
||||
"id": "camera",
|
||||
"typeName": "camera"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Store tldraw file in Supabase storage
|
||||
file_path = f"{school_data['urn']}/tldraw.json"
|
||||
file_options = {
|
||||
"content-type": "application/json",
|
||||
"x-upsert": "true",
|
||||
"metadata": {
|
||||
"establishment_urn": school_data['urn'],
|
||||
"establishment_name": school_data['establishment_name']
|
||||
}
|
||||
}
|
||||
|
||||
# Upload file
|
||||
self.storage.upload_file(
|
||||
bucket_id="cc.institutes",
|
||||
file_path=file_path,
|
||||
file_data=json.dumps(tldraw_data).encode(),
|
||||
content_type="application/json",
|
||||
upsert=True
|
||||
)
|
||||
|
||||
# Create node in Neo4j
|
||||
with self.neontology as neo:
|
||||
self.logger.info(f"Creating school node in Neo4j: {school_node.to_dict()}")
|
||||
neo.create_or_merge_node(school_node, database="cc.institutes", operation="merge")
|
||||
return {"status": "success", "node": school_node}
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error creating school node: {str(e)}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
def create_private_database(self, school_data: Dict) -> Dict:
|
||||
"""Creates a private database for a specific school"""
|
||||
try:
|
||||
private_db_name = f"cc.institutes.{school_data['urn']}"
|
||||
with self.driver.session() as session:
|
||||
session_tools.create_database(session, private_db_name)
|
||||
self.logger.info(f"Created private database {private_db_name}")
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"Database {private_db_name} created successfully"
|
||||
}
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error creating private database: {str(e)}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
def create_basic_structure(self, school_node: entities.SchoolNode, database_name: str) -> Dict:
|
||||
"""Creates basic structural nodes in the specified database"""
|
||||
try:
|
||||
# Create filesystem paths
|
||||
fs_handler = ClassroomCopilotFilesystem(database_name, init_run_type="school")
|
||||
|
||||
# Create Department Structure node
|
||||
department_structure_node_unique_id = f"DepartmentStructure_{school_node.unique_id}"
|
||||
_, department_path = fs_handler.create_school_department_directory(school_node.path, "departments")
|
||||
department_structure_node = entities.DepartmentStructureNode(
|
||||
unique_id=department_structure_node_unique_id,
|
||||
path=department_path
|
||||
)
|
||||
|
||||
# Create Curriculum Structure node
|
||||
_, curriculum_path = fs_handler.create_school_curriculum_directory(school_node.path)
|
||||
curriculum_node = curriculum_neo.CurriculumStructureNode(
|
||||
unique_id=f"CurriculumStructure_{school_node.unique_id}",
|
||||
path=curriculum_path
|
||||
)
|
||||
|
||||
# Create Pastoral Structure node
|
||||
_, pastoral_path = fs_handler.create_school_pastoral_directory(school_node.path)
|
||||
pastoral_node = curriculum_neo.PastoralStructureNode(
|
||||
unique_id=f"PastoralStructure_{school_node.unique_id}",
|
||||
path=pastoral_path
|
||||
)
|
||||
|
||||
with self.neontology as neo:
|
||||
# Create nodes
|
||||
neo.create_or_merge_node(department_structure_node, database=str(database_name), operation='merge')
|
||||
fs_handler.create_default_tldraw_file(department_structure_node.path, department_structure_node.to_dict())
|
||||
|
||||
neo.create_or_merge_node(curriculum_node, database=str(database_name), operation='merge')
|
||||
fs_handler.create_default_tldraw_file(curriculum_node.path, curriculum_node.to_dict())
|
||||
|
||||
neo.create_or_merge_node(pastoral_node, database=database_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(pastoral_node.path, pastoral_node.to_dict())
|
||||
|
||||
# Create relationships
|
||||
neo.create_or_merge_relationship(
|
||||
entity_relationships.SchoolHasDepartmentStructure(source=school_node, target=department_structure_node),
|
||||
database=database_name, operation='merge'
|
||||
)
|
||||
|
||||
neo.create_or_merge_relationship(
|
||||
entity_curriculum_rels.SchoolHasCurriculumStructure(source=school_node, target=curriculum_node),
|
||||
database=database_name, operation='merge'
|
||||
)
|
||||
|
||||
neo.create_or_merge_relationship(
|
||||
entity_curriculum_rels.SchoolHasPastoralStructure(source=school_node, target=pastoral_node),
|
||||
database=database_name, operation='merge'
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Basic structure created successfully",
|
||||
"nodes": {
|
||||
"department_structure": department_structure_node,
|
||||
"curriculum_structure": curriculum_node,
|
||||
"pastoral_structure": pastoral_node
|
||||
}
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error creating basic structure: {str(e)}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
def create_detailed_structure(self, school_node: entities.SchoolNode, database_name: str, excel_file: BinaryIO) -> Dict:
|
||||
"""Creates detailed structural nodes from Excel file"""
|
||||
try:
|
||||
# Store Excel file in Supabase
|
||||
file_path = f"{school_node.urn}/structure.xlsx"
|
||||
|
||||
# Upload Excel file
|
||||
self.storage.upload_file(
|
||||
bucket_id="cc.institutes",
|
||||
file_path=file_path,
|
||||
file_data=excel_file.read(),
|
||||
content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
upsert=True
|
||||
)
|
||||
|
||||
# Process Excel file
|
||||
dataframes = pd.read_excel(excel_file, sheet_name=None)
|
||||
|
||||
# Get existing basic structure nodes
|
||||
with self.neontology as neo:
|
||||
result = neo.cypher_read("""
|
||||
MATCH (s:School {unique_id: $school_id})
|
||||
OPTIONAL MATCH (s)-[:HAS_DEPARTMENT_STRUCTURE]->(ds:DepartmentStructure)
|
||||
OPTIONAL MATCH (s)-[:HAS_CURRICULUM_STRUCTURE]->(cs:CurriculumStructure)
|
||||
OPTIONAL MATCH (s)-[:HAS_PASTORAL_STRUCTURE]->(ps:PastoralStructure)
|
||||
RETURN ds, cs, ps
|
||||
""", {"school_id": school_node.unique_id}, database=database_name)
|
||||
|
||||
if not result:
|
||||
raise Exception("Basic structure not found")
|
||||
|
||||
department_structure = result['ds']
|
||||
curriculum_structure = result['cs']
|
||||
pastoral_structure = result['ps']
|
||||
|
||||
# Create departments and subjects
|
||||
unique_departments = dataframes['keystagesyllabuses']['Department'].dropna().unique()
|
||||
|
||||
fs_handler = ClassroomCopilotFilesystem(database_name, init_run_type="school")
|
||||
node_library = {}
|
||||
|
||||
with self.neontology as neo:
|
||||
for department_name in unique_departments:
|
||||
_, department_path = fs_handler.create_school_department_directory(school_node.path, department_name)
|
||||
|
||||
department_node = entities.DepartmentNode(
|
||||
unique_id=f"Department_{school_node.unique_id}_{department_name.replace(' ', '_')}",
|
||||
department_name=department_name,
|
||||
path=department_path
|
||||
)
|
||||
neo.create_or_merge_node(department_node, database=database_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(department_node.path, department_node.to_dict())
|
||||
node_library[f'department_{department_name}'] = department_node
|
||||
|
||||
# Link to department structure
|
||||
neo.create_or_merge_relationship(
|
||||
entity_relationships.DepartmentStructureHasDepartment(
|
||||
source=department_structure,
|
||||
target=department_node
|
||||
),
|
||||
database=database_name,
|
||||
operation='merge'
|
||||
)
|
||||
|
||||
# Create year groups
|
||||
year_groups = self.sort_year_groups(dataframes['yeargroupsyllabuses'])['YearGroup'].unique()
|
||||
last_year_group_node = None
|
||||
|
||||
for year_group in year_groups:
|
||||
numeric_year_group = pd.to_numeric(year_group, errors='coerce')
|
||||
if pd.notna(numeric_year_group):
|
||||
_, year_group_path = fs_handler.create_pastoral_year_group_directory(
|
||||
pastoral_structure.path,
|
||||
str(int(numeric_year_group))
|
||||
)
|
||||
|
||||
year_group_node = curriculum_neo.YearGroupNode(
|
||||
unique_id=f"YearGroup_{school_node.unique_id}_YGrp{int(numeric_year_group)}",
|
||||
year_group=str(int(numeric_year_group)),
|
||||
year_group_name=f"Year {int(numeric_year_group)}",
|
||||
path=year_group_path
|
||||
)
|
||||
neo.create_or_merge_node(year_group_node, database=database_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(year_group_node.path, year_group_node.to_dict())
|
||||
node_library[f'year_group_{int(numeric_year_group)}'] = year_group_node
|
||||
|
||||
# Create sequential relationship
|
||||
if last_year_group_node:
|
||||
neo.create_or_merge_relationship(
|
||||
curriculum_relationships.YearGroupFollowsYearGroup(
|
||||
source=last_year_group_node,
|
||||
target=year_group_node
|
||||
),
|
||||
database=database_name,
|
||||
operation='merge'
|
||||
)
|
||||
last_year_group_node = year_group_node
|
||||
|
||||
# Link to pastoral structure
|
||||
neo.create_or_merge_relationship(
|
||||
curriculum_relationships.PastoralStructureIncludesYearGroup(
|
||||
source=pastoral_structure,
|
||||
target=year_group_node
|
||||
),
|
||||
database=database_name,
|
||||
operation='merge'
|
||||
)
|
||||
|
||||
# Create key stages
|
||||
key_stages = dataframes['keystagesyllabuses']['KeyStage'].unique()
|
||||
last_key_stage_node = None
|
||||
|
||||
for key_stage in sorted(key_stages):
|
||||
_, key_stage_path = fs_handler.create_curriculum_key_stage_directory(
|
||||
curriculum_structure.path,
|
||||
str(key_stage)
|
||||
)
|
||||
|
||||
key_stage_node = curriculum_neo.KeyStageNode(
|
||||
unique_id=f"KeyStage_{curriculum_structure.unique_id}_KStg{key_stage}",
|
||||
key_stage_name=f"Key Stage {key_stage}",
|
||||
key_stage=str(key_stage),
|
||||
path=key_stage_path
|
||||
)
|
||||
neo.create_or_merge_node(key_stage_node, database=database_name, operation='merge')
|
||||
fs_handler.create_default_tldraw_file(key_stage_node.path, key_stage_node.to_dict())
|
||||
node_library[f'key_stage_{key_stage}'] = key_stage_node
|
||||
|
||||
# Create sequential relationship
|
||||
if last_key_stage_node:
|
||||
neo.create_or_merge_relationship(
|
||||
curriculum_relationships.KeyStageFollowsKeyStage(
|
||||
source=last_key_stage_node,
|
||||
target=key_stage_node
|
||||
),
|
||||
database=database_name,
|
||||
operation='merge'
|
||||
)
|
||||
last_key_stage_node = key_stage_node
|
||||
|
||||
# Link to curriculum structure
|
||||
neo.create_or_merge_relationship(
|
||||
curriculum_relationships.CurriculumStructureIncludesKeyStage(
|
||||
source=curriculum_structure,
|
||||
target=key_stage_node
|
||||
),
|
||||
database=database_name,
|
||||
operation='merge'
|
||||
)
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Detailed structure created successfully",
|
||||
"node_library": node_library
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error creating detailed structure: {str(e)}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
|
||||
def sort_year_groups(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Helper function to sort year groups numerically"""
|
||||
df = df.copy()
|
||||
df['YearGroupNumeric'] = pd.to_numeric(df['YearGroup'], errors='coerce')
|
||||
return df.sort_values(by='YearGroupNumeric')
|
||||
|
||||
def check_schools_database(self) -> Dict:
|
||||
"""Check if the schools database exists and has been initialized"""
|
||||
try:
|
||||
db_name = "cc.institutes"
|
||||
with self.driver.session() as session:
|
||||
# Check if database exists
|
||||
databases = session_tools.list_databases(session)
|
||||
if db_name not in databases:
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Database {db_name} does not exist"
|
||||
}
|
||||
|
||||
# Check if database has any nodes (indicating it's been initialized)
|
||||
session.run("USE " + db_name)
|
||||
result = session.run("MATCH (n) RETURN count(n) as count").single()
|
||||
node_count = result["count"] if result else 0
|
||||
|
||||
if node_count == 0:
|
||||
return {
|
||||
"status": "error",
|
||||
"message": f"Database {db_name} exists but has no nodes"
|
||||
}
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"message": f"Database {db_name} exists and has {node_count} nodes"
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error checking schools database: {str(e)}")
|
||||
return {"status": "error", "message": str(e)}
|
||||
@@ -14,14 +14,13 @@ class CreateBucketOptions(TypedDict, total=False):
|
||||
allowed_mime_types: List[str]
|
||||
name: str
|
||||
|
||||
def _create_base_client(url: str, key: str, options: Optional[Dict[str, Any]] = None, access_token: Optional[str] = None) -> Client:
|
||||
def _create_base_client(url: str, key: str, options: Optional[Dict[str, Any]] = None) -> Client:
|
||||
"""Create a base Supabase client with given configuration."""
|
||||
client_options = SyncClientOptions(
|
||||
schema="public",
|
||||
storage=SyncMemoryStorage(),
|
||||
headers={
|
||||
"apikey": key,
|
||||
"Authorization": f"Bearer {access_token if access_token else key}"
|
||||
"Authorization": f"Bearer {key}"
|
||||
}
|
||||
)
|
||||
return create_client(url, key, options=client_options)
|
||||
@@ -29,16 +28,16 @@ def _create_base_client(url: str, key: str, options: Optional[Dict[str, Any]] =
|
||||
class SupabaseServiceRoleClient:
|
||||
"""Supabase client for making authenticated requests using the service role key"""
|
||||
|
||||
def __init__(self, url: Optional[str] = None, service_role_key: Optional[str] = None, access_token: Optional[str] = None):
|
||||
def __init__(self, url: Optional[str] = None, service_role_key: Optional[str] = None):
|
||||
"""Initialize the Supabase client with URL and service role key"""
|
||||
self.url = url or os.environ.get("SUPABASE_URL", "http://kong:8000")
|
||||
self.url = url or os.environ.get("SUPABASE_URL", "http://localhost:8000")
|
||||
self.service_role_key = service_role_key or os.environ.get("SERVICE_ROLE_KEY")
|
||||
|
||||
if not self.url or not self.service_role_key:
|
||||
raise ValueError("SUPABASE_URL and SERVICE_ROLE_KEY must be provided")
|
||||
|
||||
# Initialize Supabase client with service role key and optional access token
|
||||
self.supabase = _create_base_client(self.url, self.service_role_key, access_token=access_token)
|
||||
self.supabase = _create_base_client(self.url, self.service_role_key)
|
||||
|
||||
def create_bucket(self, id: str, options: Optional[CreateBucketOptions] = None) -> Dict[str, Any]:
|
||||
"""Create a storage bucket with the given ID and options"""
|
||||
@@ -48,17 +47,12 @@ class SupabaseServiceRoleClient:
|
||||
options['name'] = id # Use ID as default name if not provided
|
||||
return self.supabase.storage.create_bucket(id, options=options)
|
||||
|
||||
@classmethod
|
||||
def for_admin(cls, access_token: str) -> 'SupabaseServiceRoleClient':
|
||||
"""Create a client instance for the super admin using their access token"""
|
||||
return cls(access_token=access_token)
|
||||
|
||||
class SupabaseAnonClient:
|
||||
"""Supabase client for making authenticated requests using the anon key"""
|
||||
|
||||
def __init__(self, url: Optional[str] = None, anon_key: Optional[str] = None, access_token: Optional[str] = None):
|
||||
"""Initialize the Supabase client with URL and anon key"""
|
||||
self.url = url or os.environ.get("SUPABASE_URL", "http://kong:8000")
|
||||
self.url = url or os.environ.get("SUPABASE_URL", "http://localhost:8000")
|
||||
self.anon_key = anon_key or os.environ.get("ANON_KEY")
|
||||
|
||||
if not self.url or not self.anon_key:
|
||||
|
||||
@@ -18,7 +18,6 @@ import re
|
||||
|
||||
class ClassroomCopilotFilesystem:
|
||||
def __init__(self, db_name: str, init_run_type: str = None):
|
||||
logging.info(f"Initializing ClassroomCopilotFilesystem with db_name: {db_name} and init_run_type: {init_run_type}")
|
||||
|
||||
self.db_name = db_name
|
||||
|
||||
@@ -27,19 +26,21 @@ class ClassroomCopilotFilesystem:
|
||||
if not self.base_path:
|
||||
raise ValueError("NODE_FILESYSTEM_PATH environment variable not set")
|
||||
|
||||
logging.info(f"Initializing ClassroomCopilotFilesystem with db_name: {db_name} and init_run_type: {init_run_type} with base path: {self.base_path}")
|
||||
|
||||
# Set root path based on init type
|
||||
if init_run_type == "school":
|
||||
self.root_path = os.path.join(self.base_path, "schools", self.db_name)
|
||||
self.root_path = os.path.join(self.base_path, "schools")
|
||||
logging.debug(f"School root path: {self.root_path}")
|
||||
elif init_run_type == "user":
|
||||
self.root_path = os.path.join(self.base_path, "users", self.db_name)
|
||||
self.root_path = os.path.join(self.base_path, "users")
|
||||
logging.debug(f"User root path: {self.root_path}")
|
||||
elif init_run_type == "multiplayer":
|
||||
self.root_path = os.path.join(self.base_path, "multiplayer")
|
||||
logging.debug(f"Multiplayer root path: {self.root_path}")
|
||||
else:
|
||||
self.root_path = os.path.join(self.base_path, self.db_name)
|
||||
logging.debug(f"Default root path: {self.root_path}")
|
||||
self.root_path = self.base_path
|
||||
logging.debug(f"Database root path: {self.root_path}")
|
||||
|
||||
# Ensure root directory exists
|
||||
os.makedirs(self.root_path, exist_ok=True)
|
||||
@@ -63,45 +64,33 @@ class ClassroomCopilotFilesystem:
|
||||
return True
|
||||
return False
|
||||
|
||||
def sanitize_username(self, username):
|
||||
return re.sub(r'[^\w\-_\.]', '_', username)
|
||||
|
||||
def create_user_directory(self, username, user_type=None, school_path=None):
|
||||
def create_private_user_directory(self, user_id):
|
||||
"""Create a directory for a specific user."""
|
||||
sanitized_username = self.sanitize_username(username)
|
||||
|
||||
if school_path:
|
||||
# For school database: /schools/[school_db]/users/[user_type]/[username]
|
||||
user_path = os.path.join(self.root_path, "users", user_type, sanitized_username)
|
||||
else:
|
||||
# For user database: /users/[user_db]/[username]
|
||||
user_path = os.path.join(self.root_path, sanitized_username)
|
||||
# For user database: /users/[user_db]/[username]
|
||||
user_path = os.path.join(self.root_path, user_id)
|
||||
|
||||
logging.info(f"Creating user directory at {user_path}")
|
||||
return self.create_directory(user_path), user_path
|
||||
|
||||
def create_user_worker_directory(self, user_path, worker_code):
|
||||
def create_user_worker_directory(self, user_path, worker_id, worker_type):
|
||||
"""Create a worker directory under the user directory."""
|
||||
# Create worker directory: [user_path]/[worker_code]
|
||||
worker_path = os.path.join(user_path, worker_code)
|
||||
worker_path = os.path.join(user_path, worker_type, worker_id)
|
||||
logging.info(f"Creating worker directory at {worker_path}")
|
||||
return self.create_directory(worker_path), worker_path
|
||||
|
||||
def create_school_worker_directory(self, school_path, worker_type):
|
||||
def create_school_worker_directory(self, school_path, worker_type, worker_id):
|
||||
"""Create a worker directory under the school directory."""
|
||||
worker_path = os.path.join(school_path, "workers", worker_type)
|
||||
logging.info(f"Creating school worker directory at {worker_path}")
|
||||
worker_path = os.path.join(school_path, "workers", worker_type, worker_id)
|
||||
logging.info(f"Creating school {worker_type} worker directory at {worker_path}")
|
||||
return self.create_directory(worker_path), worker_path
|
||||
|
||||
def create_school_directory(self, school_uuid=None):
|
||||
def create_school_directory(self, school_uuid_string):
|
||||
"""Create a directory for a specific school."""
|
||||
logging.info(f"Creating school directory with school_uuid: {school_uuid}")
|
||||
if school_uuid is None:
|
||||
logging.debug(f"School UUID is None, creating school directory at {self.root_path}")
|
||||
school_path = self.root_path
|
||||
else:
|
||||
logging.debug(f"School UUID is not None, creating school directory at {os.path.join(self.root_path, school_uuid)}")
|
||||
school_path = os.path.join(self.root_path, school_uuid)
|
||||
logging.info(f"Creating school directory with uuid_string: {school_uuid_string}")
|
||||
logging.debug(f"School UUID is not None, creating school directory at {os.path.join(self.root_path, school_uuid_string)}")
|
||||
school_path = os.path.join(self.root_path, school_uuid_string)
|
||||
return self.create_directory(school_path), school_path
|
||||
|
||||
def create_year_directory(self, year, calendar_path=None):
|
||||
@@ -248,7 +237,12 @@ class ClassroomCopilotFilesystem:
|
||||
"""Create a directory for a specific topic under a year group syllabus."""
|
||||
topic_path = os.path.join(year_group_syllabus_path, "topics", f"{topic_id}")
|
||||
return self.create_directory(topic_path), topic_path
|
||||
|
||||
|
||||
def create_curriculum_keystage_topic_directory(self, keystage_syllabus_path, topic_id):
|
||||
"""Create a directory for a specific key stage topic under a key stage group syllabus."""
|
||||
topic_path = os.path.join(keystage_syllabus_path, "core_topics", f"{topic_id}")
|
||||
return self.create_directory(topic_path), topic_path
|
||||
|
||||
def create_curriculum_lesson_directory(self, topic_path, lesson_id):
|
||||
"""Create a directory for a specific lesson under a topic."""
|
||||
lesson_path = os.path.join(topic_path, "lessons", f"{lesson_id}")
|
||||
@@ -276,285 +270,4 @@ class ClassroomCopilotFilesystem:
|
||||
|
||||
def create_teacher_planned_lesson_directory(self, class_path, lesson_id):
|
||||
planned_lesson_path = os.path.join(class_path, "planned_lessons", lesson_id)
|
||||
return self.create_directory(planned_lesson_path), planned_lesson_path
|
||||
|
||||
# TLDraw File Creation
|
||||
def create_default_tldraw_file(self, node_path, node_data):
|
||||
"""Create a tldraw file for a node."""
|
||||
logging.info(f"Creating tldraw file for node at {node_path}")
|
||||
|
||||
# Ensure the directory exists
|
||||
os.makedirs(node_path, exist_ok=True)
|
||||
|
||||
tldraw_path = os.path.join(node_path, 'tldraw_file.json')
|
||||
|
||||
# Create default tldraw content
|
||||
tldraw_content = {
|
||||
"document": {
|
||||
"store": {
|
||||
"document:document": {
|
||||
"gridSize": 10,
|
||||
"name": "",
|
||||
"meta": {},
|
||||
"id": "document:document",
|
||||
"typeName": "document"
|
||||
},
|
||||
"page:page": {
|
||||
"meta": {},
|
||||
"id": "page:page",
|
||||
"name": "Page 1",
|
||||
"index": "a1",
|
||||
"typeName": "page"
|
||||
}
|
||||
},
|
||||
"schema":
|
||||
{"schemaVersion":2,
|
||||
"sequences": {
|
||||
"com.tldraw.store":4,
|
||||
"com.tldraw.asset":1,
|
||||
"com.tldraw.camera":1,
|
||||
"com.tldraw.document":2,
|
||||
"com.tldraw.instance":25,
|
||||
"com.tldraw.instance_page_state":5,
|
||||
"com.tldraw.page":1,
|
||||
"com.tldraw.instance_presence":5,
|
||||
"com.tldraw.pointer":1,
|
||||
"com.tldraw.shape":4,
|
||||
"com.tldraw.asset.bookmark":2,
|
||||
"com.tldraw.asset.image":5,
|
||||
"com.tldraw.asset.video":5,
|
||||
"com.tldraw.shape.arrow":5,
|
||||
"com.tldraw.shape.bookmark":2,
|
||||
"com.tldraw.shape.draw":2,
|
||||
"com.tldraw.shape.embed":4,
|
||||
"com.tldraw.shape.frame":0,
|
||||
"com.tldraw.shape.geo":9,
|
||||
"com.tldraw.shape.group":0,
|
||||
"com.tldraw.shape.highlight":1,
|
||||
"com.tldraw.shape.image":4,
|
||||
"com.tldraw.shape.line":5,
|
||||
"com.tldraw.shape.note":8,
|
||||
"com.tldraw.shape.text":2,
|
||||
"com.tldraw.shape.video":2,
|
||||
"com.tldraw.shape.youtube-embed":0,
|
||||
"com.tldraw.shape.calendar":0,
|
||||
"com.tldraw.shape.microphone":1,
|
||||
"com.tldraw.shape.transcriptionText":0,
|
||||
"com.tldraw.shape.slide":0,"com.tldraw.shape.slideshow":0,
|
||||
"com.tldraw.shape.user_node":1,
|
||||
"com.tldraw.shape.developer_node":1,
|
||||
"com.tldraw.shape.student_node":1,
|
||||
"com.tldraw.shape.teacher_node":1,
|
||||
"com.tldraw.shape.calendar_node":1,
|
||||
"com.tldraw.shape.calendar_year_node":1,
|
||||
"com.tldraw.shape.calendar_month_node":1,
|
||||
"com.tldraw.shape.calendar_week_node":1,
|
||||
"com.tldraw.shape.calendar_day_node":1,
|
||||
"com.tldraw.shape.calendar_time_chunk_node":1,
|
||||
"com.tldraw.shape.teacher_timetable_node":1,
|
||||
"com.tldraw.shape.timetable_lesson_node":1,
|
||||
"com.tldraw.shape.planned_lesson_node":1,
|
||||
"com.tldraw.shape.pastoral_structure_node":1,
|
||||
"com.tldraw.shape.year_group_node":1,
|
||||
"com.tldraw.shape.curriculum_structure_node":1,
|
||||
"com.tldraw.shape.key_stage_node":1,
|
||||
"com.tldraw.shape.key_stage_syllabus_node":1,
|
||||
"com.tldraw.shape.year_group_syllabus_node":1,
|
||||
"com.tldraw.shape.subject_node":1,
|
||||
"com.tldraw.shape.topic_node":1,
|
||||
"com.tldraw.shape.topic_lesson_node":1,
|
||||
"com.tldraw.shape.learning_statement_node":1,
|
||||
"com.tldraw.shape.science_lab_node":1,
|
||||
"com.tldraw.shape.school_timetable_node":1,
|
||||
"com.tldraw.shape.academic_year_node":1,
|
||||
"com.tldraw.shape.academic_term_node":1,
|
||||
"com.tldraw.shape.academic_week_node":1,
|
||||
"com.tldraw.shape.academic_day_node":1,
|
||||
"com.tldraw.shape.academic_period_node":1,
|
||||
"com.tldraw.shape.registration_period_node":1,
|
||||
"com.tldraw.shape.school_node":1,
|
||||
"com.tldraw.shape.department_node":1,
|
||||
"com.tldraw.shape.room_node":1,
|
||||
"com.tldraw.shape.subject_class_node":1,
|
||||
"com.tldraw.shape.general_relationship":1,
|
||||
"com.tldraw.binding.arrow":0,
|
||||
"com.tldraw.binding.slide-layout":0
|
||||
}
|
||||
},
|
||||
"recordVersions": {
|
||||
"asset": { "version": 1, "subTypeKey": "type", "subTypeVersions": {} },
|
||||
"camera": { "version": 1 },
|
||||
"document": { "version": 2 },
|
||||
"instance": { "version": 21 },
|
||||
"instance_page_state": { "version": 5 },
|
||||
"page": { "version": 1 },
|
||||
"shape": { "version": 3, "subTypeKey": "type", "subTypeVersions": {} },
|
||||
"instance_presence": { "version": 5 },
|
||||
"pointer": { "version": 1 }
|
||||
},
|
||||
"rootShapeIds":[],
|
||||
"bindings":[],
|
||||
"assets":[]
|
||||
},
|
||||
"session": {
|
||||
"version": 0,
|
||||
"currentPageId": "page:page",
|
||||
"pageStates": [{
|
||||
"pageId": "page:page",
|
||||
"camera": {"x": 0, "y": 0, "z": 1},
|
||||
"selectedShapeIds": []
|
||||
}]
|
||||
},
|
||||
"node_data": node_data
|
||||
}
|
||||
|
||||
with open(tldraw_path, 'w') as f:
|
||||
json.dump(tldraw_content, f, indent=4)
|
||||
|
||||
logging.info(f"tldraw file created at {tldraw_path}")
|
||||
return tldraw_path
|
||||
|
||||
def create_default_tldraw_file_in_storage(self, admin_supabase, bucket_id, file_path, node_data):
|
||||
"""Create a tldraw file in Supabase storage."""
|
||||
logging.info(f"Creating tldraw file in storage at {file_path}")
|
||||
|
||||
# Create default tldraw content
|
||||
tldraw_content = {
|
||||
"document": {
|
||||
"store": {
|
||||
"document:document": {
|
||||
"gridSize": 10,
|
||||
"name": "",
|
||||
"meta": {},
|
||||
"id": "document:document",
|
||||
"typeName": "document"
|
||||
},
|
||||
"page:page": {
|
||||
"meta": {},
|
||||
"id": "page:page",
|
||||
"name": "Page 1",
|
||||
"index": "a1",
|
||||
"typeName": "page"
|
||||
}
|
||||
},
|
||||
"schema":
|
||||
{"schemaVersion":2,
|
||||
"sequences": {
|
||||
"com.tldraw.store":4,
|
||||
"com.tldraw.asset":1,
|
||||
"com.tldraw.camera":1,
|
||||
"com.tldraw.document":2,
|
||||
"com.tldraw.instance":25,
|
||||
"com.tldraw.instance_page_state":5,
|
||||
"com.tldraw.page":1,
|
||||
"com.tldraw.instance_presence":5,
|
||||
"com.tldraw.pointer":1,
|
||||
"com.tldraw.shape":4,
|
||||
"com.tldraw.asset.bookmark":2,
|
||||
"com.tldraw.asset.image":5,
|
||||
"com.tldraw.asset.video":5,
|
||||
"com.tldraw.shape.arrow":5,
|
||||
"com.tldraw.shape.bookmark":2,
|
||||
"com.tldraw.shape.draw":2,
|
||||
"com.tldraw.shape.embed":4,
|
||||
"com.tldraw.shape.frame":0,
|
||||
"com.tldraw.shape.geo":9,
|
||||
"com.tldraw.shape.group":0,
|
||||
"com.tldraw.shape.highlight":1,
|
||||
"com.tldraw.shape.image":4,
|
||||
"com.tldraw.shape.line":5,
|
||||
"com.tldraw.shape.note":8,
|
||||
"com.tldraw.shape.text":2,
|
||||
"com.tldraw.shape.video":2,
|
||||
"com.tldraw.shape.youtube-embed":0,
|
||||
"com.tldraw.shape.calendar":0,
|
||||
"com.tldraw.shape.microphone":1,
|
||||
"com.tldraw.shape.transcriptionText":0,
|
||||
"com.tldraw.shape.slide":0,"com.tldraw.shape.slideshow":0,
|
||||
"com.tldraw.shape.user_node":1,
|
||||
"com.tldraw.shape.developer_node":1,
|
||||
"com.tldraw.shape.student_node":1,
|
||||
"com.tldraw.shape.teacher_node":1,
|
||||
"com.tldraw.shape.calendar_node":1,
|
||||
"com.tldraw.shape.calendar_year_node":1,
|
||||
"com.tldraw.shape.calendar_month_node":1,
|
||||
"com.tldraw.shape.calendar_week_node":1,
|
||||
"com.tldraw.shape.calendar_day_node":1,
|
||||
"com.tldraw.shape.calendar_time_chunk_node":1,
|
||||
"com.tldraw.shape.teacher_timetable_node":1,
|
||||
"com.tldraw.shape.timetable_lesson_node":1,
|
||||
"com.tldraw.shape.planned_lesson_node":1,
|
||||
"com.tldraw.shape.pastoral_structure_node":1,
|
||||
"com.tldraw.shape.year_group_node":1,
|
||||
"com.tldraw.shape.curriculum_structure_node":1,
|
||||
"com.tldraw.shape.key_stage_node":1,
|
||||
"com.tldraw.shape.key_stage_syllabus_node":1,
|
||||
"com.tldraw.shape.year_group_syllabus_node":1,
|
||||
"com.tldraw.shape.subject_node":1,
|
||||
"com.tldraw.shape.topic_node":1,
|
||||
"com.tldraw.shape.topic_lesson_node":1,
|
||||
"com.tldraw.shape.learning_statement_node":1,
|
||||
"com.tldraw.shape.science_lab_node":1,
|
||||
"com.tldraw.shape.school_timetable_node":1,
|
||||
"com.tldraw.shape.academic_year_node":1,
|
||||
"com.tldraw.shape.academic_term_node":1,
|
||||
"com.tldraw.shape.academic_week_node":1,
|
||||
"com.tldraw.shape.academic_day_node":1,
|
||||
"com.tldraw.shape.academic_period_node":1,
|
||||
"com.tldraw.shape.registration_period_node":1,
|
||||
"com.tldraw.shape.school_node":1,
|
||||
"com.tldraw.shape.department_node":1,
|
||||
"com.tldraw.shape.room_node":1,
|
||||
"com.tldraw.shape.subject_class_node":1,
|
||||
"com.tldraw.shape.general_relationship":1,
|
||||
"com.tldraw.binding.arrow":0,
|
||||
"com.tldraw.binding.slide-layout":0
|
||||
}
|
||||
},
|
||||
"recordVersions": {
|
||||
"asset": { "version": 1, "subTypeKey": "type", "subTypeVersions": {} },
|
||||
"camera": { "version": 1 },
|
||||
"document": { "version": 2 },
|
||||
"instance": { "version": 21 },
|
||||
"instance_page_state": { "version": 5 },
|
||||
"page": { "version": 1 },
|
||||
"shape": { "version": 3, "subTypeKey": "type", "subTypeVersions": {} },
|
||||
"instance_presence": { "version": 5 },
|
||||
"pointer": { "version": 1 }
|
||||
},
|
||||
"rootShapeIds":[],
|
||||
"bindings":[],
|
||||
"assets":[]
|
||||
},
|
||||
"session": {
|
||||
"version": 0,
|
||||
"currentPageId": "page:page",
|
||||
"pageStates": [{
|
||||
"pageId": "page:page",
|
||||
"camera": {"x": 0, "y": 0, "z": 1},
|
||||
"selectedShapeIds": []
|
||||
}]
|
||||
},
|
||||
"node_data": node_data
|
||||
}
|
||||
|
||||
# Convert the content to JSON string
|
||||
tldraw_json = json.dumps(tldraw_content, indent=4)
|
||||
|
||||
try:
|
||||
# Upload the file to Supabase storage
|
||||
result = admin_supabase.storage.from_(bucket_id).upload(
|
||||
path=file_path,
|
||||
file=tldraw_json,
|
||||
file_options={"content-type": "application/json"}
|
||||
)
|
||||
|
||||
if result.get('error'):
|
||||
logging.error(f"Error creating tldraw file in storage: {result['error']}")
|
||||
raise Exception(f"Failed to create tldraw file: {result['error']}")
|
||||
|
||||
logging.info(f"tldraw file created in storage at {file_path}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logging.error(f"Error creating tldraw file in storage: {str(e)}")
|
||||
raise e
|
||||
return self.create_directory(planned_lesson_path), planned_lesson_path
|
||||
@@ -13,8 +13,8 @@ def get_static_nodes(context: str, db_name: str) -> List[Dict[str, Any]]:
|
||||
query = """
|
||||
MATCH (t:Teacher)
|
||||
RETURN DISTINCT {
|
||||
id: t.unique_id,
|
||||
path: t.path,
|
||||
id: t.uuid_string,
|
||||
path: t.node_storage_path,
|
||||
label: t.teacher_name_formal,
|
||||
type: 'Teacher',
|
||||
isStatic: true,
|
||||
@@ -24,8 +24,8 @@ def get_static_nodes(context: str, db_name: str) -> List[Dict[str, Any]]:
|
||||
UNION ALL
|
||||
MATCH (t:UserTeacherTimetable)
|
||||
RETURN DISTINCT {
|
||||
id: t.unique_id,
|
||||
path: t.path,
|
||||
id: t.uuid_string,
|
||||
path: t.node_storage_path,
|
||||
label: t.name,
|
||||
type: 'UserTeacherTimetable',
|
||||
isStatic: true,
|
||||
@@ -35,8 +35,8 @@ def get_static_nodes(context: str, db_name: str) -> List[Dict[str, Any]]:
|
||||
UNION ALL
|
||||
MATCH (t:UserTeacherTimetable)-[:HAS_CLASS]->(c:Class)
|
||||
RETURN DISTINCT {
|
||||
id: c.unique_id,
|
||||
path: c.path,
|
||||
id: c.uuid_string,
|
||||
path: c.node_storage_path,
|
||||
label: c.name,
|
||||
type: 'Class',
|
||||
isStatic: true,
|
||||
@@ -49,8 +49,8 @@ def get_static_nodes(context: str, db_name: str) -> List[Dict[str, Any]]:
|
||||
query = """
|
||||
MATCH (u:User)
|
||||
RETURN DISTINCT {
|
||||
id: u.unique_id,
|
||||
path: u.path,
|
||||
id: u.uuid_string,
|
||||
path: u.node_storage_path,
|
||||
label: u.user_name,
|
||||
type: 'User',
|
||||
isStatic: true,
|
||||
@@ -70,8 +70,8 @@ def get_static_nodes(context: str, db_name: str) -> List[Dict[str, Any]]:
|
||||
ELSE 1
|
||||
END as nodeOrder
|
||||
RETURN DISTINCT {
|
||||
id: n.unique_id,
|
||||
path: n.path,
|
||||
id: n.uuid_string,
|
||||
path: n.node_storage_path,
|
||||
label: n.name,
|
||||
type: 'Calendar',
|
||||
isStatic: true,
|
||||
@@ -98,7 +98,7 @@ def get_today_calendar_node(db_name: str) -> Optional[Dict[str, Any]]:
|
||||
query = """
|
||||
MATCH (n:Calendar)
|
||||
WHERE date($today) >= date(n.start_date) AND date($today) <= date(n.end_date)
|
||||
RETURN n.unique_id as id, n.path as path, n.name as label,
|
||||
RETURN n.uuid_string as id, n.path as path, n.name as label,
|
||||
'Calendar' as type
|
||||
LIMIT 1
|
||||
"""
|
||||
@@ -117,7 +117,7 @@ def get_relative_calendar_node(day_offset: int, db_name: str) -> Optional[Dict[s
|
||||
query = """
|
||||
MATCH (n:Calendar)
|
||||
WHERE date($target_date) >= date(n.start_date) AND date($target_date) <= date(n.end_date)
|
||||
RETURN n.unique_id as id, n.path as path, n.name as label,
|
||||
RETURN n.uuid_string as id, n.node_storage_path as path, n.name as label,
|
||||
'Calendar' as type
|
||||
LIMIT 1
|
||||
"""
|
||||
@@ -136,7 +136,7 @@ def get_next_month_node(db_name: str) -> Optional[Dict[str, Any]]:
|
||||
query = """
|
||||
MATCH (n:Calendar)
|
||||
WHERE date($next_month_start) >= date(n.start_date) AND date($next_month_start) <= date(n.end_date)
|
||||
RETURN n.unique_id as id, n.path as path, n.name as label,
|
||||
RETURN n.uuid_string as id, n.node_storage_path as path, n.name as label,
|
||||
'Calendar' as type
|
||||
LIMIT 1
|
||||
"""
|
||||
@@ -155,7 +155,7 @@ def get_previous_month_node(db_name: str) -> Optional[Dict[str, Any]]:
|
||||
query = """
|
||||
MATCH (n:Calendar)
|
||||
WHERE date($prev_month_start) >= date(n.start_date) AND date($prev_month_start) <= date(n.end_date)
|
||||
RETURN n.unique_id as id, n.path as path, n.name as label,
|
||||
RETURN n.uuid_string as id, n.node_storage_path as path, n.name as label,
|
||||
'Calendar' as type
|
||||
LIMIT 1
|
||||
"""
|
||||
@@ -172,7 +172,7 @@ def get_user_timetables(db_name: str) -> List[Dict[str, Any]]:
|
||||
"""Get user's timetables."""
|
||||
query = """
|
||||
MATCH (t:UserTeacherTimetable)
|
||||
RETURN t.unique_id as id, t.path as path, t.name as label,
|
||||
RETURN t.uuid_string as id, t.node_storage_path as path, t.name as label,
|
||||
'UserTeacherTimetable' as type
|
||||
"""
|
||||
try:
|
||||
@@ -186,8 +186,8 @@ def get_user_timetables(db_name: str) -> List[Dict[str, Any]]:
|
||||
def get_timetable_classes(timetable_id: str, db_name: str) -> List[Dict[str, Any]]:
|
||||
"""Get classes for a timetable."""
|
||||
query = """
|
||||
MATCH (t:UserTeacherTimetable {unique_id: $timetable_id})-[:HAS_CLASS]->(c:Class)
|
||||
RETURN c.unique_id as id, c.path as path, c.name as label,
|
||||
MATCH (t:UserTeacherTimetable {uuid_string: $timetable_id})-[:HAS_CLASS]->(c:Class)
|
||||
RETURN c.uuid_string as id, c.node_storage_path as path, c.name as label,
|
||||
'Class' as type
|
||||
"""
|
||||
try:
|
||||
@@ -202,9 +202,9 @@ def get_next_lesson(class_id: str, db_name: str) -> Optional[Dict[str, Any]]:
|
||||
"""Get next lesson for a class."""
|
||||
now = datetime.now().strftime("%Y-%m-%d %H:%M")
|
||||
query = """
|
||||
MATCH (c:Class {unique_id: $class_id})-[:HAS_LESSON]->(l:Lesson)
|
||||
MATCH (c:Class {uuid_string: $class_id})-[:HAS_LESSON]->(l:Lesson)
|
||||
WHERE l.start_time > $now
|
||||
RETURN l.unique_id as id, l.path as path, l.name as label,
|
||||
RETURN l.uuid_string as id, l.node_storage_path as path, l.name as label,
|
||||
'Lesson' as type
|
||||
ORDER BY l.start_time ASC
|
||||
LIMIT 1
|
||||
@@ -222,9 +222,9 @@ def get_previous_lesson(class_id: str, db_name: str) -> Optional[Dict[str, Any]]
|
||||
"""Get previous lesson for a class."""
|
||||
now = datetime.now().strftime("%Y-%m-%d %H:%M")
|
||||
query = """
|
||||
MATCH (c:Class {unique_id: $class_id})-[:HAS_LESSON]->(l:Lesson)
|
||||
MATCH (c:Class {uuid_string: $class_id})-[:HAS_LESSON]->(l:Lesson)
|
||||
WHERE l.start_time < $now
|
||||
RETURN l.unique_id as id, l.path as path, l.name as label,
|
||||
RETURN l.uuid_string as id, l.path as path, l.name as label,
|
||||
'Lesson' as type
|
||||
ORDER BY l.start_time DESC
|
||||
LIMIT 1
|
||||
@@ -251,20 +251,20 @@ def save_shared_snapshot(path: str, room_id: str, snapshot: Dict[str, Any]) -> b
|
||||
def get_connected_nodes_for_workers(node_id: str, db_name: str) -> List[Dict[str, Any]]:
|
||||
"""Get connected nodes specific to the workers context."""
|
||||
query = """
|
||||
MATCH (n {unique_id: $node_id})
|
||||
MATCH (n {uuid_string: $node_id})
|
||||
WITH n
|
||||
CALL {
|
||||
WITH n
|
||||
MATCH (n:UserTeacherTimetable)-[:HAS_CLASS]->(c:Class)
|
||||
RETURN c.unique_id as id, c.path as path, c.name as label,
|
||||
RETURN c.uuid_string as id, c.node_storage_path as path, c.name as label,
|
||||
'Class' as type
|
||||
UNION
|
||||
MATCH (n:Class)<-[:HAS_CLASS]-(t:UserTeacherTimetable)
|
||||
RETURN t.unique_id as id, t.path as path, t.name as label,
|
||||
RETURN t.uuid_string as id, t.node_storage_path as path, t.name as label,
|
||||
'UserTeacherTimetable' as type
|
||||
UNION
|
||||
MATCH (n:Class)-[:HAS_LESSON]->(l:Lesson)
|
||||
RETURN l.unique_id as id, l.path as path, l.name as label,
|
||||
RETURN l.uuid_string as id, l.node_storage_path as path, l.name as label,
|
||||
'Lesson' as type
|
||||
}
|
||||
RETURN DISTINCT id, path, label, type
|
||||
@@ -284,8 +284,8 @@ def get_connected_nodes(node_id: str, db_name: str, context: str = None) -> List
|
||||
|
||||
# Default query for other contexts
|
||||
query = """
|
||||
MATCH (n {unique_id: $node_id})-[r]-(connected)
|
||||
RETURN DISTINCT connected.unique_id as id, connected.path as path,
|
||||
MATCH (n {uuid_string: $node_id})-[r]-(connected)
|
||||
RETURN DISTINCT connected.uuid_string as id, connected.path as path,
|
||||
connected.name as label, labels(connected)[0] as type
|
||||
"""
|
||||
try:
|
||||
@@ -314,37 +314,37 @@ def get_worker_structure(db_name: str) -> Dict[str, Any]:
|
||||
// Collect all nodes
|
||||
RETURN {
|
||||
schools: collect(DISTINCT {
|
||||
id: s.unique_id,
|
||||
path: s.path,
|
||||
id: s.uuid_string,
|
||||
path: s.node_storage_path,
|
||||
name: s.school_name,
|
||||
__primarylabel__: 'School'
|
||||
}),
|
||||
departments: collect(DISTINCT {
|
||||
id: d.unique_id,
|
||||
path: d.path,
|
||||
id: d.uuid_string,
|
||||
path: d.node_storage_path,
|
||||
code: d.department_code,
|
||||
school_id: s.unique_id,
|
||||
school_id: s.uuid_string,
|
||||
__primarylabel__: 'Department'
|
||||
}),
|
||||
timetables: collect(DISTINCT {
|
||||
id: t.unique_id,
|
||||
path: t.path,
|
||||
id: t.uuid_string,
|
||||
path: t.node_storage_path,
|
||||
name: t.name,
|
||||
department_id: d.unique_id,
|
||||
department_id: d.uuid_string,
|
||||
__primarylabel__: 'UserTeacherTimetable'
|
||||
}),
|
||||
classes: collect(DISTINCT {
|
||||
id: c.unique_id,
|
||||
path: c.path,
|
||||
id: c.uuid_string,
|
||||
path: c.node_storage_path,
|
||||
code: c.class_code,
|
||||
timetable_id: t.unique_id,
|
||||
timetable_id: t.uuid_string,
|
||||
__primarylabel__: 'Class'
|
||||
}),
|
||||
lessons: collect(DISTINCT {
|
||||
id: l.unique_id,
|
||||
path: l.path,
|
||||
id: l.uuid_string,
|
||||
path: l.node_storage_path,
|
||||
start_time: l.start_time,
|
||||
class_id: c.unique_id,
|
||||
class_id: c.uuid_string,
|
||||
__primarylabel__: 'TimetableLesson'
|
||||
})
|
||||
} as structure
|
||||
@@ -369,10 +369,10 @@ def get_worker_structure(db_name: str) -> Dict[str, Any]:
|
||||
def get_school_node(school_id: str, db_name: str) -> Optional[Dict[str, Any]]:
|
||||
"""Get a specific school node."""
|
||||
query = """
|
||||
MATCH (s:School {unique_id: $school_id})
|
||||
MATCH (s:School {uuid_string: $school_id})
|
||||
RETURN {
|
||||
id: s.unique_id,
|
||||
path: s.path,
|
||||
id: s.uuid_string,
|
||||
path: s.node_storage_path,
|
||||
name: s.school_name,
|
||||
__primarylabel__: 'School'
|
||||
} as node
|
||||
@@ -389,10 +389,10 @@ def get_school_node(school_id: str, db_name: str) -> Optional[Dict[str, Any]]:
|
||||
def get_department_node(dept_id: str, db_name: str) -> Optional[Dict[str, Any]]:
|
||||
"""Get a specific department node."""
|
||||
query = """
|
||||
MATCH (d:Department {unique_id: $dept_id})
|
||||
MATCH (d:Department {uuid_string: $dept_id})
|
||||
RETURN {
|
||||
id: d.unique_id,
|
||||
path: d.path,
|
||||
id: d.uuid_string,
|
||||
path: d.node_storage_path,
|
||||
code: d.department_code,
|
||||
__primarylabel__: 'Department'
|
||||
} as node
|
||||
@@ -409,10 +409,10 @@ def get_department_node(dept_id: str, db_name: str) -> Optional[Dict[str, Any]]:
|
||||
def get_timetable_node(timetable_id: str, db_name: str) -> Optional[Dict[str, Any]]:
|
||||
"""Get a specific timetable node."""
|
||||
query = """
|
||||
MATCH (t:UserTeacherTimetable {unique_id: $timetable_id})
|
||||
MATCH (t:UserTeacherTimetable {uuid_string: $timetable_id})
|
||||
RETURN {
|
||||
id: t.unique_id,
|
||||
path: t.path,
|
||||
id: t.uuid_string,
|
||||
path: t.node_storage_path,
|
||||
name: t.name,
|
||||
__primarylabel__: 'UserTeacherTimetable'
|
||||
} as node
|
||||
@@ -429,10 +429,10 @@ def get_timetable_node(timetable_id: str, db_name: str) -> Optional[Dict[str, An
|
||||
def get_class_node(class_id: str, db_name: str) -> Optional[Dict[str, Any]]:
|
||||
"""Get a specific class node."""
|
||||
query = """
|
||||
MATCH (c:Class {unique_id: $class_id})
|
||||
MATCH (c:Class {uuid_string: $class_id})
|
||||
RETURN {
|
||||
id: c.unique_id,
|
||||
path: c.path,
|
||||
id: c.uuid_string,
|
||||
path: c.node_storage_path,
|
||||
code: c.class_code,
|
||||
__primarylabel__: 'Class'
|
||||
} as node
|
||||
@@ -449,10 +449,10 @@ def get_class_node(class_id: str, db_name: str) -> Optional[Dict[str, Any]]:
|
||||
def get_lesson_node(lesson_id: str, db_name: str) -> Optional[Dict[str, Any]]:
|
||||
"""Get a specific lesson node."""
|
||||
query = """
|
||||
MATCH (l:TimetableLesson {unique_id: $lesson_id})
|
||||
MATCH (l:TimetableLesson {uuid_string: $lesson_id})
|
||||
RETURN {
|
||||
id: l.unique_id,
|
||||
path: l.path,
|
||||
id: l.uuid_string,
|
||||
path: l.node_storage_path,
|
||||
start_time: l.start_time,
|
||||
__primarylabel__: 'TimetableLesson'
|
||||
} as node
|
||||
@@ -473,8 +473,8 @@ def get_current_lesson(db_name: str) -> Optional[Dict[str, Any]]:
|
||||
MATCH (l:TimetableLesson)
|
||||
WHERE l.start_time >= $now
|
||||
RETURN {
|
||||
id: l.unique_id,
|
||||
path: l.path,
|
||||
id: l.uuid_string,
|
||||
path: l.node_storage_path,
|
||||
start_time: l.start_time,
|
||||
__primarylabel__: 'TimetableLesson'
|
||||
} as node
|
||||
|
||||
@@ -15,8 +15,6 @@ logging = logger.get_logger(
|
||||
import requests
|
||||
import base64
|
||||
|
||||
dev_mode = os.getenv('DEV_MODE', 'false')
|
||||
|
||||
def send_query(query, encoded_credentials=None, params=None, method='POST', database="system", endpoint="/tx/commit"):
|
||||
if encoded_credentials is None:
|
||||
logging.debug(f"Sending query to Neo4j: {query}")
|
||||
|
||||
@@ -15,16 +15,16 @@ logging = logger.get_logger(
|
||||
import modules.database.tools.queries as query
|
||||
from contextlib import suppress
|
||||
|
||||
def get_node_by_unique_id_and_adjacent_nodes(session, unique_id):
|
||||
return session.read_transaction(_get_node_by_unique_id_and_adjacent_nodes, unique_id)
|
||||
def get_node_by_uuid_string_and_adjacent_nodes(session, uuid_string):
|
||||
return session.read_transaction(_get_node_by_uuid_string_and_adjacent_nodes, uuid_string)
|
||||
|
||||
def _get_node_by_unique_id_and_adjacent_nodes(tx, unique_id):
|
||||
def _get_node_by_uuid_string_and_adjacent_nodes(tx, uuid_string):
|
||||
query = """
|
||||
MATCH (n {unique_id: $unique_id})
|
||||
MATCH (n {uuid_string: $uuid_string})
|
||||
OPTIONAL MATCH (n)-[r]-(adjacent)
|
||||
RETURN n AS node, COLLECT(DISTINCT {node: adjacent, relationship: r}) AS connected_nodes
|
||||
"""
|
||||
result = tx.run(query, unique_id=unique_id)
|
||||
result = tx.run(query, uuid_string=uuid_string)
|
||||
record = result.single()
|
||||
if record:
|
||||
node = record["node"]
|
||||
@@ -242,20 +242,20 @@ def _find_nodes_by_label(tx, label):
|
||||
result = tx.run(query)
|
||||
return [record["n"] for record in result]
|
||||
|
||||
def get_node_by_unique_id(session, unique_id):
|
||||
return session.read_transaction(_get_node_by_unique_id, unique_id)
|
||||
def get_node_by_uuid_string(session, uuid_string):
|
||||
return session.read_transaction(_get_node_by_uuid_string, uuid_string)
|
||||
|
||||
def _get_node_by_unique_id(tx, unique_id):
|
||||
def _get_node_by_uuid_string(tx, uuid_string):
|
||||
query = f"""
|
||||
MATCH (n)
|
||||
WHERE n.unique_id = $unique_id
|
||||
WHERE n.uuid_string = $uuid_string
|
||||
RETURN n
|
||||
"""
|
||||
logging.debug(f"Executing query with unique_id: {unique_id}")
|
||||
result = tx.run(query, unique_id=unique_id)
|
||||
logging.debug(f"Executing query with uuid_string: {uuid_string}")
|
||||
result = tx.run(query, uuid_string=uuid_string)
|
||||
record = result.single()
|
||||
if record is None:
|
||||
logging.warning(f"No node found with unique_id: {unique_id}")
|
||||
logging.warning(f"No node found with uuid_string: {uuid_string}")
|
||||
return None
|
||||
return record[0]
|
||||
|
||||
|
||||
@@ -70,11 +70,10 @@ class BaseNode(CommonModel): # pyre-ignore[13]
|
||||
|
||||
def merge(self, database: str = 'neo4j') -> None:
|
||||
"""Merge this node into the graph."""
|
||||
|
||||
|
||||
params = self._get_merge_parameters()
|
||||
|
||||
all_labels = [self.__primarylabel__] + self.__secondarylabels__
|
||||
|
||||
|
||||
cypher = f"""
|
||||
MERGE (n:{":".join(all_labels)} {{ {self.__primaryproperty__}: $pp }})
|
||||
ON MATCH SET n += $set_on_match
|
||||
@@ -82,10 +81,15 @@ class BaseNode(CommonModel): # pyre-ignore[13]
|
||||
SET n += $always_set
|
||||
RETURN n
|
||||
"""
|
||||
|
||||
|
||||
print(f"DEBUG: Executing merge query: {cypher}")
|
||||
print(f"DEBUG: With params: {params}")
|
||||
print(f"DEBUG: Database: {database}")
|
||||
|
||||
graph = GraphConnection()
|
||||
with graph.driver.session(database=database) as session:
|
||||
result = session.run(cypher, params).single()
|
||||
print(f"DEBUG: Merge result: {result}")
|
||||
if result:
|
||||
return self.__class__(**dict(result["n"]))
|
||||
return None
|
||||
|
||||
@@ -60,14 +60,20 @@ def create_or_merge_neontology_node(node: BaseNode, database: str = 'neo4j', ope
|
||||
operation (str): The operation to perform ('create' or 'merge'). Defaults to 'merge'.
|
||||
"""
|
||||
try:
|
||||
logging.debug(f"Creating/merging node: {node.__class__.__name__} with label '{node.__primarylabel__}' in database '{database}'")
|
||||
logging.debug(f"Node data: {node.to_dict()}")
|
||||
|
||||
if operation == "create":
|
||||
node.create(database=database)
|
||||
result = node.create(database=database)
|
||||
logging.debug(f"Create result: {result}")
|
||||
elif operation == "merge":
|
||||
node.merge(database=database)
|
||||
result = node.merge(database=database)
|
||||
logging.debug(f"Merge result: {result}")
|
||||
else:
|
||||
logging.error(f"Invalid operation: {operation}")
|
||||
except Exception as e:
|
||||
logging.error(f"Error in processing node: {e}")
|
||||
raise # Re-raise to see the actual error
|
||||
|
||||
# Create or merge a Neontology node in the Neo4j database. If a ValidationError occurs
|
||||
# due to a NaN value, replace it with a default value and retry.
|
||||
|
||||
@@ -0,0 +1,451 @@
|
||||
"""
|
||||
Supabase Storage Tools for ClassroomCopilot
|
||||
Replaces local filesystem paths with Supabase Storage bucket paths
|
||||
"""
|
||||
import os
|
||||
from modules.logger_tool import initialise_logger
|
||||
from typing import Tuple, Optional
|
||||
|
||||
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
|
||||
class SupabaseStorageTools:
|
||||
"""
|
||||
Generates Supabase Storage paths for TLDraw snapshots and other files.
|
||||
|
||||
Path format: bucket/nodetype/node_unique_id
|
||||
Example: cc.snapshots/User/fda8dca7-4d18-43c9-bb74-260777043447
|
||||
"""
|
||||
|
||||
def __init__(self, db_name: str, init_run_type: str = None):
|
||||
self.db_name = db_name
|
||||
self.init_run_type = init_run_type
|
||||
|
||||
# Define bucket mappings based on node types
|
||||
self.bucket_mappings = {
|
||||
'User': 'cc.public.snapshots',
|
||||
'Teacher': 'cc.public.snapshots',
|
||||
'Student': 'cc.public.snapshots',
|
||||
'School': 'cc.public.snapshots',
|
||||
'Department': 'cc.public.snapshots',
|
||||
'Subject': 'cc.public.snapshots',
|
||||
'CalendarYear': 'cc.public.snapshots',
|
||||
'CalendarMonth': 'cc.public.snapshots',
|
||||
'CalendarWeek': 'cc.public.snapshots',
|
||||
'CalendarDay': 'cc.public.snapshots',
|
||||
'CalendarTimeChunk': 'cc.public.snapshots',
|
||||
'KeyStage': 'cc.public.snapshots',
|
||||
'YearGroup': 'cc.public.snapshots',
|
||||
'KeyStageSyllabus': 'cc.public.snapshots',
|
||||
'YearGroupSyllabus': 'cc.public.snapshots',
|
||||
'Topic': 'cc.public.snapshots',
|
||||
'TopicLesson': 'cc.public.snapshots',
|
||||
'LearningStatement': 'cc.public.snapshots',
|
||||
'UserTeacherTimetable': 'cc.public.snapshots',
|
||||
'Class': 'cc.public.snapshots',
|
||||
'TimetableLesson': 'cc.public.snapshots',
|
||||
'SuperAdmin': 'cc.public.snapshots',
|
||||
'Developer': 'cc.public.snapshots',
|
||||
'CurriculumStructure': 'cc.public.snapshots',
|
||||
'PastoralStructure': 'cc.public.snapshots',
|
||||
'DepartmentStructure': 'cc.public.snapshots',
|
||||
}
|
||||
|
||||
logger.info(f"Initializing SupabaseStorageTools with db_name: {db_name} and init_run_type: {init_run_type}")
|
||||
|
||||
def get_storage_path(self, node_type: str, node_id: str) -> str:
|
||||
"""
|
||||
Generate Supabase Storage path for a node.
|
||||
|
||||
Args:
|
||||
node_type: The type of node (e.g., 'User', 'Teacher', 'School')
|
||||
node_id: The unique identifier for the node
|
||||
|
||||
Returns:
|
||||
str: Storage path in format bucket/nodetype/node_id
|
||||
"""
|
||||
bucket = self.bucket_mappings.get(node_type, 'cc.public.snapshots')
|
||||
path = f"{bucket}/{node_type}/{node_id}"
|
||||
|
||||
logger.debug(f"Generated storage path for {node_type} {node_id}: {path}")
|
||||
return path
|
||||
|
||||
def create_user_storage_path(self, user_id: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a user node.
|
||||
|
||||
Args:
|
||||
user_id: The user's unique identifier
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
path = self.get_storage_path('User', user_id)
|
||||
logger.info(f"Created user storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_teacher_storage_path(self, teacher_id: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a teacher node.
|
||||
|
||||
Args:
|
||||
teacher_id: The teacher's unique identifier
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
path = self.get_storage_path('Teacher', teacher_id)
|
||||
logger.info(f"Created teacher storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_student_storage_path(self, student_id: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a student node.
|
||||
|
||||
Args:
|
||||
student_id: The student's unique identifier
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
path = self.get_storage_path('Student', student_id)
|
||||
logger.info(f"Created student storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_school_storage_path(self, school_id: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a school node.
|
||||
|
||||
Args:
|
||||
school_id: The school's unique identifier
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
path = self.get_storage_path('School', school_id)
|
||||
logger.info(f"Created school storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_calendar_year_storage_path(self, year: int) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a calendar year node.
|
||||
|
||||
Args:
|
||||
year: The year
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
path = self.get_storage_path('CalendarYear', str(year))
|
||||
logger.info(f"Created calendar year storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_calendar_month_storage_path(self, year: int, month: int) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a calendar month node.
|
||||
|
||||
Args:
|
||||
year: The year
|
||||
month: The month
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
month_id = f"{year}_{month:02d}"
|
||||
path = self.get_storage_path('CalendarMonth', month_id)
|
||||
logger.info(f"Created calendar month storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_calendar_week_storage_path(self, year: int, week: int) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a calendar week node.
|
||||
|
||||
Args:
|
||||
year: The ISO year
|
||||
week: The ISO week number
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
week_id = f"{year}_{week:02d}"
|
||||
path = self.get_storage_path('CalendarWeek', week_id)
|
||||
logger.info(f"Created calendar week storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_calendar_day_storage_path(self, year: int, month: int, day: int) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a calendar day node.
|
||||
|
||||
Args:
|
||||
year: The year
|
||||
month: The month
|
||||
day: The day
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
day_id = f"{year}_{month:02d}_{day:02d}"
|
||||
path = self.get_storage_path('CalendarDay', day_id)
|
||||
logger.info(f"Created calendar day storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_calendar_time_chunk_storage_path(self, day_id: str, chunk_index: int) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a calendar time chunk node.
|
||||
|
||||
Args:
|
||||
day_id: The day identifier (e.g., "2025_01_15")
|
||||
chunk_index: The time chunk index within the day
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
chunk_id = f"{day_id}_{chunk_index:02d}"
|
||||
path = self.get_storage_path('CalendarTimeChunk', chunk_id)
|
||||
logger.info(f"Created calendar time chunk storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_department_storage_path(self, department_id: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a department node.
|
||||
|
||||
Args:
|
||||
department_id: The department's unique identifier
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
path = self.get_storage_path('Department', department_id)
|
||||
logger.info(f"Created department storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_subject_storage_path(self, subject_id: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a subject node.
|
||||
|
||||
Args:
|
||||
subject_id: The subject's unique identifier
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
path = self.get_storage_path('Subject', subject_id)
|
||||
logger.info(f"Created subject storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_key_stage_storage_path(self, key_stage_id: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a key stage node.
|
||||
|
||||
Args:
|
||||
key_stage_id: The key stage's unique identifier
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
path = self.get_storage_path('KeyStage', key_stage_id)
|
||||
logger.info(f"Created key stage storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_year_group_storage_path(self, year_group_id: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a year group node.
|
||||
|
||||
Args:
|
||||
year_group_id: The year group's unique identifier
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
path = self.get_storage_path('YearGroup', year_group_id)
|
||||
logger.info(f"Created year group storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_key_stage_syllabus_storage_path(self, syllabus_id: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a key stage syllabus node.
|
||||
|
||||
Args:
|
||||
syllabus_id: The syllabus's unique identifier
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
path = self.get_storage_path('KeyStageSyllabus', syllabus_id)
|
||||
logger.info(f"Created key stage syllabus storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_year_group_syllabus_storage_path(self, syllabus_id: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a year group syllabus node.
|
||||
|
||||
Args:
|
||||
syllabus_id: The syllabus's unique identifier
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
path = self.get_storage_path('YearGroupSyllabus', syllabus_id)
|
||||
logger.info(f"Created year group syllabus storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_topic_storage_path(self, topic_id: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a topic node.
|
||||
|
||||
Args:
|
||||
topic_id: The topic's unique identifier
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
path = self.get_storage_path('Topic', topic_id)
|
||||
logger.info(f"Created topic storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_topic_lesson_storage_path(self, lesson_id: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a topic lesson node.
|
||||
|
||||
Args:
|
||||
lesson_id: The lesson's unique identifier
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
path = self.get_storage_path('TopicLesson', lesson_id)
|
||||
logger.info(f"Created topic lesson storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_learning_statement_storage_path(self, statement_id: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a learning statement node.
|
||||
|
||||
Args:
|
||||
statement_id: The statement's unique identifier
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
path = self.get_storage_path('LearningStatement', statement_id)
|
||||
logger.info(f"Created learning statement storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_timetable_storage_path(self, timetable_id: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a timetable node.
|
||||
|
||||
Args:
|
||||
timetable_id: The timetable's unique identifier
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
path = self.get_storage_path('UserTeacherTimetable', timetable_id)
|
||||
logger.info(f"Created timetable storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_class_storage_path(self, class_id: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a class node.
|
||||
|
||||
Args:
|
||||
class_id: The class's unique identifier
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
path = self.get_storage_path('Class', class_id)
|
||||
logger.info(f"Created class storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_timetable_lesson_storage_path(self, lesson_id: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a timetable lesson node.
|
||||
|
||||
Args:
|
||||
lesson_id: The lesson's unique identifier
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
path = self.get_storage_path('TimetableLesson', lesson_id)
|
||||
logger.info(f"Created timetable lesson storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_super_admin_storage_path(self, admin_id: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a super admin node.
|
||||
|
||||
Args:
|
||||
admin_id: The admin's unique identifier
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
path = self.get_storage_path('SuperAdmin', admin_id)
|
||||
logger.info(f"Created super admin storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_curriculum_storage_path(self, curriculum_id: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a curriculum structure node.
|
||||
|
||||
Args:
|
||||
curriculum_id: The curriculum's unique identifier
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
path = self.get_storage_path('CurriculumStructure', curriculum_id)
|
||||
logger.info(f"Created curriculum structure storage path: {path}")
|
||||
return True, path
|
||||
|
||||
def create_pastoral_storage_path(self, pastoral_id: str) -> Tuple[bool, str]:
|
||||
"""
|
||||
Create storage path for a pastoral structure node.
|
||||
|
||||
Args:
|
||||
pastoral_id: The pastoral's unique identifier
|
||||
|
||||
Returns:
|
||||
Tuple[bool, str]: (success, storage_path)
|
||||
"""
|
||||
path = self.get_storage_path('PastoralStructure', pastoral_id)
|
||||
logger.info(f"Created pastoral structure storage path: {path}")
|
||||
return True, path
|
||||
|
||||
# Legacy compatibility methods that return the same interface as filesystem tools
|
||||
def create_private_user_directory(self, user_id: str) -> Tuple[bool, str]:
|
||||
"""Legacy compatibility method."""
|
||||
return self.create_user_storage_path(user_id)
|
||||
|
||||
def create_user_worker_directory(self, user_path: str, worker_id: str, worker_type: str) -> Tuple[bool, str]:
|
||||
"""Legacy compatibility method."""
|
||||
if worker_type in ['teacher', 'email_teacher', 'ms_teacher']:
|
||||
return self.create_teacher_storage_path(worker_id)
|
||||
elif worker_type in ['student', 'email_student', 'ms_student']:
|
||||
return self.create_student_storage_path(worker_id)
|
||||
elif worker_type == 'superadmin':
|
||||
return self.create_super_admin_storage_path(worker_id)
|
||||
elif worker_type == 'developer':
|
||||
return self.create_developer_storage_path(worker_id)
|
||||
else:
|
||||
# Default to generic storage path
|
||||
path = self.get_storage_path(worker_type.title(), worker_id)
|
||||
return True, path
|
||||
|
||||
def create_school_directory(self, school_uuid_string: str) -> Tuple[bool, str]:
|
||||
"""Legacy compatibility method."""
|
||||
return self.create_school_storage_path(school_uuid_string)
|
||||
|
||||
def create_school_curriculum_directory(self, school_path: Optional[str] = None) -> Tuple[bool, str]:
|
||||
"""Legacy compatibility method - returns empty path since curriculum is handled by individual nodes."""
|
||||
return True, ""
|
||||
|
||||
def create_school_pastoral_directory(self, school_path: Optional[str] = None) -> Tuple[bool, str]:
|
||||
"""Legacy compatibility method - returns empty path since pastoral is handled by individual nodes."""
|
||||
return True, ""
|
||||
|
||||
def create_directory(self, path: str) -> bool:
|
||||
"""Legacy compatibility method - always returns True since we don't create physical directories."""
|
||||
return True
|
||||
@@ -0,0 +1,544 @@
|
||||
"""
|
||||
Enhanced Document Analysis Module
|
||||
|
||||
This module provides comprehensive document structure analysis beyond basic split maps,
|
||||
including multi-level hierarchies, numbering system detection, and content type analysis.
|
||||
"""
|
||||
|
||||
import re
|
||||
import json
|
||||
import uuid
|
||||
import datetime
|
||||
from typing import Dict, List, Any, Optional, Tuple
|
||||
from dataclasses import dataclass, asdict
|
||||
from pathlib import Path
|
||||
import fitz # PyMuPDF
|
||||
from modules.logger_tool import initialise_logger
|
||||
import os
|
||||
|
||||
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
|
||||
@dataclass
|
||||
class DocumentSection:
|
||||
"""Represents a document section at any hierarchy level"""
|
||||
id: str
|
||||
title: str
|
||||
level: int
|
||||
start_page: int
|
||||
end_page: int
|
||||
numbering: Optional[str] = None
|
||||
parent_id: Optional[str] = None
|
||||
children: List[str] = None
|
||||
content_types: Dict[str, int] = None
|
||||
confidence: float = 0.8
|
||||
|
||||
def __post_init__(self):
|
||||
if self.children is None:
|
||||
self.children = []
|
||||
if self.content_types is None:
|
||||
self.content_types = {}
|
||||
|
||||
@dataclass
|
||||
class NumberingSystem:
|
||||
"""Represents a detected numbering or coding system"""
|
||||
system_id: str
|
||||
pattern: str
|
||||
description: str
|
||||
examples: List[str]
|
||||
applies_to_levels: List[int]
|
||||
confidence: float
|
||||
|
||||
@dataclass
|
||||
class ContentBlock:
|
||||
"""Represents a content block with type information"""
|
||||
block_id: str
|
||||
page: int
|
||||
bbox: Tuple[float, float, float, float] # x0, y0, x1, y1
|
||||
content_type: str # 'text', 'image', 'table', 'formula', 'diagram'
|
||||
text_content: Optional[str] = None
|
||||
metadata: Dict[str, Any] = None
|
||||
|
||||
class DocumentAnalyzer:
|
||||
"""Enhanced document analyzer for comprehensive structure detection"""
|
||||
|
||||
def __init__(self):
|
||||
self.numbering_patterns = [
|
||||
# Roman numerals
|
||||
(r'^([IVX]+)\.?\s+(.+)', 'roman_numerals', 'Roman numeral chapters (I, II, III, ...)'),
|
||||
# Decimal numbering
|
||||
(r'^(\d+(?:\.\d+)*)\.?\s+(.+)', 'decimal_numbering', 'Decimal numbering (1.1, 1.2.1, ...)'),
|
||||
# Letter numbering
|
||||
(r'^([A-Z])\.?\s+(.+)', 'letter_chapters', 'Letter chapters (A, B, C, ...)'),
|
||||
(r'^([a-z])\.?\s+(.+)', 'letter_sections', 'Letter sections (a, b, c, ...)'),
|
||||
# Bracketed numbering
|
||||
(r'^\((\d+)\)\s+(.+)', 'bracketed_numbers', 'Bracketed numbers ((1), (2), ...)'),
|
||||
# Legal numbering
|
||||
(r'^§\s*(\d+(?:\.\d+)*)\s+(.+)', 'legal_sections', 'Legal sections (§1, §1.1, ...)'),
|
||||
# Article numbering
|
||||
(r'^(?:Article|Art\.?)\s+(\d+(?:\.\d+)*)\s+(.+)', 'articles', 'Article numbering'),
|
||||
]
|
||||
|
||||
def analyze_document_structure(self, pdf_bytes: bytes, tika_json: Dict = None,
|
||||
docling_json: Dict = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Fast, header-only structure analysis.
|
||||
- Prefer PDF outline/bookmarks
|
||||
- Otherwise, use Docling heading roles from existing artefact JSON
|
||||
- No full-text scans; no per-page content analysis
|
||||
"""
|
||||
logger.info("Starting FAST document structure analysis (headings only)")
|
||||
|
||||
doc = fitz.open(stream=pdf_bytes, filetype="pdf")
|
||||
page_count = len(doc)
|
||||
|
||||
# Try PDF outline first
|
||||
sections: List[DocumentSection] = self._extract_from_pdf_outline(doc)
|
||||
|
||||
# Fallback to Docling headings if outline inadequate
|
||||
if (not sections) and docling_json:
|
||||
sections = self._extract_from_docling(docling_json)
|
||||
|
||||
# Final fallback: coarse windows
|
||||
if not sections:
|
||||
sections = []
|
||||
step = max(10, min(30, page_count // 5 or 1))
|
||||
i = 1
|
||||
idx = 1
|
||||
while i <= page_count:
|
||||
end = min(page_count, i + step - 1)
|
||||
sections.append(DocumentSection(
|
||||
id=f"sec{idx:02d}",
|
||||
title=f"Pages {i}-{end}",
|
||||
level=1,
|
||||
start_page=i,
|
||||
end_page=end,
|
||||
confidence=0.2
|
||||
))
|
||||
i = end + 1
|
||||
idx += 1
|
||||
|
||||
# Build hierarchy relationships and adjust parent end-pages using sibling boundaries
|
||||
sections = self._build_section_hierarchy(sections)
|
||||
# Normalize and finalize sections (clamp, front matter, last-page coverage)
|
||||
sections = self._normalize_and_cover(sections, page_count)
|
||||
|
||||
doc.close()
|
||||
|
||||
return {
|
||||
"version": 2,
|
||||
"analysis_timestamp": datetime.datetime.utcnow().isoformat() + "Z",
|
||||
"page_count": page_count,
|
||||
"sections": [asdict(section) for section in sections],
|
||||
"metadata": {
|
||||
"analyzer_version": "2.1-fast",
|
||||
"analysis_methods": ["pdf_outline", "docling_headings"],
|
||||
}
|
||||
}
|
||||
|
||||
def _extract_hierarchical_structure(self, doc: fitz.Document, tika_json: Dict = None,
|
||||
docling_json: Dict = None) -> List[DocumentSection]:
|
||||
# Kept for backward compat; delegate to outline + docling only
|
||||
sections = self._extract_from_pdf_outline(doc)
|
||||
if (not sections) and docling_json:
|
||||
sections = self._extract_from_docling(docling_json)
|
||||
return sections
|
||||
|
||||
def _extract_from_pdf_outline(self, doc: fitz.Document) -> List[DocumentSection]:
|
||||
"""Extract sections from PDF outline/bookmarks"""
|
||||
sections = []
|
||||
toc = doc.get_toc(simple=False)
|
||||
|
||||
for i, (level, title, page, dest) in enumerate(toc):
|
||||
if page < 1:
|
||||
continue
|
||||
|
||||
section_id = f"outline_{i:03d}"
|
||||
|
||||
# Calculate end page (next section's start - 1, or last page)
|
||||
end_page = page
|
||||
for j in range(i + 1, len(toc)):
|
||||
if toc[j][2] > 0: # Valid page number
|
||||
end_page = toc[j][2] - 1
|
||||
break
|
||||
else:
|
||||
end_page = len(doc)
|
||||
|
||||
section = DocumentSection(
|
||||
id=section_id,
|
||||
title=title.strip(),
|
||||
level=level,
|
||||
start_page=page,
|
||||
end_page=end_page,
|
||||
confidence=0.95
|
||||
)
|
||||
sections.append(section)
|
||||
|
||||
return sections
|
||||
|
||||
def _extract_from_docling(self, docling_json: Dict) -> List[DocumentSection]:
|
||||
"""Extract sections from Docling analysis"""
|
||||
sections: List[DocumentSection] = []
|
||||
blocks = docling_json.get("blocks", []) or docling_json.get("elements", [])
|
||||
|
||||
# 1) Collect headings with page and level, preserving order
|
||||
heading_items: List[Tuple[int, str, int, str]] = [] # (page, level, order_index, title)
|
||||
order_index = 0
|
||||
for block in blocks:
|
||||
role_raw = (block.get("role") or block.get("type") or "").lower()
|
||||
if not ("heading" in role_raw or role_raw in ("h1", "h2", "h3", "h4", "h5", "h6", "title")):
|
||||
continue
|
||||
text = (block.get("text") or block.get("content") or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
page_val = block.get("page", None)
|
||||
if page_val is None:
|
||||
page_val = block.get("page_no", None)
|
||||
if page_val is None:
|
||||
page_val = block.get("pageIndex", None)
|
||||
try:
|
||||
page_int = int(page_val) if page_val is not None else 1
|
||||
except Exception:
|
||||
page_int = 1
|
||||
# Normalize to 1-based
|
||||
page_int = page_int + 1 if page_int == 0 else page_int
|
||||
|
||||
# Determine heading level
|
||||
level = 1
|
||||
if "1" in role_raw or role_raw == "title":
|
||||
level = 1
|
||||
elif "2" in role_raw:
|
||||
level = 2
|
||||
elif "3" in role_raw:
|
||||
level = 3
|
||||
elif "4" in role_raw:
|
||||
level = 4
|
||||
elif "5" in role_raw:
|
||||
level = 5
|
||||
elif "6" in role_raw:
|
||||
level = 6
|
||||
|
||||
heading_items.append((page_int, level, order_index, text))
|
||||
order_index += 1
|
||||
|
||||
if not heading_items:
|
||||
return sections
|
||||
|
||||
# 2) Sort by page then order_index to preserve within-page order
|
||||
heading_items.sort(key=lambda h: (h[0], h[2]))
|
||||
|
||||
# 3) Build sections with hierarchical end-page computation
|
||||
stack: List[DocumentSection] = []
|
||||
idx_counter = 0
|
||||
|
||||
def close_until(level_threshold: int, next_page: int):
|
||||
nonlocal sections, stack
|
||||
while stack and stack[-1].level >= level_threshold:
|
||||
cur = stack.pop()
|
||||
# If next heading on same page, close at same page; else previous page
|
||||
if next_page <= cur.start_page:
|
||||
cur.end_page = cur.start_page
|
||||
else:
|
||||
cur.end_page = next_page - 1
|
||||
if cur.end_page < cur.start_page:
|
||||
cur.end_page = cur.start_page
|
||||
sections.append(cur)
|
||||
|
||||
for page_int, level, _, text in heading_items:
|
||||
# Close siblings and deeper levels
|
||||
close_until(level, page_int)
|
||||
# Open new heading
|
||||
section_id = f"docling_{idx_counter:03d}"
|
||||
idx_counter += 1
|
||||
new_sec = DocumentSection(
|
||||
id=section_id,
|
||||
title=text,
|
||||
level=level,
|
||||
start_page=page_int,
|
||||
end_page=page_int, # temporary; will finalize when closing
|
||||
confidence=0.8
|
||||
)
|
||||
stack.append(new_sec)
|
||||
|
||||
# Close any remaining open sections at document end later in normalization
|
||||
while stack:
|
||||
cur = stack.pop()
|
||||
sections.append(cur)
|
||||
|
||||
return sections
|
||||
|
||||
def _normalize_and_cover(self, sections: List[DocumentSection], page_count: int) -> List[DocumentSection]:
|
||||
"""Harden outline sections while preserving hierarchy:
|
||||
- clamp each section to [1, page_count]
|
||||
- fix inverted ranges (but DO NOT remove hierarchical overlaps)
|
||||
- ensure coverage from page 1 with synthetic front-matter if needed
|
||||
- ensure last top-level section extends to page_count
|
||||
- compute numbering hint if missing
|
||||
"""
|
||||
if not sections:
|
||||
return sections
|
||||
|
||||
# Clamp values per section; do not modify overlap relationships
|
||||
for s in sections:
|
||||
s.start_page = max(1, min(s.start_page or 1, page_count))
|
||||
s.end_page = max(1, min(s.end_page or s.start_page, page_count))
|
||||
if s.end_page < s.start_page:
|
||||
s.end_page = s.start_page
|
||||
|
||||
# Maintain original order (as produced by extractor and hierarchy builder)
|
||||
|
||||
# Insert synthetic front matter if needed
|
||||
if sections and sections[0].start_page > 1:
|
||||
# Generate a unique synthetic id that won't collide with existing ids
|
||||
existing_ids = {s.id for s in sections}
|
||||
base_id = "outline_front_matter"
|
||||
syn_id = base_id
|
||||
idx = 1
|
||||
while syn_id in existing_ids:
|
||||
syn_id = f"{base_id}_{idx}"
|
||||
idx += 1
|
||||
front = DocumentSection(
|
||||
id=syn_id,
|
||||
title="Front matter",
|
||||
level=1,
|
||||
start_page=1,
|
||||
end_page=sections[0].start_page - 1,
|
||||
confidence=0.6
|
||||
)
|
||||
sections.insert(0, front)
|
||||
|
||||
# Ensure last top-level section covers to page_count
|
||||
top_levels = [s for s in sections if s.parent_id is None]
|
||||
if top_levels:
|
||||
last_top = top_levels[-1]
|
||||
if last_top.end_page < page_count:
|
||||
last_top.end_page = page_count
|
||||
|
||||
# Light numbering extraction based on heading text prefix
|
||||
for s in sections:
|
||||
m = re.match(r"^\s*([A-Za-z]+|[IVXLCM]+|\d+(?:\.\d+)*)\.?\s+", s.title)
|
||||
if m:
|
||||
s.numbering = s.numbering or m.group(1)
|
||||
|
||||
return sections
|
||||
|
||||
def _extract_from_text_patterns(self, doc: fitz.Document) -> List[DocumentSection]:
|
||||
# Disabled in fast mode
|
||||
return []
|
||||
|
||||
def _estimate_section_level(self, numbering: str, system_type: str) -> int:
|
||||
"""Estimate section level based on numbering pattern"""
|
||||
if system_type == 'roman_numerals':
|
||||
return 1 # Typically chapter level
|
||||
elif system_type == 'decimal_numbering':
|
||||
dots = numbering.count('.')
|
||||
return min(dots + 1, 6) # 1.1.1 = level 3
|
||||
elif system_type == 'letter_chapters':
|
||||
return 1
|
||||
elif system_type == 'letter_sections':
|
||||
return 2
|
||||
else:
|
||||
return 2 # Default
|
||||
|
||||
def _detect_numbering_systems(self, sections: List[DocumentSection]) -> List[NumberingSystem]:
|
||||
"""Detect numbering systems used in the document"""
|
||||
systems = []
|
||||
|
||||
# Group sections by their numbering patterns
|
||||
pattern_groups = {}
|
||||
for section in sections:
|
||||
if section.numbering:
|
||||
for pattern, system_type, description in self.numbering_patterns:
|
||||
if re.match(pattern.replace(r'^(.+)', section.numbering), section.numbering):
|
||||
if system_type not in pattern_groups:
|
||||
pattern_groups[system_type] = {
|
||||
'pattern': pattern,
|
||||
'description': description,
|
||||
'examples': [],
|
||||
'levels': set(),
|
||||
'count': 0
|
||||
}
|
||||
pattern_groups[system_type]['examples'].append(section.numbering)
|
||||
pattern_groups[system_type]['levels'].add(section.level)
|
||||
pattern_groups[system_type]['count'] += 1
|
||||
break
|
||||
|
||||
# Create NumberingSystem objects
|
||||
for system_type, data in pattern_groups.items():
|
||||
if data['count'] >= 2: # At least 2 examples to be confident
|
||||
system = NumberingSystem(
|
||||
system_id=system_type,
|
||||
pattern=data['pattern'],
|
||||
description=data['description'],
|
||||
examples=data['examples'][:5], # First 5 examples
|
||||
applies_to_levels=list(data['levels']),
|
||||
confidence=min(0.9, 0.5 + (data['count'] * 0.1))
|
||||
)
|
||||
systems.append(system)
|
||||
|
||||
return systems
|
||||
|
||||
def _analyze_content_types(self, doc: fitz.Document, sections: List[DocumentSection]) -> Dict[str, Any]:
|
||||
# Disabled in fast mode
|
||||
return {"total_blocks": 0, "content_types": {}, "sections": {}}
|
||||
|
||||
def _detect_tables_in_page(self, page) -> int:
|
||||
return 0
|
||||
|
||||
def _has_complex_formatting(self, page) -> bool:
|
||||
return False
|
||||
|
||||
def _merge_and_deduplicate_sections(self, sections: List[DocumentSection]) -> List[DocumentSection]:
|
||||
"""Merge overlapping sections and remove duplicates"""
|
||||
if not sections:
|
||||
return []
|
||||
|
||||
# Sort by start page, then by level
|
||||
sections.sort(key=lambda s: (s.start_page, s.level))
|
||||
|
||||
merged = []
|
||||
for section in sections:
|
||||
# Check if this section overlaps significantly with existing ones
|
||||
is_duplicate = False
|
||||
for existing in merged:
|
||||
if (existing.start_page == section.start_page and
|
||||
abs(existing.level - section.level) <= 1 and
|
||||
self._text_similarity(existing.title, section.title) > 0.8):
|
||||
# This is likely a duplicate, merge information
|
||||
if section.confidence > existing.confidence:
|
||||
existing.title = section.title
|
||||
existing.numbering = section.numbering or existing.numbering
|
||||
existing.confidence = section.confidence
|
||||
is_duplicate = True
|
||||
break
|
||||
|
||||
if not is_duplicate:
|
||||
merged.append(section)
|
||||
|
||||
return merged
|
||||
|
||||
def _text_similarity(self, text1: str, text2: str) -> float:
|
||||
"""Calculate text similarity (simple implementation)"""
|
||||
if not text1 or not text2:
|
||||
return 0.0
|
||||
|
||||
# Simple word-based similarity
|
||||
words1 = set(text1.lower().split())
|
||||
words2 = set(text2.lower().split())
|
||||
|
||||
if not words1 and not words2:
|
||||
return 1.0
|
||||
|
||||
intersection = words1.intersection(words2)
|
||||
union = words1.union(words2)
|
||||
|
||||
return len(intersection) / len(union) if union else 0.0
|
||||
|
||||
def _build_section_hierarchy(self, sections: List[DocumentSection]) -> List[DocumentSection]:
|
||||
"""Build parent-child relationships between sections"""
|
||||
if not sections:
|
||||
return []
|
||||
|
||||
# Sort by start page and level
|
||||
sections.sort(key=lambda s: (s.start_page, s.level))
|
||||
|
||||
# Build hierarchy
|
||||
for i, section in enumerate(sections):
|
||||
# Find parent (previous section with lower level)
|
||||
for j in range(i - 1, -1, -1):
|
||||
potential_parent = sections[j]
|
||||
if (potential_parent.level < section.level and
|
||||
potential_parent.start_page <= section.start_page and
|
||||
potential_parent.end_page >= section.start_page):
|
||||
section.parent_id = potential_parent.id
|
||||
potential_parent.children.append(section.id)
|
||||
break
|
||||
|
||||
# Update end pages based on children
|
||||
for j in range(i + 1, len(sections)):
|
||||
next_section = sections[j]
|
||||
if (next_section.level <= section.level and
|
||||
next_section.start_page > section.start_page):
|
||||
section.end_page = min(section.end_page, next_section.start_page - 1)
|
||||
break
|
||||
|
||||
return sections
|
||||
|
||||
def _generate_processing_recommendations(self, sections: List[DocumentSection],
|
||||
content_analysis: Dict) -> Dict[str, Any]:
|
||||
"""Generate intelligent processing recommendations"""
|
||||
recommendations = {
|
||||
"document_length_category": "short", # short, medium, long
|
||||
"suggested_processing_approach": "full_document", # full_document, section_by_section
|
||||
"high_priority_sections": [],
|
||||
"ocr_recommended_sections": [],
|
||||
"image_analysis_sections": [],
|
||||
"table_extraction_sections": [],
|
||||
"estimated_total_time": 0
|
||||
}
|
||||
|
||||
total_pages = max(s.end_page for s in sections) if sections else 0
|
||||
|
||||
# Categorize document length
|
||||
if total_pages <= 10:
|
||||
recommendations["document_length_category"] = "short"
|
||||
recommendations["suggested_processing_approach"] = "full_document"
|
||||
elif total_pages <= 50:
|
||||
recommendations["document_length_category"] = "medium"
|
||||
recommendations["suggested_processing_approach"] = "section_by_section"
|
||||
else:
|
||||
recommendations["document_length_category"] = "long"
|
||||
recommendations["suggested_processing_approach"] = "section_by_section"
|
||||
|
||||
# Analyze each section for recommendations
|
||||
total_estimated_time = 0
|
||||
for section in sections:
|
||||
section_data = content_analysis["sections"].get(section.id, {})
|
||||
|
||||
# High priority sections (first few sections, or sections with important titles)
|
||||
if (section.start_page <= 5 or
|
||||
any(keyword in section.title.lower() for keyword in
|
||||
['abstract', 'summary', 'introduction', 'conclusion', 'executive'])):
|
||||
recommendations["high_priority_sections"].append(section.id)
|
||||
|
||||
# OCR recommendations
|
||||
if section_data.get("images", 0) > 0 or section_data.get("has_complex_formatting", False):
|
||||
recommendations["ocr_recommended_sections"].append(section.id)
|
||||
|
||||
# Image analysis recommendations
|
||||
if section_data.get("images", 0) > 2: # Sections with multiple images
|
||||
recommendations["image_analysis_sections"].append(section.id)
|
||||
|
||||
# Table extraction recommendations
|
||||
if section_data.get("tables", 0) > 0:
|
||||
recommendations["table_extraction_sections"].append(section.id)
|
||||
|
||||
total_estimated_time += section_data.get("estimated_processing_time", 0)
|
||||
|
||||
recommendations["estimated_total_time"] = total_estimated_time
|
||||
|
||||
return recommendations
|
||||
|
||||
def create_document_outline_hierarchy_artefact(file_id: str, pdf_bytes: bytes,
|
||||
tika_json: Dict = None,
|
||||
docling_json: Dict = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Create a comprehensive document outline hierarchy artefact
|
||||
|
||||
Args:
|
||||
file_id: File ID
|
||||
pdf_bytes: PDF file content
|
||||
tika_json: Optional Tika analysis results
|
||||
docling_json: Optional Docling analysis results
|
||||
|
||||
Returns:
|
||||
Document outline hierarchy artefact
|
||||
"""
|
||||
analyzer = DocumentAnalyzer()
|
||||
analysis = analyzer.analyze_document_structure(pdf_bytes, tika_json, docling_json)
|
||||
|
||||
# Add file metadata
|
||||
analysis["file_id"] = file_id
|
||||
analysis["artefact_id"] = str(uuid.uuid4())
|
||||
analysis["artefact_type"] = "document_outline_hierarchy"
|
||||
|
||||
return analysis
|
||||
@@ -39,28 +39,41 @@ class DocumentProcessor:
|
||||
|
||||
# Use LibreOffice for conversion
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
output_file = Path(temp_dir) / f"{input_file.stem}.pdf"
|
||||
|
||||
# Convert using LibreOffice
|
||||
cmd = [
|
||||
'libreoffice',
|
||||
'--headless',
|
||||
'--convert-to', 'pdf',
|
||||
'--outdir', str(temp_dir),
|
||||
str(input_file)
|
||||
]
|
||||
|
||||
try:
|
||||
subprocess.run(cmd, check=True, capture_output=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
raise RuntimeError(f"Conversion failed: {e.stderr.decode()}")
|
||||
with tempfile.TemporaryDirectory() as profile_dir:
|
||||
output_file = Path(temp_dir) / f"{input_file.stem}.pdf"
|
||||
|
||||
# Convert using LibreOffice with explicit profile directory
|
||||
cmd = [
|
||||
'/Applications/LibreOffice.app/Contents/MacOS/soffice',
|
||||
'--headless',
|
||||
'--invisible',
|
||||
'--nodefault',
|
||||
'--nolockcheck',
|
||||
'--nologo',
|
||||
'--norestore',
|
||||
f'-env:UserInstallation=file://{profile_dir}',
|
||||
'--convert-to', 'pdf',
|
||||
'--outdir', str(temp_dir),
|
||||
str(input_file)
|
||||
]
|
||||
|
||||
try:
|
||||
result = subprocess.run(cmd, check=True, capture_output=True, timeout=120)
|
||||
except subprocess.CalledProcessError as e:
|
||||
stderr_msg = e.stderr.decode() if e.stderr else "No stderr output"
|
||||
stdout_msg = e.stdout.decode() if e.stdout else "No stdout output"
|
||||
raise RuntimeError(f"Conversion failed: {stderr_msg}. Stdout: {stdout_msg}")
|
||||
except subprocess.TimeoutExpired:
|
||||
raise RuntimeError("Conversion failed: Process timed out after 120 seconds")
|
||||
|
||||
if not output_file.exists():
|
||||
raise RuntimeError("Conversion failed: Output file not created")
|
||||
if not output_file.exists():
|
||||
# List all files in temp_dir for debugging
|
||||
files_in_dir = list(Path(temp_dir).glob('*'))
|
||||
raise RuntimeError(f"Conversion failed: Output file not created. Files in output dir: {files_in_dir}")
|
||||
|
||||
# Read and return the PDF content
|
||||
with open(output_file, 'rb') as f:
|
||||
return f.read()
|
||||
# Read and return the PDF content
|
||||
with open(output_file, 'rb') as f:
|
||||
return f.read()
|
||||
|
||||
def batch_convert_directory(self, directory: str) -> List[Dict]:
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,411 @@
|
||||
"""
|
||||
Memory-Aware Queue Management System
|
||||
====================================
|
||||
|
||||
Provides intelligent queue management based on memory usage and file sizes
|
||||
rather than simple task count limits. Supports multiple users with fair
|
||||
queuing and capacity management.
|
||||
|
||||
Features:
|
||||
- Memory-based queue limits (not just task count)
|
||||
- Fair queuing across multiple users
|
||||
- Upload capacity checking with user feedback
|
||||
- Graceful degradation under load
|
||||
- Service-specific memory tracking
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import json
|
||||
import uuid
|
||||
import logging
|
||||
from typing import Dict, List, Optional, Any, Tuple
|
||||
from dataclasses import dataclass, asdict
|
||||
from enum import Enum
|
||||
import redis
|
||||
from .redis_manager import get_redis_manager
|
||||
import psutil
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class QueueStatus(Enum):
|
||||
ACCEPTING = "accepting" # Normal operation
|
||||
BUSY = "busy" # High load, warn users
|
||||
OVERLOADED = "overloaded" # Reject new uploads
|
||||
MAINTENANCE = "maintenance" # Manual override
|
||||
|
||||
@dataclass
|
||||
class MemoryConfig:
|
||||
"""Memory configuration for queue management."""
|
||||
max_total_memory_mb: int = 2048 # 2GB total queue memory
|
||||
max_user_memory_mb: int = 512 # 512MB per user
|
||||
max_file_size_mb: int = 100 # 100MB max file size
|
||||
memory_warning_threshold: float = 0.8 # Warn at 80%
|
||||
memory_reject_threshold: float = 0.95 # Reject at 95%
|
||||
|
||||
@dataclass
|
||||
class QueuedFile:
|
||||
"""Represents a file waiting in the queue."""
|
||||
file_id: str
|
||||
user_id: str
|
||||
filename: str
|
||||
size_bytes: int
|
||||
mime_type: str
|
||||
cabinet_id: str
|
||||
priority: int = 1
|
||||
queued_at: float = 0
|
||||
estimated_processing_time: int = 300 # seconds
|
||||
memory_estimate_mb: float = 0
|
||||
|
||||
def __post_init__(self):
|
||||
if self.queued_at == 0:
|
||||
self.queued_at = time.time()
|
||||
|
||||
# Estimate memory usage (rough heuristic)
|
||||
self.memory_estimate_mb = self._estimate_memory_usage()
|
||||
|
||||
def _estimate_memory_usage(self) -> float:
|
||||
"""Estimate memory usage for this file during processing."""
|
||||
base_mb = self.size_bytes / (1024 * 1024)
|
||||
|
||||
# Processing multipliers based on operations
|
||||
if self.mime_type == 'application/pdf':
|
||||
# PDF: original + extracted text + images + thumbnails
|
||||
return base_mb * 3.5
|
||||
elif self.mime_type.startswith('image/'):
|
||||
# Images: original + resized variants + OCR text
|
||||
return base_mb * 2.5
|
||||
else:
|
||||
# Other docs: original + PDF conversion + processing
|
||||
return base_mb * 4.0
|
||||
|
||||
class MemoryAwareQueue:
|
||||
"""Memory-aware queue management system."""
|
||||
|
||||
def __init__(self, environment: str = "dev"):
|
||||
self.redis_manager = get_redis_manager(environment)
|
||||
self.redis_client = self.redis_manager.client
|
||||
self.config = self._load_config()
|
||||
|
||||
# Redis keys
|
||||
self.upload_queue_key = "upload_queue"
|
||||
self.processing_memory_key = "processing_memory"
|
||||
self.user_quota_key = "user_quotas"
|
||||
self.system_status_key = "system_status"
|
||||
|
||||
logger.info(f"🧠 Memory-aware queue initialized (max: {self.config.max_total_memory_mb}MB)")
|
||||
|
||||
def _load_config(self) -> MemoryConfig:
|
||||
"""Load memory configuration from environment."""
|
||||
return MemoryConfig(
|
||||
max_total_memory_mb=int(os.getenv('QUEUE_MAX_MEMORY_MB', '2048')),
|
||||
max_user_memory_mb=int(os.getenv('QUEUE_MAX_USER_MEMORY_MB', '512')),
|
||||
max_file_size_mb=int(os.getenv('MAX_FILE_SIZE_MB', '100')),
|
||||
memory_warning_threshold=float(os.getenv('MEMORY_WARNING_THRESHOLD', '0.8')),
|
||||
memory_reject_threshold=float(os.getenv('MEMORY_REJECT_THRESHOLD', '0.95'))
|
||||
)
|
||||
|
||||
def check_upload_capacity(self, user_id: str, file_size_bytes: int,
|
||||
mime_type: str) -> Tuple[bool, str, Dict[str, Any]]:
|
||||
"""
|
||||
Check if system can accept a new upload.
|
||||
|
||||
Returns:
|
||||
(can_accept, message, queue_info)
|
||||
"""
|
||||
|
||||
# Create temporary QueuedFile to estimate memory
|
||||
temp_file = QueuedFile(
|
||||
file_id="temp",
|
||||
user_id=user_id,
|
||||
filename="temp",
|
||||
size_bytes=file_size_bytes,
|
||||
mime_type=mime_type,
|
||||
cabinet_id="temp"
|
||||
)
|
||||
|
||||
file_memory_mb = temp_file.memory_estimate_mb
|
||||
|
||||
# Check file size limit
|
||||
if file_size_bytes > (self.config.max_file_size_mb * 1024 * 1024):
|
||||
return False, f"File too large (max: {self.config.max_file_size_mb}MB)", {}
|
||||
|
||||
# Get current memory usage
|
||||
current_memory = self._get_current_memory_usage()
|
||||
user_memory = self._get_user_memory_usage(user_id)
|
||||
|
||||
# Check user quota
|
||||
if user_memory + file_memory_mb > self.config.max_user_memory_mb:
|
||||
return False, f"User quota exceeded (limit: {self.config.max_user_memory_mb}MB)", {
|
||||
'user_current': user_memory,
|
||||
'user_limit': self.config.max_user_memory_mb
|
||||
}
|
||||
|
||||
# Check system capacity
|
||||
total_after = current_memory + file_memory_mb
|
||||
max_memory = self.config.max_total_memory_mb
|
||||
|
||||
if total_after > (max_memory * self.config.memory_reject_threshold):
|
||||
queue_info = self._get_queue_info()
|
||||
return False, "System overloaded. Please try again later.", {
|
||||
'current_memory': current_memory,
|
||||
'max_memory': max_memory,
|
||||
'utilization': current_memory / max_memory,
|
||||
'queue_position': queue_info['total_queued'] + 1
|
||||
}
|
||||
|
||||
# Calculate wait time estimate
|
||||
wait_estimate = self._estimate_wait_time(user_id)
|
||||
|
||||
status = "ready"
|
||||
message = "Upload accepted"
|
||||
|
||||
if total_after > (max_memory * self.config.memory_warning_threshold):
|
||||
status = "busy"
|
||||
message = f"System busy. Estimated wait: {wait_estimate // 60}m {wait_estimate % 60}s"
|
||||
|
||||
return True, message, {
|
||||
'status': status,
|
||||
'estimated_wait_seconds': wait_estimate,
|
||||
'memory_usage': {
|
||||
'current': current_memory,
|
||||
'after_upload': total_after,
|
||||
'limit': max_memory,
|
||||
'utilization': total_after / max_memory
|
||||
},
|
||||
'user_quota': {
|
||||
'used': user_memory,
|
||||
'after_upload': user_memory + file_memory_mb,
|
||||
'limit': self.config.max_user_memory_mb
|
||||
}
|
||||
}
|
||||
|
||||
def enqueue_file(self, file_id: str, user_id: str, filename: str,
|
||||
size_bytes: int, mime_type: str, cabinet_id: str,
|
||||
priority: int = 1) -> Dict[str, Any]:
|
||||
"""
|
||||
Add file to upload queue.
|
||||
|
||||
Returns:
|
||||
Queue information including position and estimated wait time
|
||||
"""
|
||||
|
||||
queued_file = QueuedFile(
|
||||
file_id=file_id,
|
||||
user_id=user_id,
|
||||
filename=filename,
|
||||
size_bytes=size_bytes,
|
||||
mime_type=mime_type,
|
||||
cabinet_id=cabinet_id,
|
||||
priority=priority
|
||||
)
|
||||
|
||||
# Serialize and add to Redis queue (priority queue: higher priority = lower score)
|
||||
score = time.time() - (priority * 1000000) # Priority affects score significantly
|
||||
|
||||
self.redis_client.zadd(
|
||||
self.upload_queue_key,
|
||||
{json.dumps(asdict(queued_file)): score}
|
||||
)
|
||||
|
||||
# Update user quota tracking
|
||||
self._update_user_quota(user_id, queued_file.memory_estimate_mb, increment=True)
|
||||
|
||||
# Get queue position and wait estimate
|
||||
position = self._get_queue_position(file_id)
|
||||
wait_estimate = self._estimate_wait_time(user_id)
|
||||
|
||||
logger.info(f"📋 Queued file {file_id} for user {user_id} (pos: {position}, wait: {wait_estimate}s)")
|
||||
|
||||
return {
|
||||
'queued': True,
|
||||
'file_id': file_id,
|
||||
'queue_position': position,
|
||||
'estimated_wait_seconds': wait_estimate,
|
||||
'memory_estimate_mb': queued_file.memory_estimate_mb
|
||||
}
|
||||
|
||||
def dequeue_next_file(self, service_name: str) -> Optional[QueuedFile]:
|
||||
"""
|
||||
Get next file from queue for processing.
|
||||
|
||||
Args:
|
||||
service_name: The service requesting work (for capacity management)
|
||||
"""
|
||||
|
||||
# Check if service has capacity
|
||||
service_memory = self._get_service_memory_usage(service_name)
|
||||
service_limit = self._get_service_memory_limit(service_name)
|
||||
|
||||
if service_memory >= service_limit:
|
||||
logger.debug(f"Service {service_name} at capacity ({service_memory}/{service_limit}MB)")
|
||||
return None
|
||||
|
||||
# Get next item from priority queue (lowest score first)
|
||||
items = self.redis_client.zrange(self.upload_queue_key, 0, 0, withscores=True)
|
||||
|
||||
if not items:
|
||||
return None
|
||||
|
||||
file_data_json, score = items[0]
|
||||
file_data = json.loads(file_data_json)
|
||||
queued_file = QueuedFile(**file_data)
|
||||
|
||||
# Check if this file would exceed service memory limit
|
||||
if service_memory + queued_file.memory_estimate_mb > service_limit:
|
||||
# Skip this file for now, try smaller ones later
|
||||
logger.debug(f"File {queued_file.file_id} too large for {service_name} capacity")
|
||||
return None
|
||||
|
||||
# Remove from queue
|
||||
self.redis_client.zrem(self.upload_queue_key, file_data_json)
|
||||
|
||||
# Update tracking
|
||||
self._update_user_quota(queued_file.user_id, queued_file.memory_estimate_mb, increment=False)
|
||||
self._update_service_memory(service_name, queued_file.memory_estimate_mb, increment=True)
|
||||
|
||||
logger.info(f"🎯 Dequeued file {queued_file.file_id} for {service_name} processing")
|
||||
|
||||
return queued_file
|
||||
|
||||
def complete_processing(self, service_name: str, file_id: str, memory_used_mb: float):
|
||||
"""Mark file processing as complete and free memory."""
|
||||
self._update_service_memory(service_name, memory_used_mb, increment=False)
|
||||
logger.info(f"✅ Completed processing {file_id} in {service_name} (freed {memory_used_mb}MB)")
|
||||
|
||||
def _get_current_memory_usage(self) -> float:
|
||||
"""Get current total memory usage across all services."""
|
||||
services = ['docling', 'tika', 'llm', 'document_analysis']
|
||||
total = 0
|
||||
|
||||
for service in services:
|
||||
service_key = f"{self.processing_memory_key}:{service}"
|
||||
memory = float(self.redis_client.get(service_key) or 0)
|
||||
total += memory
|
||||
|
||||
return total
|
||||
|
||||
def _get_user_memory_usage(self, user_id: str) -> float:
|
||||
"""Get current memory usage for a specific user."""
|
||||
user_key = f"{self.user_quota_key}:{user_id}"
|
||||
return float(self.redis_client.get(user_key) or 0)
|
||||
|
||||
def _get_service_memory_usage(self, service_name: str) -> float:
|
||||
"""Get current memory usage for a service."""
|
||||
service_key = f"{self.processing_memory_key}:{service_name}"
|
||||
return float(self.redis_client.get(service_key) or 0)
|
||||
|
||||
def _get_service_memory_limit(self, service_name: str) -> float:
|
||||
"""Get memory limit for a service."""
|
||||
# Service-specific memory limits as percentage of total
|
||||
limits = {
|
||||
'docling': 0.4, # 40% for Docling (memory-intensive)
|
||||
'tika': 0.2, # 20% for Tika
|
||||
'llm': 0.3, # 30% for LLM processing
|
||||
'document_analysis': 0.1 # 10% for document analysis
|
||||
}
|
||||
|
||||
percentage = limits.get(service_name, 0.1)
|
||||
return self.config.max_total_memory_mb * percentage
|
||||
|
||||
def _update_user_quota(self, user_id: str, memory_mb: float, increment: bool):
|
||||
"""Update user memory quota tracking."""
|
||||
user_key = f"{self.user_quota_key}:{user_id}"
|
||||
|
||||
if increment:
|
||||
self.redis_client.incrbyfloat(user_key, memory_mb)
|
||||
else:
|
||||
current = float(self.redis_client.get(user_key) or 0)
|
||||
new_value = max(0, current - memory_mb)
|
||||
self.redis_client.set(user_key, new_value)
|
||||
|
||||
# Set expiration for cleanup
|
||||
self.redis_client.expire(user_key, 86400) # 24 hours
|
||||
|
||||
def _update_service_memory(self, service_name: str, memory_mb: float, increment: bool):
|
||||
"""Update service memory usage tracking."""
|
||||
service_key = f"{self.processing_memory_key}:{service_name}"
|
||||
|
||||
if increment:
|
||||
self.redis_client.incrbyfloat(service_key, memory_mb)
|
||||
else:
|
||||
current = float(self.redis_client.get(service_key) or 0)
|
||||
new_value = max(0, current - memory_mb)
|
||||
self.redis_client.set(service_key, new_value)
|
||||
|
||||
# Set expiration for cleanup
|
||||
self.redis_client.expire(service_key, 3600) # 1 hour
|
||||
|
||||
def _get_queue_position(self, file_id: str) -> int:
|
||||
"""Get position of file in queue."""
|
||||
items = self.redis_client.zrange(self.upload_queue_key, 0, -1)
|
||||
for i, item in enumerate(items):
|
||||
file_data = json.loads(item)
|
||||
if file_data['file_id'] == file_id:
|
||||
return i + 1
|
||||
return 0
|
||||
|
||||
def _estimate_wait_time(self, user_id: str) -> int:
|
||||
"""Estimate wait time for user's next file."""
|
||||
# Simple estimation based on queue position and average processing time
|
||||
queue_size = self.redis_client.zcard(self.upload_queue_key)
|
||||
avg_processing_time = 300 # 5 minutes average
|
||||
|
||||
return int(queue_size * avg_processing_time * 0.5) # Assume parallel processing
|
||||
|
||||
def _get_queue_info(self) -> Dict[str, Any]:
|
||||
"""Get comprehensive queue information."""
|
||||
total_queued = self.redis_client.zcard(self.upload_queue_key)
|
||||
current_memory = self._get_current_memory_usage()
|
||||
max_memory = self.config.max_total_memory_mb
|
||||
|
||||
return {
|
||||
'total_queued': total_queued,
|
||||
'memory_usage': {
|
||||
'current_mb': current_memory,
|
||||
'max_mb': max_memory,
|
||||
'utilization': current_memory / max_memory if max_memory > 0 else 0
|
||||
},
|
||||
'status': self._determine_system_status(current_memory, max_memory)
|
||||
}
|
||||
|
||||
def _determine_system_status(self, current_memory: float, max_memory: float) -> str:
|
||||
"""Determine current system status based on memory usage."""
|
||||
utilization = current_memory / max_memory if max_memory > 0 else 0
|
||||
|
||||
if utilization >= self.config.memory_reject_threshold:
|
||||
return "overloaded"
|
||||
elif utilization >= self.config.memory_warning_threshold:
|
||||
return "busy"
|
||||
else:
|
||||
return "ready"
|
||||
|
||||
def get_system_status(self) -> Dict[str, Any]:
|
||||
"""Get comprehensive system status for monitoring."""
|
||||
queue_info = self._get_queue_info()
|
||||
|
||||
# Service-specific info
|
||||
services = {}
|
||||
for service_name in ['docling', 'tika', 'llm', 'document_analysis']:
|
||||
services[service_name] = {
|
||||
'memory_used_mb': self._get_service_memory_usage(service_name),
|
||||
'memory_limit_mb': self._get_service_memory_limit(service_name),
|
||||
'utilization': self._get_service_memory_usage(service_name) / self._get_service_memory_limit(service_name)
|
||||
}
|
||||
|
||||
return {
|
||||
'status': queue_info['status'],
|
||||
'queue': queue_info,
|
||||
'services': services,
|
||||
'config': asdict(self.config)
|
||||
}
|
||||
|
||||
# Convenience functions
|
||||
def get_memory_queue(environment: str = "dev") -> MemoryAwareQueue:
|
||||
"""Get memory-aware queue instance."""
|
||||
return MemoryAwareQueue(environment)
|
||||
|
||||
def check_upload_capacity(user_id: str, file_size: int, mime_type: str, environment: str = "dev") -> Tuple[bool, str, Dict]:
|
||||
"""Quick capacity check for upload."""
|
||||
queue = get_memory_queue(environment)
|
||||
return queue.check_upload_capacity(user_id, file_size, mime_type)
|
||||
@@ -0,0 +1,223 @@
|
||||
"""
|
||||
Page Image Generation Module
|
||||
|
||||
This module generates full-resolution page images and thumbnails from PDF documents
|
||||
for use in the document viewer UI.
|
||||
"""
|
||||
|
||||
import io
|
||||
import uuid
|
||||
from typing import Dict, List, Any, Tuple
|
||||
from pathlib import Path
|
||||
import fitz # PyMuPDF
|
||||
from PIL import Image
|
||||
from modules.logger_tool import initialise_logger
|
||||
import os
|
||||
|
||||
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
|
||||
class PageImageGenerator:
|
||||
"""Generates page images and thumbnails from PDF documents"""
|
||||
|
||||
def __init__(self):
|
||||
# Image generation settings
|
||||
self.full_image_dpi = 200 # High quality for full images
|
||||
self.thumbnail_dpi = 100 # Lower quality for thumbnails
|
||||
self.thumbnail_max_width = 300
|
||||
self.thumbnail_max_height = 400
|
||||
self.image_format = "PNG"
|
||||
self.thumbnail_format = "WEBP" # More efficient for thumbnails
|
||||
|
||||
def generate_page_images(self, file_id: str, cabinet_id: str, pdf_bytes: bytes) -> Dict[str, Any]:
|
||||
"""
|
||||
Generate full page images and thumbnails for a PDF document
|
||||
|
||||
Args:
|
||||
file_id: File ID
|
||||
cabinet_id: Cabinet ID for storage path
|
||||
pdf_bytes: PDF file content
|
||||
|
||||
Returns:
|
||||
Page images artefact data
|
||||
"""
|
||||
logger.info(f"Starting page image generation for file_id={file_id}")
|
||||
|
||||
doc = fitz.open(stream=pdf_bytes, filetype="pdf")
|
||||
page_count = len(doc)
|
||||
|
||||
artefact_id = str(uuid.uuid4())
|
||||
page_images = []
|
||||
|
||||
for page_num in range(page_count):
|
||||
page = doc[page_num]
|
||||
page_number = page_num + 1
|
||||
|
||||
logger.debug(f"Processing page {page_number}/{page_count} for file_id={file_id}")
|
||||
|
||||
# Generate full resolution image
|
||||
full_image_data, full_dimensions = self._generate_full_image(page, page_number)
|
||||
full_image_path = f"{cabinet_id}/{file_id}/{artefact_id}/page_{page_number:03d}_full.png"
|
||||
|
||||
# Generate thumbnail
|
||||
thumbnail_data, thumbnail_dimensions = self._generate_thumbnail(page, page_number)
|
||||
thumbnail_path = f"{cabinet_id}/{file_id}/{artefact_id}/page_{page_number:03d}_thumb.webp"
|
||||
|
||||
page_info = {
|
||||
"page": page_number,
|
||||
"full_image_path": full_image_path,
|
||||
"full_image_data": full_image_data, # Will be uploaded separately
|
||||
"full_dimensions": full_dimensions,
|
||||
"thumbnail_path": thumbnail_path,
|
||||
"thumbnail_data": thumbnail_data, # Will be uploaded separately
|
||||
"thumbnail_dimensions": thumbnail_dimensions,
|
||||
"rotation": page.rotation,
|
||||
"has_text": bool(page.get_text().strip()),
|
||||
"has_images": len(page.get_images()) > 0,
|
||||
"has_drawings": len(page.get_drawings()) > 0
|
||||
}
|
||||
|
||||
page_images.append(page_info)
|
||||
|
||||
doc.close()
|
||||
|
||||
artefact_data = {
|
||||
"version": 1,
|
||||
"file_id": file_id,
|
||||
"artefact_id": artefact_id,
|
||||
"artefact_type": "page_images",
|
||||
"generation_timestamp": self._get_timestamp(),
|
||||
"page_count": page_count,
|
||||
"page_images": page_images,
|
||||
"generation_settings": {
|
||||
"full_image_dpi": self.full_image_dpi,
|
||||
"thumbnail_dpi": self.thumbnail_dpi,
|
||||
"thumbnail_max_width": self.thumbnail_max_width,
|
||||
"thumbnail_max_height": self.thumbnail_max_height,
|
||||
"image_format": self.image_format,
|
||||
"thumbnail_format": self.thumbnail_format
|
||||
},
|
||||
"storage_info": {
|
||||
"total_full_images": page_count,
|
||||
"total_thumbnails": page_count,
|
||||
"estimated_storage_mb": self._estimate_storage_size(page_images)
|
||||
}
|
||||
}
|
||||
|
||||
logger.info(f"Generated {page_count} page images for file_id={file_id}")
|
||||
return artefact_data
|
||||
|
||||
def _generate_full_image(self, page: fitz.Page, page_number: int) -> Tuple[bytes, Dict[str, int]]:
|
||||
"""Generate full resolution page image"""
|
||||
# Create transformation matrix for high DPI
|
||||
mat = fitz.Matrix(self.full_image_dpi / 72, self.full_image_dpi / 72)
|
||||
|
||||
# Render page to pixmap
|
||||
pix = page.get_pixmap(matrix=mat, alpha=False)
|
||||
|
||||
# Convert to PNG bytes
|
||||
img_data = pix.tobytes("png")
|
||||
|
||||
dimensions = {
|
||||
"width": pix.width,
|
||||
"height": pix.height
|
||||
}
|
||||
|
||||
pix = None # Free memory
|
||||
|
||||
return img_data, dimensions
|
||||
|
||||
def _generate_thumbnail(self, page: fitz.Page, page_number: int) -> Tuple[bytes, Dict[str, int]]:
|
||||
"""Generate thumbnail image"""
|
||||
# Create transformation matrix for lower DPI
|
||||
mat = fitz.Matrix(self.thumbnail_dpi / 72, self.thumbnail_dpi / 72)
|
||||
|
||||
# Render page to pixmap
|
||||
pix = page.get_pixmap(matrix=mat, alpha=False)
|
||||
|
||||
# Convert to PIL Image for resizing
|
||||
img_data = pix.tobytes("png")
|
||||
pil_image = Image.open(io.BytesIO(img_data))
|
||||
|
||||
# Resize to thumbnail dimensions while maintaining aspect ratio
|
||||
pil_image.thumbnail((self.thumbnail_max_width, self.thumbnail_max_height), Image.Resampling.LANCZOS)
|
||||
|
||||
# Convert to WebP for better compression
|
||||
thumbnail_buffer = io.BytesIO()
|
||||
pil_image.save(thumbnail_buffer, format="WEBP", quality=85, optimize=True)
|
||||
thumbnail_data = thumbnail_buffer.getvalue()
|
||||
|
||||
dimensions = {
|
||||
"width": pil_image.width,
|
||||
"height": pil_image.height
|
||||
}
|
||||
|
||||
pix = None # Free memory
|
||||
pil_image.close()
|
||||
|
||||
return thumbnail_data, dimensions
|
||||
|
||||
def _estimate_storage_size(self, page_images: List[Dict]) -> float:
|
||||
"""Estimate total storage size in MB"""
|
||||
total_bytes = 0
|
||||
|
||||
for page_info in page_images:
|
||||
# Estimate full image size (PNG is roughly 3-4 bytes per pixel)
|
||||
full_dims = page_info["full_dimensions"]
|
||||
full_size = full_dims["width"] * full_dims["height"] * 3.5
|
||||
|
||||
# Estimate thumbnail size (WebP is much more efficient)
|
||||
thumb_dims = page_info["thumbnail_dimensions"]
|
||||
thumb_size = thumb_dims["width"] * thumb_dims["height"] * 0.5
|
||||
|
||||
total_bytes += full_size + thumb_size
|
||||
|
||||
return round(total_bytes / (1024 * 1024), 2) # Convert to MB
|
||||
|
||||
def _get_timestamp(self) -> str:
|
||||
"""Get current timestamp in ISO format"""
|
||||
import datetime
|
||||
return datetime.datetime.utcnow().isoformat() + "Z"
|
||||
|
||||
def generate_single_page_image(self, pdf_bytes: bytes, page_number: int,
|
||||
image_type: str = "full") -> Tuple[bytes, Dict[str, int]]:
|
||||
"""
|
||||
Generate a single page image (for on-demand generation)
|
||||
|
||||
Args:
|
||||
pdf_bytes: PDF file content
|
||||
page_number: Page number (1-based)
|
||||
image_type: "full" or "thumbnail"
|
||||
|
||||
Returns:
|
||||
Tuple of (image_bytes, dimensions)
|
||||
"""
|
||||
doc = fitz.open(stream=pdf_bytes, filetype="pdf")
|
||||
|
||||
if page_number < 1 or page_number > len(doc):
|
||||
doc.close()
|
||||
raise ValueError(f"Page number {page_number} is out of range (1-{len(doc)})")
|
||||
|
||||
page = doc[page_number - 1] # Convert to 0-based index
|
||||
|
||||
if image_type == "thumbnail":
|
||||
image_data, dimensions = self._generate_thumbnail(page, page_number)
|
||||
else:
|
||||
image_data, dimensions = self._generate_full_image(page, page_number)
|
||||
|
||||
doc.close()
|
||||
return image_data, dimensions
|
||||
|
||||
def create_page_images_artefact(file_id: str, cabinet_id: str, pdf_bytes: bytes) -> Dict[str, Any]:
|
||||
"""
|
||||
Create page images artefact for a PDF document
|
||||
|
||||
Args:
|
||||
file_id: File ID
|
||||
cabinet_id: Cabinet ID
|
||||
pdf_bytes: PDF file content
|
||||
|
||||
Returns:
|
||||
Page images artefact data
|
||||
"""
|
||||
generator = PageImageGenerator()
|
||||
return generator.generate_page_images(file_id, cabinet_id, pdf_bytes)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,679 @@
|
||||
"""
|
||||
Robust Document Processing Queue System
|
||||
|
||||
This module provides a Redis-based queuing system for document processing tasks
|
||||
to prevent server overload and handle concurrent processing efficiently.
|
||||
|
||||
Features:
|
||||
- Priority-based queuing (high, normal, low)
|
||||
- Rate limiting per service (Tika, Docling, LLM)
|
||||
- Concurrent processing limits
|
||||
- Task retry mechanism with exponential backoff
|
||||
- Dead letter queue for failed tasks
|
||||
- Health monitoring and metrics
|
||||
- Graceful shutdown handling
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
import asyncio
|
||||
import threading
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Optional, Any, Callable, Union
|
||||
from urllib.parse import urlparse
|
||||
from dataclasses import dataclass, asdict
|
||||
from enum import Enum
|
||||
import redis
|
||||
import os
|
||||
from modules.logger_tool import initialise_logger
|
||||
from .redis_manager import get_redis_manager, Environment
|
||||
|
||||
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
|
||||
# Custom exceptions
|
||||
class QueueConnectionError(Exception):
|
||||
"""Raised when the queue system cannot connect to Redis"""
|
||||
pass
|
||||
|
||||
class QueueFullError(Exception):
|
||||
"""Raised when a service queue is at capacity"""
|
||||
pass
|
||||
|
||||
class RateLimitError(Exception):
|
||||
"""Raised when rate limits are exceeded"""
|
||||
pass
|
||||
|
||||
class TaskPriority(Enum):
|
||||
HIGH = "high" # Interactive uploads, user waiting
|
||||
NORMAL = "normal" # Regular batch processing
|
||||
LOW = "low" # Background/cleanup tasks
|
||||
|
||||
class TaskStatus(Enum):
|
||||
PENDING = "pending"
|
||||
PROCESSING = "processing"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
RETRYING = "retrying"
|
||||
DEAD = "dead"
|
||||
|
||||
class ServiceType(Enum):
|
||||
TIKA = "tika"
|
||||
DOCLING = "docling"
|
||||
LLM = "llm"
|
||||
SPLIT_MAP = "split_map"
|
||||
DOCUMENT_ANALYSIS = "document_analysis"
|
||||
PAGE_IMAGES = "page_images"
|
||||
|
||||
@dataclass
|
||||
class QueueTask:
|
||||
"""Represents a task in the processing queue."""
|
||||
id: str
|
||||
service: ServiceType
|
||||
priority: TaskPriority
|
||||
file_id: str
|
||||
task_type: str # e.g., "tika_metadata", "docling_frontmatter", "llm_classify"
|
||||
payload: Dict[str, Any]
|
||||
created_at: float
|
||||
scheduled_at: float = None # For delayed/retry tasks
|
||||
attempts: int = 0
|
||||
max_attempts: int = 3
|
||||
timeout: int = 300 # seconds
|
||||
callback_url: Optional[str] = None
|
||||
user_id: Optional[str] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.scheduled_at is None:
|
||||
self.scheduled_at = time.time()
|
||||
|
||||
class DocumentProcessingQueue:
|
||||
"""Redis-based document processing queue with rate limiting and priorities."""
|
||||
|
||||
def __init__(self, environment: str = None, redis_url: str = None):
|
||||
"""Initialize queue with Redis connection using new Redis manager.
|
||||
|
||||
Args:
|
||||
environment: 'dev', 'prod', or 'test' (auto-detected if not provided)
|
||||
redis_url: Legacy parameter for backward compatibility
|
||||
"""
|
||||
# Auto-detect environment from startup mode if not provided
|
||||
if not environment:
|
||||
environment = 'dev' if os.getenv('BACKEND_DEV_MODE', 'true').lower() == 'true' else 'prod'
|
||||
|
||||
# Initialize Redis manager for this environment
|
||||
self.redis_manager = get_redis_manager(environment)
|
||||
|
||||
# Initialize Redis environment (ensures service running and connects)
|
||||
if not self.redis_manager.initialize_environment():
|
||||
raise ConnectionError(f"Failed to initialize Redis for {environment} environment")
|
||||
|
||||
# Use the managed Redis client
|
||||
self.redis_client = self.redis_manager.client
|
||||
self.environment = environment
|
||||
|
||||
logger.info(f"🎯 Queue initialized for {environment} environment (db={self.redis_manager.config.db})")
|
||||
|
||||
# Queue configuration
|
||||
self.service_limits = {
|
||||
ServiceType.TIKA: int(os.getenv('QUEUE_TIKA_LIMIT', '3')),
|
||||
ServiceType.DOCLING: int(os.getenv('QUEUE_DOCLING_LIMIT', '2')),
|
||||
ServiceType.LLM: int(os.getenv('QUEUE_LLM_LIMIT', '5')),
|
||||
ServiceType.SPLIT_MAP: int(os.getenv('QUEUE_SPLIT_MAP_LIMIT', '10')),
|
||||
ServiceType.DOCUMENT_ANALYSIS: int(os.getenv('QUEUE_DOCUMENT_ANALYSIS_LIMIT', '5')),
|
||||
ServiceType.PAGE_IMAGES: int(os.getenv('QUEUE_PAGE_IMAGES_LIMIT', '3'))
|
||||
}
|
||||
|
||||
# Rate limiting (requests per minute)
|
||||
self.rate_limits = {
|
||||
ServiceType.TIKA: int(os.getenv('QUEUE_TIKA_RATE', '60')),
|
||||
ServiceType.DOCLING: int(os.getenv('QUEUE_DOCLING_RATE', '30')),
|
||||
ServiceType.LLM: int(os.getenv('QUEUE_LLM_RATE', '120')),
|
||||
ServiceType.SPLIT_MAP: int(os.getenv('QUEUE_SPLIT_MAP_RATE', '100'))
|
||||
}
|
||||
|
||||
# Queue names
|
||||
self.queue_keys = {
|
||||
priority: f"queue:{priority.value}" for priority in TaskPriority
|
||||
}
|
||||
self.processing_key = "processing"
|
||||
self.dead_letter_key = "dead_letter"
|
||||
self.metrics_key = "metrics"
|
||||
|
||||
# Worker control
|
||||
self.workers_running = {}
|
||||
self.shutdown_event = threading.Event()
|
||||
|
||||
logger.info(f"⚙️ Queue service limits: {dict(self.service_limits)}")
|
||||
|
||||
def _get_task_key(self, task_id: str) -> str:
|
||||
"""Get Redis key for task data."""
|
||||
return f"task:{task_id}"
|
||||
|
||||
def _get_service_processing_key(self, service: ServiceType) -> str:
|
||||
"""Get Redis key for service processing counter."""
|
||||
return f"processing:{service.value}"
|
||||
|
||||
def _get_rate_limit_key(self, service: ServiceType) -> str:
|
||||
"""Get Redis key for rate limiting."""
|
||||
return f"rate_limit:{service.value}:{int(time.time() // 60)}"
|
||||
|
||||
def enqueue_task(self,
|
||||
service: ServiceType,
|
||||
task_type: str,
|
||||
file_id: str,
|
||||
payload: Dict[str, Any],
|
||||
priority: TaskPriority = TaskPriority.NORMAL,
|
||||
user_id: str = None,
|
||||
timeout: int = 300,
|
||||
max_attempts: int = 3) -> str:
|
||||
"""
|
||||
Enqueue a new processing task.
|
||||
|
||||
Returns:
|
||||
str: Task ID
|
||||
"""
|
||||
task_id = str(uuid.uuid4())
|
||||
task = QueueTask(
|
||||
id=task_id,
|
||||
service=service,
|
||||
priority=priority,
|
||||
file_id=file_id,
|
||||
task_type=task_type,
|
||||
payload=payload,
|
||||
created_at=time.time(),
|
||||
timeout=timeout,
|
||||
max_attempts=max_attempts,
|
||||
user_id=user_id
|
||||
)
|
||||
|
||||
try:
|
||||
# Store task data (convert enums to strings for Redis)
|
||||
task_dict = asdict(task)
|
||||
task_dict['service'] = task_dict['service'].value
|
||||
task_dict['priority'] = task_dict['priority'].value
|
||||
task_dict['payload'] = json.dumps(task_dict['payload'])
|
||||
# Convert None values to empty strings for Redis
|
||||
for key, value in task_dict.items():
|
||||
if value is None:
|
||||
task_dict[key] = ''
|
||||
|
||||
self.redis_client.hset(
|
||||
self._get_task_key(task_id),
|
||||
mapping=task_dict
|
||||
)
|
||||
self.redis_client.expire(self._get_task_key(task_id), 86400) # 24 hours TTL
|
||||
|
||||
# Add to priority queue
|
||||
queue_key = self.queue_keys[priority]
|
||||
self.redis_client.lpush(queue_key, task_id)
|
||||
|
||||
except redis.ConnectionError as e:
|
||||
logger.error(f"Redis connection failed when enqueueing task: {e}")
|
||||
logger.error("Please ensure Redis is running. Start the API server with './start.sh dev' to auto-start Redis.")
|
||||
raise QueueConnectionError(f"Queue system unavailable: Redis connection failed")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to enqueue task {task_id}: {e}")
|
||||
raise
|
||||
|
||||
# Update metrics
|
||||
self._update_metrics("enqueued", service.value, priority.value)
|
||||
|
||||
logger.info(f"Enqueued task {task_id}: {service.value}/{task_type} for file {file_id}")
|
||||
return task_id
|
||||
|
||||
def dequeue_task(self, timeout: int = 10) -> Optional[QueueTask]:
|
||||
"""
|
||||
Dequeue next available task respecting service limits and rate limits.
|
||||
|
||||
Args:
|
||||
timeout: Blocking timeout in seconds
|
||||
|
||||
Returns:
|
||||
QueueTask or None if no task available
|
||||
"""
|
||||
# Check all priority queues in order
|
||||
for priority in [TaskPriority.HIGH, TaskPriority.NORMAL, TaskPriority.LOW]:
|
||||
queue_key = self.queue_keys[priority]
|
||||
|
||||
# Non-blocking pop to check availability
|
||||
task_id = self.redis_client.rpop(queue_key)
|
||||
if not task_id:
|
||||
continue
|
||||
|
||||
# Get task data
|
||||
task_data = self.redis_client.hgetall(self._get_task_key(task_id))
|
||||
if not task_data:
|
||||
logger.warning(f"Task {task_id} data not found, skipping")
|
||||
continue
|
||||
|
||||
# Reconstruct task object (ignore non-dataclass keys like 'status', 'result', timestamps)
|
||||
task_data['service'] = ServiceType(task_data['service'])
|
||||
task_data['priority'] = TaskPriority(task_data['priority'])
|
||||
task_data['created_at'] = float(task_data['created_at'])
|
||||
task_data['scheduled_at'] = float(task_data['scheduled_at'])
|
||||
task_data['attempts'] = int(task_data['attempts'])
|
||||
task_data['max_attempts'] = int(task_data['max_attempts'])
|
||||
task_data['timeout'] = int(task_data['timeout'])
|
||||
task_data['payload'] = json.loads(task_data['payload'])
|
||||
|
||||
# Drop extraneous keys that may be present in Redis
|
||||
for k in ['status', 'result', 'completed_at', 'failed_at', 'last_error', 'final_error']:
|
||||
task_data.pop(k, None)
|
||||
|
||||
task = QueueTask(**task_data)
|
||||
|
||||
# Check if task is ready (for delayed/retry tasks)
|
||||
if task.scheduled_at > time.time():
|
||||
# Put back in queue for later
|
||||
self.redis_client.lpush(queue_key, task_id)
|
||||
continue
|
||||
|
||||
# Enforce simple dependency ordering using optional depends_on array in payload
|
||||
try:
|
||||
depends_on = []
|
||||
if isinstance(task.payload, dict):
|
||||
depends_on = task.payload.get('depends_on') or []
|
||||
if isinstance(depends_on, list) and len(depends_on) > 0:
|
||||
logger.info(f"Checking {len(depends_on)} dependencies for task {task_id}")
|
||||
unmet = []
|
||||
for dep_id in depends_on:
|
||||
if not dep_id:
|
||||
continue
|
||||
dep_key = self._get_task_key(dep_id)
|
||||
dep_status = self.redis_client.hget(dep_key, 'status') or TaskStatus.PENDING.value
|
||||
if dep_status != TaskStatus.COMPLETED.value:
|
||||
unmet.append((dep_id, dep_status))
|
||||
if len(unmet) > 0:
|
||||
# Reschedule this task a bit later to avoid tight loops
|
||||
next_time = time.time() + 10
|
||||
self.redis_client.hset(
|
||||
self._get_task_key(task_id),
|
||||
mapping={'scheduled_at': next_time}
|
||||
)
|
||||
# Put back for later processing
|
||||
self.redis_client.lpush(queue_key, task_id)
|
||||
logger.info(f"Deferring task {task_id} due to unmet dependencies: {unmet}")
|
||||
continue
|
||||
else:
|
||||
logger.info(f"All {len(depends_on)} dependencies satisfied for task {task_id}")
|
||||
except Exception as dep_e:
|
||||
logger.warning(f"Dependency check failed for task {task_id}: {dep_e}")
|
||||
|
||||
# Check service limits
|
||||
service_processing_key = self._get_service_processing_key(task.service)
|
||||
current_processing = int(self.redis_client.get(service_processing_key) or 0)
|
||||
|
||||
if current_processing >= self.service_limits[task.service]:
|
||||
# Put back in queue with delay to prevent infinite loops
|
||||
logger.warning(f"🚨 SERVICE LIMIT: Task {task_id} re-queued due to service limit exceeded: {current_processing}/{self.service_limits[task.service]} for {task.service.value}")
|
||||
|
||||
# Add delay before re-queueing to prevent tight loops
|
||||
next_time = time.time() + 5 # Wait 5 seconds before retrying
|
||||
self.redis_client.hset(
|
||||
self._get_task_key(task_id),
|
||||
mapping={'scheduled_at': next_time}
|
||||
)
|
||||
self.redis_client.lpush(queue_key, task_id)
|
||||
continue
|
||||
|
||||
# Check rate limits
|
||||
rate_key = self._get_rate_limit_key(task.service)
|
||||
current_rate = int(self.redis_client.get(rate_key) or 0)
|
||||
|
||||
if current_rate >= self.rate_limits[task.service]:
|
||||
# Put back in queue with delay to prevent infinite loops
|
||||
logger.warning(f"🚨 RATE LIMIT: Task {task_id} re-queued due to rate limit exceeded: {current_rate}/{self.rate_limits[task.service]} for {task.service.value}")
|
||||
|
||||
# Add delay before re-queueing to prevent tight loops
|
||||
next_time = time.time() + 60 # Wait 60 seconds for rate limit reset
|
||||
self.redis_client.hset(
|
||||
self._get_task_key(task_id),
|
||||
mapping={'scheduled_at': next_time}
|
||||
)
|
||||
self.redis_client.lpush(queue_key, task_id)
|
||||
continue
|
||||
|
||||
# Task can be processed
|
||||
# Increment processing counter
|
||||
self.redis_client.incr(service_processing_key)
|
||||
self.redis_client.expire(service_processing_key, 3600) # 1 hour
|
||||
|
||||
# Increment rate limit counter
|
||||
self.redis_client.incr(rate_key)
|
||||
self.redis_client.expire(rate_key, 60) # 1 minute
|
||||
|
||||
# Add to processing set
|
||||
self.redis_client.hset(
|
||||
self.processing_key,
|
||||
task_id,
|
||||
json.dumps({
|
||||
'service': task.service.value,
|
||||
'started_at': time.time(),
|
||||
'worker_id': threading.current_thread().ident
|
||||
})
|
||||
)
|
||||
|
||||
# Update metrics
|
||||
self._update_metrics("dequeued", task.service.value, task.priority.value)
|
||||
|
||||
logger.debug(f"Dequeued task {task_id}: {task.service.value}/{task.task_type}")
|
||||
return task
|
||||
|
||||
return None
|
||||
|
||||
def complete_task(self, task: QueueTask, result: Dict[str, Any] = None):
|
||||
"""Mark task as completed and clean up."""
|
||||
# Remove from processing
|
||||
self.redis_client.hdel(self.processing_key, task.id)
|
||||
|
||||
# Decrement processing counter
|
||||
service_processing_key = self._get_service_processing_key(task.service)
|
||||
self.redis_client.decr(service_processing_key)
|
||||
|
||||
# Update task status
|
||||
self.redis_client.hset(
|
||||
self._get_task_key(task.id),
|
||||
mapping={
|
||||
'status': TaskStatus.COMPLETED.value,
|
||||
'completed_at': time.time(),
|
||||
'result': json.dumps(result or {})
|
||||
}
|
||||
)
|
||||
|
||||
# Update metrics
|
||||
self._update_metrics("completed", task.service.value, task.priority.value)
|
||||
|
||||
logger.info(f"Completed task {task.id}: {task.service.value}/{task.task_type}")
|
||||
|
||||
def fail_task(self, task: QueueTask, error: str, retry: bool = True):
|
||||
"""Handle task failure with retry logic."""
|
||||
# Remove from processing
|
||||
self.redis_client.hdel(self.processing_key, task.id)
|
||||
|
||||
# Decrement processing counter
|
||||
service_processing_key = self._get_service_processing_key(task.service)
|
||||
self.redis_client.decr(service_processing_key)
|
||||
|
||||
task.attempts += 1
|
||||
|
||||
if retry and task.attempts < task.max_attempts:
|
||||
# Enhanced retry logic with progress-aware delays for comparison tasks
|
||||
if task.task_type == 'docling_comparison_analysis':
|
||||
# Special handling for comparison analysis tasks
|
||||
payload = getattr(task, 'payload', {})
|
||||
|
||||
# Check if this is a progress-aware retry
|
||||
if hasattr(error, 'is_progress_retry') and error.is_progress_retry:
|
||||
# Active progress - shorter delay (30s to 2 minutes)
|
||||
delay = min(120, 30 + task.attempts * 15)
|
||||
logger.info(f"Comparison task {task.id}: Active progress detected, shorter retry delay: {delay}s")
|
||||
elif hasattr(error, 'is_alignment_retry') and error.is_alignment_retry:
|
||||
# Alignment issues - medium delay (1-3 minutes)
|
||||
delay = min(180, 60 + task.attempts * 20)
|
||||
logger.info(f"Comparison task {task.id}: Alignment retry, medium delay: {delay}s")
|
||||
elif hasattr(error, 'is_stalled_retry') and error.is_stalled_retry:
|
||||
# Stalled processing - longer delay (2-10 minutes)
|
||||
delay = min(600, 120 + task.attempts * 30)
|
||||
logger.info(f"Comparison task {task.id}: Stalled retry, longer delay: {delay}s")
|
||||
else:
|
||||
# Default comparison delay - extended for large PDFs (5-20 minutes)
|
||||
delay = min(1200, 300 + task.attempts * 60)
|
||||
logger.info(f"Comparison task {task.id}: Standard retry, extended delay: {delay}s")
|
||||
|
||||
# Update progress tracking in payload for next attempt
|
||||
if hasattr(error, 'current_progress'):
|
||||
payload['previous_progress'] = error.current_progress
|
||||
# Update the task payload in Redis
|
||||
import json
|
||||
self.redis_client.hset(
|
||||
self._get_task_key(task.id),
|
||||
'payload', json.dumps(payload)
|
||||
)
|
||||
else:
|
||||
# Standard retry with exponential backoff for other tasks
|
||||
delay = min(300, 2 ** task.attempts * 10) # Max 5 minutes
|
||||
|
||||
task.scheduled_at = time.time() + delay
|
||||
task.status = TaskStatus.RETRYING
|
||||
|
||||
# Update task data
|
||||
self.redis_client.hset(
|
||||
self._get_task_key(task.id),
|
||||
mapping={
|
||||
'attempts': task.attempts,
|
||||
'scheduled_at': task.scheduled_at,
|
||||
'status': TaskStatus.RETRYING.value,
|
||||
'last_error': error
|
||||
}
|
||||
)
|
||||
|
||||
# Re-queue for retry
|
||||
queue_key = self.queue_keys[task.priority]
|
||||
self.redis_client.lpush(queue_key, task.id)
|
||||
|
||||
# Update metrics
|
||||
self._update_metrics("retried", task.service.value, task.priority.value)
|
||||
|
||||
logger.warning(f"Retrying task {task.id} in {delay}s (attempt {task.attempts}/{task.max_attempts}): {error}")
|
||||
else:
|
||||
# Move to dead letter queue
|
||||
task.status = TaskStatus.DEAD
|
||||
|
||||
self.redis_client.hset(
|
||||
self._get_task_key(task.id),
|
||||
mapping={
|
||||
'attempts': task.attempts,
|
||||
'status': TaskStatus.DEAD.value,
|
||||
'failed_at': time.time(),
|
||||
'final_error': error
|
||||
}
|
||||
)
|
||||
|
||||
self.redis_client.lpush(self.dead_letter_key, task.id)
|
||||
|
||||
# Update metrics
|
||||
self._update_metrics("failed", task.service.value, task.priority.value)
|
||||
|
||||
logger.error(f"Task {task.id} moved to dead letter queue after {task.attempts} attempts: {error}")
|
||||
|
||||
def get_queue_stats(self) -> Dict[str, Any]:
|
||||
"""Get comprehensive queue statistics."""
|
||||
stats = {
|
||||
'queues': {},
|
||||
'processing': {},
|
||||
'service_limits': dict(self.service_limits),
|
||||
'rate_limits': dict(self.rate_limits),
|
||||
'dead_letter_count': self.redis_client.llen(self.dead_letter_key)
|
||||
}
|
||||
|
||||
# Queue lengths
|
||||
for priority in TaskPriority:
|
||||
queue_key = self.queue_keys[priority]
|
||||
stats['queues'][priority.value] = self.redis_client.llen(queue_key)
|
||||
|
||||
# Processing counts
|
||||
for service in ServiceType:
|
||||
service_key = self._get_service_processing_key(service)
|
||||
stats['processing'][service.value] = int(self.redis_client.get(service_key) or 0)
|
||||
|
||||
# Total processing
|
||||
stats['total_processing'] = self.redis_client.hlen(self.processing_key)
|
||||
|
||||
return stats
|
||||
|
||||
def _update_metrics(self, action: str, service: str, priority: str):
|
||||
"""Update queue metrics."""
|
||||
timestamp = int(time.time())
|
||||
metric_key = f"{self.metrics_key}:{action}:{service}:{priority}:{timestamp // 60}"
|
||||
self.redis_client.incr(metric_key)
|
||||
self.redis_client.expire(metric_key, 3600) # 1 hour
|
||||
|
||||
def start_worker(self, worker_id: str = None, services: List[ServiceType] = None):
|
||||
"""Start a queue worker thread."""
|
||||
if worker_id is None:
|
||||
worker_id = f"worker-{uuid.uuid4().hex[:8]}"
|
||||
|
||||
if services is None:
|
||||
services = list(ServiceType)
|
||||
|
||||
def worker_loop():
|
||||
logger.info(f"Starting worker {worker_id} for services: {[s.value for s in services]}")
|
||||
|
||||
while not self.shutdown_event.is_set():
|
||||
try:
|
||||
task = self.dequeue_task(timeout=5)
|
||||
if task is None:
|
||||
continue
|
||||
|
||||
# DEBUG: Log task details immediately after dequeue
|
||||
logger.info(f"🔍 WORKER DEBUG: Dequeued task {task.id}, service={task.service.value}, task_type={task.task_type}")
|
||||
logger.info(f"🔍 WORKER DEBUG: Worker {worker_id} handles services: {[s.value for s in services]}")
|
||||
|
||||
if task.service not in services:
|
||||
# Put back in queue if worker doesn't handle this service
|
||||
# But first clean up the processing state to avoid infinite loops
|
||||
logger.warning(f"🚨 WORKER DEBUG: Task {task.id} service {task.service.value} NOT in worker services {[s.value for s in services]}")
|
||||
self.redis_client.hdel(self.processing_key, task.id)
|
||||
# NOTE: Do NOT decrement service_processing_key here - task will be re-processed by correct worker
|
||||
# service_processing_key = self._get_service_processing_key(task.service)
|
||||
# self.redis_client.decr(service_processing_key) # REMOVED: Caused negative counters
|
||||
|
||||
queue_key = self.queue_keys[task.priority]
|
||||
self.redis_client.lpush(queue_key, task.id)
|
||||
logger.warning(f"🚨 WORKER DEBUG: Task {task.id} re-queued due to service mismatch")
|
||||
continue
|
||||
|
||||
# DEBUG: Confirm we're about to process
|
||||
logger.info(f"✅ WORKER DEBUG: About to process task {task.id}")
|
||||
|
||||
# Process the task
|
||||
self._process_task(task)
|
||||
|
||||
# DEBUG: Confirm processing completed
|
||||
logger.info(f"✅ WORKER DEBUG: Finished processing task {task.id}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Worker {worker_id} error: {e}")
|
||||
time.sleep(1)
|
||||
|
||||
logger.info(f"Worker {worker_id} shutting down")
|
||||
|
||||
thread = threading.Thread(target=worker_loop, name=f"QueueWorker-{worker_id}")
|
||||
thread.daemon = True
|
||||
thread.start()
|
||||
|
||||
self.workers_running[worker_id] = thread
|
||||
return worker_id
|
||||
|
||||
def _process_task(self, task: QueueTask):
|
||||
"""Process a single task (to be overridden by specific implementations)."""
|
||||
logger.warning(f"No processor implemented for task {task.id}: {task.service.value}/{task.task_type}")
|
||||
self.fail_task(task, "No processor implemented", retry=False)
|
||||
|
||||
def shutdown(self, timeout: int = 30):
|
||||
"""Gracefully shutdown all workers."""
|
||||
logger.info("Shutting down queue workers...")
|
||||
self.shutdown_event.set()
|
||||
|
||||
# Wait for workers to finish
|
||||
for worker_id, thread in self.workers_running.items():
|
||||
thread.join(timeout=timeout)
|
||||
if thread.is_alive():
|
||||
logger.warning(f"Worker {worker_id} did not shut down gracefully")
|
||||
|
||||
self.workers_running.clear()
|
||||
logger.info("Queue shutdown complete")
|
||||
|
||||
# Global queue instance
|
||||
_queue_instance = None
|
||||
|
||||
def get_queue() -> DocumentProcessingQueue:
|
||||
"""Get the global queue instance."""
|
||||
global _queue_instance
|
||||
if _queue_instance is None:
|
||||
_queue_instance = DocumentProcessingQueue()
|
||||
return _queue_instance
|
||||
|
||||
# Convenience functions for common operations
|
||||
def enqueue_tika_task(file_id: str, payload: Dict[str, Any], priority: TaskPriority = TaskPriority.NORMAL) -> str:
|
||||
"""Enqueue a Tika processing task."""
|
||||
return get_queue().enqueue_task(
|
||||
service=ServiceType.TIKA,
|
||||
task_type="metadata_extraction",
|
||||
file_id=file_id,
|
||||
payload=payload,
|
||||
priority=priority,
|
||||
timeout=int(os.getenv('TIKA_TIMEOUT', '300'))
|
||||
)
|
||||
|
||||
def enqueue_docling_task(file_id: str, task_type: str, payload: Dict[str, Any],
|
||||
priority: TaskPriority = TaskPriority.NORMAL, timeout: int = 1800,
|
||||
max_attempts: int = None) -> str:
|
||||
"""Enqueue a Docling processing task with intelligent retry limits."""
|
||||
|
||||
# Auto-configure max_attempts based on task type and payload
|
||||
if max_attempts is None:
|
||||
if task_type == 'docling_comparison_analysis':
|
||||
# Use the max_retry_attempts from payload, or default to high limit for comparisons
|
||||
max_attempts = payload.get('max_retry_attempts', 50)
|
||||
else:
|
||||
# Standard retry limit for other docling tasks
|
||||
max_attempts = 3
|
||||
|
||||
return get_queue().enqueue_task(
|
||||
service=ServiceType.DOCLING,
|
||||
task_type=task_type,
|
||||
file_id=file_id,
|
||||
payload=payload,
|
||||
priority=priority,
|
||||
timeout=timeout,
|
||||
max_attempts=max_attempts
|
||||
)
|
||||
|
||||
def enqueue_llm_task(file_id: str, task_type: str, payload: Dict[str, Any],
|
||||
priority: TaskPriority = TaskPriority.NORMAL) -> str:
|
||||
"""Enqueue an LLM processing task."""
|
||||
return get_queue().enqueue_task(
|
||||
service=ServiceType.LLM,
|
||||
task_type=task_type,
|
||||
file_id=file_id,
|
||||
payload=payload,
|
||||
priority=priority,
|
||||
timeout=int(os.getenv('LLM_TIMEOUT', '180'))
|
||||
)
|
||||
|
||||
def enqueue_split_map_task(file_id: str, payload: Dict[str, Any],
|
||||
priority: TaskPriority = TaskPriority.NORMAL) -> str:
|
||||
"""Enqueue a split map generation task."""
|
||||
return get_queue().enqueue_task(
|
||||
service=ServiceType.SPLIT_MAP,
|
||||
task_type="generate_split_map",
|
||||
file_id=file_id,
|
||||
payload=payload,
|
||||
priority=priority,
|
||||
timeout=120
|
||||
)
|
||||
|
||||
def enqueue_document_analysis_task(file_id: str, payload: Dict[str, Any],
|
||||
priority: TaskPriority = TaskPriority.NORMAL) -> str:
|
||||
"""Enqueue a document structure analysis task."""
|
||||
return get_queue().enqueue_task(
|
||||
service=ServiceType.DOCUMENT_ANALYSIS,
|
||||
task_type="document_structure_analysis",
|
||||
file_id=file_id,
|
||||
payload=payload,
|
||||
priority=priority,
|
||||
timeout=int(os.getenv('DOCUMENT_ANALYSIS_TIMEOUT', '300'))
|
||||
)
|
||||
|
||||
def enqueue_page_images_task(file_id: str, payload: Dict[str, Any],
|
||||
priority: TaskPriority = TaskPriority.NORMAL) -> str:
|
||||
"""Enqueue a page images generation task."""
|
||||
return get_queue().enqueue_task(
|
||||
service=ServiceType.PAGE_IMAGES,
|
||||
task_type="generate_page_images",
|
||||
file_id=file_id,
|
||||
payload=payload,
|
||||
priority=priority,
|
||||
timeout=int(os.getenv('PAGE_IMAGES_TIMEOUT', '600'))
|
||||
)
|
||||
@@ -0,0 +1,523 @@
|
||||
"""
|
||||
Comprehensive Redis Management System
|
||||
=====================================
|
||||
|
||||
Handles environment-specific Redis configuration, service management,
|
||||
task recovery, and health monitoring for ClassroomCopilot.
|
||||
|
||||
Features:
|
||||
- Environment isolation (dev/prod/test databases)
|
||||
- Automatic service management and health checks
|
||||
- Task recovery and persistence strategies
|
||||
- Graceful degradation and error handling
|
||||
"""
|
||||
|
||||
import os
|
||||
import redis
|
||||
import subprocess
|
||||
import time
|
||||
import signal
|
||||
import json
|
||||
import logging
|
||||
from typing import Dict, List, Optional, Any, Tuple
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class Environment(Enum):
|
||||
DEV = "dev"
|
||||
PROD = "prod"
|
||||
TEST = "test"
|
||||
|
||||
@dataclass
|
||||
class RedisConfig:
|
||||
host: str
|
||||
port: int
|
||||
db: int
|
||||
password: Optional[str]
|
||||
ssl: bool
|
||||
persist: bool
|
||||
task_ttl: int
|
||||
url: str
|
||||
|
||||
class RedisManager:
|
||||
"""Comprehensive Redis management with environment isolation and recovery."""
|
||||
|
||||
def __init__(self, environment: Environment = Environment.DEV):
|
||||
self.environment = environment
|
||||
self.config = self._load_config()
|
||||
self.client: Optional[redis.Redis] = None
|
||||
self._subprocess: Optional[subprocess.Popen] = None
|
||||
self._health_check_enabled = True
|
||||
|
||||
logger.info(f"🔧 Redis Manager initialized for {environment.value} environment")
|
||||
logger.info(f"📡 Target: {self.config.host}:{self.config.port}/db{self.config.db}")
|
||||
|
||||
def _load_config(self) -> RedisConfig:
|
||||
"""Load environment-specific Redis configuration."""
|
||||
|
||||
# Base configuration
|
||||
host = os.getenv('REDIS_HOST', 'localhost')
|
||||
port = int(os.getenv('REDIS_PORT', '6379'))
|
||||
password = os.getenv('REDIS_PASSWORD') or None
|
||||
ssl = os.getenv('REDIS_SSL', 'false').lower() == 'true'
|
||||
|
||||
# Environment-specific settings
|
||||
if self.environment == Environment.DEV:
|
||||
db = int(os.getenv('REDIS_DB_DEV', '0'))
|
||||
persist = os.getenv('REDIS_PERSIST_DEV', 'false').lower() == 'true'
|
||||
task_ttl = int(os.getenv('REDIS_TASK_TTL_DEV', '3600'))
|
||||
elif self.environment == Environment.PROD:
|
||||
db = int(os.getenv('REDIS_DB_PROD', '1'))
|
||||
persist = os.getenv('REDIS_PERSIST_PROD', 'true').lower() == 'true'
|
||||
task_ttl = int(os.getenv('REDIS_TASK_TTL_PROD', '86400'))
|
||||
else: # TEST
|
||||
db = int(os.getenv('REDIS_DB_TEST', '2'))
|
||||
persist = False
|
||||
task_ttl = int(os.getenv('REDIS_TASK_TTL_TEST', '1800'))
|
||||
|
||||
# Construct URL
|
||||
auth_part = f":{password}@" if password else ""
|
||||
url = f"redis://{auth_part}{host}:{port}/{db}"
|
||||
|
||||
return RedisConfig(
|
||||
host=host,
|
||||
port=port,
|
||||
db=db,
|
||||
password=password,
|
||||
ssl=ssl,
|
||||
persist=persist,
|
||||
task_ttl=task_ttl,
|
||||
url=url
|
||||
)
|
||||
|
||||
def ensure_service_running(self) -> bool:
|
||||
"""Ensure Redis service is running, start if needed."""
|
||||
|
||||
# Check if Redis is already running
|
||||
if self._is_redis_running():
|
||||
logger.info("✅ Redis service already running")
|
||||
return True
|
||||
|
||||
# Try to start Redis service
|
||||
logger.info("🚀 Starting Redis service...")
|
||||
|
||||
# Try systemctl first (Linux production)
|
||||
if self._try_systemctl_start():
|
||||
return True
|
||||
|
||||
# Try brew services (macOS)
|
||||
if self._try_brew_start():
|
||||
return True
|
||||
|
||||
# Try direct Redis server start
|
||||
if self._try_direct_start():
|
||||
return True
|
||||
|
||||
# Try Docker fallback
|
||||
if self._try_docker_start():
|
||||
return True
|
||||
|
||||
logger.error("❌ Failed to start Redis service with all methods")
|
||||
return False
|
||||
|
||||
def _is_redis_running(self) -> bool:
|
||||
"""Check if Redis is accessible."""
|
||||
try:
|
||||
test_client = redis.Redis(
|
||||
host=self.config.host,
|
||||
port=self.config.port,
|
||||
socket_connect_timeout=2,
|
||||
socket_timeout=2
|
||||
)
|
||||
test_client.ping()
|
||||
test_client.close()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _try_systemctl_start(self) -> bool:
|
||||
"""Try starting Redis with systemctl."""
|
||||
try:
|
||||
if not os.path.exists('/usr/bin/systemctl'):
|
||||
return False
|
||||
|
||||
subprocess.run(['sudo', 'systemctl', 'start', 'redis'],
|
||||
check=True, capture_output=True, timeout=10)
|
||||
time.sleep(2)
|
||||
return self._is_redis_running()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _try_brew_start(self) -> bool:
|
||||
"""Try starting Redis with brew services."""
|
||||
try:
|
||||
if not os.path.exists('/opt/homebrew/bin/brew'):
|
||||
return False
|
||||
|
||||
subprocess.run(['/opt/homebrew/bin/brew', 'services', 'start', 'redis'],
|
||||
check=True, capture_output=True, timeout=10)
|
||||
time.sleep(2)
|
||||
return self._is_redis_running()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _try_direct_start(self) -> bool:
|
||||
"""Try starting Redis server directly."""
|
||||
try:
|
||||
# Find redis-server binary
|
||||
redis_cmd = None
|
||||
for path in ['/opt/homebrew/bin/redis-server', '/usr/local/bin/redis-server', 'redis-server']:
|
||||
try:
|
||||
subprocess.run([path, '--version'], capture_output=True, check=True, timeout=5)
|
||||
redis_cmd = path
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not redis_cmd:
|
||||
return False
|
||||
|
||||
# Start Redis with appropriate config
|
||||
config_args = [
|
||||
redis_cmd,
|
||||
'--port', str(self.config.port),
|
||||
'--bind', self.config.host,
|
||||
'--protected-mode', 'no',
|
||||
'--loglevel', 'notice',
|
||||
'--daemonize', 'yes' # Run as daemon
|
||||
]
|
||||
|
||||
# Add persistence settings
|
||||
if not self.config.persist:
|
||||
config_args.extend(['--save', '', '--appendonly', 'no'])
|
||||
else:
|
||||
config_args.extend(['--save', '60 1000', '--appendonly', 'yes'])
|
||||
|
||||
subprocess.run(config_args, check=True, capture_output=True, timeout=10)
|
||||
time.sleep(3)
|
||||
return self._is_redis_running()
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Direct Redis start failed: {e}")
|
||||
return False
|
||||
|
||||
def _try_docker_start(self) -> bool:
|
||||
"""Try starting Redis with Docker."""
|
||||
try:
|
||||
subprocess.run(['docker', '--version'], capture_output=True, check=True, timeout=5)
|
||||
|
||||
# Check if Redis container already exists
|
||||
result = subprocess.run(
|
||||
['docker', 'ps', '-a', '--filter', 'name=classroomcopilot-redis', '--format', '{{.Names}}'],
|
||||
capture_output=True, text=True, timeout=10
|
||||
)
|
||||
|
||||
if 'classroomcopilot-redis' in result.stdout:
|
||||
# Start existing container
|
||||
subprocess.run(['docker', 'start', 'classroomcopilot-redis'],
|
||||
check=True, capture_output=True, timeout=10)
|
||||
else:
|
||||
# Create new container
|
||||
docker_cmd = [
|
||||
'docker', 'run', '-d',
|
||||
'--name', 'classroomcopilot-redis',
|
||||
'-p', f'{self.config.port}:6379',
|
||||
'redis:alpine'
|
||||
]
|
||||
|
||||
if not self.config.persist:
|
||||
docker_cmd.extend(['redis-server', '--save', '', '--appendonly', 'no'])
|
||||
|
||||
subprocess.run(docker_cmd, check=True, capture_output=True, timeout=30)
|
||||
|
||||
time.sleep(3)
|
||||
return self._is_redis_running()
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"Docker Redis start failed: {e}")
|
||||
return False
|
||||
|
||||
def connect(self) -> bool:
|
||||
"""Establish Redis connection with retry logic."""
|
||||
max_attempts = int(os.getenv('REDIS_MAX_RETRY_ATTEMPTS', '3'))
|
||||
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
logger.info(f"🔌 Connecting to Redis... (attempt {attempt}/{max_attempts})")
|
||||
|
||||
self.client = redis.Redis(
|
||||
host=self.config.host,
|
||||
port=self.config.port,
|
||||
db=self.config.db,
|
||||
password=self.config.password,
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=5,
|
||||
socket_timeout=5,
|
||||
retry_on_timeout=True
|
||||
)
|
||||
|
||||
# Test connection
|
||||
self.client.ping()
|
||||
|
||||
logger.info(f"✅ Connected to Redis {self.config.host}:{self.config.port}/db{self.config.db}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"❌ Connection attempt {attempt} failed: {e}")
|
||||
if attempt < max_attempts:
|
||||
logger.info("⏳ Retrying in 2 seconds...")
|
||||
time.sleep(2)
|
||||
else:
|
||||
logger.error("💥 All connection attempts failed")
|
||||
return False
|
||||
|
||||
return False
|
||||
|
||||
def initialize_environment(self) -> bool:
|
||||
"""Initialize Redis environment based on mode."""
|
||||
|
||||
if not self.ensure_service_running():
|
||||
logger.error("Cannot initialize - Redis service not available")
|
||||
return False
|
||||
|
||||
if not self.connect():
|
||||
logger.error("Cannot initialize - Connection failed")
|
||||
return False
|
||||
|
||||
if self.environment == Environment.DEV:
|
||||
return self._initialize_dev_environment()
|
||||
elif self.environment == Environment.PROD:
|
||||
return self._initialize_prod_environment()
|
||||
else:
|
||||
return self._initialize_test_environment()
|
||||
|
||||
def _initialize_dev_environment(self) -> bool:
|
||||
"""Initialize development environment - clean slate."""
|
||||
try:
|
||||
logger.info("🧹 DEV MODE: Clearing all data for clean startup...")
|
||||
|
||||
# Get all keys in this database
|
||||
all_keys = self.client.keys('*')
|
||||
|
||||
if all_keys:
|
||||
# Nuclear option - clear everything in this DB
|
||||
self.client.flushdb()
|
||||
logger.info(f"💥 DEV MODE: Nuked {len(all_keys)} keys for clean startup")
|
||||
else:
|
||||
logger.info("✅ DEV MODE: Database already clean")
|
||||
|
||||
# Set up development-specific config
|
||||
try:
|
||||
self.client.config_set('save', '') # Disable RDB snapshots
|
||||
self.client.config_set('appendonly', 'no') # Disable AOF
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not set Redis config (may not have permissions): {e}")
|
||||
|
||||
logger.info("🎯 DEV MODE: Environment initialized for fast iteration")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize dev environment: {e}")
|
||||
return False
|
||||
|
||||
def _initialize_prod_environment(self) -> bool:
|
||||
"""Initialize production environment - preserve data."""
|
||||
try:
|
||||
logger.info("🏭 PROD MODE: Initializing with data preservation...")
|
||||
|
||||
# Check for existing tasks and report
|
||||
task_keys = self.client.keys('task:*')
|
||||
queue_keys = self.client.keys('queue:*')
|
||||
processing_keys = self.client.keys('processing:*')
|
||||
|
||||
logger.info(f"📊 PROD MODE: Found {len(task_keys)} tasks, {len(queue_keys)} queues, {len(processing_keys)} processing counters")
|
||||
|
||||
# Enable persistence
|
||||
if self.config.persist:
|
||||
try:
|
||||
self.client.config_set('save', '60 1000') # Save every 60s if ≥1000 changes
|
||||
self.client.config_set('appendonly', 'yes') # Enable AOF
|
||||
logger.info("💾 PROD MODE: Persistence enabled")
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not set Redis config (may not have permissions): {e}")
|
||||
|
||||
# Recover any stuck tasks
|
||||
self._recover_stuck_tasks()
|
||||
|
||||
logger.info("✅ PROD MODE: Environment initialized with data recovery")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize prod environment: {e}")
|
||||
return False
|
||||
|
||||
def _initialize_test_environment(self) -> bool:
|
||||
"""Initialize test environment - isolated and clean."""
|
||||
try:
|
||||
logger.info("🧪 TEST MODE: Setting up isolated test environment...")
|
||||
|
||||
# Clear test database
|
||||
self.client.flushdb()
|
||||
|
||||
# Disable persistence for speed
|
||||
try:
|
||||
self.client.config_set('save', '')
|
||||
self.client.config_set('appendonly', 'no')
|
||||
except Exception as e:
|
||||
logger.debug(f"Could not set Redis config (may not have permissions): {e}")
|
||||
|
||||
logger.info("✅ TEST MODE: Clean, isolated environment ready")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize test environment: {e}")
|
||||
return False
|
||||
|
||||
def _recover_stuck_tasks(self):
|
||||
"""Recover tasks that were processing when system shut down."""
|
||||
try:
|
||||
logger.info("🔄 Recovering stuck tasks from previous session...")
|
||||
|
||||
# Get all processing tasks
|
||||
processing_data = self.client.hgetall('processing')
|
||||
|
||||
if not processing_data:
|
||||
logger.info("✅ No stuck tasks to recover")
|
||||
return
|
||||
|
||||
recovered_count = 0
|
||||
|
||||
for task_id, task_info in processing_data.items():
|
||||
try:
|
||||
# Parse task info
|
||||
info = json.loads(task_info)
|
||||
|
||||
# Check if task still exists
|
||||
task_key = f"task:{task_id}"
|
||||
if not self.client.exists(task_key):
|
||||
logger.warning(f"⚠️ Task {task_id} data missing, removing from processing")
|
||||
self.client.hdel('processing', task_id)
|
||||
continue
|
||||
|
||||
# Reset task to pending and re-queue
|
||||
self.client.hset(task_key, 'status', 'pending')
|
||||
self.client.hdel(task_key, 'started_at')
|
||||
|
||||
# Add back to appropriate queue
|
||||
priority = info.get('priority', 'normal')
|
||||
queue_key = f"queue:{priority}"
|
||||
self.client.lpush(queue_key, task_id)
|
||||
|
||||
# Remove from processing
|
||||
self.client.hdel('processing', task_id)
|
||||
|
||||
# Reset service counters
|
||||
service = info.get('service')
|
||||
if service:
|
||||
service_key = f"processing:{service}"
|
||||
current_count = int(self.client.get(service_key) or 0)
|
||||
if current_count > 0:
|
||||
self.client.decr(service_key)
|
||||
|
||||
recovered_count += 1
|
||||
logger.info(f"🔄 Recovered stuck task {task_id} ({service}/{priority})")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to recover task {task_id}: {e}")
|
||||
# Remove problematic entry
|
||||
self.client.hdel('processing', task_id)
|
||||
|
||||
logger.info(f"✅ Recovered {recovered_count} stuck tasks")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Task recovery failed: {e}")
|
||||
|
||||
def health_check(self) -> Dict[str, Any]:
|
||||
"""Comprehensive health check."""
|
||||
health = {
|
||||
'status': 'healthy',
|
||||
'environment': self.environment.value,
|
||||
'database': self.config.db,
|
||||
'connection': False,
|
||||
'memory_usage': None,
|
||||
'queue_stats': {},
|
||||
'error': None
|
||||
}
|
||||
|
||||
try:
|
||||
if not self.client:
|
||||
raise Exception("No Redis connection")
|
||||
|
||||
# Test connection
|
||||
self.client.ping()
|
||||
health['connection'] = True
|
||||
|
||||
# Get memory usage
|
||||
info = self.client.info('memory')
|
||||
health['memory_usage'] = {
|
||||
'used_memory_human': info.get('used_memory_human', 'unknown'),
|
||||
'used_memory_peak_human': info.get('used_memory_peak_human', 'unknown')
|
||||
}
|
||||
|
||||
# Get queue statistics
|
||||
health['queue_stats'] = {
|
||||
'total_keys': len(self.client.keys('*')),
|
||||
'tasks': len(self.client.keys('task:*')),
|
||||
'queues': {
|
||||
'high': self.client.llen('queue:high'),
|
||||
'normal': self.client.llen('queue:normal'),
|
||||
'low': self.client.llen('queue:low')
|
||||
},
|
||||
'processing': self.client.hlen('processing'),
|
||||
'dead_letter': self.client.llen('dead_letter')
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
health['status'] = 'unhealthy'
|
||||
health['error'] = str(e)
|
||||
|
||||
return health
|
||||
|
||||
def shutdown(self, force: bool = False):
|
||||
"""Graceful shutdown with optional data preservation."""
|
||||
|
||||
if self.environment == Environment.DEV and not force:
|
||||
logger.info("🧹 DEV MODE: Clearing data on shutdown...")
|
||||
try:
|
||||
if self.client:
|
||||
self.client.flushdb()
|
||||
logger.info("✅ Dev data cleared")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to clear dev data: {e}")
|
||||
|
||||
# Close connection
|
||||
if self.client:
|
||||
try:
|
||||
self.client.close()
|
||||
logger.info("🔌 Redis connection closed")
|
||||
except Exception as e:
|
||||
logger.warning(f"Error closing Redis connection: {e}")
|
||||
|
||||
# Stop subprocess if we started it
|
||||
if self._subprocess:
|
||||
try:
|
||||
self._subprocess.terminate()
|
||||
self._subprocess.wait(timeout=5)
|
||||
logger.info("🛑 Redis subprocess stopped")
|
||||
except Exception as e:
|
||||
logger.warning(f"Error stopping Redis subprocess: {e}")
|
||||
|
||||
logger.info(f"✅ Redis manager shutdown complete ({self.environment.value})")
|
||||
|
||||
# Convenience functions for backward compatibility
|
||||
def get_redis_manager(environment: str = "dev") -> RedisManager:
|
||||
"""Get a Redis manager instance for the specified environment."""
|
||||
env = Environment(environment.lower())
|
||||
return RedisManager(env)
|
||||
|
||||
def ensure_redis_running(environment: str = "dev") -> bool:
|
||||
"""Ensure Redis is running for the specified environment."""
|
||||
manager = get_redis_manager(environment)
|
||||
return manager.ensure_service_running()
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user