latest
This commit is contained in:
@@ -1,20 +1,22 @@
|
||||
import os
|
||||
import argparse
|
||||
import sys
|
||||
import subprocess
|
||||
import signal
|
||||
import atexit
|
||||
import time
|
||||
from modules.logger_tool import initialise_logger
|
||||
logger = initialise_logger(__name__, os.getenv("LOG_LEVEL"), os.getenv("LOG_PATH"), 'default', True)
|
||||
from fastapi import FastAPI, HTTPException
|
||||
import uvicorn
|
||||
import requests
|
||||
from typing import Dict, Any
|
||||
from typing import Dict, Any, Optional
|
||||
from modules.database.tools.neo4j_driver_tools import get_driver
|
||||
from run.initialization.initialization import InitializationSystem
|
||||
import time
|
||||
import ssl
|
||||
|
||||
from run.setup import setup_cors
|
||||
from run.routers import register_routes
|
||||
from run.initialization import initialize_system
|
||||
from modules.task_processors import get_processor
|
||||
from modules.queue_system import ServiceType
|
||||
|
||||
# FastAPI App Setup
|
||||
app = FastAPI()
|
||||
@@ -28,7 +30,8 @@ async def health_check() -> Dict[str, Any]:
|
||||
"status": "healthy",
|
||||
"services": {
|
||||
"neo4j": {"status": "healthy", "message": "Connected"},
|
||||
"supabase": {"status": "healthy", "message": "Connected"}
|
||||
"supabase": {"status": "healthy", "message": "Connected"},
|
||||
"redis": {"status": "healthy", "message": "Connected"}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -70,6 +73,35 @@ async def health_check() -> Dict[str, Any]:
|
||||
}
|
||||
health_status["status"] = "unhealthy"
|
||||
|
||||
try:
|
||||
# Check Redis using new Redis manager
|
||||
from modules.redis_manager import get_redis_manager
|
||||
|
||||
# Determine environment
|
||||
environment = 'dev' if os.getenv('BACKEND_DEV_MODE', 'true').lower() == 'true' else 'prod'
|
||||
redis_manager = get_redis_manager(environment)
|
||||
|
||||
# Get comprehensive health check
|
||||
redis_health = redis_manager.health_check()
|
||||
|
||||
health_status["services"]["redis"] = {
|
||||
"status": redis_health['status'],
|
||||
"message": redis_health.get('error', f"Connected to {environment} environment (db={redis_health['database']})"),
|
||||
"environment": redis_health['environment'],
|
||||
"database": redis_health['database'],
|
||||
"queue_stats": redis_health.get('queue_stats', {})
|
||||
}
|
||||
|
||||
if redis_health['status'] != 'healthy':
|
||||
health_status["status"] = "unhealthy"
|
||||
|
||||
except Exception as e:
|
||||
health_status["services"]["redis"] = {
|
||||
"status": "unhealthy",
|
||||
"message": f"Error checking Redis: {str(e)}"
|
||||
}
|
||||
health_status["status"] = "unhealthy"
|
||||
|
||||
if health_status["status"] == "unhealthy":
|
||||
raise HTTPException(status_code=503, detail=health_status)
|
||||
|
||||
@@ -78,71 +110,253 @@ async def health_check() -> Dict[str, Any]:
|
||||
# Register routes
|
||||
register_routes(app)
|
||||
|
||||
# Initialize system with retry logic
|
||||
def initialize_with_retry(max_attempts: int = 3, initial_delay: int = 5) -> bool:
|
||||
"""Initialize the system with retry logic"""
|
||||
attempt = 0
|
||||
delay = initial_delay
|
||||
|
||||
while attempt < max_attempts:
|
||||
try:
|
||||
logger.info(f"Attempting system initialization (attempt {attempt + 1}/{max_attempts})")
|
||||
initialize_system()
|
||||
logger.info("System initialization completed successfully")
|
||||
return True
|
||||
except Exception as e:
|
||||
attempt += 1
|
||||
if attempt == max_attempts:
|
||||
logger.error(f"System initialization failed after {max_attempts} attempts: {str(e)}")
|
||||
return False
|
||||
|
||||
logger.warning(f"Initialization attempt {attempt} failed: {str(e)}. Retrying in {delay} seconds...")
|
||||
time.sleep(delay)
|
||||
delay *= 2 # Exponential backoff
|
||||
|
||||
return False
|
||||
# Start workers in the application process to avoid uvicorn reload issues
|
||||
@app.on_event("startup")
|
||||
async def _start_workers_event():
|
||||
try:
|
||||
if os.getenv('AUTO_START_QUEUE_WORKERS', 'true').lower() != 'true':
|
||||
logger.info("AUTO_START_QUEUE_WORKERS=false, not starting in-process workers")
|
||||
return
|
||||
workers = int(os.getenv('QUEUE_WORKERS', '3'))
|
||||
services_csv = os.getenv('QUEUE_SERVICES', 'tika,docling,split_map,document_analysis,page_images')
|
||||
service_names = [s.strip().lower() for s in services_csv.split(',') if s.strip()]
|
||||
service_enums = []
|
||||
for name in service_names:
|
||||
try:
|
||||
service_enums.append(ServiceType(name))
|
||||
except Exception:
|
||||
pass
|
||||
if not service_enums:
|
||||
service_enums = list(ServiceType)
|
||||
processor = get_processor()
|
||||
started = []
|
||||
for i in range(workers):
|
||||
wid = processor.start_worker(worker_id=f"app-worker-{i+1}", services=service_enums)
|
||||
started.append(wid)
|
||||
logger.info(f"In-process queue workers started: {started} for services {[s.value for s in service_enums]}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start in-process workers: {e}")
|
||||
|
||||
def run_initialization_mode():
|
||||
"""Run only the initialization process"""
|
||||
logger.info("Running in initialization mode")
|
||||
logger.info("Starting system initialization...")
|
||||
@app.on_event("shutdown")
|
||||
async def _shutdown_workers_event():
|
||||
try:
|
||||
processor = get_processor()
|
||||
processor.shutdown(timeout=30)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error during workers shutdown: {e}")
|
||||
|
||||
# Global subprocess handles (only for workers now)
|
||||
workers_process: Optional[subprocess.Popen] = None
|
||||
|
||||
# Global Redis manager for cleanup
|
||||
redis_manager = None
|
||||
|
||||
def start_queue_workers():
|
||||
"""Start queue workers as a subprocess (tied to API lifecycle)."""
|
||||
global workers_process
|
||||
if os.getenv('AUTO_START_QUEUE_WORKERS', 'true').lower() != 'true':
|
||||
logger.info("AUTO_START_QUEUE_WORKERS=false, not starting workers")
|
||||
return
|
||||
|
||||
# If already started, skip
|
||||
if workers_process is not None and workers_process.poll() is None:
|
||||
logger.info("Queue workers already running")
|
||||
return
|
||||
|
||||
services = os.getenv(
|
||||
'QUEUE_SERVICES',
|
||||
'tika,docling,split_map,document_analysis,page_images'
|
||||
)
|
||||
workers = int(os.getenv('QUEUE_WORKERS', '3'))
|
||||
check_interval = os.getenv('QUEUE_CHECK_INTERVAL', '15')
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
'start_queue_workers.py',
|
||||
'--workers', str(workers),
|
||||
'--services', services,
|
||||
'--check-interval', check_interval,
|
||||
]
|
||||
# Workers will auto-detect environment and use appropriate Redis database
|
||||
|
||||
log_path = os.getenv('QUEUE_WORKERS_LOG', './queue_workers.log')
|
||||
try:
|
||||
log_file = open(log_path, 'a')
|
||||
logger.info(f"Starting queue workers ({workers}) for services [{services}] → {log_path}")
|
||||
workers_process = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=log_file,
|
||||
stderr=log_file,
|
||||
preexec_fn=os.setsid if os.name != 'nt' else None
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to start queue workers: {e}")
|
||||
|
||||
def stop_queue_workers():
|
||||
"""Stop queue workers subprocess."""
|
||||
global workers_process
|
||||
if workers_process is not None:
|
||||
try:
|
||||
logger.info("Stopping queue workers...")
|
||||
if os.name != 'nt':
|
||||
os.killpg(os.getpgid(workers_process.pid), signal.SIGTERM)
|
||||
else:
|
||||
workers_process.terminate()
|
||||
try:
|
||||
workers_process.wait(timeout=10)
|
||||
logger.info("Queue workers stopped gracefully")
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("Queue workers did not stop gracefully, forcing shutdown...")
|
||||
if os.name != 'nt':
|
||||
os.killpg(os.getpgid(workers_process.pid), signal.SIGKILL)
|
||||
else:
|
||||
workers_process.kill()
|
||||
workers_process.wait()
|
||||
logger.info("Queue workers force stopped")
|
||||
except Exception as e:
|
||||
logger.error(f"Error stopping queue workers: {e}")
|
||||
finally:
|
||||
workers_process = None
|
||||
|
||||
def _install_signal_handlers():
|
||||
def signal_handler(signum, frame):
|
||||
logger.info(f"Received signal {signum}, shutting down...")
|
||||
stop_queue_workers()
|
||||
# Gracefully shutdown Redis manager if it exists
|
||||
global redis_manager
|
||||
if redis_manager:
|
||||
redis_manager.shutdown()
|
||||
sys.exit(0)
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
|
||||
def run_infrastructure_mode():
|
||||
"""Run infrastructure setup: Neo4j schema, calendar, and Supabase buckets"""
|
||||
logger.info("Running in infrastructure mode")
|
||||
logger.info("Starting infrastructure setup...")
|
||||
|
||||
if initialize_with_retry():
|
||||
logger.info("Initialization completed successfully")
|
||||
try:
|
||||
from run.initialization import initialize_infrastructure_mode
|
||||
initialize_infrastructure_mode()
|
||||
logger.info("Infrastructure setup completed successfully")
|
||||
return True
|
||||
else:
|
||||
logger.error("Initialization failed after multiple attempts")
|
||||
except Exception as e:
|
||||
logger.error(f"Infrastructure setup failed: {str(e)}")
|
||||
return False
|
||||
|
||||
def run_demo_school_mode():
|
||||
"""Run demo school creation"""
|
||||
logger.info("Running in demo school mode")
|
||||
logger.info("Starting demo school creation...")
|
||||
|
||||
try:
|
||||
from run.initialization import initialize_demo_school_mode
|
||||
initialize_demo_school_mode()
|
||||
logger.info("Demo school creation completed successfully")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Demo school creation failed: {str(e)}")
|
||||
return False
|
||||
|
||||
def run_demo_users_mode():
|
||||
"""Run demo users creation"""
|
||||
logger.info("Running in demo users mode")
|
||||
logger.info("Starting demo users creation...")
|
||||
|
||||
try:
|
||||
from run.initialization import initialize_demo_users_mode
|
||||
initialize_demo_users_mode()
|
||||
logger.info("Demo users creation completed successfully")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Demo users creation failed: {str(e)}")
|
||||
return False
|
||||
|
||||
def run_gais_data_mode():
|
||||
"""Run GAIS data import"""
|
||||
logger.info("Running in GAIS data import mode")
|
||||
logger.info("Starting GAIS data import...")
|
||||
|
||||
try:
|
||||
from run.initialization import initialize_gais_data_mode
|
||||
initialize_gais_data_mode()
|
||||
logger.info("GAIS data import completed successfully")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"GAIS data import failed: {str(e)}")
|
||||
return False
|
||||
|
||||
# Old clear_dev_redis_queue function removed - now handled by Redis Manager
|
||||
|
||||
def run_development_mode():
|
||||
"""Run the server in development mode with auto-reload"""
|
||||
logger.info("Running in development mode")
|
||||
|
||||
# Initialize Redis manager for development (auto-clears data)
|
||||
global redis_manager
|
||||
from modules.redis_manager import get_redis_manager
|
||||
redis_manager = get_redis_manager('dev')
|
||||
|
||||
if not redis_manager.initialize_environment():
|
||||
logger.error("Failed to initialize Redis for development")
|
||||
return False
|
||||
|
||||
# Workers are started in app startup event
|
||||
|
||||
logger.info("Starting uvicorn server with auto-reload...")
|
||||
|
||||
uvicorn.run(
|
||||
"main:app",
|
||||
host="0.0.0.0",
|
||||
port=int(os.getenv('UVICORN_PORT', 8000)),
|
||||
log_level=os.getenv('LOG_LEVEL', 'info'),
|
||||
proxy_headers=True,
|
||||
timeout_keep_alive=10,
|
||||
reload=True
|
||||
)
|
||||
# Install signal handlers for graceful shutdown
|
||||
_install_signal_handlers()
|
||||
|
||||
try:
|
||||
uvicorn.run(
|
||||
"main:app",
|
||||
host="0.0.0.0",
|
||||
port=int(os.getenv('UVICORN_PORT', 8080)),
|
||||
log_level=os.getenv('LOG_LEVEL', 'info'),
|
||||
proxy_headers=True,
|
||||
timeout_keep_alive=10,
|
||||
reload=True
|
||||
)
|
||||
finally:
|
||||
stop_queue_workers()
|
||||
if redis_manager:
|
||||
redis_manager.shutdown()
|
||||
|
||||
def run_production_mode():
|
||||
"""Run the server in production mode"""
|
||||
logger.info("Running in production mode")
|
||||
|
||||
# Initialize Redis manager for production (preserves data, recovers tasks)
|
||||
global redis_manager
|
||||
from modules.redis_manager import get_redis_manager
|
||||
redis_manager = get_redis_manager('prod')
|
||||
|
||||
if not redis_manager.initialize_environment():
|
||||
logger.error("Failed to initialize Redis for production")
|
||||
return False
|
||||
|
||||
# Workers are started in app startup event
|
||||
|
||||
logger.info("Starting uvicorn server in production mode...")
|
||||
|
||||
uvicorn.run(
|
||||
"main:app",
|
||||
host="0.0.0.0",
|
||||
port=int(os.getenv('UVICORN_PORT', 8000)),
|
||||
log_level=os.getenv('LOG_LEVEL', 'info'),
|
||||
proxy_headers=True,
|
||||
timeout_keep_alive=10,
|
||||
workers=int(os.getenv('UVICORN_WORKERS', '1'))
|
||||
)
|
||||
# Install signal handlers for graceful shutdown
|
||||
_install_signal_handlers()
|
||||
|
||||
try:
|
||||
uvicorn.run(
|
||||
"main:app",
|
||||
host="0.0.0.0",
|
||||
port=int(os.getenv('UVICORN_PORT', 8080)),
|
||||
log_level=os.getenv('LOG_LEVEL', 'info'),
|
||||
proxy_headers=True,
|
||||
timeout_keep_alive=10,
|
||||
workers=int(os.getenv('UVICORN_WORKERS', '1'))
|
||||
)
|
||||
finally:
|
||||
stop_queue_workers()
|
||||
if redis_manager:
|
||||
redis_manager.shutdown()
|
||||
|
||||
def parse_arguments():
|
||||
"""Parse command line arguments"""
|
||||
@@ -151,15 +365,18 @@ def parse_arguments():
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Startup modes:
|
||||
init - Run initialization scripts (database setup, etc.)
|
||||
dev - Run development server with auto-reload
|
||||
prod - Run production server (for Docker/containerized deployment)
|
||||
infra - Setup infrastructure (Neo4j schema, calendar, Supabase buckets)
|
||||
demo-school - Create demo school (KevlarAI)
|
||||
demo-users - Create demo users
|
||||
gais-data - Import GAIS data (Edubase, etc.)
|
||||
dev - Run development server with auto-reload
|
||||
prod - Run production server (for Docker/containerized deployment)
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--mode', '-m',
|
||||
choices=['init', 'dev', 'prod'],
|
||||
choices=['infra', 'demo-school', 'demo-users', 'gais-data', 'dev', 'prod'],
|
||||
default='dev',
|
||||
help='Startup mode (default: dev)'
|
||||
)
|
||||
@@ -177,9 +394,24 @@ if __name__ == "__main__":
|
||||
|
||||
logger.info(f"Starting ClassroomCopilot API in {args.mode} mode")
|
||||
|
||||
if args.mode == 'init':
|
||||
# Run initialization only
|
||||
success = run_initialization_mode()
|
||||
if args.mode == 'infra':
|
||||
# Run infrastructure setup
|
||||
success = run_infrastructure_mode()
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
elif args.mode == 'demo-school':
|
||||
# Run demo school creation
|
||||
success = run_demo_school_mode()
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
elif args.mode == 'demo-users':
|
||||
# Run demo users creation
|
||||
success = run_demo_users_mode()
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
elif args.mode == 'gais-data':
|
||||
# Run GAIS data import
|
||||
success = run_gais_data_mode()
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
elif args.mode == 'dev':
|
||||
|
||||
Reference in New Issue
Block a user