diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml new file mode 100644 index 0000000..ecb8d68 --- /dev/null +++ b/.gitea/workflows/deploy.yml @@ -0,0 +1,35 @@ +name: api-ci-deploy + +on: + push: + branches: [master] + workflow_dispatch: + +jobs: + test-build-deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build API image + run: docker build -t cc-api-ci:${{ github.sha }} . + + - name: Configure SSH + run: | + mkdir -p ~/.ssh + printf '%s\n' "${{ secrets.DEPLOY_SSH_PRIVATE_KEY }}" > ~/.ssh/deploy_key + chmod 600 ~/.ssh/deploy_key + printf '%s\n' "${{ secrets.DEPLOY_KNOWN_HOSTS }}" > ~/.ssh/known_hosts + + - name: Deploy API + run: | + ssh -i ~/.ssh/deploy_key "${{ secrets.DEPLOY_USER }}@${{ secrets.API_DEPLOY_HOST }}" ' + set -euo pipefail + cd /home/kcar/api + git fetch origin master + git reset --hard origin/master + docker network inspect kevlarai-network >/dev/null 2>&1 || docker network create kevlarai-network + docker compose -p api -f docker-compose.yml up -d --build + docker compose -p api -f docker-compose.yml ps + curl -fsS http://127.0.0.1:8000/health >/dev/null + ' diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index e66d344..c3b82e1 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -36,6 +36,26 @@ services: - kevlarai-network restart: unless-stopped + backend-test: + image: cc-api-dev:latest + build: + context: . + dockerfile: Dockerfile + env_file: + - .env.dev + environment: + - REDIS_HOST=redis-dev + - API_HEALTH_URL=http://192.168.0.64:18000/health + depends_on: + redis-dev: + condition: service_healthy + networks: + - kevlarai-network + entrypoint: ["python", "-m", "pytest"] + command: ["-q", "tests"] + profiles: + - test + volumes: redis-dev-data: diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..3b2c446 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +testpaths = tests +python_files = test_*.py +addopts = -q diff --git a/run/initialization/sync_users_neo4j.py b/run/initialization/sync_users_neo4j.py new file mode 100644 index 0000000..fab70c2 --- /dev/null +++ b/run/initialization/sync_users_neo4j.py @@ -0,0 +1,238 @@ +"""Sync Supabase profile users into the central Neo4j cc.users database. + +This script is intentionally idempotent. It defaults to --dry-run so it can be +used safely during diagnostics. Use --apply to write/update Neo4j nodes. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path +from dataclasses import dataclass +from typing import Any + +SCRIPT_DIR = str(Path(__file__).resolve().parent) +if SCRIPT_DIR in sys.path: + sys.path.remove(SCRIPT_DIR) + +import requests +from dotenv import load_dotenv +from neo4j import GraphDatabase + + +@dataclass(frozen=True) +class UserRecord: + uuid_string: str + user_email: str + cc_username: str + user_type: str + user_name: str + display_name: str | None + institute_id: str | None + institute_name: str | None + institute_role: str | None + institute_neo4j_uuid_string: str | None + user_db_name: str + institute_db_name: str | None + node_storage_path: str + + +def load_environment(env_file: str | None) -> None: + if env_file: + load_dotenv(env_file, override=True) + else: + load_dotenv(override=False) + + +def supabase_headers() -> dict[str, str]: + key = os.getenv('SERVICE_ROLE_KEY') or os.getenv('ANON_KEY') + if not key: + raise RuntimeError('SERVICE_ROLE_KEY or ANON_KEY is required') + return {'apikey': key, 'Authorization': f'Bearer {key}'} + + +def fetch_table(table: str, select: str) -> list[dict[str, Any]]: + base = os.getenv('SUPABASE_URL') + if not base: + raise RuntimeError('SUPABASE_URL is required') + response = requests.get( + f'{base.rstrip("/")}/rest/v1/{table}', + headers=supabase_headers(), + params={'select': select}, + timeout=30, + ) + if response.status_code != 200: + raise RuntimeError(f'Supabase {table} query failed: {response.status_code} {response.text[:500]}') + return response.json() + + +def normalize_uuid(value: str | None) -> str | None: + return value.replace('-', '') if value else None + + +def build_records() -> list[UserRecord]: + profiles = fetch_table('profiles', 'id,email,username,full_name,display_name,user_type,metadata,user_db_name,school_db_name') + memberships = fetch_table('institute_memberships', 'profile_id,institute_id,role') + institutes = fetch_table('institutes', 'id,name,urn,neo4j_uuid_string') + membership_by_profile = {m['profile_id']: m for m in memberships} + institute_by_id = {i['id']: i for i in institutes} + records: list[UserRecord] = [] + for profile in profiles: + profile_id = profile['id'] + user_type = profile.get('user_type') or 'unknown' + uuid_no_dash = normalize_uuid(profile_id) + email = profile.get('email') or '' + username = profile.get('username') or (email.split('@', 1)[0] if email else uuid_no_dash) + user_name = profile.get('full_name') or profile.get('display_name') or username or email or profile_id + membership = membership_by_profile.get(profile_id, {}) + institute = institute_by_id.get(membership.get('institute_id'), {}) if membership else {} + institute_uuid = institute.get('neo4j_uuid_string') or normalize_uuid(institute.get('id')) + user_db_name = profile.get('user_db_name') or f'cc.users.{user_type}.{uuid_no_dash}' + institute_db_name = profile.get('school_db_name') or (f'cc.institutes.{institute_uuid}' if institute_uuid else None) + records.append(UserRecord( + uuid_string=profile_id, + user_email=email, + cc_username=username or profile_id, + user_type=user_type, + user_name=user_name, + display_name=profile.get('display_name'), + institute_id=membership.get('institute_id') if membership else None, + institute_name=institute.get('name') if institute else None, + institute_role=membership.get('role') if membership else None, + institute_neo4j_uuid_string=institute_uuid, + user_db_name=user_db_name, + institute_db_name=institute_db_name, + node_storage_path=f'neo4j://{user_db_name}/User/{profile_id}', + )) + return records + + +def neo4j_driver(): + url = os.getenv('APP_BOLT_URL') + username = os.getenv('USER_NEO4J') + password = os.getenv('PASSWORD_NEO4J') + if not url or not username or not password: + raise RuntimeError('APP_BOLT_URL, USER_NEO4J, and PASSWORD_NEO4J are required') + return GraphDatabase.driver(url, auth=(username, password)) + + +def ensure_schema(session) -> None: + statements = [ + 'CREATE CONSTRAINT user_uuid_unique IF NOT EXISTS FOR (u:User) REQUIRE u.uuid_string IS UNIQUE', + 'CREATE INDEX user_email_index IF NOT EXISTS FOR (u:User) ON (u.user_email)', + 'CREATE INDEX user_username_index IF NOT EXISTS FOR (u:User) ON (u.cc_username)', + 'CREATE INDEX user_type_index IF NOT EXISTS FOR (u:User) ON (u.user_type)', + 'CREATE INDEX user_institute_index IF NOT EXISTS FOR (u:User) ON (u.institute_id)', + 'CREATE CONSTRAINT institute_uuid_unique IF NOT EXISTS FOR (i:Institute) REQUIRE i.uuid_string IS UNIQUE', + ] + for statement in statements: + session.run(statement).consume() + + +def merge_user(session, record: UserRecord) -> None: + labels = ':User' + if record.user_type == 'teacher': + labels = ':User:Teacher' + elif record.user_type == 'student': + labels = ':User:Student' + session.run( + f""" + MERGE (u{labels} {{uuid_string: $uuid_string}}) + SET u.user_email = $user_email, + u.cc_username = $cc_username, + u.user_type = $user_type, + u.user_name = $user_name, + u.display_name = $display_name, + u.institute_id = $institute_id, + u.institute_name = $institute_name, + u.institute_role = $institute_role, + u.institute_neo4j_uuid_string = $institute_neo4j_uuid_string, + u.user_db_name = $user_db_name, + u.institute_db_name = $institute_db_name, + u.node_storage_path = $node_storage_path, + u.merged = true, + u.source = 'supabase.profiles', + u.synced_at = datetime() + """, + **record.__dict__, + ).consume() + if record.institute_neo4j_uuid_string: + session.run( + """ + MATCH (u:User {uuid_string: $uuid_string}) + MERGE (i:Institute {uuid_string: $institute_neo4j_uuid_string}) + SET i.supabase_id = $institute_id, + i.name = $institute_name + MERGE (u)-[r:MEMBER_OF]->(i) + SET r.role = $institute_role, + r.synced_at = datetime() + """, + **record.__dict__, + ).consume() + + +def verify(session) -> dict[str, Any]: + result: dict[str, Any] = {} + result['users'] = session.run('MATCH (u:User) RETURN count(u) AS c').single()['c'] + result['teachers'] = session.run("MATCH (u:User {user_type: 'teacher'}) RETURN count(u) AS c").single()['c'] + result['students'] = session.run("MATCH (u:User {user_type: 'student'}) RETURN count(u) AS c").single()['c'] + result['bad_users'] = session.run(""" + MATCH (u:User) + WHERE u.uuid_string IS NULL OR u.user_email IS NULL OR u.cc_username IS NULL + OR u.user_type IS NULL OR u.user_name IS NULL OR u.user_db_name IS NULL + RETURN count(u) AS c + """).single()['c'] + result['duplicate_uuids'] = session.run(""" + MATCH (u:User) + WITH u.uuid_string AS uuid, count(*) AS c + WHERE c > 1 + RETURN count(*) AS c + """).single()['c'] + result['memberships'] = session.run('MATCH (:User)-[r:MEMBER_OF]->(:Institute) RETURN count(r) AS c').single()['c'] + result['institutes'] = session.run('MATCH (i:Institute) RETURN count(i) AS c').single()['c'] + return result + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument('--env-file', default=None) + parser.add_argument('--database', default='cc.users') + parser.add_argument('--apply', action='store_true', help='Write to Neo4j. Defaults to dry-run.') + args = parser.parse_args() + load_environment(args.env_file) + records = build_records() + by_type: dict[str, int] = {} + for record in records: + by_type[record.user_type] = by_type.get(record.user_type, 0) + 1 + print(f'Supabase source records: {len(records)} {by_type}') + if not args.apply: + for record in records[:5]: + print(f'DRY RUN {record.uuid_string} {record.user_email} -> {record.user_db_name}') + print('Dry run only. Re-run with --apply to write Neo4j cc.users.') + return 0 + with neo4j_driver() as driver: + with driver.session(database=args.database) as session: + ensure_schema(session) + for record in records: + merge_user(session, record) + result = verify(session) + print(f'Neo4j verification: {result}') + expected = { + 'users': len(records), + 'teachers': by_type.get('teacher', 0), + 'students': by_type.get('student', 0), + 'bad_users': 0, + 'duplicate_uuids': 0, + 'memberships': len(records), + 'institutes': 2, + } + for key, value in expected.items(): + if result.get(key) != value: + raise RuntimeError(f'Verification failed for {key}: expected {value}, got {result.get(key)}') + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tests/test_dev_stack.py b/tests/test_dev_stack.py new file mode 100644 index 0000000..ce68b43 --- /dev/null +++ b/tests/test_dev_stack.py @@ -0,0 +1,53 @@ +import os + +import requests + + +def _supabase_headers(): + key = os.getenv('SERVICE_ROLE_KEY') or os.getenv('ANON_KEY') + assert key, 'SERVICE_ROLE_KEY or ANON_KEY must be set for Supabase integration tests' + return { + 'apikey': key, + 'Authorization': f'Bearer {key}', + 'Prefer': 'count=exact', + } + + +def _rest_count(table: str) -> int: + supabase_url = os.getenv('SUPABASE_URL') + assert supabase_url, 'SUPABASE_URL must be set' + response = requests.get( + f'{supabase_url.rstrip("/")}/rest/v1/{table}', + headers=_supabase_headers(), + params={'select': 'id'}, + timeout=15, + ) + assert response.status_code in (200, 206), response.text[:500] + content_range = response.headers.get('content-range', '') + assert '/' in content_range, f'missing exact content-range count for {table}: {content_range!r}' + return int(content_range.rsplit('/', 1)[1]) + + +def test_dev_environment_points_at_dev_supabase(): + assert os.getenv('SUPABASE_URL') == 'http://192.168.0.94:8000' + + +def test_dev_api_health_endpoint_is_healthy(): + health_url = os.getenv('API_HEALTH_URL', 'http://192.168.0.64:18000/health') + response = requests.get(health_url, timeout=15) + assert response.status_code == 200 + payload = response.json() + assert payload['status'] == 'healthy' + assert payload['services']['supabase']['status'] == 'healthy' + assert payload['services']['redis']['status'] == 'healthy' + + +def test_supabase_dev_seed_core_counts(): + assert _rest_count('profiles') == 21 + assert _rest_count('institute_memberships') == 21 + assert _rest_count('institutes') == 2 + + +def test_supabase_dev_seed_timetable_counts(): + assert _rest_count('classes') == 17 + assert _rest_count('taught_lessons') == 1462 diff --git a/tests/test_neo4j_users.py b/tests/test_neo4j_users.py new file mode 100644 index 0000000..ae9b686 --- /dev/null +++ b/tests/test_neo4j_users.py @@ -0,0 +1,34 @@ +import os + +from neo4j import GraphDatabase + + +def test_cc_users_database_is_populated_from_seed_profiles(): + url = os.getenv('APP_BOLT_URL') + user = os.getenv('USER_NEO4J') + password = os.getenv('PASSWORD_NEO4J') + assert url and user and password + with GraphDatabase.driver(url, auth=(user, password)) as driver: + with driver.session(database='cc.users') as session: + result = session.run(''' + CALL { MATCH (u:User) RETURN count(u) AS users } + CALL { MATCH (u:User {user_type: 'teacher'}) RETURN count(u) AS teachers } + CALL { MATCH (u:User {user_type: 'student'}) RETURN count(u) AS students } + CALL { MATCH (:User)-[r:MEMBER_OF]->(:Institute) RETURN count(r) AS memberships } + CALL { MATCH (i:Institute) RETURN count(i) AS institutes } + CALL { + MATCH (bad:User) + WHERE bad.uuid_string IS NULL OR bad.user_email IS NULL OR bad.cc_username IS NULL + OR bad.user_type IS NULL OR bad.user_name IS NULL OR bad.user_db_name IS NULL + RETURN count(bad) AS bad_users + } + RETURN users, teachers, students, memberships, institutes, bad_users + ''').single() + assert dict(result) == { + 'users': 21, + 'teachers': 15, + 'students': 6, + 'memberships': 21, + 'institutes': 2, + 'bad_users': 0, + }