diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..c1a97b0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +volumes/db/data +volumes/storage +volumes/logs +.git diff --git a/.gitignore b/.gitignore index e69de29..c9641d0 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1,16 @@ +# Environment files +.env +.env.* +!.env.example + +# Docker volumes (large runtime data) +volumes/ + +# Backup files +*.bak +*.bak.* +backups/ + +# Logs +logs/ +*.log diff --git a/api/kong.yml b/api/kong.yml deleted file mode 100644 index 00f81d6..0000000 --- a/api/kong.yml +++ /dev/null @@ -1,226 +0,0 @@ -_format_version: '2.1' -_transform: true - -### -### Consumers / Users -### -consumers: - - username: DASHBOARD - - username: anon - keyauth_credentials: - - key: $SUPABASE_ANON_KEY - - username: service_role - keyauth_credentials: - - key: $SUPABASE_SERVICE_KEY - -### -### Access Control List -### -acls: - - consumer: anon - group: anon - - consumer: service_role - group: admin - -### -### Dashboard credentials -### -basicauth_credentials: - - consumer: DASHBOARD - username: $DASHBOARD_USERNAME - password: $DASHBOARD_PASSWORD - -### -### API Routes -### -services: - ## Open Auth routes - - name: auth-v1-open - url: http://auth:9999/verify - routes: - - name: auth-v1-open - strip_path: true - paths: - - /auth/v1/verify - plugins: - - name: cors - - name: auth-v1-open-callback - url: http://auth:9999/callback - routes: - - name: auth-v1-open-callback - strip_path: true - paths: - - /auth/v1/callback - plugins: - - name: cors - - name: auth-v1-open-authorize - url: http://auth:9999/authorize - routes: - - name: auth-v1-open-authorize - strip_path: true - paths: - - /auth/v1/authorize - plugins: - - name: cors - - ## Secure Auth routes - - name: auth-v1 - _comment: 'GoTrue: /auth/v1/* -> http://auth:9999/*' - url: http://auth:9999/ - routes: - - name: auth-v1-all - strip_path: true - paths: - - /auth/v1/ - plugins: - - name: cors - - name: key-auth - config: - hide_credentials: false - - name: acl - config: - hide_groups_header: true - allow: - - admin - - anon - - ## Secure REST routes - - name: rest-v1 - _comment: 'PostgREST: /rest/v1/* -> http://rest:3000/*' - url: http://rest:3000/ - routes: - - name: rest-v1-all - strip_path: true - paths: - - /rest/v1/ - plugins: - - name: cors - - name: key-auth - config: - hide_credentials: true - - name: acl - config: - hide_groups_header: true - allow: - - admin - - anon - - ## Secure GraphQL routes - - name: graphql-v1 - _comment: 'PostgREST: /graphql/v1/* -> http://rest:3000/rpc/graphql' - url: http://rest:3000/rpc/graphql - routes: - - name: graphql-v1-all - strip_path: true - paths: - - /graphql/v1 - plugins: - - name: cors - - name: key-auth - config: - hide_credentials: true - - name: request-transformer - config: - add: - headers: - - Content-Profile:graphql_public - - name: acl - config: - hide_groups_header: true - allow: - - admin - - anon - - ## Secure Realtime routes - - name: realtime-v1-ws - _comment: 'Realtime: /realtime/v1/* -> ws://realtime:4000/socket/*' - url: http://realtime-dev.supabase-realtime:4000/socket - protocol: ws - routes: - - name: realtime-v1-ws - strip_path: true - paths: - - /realtime/v1/ - plugins: - - name: cors - - name: key-auth - config: - hide_credentials: false - - name: acl - config: - hide_groups_header: true - allow: - - admin - - anon - - name: realtime-v1-rest - _comment: 'Realtime: /realtime/v1/* -> ws://realtime:4000/socket/*' - url: http://realtime-dev.supabase-realtime:4000/api - protocol: http - routes: - - name: realtime-v1-rest - strip_path: true - paths: - - /realtime/v1/api - plugins: - - name: cors - - name: key-auth - config: - hide_credentials: false - - name: acl - config: - hide_groups_header: true - allow: - - admin - - anon - ## Storage routes: the storage server manages its own auth - - name: storage-v1 - _comment: 'Storage: /storage/v1/* -> http://storage:5000/*' - url: http://storage:5000/ - routes: - - name: storage-v1-all - strip_path: true - paths: - - /storage/v1/ - plugins: - - name: cors - - ## Edge Functions routes - - name: functions-v1 - _comment: 'Edge Functions: /functions/v1/* -> http://functions:9000/*' - url: http://functions:9000/ - routes: - - name: functions-v1-all - strip_path: true - paths: - - /functions/v1/ - plugins: - - name: cors - - ## Analytics routes - - name: analytics-v1 - _comment: 'Analytics: /analytics/v1/* -> http://logflare:4000/*' - url: http://analytics:4000/ - routes: - - name: analytics-v1-all - strip_path: true - paths: - - /analytics/v1/ - - ## Secure Database routes - - name: meta - _comment: 'pg-meta: /pg/* -> http://pg-meta:8080/*' - url: http://meta:8080/ - routes: - - name: meta-all - strip_path: true - paths: - - /pg/ - plugins: - - name: key-auth - config: - hide_credentials: false - - name: acl - config: - hide_groups_header: true - allow: - - admin \ No newline at end of file diff --git a/db/init-scripts/51-webhooks.sql b/db/init-scripts/51-webhooks.sql deleted file mode 100644 index 886a6a7..0000000 --- a/db/init-scripts/51-webhooks.sql +++ /dev/null @@ -1,131 +0,0 @@ -BEGIN; - -- Create pg_net extension - CREATE EXTENSION IF NOT EXISTS pg_net SCHEMA extensions; - - -- Create supabase_functions schema - CREATE SCHEMA IF NOT EXISTS supabase_functions AUTHORIZATION postgres; - - -- Grant basic permissions - GRANT USAGE ON SCHEMA supabase_functions TO postgres, anon, authenticated, service_role; - ALTER DEFAULT PRIVILEGES IN SCHEMA supabase_functions GRANT ALL ON TABLES TO postgres, anon, authenticated, service_role; - ALTER DEFAULT PRIVILEGES IN SCHEMA supabase_functions GRANT ALL ON FUNCTIONS TO postgres, anon, authenticated, service_role; - ALTER DEFAULT PRIVILEGES IN SCHEMA supabase_functions GRANT ALL ON SEQUENCES TO postgres, anon, authenticated, service_role; - - -- supabase_functions.migrations definition - CREATE TABLE IF NOT EXISTS supabase_functions.migrations ( - version text PRIMARY KEY, - inserted_at timestamptz NOT NULL DEFAULT NOW() - ); - - -- Initial supabase_functions migration - INSERT INTO supabase_functions.migrations (version) VALUES ('initial') ON CONFLICT DO NOTHING; - - -- supabase_functions.hooks definition - CREATE TABLE IF NOT EXISTS supabase_functions.hooks ( - id bigserial PRIMARY KEY, - hook_table_id integer NOT NULL, - hook_name text NOT NULL, - created_at timestamptz NOT NULL DEFAULT NOW(), - request_id bigint - ); - - -- Create indexes if they don't exist - CREATE INDEX IF NOT EXISTS supabase_functions_hooks_request_id_idx ON supabase_functions.hooks USING btree (request_id); - CREATE INDEX IF NOT EXISTS supabase_functions_hooks_h_table_id_h_name_idx ON supabase_functions.hooks USING btree (hook_table_id, hook_name); - - COMMENT ON TABLE supabase_functions.hooks IS 'Supabase Functions Hooks: Audit trail for triggered hooks.'; - - -- Create the http_request function - CREATE OR REPLACE FUNCTION supabase_functions.http_request() - RETURNS trigger - LANGUAGE plpgsql - AS $function$ - DECLARE - request_id bigint; - payload jsonb; - url text := TG_ARGV[0]::text; - method text := TG_ARGV[1]::text; - headers jsonb DEFAULT '{}'::jsonb; - params jsonb DEFAULT '{}'::jsonb; - timeout_ms integer DEFAULT 1000; - BEGIN - IF url IS NULL OR url = 'null' THEN - RAISE EXCEPTION 'url argument is missing'; - END IF; - - IF method IS NULL OR method = 'null' THEN - RAISE EXCEPTION 'method argument is missing'; - END IF; - - IF TG_ARGV[2] IS NULL OR TG_ARGV[2] = 'null' THEN - headers = '{"Content-Type": "application/json"}'::jsonb; - ELSE - headers = TG_ARGV[2]::jsonb; - END IF; - - IF TG_ARGV[3] IS NULL OR TG_ARGV[3] = 'null' THEN - params = '{}'::jsonb; - ELSE - params = TG_ARGV[3]::jsonb; - END IF; - - IF TG_ARGV[4] IS NULL OR TG_ARGV[4] = 'null' THEN - timeout_ms = 1000; - ELSE - timeout_ms = TG_ARGV[4]::integer; - END IF; - - CASE - WHEN method = 'GET' THEN - SELECT http_get INTO request_id FROM net.http_get( - url, - params, - headers, - timeout_ms - ); - WHEN method = 'POST' THEN - payload = jsonb_build_object( - 'old_record', OLD, - 'record', NEW, - 'type', TG_OP, - 'table', TG_TABLE_NAME, - 'schema', TG_TABLE_SCHEMA - ); - - SELECT http_post INTO request_id FROM net.http_post( - url, - payload, - params, - headers, - timeout_ms - ); - ELSE - RAISE EXCEPTION 'method argument % is invalid', method; - END CASE; - - INSERT INTO supabase_functions.hooks - (hook_table_id, hook_name, request_id) - VALUES - (TG_RELID, TG_NAME, request_id); - - RETURN NEW; - END - $function$; - - -- Set function properties - ALTER FUNCTION supabase_functions.http_request() SECURITY DEFINER; - ALTER FUNCTION supabase_functions.http_request() SET search_path = supabase_functions; - - -- Grant execute permissions - REVOKE ALL ON FUNCTION supabase_functions.http_request() FROM PUBLIC; - GRANT EXECUTE ON FUNCTION supabase_functions.http_request() TO postgres, anon, authenticated, service_role; - - -- Grant pg_net permissions - GRANT USAGE ON SCHEMA net TO postgres, anon, authenticated, service_role; - GRANT EXECUTE ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) TO postgres, anon, authenticated, service_role; - GRANT EXECUTE ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) TO postgres, anon, authenticated, service_role; - - -- Add migration record - INSERT INTO supabase_functions.migrations (version) VALUES ('20210809183423_update_grants') ON CONFLICT DO NOTHING; - -COMMIT; diff --git a/db/init-scripts/52-jwt.sql b/db/init-scripts/52-jwt.sql deleted file mode 100644 index 0312581..0000000 --- a/db/init-scripts/52-jwt.sql +++ /dev/null @@ -1,6 +0,0 @@ --- Set JWT configuration for the database --- These settings will be configured through environment variables in the Supabase setup - --- Note: JWT configuration is handled by Supabase's internal configuration --- This file is kept for reference but the actual JWT settings are managed --- through the Supabase configuration and environment variables diff --git a/db/init-scripts/52-roles.sql b/db/init-scripts/52-roles.sql deleted file mode 100644 index 0f47c5b..0000000 --- a/db/init-scripts/52-roles.sql +++ /dev/null @@ -1,11 +0,0 @@ --- NOTE: change to your own passwords for production environments --- Password configuration is handled by Supabase's internal setup --- This file is kept for reference but the actual password settings are managed --- through the Supabase configuration and environment variables - --- The following users are created and configured by Supabase automatically: --- - authenticator --- - pgbouncer --- - supabase_auth_admin --- - supabase_functions_admin --- - supabase_storage_admin diff --git a/db/init/seed.sql b/db/init/seed.sql deleted file mode 100644 index e69de29..0000000 diff --git a/db/migrations/core/61-core-schema.sql b/db/migrations/core/61-core-schema.sql deleted file mode 100644 index d49d7b6..0000000 --- a/db/migrations/core/61-core-schema.sql +++ /dev/null @@ -1,364 +0,0 @@ ---[ Database Schema Version ]-- --- Version: 1.0.0 --- Last Updated: 2024-02-24 --- Description: Core schema setup for ClassConcepts with neoFS filesystem integration --- Dependencies: auth.users (Supabase Auth) - ---[ Validation ]-- -do $$ -begin - -- Verify required extensions - if not exists (select 1 from pg_extension where extname = 'uuid-ossp') then - raise exception 'Required extension uuid-ossp is not installed'; - end if; - - -- Verify auth schema exists - if not exists (select 1 from information_schema.schemata where schema_name = 'auth') then - raise exception 'Required auth schema is not available'; - end if; - - -- Verify storage schema exists - if not exists (select 1 from information_schema.schemata where schema_name = 'storage') then - raise exception 'Required storage schema is not available'; - end if; -end $$; - ---[ 1. Extensions ]-- -create extension if not exists "uuid-ossp"; - --- Create rpc schema if it doesn't exist -create schema if not exists rpc; -grant usage on schema rpc to anon, authenticated; - --- Create exec_sql function for admin operations -create or replace function exec_sql(query text) -returns void as $$ -begin - execute query; -end; -$$ language plpgsql security definer; - --- Create updated_at trigger function -create or replace function public.handle_updated_at() -returns trigger as $$ -begin - new.updated_at = timezone('utc'::text, now()); - return new; -end; -$$ language plpgsql security definer; - --- Create completed_at trigger function for document artefacts -create or replace function public.set_completed_at() -returns trigger as $$ -begin - if NEW.status = 'completed' and OLD.status != 'completed' then - NEW.completed_at = now(); - end if; - return NEW; -end; -$$ language plpgsql security definer; - ---[ 5. Core Tables ]-- --- Base user profiles -create table if not exists public.profiles ( - id uuid primary key references auth.users(id) on delete cascade, - email text not null unique, - user_type text not null check ( - user_type in ( - 'teacher', - 'student', - 'email_teacher', - 'email_student', - 'developer', - 'superadmin' - ) - ), - username text not null unique, - full_name text, - display_name text, - metadata jsonb default '{}'::jsonb, - user_db_name text, - school_db_name text, - neo4j_sync_status text default 'pending' check (neo4j_sync_status in ('pending', 'ready', 'failed')), - neo4j_synced_at timestamp with time zone, - last_login timestamp with time zone, - created_at timestamp with time zone default timezone('utc'::text, now()), - updated_at timestamp with time zone default timezone('utc'::text, now()) -); -comment on table public.profiles is 'User profiles linked to Supabase auth.users'; -comment on column public.profiles.user_type is 'Type of user: teacher or student'; - --- Active institutes -create table if not exists public.institutes ( - id uuid primary key default uuid_generate_v4(), - name text not null, - urn text unique, - status text not null default 'active' check (status in ('active', 'inactive', 'pending')), - address jsonb default '{}'::jsonb, - website text, - metadata jsonb default '{}'::jsonb, - geo_coordinates jsonb default '{}'::jsonb, - neo4j_uuid_string text, - neo4j_public_sync_status text default 'pending' check (neo4j_public_sync_status in ('pending', 'synced', 'failed')), - neo4j_public_sync_at timestamp with time zone, - neo4j_private_sync_status text default 'not_started' check (neo4j_private_sync_status in ('not_started', 'pending', 'synced', 'failed')), - neo4j_private_sync_at timestamp with time zone, - created_at timestamp with time zone default timezone('utc'::text, now()), - updated_at timestamp with time zone default timezone('utc'::text, now()) -); -comment on table public.institutes is 'Active institutes in the system'; -comment on column public.institutes.geo_coordinates is 'Geospatial coordinates from OSM search (latitude, longitude, boundingbox)'; - ---[ 6. neoFS Filesystem Tables ]-- --- File cabinets for organizing files -create table if not exists public.file_cabinets ( - id uuid primary key default uuid_generate_v4(), - user_id uuid not null references public.profiles(id) on delete cascade, - name text not null, - created_at timestamp with time zone default timezone('utc'::text, now()) -); -comment on table public.file_cabinets is 'User file cabinets for organizing documents and files'; - --- Files stored in cabinets -create table if not exists public.files ( - id uuid primary key default uuid_generate_v4(), - cabinet_id uuid not null references public.file_cabinets(id) on delete cascade, - name text not null, - path text not null, - bucket text default 'file-cabinets' not null, - created_at timestamp with time zone default timezone('utc'::text, now()), - mime_type text, - metadata jsonb default '{}'::jsonb, - size text, - category text generated always as ( - case - when mime_type like 'image/%' then 'image' - when mime_type = 'application/pdf' then 'document' - when mime_type in ('application/msword', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') then 'document' - when mime_type in ('application/vnd.ms-excel', 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet') then 'spreadsheet' - when mime_type in ('application/vnd.ms-powerpoint', 'application/vnd.openxmlformats-officedocument.presentationml.presentation') then 'presentation' - when mime_type like 'audio/%' then 'audio' - when mime_type like 'video/%' then 'video' - else 'other' - end - ) stored -); -comment on table public.files is 'Files stored in user cabinets with automatic categorization'; -comment on column public.files.category is 'Automatically determined file category based on MIME type'; - --- AI brains for processing files -create table if not exists public.brains ( - id uuid primary key default uuid_generate_v4(), - user_id uuid not null references public.profiles(id) on delete cascade, - name text not null, - purpose text, - created_at timestamp with time zone default timezone('utc'::text, now()) -); -comment on table public.brains is 'AI brains for processing and analyzing user files'; - --- Brain-file associations -create table if not exists public.brain_files ( - brain_id uuid not null references public.brains(id) on delete cascade, - file_id uuid not null references public.files(id) on delete cascade, - primary key (brain_id, file_id) -); -comment on table public.brain_files is 'Associations between AI brains and files for processing'; - --- Document artefacts from file processing -create table if not exists public.document_artefacts ( - id uuid primary key default uuid_generate_v4(), - file_id uuid references public.files(id) on delete cascade, - page_number integer default 0 not null, - type text not null, - rel_path text not null, - size_tag text, - language text, - chunk_index integer, - extra jsonb, - created_at timestamp with time zone default timezone('utc'::text, now()), - status text default 'completed' not null check (status in ('pending', 'processing', 'completed', 'failed')), - started_at timestamp with time zone default timezone('utc'::text, now()), - completed_at timestamp with time zone, - error_message text -); -comment on table public.document_artefacts is 'Extracted artefacts from document processing'; -comment on column public.document_artefacts.status is 'Extraction status: pending, processing, completed, or failed'; -comment on column public.document_artefacts.started_at is 'Timestamp when extraction process started'; -comment on column public.document_artefacts.completed_at is 'Timestamp when extraction process completed (success or failure)'; -comment on column public.document_artefacts.error_message is 'Error details if extraction failed'; - --- Function execution logs -create table if not exists public.function_logs ( - id serial primary key, - file_id uuid references public.files(id) on delete cascade, - timestamp timestamp with time zone default timezone('utc'::text, now()), - step text, - message text, - data jsonb -); -comment on table public.function_logs is 'Logs of function executions and processing steps'; - ---[ 7. Relationship Tables ]-- --- Institute memberships -create table if not exists public.institute_memberships ( - id uuid primary key default uuid_generate_v4(), - profile_id uuid references public.profiles(id) on delete cascade, - institute_id uuid references public.institutes(id) on delete cascade, - role text not null check (role in ('teacher', 'student')), - tldraw_preferences jsonb default '{}'::jsonb, - metadata jsonb default '{}'::jsonb, - created_at timestamp with time zone default timezone('utc'::text, now()), - updated_at timestamp with time zone default timezone('utc'::text, now()), - unique(profile_id, institute_id) -); -comment on table public.institute_memberships is 'Manages user roles and relationships with institutes'; - --- Membership requests -create table if not exists public.institute_membership_requests ( - id uuid primary key default uuid_generate_v4(), - profile_id uuid references public.profiles(id) on delete cascade, - institute_id uuid references public.institutes(id) on delete cascade, - requested_role text check (requested_role in ('teacher', 'student')), - status text default 'pending' check (status in ('pending', 'approved', 'rejected')), - metadata jsonb default '{}'::jsonb, - created_at timestamp with time zone default timezone('utc'::text, now()), - updated_at timestamp with time zone default timezone('utc'::text, now()) -); -comment on table public.institute_membership_requests is 'Tracks requests to join institutes'; - ---[ 8. Audit Tables ]-- --- System audit logs -create table if not exists public.audit_logs ( - id uuid primary key default uuid_generate_v4(), - profile_id uuid references public.profiles(id) on delete set null, - action_type text, - table_name text, - record_id uuid, - changes jsonb, - created_at timestamp with time zone default timezone('utc'::text, now()) -); -comment on table public.audit_logs is 'System-wide audit trail for important operations'; - ---[ 9. Exam Specifications ]-- -create table if not exists public.eb_specifications ( - id uuid primary key default uuid_generate_v4(), - spec_code text unique, - exam_board_code text, - award_code text, - subject_code text, - first_teach text, - spec_ver text, - - -- Document storage details - storage_loc text, - doc_type text check (doc_type in ('pdf', 'json', 'md', 'html', 'txt', 'doctags')), - doc_details jsonb default '{}'::jsonb, -- e.g. Tika extract - docling_docs jsonb default '{}'::jsonb, -- e.g. Docling extracts settings and storage locations - - created_at timestamp with time zone default timezone('utc'::text, now()), - updated_at timestamp with time zone default timezone('utc'::text, now()) -); - -comment on table public.eb_specifications is 'Exam board specifications and their primary document'; -comment on column public.eb_specifications.spec_code is 'Unique code for the specification, used for linking exams'; -comment on column public.eb_specifications.doc_details is 'Tika extract of the specification document'; -comment on column public.eb_specifications.docling_docs is 'Docling extracts settings and storage locations for the specification document'; - ---[ 10. Exam Papers / Entries ]-- -create table if not exists public.eb_exams ( - id uuid primary key default uuid_generate_v4(), - exam_code text unique, - spec_code text references public.eb_specifications(spec_code) on delete cascade, - paper_code text, - tier text, - session text, - type_code text, - - -- Document storage details - storage_loc text, - doc_type text check (doc_type in ('pdf', 'json', 'md', 'html', 'txt', 'doctags')), - doc_details jsonb default '{}'::jsonb, -- e.g. Tika extract - docling_docs jsonb default '{}'::jsonb, -- e.g. Docling extracts settings and storage locations - - created_at timestamp with time zone default timezone('utc'::text, now()), - updated_at timestamp with time zone default timezone('utc'::text, now()) -); - -comment on table public.eb_exams is 'Exam papers and related documents linked to specifications'; -comment on column public.eb_exams.exam_code is 'Unique code for the exam paper, used for linking questions'; -comment on column public.eb_exams.type_code is 'Type code for the exam document: Question Paper (QP), Mark Scheme (MS), Examiner Report (ER), Other (OT)'; -comment on column public.eb_exams.doc_details is 'Tika extract of the exam paper document'; -comment on column public.eb_exams.docling_docs is 'Docling extracts settings and storage locations for the exam paper document'; - ---[ 11. Indexes ]-- --- Index for geospatial queries -create index if not exists idx_institutes_geo_coordinates on public.institutes using gin(geo_coordinates); -create index if not exists idx_institutes_urn on public.institutes(urn); - --- Document artefacts indexes -create index if not exists idx_document_artefacts_file_status on public.document_artefacts(file_id, status); -create index if not exists idx_document_artefacts_file_type on public.document_artefacts(file_id, type); -create index if not exists idx_document_artefacts_status on public.document_artefacts(status); - --- File indexes -create index if not exists idx_files_cabinet_id on public.files(cabinet_id); -create index if not exists idx_files_mime_type on public.files(mime_type); -create index if not exists idx_files_category on public.files(category); - --- Brain indexes -create index if not exists idx_brains_user_id on public.brains(user_id); - --- Exam board indexes -create index if not exists idx_eb_exams_exam_code on public.eb_exams(exam_code); -create index if not exists idx_eb_exams_spec_code on public.eb_exams(spec_code); -create index if not exists idx_eb_exams_paper_code on public.eb_exams(paper_code); -create index if not exists idx_eb_exams_tier on public.eb_exams(tier); -create index if not exists idx_eb_exams_session on public.eb_exams(session); -create index if not exists idx_eb_exams_type_code on public.eb_exams(type_code); -create index if not exists idx_eb_specifications_spec_code on public.eb_specifications(spec_code); -create index if not exists idx_eb_specifications_exam_board_code on public.eb_specifications(exam_board_code); -create index if not exists idx_eb_specifications_award_code on public.eb_specifications(award_code); -create index if not exists idx_eb_specifications_subject_code on public.eb_specifications(subject_code); - ---[ 12. Triggers ]-- --- Set completed_at when document artefact status changes to completed -create trigger trigger_set_completed_at - before update on public.document_artefacts - for each row - execute function public.set_completed_at(); - --- Set updated_at on profile updates -create trigger trigger_profiles_updated_at - before update on public.profiles - for each row - execute function public.handle_updated_at(); - --- Set updated_at on institute updates -create trigger trigger_institutes_updated_at - before update on public.institutes - for each row - execute function public.handle_updated_at(); - --- Set updated_at on institute_memberships updates -create trigger trigger_institute_memberships_updated_at - before update on public.institute_memberships - for each row - execute function public.handle_updated_at(); - --- Set updated_at on institute_membership_requests updates -create trigger trigger_institute_membership_requests_updated_at - before update on public.institute_memberships - for each row - execute function public.handle_updated_at(); - --- Set updated_at on eb_specifications updates -create trigger trigger_eb_specifications_updated_at - before update on public.eb_specifications - for each row - execute function public.handle_updated_at(); - --- Set updated_at on eb_exams updates -create trigger trigger_eb_exams_updated_at - before update on public.eb_exams - for each row - execute function public.handle_updated_at(); diff --git a/db/migrations/core/62-functions-triggers.sql b/db/migrations/core/62-functions-triggers.sql deleted file mode 100644 index 8c13d65..0000000 --- a/db/migrations/core/62-functions-triggers.sql +++ /dev/null @@ -1,191 +0,0 @@ ---[ 8. Auth Functions ]-- --- Create a secure function to check admin status -create or replace function public.is_admin() -returns boolean as $$ - select coalesce( - (select true - from public.profiles - where id = auth.uid() - and user_type = 'admin'), - false - ); -$$ language sql security definer; - --- Create a secure function to check super admin status -create or replace function public.is_super_admin() -returns boolean as $$ - select coalesce( - (select true - from public.profiles - where id = auth.uid() - and user_type = 'admin'), - false - ); -$$ language sql security definer; - --- Create public wrapper functions --- Note: These are now the main implementation functions, not wrappers --- The original auth schema functions have been moved to public schema - --- Grant execute permissions -grant execute on function public.is_admin to authenticated; -grant execute on function public.is_super_admin to authenticated; - --- Initial admin setup function -create or replace function public.setup_initial_admin(admin_email text) -returns json -language plpgsql -security definer -as $$ -declare - result json; -begin - -- Only allow this to run as service role or superuser - if not ( - current_user = 'service_role' - or exists ( - select 1 from pg_roles - where rolname = current_user - and rolsuper - ) - ) then - raise exception 'Must be run as service_role or superuser'; - end if; - - -- Update user_type and username for admin - update public.profiles - set user_type = 'admin', - username = coalesce(username, 'superadmin'), - display_name = coalesce(display_name, 'Super Admin') - where email = admin_email - returning json_build_object( - 'id', id, - 'email', email, - 'user_type', user_type, - 'username', username, - 'display_name', display_name - ) into result; - - if result is null then - raise exception 'Admin user with email % not found', admin_email; - end if; - - return result; -end; -$$; - --- Grant execute permissions -revoke execute on function public.setup_initial_admin from public; -grant execute on function public.setup_initial_admin to authenticated, service_role; - --- Create RPC wrapper for REST API access -create or replace function rpc.setup_initial_admin(admin_email text) -returns json -language plpgsql -security definer -as $$ -begin - return public.setup_initial_admin(admin_email); -end; -$$; - --- Grant execute permissions for RPC wrapper -grant execute on function rpc.setup_initial_admin to authenticated, service_role; - ---[ 9. Utility Functions ]-- --- Check if database is ready -create or replace function check_db_ready() -returns boolean -language plpgsql -security definer -as $$ -begin - -- Check if essential schemas exist - if not exists ( - select 1 - from information_schema.schemata - where schema_name in ('auth', 'storage', 'public') - ) then - return false; - end if; - - -- Check if essential tables exist - if not exists ( - select 1 - from information_schema.tables - where table_schema = 'auth' - and table_name = 'users' - ) then - return false; - end if; - - -- Check if RLS is enabled on public.profiles - if not exists ( - select 1 - from pg_tables - where schemaname = 'public' - and tablename = 'profiles' - and rowsecurity = true - ) then - return false; - end if; - - return true; -end; -$$; - --- Grant execute permission -grant execute on function check_db_ready to anon, authenticated, service_role; - --- Function to handle new user registration -create or replace function public.handle_new_user() -returns trigger -language plpgsql -security definer set search_path = public -as $$ -declare - default_user_type text := 'email_student'; - default_username text; -begin - -- Generate username from email - default_username := split_part(new.email, '@', 1); - - insert into public.profiles ( - id, - email, - user_type, - username, - display_name - ) - values ( - new.id, - new.email, - coalesce(new.raw_user_meta_data->>'user_type', default_user_type), - coalesce(new.raw_user_meta_data->>'username', default_username), - coalesce(new.raw_user_meta_data->>'display_name', default_username) - ); - return new; -end; -$$; - --- Trigger for new user creation -drop trigger if exists on_auth_user_created on auth.users; -create trigger on_auth_user_created - after insert on auth.users - for each row execute procedure public.handle_new_user(); - ---[ 11. Database Triggers ]-- -drop trigger if exists handle_profiles_updated_at on public.profiles; -create trigger handle_profiles_updated_at - before update on public.profiles - for each row execute function public.handle_updated_at(); - -drop trigger if exists handle_institute_memberships_updated_at on public.institute_memberships; -create trigger handle_institute_memberships_updated_at - before update on public.institute_memberships - for each row execute function public.handle_updated_at(); - -drop trigger if exists handle_membership_requests_updated_at on public.institute_membership_requests; -create trigger handle_membership_requests_updated_at - before update on public.institute_membership_requests - for each row execute function public.handle_updated_at(); \ No newline at end of file diff --git a/db/migrations/core/63-storage-policies.sql b/db/migrations/core/63-storage-policies.sql deleted file mode 100644 index 952e9b4..0000000 --- a/db/migrations/core/63-storage-policies.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Storage policies configuration for Supabase --- Note: Storage bucket policies are managed by Supabase internally --- This file provides guidance on what should be configured - --- Storage bucket policies should be configured through: --- 1. Supabase Dashboard > Storage > Policies --- 2. Or via SQL with proper permissions (requires service_role or owner access) - --- Recommended policies for storage.buckets: --- - Super admin has full access to buckets --- - Users can create their own buckets --- - Users can view their own buckets or public buckets - --- Recommended policies for storage.objects: --- - Users can upload to buckets they own --- - Users can view objects in public buckets --- - Users can manage objects in buckets they own - --- Note: These policies require the service_role or appropriate permissions --- to be applied to the storage schema tables \ No newline at end of file diff --git a/db/migrations/core/64-initial-admin.sql b/db/migrations/core/64-initial-admin.sql deleted file mode 100644 index d4aa11f..0000000 --- a/db/migrations/core/64-initial-admin.sql +++ /dev/null @@ -1,20 +0,0 @@ --- Initial admin setup for ClassroomCopilot --- This file handles basic database setup and permissions - --- Ensure uuid-ossp extension is enabled -create extension if not exists "uuid-ossp" schema extensions; - --- Grant basic permissions to authenticated users for public schema --- Note: These permissions are granted to allow users to work with the application -grant usage on schema public to authenticated; -grant all on all tables in schema public to authenticated; -grant all on all sequences in schema public to authenticated; -grant all on all functions in schema public to authenticated; - --- Set default privileges for future objects -alter default privileges in schema public grant all on tables to authenticated; -alter default privileges in schema public grant all on sequences to authenticated; -alter default privileges in schema public grant all on functions to authenticated; - --- Note: The setup_initial_admin function is defined in 62-functions-triggers.sql --- and should be called with an admin email parameter when needed \ No newline at end of file diff --git a/db/migrations/core/65-filesystem-augments.sql b/db/migrations/core/65-filesystem-augments.sql deleted file mode 100644 index bb07296..0000000 --- a/db/migrations/core/65-filesystem-augments.sql +++ /dev/null @@ -1,95 +0,0 @@ --- Files table augments and storage GC hooks - --- 1) Add columns to files if missing -do $$ -begin - if not exists ( - select 1 from information_schema.columns - where table_schema='public' and table_name='files' and column_name='uploaded_by' - ) then - alter table public.files add column uploaded_by uuid references public.profiles(id); - end if; - if not exists ( - select 1 from information_schema.columns - where table_schema='public' and table_name='files' and column_name='size_bytes' - ) then - alter table public.files add column size_bytes bigint; - end if; - if not exists ( - select 1 from information_schema.columns - where table_schema='public' and table_name='files' and column_name='source' - ) then - alter table public.files add column source text default 'uploader-web'; - end if; -end $$; - --- 2) Unique index for cabinet/path combo -create unique index if not exists uq_files_cabinet_path on public.files(cabinet_id, path); - --- 3) Storage GC helpers (ported from neoFS with storage schema) -create or replace function public._delete_storage_objects(p_bucket text, p_path text) -returns void -language plpgsql security definer -set search_path to 'public', 'storage' -as $$ -begin - if p_bucket is null or p_path is null then - return; - end if; - delete from storage.objects where bucket_id = p_bucket and name = p_path; - delete from storage.objects where bucket_id = p_bucket and name like p_path || '/%'; -end -$$; - -create or replace function public._storage_gc_sql() -returns trigger -language plpgsql security definer -set search_path to 'public', 'storage' -as $$ -begin - if tg_op = 'DELETE' then - perform public._delete_storage_objects(old.bucket, old.path); - elsif tg_op = 'UPDATE' then - if (old.bucket is distinct from new.bucket) or (old.path is distinct from new.path) then - perform public._delete_storage_objects(old.bucket, old.path); - end if; - end if; - return null; -end -$$; - --- 4) Attach GC trigger to files bucket/path changes -drop trigger if exists trg_files_gc on public.files; -create trigger trg_files_gc -after delete or update of bucket, path on public.files -for each row execute function public._storage_gc_sql(); - --- 5) Document artefacts GC: remove artefact objects from storage when rows change/delete -create or replace function public._artefact_gc_sql() -returns trigger -language plpgsql security definer -set search_path to 'public', 'storage' -as $$ -declare - v_bucket text; -begin - if tg_op = 'DELETE' then - select f.bucket into v_bucket from public.files f where f.id = old.file_id; - perform public._delete_storage_objects(v_bucket, old.rel_path); - return old; - elsif tg_op = 'UPDATE' then - if (old.rel_path is distinct from new.rel_path) or (old.file_id is distinct from new.file_id) then - select f.bucket into v_bucket from public.files f where f.id = old.file_id; - perform public._delete_storage_objects(v_bucket, old.rel_path); - end if; - return new; - end if; -end -$$; - -drop trigger if exists trg_document_artefacts_gc on public.document_artefacts; -create trigger trg_document_artefacts_gc -before delete or update of file_id, rel_path on public.document_artefacts -for each row execute function public._artefact_gc_sql(); - - diff --git a/db/migrations/core/66-rls-policies.sql b/db/migrations/core/66-rls-policies.sql deleted file mode 100644 index d76126d..0000000 --- a/db/migrations/core/66-rls-policies.sql +++ /dev/null @@ -1,84 +0,0 @@ --- Enable RLS and define policies for filesystem tables - --- 1) Enable RLS -alter table if exists public.file_cabinets enable row level security; -alter table if exists public.files enable row level security; -alter table if exists public.brain_files enable row level security; -alter table if exists public.document_artefacts enable row level security; - -drop policy if exists "User can access own cabinets" on public.file_cabinets; -create policy "User can access own cabinets" on public.file_cabinets -using (user_id = auth.uid()) -with check (user_id = auth.uid()); - -drop policy if exists "User can access files in own cabinet" on public.files; -create policy "User can access files in own cabinet" on public.files -using (exists ( - select 1 from public.file_cabinets c - where c.id = files.cabinet_id and c.user_id = auth.uid() -)) -with check (exists ( - select 1 from public.file_cabinets c - where c.id = files.cabinet_id and c.user_id = auth.uid() -)); - -drop policy if exists "User can insert files into own cabinet" on public.files; -create policy "User can insert files into own cabinet" on public.files for insert to authenticated -with check (exists ( - select 1 from public.file_cabinets c - where c.id = files.cabinet_id and c.user_id = auth.uid() -)); - -drop policy if exists "User can update files in own cabinet" on public.files; -create policy "User can update files in own cabinet" on public.files for update to authenticated -using (exists ( - select 1 from public.file_cabinets c - where c.id = files.cabinet_id and c.user_id = auth.uid() -)) -with check (exists ( - select 1 from public.file_cabinets c - where c.id = files.cabinet_id and c.user_id = auth.uid() -)); - -drop policy if exists "User can delete files from own cabinet" on public.files; -create policy "User can delete files from own cabinet" on public.files for delete -using (exists ( - select 1 from public.file_cabinets c - where c.id = files.cabinet_id and c.user_id = auth.uid() -)); - --- 4) Brain-files: allow linking owned files to owned brains -drop policy if exists "User can link files they own to their brains" on public.brain_files; -create policy "User can link files they own to their brains" on public.brain_files -using ( - exists (select 1 from public.brains b where b.id = brain_files.brain_id and b.user_id = auth.uid()) - and exists ( - select 1 from public.files f join public.file_cabinets c on f.cabinet_id = c.id - where f.id = brain_files.file_id and c.user_id = auth.uid() - ) -) -with check (true); - --- 5) Document artefacts: allow reads to owners via file cabinet, writes via service_role -drop policy if exists "artefacts_read_by_owner" on public.document_artefacts; -create policy "artefacts_read_by_owner" on public.document_artefacts for select to authenticated -using (exists ( - select 1 from public.files f join public.file_cabinets c on f.cabinet_id = c.id - where f.id = document_artefacts.file_id and c.user_id = auth.uid() -)); - -drop policy if exists "artefacts_rw_service" on public.document_artefacts; -create policy "artefacts_rw_service" on public.document_artefacts to service_role -using (true) with check (true); - --- Allow owners to delete their artefacts (needed for cascades under RLS) -drop policy if exists "artefacts_delete_by_owner" on public.document_artefacts; -create policy "artefacts_delete_by_owner" on public.document_artefacts for delete to authenticated -using (exists ( - select 1 from public.files f join public.file_cabinets c on f.cabinet_id = c.id - where f.id = document_artefacts.file_id and c.user_id = auth.uid() -)); - --- File vectors RLS and policies are defined in 67-vectors.sql after the table is created - - diff --git a/db/migrations/core/67-vectors.sql b/db/migrations/core/67-vectors.sql deleted file mode 100644 index bf9a172..0000000 --- a/db/migrations/core/67-vectors.sql +++ /dev/null @@ -1,79 +0,0 @@ --- Vectors: file_vectors table and similarity search function - --- 1) Ensure pgvector extension is available -create extension if not exists vector; - --- 2) File vectors table -create table if not exists public.file_vectors ( - id bigint generated by default as identity primary key, - created_at timestamp with time zone default now() not null, - embedding public.vector, - metadata jsonb, - content text -); - --- 3) ANN index (skipped until embedding dimension is fixed) --- To enable: set column type to public.vector() and uncomment: --- create index if not exists file_vectors_embedding_idx --- on public.file_vectors using ivfflat (embedding public.vector_cosine_ops) --- with (lists='100'); - --- 3b) Enable RLS and set policies (moved here to avoid ordering issues) -alter table if exists public.file_vectors enable row level security; - -drop policy if exists "vectors_read_by_owner" on public.file_vectors; -create policy "vectors_read_by_owner" on public.file_vectors for select to authenticated -using (coalesce((metadata->>'file_id')::uuid, null) is null or exists ( - select 1 from public.files f join public.file_cabinets c on f.cabinet_id = c.id - where f.id = (metadata->>'file_id')::uuid and c.user_id = auth.uid() -)); - -drop policy if exists "vectors_rw_service" on public.file_vectors; -create policy "vectors_rw_service" on public.file_vectors to service_role -using (true) with check (true); - --- 4) Match function mirrored from neoFS (generic metadata mapping) -create or replace function public.match_file_vectors( - filter jsonb, - match_count integer, - query_embedding public.vector -) -returns table ( - id bigint, - file_id uuid, - cabinet_id uuid, - artefact_type text, - artefact_is text, - original_path_prefix text, - original_filename text, - content text, - metadata jsonb, - similarity double precision -) -language sql stable as $$ - select - fv.id, - nullif(fv.metadata->>'file_id','')::uuid as file_id, - nullif(fv.metadata->>'cabinet_id','')::uuid as cabinet_id, - nullif(fv.metadata->>'artefact_type','') as artefact_type, - nullif(fv.metadata->>'artefact_is','') as artefact_is, - nullif(fv.metadata->>'original_path_prefix','') as original_path_prefix, - nullif(fv.metadata->>'original_filename','') as original_filename, - fv.content, - fv.metadata, - 1 - (fv.embedding <=> query_embedding) as similarity - from public.file_vectors fv - where - (coalesce(filter ? 'file_id', false) = false or (fv.metadata->>'file_id')::uuid = (filter->>'file_id')::uuid) - and (coalesce(filter ? 'cabinet_id', false) = false or (fv.metadata->>'cabinet_id')::uuid = (filter->>'cabinet_id')::uuid) - and (coalesce(filter ? 'artefact_type', false) = false or (fv.metadata->>'artefact_type') = (filter->>'artefact_type')) - and (coalesce(filter ? 'artefact_id', false) = false or (fv.metadata->>'artefact_id') = (filter->>'artefact_id')) - and (coalesce(filter ? 'original_path_prefix', false) = false or (fv.metadata->>'original_path_prefix') like (filter->>'original_path_prefix') || '%') - and (coalesce(filter ? 'original_path_prefix_ilike', false)= false or (fv.metadata->>'original_path_prefix') ilike (filter->>'original_path_prefix_ilike') || '%') - and (coalesce(filter ? 'original_filename', false) = false or (fv.metadata->>'original_filename') = (filter->>'original_filename')) - and (coalesce(filter ? 'original_filename_ilike', false)= false or (fv.metadata->>'original_filename') ilike (filter->>'original_filename_ilike')) - order by fv.embedding <=> query_embedding - limit greatest(coalesce(match_count, 10), 1) -$$; - - diff --git a/db/migrations/core/68-cabinet-memberships.sql b/db/migrations/core/68-cabinet-memberships.sql deleted file mode 100644 index 0d0ef3e..0000000 --- a/db/migrations/core/68-cabinet-memberships.sql +++ /dev/null @@ -1,73 +0,0 @@ --- Cabinet memberships for sharing access - -create table if not exists public.cabinet_memberships ( - id uuid default uuid_generate_v4() primary key, - cabinet_id uuid not null references public.file_cabinets(id) on delete cascade, - profile_id uuid not null references public.profiles(id) on delete cascade, - role text not null check (role in ('owner','editor','viewer')), - created_at timestamp with time zone default timezone('utc'::text, now()), - updated_at timestamp with time zone default timezone('utc'::text, now()), - unique(cabinet_id, profile_id) -); - -create index if not exists idx_cabinet_memberships_cabinet on public.cabinet_memberships(cabinet_id); -create index if not exists idx_cabinet_memberships_profile on public.cabinet_memberships(profile_id); - --- Updated at trigger -drop trigger if exists trg_cabinet_memberships_updated_at on public.cabinet_memberships; -create trigger trg_cabinet_memberships_updated_at - before update on public.cabinet_memberships - for each row execute function public.handle_updated_at(); - --- RLS and policies -alter table if exists public.cabinet_memberships enable row level security; - --- Members can select their own memberships; cabinet owners can also see memberships -drop policy if exists cm_read_self_or_owner on public.cabinet_memberships; -create policy cm_read_self_or_owner on public.cabinet_memberships for select to authenticated -using ( - profile_id = auth.uid() or exists ( - select 1 from public.file_cabinets c where c.id = cabinet_memberships.cabinet_id and c.user_id = auth.uid() - ) -); - --- Cabinet owners can insert memberships -drop policy if exists cm_insert_by_owner on public.cabinet_memberships; -create policy cm_insert_by_owner on public.cabinet_memberships for insert to authenticated -with check (exists ( - select 1 from public.file_cabinets c where c.id = cabinet_memberships.cabinet_id and c.user_id = auth.uid() -)); - --- Cabinet owners can update memberships (e.g., role) -drop policy if exists cm_update_by_owner on public.cabinet_memberships; -create policy cm_update_by_owner on public.cabinet_memberships for update to authenticated -using (exists ( - select 1 from public.file_cabinets c where c.id = cabinet_memberships.cabinet_id and c.user_id = auth.uid() -)) -with check (exists ( - select 1 from public.file_cabinets c where c.id = cabinet_memberships.cabinet_id and c.user_id = auth.uid() -)); - --- Cabinet owners can delete memberships -drop policy if exists cm_delete_by_owner on public.cabinet_memberships; -create policy cm_delete_by_owner on public.cabinet_memberships for delete to authenticated -using (exists ( - select 1 from public.file_cabinets c where c.id = cabinet_memberships.cabinet_id and c.user_id = auth.uid() -)); - --- Extend access to cabinets/files for members (after table exists) -drop policy if exists "User can access cabinets via membership" on public.file_cabinets; -create policy "User can access cabinets via membership" on public.file_cabinets for select to authenticated -using (exists ( - select 1 from public.cabinet_memberships m - where m.cabinet_id = file_cabinets.id and m.profile_id = auth.uid() -)); - -drop policy if exists "User can access files via membership" on public.files; -create policy "User can access files via membership" on public.files for select to authenticated -using (exists ( - select 1 from public.cabinet_memberships m - where m.cabinet_id = files.cabinet_id and m.profile_id = auth.uid() -)); - - diff --git a/db/migrations/core/69-gc-prefix-cleanup.sql b/db/migrations/core/69-gc-prefix-cleanup.sql deleted file mode 100644 index 1f53f1a..0000000 --- a/db/migrations/core/69-gc-prefix-cleanup.sql +++ /dev/null @@ -1,48 +0,0 @@ --- Ensure storage objects for all artefacts are removed when a file is deleted --- by deleting the entire "cabinet_id/file_id" directory prefix in Storage. - --- Helper to delete all objects under a prefix -create or replace function public._delete_storage_prefix(p_bucket text, p_prefix text) -returns void -language plpgsql security definer -set search_path to 'public', 'storage' -as $$ -begin - if p_bucket is null or p_prefix is null then - return; - end if; - -- Delete any objects whose name starts with the prefix + '/' - delete from storage.objects where bucket_id = p_bucket and name like p_prefix || '/%'; - -- In case an object exists exactly at the prefix (rare but safe) - delete from storage.objects where bucket_id = p_bucket and name = p_prefix; -end -$$; - --- Update file-level GC to also delete the parent directory prefix (cabinet_id/file_id) -create or replace function public._storage_gc_sql() -returns trigger -language plpgsql security definer -set search_path to 'public', 'storage' -as $$ -declare - v_prefix text; -begin - -- Derive directory prefix from the file path by removing the last segment (filename) - -- Example: 'cabinet_id/file_id/filename.ext' -> 'cabinet_id/file_id' - v_prefix := regexp_replace(old.path, '/[^/]+$', ''); - - if tg_op = 'DELETE' then - -- Delete the original object and any artefacts under the file's directory - perform public._delete_storage_objects(old.bucket, old.path); - perform public._delete_storage_prefix(old.bucket, v_prefix); - elsif tg_op = 'UPDATE' then - if (old.bucket is distinct from new.bucket) or (old.path is distinct from new.path) then - perform public._delete_storage_objects(old.bucket, old.path); - perform public._delete_storage_prefix(old.bucket, v_prefix); - end if; - end if; - return null; -end -$$; - - diff --git a/db/migrations/core/70_add_directory_support.sql b/db/migrations/core/70_add_directory_support.sql deleted file mode 100644 index 8f42b9b..0000000 --- a/db/migrations/core/70_add_directory_support.sql +++ /dev/null @@ -1,41 +0,0 @@ --- Add directory support to files table --- Migration: Add directory support for folder uploads - --- Add new columns to files table -ALTER TABLE files -ADD COLUMN IF NOT EXISTS is_directory BOOLEAN DEFAULT FALSE, -ADD COLUMN IF NOT EXISTS parent_directory_id UUID REFERENCES files(id) ON DELETE CASCADE, -ADD COLUMN IF NOT EXISTS relative_path TEXT, -ADD COLUMN IF NOT EXISTS directory_manifest JSONB, -ADD COLUMN IF NOT EXISTS upload_session_id UUID, -ADD COLUMN IF NOT EXISTS processing_status TEXT DEFAULT 'uploaded' CHECK (processing_status IN ('uploaded', 'processing', 'completed', 'failed', 'queued')); - --- Create index for efficient directory queries -CREATE INDEX IF NOT EXISTS idx_files_parent_directory ON files(parent_directory_id); -CREATE INDEX IF NOT EXISTS idx_files_upload_session ON files(upload_session_id); -CREATE INDEX IF NOT EXISTS idx_files_processing_status ON files(processing_status); -CREATE INDEX IF NOT EXISTS idx_files_is_directory ON files(is_directory); - --- Create directory manifest structure -COMMENT ON COLUMN files.is_directory IS 'True if this record represents a directory/folder'; -COMMENT ON COLUMN files.parent_directory_id IS 'ID of parent directory if this file is inside an uploaded folder'; -COMMENT ON COLUMN files.relative_path IS 'Relative path within the uploaded directory structure'; -COMMENT ON COLUMN files.directory_manifest IS 'JSON manifest of directory contents including file count, total size, structure'; -COMMENT ON COLUMN files.upload_session_id IS 'Groups files uploaded together in a single directory upload session'; -COMMENT ON COLUMN files.processing_status IS 'Simple status tracking without auto-processing'; - --- Example directory_manifest structure: --- { --- "total_files": 15, --- "total_size_bytes": 12345678, --- "directory_structure": { --- "documents/": { --- "file1.pdf": {"size": 123456, "mime_type": "application/pdf"}, --- "subdirectory/": { --- "file2.docx": {"size": 234567, "mime_type": "application/vnd.openxmlformats-officedocument.wordprocessingml.document"} --- } --- } --- }, --- "upload_timestamp": "2024-09-23T12:00:00Z", --- "upload_method": "directory_picker" --- } diff --git a/db/migrations/supabase/50-_supabase.sql b/db/migrations/supabase/50-_supabase.sql deleted file mode 100644 index 0871c34..0000000 --- a/db/migrations/supabase/50-_supabase.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Create _supabase database for internal Supabase operations --- This database is created automatically by Supabase's internal setup --- This file is kept for reference but the actual database creation is managed --- through the Supabase configuration and environment variables - --- Note: The _supabase database is created with the postgres user as owner --- by default during Supabase initialization diff --git a/db/migrations/supabase/52-logs.sql b/db/migrations/supabase/52-logs.sql deleted file mode 100644 index 349ccca..0000000 --- a/db/migrations/supabase/52-logs.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Create _analytics schema for Supabase analytics --- This schema is created automatically by Supabase's internal setup --- This file is kept for reference but the actual schema creation is managed --- through the Supabase configuration and environment variables - --- Note: The _analytics schema is created in the _supabase database --- with appropriate ownership during Supabase initialization diff --git a/db/migrations/supabase/52-pooler.sql b/db/migrations/supabase/52-pooler.sql deleted file mode 100644 index 2c3c8bb..0000000 --- a/db/migrations/supabase/52-pooler.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Create _supavisor schema for Supabase connection pooling --- This schema is created automatically by Supabase's internal setup --- This file is kept for reference but the actual schema creation is managed --- through the Supabase configuration and environment variables - --- Note: The _supavisor schema is created in the _supabase database --- with appropriate ownership during Supabase initialization diff --git a/db/migrations/supabase/52-realtime.sql b/db/migrations/supabase/52-realtime.sql deleted file mode 100644 index 7ab2d94..0000000 --- a/db/migrations/supabase/52-realtime.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Create _realtime schema for Supabase realtime functionality --- This schema is created automatically by Supabase's internal setup --- This file is kept for reference but the actual schema creation is managed --- through the Supabase configuration and environment variables - --- Note: The _realtime schema is created with appropriate ownership --- during Supabase initialization diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..5363f6f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,479 @@ +# Usage +# Start: docker compose up +# With helpers: docker compose -f docker-compose.yml -f ./dev/docker-compose.dev.yml up +# Stop: docker compose down +# Destroy: docker compose -f docker-compose.yml -f ./dev/docker-compose.dev.yml down -v --remove-orphans +# Reset everything: ./reset.sh + +name: supabase + +services: + + studio: + container_name: supabase-studio + image: supabase/studio:2025.06.30-sha-6f5982d + restart: unless-stopped + ports: + - ${STUDIO_PORT}:3000 + healthcheck: + test: [ "CMD", "node", "-e", "fetch('http://studio:3000/api/platform/profile').then((r) => {if (r.status !== 200) throw new Error(r.status)})" ] + timeout: 10s + interval: 5s + retries: 3 + depends_on: + analytics: + condition: service_healthy + environment: + STUDIO_PG_META_URL: http://meta:8080 + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + + DEFAULT_ORGANIZATION_NAME: ${STUDIO_DEFAULT_ORGANIZATION} + DEFAULT_PROJECT_NAME: ${STUDIO_DEFAULT_PROJECT} + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + + SUPABASE_URL: http://kong:8000 + SUPABASE_PUBLIC_URL: ${SUPABASE_PUBLIC_URL} + SUPABASE_ANON_KEY: ${ANON_KEY} + SUPABASE_SERVICE_KEY: ${SERVICE_ROLE_KEY} + AUTH_JWT_SECRET: ${JWT_SECRET} + + LOGFLARE_PRIVATE_ACCESS_TOKEN: ${LOGFLARE_PRIVATE_ACCESS_TOKEN} + LOGFLARE_URL: http://analytics:4000 + NEXT_PUBLIC_ENABLE_LOGS: true + # Comment to use Big Query backend for analytics + NEXT_ANALYTICS_BACKEND_PROVIDER: postgres + # Uncomment to use Big Query backend for analytics + # NEXT_ANALYTICS_BACKEND_PROVIDER: bigquery + + kong: + container_name: supabase-kong + image: kong:2.8.1 + restart: unless-stopped + ports: + - ${KONG_HTTP_PORT}:8000/tcp + - ${KONG_HTTPS_PORT}:8443/tcp + volumes: + # https://github.com/supabase/supabase/issues/12661 + - ./volumes/api/kong.yml:/home/kong/temp.yml:ro,z + depends_on: + analytics: + condition: service_healthy + environment: + KONG_DATABASE: "off" + KONG_DECLARATIVE_CONFIG: /home/kong/kong.yml + # https://github.com/supabase/cli/issues/14 + KONG_DNS_ORDER: LAST,A,CNAME + KONG_PLUGINS: request-transformer,cors,key-auth,acl,basic-auth + KONG_NGINX_PROXY_PROXY_BUFFER_SIZE: 160k + KONG_NGINX_PROXY_PROXY_BUFFERS: 64 160k + SUPABASE_ANON_KEY: ${ANON_KEY} + SUPABASE_SERVICE_KEY: ${SERVICE_ROLE_KEY} + DASHBOARD_USERNAME: ${DASHBOARD_USERNAME} + DASHBOARD_PASSWORD: ${DASHBOARD_PASSWORD} + # https://unix.stackexchange.com/a/294837 + entrypoint: bash -c 'eval "echo \"$$(cat ~/temp.yml)\"" > ~/kong.yml && /docker-entrypoint.sh kong docker-start' + + auth: + container_name: supabase-auth + image: supabase/gotrue:v2.177.0 + restart: unless-stopped + healthcheck: + test: [ "CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:9999/health" ] + timeout: 5s + interval: 5s + retries: 3 + depends_on: + db: + # Disable this if you are using an external Postgres database + condition: service_healthy + analytics: + condition: service_healthy + environment: + GOTRUE_API_HOST: 0.0.0.0 + GOTRUE_API_PORT: 9999 + API_EXTERNAL_URL: ${API_EXTERNAL_URL} + + GOTRUE_DB_DRIVER: postgres + GOTRUE_DB_DATABASE_URL: postgres://supabase_auth_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} + + GOTRUE_SITE_URL: ${SITE_URL} + GOTRUE_URI_ALLOW_LIST: ${ADDITIONAL_REDIRECT_URLS} + GOTRUE_DISABLE_SIGNUP: ${DISABLE_SIGNUP} + + GOTRUE_JWT_ADMIN_ROLES: service_role + GOTRUE_JWT_AUD: authenticated + GOTRUE_JWT_DEFAULT_GROUP_NAME: authenticated + GOTRUE_JWT_EXP: ${JWT_EXPIRY} + GOTRUE_JWT_SECRET: ${JWT_SECRET} + + GOTRUE_EXTERNAL_EMAIL_ENABLED: ${ENABLE_EMAIL_SIGNUP} + GOTRUE_EXTERNAL_ANONYMOUS_USERS_ENABLED: ${ENABLE_ANONYMOUS_USERS} + GOTRUE_MAILER_AUTOCONFIRM: ${ENABLE_EMAIL_AUTOCONFIRM} + + # Uncomment to bypass nonce check in ID Token flow. Commonly set to true when using Google Sign In on mobile. + # GOTRUE_EXTERNAL_SKIP_NONCE_CHECK: true + + # GOTRUE_MAILER_SECURE_EMAIL_CHANGE_ENABLED: true + # GOTRUE_SMTP_MAX_FREQUENCY: 1s + GOTRUE_SMTP_ADMIN_EMAIL: ${SMTP_ADMIN_EMAIL} + GOTRUE_SMTP_HOST: ${SMTP_HOST} + GOTRUE_SMTP_PORT: ${SMTP_PORT} + GOTRUE_SMTP_USER: ${SMTP_USER} + GOTRUE_SMTP_PASS: ${SMTP_PASS} + GOTRUE_SMTP_SENDER_NAME: ${SMTP_SENDER_NAME} + GOTRUE_MAILER_URLPATHS_INVITE: ${MAILER_URLPATHS_INVITE} + GOTRUE_MAILER_URLPATHS_CONFIRMATION: ${MAILER_URLPATHS_CONFIRMATION} + GOTRUE_MAILER_URLPATHS_RECOVERY: ${MAILER_URLPATHS_RECOVERY} + GOTRUE_MAILER_URLPATHS_EMAIL_CHANGE: ${MAILER_URLPATHS_EMAIL_CHANGE} + + GOTRUE_EXTERNAL_PHONE_ENABLED: ${ENABLE_PHONE_SIGNUP} + GOTRUE_SMS_AUTOCONFIRM: ${ENABLE_PHONE_AUTOCONFIRM} + # Uncomment to enable custom access token hook. Please see: https://supabase.com/docs/guides/auth/auth-hooks for full list of hooks and additional details about custom_access_token_hook + + # GOTRUE_HOOK_CUSTOM_ACCESS_TOKEN_ENABLED: "true" + # GOTRUE_HOOK_CUSTOM_ACCESS_TOKEN_URI: "pg-functions://postgres/public/custom_access_token_hook" + # GOTRUE_HOOK_CUSTOM_ACCESS_TOKEN_SECRETS: "" + + # GOTRUE_HOOK_MFA_VERIFICATION_ATTEMPT_ENABLED: "true" + # GOTRUE_HOOK_MFA_VERIFICATION_ATTEMPT_URI: "pg-functions://postgres/public/mfa_verification_attempt" + + # GOTRUE_HOOK_PASSWORD_VERIFICATION_ATTEMPT_ENABLED: "true" + # GOTRUE_HOOK_PASSWORD_VERIFICATION_ATTEMPT_URI: "pg-functions://postgres/public/password_verification_attempt" + + # GOTRUE_HOOK_SEND_SMS_ENABLED: "false" + # GOTRUE_HOOK_SEND_SMS_URI: "pg-functions://postgres/public/custom_access_token_hook" + # GOTRUE_HOOK_SEND_SMS_SECRETS: "v1,whsec_VGhpcyBpcyBhbiBleGFtcGxlIG9mIGEgc2hvcnRlciBCYXNlNjQgc3RyaW5n" + + # GOTRUE_HOOK_SEND_EMAIL_ENABLED: "false" + # GOTRUE_HOOK_SEND_EMAIL_URI: "http://host.docker.internal:54321/functions/v1/email_sender" + # GOTRUE_HOOK_SEND_EMAIL_SECRETS: "v1,whsec_VGhpcyBpcyBhbiBleGFtcGxlIG9mIGEgc2hvcnRlciBCYXNlNjQgc3RyaW5n" + + rest: + container_name: supabase-rest + image: postgrest/postgrest:v12.2.12 + restart: unless-stopped + depends_on: + db: + # Disable this if you are using an external Postgres database + condition: service_healthy + analytics: + condition: service_healthy + environment: + PGRST_DB_URI: postgres://authenticator:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} + PGRST_DB_SCHEMAS: ${PGRST_DB_SCHEMAS} + PGRST_DB_ANON_ROLE: anon + PGRST_JWT_SECRET: ${JWT_SECRET} + PGRST_DB_USE_LEGACY_GUCS: "false" + PGRST_APP_SETTINGS_JWT_SECRET: ${JWT_SECRET} + PGRST_APP_SETTINGS_JWT_EXP: ${JWT_EXPIRY} + command: [ "postgrest" ] + + realtime: + # This container name looks inconsistent but is correct because realtime constructs tenant id by parsing the subdomain + container_name: realtime-dev.supabase-realtime + image: supabase/realtime:v2.34.47 + restart: unless-stopped + depends_on: + db: + # Disable this if you are using an external Postgres database + condition: service_healthy + analytics: + condition: service_healthy + healthcheck: + test: [ "CMD", "curl", "-sSfL", "--head", "-o", "/dev/null", "-H", "Authorization: Bearer ${ANON_KEY}", "http://localhost:4000/api/tenants/realtime-dev/health" ] + timeout: 5s + interval: 5s + retries: 3 + environment: + PORT: 4000 + DB_HOST: ${POSTGRES_HOST} + DB_PORT: ${POSTGRES_PORT} + DB_USER: supabase_admin + DB_PASSWORD: ${POSTGRES_PASSWORD} + DB_NAME: ${POSTGRES_DB} + DB_AFTER_CONNECT_QUERY: 'SET search_path TO _realtime' + DB_ENC_KEY: supabaserealtime + API_JWT_SECRET: ${JWT_SECRET} + SECRET_KEY_BASE: ${SECRET_KEY_BASE} + ERL_AFLAGS: -proto_dist inet_tcp + DNS_NODES: "''" + RLIMIT_NOFILE: "10000" + APP_NAME: realtime + SEED_SELF_HOST: true + RUN_JANITOR: true + + # To use S3 backed storage: docker compose -f docker-compose.yml -f docker-compose.s3.yml up + storage: + container_name: supabase-storage + image: supabase/storage-api:v1.25.7 + restart: unless-stopped + volumes: + - ./volumes/storage:/var/lib/storage:z + healthcheck: + test: [ "CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://storage:5000/status" ] + timeout: 5s + interval: 5s + retries: 3 + depends_on: + db: + # Disable this if you are using an external Postgres database + condition: service_healthy + rest: + condition: service_started + imgproxy: + condition: service_started + environment: + ANON_KEY: ${ANON_KEY} + SERVICE_KEY: ${SERVICE_ROLE_KEY} + POSTGREST_URL: http://rest:3000 + PGRST_JWT_SECRET: ${JWT_SECRET} + DATABASE_URL: postgres://supabase_storage_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} + FILE_SIZE_LIMIT: 2147483648 + STORAGE_BACKEND: file + FILE_STORAGE_BACKEND_PATH: /var/lib/storage + TENANT_ID: stub + # TODO: https://github.com/supabase/storage-api/issues/55 + REGION: stub + GLOBAL_S3_BUCKET: stub + ENABLE_IMAGE_TRANSFORMATION: "true" + IMGPROXY_URL: http://imgproxy:5001 + + imgproxy: + container_name: supabase-imgproxy + image: darthsim/imgproxy:v3.8.0 + restart: unless-stopped + volumes: + - ./volumes/storage:/var/lib/storage:z + healthcheck: + test: [ "CMD", "imgproxy", "health" ] + timeout: 5s + interval: 5s + retries: 3 + environment: + IMGPROXY_BIND: ":5001" + IMGPROXY_LOCAL_FILESYSTEM_ROOT: / + IMGPROXY_USE_ETAG: "true" + IMGPROXY_ENABLE_WEBP_DETECTION: ${IMGPROXY_ENABLE_WEBP_DETECTION} + + meta: + container_name: supabase-meta + image: supabase/postgres-meta:v0.91.0 + restart: unless-stopped + depends_on: + db: + # Disable this if you are using an external Postgres database + condition: service_healthy + analytics: + condition: service_healthy + environment: + PG_META_PORT: 8080 + PG_META_DB_HOST: ${POSTGRES_HOST} + PG_META_DB_PORT: ${POSTGRES_PORT} + PG_META_DB_NAME: ${POSTGRES_DB} + PG_META_DB_USER: supabase_admin + PG_META_DB_PASSWORD: ${POSTGRES_PASSWORD} + + functions: + container_name: supabase-edge-functions + image: supabase/edge-runtime:v1.67.4 + restart: unless-stopped + volumes: + - ./volumes/functions:/home/deno/functions:Z + depends_on: + analytics: + condition: service_healthy + environment: + JWT_SECRET: ${JWT_SECRET} + SUPABASE_URL: http://kong:8000 + SUPABASE_ANON_KEY: ${ANON_KEY} + SUPABASE_SERVICE_ROLE_KEY: ${SERVICE_ROLE_KEY} + SUPABASE_DB_URL: postgresql://postgres:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} + # TODO: Allow configuring VERIFY_JWT per function. This PR might help: https://github.com/supabase/cli/pull/786 + VERIFY_JWT: "${FUNCTIONS_VERIFY_JWT}" + command: [ "start", "--main-service", "/home/deno/functions/main" ] + + analytics: + container_name: supabase-analytics + image: supabase/logflare:1.14.2 + restart: unless-stopped + ports: + - 4000:4000 + # Uncomment to use Big Query backend for analytics + # volumes: + # - type: bind + # source: ${PWD}/gcloud.json + # target: /opt/app/rel/logflare/bin/gcloud.json + # read_only: true + healthcheck: + test: [ "CMD", "curl", "http://localhost:4000/health" ] + timeout: 5s + interval: 5s + retries: 10 + depends_on: + db: + # Disable this if you are using an external Postgres database + condition: service_healthy + environment: + LOGFLARE_NODE_HOST: 127.0.0.1 + DB_USERNAME: supabase_admin + DB_DATABASE: _supabase + DB_HOSTNAME: ${POSTGRES_HOST} + DB_PORT: ${POSTGRES_PORT} + DB_PASSWORD: ${POSTGRES_PASSWORD} + DB_SCHEMA: _analytics + LOGFLARE_PUBLIC_ACCESS_TOKEN: ${LOGFLARE_PUBLIC_ACCESS_TOKEN} + LOGFLARE_API_KEY: ${LOGFLARE_API_KEY} + LOGFLARE_PRIVATE_ACCESS_TOKEN: ${LOGFLARE_PRIVATE_ACCESS_TOKEN} + LOGFLARE_SINGLE_TENANT: true + LOGFLARE_SUPABASE_MODE: true + LOGFLARE_MIN_CLUSTER_SIZE: 1 + + # Comment variables to use Big Query backend for analytics + POSTGRES_BACKEND_URL: postgresql://supabase_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/_supabase + POSTGRES_BACKEND_SCHEMA: _analytics + LOGFLARE_FEATURE_FLAG_OVERRIDE: multibackend=true + # Uncomment to use Big Query backend for analytics + # GOOGLE_PROJECT_ID: ${GOOGLE_PROJECT_ID} + # GOOGLE_PROJECT_NUMBER: ${GOOGLE_PROJECT_NUMBER} + + # Comment out everything below this point if you are using an external Postgres database + db: + container_name: supabase-db + image: supabase/postgres:15.8.1.060 + restart: unless-stopped + volumes: + - ./volumes/db/realtime.sql:/docker-entrypoint-initdb.d/migrations/99-realtime.sql:Z + # Must be superuser to create event trigger + - ./volumes/db/webhooks.sql:/docker-entrypoint-initdb.d/init-scripts/98-webhooks.sql:Z + # Must be superuser to alter reserved role + - ./volumes/db/roles.sql:/docker-entrypoint-initdb.d/init-scripts/99-roles.sql:Z + # Initialize the database settings with JWT_SECRET and JWT_EXP + - ./volumes/db/jwt.sql:/docker-entrypoint-initdb.d/init-scripts/99-jwt.sql:Z + # PGDATA directory is persisted between restarts + - ./volumes/db/data:/var/lib/postgresql/data:Z + # Changes required for internal supabase data such as _analytics + - ./volumes/db/_supabase.sql:/docker-entrypoint-initdb.d/migrations/97-_supabase.sql:Z + # Changes required for Analytics support + - ./volumes/db/logs.sql:/docker-entrypoint-initdb.d/migrations/99-logs.sql:Z + # Changes required for Pooler support + - ./volumes/db/pooler.sql:/docker-entrypoint-initdb.d/migrations/99-pooler.sql:Z + # Use named volume to persist pgsodium decryption key between restarts + - db-config:/etc/postgresql-custom + healthcheck: + test: [ "CMD", "pg_isready", "-U", "postgres", "-h", "localhost" ] + interval: 5s + timeout: 5s + retries: 10 + depends_on: + vector: + condition: service_healthy + environment: + POSTGRES_HOST: /var/run/postgresql + PGPORT: ${POSTGRES_PORT} + POSTGRES_PORT: ${POSTGRES_PORT} + PGPASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + PGDATABASE: ${POSTGRES_DB} + POSTGRES_DB: ${POSTGRES_DB} + JWT_SECRET: ${JWT_SECRET} + JWT_EXP: ${JWT_EXPIRY} + command: + [ + "postgres", + "-c", + "config_file=/etc/postgresql/postgresql.conf", + "-c", + "log_min_messages=fatal" # prevents Realtime polling queries from appearing in logs + ] + ports: + - ${PORT_SUPABASE_POSTGRES_TEST}:${POSTGRES_PORT} + + vector: + container_name: supabase-vector + image: timberio/vector:0.28.1-alpine + restart: unless-stopped + volumes: + - ./volumes/logs/vector.yml:/etc/vector/vector.yml:ro,z + - ${DOCKER_SOCKET_LOCATION}:/var/run/docker.sock:ro,z + healthcheck: + test: [ "CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://vector:9001/health" ] + timeout: 5s + interval: 5s + retries: 3 + environment: + LOGFLARE_PUBLIC_ACCESS_TOKEN: ${LOGFLARE_PUBLIC_ACCESS_TOKEN} + LOGFLARE_API_KEY: ${LOGFLARE_API_KEY} + command: [ "--config", "/etc/vector/vector.yml" ] + security_opt: + - "label=disable" + + # Update the DATABASE_URL if you are using an external Postgres database + supavisor: + container_name: supabase-pooler + image: supabase/supavisor:2.5.7 + restart: unless-stopped + ports: + - ${POSTGRES_PORT}:5432 + - ${POOLER_PROXY_PORT_TRANSACTION}:6543 + volumes: + - ./volumes/pooler/pooler.exs:/etc/pooler/pooler.exs:ro,z + healthcheck: + test: [ "CMD", "curl", "-sSfL", "--head", "-o", "/dev/null", "http://127.0.0.1:4000/api/health" ] + interval: 10s + timeout: 5s + retries: 5 + depends_on: + db: + condition: service_healthy + analytics: + condition: service_healthy + environment: + PORT: 4000 + POSTGRES_PORT: ${POSTGRES_PORT} + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + DATABASE_URL: ecto://supabase_admin:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/_supabase + CLUSTER_POSTGRES: true + SECRET_KEY_BASE: ${SECRET_KEY_BASE} + VAULT_ENC_KEY: ${VAULT_ENC_KEY} + API_JWT_SECRET: ${JWT_SECRET} + METRICS_JWT_SECRET: ${JWT_SECRET} + REGION: local + ERL_AFLAGS: -proto_dist inet_tcp + POOLER_TENANT_ID: ${POOLER_TENANT_ID} + POOLER_DEFAULT_POOL_SIZE: ${POOLER_DEFAULT_POOL_SIZE} + POOLER_MAX_CLIENT_CONN: ${POOLER_MAX_CLIENT_CONN} + POOLER_POOL_MODE: transaction + DB_POOL_SIZE: ${POOLER_DB_POOL_SIZE} + command: [ "/bin/sh", "-c", "/app/bin/migrate && /app/bin/supavisor eval \"$$(cat /etc/pooler/pooler.exs)\" && /app/bin/server" ] + + ## MCP Server - Model Context Protocol for AI integrations + ## DISABLED BY DEFAULT - Add 'mcp' to COMPOSE_PROFILES to enable + mcp: + container_name: mcp + profiles: + - mcp + build: + context: . + dockerfile: ./volumes/mcp/Dockerfile + restart: unless-stopped + healthcheck: + test: [ "CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://127.0.0.1:3100/health" ] + timeout: 5s + interval: 10s + retries: 3 + depends_on: + db: + condition: service_healthy + rest: + condition: service_started + environment: + SUPABASE_URL: http://kong:8000 + SUPABASE_ANON_KEY: ${ANON_KEY} + SUPABASE_SERVICE_ROLE_KEY: ${SERVICE_ROLE_KEY} + SUPABASE_AUTH_JWT_SECRET: ${JWT_SECRET} + DATABASE_URL: postgresql://postgres:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} + command: [ "bun", "run", "dist/index.js", "--transport", "http", "--port", "3100", "--host", "0.0.0.0", "--url", "http://kong:8000", "--anon-key", "${ANON_KEY}", "--service-key", "${SERVICE_ROLE_KEY}", "--jwt-secret", "${JWT_SECRET}", "--db-url", "postgresql://postgres:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}" ] + +volumes: + db-config: diff --git a/functions/FUNCTIONS_README.md b/functions/FUNCTIONS_README.md deleted file mode 100644 index 28f5c2c..0000000 --- a/functions/FUNCTIONS_README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Supabase Edge Functions - -This document describes the available Edge Functions in this self-hosted Supabase instance. - -## institute-geocoder - -Institute address geocoding using SearXNG/OpenStreetMap - -**Endpoints:** -- `/functions/v1/institute-geocoder` -- `/functions/v1/institute-geocoder/batch` - -**Usage:** POST with institute_id and optional address data - -**Dependencies:** SearXNG service, OpenStreetMap data - diff --git a/functions/hello/index.ts b/functions/hello/index.ts deleted file mode 100644 index f1e20b9..0000000 --- a/functions/hello/index.ts +++ /dev/null @@ -1,16 +0,0 @@ -// Follow this setup guide to integrate the Deno language server with your editor: -// https://deno.land/manual/getting_started/setup_your_environment -// This enables autocomplete, go to definition, etc. - -import { serve } from "https://deno.land/std@0.177.1/http/server.ts" - -serve(async () => { - return new Response( - `"Hello from Edge Functions!"`, - { headers: { "Content-Type": "application/json" } }, - ) -}) - -// To invoke: -// curl 'http://localhost:/functions/v1/hello' \ -// --header 'Authorization: Bearer ' diff --git a/functions/institute-geocoder-batch/index.ts b/functions/institute-geocoder-batch/index.ts deleted file mode 100644 index 638eca9..0000000 --- a/functions/institute-geocoder-batch/index.ts +++ /dev/null @@ -1,391 +0,0 @@ -import { serve } from 'https://deno.land/std@0.131.0/http/server.ts' -import { createClient } from 'https://esm.sh/@supabase/supabase-js@2' - -const corsHeaders = { - 'Access-Control-Allow-Origin': '*', - 'Access-Control-Allow-Headers': 'authorization, x-client-info, apikey, content-type', -} - -interface BatchGeocodingRequest { - limit?: number - force_refresh?: boolean - institute_ids?: string[] -} - -interface GeocodingResult { - institute_id: string - success: boolean - message: string - coordinates?: { - latitude: number - longitude: number - boundingbox: string[] - geojson?: any - osm?: any - } - error?: string -} - -serve(async (req: Request) => { - // Handle CORS preflight requests - if (req.method === 'OPTIONS') { - return new Response('ok', { headers: corsHeaders }) - } - - try { - // Get environment variables - const supabaseUrl = Deno.env.get('SUPABASE_URL') - const supabaseServiceKey = Deno.env.get('SUPABASE_SERVICE_ROLE_KEY') - const searxngUrl = Deno.env.get('SEARXNG_URL') || 'https://search.kevlarai.com' - - if (!supabaseUrl || !supabaseServiceKey) { - throw new Error('Missing required environment variables') - } - - // Create Supabase client - const supabase = createClient(supabaseUrl, supabaseServiceKey) - - // Parse request body - const body: BatchGeocodingRequest = await req.json() - const limit = body.limit || 10 - const forceRefresh = body.force_refresh || false - - // Get institutes that need geocoding - let query = supabase - .from('institutes') - .select('id, name, address, geo_coordinates') - .not('import_id', 'is', null) - - if (!forceRefresh) { - // Only get institutes without coordinates or with empty coordinates - query = query.or('geo_coordinates.is.null,geo_coordinates.eq.{}') - } - - if (body.institute_ids && body.institute_ids.length > 0) { - query = query.in('id', body.institute_ids) - } - - const { data: institutes, error: fetchError } = await query.limit(limit) - - if (fetchError) { - throw new Error(`Failed to fetch institutes: ${fetchError.message}`) - } - - if (!institutes || institutes.length === 0) { - return new Response( - JSON.stringify({ - success: true, - message: 'No institutes found that need geocoding', - processed: 0 - }), - { - status: 200, - headers: { ...corsHeaders, 'Content-Type': 'application/json' } - } - ) - } - - console.log(`Processing ${institutes.length} institutes for geocoding`) - - const results: GeocodingResult[] = [] - let successCount = 0 - let errorCount = 0 - - // Process institutes sequentially to avoid overwhelming the SearXNG service - let processedCount = 0 - for (const institute of institutes) { - try { - const address = institute.address as any - if (!address) { - results.push({ - institute_id: institute.id, - success: false, - message: 'No address information available', - error: 'Missing address data' - }) - errorCount++ - processedCount++ - continue - } - - // Build search query from address components - const addressParts = [ - address.street, - address.town, - address.county, - address.postcode, - address.country - ].filter(Boolean) - - if (addressParts.length === 0) { - results.push({ - institute_id: institute.id, - success: false, - message: 'No valid address components found', - error: 'Empty address parts' - }) - errorCount++ - processedCount++ - continue - } - - const searchQuery = addressParts.join(', ') - console.log(`Geocoding institute ${institute.id}: ${searchQuery}`) - - // Query SearXNG for geocoding with fallback strategy - const geocodingResult = await geocodeAddressWithFallback(address, searxngUrl) - - if (geocodingResult.success && geocodingResult.coordinates) { - // Update institute with geospatial coordinates - const { error: updateError } = await supabase - .from('institutes') - .update({ - geo_coordinates: { - latitude: geocodingResult.coordinates.latitude, - longitude: geocodingResult.coordinates.longitude, - boundingbox: geocodingResult.coordinates.boundingbox, - geojson: geocodingResult.coordinates.geojson, - osm: geocodingResult.coordinates.osm, - search_query: searchQuery, - geocoded_at: new Date().toISOString() - } - }) - .eq('id', institute.id) - - if (updateError) { - throw new Error(`Failed to update institute: ${updateError.message}`) - } - - results.push({ - institute_id: institute.id, - success: true, - message: 'Successfully geocoded', - coordinates: geocodingResult.coordinates - }) - successCount++ - - // Log the successful geocoding - await supabase - .from('function_logs') - .insert({ - file_id: null, - step: 'batch_geocoding', - message: 'Successfully geocoded institute address in batch', - data: { - institute_id: institute.id, - search_query: searchQuery, - coordinates: geocodingResult.coordinates - } - }) - - } else { - results.push({ - institute_id: institute.id, - success: false, - message: 'Geocoding failed', - error: geocodingResult.error || 'Unknown error' - }) - errorCount++ - } - - processedCount++ - - // Add a small delay between requests to be respectful to the SearXNG service - // Optimize delay based on batch size for better performance - if (processedCount < institutes.length) { // Don't delay after the last institute - const delay = institutes.length > 200 ? 50 : 100; // Faster processing for large batches - await new Promise(resolve => setTimeout(resolve, delay)) - } - - } catch (error) { - console.error(`Error processing institute ${institute.id}:`, error) - results.push({ - institute_id: institute.id, - success: false, - message: 'Processing error', - error: error.message - }) - errorCount++ - } - } - - // Log the batch operation - await supabase - .from('function_logs') - .insert({ - file_id: null, - step: 'batch_geocoding_complete', - message: 'Batch geocoding operation completed', - data: { - total_processed: institutes.length, - successful: successCount, - failed: errorCount, - results: results - } - }) - - return new Response( - JSON.stringify({ - success: true, - message: 'Batch geocoding completed', - summary: { - total_processed: institutes.length, - successful: successCount, - failed: errorCount - }, - results: results - }), - { - status: 200, - headers: { ...corsHeaders, 'Content-Type': 'application/json' } - } - ) - - } catch (error) { - console.error('Error in batch institute geocoder:', error) - - return new Response( - JSON.stringify({ - error: 'Internal server error', - details: error.message - }), - { - status: 500, - headers: { ...corsHeaders, 'Content-Type': 'application/json' } - } - ) - } -}) - -async function geocodeAddress(searchQuery: string, searxngUrl: string): Promise<{ - success: boolean - coordinates?: { - latitude: number - longitude: number - boundingbox: string[] - geojson?: any - osm?: any - } - error?: string -}> { - try { - // Format search query for OSM - const osmQuery = `!osm ${searchQuery}` - const searchUrl = `${searxngUrl}/search?q=${encodeURIComponent(osmQuery)}&format=json` - - const response = await fetch(searchUrl, { - method: 'GET', - headers: { - 'Accept': 'application/json', - 'User-Agent': 'ClassroomCopilot-BatchGeocoder/1.0' - } - }) - - if (!response.ok) { - throw new Error(`SearXNG request failed: ${response.status} ${response.statusText}`) - } - - const data = await response.json() - - // Check if we have results - the number_of_results field might be unreliable - // so we check the results array directly - if (!data.results || data.results.length === 0) { - return { - success: false, - error: 'No results returned from SearXNG' - } - } - - const result = data.results[0] - - if (!result.latitude || !result.longitude) { - return { - success: false, - error: 'Missing latitude or longitude in SearXNG response' - } - } - - return { - success: true, - coordinates: { - latitude: parseFloat(result.latitude), - longitude: parseFloat(result.longitude), - boundingbox: result.boundingbox || [], - geojson: result.geojson, - osm: result.osm - } - } - - } catch (error) { - console.error('Geocoding error:', error) - return { - success: false, - error: error.message - } - } -} - -async function geocodeAddressWithFallback(address: any, searxngUrl: string): Promise<{ - success: boolean - coordinates?: { - latitude: number - longitude: number - boundingbox: string[] - geojson?: any - osm?: any - } - error?: string -}> { - // Strategy 1: Try full address (street + town + county + postcode) - if (address.street && address.town && address.county && address.postcode) { - const fullQuery = `${address.street}, ${address.town}, ${address.county}, ${address.postcode}` - console.log(`Trying full address: ${fullQuery}`) - - const result = await geocodeAddress(fullQuery, searxngUrl) - if (result.success && result.coordinates) { - console.log('Full address geocoding successful') - return result - } - } - - // Strategy 2: Try town + county + postcode - if (address.town && address.county && address.postcode) { - const mediumQuery = `${address.town}, ${address.county}, ${address.postcode}` - console.log(`Trying medium address: ${mediumQuery}`) - - const result = await geocodeAddress(mediumQuery, searxngUrl) - if (result.success && result.coordinates) { - console.log('Medium address geocoding successful') - return result - } - } - - // Strategy 3: Try just postcode - if (address.postcode) { - console.log(`Trying postcode only: ${address.postcode}`) - - const result = await geocodeAddress(address.postcode, searxngUrl) - if (result.success && result.coordinates) { - console.log('Postcode geocoding successful') - return result - } - } - - // Strategy 4: Try town + postcode - if (address.town && address.postcode) { - const simpleQuery = `${address.town}, ${address.postcode}` - console.log(`Trying simple address: ${simpleQuery}`) - - const result = await geocodeAddress(simpleQuery, searxngUrl) - if (result.success && result.coordinates) { - console.log('Simple address geocoding successful') - return result - } - } - - // All strategies failed - return { - success: false, - error: 'No coordinates found with any address combination' - } -} diff --git a/functions/institute-geocoder/batch.ts b/functions/institute-geocoder/batch.ts deleted file mode 100644 index e797a05..0000000 --- a/functions/institute-geocoder/batch.ts +++ /dev/null @@ -1,317 +0,0 @@ -import { serve } from 'https://deno.land/std@0.131.0/http/server.ts' -import { createClient } from 'https://esm.sh/@supabase/supabase-js@2' - -const corsHeaders = { - 'Access-Control-Allow-Origin': '*', - 'Access-Control-Allow-Headers': 'authorization, x-client-info, apikey, content-type', -} - -interface BatchGeocodingRequest { - limit?: number - force_refresh?: boolean - institute_ids?: string[] -} - -interface GeocodingResult { - institute_id: string - success: boolean - message: string - coordinates?: { - latitude: number - longitude: number - boundingbox: string[] - geojson?: any - osm?: any - } - error?: string -} - -serve(async (req: Request) => { - // Handle CORS preflight requests - if (req.method === 'OPTIONS') { - return new Response('ok', { headers: corsHeaders }) - } - - try { - // Get environment variables - const supabaseUrl = Deno.env.get('SUPABASE_URL') - const supabaseServiceKey = Deno.env.get('SUPABASE_SERVICE_RATE_KEY') - const searxngUrl = Deno.env.get('SEARXNG_URL') || 'https://search.kevlarai.com' - - if (!supabaseUrl || !supabaseServiceKey) { - throw new Error('Missing required environment variables') - } - - // Create Supabase client - const supabase = createClient(supabaseUrl, supabaseServiceKey) - - // Parse request body - const body: BatchGeocodingRequest = await req.json() - const limit = body.limit || 10 - const forceRefresh = body.force_refresh || false - - // Get institutes that need geocoding - let query = supabase - .from('institutes') - .select('id, name, address, geo_coordinates') - .not('import_id', 'is', null) - - if (!forceRefresh) { - // Only get institutes without coordinates or with empty coordinates - query = query.or('geo_coordinates.is.null,geo_coordinates.eq.{}') - } - - if (body.institute_ids && body.institute_ids.length > 0) { - query = query.in('id', body.institute_ids) - } - - const { data: institutes, error: fetchError } = await query.limit(limit) - - if (fetchError) { - throw new Error(`Failed to fetch institutes: ${fetchError.message}`) - } - - if (!institutes || institutes.length === 0) { - return new Response( - JSON.stringify({ - success: true, - message: 'No institutes found that need geocoding', - processed: 0 - }), - { - status: 200, - headers: { ...corsHeaders, 'Content-Type': 'application/json' } - } - ) - } - - console.log(`Processing ${institutes.length} institutes for geocoding`) - - const results: GeocodingResult[] = [] - let successCount = 0 - let errorCount = 0 - - // Process institutes sequentially to avoid overwhelming the SearXNG service - for (const institute of institutes) { - try { - const address = institute.address as any - if (!address) { - results.push({ - institute_id: institute.id, - success: false, - message: 'No address information available', - error: 'Missing address data' - }) - errorCount++ - continue - } - - // Build search query from address components - const addressParts = [ - address.street, - address.town, - address.county, - address.postcode, - address.country - ].filter(Boolean) - - if (addressParts.length === 0) { - results.push({ - institute_id: institute.id, - success: false, - message: 'No valid address components found', - error: 'Empty address parts' - }) - errorCount++ - continue - } - - const searchQuery = addressParts.join(', ') - console.log(`Geocoding institute ${institute.id}: ${searchQuery}`) - - // Query SearXNG for geocoding - const geocodingResult = await geocodeAddress(searchQuery, searxngUrl) - - if (geocodingResult.success && geocodingResult.coordinates) { - // Update institute with geospatial coordinates - const { error: updateError } = await supabase - .from('institutes') - .update({ - geo_coordinates: { - latitude: geocodingResult.coordinates.latitude, - longitude: geocodingResult.coordinates.longitude, - boundingbox: geocodingResult.coordinates.boundingbox, - geojson: geocodingResult.coordinates.geojson, - osm: geocodingResult.coordinates.osm, - search_query: searchQuery, - geocoded_at: new Date().toISOString() - } - }) - .eq('id', institute.id) - - if (updateError) { - throw new Error(`Failed to update institute: ${updateError.message}`) - } - - results.push({ - institute_id: institute.id, - success: true, - message: 'Successfully geocoded', - coordinates: geocodingResult.coordinates - }) - successCount++ - - // Log the successful geocoding - await supabase - .from('function_logs') - .insert({ - file_id: null, - step: 'batch_geocoding', - message: 'Successfully geocoded institute address in batch', - data: { - institute_id: institute.id, - search_query: searchQuery, - coordinates: geocodingResult.coordinates - } - }) - - } else { - results.push({ - institute_id: institute.id, - success: false, - message: 'Geocoding failed', - error: geocodingResult.error || 'Unknown error' - }) - errorCount++ - } - - // Add a small delay between requests to be respectful to the SearXNG service - await new Promise(resolve => setTimeout(resolve, 100)) - - } catch (error) { - console.error(`Error processing institute ${institute.id}:`, error) - results.push({ - institute_id: institute.id, - success: false, - message: 'Processing error', - error: error.message - }) - errorCount++ - } - } - - // Log the batch operation - await supabase - .from('function_logs') - .insert({ - file_id: null, - step: 'batch_geocoding_complete', - message: 'Batch geocoding operation completed', - data: { - total_processed: institutes.length, - successful: successCount, - failed: errorCount, - results: results - } - }) - - return new Response( - JSON.stringify({ - success: true, - message: 'Batch geocoding completed', - summary: { - total_processed: institutes.length, - successful: successCount, - failed: errorCount - }, - results: results - }), - { - status: 200, - headers: { ...corsHeaders, 'Content-Type': 'application/json' } - } - ) - - } catch (error) { - console.error('Error in batch institute geocoder:', error) - - return new Response( - JSON.stringify({ - error: 'Internal server error', - details: error.message - }), - { - status: 500, - headers: { ...corsHeaders, 'Content-Type': 'application/json' } - } - ) - } -}) - -async function geocodeAddress(searchQuery: string, searxngUrl: string): Promise<{ - success: boolean - coordinates?: { - latitude: number - longitude: number - boundingbox: string[] - geojson?: any - osm?: any - } - error?: string -}> { - try { - // Format search query for OSM - const osmQuery = `!osm ${searchQuery}` - const searchUrl = `${searxngUrl}/search?q=${encodeURIComponent(osmQuery)}&format=json` - - const response = await fetch(searchUrl, { - method: 'GET', - headers: { - 'Accept': 'application/json', - 'User-Agent': 'ClassroomCopilot-BatchGeocoder/1.0' - } - }) - - if (!response.ok) { - throw new Error(`SearXNG request failed: ${response.status} ${response.statusText}`) - } - - const data = await response.json() - - // Check if we have results - the number_of_results field might be unreliable - // so we check the results array directly - if (!data.results || data.results.length === 0) { - return { - success: false, - error: 'No results returned from SearXNG' - } - } - - const result = data.results[0] - - if (!result.latitude || !result.longitude) { - return { - success: false, - error: 'Missing latitude or longitude in SearXNG response' - } - } - - return { - success: true, - coordinates: { - latitude: parseFloat(result.latitude), - longitude: parseFloat(result.longitude), - boundingbox: result.boundingbox || [], - geojson: result.geojson, - osm: result.osm - } - } - - } catch (error) { - console.error('Geocoding error:', error) - return { - success: false, - error: error.message - } - } -} diff --git a/functions/institute-geocoder/example-usage.ts b/functions/institute-geocoder/example-usage.ts deleted file mode 100644 index 619c345..0000000 --- a/functions/institute-geocoder/example-usage.ts +++ /dev/null @@ -1,315 +0,0 @@ -// Example usage of Institute Geocoder functions -// This file demonstrates how to integrate the geocoding functions in your frontend - -import { createClient } from '@supabase/supabase-js' - -// Initialize Supabase client -const supabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL! -const supabaseAnonKey = process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY! -const supabase = createClient(supabaseUrl, supabaseAnonKey) - -// Types for institute data -interface Institute { - id: string - name: string - address: { - street?: string - town?: string - county?: string - postcode?: string - country?: string - } - geo_coordinates?: { - latitude: number - longitude: number - boundingbox: string[] - search_query: string - geocoded_at: string - } -} - -interface GeocodingResult { - success: boolean - message: string - coordinates?: { - latitude: number - longitude: number - boundingbox: string[] - } - error?: string -} - -// 1. Geocode a single institute -export async function geocodeInstitute(instituteId: string): Promise { - try { - const { data, error } = await supabase.functions.invoke('institute-geocoder', { - body: { institute_id: instituteId } - }) - - if (error) { - throw new Error(error.message) - } - - return data - } catch (error) { - console.error('Geocoding failed:', error) - return { - success: false, - message: 'Geocoding failed', - error: error instanceof Error ? error.message : 'Unknown error' - } - } -} - -// 2. Batch geocode multiple institutes -export async function batchGeocodeInstitutes( - limit: number = 10, - forceRefresh: boolean = false -): Promise { - try { - const { data, error } = await supabase.functions.invoke('institute-geocoder/batch', { - body: { - limit, - force_refresh: forceRefresh - } - }) - - if (error) { - throw new Error(error.message) - } - - return data - } catch (error) { - console.error('Batch geocoding failed:', error) - throw error - } -} - -// 3. Get institutes that need geocoding -export async function getInstitutesNeedingGeocoding(): Promise { - try { - const { data, error } = await supabase - .from('institutes') - .select('id, name, address, geo_coordinates') - .or('geo_coordinates.is.null,geo_coordinates.eq.{}') - .not('import_id', 'is', null) - - if (error) { - throw new Error(error.message) - } - - return data || [] - } catch (error) { - console.error('Failed to fetch institutes:', error) - return [] - } -} - -// 4. Display institute on a map (example with Leaflet) -export function displayInstituteOnMap( - institute: Institute, - mapElement: HTMLElement -): void { - if (!institute.geo_coordinates) { - console.warn('Institute has no coordinates:', institute.name) - return - } - - // This is a placeholder - you'd need to implement actual map rendering - // For example, using Leaflet, Mapbox, or Google Maps - const { latitude, longitude } = institute.geo_coordinates - - console.log(`Displaying ${institute.name} at ${latitude}, ${longitude}`) - - // Example map implementation: - // const map = L.map(mapElement).setView([latitude, longitude], 13) - // L.marker([latitude, longitude]).addTo(map).bindPopup(institute.name) -} - -// 5. React component example -export function InstituteGeocoder() { - const [institutes, setInstitutes] = useState([]) - const [loading, setLoading] = useState(false) - const [geocodingProgress, setGeocodingProgress] = useState(0) - - // Load institutes that need geocoding - useEffect(() => { - loadInstitutes() - }, []) - - async function loadInstitutes() { - const data = await getInstitutesNeedingGeocoding() - setInstitutes(data) - } - - // Geocode all institutes - async function geocodeAllInstitutes() { - setLoading(true) - setGeocodingProgress(0) - - try { - const result = await batchGeocodeInstitutes(institutes.length, false) - - if (result.success) { - setGeocodingProgress(100) - // Reload institutes to show updated coordinates - await loadInstitutes() - } - } catch (error) { - console.error('Batch geocoding failed:', error) - } finally { - setLoading(false) - } - } - - // Geocode single institute - async function geocodeSingleInstitute(instituteId: string) { - try { - const result = await geocodeInstitute(instituteId) - if (result.success) { - // Reload institutes to show updated coordinates - await loadInstitutes() - } - } catch (error) { - console.error('Single geocoding failed:', error) - } - } - - return ( -
-

Institute Geocoding

- -
- - - {loading && ( -
-
-
- )} -
- -
- {institutes.map(institute => ( -
-

{institute.name}

-

- {institute.address.street && `${institute.address.street}, `} - {institute.address.town && `${institute.address.town}, `} - {institute.address.county && `${institute.address.county}, `} - {institute.address.postcode} -

- - {institute.geo_coordinates ? ( -
- 📍 {institute.geo_coordinates.latitude}, {institute.geo_coordinates.longitude} - Geocoded: {new Date(institute.geo_coordinates.geocoded_at).toLocaleDateString()} -
- ) : ( - - )} -
- ))} -
-
- ) -} - -// 6. Utility functions for working with coordinates -export class CoordinateUtils { - // Calculate distance between two points (Haversine formula) - static calculateDistance( - lat1: number, - lon1: number, - lat2: number, - lon2: number - ): number { - const R = 6371 // Earth's radius in kilometers - const dLat = this.toRadians(lat2 - lat1) - const dLon = this.toRadians(lon2 - lon1) - - const a = - Math.sin(dLat / 2) * Math.sin(dLat / 2) + - Math.cos(this.toRadians(lat1)) * Math.cos(this.toRadians(lat2)) * - Math.sin(dLon / 2) * Math.sin(dLon / 2) - - const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a)) - return R * c - } - - // Convert degrees to radians - private static toRadians(degrees: number): number { - return degrees * (Math.PI / 180) - } - - // Check if coordinates are within a bounding box - static isWithinBounds( - lat: number, - lon: number, - bounds: [number, number, number, number] // [minLat, maxLat, minLon, maxLon] - ): boolean { - return lat >= bounds[0] && lat <= bounds[1] && - lon >= bounds[2] && lon <= bounds[3] - } - - // Format coordinates for display - static formatCoordinates(lat: number, lon: number): string { - const latDir = lat >= 0 ? 'N' : 'S' - const lonDir = lon >= 0 ? 'E' : 'W' - return `${Math.abs(lat).toFixed(6)}°${latDir}, ${Math.abs(lon).toFixed(6)}°${lonDir}` - } -} - -// 7. Example of using coordinates in Neo4j queries -export const neo4jQueries = { - // Create institute node with location - createInstituteWithLocation: ` - CREATE (i:Institute { - id: $institute_id, - name: $name, - location: point({latitude: $latitude, longitude: $longitude}) - }) - RETURN i - `, - - // Find institutes within radius - findInstitutesWithinRadius: ` - MATCH (i:Institute) - WHERE distance(i.location, point({latitude: $centerLat, longitude: $centerLon})) < $radiusMeters - RETURN i, distance(i.location, point({latitude: $centerLat, longitude: $centerLon})) as distance - ORDER BY distance - `, - - // Find institutes in bounding box - findInstitutesInBounds: ` - MATCH (i:Institute) - WHERE i.location.latitude >= $minLat - AND i.location.latitude <= $maxLat - AND i.location.longitude >= $minLon - AND i.location.longitude <= $maxLon - RETURN i - ` -} - -export default { - geocodeInstitute, - batchGeocodeInstitutes, - getInstitutesNeedingGeocoding, - displayInstituteOnMap, - InstituteGeocoder, - CoordinateUtils, - neo4jQueries -} diff --git a/functions/institute-geocoder/index.ts b/functions/institute-geocoder/index.ts deleted file mode 100644 index 2e8cb52..0000000 --- a/functions/institute-geocoder/index.ts +++ /dev/null @@ -1,325 +0,0 @@ -import { serve } from 'https://deno.land/std@0.131.0/http/server.ts' -import { createClient } from 'https://esm.sh/@supabase/supabase-js@2' - -const corsHeaders = { - 'Access-Control-Allow-Origin': '*', - 'Access-Control-Allow-Headers': 'authorization, x-client-info, apikey, content-type', -} - -interface GeocodingRequest { - institute_id: string - address?: string - street?: string - town?: string - county?: string - postcode?: string - country?: string -} - -interface SearXNGResponse { - query: string - number_of_results: number - results: Array<{ - title: string - longitude: string - latitude: string - boundingbox: string[] - geojson?: any - osm?: any - }> -} - -interface GeocodingResult { - success: boolean - message: string - coordinates?: { - latitude: number - longitude: number - boundingbox: string[] - geojson?: any - osm?: any - } - error?: string -} - -serve(async (req: Request) => { - // Handle CORS preflight requests - if (req.method === 'OPTIONS') { - return new Response('ok', { headers: corsHeaders }) - } - - try { - // Get environment variables - const supabaseUrl = Deno.env.get('SUPABASE_URL') - const supabaseServiceKey = Deno.env.get('SUPABASE_SERVICE_ROLE_KEY') - const searxngUrl = Deno.env.get('SEARXNG_URL') || 'https://search.kevlarai.com' - - if (!supabaseUrl || !supabaseServiceKey) { - throw new Error('Missing required environment variables') - } - - // Create Supabase client - const supabase = createClient(supabaseUrl, supabaseServiceKey) - - // Parse request body - const body: GeocodingRequest = await req.json() - - if (!body.institute_id) { - return new Response( - JSON.stringify({ error: 'institute_id is required' }), - { - status: 400, - headers: { ...corsHeaders, 'Content-Type': 'application/json' } - } - ) - } - - // Get institute data from database - const { data: institute, error: fetchError } = await supabase - .from('institutes') - .select('*') - .eq('id', body.institute_id) - .single() - - if (fetchError || !institute) { - return new Response( - JSON.stringify({ error: 'Institute not found' }), - { - status: 404, - headers: { ...corsHeaders, 'Content-Type': 'application/json' } - } - ) - } - - // Build search query from address components - let searchQuery = '' - if (body.address) { - searchQuery = body.address - } else { - const addressParts = [ - body.street, - body.town, - body.county, - body.postcode, - body.country - ].filter(Boolean) - searchQuery = addressParts.join(', ') - } - - // If no search query provided, try to build from institute data - if (!searchQuery && institute.address) { - const address = institute.address as any - const addressParts = [ - address.street, - address.town, - address.county, - address.postcode, - address.country - ].filter(Boolean) - searchQuery = addressParts.join(', ') - } - - if (!searchQuery) { - return new Response( - JSON.stringify({ error: 'No address information available for geocoding' }), - { - status: 400, - headers: { ...corsHeaders, 'Content-Type': 'application/json' } - } - ) - } - - // Query SearXNG for geocoding - const geocodingResult = await geocodeAddressWithFallback(institute.address, searxngUrl) - - if (!geocodingResult.success) { - return new Response( - JSON.stringify({ - error: 'Geocoding failed', - details: geocodingResult.error - }), - { - status: 500, - headers: { ...corsHeaders, 'Content-Type': 'application/json' } - } - ) - } - - // Update institute with geospatial coordinates - const { error: updateError } = await supabase - .from('institutes') - .update({ - geo_coordinates: { - latitude: geocodingResult.coordinates!.latitude, - longitude: geocodingResult.coordinates!.longitude, - boundingbox: geocodingResult.coordinates!.boundingbox, - geojson: geocodingResult.coordinates!.geojson, - osm: geocodingResult.coordinates!.osm, - search_query: searchQuery, - geocoded_at: new Date().toISOString() - } - }) - .eq('id', body.institute_id) - - if (updateError) { - throw new Error(`Failed to update institute: ${updateError.message}`) - } - - // Log the geocoding operation - await supabase - .from('function_logs') - .insert({ - file_id: null, - step: 'geocoding', - message: 'Successfully geocoded institute address', - data: { - institute_id: body.institute_id, - search_query: searchQuery, - coordinates: geocodingResult.coordinates - } - }) - - return new Response( - JSON.stringify({ - success: true, - message: 'Institute geocoded successfully', - institute_id: body.institute_id, - coordinates: geocodingResult.coordinates - }), - { - status: 200, - headers: { ...corsHeaders, 'Content-Type': 'application/json' } - } - ) - - } catch (error) { - console.error('Error in institute geocoder:', error) - - return new Response( - JSON.stringify({ - error: 'Internal server error', - details: error.message - }), - { - status: 500, - headers: { ...corsHeaders, 'Content-Type': 'application/json' } - } - ) - } -}) - -async function geocodeAddress(searchQuery: string, searxngUrl: string): Promise { - try { - console.log(`Geocoding address: ${searchQuery}`) - - // Build the SearXNG query - const query = `!osm ${searchQuery}` - const url = `${searxngUrl}/search?q=${encodeURIComponent(query)}&format=json` - - console.log(`SearXNG URL: ${url}`) - - const response = await fetch(url) - if (!response.ok) { - throw new Error(`SearXNG request failed: ${response.status} ${response.statusText}`) - } - - const data: SearXNGResponse = await response.json() - console.log(`SearXNG response: ${JSON.stringify(data, null, 2)}`) - - // Check if we have results - if (!data.results || data.results.length === 0) { - return { - success: false, - message: 'No results returned from SearXNG', - error: 'No results returned from SearXNG' - } - } - - // Get the best result (first one) - const bestResult = data.results[0] - - if (!bestResult.latitude || !bestResult.longitude) { - return { - success: false, - message: 'Result missing coordinates', - error: 'Result missing coordinates' - } - } - - return { - success: true, - message: 'Geocoding successful', - coordinates: { - latitude: parseFloat(bestResult.latitude), - longitude: parseFloat(bestResult.longitude), - boundingbox: bestResult.boundingbox || [], - geojson: bestResult.geojson || null, - osm: bestResult.osm || null - } - } - - } catch (error) { - console.error('Error in geocodeAddress:', error) - return { - success: false, - message: 'Geocoding failed', - error: error.message - } - } -} - -async function geocodeAddressWithFallback(address: any, searxngUrl: string): Promise { - // Strategy 1: Try full address (street + town + county + postcode) - if (address.street && address.town && address.county && address.postcode) { - const fullQuery = `${address.street}, ${address.town}, ${address.county}, ${address.postcode}` - console.log(`Trying full address: ${fullQuery}`) - - const result = await geocodeAddress(fullQuery, searxngUrl) - if (result.success) { - console.log('Full address geocoding successful') - return result - } - } - - // Strategy 2: Try town + county + postcode - if (address.town && address.county && address.postcode) { - const mediumQuery = `${address.town}, ${address.county}, ${address.postcode}` - console.log(`Trying medium address: ${mediumQuery}`) - - const result = await geocodeAddress(mediumQuery, searxngUrl) - if (result.success) { - console.log('Medium address geocoding successful') - return result - } - } - - // Strategy 3: Try just postcode - if (address.postcode) { - console.log(`Trying postcode only: ${address.postcode}`) - - const result = await geocodeAddress(address.postcode, searxngUrl) - if (result.success) { - console.log('Postcode geocoding successful') - return result - } - } - - // Strategy 4: Try town + postcode - if (address.town && address.postcode) { - const simpleQuery = `${address.town}, ${address.postcode}` - console.log(`Trying simple address: ${simpleQuery}`) - - const result = await geocodeAddress(simpleQuery, searxngUrl) - if (result.success) { - console.log('Simple address geocoding successful') - return result - } - } - - // All strategies failed - return { - success: false, - message: 'All geocoding strategies failed', - error: 'No coordinates found with any address combination' - } -} diff --git a/functions/institute-geocoder/test.ts b/functions/institute-geocoder/test.ts deleted file mode 100644 index bb5110a..0000000 --- a/functions/institute-geocoder/test.ts +++ /dev/null @@ -1,142 +0,0 @@ -// Test script for institute geocoder functions -// This can be run in the browser console or as a standalone test - -interface TestCase { - name: string - address: string - expected_coords?: { - latitude: number - longitude: number - } -} - -const testCases: TestCase[] = [ - { - name: "10 Downing Street, London", - address: "10 Downing Street, London", - expected_coords: { - latitude: 51.5034878, - longitude: -0.1276965 - } - }, - { - name: "Buckingham Palace, London", - address: "Buckingham Palace, London", - expected_coords: { - latitude: 51.501364, - longitude: -0.124432 - } - }, - { - name: "Big Ben, London", - address: "Big Ben, London", - expected_coords: { - latitude: 51.499479, - longitude: -0.124809 - } - } -] - -async function testGeocoding() { - console.log("🧪 Starting Institute Geocoder Tests...") - - for (const testCase of testCases) { - console.log(`\n📍 Testing: ${testCase.name}`) - - try { - // Test the SearXNG service directly - const searchQuery = `!osm ${testCase.address}` - const searchUrl = `https://search.kevlarai.com/search?q=${encodeURIComponent(searchQuery)}&format=json` - - console.log(`🔍 Searching: ${searchUrl}`) - - const response = await fetch(searchUrl) - if (!response.ok) { - throw new Error(`HTTP ${response.status}: ${response.statusText}`) - } - - const data = await response.json() - console.log(`📊 Results: ${data.number_of_results} found`) - - if (data.results && data.results.length > 0) { - const result = data.results[0] - const coords = { - latitude: parseFloat(result.latitude), - longitude: parseFloat(result.longitude) - } - - console.log(`✅ Coordinates: ${coords.latitude}, ${coords.longitude}`) - - if (testCase.expected_coords) { - const latDiff = Math.abs(coords.latitude - testCase.expected_coords.latitude) - const lonDiff = Math.abs(coords.longitude - testCase.expected_coords.longitude) - - if (latDiff < 0.01 && lonDiff < 0.01) { - console.log(`🎯 Accuracy: High (within 0.01 degrees)`) - } else if (latDiff < 0.1 && lonDiff < 0.1) { - console.log(`🎯 Accuracy: Medium (within 0.1 degrees)`) - } else { - console.log(`⚠️ Accuracy: Low (difference > 0.1 degrees)`) - } - } - - if (result.boundingbox) { - console.log(`🗺️ Bounding Box: ${result.boundingbox.join(', ')}`) - } - - if (result.geojson) { - console.log(`🗺️ GeoJSON: ${result.geojson.type} with ${result.geojson.coordinates?.[0]?.length || 0} points`) - } - - } else { - console.log(`❌ No results found`) - } - - } catch (error) { - console.error(`❌ Test failed: ${error.message}`) - } - } - - console.log("\n🏁 Testing completed!") -} - -// Test address parsing function -function testAddressParsing() { - console.log("\n🔧 Testing Address Parsing...") - - const testAddresses = [ - { - street: "10 Downing Street", - town: "London", - county: "Greater London", - postcode: "SW1A 2AA", - country: "United Kingdom" - }, - { - street: "Buckingham Palace", - town: "London", - county: "Greater London", - postcode: "SW1A 1AA", - country: "United Kingdom" - } - ] - - for (const addr of testAddresses) { - const parts = [addr.street, addr.town, addr.county, addr.postcode, addr.country].filter(Boolean) - const searchQuery = parts.join(', ') - console.log(`📍 Address: ${searchQuery}`) - } -} - -// Run tests if this script is executed directly -if (typeof window !== 'undefined') { - // Browser environment - window.testGeocoding = testGeocoding - window.testAddressParsing = testAddressParsing - console.log("🧪 Institute Geocoder tests loaded. Run testGeocoding() or testAddressParsing() to test.") -} else { - // Node.js environment - console.log("🧪 Institute Geocoder tests loaded.") -} - -export { testGeocoding, testAddressParsing } diff --git a/functions/main/index.ts b/functions/main/index.ts deleted file mode 100644 index a094010..0000000 --- a/functions/main/index.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { serve } from 'https://deno.land/std@0.131.0/http/server.ts' -import * as jose from 'https://deno.land/x/jose@v4.14.4/index.ts' - -console.log('main function started') - -const JWT_SECRET = Deno.env.get('JWT_SECRET') -const VERIFY_JWT = Deno.env.get('VERIFY_JWT') === 'true' - -function getAuthToken(req: Request) { - const authHeader = req.headers.get('authorization') - if (!authHeader) { - throw new Error('Missing authorization header') - } - const [bearer, token] = authHeader.split(' ') - if (bearer !== 'Bearer') { - throw new Error(`Auth header is not 'Bearer {token}'`) - } - return token -} - -async function verifyJWT(jwt: string): Promise { - const encoder = new TextEncoder() - const secretKey = encoder.encode(JWT_SECRET) - try { - await jose.jwtVerify(jwt, secretKey) - } catch (err) { - console.error(err) - return false - } - return true -} - -serve(async (req: Request) => { - if (req.method !== 'OPTIONS' && VERIFY_JWT) { - try { - const token = getAuthToken(req) - const isValidJWT = await verifyJWT(token) - - if (!isValidJWT) { - return new Response(JSON.stringify({ msg: 'Invalid JWT' }), { - status: 401, - headers: { 'Content-Type': 'application/json' }, - }) - } - } catch (e) { - console.error(e) - return new Response(JSON.stringify({ msg: e.toString() }), { - status: 401, - headers: { 'Content-Type': 'application/json' }, - }) - } - } - - const url = new URL(req.url) - const { pathname } = url - const path_parts = pathname.split('/') - const service_name = path_parts[1] - - if (!service_name || service_name === '') { - const error = { msg: 'missing function name in request' } - return new Response(JSON.stringify(error), { - status: 400, - headers: { 'Content-Type': 'application/json' }, - }) - } - - const servicePath = `/home/deno/functions/${service_name}` - console.error(`serving the request with ${servicePath}`) - - const memoryLimitMb = 150 - const workerTimeoutMs = 1 * 60 * 1000 - const noModuleCache = false - const importMapPath = null - const envVarsObj = Deno.env.toObject() - const envVars = Object.keys(envVarsObj).map((k) => [k, envVarsObj[k]]) - - try { - const worker = await EdgeRuntime.userWorkers.create({ - servicePath, - memoryLimitMb, - workerTimeoutMs, - noModuleCache, - importMapPath, - envVars, - }) - return await worker.fetch(req) - } catch (e) { - const error = { msg: e.toString() } - return new Response(JSON.stringify(error), { - status: 500, - headers: { 'Content-Type': 'application/json' }, - }) - } -}) diff --git a/logs/vector.yml b/logs/vector.yml deleted file mode 100644 index cce46df..0000000 --- a/logs/vector.yml +++ /dev/null @@ -1,232 +0,0 @@ -api: - enabled: true - address: 0.0.0.0:9001 - -sources: - docker_host: - type: docker_logs - exclude_containers: - - supabase-vector - -transforms: - project_logs: - type: remap - inputs: - - docker_host - source: |- - .project = "default" - .event_message = del(.message) - .appname = del(.container_name) - del(.container_created_at) - del(.container_id) - del(.source_type) - del(.stream) - del(.label) - del(.image) - del(.host) - del(.stream) - router: - type: route - inputs: - - project_logs - route: - kong: '.appname == "supabase-kong"' - auth: '.appname == "supabase-auth"' - rest: '.appname == "supabase-rest"' - realtime: '.appname == "supabase-realtime"' - storage: '.appname == "supabase-storage"' - functions: '.appname == "supabase-functions"' - db: '.appname == "supabase-db"' - # Ignores non nginx errors since they are related with kong booting up - kong_logs: - type: remap - inputs: - - router.kong - source: |- - req, err = parse_nginx_log(.event_message, "combined") - if err == null { - .timestamp = req.timestamp - .metadata.request.headers.referer = req.referer - .metadata.request.headers.user_agent = req.agent - .metadata.request.headers.cf_connecting_ip = req.client - .metadata.request.method = req.method - .metadata.request.path = req.path - .metadata.request.protocol = req.protocol - .metadata.response.status_code = req.status - } - if err != null { - abort - } - # Ignores non nginx errors since they are related with kong booting up - kong_err: - type: remap - inputs: - - router.kong - source: |- - .metadata.request.method = "GET" - .metadata.response.status_code = 200 - parsed, err = parse_nginx_log(.event_message, "error") - if err == null { - .timestamp = parsed.timestamp - .severity = parsed.severity - .metadata.request.host = parsed.host - .metadata.request.headers.cf_connecting_ip = parsed.client - url, err = split(parsed.request, " ") - if err == null { - .metadata.request.method = url[0] - .metadata.request.path = url[1] - .metadata.request.protocol = url[2] - } - } - if err != null { - abort - } - # Gotrue logs are structured json strings which frontend parses directly. But we keep metadata for consistency. - auth_logs: - type: remap - inputs: - - router.auth - source: |- - parsed, err = parse_json(.event_message) - if err == null { - .metadata.timestamp = parsed.time - .metadata = merge!(.metadata, parsed) - } - # PostgREST logs are structured so we separate timestamp from message using regex - rest_logs: - type: remap - inputs: - - router.rest - source: |- - parsed, err = parse_regex(.event_message, r'^(?P