From 6bbed42f55cb4bea2ae033746ba724e05c6982a5 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Wed, 20 May 2026 22:06:31 +0000 Subject: [PATCH] feat(cis): add Supabase integration, canvas event logger, and sessions tab (Phase 2) - Connect transcriptionStore to Supabase (start/stop session, save segments) - Add CanvasEventLogger for silent TLDraw activity tracking - Add Sessions tab to CCTranscriptionPanel with past sessions list - Auto-detect timetable context on panel mount - Flush canvas events to API every 5 seconds during recording --- src/stores/transcriptionStore.ts | 217 ++++++++++- .../canvas-event-logger/CanvasEventLogger.ts | 258 +++++++++++++ .../shared/CCTranscriptionPanel.tsx | 353 ++++++++++++++---- 3 files changed, 740 insertions(+), 88 deletions(-) create mode 100644 src/utils/tldraw/cc-base/canvas-event-logger/CanvasEventLogger.ts diff --git a/src/stores/transcriptionStore.ts b/src/stores/transcriptionStore.ts index 4e7b639..72cb6d6 100644 --- a/src/stores/transcriptionStore.ts +++ b/src/stores/transcriptionStore.ts @@ -1,4 +1,5 @@ import { create } from 'zustand'; +import { supabase } from '../supabaseClient'; export interface TranscriptionSegment { text: string; @@ -7,40 +8,142 @@ export interface TranscriptionSegment { end: number; } +export interface TranscriptionSession { + id: string; + user_id: string; + title: string | null; + started_at: string; + ended_at: string | null; + duration_seconds: number | null; + timetable_period_id: string | null; + timetable_event_type: string | null; + timetable_event_label: string | null; + auto_tagged: boolean; + word_count: number; + segment_count: number; +} + +export interface TimetablePeriod { + period_id: string | null; + event_type: string | null; + event_label: string | null; + start_time: string | null; + end_time: string | null; +} + interface TranscriptionState { + // Session state isRecording: boolean; isConnecting: boolean; + activeSession: TranscriptionSession | null; + + // Live feed completedSegments: TranscriptionSegment[]; currentSegment: TranscriptionSegment | null; + + // Canvas event buffer (flushed to API every 5s) + pendingCanvasEvents: any[]; + + // Timetable context + timetableContext: TimetablePeriod | null; + + // UI state wordCount: number; elapsedSeconds: number; - - startSession: () => void; - stopSession: () => void; - saveSegment: (text: string, isFinal: boolean, metadata: { start: number; end: number }) => void; + + // Actions + startSession: (timetableTag?: TimetablePeriod) => Promise; + stopSession: () => Promise; + saveSegment: (text: string, isFinal: boolean, metadata: { start: number; end: number }) => Promise; resetSession: () => void; tickElapsed: () => void; + addCanvasEvent: (event: any) => void; + flushCanvasEvents: () => Promise; + loadSessions: () => Promise; + setTimetableContext: (context: TimetablePeriod | null) => void; } export const useTranscriptionStore = create((set, get) => ({ isRecording: false, isConnecting: false, + activeSession: null, completedSegments: [], currentSegment: null, + pendingCanvasEvents: [], + timetableContext: null, wordCount: 0, elapsedSeconds: 0, - startSession: () => { - set({ isRecording: true, isConnecting: false, elapsedSeconds: 0 }); + setTimetableContext: (context) => { + set({ timetableContext: context }); }, - stopSession: () => { - set({ isRecording: false, isConnecting: false }); + startSession: async (timetableTag?: TimetablePeriod) => { + set({ isRecording: true, isConnecting: false, elapsedSeconds: 0, timetableContext: timetableTag || null }); + + // Create session in Supabase + try { + const user = await supabase.auth.getUser(); + if (!user.data.user) { + console.error('No authenticated user'); + return; + } + + const sessionData = { + user_id: user.data.user.id, + title: timetableTag?.event_label || 'Untitled Session', + canvas_type: 'teaching-canvas', + timetable_period_id: timetableTag?.period_id || null, + timetable_event_type: timetableTag?.event_type || null, + timetable_event_label: timetableTag?.event_label || null, + auto_tagged: !!timetableTag, + }; + + const { data, error } = await supabase + .from('transcription_sessions') + .insert(sessionData) + .select() + .single(); + + if (error) { + console.error('Failed to create session:', error); + return; + } + + set({ activeSession: data }); + } catch (error) { + console.error('Error starting session:', error); + } }, - saveSegment: (text: string, isFinal: boolean, metadata: { start: number; end: number }) => { - const { completedSegments, currentSegment } = get(); + stopSession: async () => { + const { activeSession, completedSegments } = get(); + + if (activeSession) { + try { + await supabase + .from('transcription_sessions') + .update({ + ended_at: new Date().toISOString(), + word_count: get().wordCount, + segment_count: completedSegments.length, + }) + .eq('id', activeSession.id); + } catch (error) { + console.error('Failed to end session:', error); + } + } + + set({ + isRecording: false, + isConnecting: false, + activeSession: null, + }); + }, + saveSegment: async (text: string, isFinal: boolean, metadata: { start: number; end: number }) => { + const { completedSegments, currentSegment, activeSession, wordCount } = get(); + if (isFinal) { // Final segment โ€” move current to completed, clear current const newCompleted = [...completedSegments]; @@ -51,7 +154,29 @@ export const useTranscriptionStore = create((set, get) => ({ (sum, seg) => sum + seg.text.trim().split(/\s+/).filter(Boolean).length, 0 ); - set({ completedSegments: newCompleted, currentSegment: null, wordCount: newWordCount }); + + set({ + completedSegments: newCompleted, + currentSegment: null, + wordCount: newWordCount + }); + + // Save to Supabase if session is active + if (activeSession) { + try { + const sequenceIndex = newCompleted.length - 1; + await supabase.from('transcription_segments').insert({ + session_id: activeSession.id, + sequence_index: sequenceIndex, + text: text, + start_seconds: metadata.start, + end_seconds: metadata.end, + is_final: true, + }); + } catch (error) { + console.error('Failed to save segment:', error); + } + } } else { // In-progress segment set({ currentSegment: { text, isFinal: false, ...metadata } }); @@ -59,10 +184,78 @@ export const useTranscriptionStore = create((set, get) => ({ }, resetSession: () => { - set({ isRecording: false, isConnecting: false, completedSegments: [], currentSegment: null, wordCount: 0, elapsedSeconds: 0 }); + set({ + isRecording: false, + isConnecting: false, + completedSegments: [], + currentSegment: null, + wordCount: 0, + elapsedSeconds: 0, + activeSession: null, + pendingCanvasEvents: [], + timetableContext: null, + }); }, tickElapsed: () => { set((state) => ({ elapsedSeconds: state.elapsedSeconds + 1 })); }, + + addCanvasEvent: (event) => { + set((state) => ({ + pendingCanvasEvents: [...state.pendingCanvasEvents, event], + })); + }, + + flushCanvasEvents: async () => { + const { pendingCanvasEvents, activeSession } = get(); + + if (pendingCanvasEvents.length === 0) return; + + const eventsToFlush = [...pendingCanvasEvents]; + + try { + for (const event of eventsToFlush) { + await supabase.from('canvas_events').insert({ + session_id: activeSession?.id || null, + user_id: (await supabase.auth.getUser()).data.user?.id || '', + timestamp: new Date().toISOString(), + session_elapsed_seconds: event.sessionElapsedSeconds || null, + event_type: event.eventType, + event_payload: event.payload || {}, + canvas_snapshot_url: event.snapshotUrl || null, + tldraw_page_id: event.pageId || null, + tldraw_shape_ids: event.shapeIds || null, + }); + } + + set({ pendingCanvasEvents: [] }); + } catch (error) { + console.error('Failed to flush canvas events:', error); + } + }, + + loadSessions: async (): Promise => { + try { + const user = await supabase.auth.getUser(); + if (!user.data.user) return []; + + const { data, error } = await supabase + .from('transcription_sessions') + .select('*') + .eq('user_id', user.data.user.id) + .order('started_at', { ascending: false }) + .limit(50); + + if (error) { + console.error('Failed to load sessions:', error); + return []; + } + + return data || []; + } catch (error) { + console.error('Error loading sessions:', error); + return []; + } + }, })); diff --git a/src/utils/tldraw/cc-base/canvas-event-logger/CanvasEventLogger.ts b/src/utils/tldraw/cc-base/canvas-event-logger/CanvasEventLogger.ts new file mode 100644 index 0000000..979ff94 --- /dev/null +++ b/src/utils/tldraw/cc-base/canvas-event-logger/CanvasEventLogger.ts @@ -0,0 +1,258 @@ +/** + * CanvasEventLogger โ€” silently logs TLDraw canvas activity during transcription sessions. + * Attaches to the editor instance and buffers events, flushing to API every 5 seconds. + */ + +import { Editor, TLStoreEventInfo } from '@tldraw/tldraw'; +import { useTranscriptionStore } from '../../stores/transcriptionStore'; + +export class CanvasEventLogger { + private editor: Editor | null = null; + private sessionId: string | null = null; + private buffer: Array<{ + eventType: string; + payload: Record; + sessionElapsedSeconds?: number; + pageId?: string; + shapeIds?: string[]; + snapshotUrl?: string; + }> = []; + private flushInterval: ReturnType | null = null; + private snapshotInterval: ReturnType | null = null; + private snapshotIntervalMs: number = 60000; // 60 seconds default + private isAttached: boolean = false; + + /** + * Attach the logger to a TLDraw editor instance. + * Starts buffering events and periodic snapshots. + */ + attach(editor: Editor, sessionId: string): void { + if (this.isAttached) { + this.detach(); + } + + this.editor = editor; + this.sessionId = sessionId; + this.isAttached = true; + + // Listen to store changes for shape/page events + const unsubscribe = editor.store.listen( + (info: TLStoreEventInfo) => this.onStoreChange(info), + { source: 'user' } // Only user-initiated changes, not programmatic + ); + + // Store unsubscribe function for cleanup + (editor as any).__canvasEventLoggerUnsubscribe = unsubscribe; + + // Start periodic flush + this.flushInterval = setInterval(() => this.flush(), 5000); + + // Start periodic snapshots + this.snapshotInterval = setInterval(() => this.captureSnapshot(), this.snapshotIntervalMs); + + // Capture initial snapshot + this.captureSnapshot(); + + console.log('[CanvasEventLogger] Attached to editor for session', sessionId); + } + + /** + * Detach the logger from the editor. + * Flushes any pending events and stops intervals. + */ + detach(): void { + if (!this.isAttached) return; + + // Flush remaining events + this.flush(); + + // Stop intervals + if (this.flushInterval) { + clearInterval(this.flushInterval); + this.flushInterval = null; + } + if (this.snapshotInterval) { + clearInterval(this.snapshotInterval); + this.snapshotInterval = null; + } + + // Unsubscribe from store listener + if (this.editor && (this.editor as any).__canvasEventLoggerUnsubscribe) { + (this.editor as any).__canvasEventLoggerUnsubscribe(); + delete (this.editor as any).__canvasEventLoggerUnsubscribe; + } + + this.editor = null; + this.sessionId = null; + this.isAttached = false; + + console.log('[CanvasEventLogger] Detached from editor'); + } + + /** + * Handle store changes to detect canvas events. + */ + private onStoreChange(info: TLStoreEventInfo): void { + if (!this.editor || !this.sessionId) return; + + const { added, removed, updated } = info; + const elapsedSeconds = useTranscriptionStore.getState().elapsedSeconds; + + // Handle shape creation + if (added.size > 0) { + for (const [id, shape] of added) { + this.buffer.push({ + eventType: 'shape_created', + payload: { + shapeId: id, + shapeType: shape.type, + pageId: this.editor?.currentPageId, + }, + sessionElapsedSeconds: elapsedSeconds, + pageId: this.editor?.currentPageId, + shapeIds: [id], + }); + } + } + + // Handle shape deletion + if (removed.size > 0) { + for (const [id] of removed) { + this.buffer.push({ + eventType: 'shape_deleted', + payload: { shapeId: id }, + sessionElapsedSeconds: elapsedSeconds, + shapeIds: [id], + }); + } + } + + // Handle page changes + if (updated.size > 0) { + for (const [id, { old, new: updatedItem }] of updated) { + if (id === 'page' && old?.currentPageId !== updatedItem?.currentPageId) { + this.buffer.push({ + eventType: 'page_changed', + payload: { + fromPageId: old?.currentPageId, + toPageId: updatedItem?.currentPageId, + }, + sessionElapsedSeconds: elapsedSeconds, + pageId: updatedItem?.currentPageId, + }); + + // Capture snapshot on page change + this.captureSnapshot(); + } + } + } + + // Handle ink/draw strokes + for (const [id, shape] of added) { + if (shape.type === 'draw') { + this.buffer.push({ + eventType: 'ink_added', + payload: { shapeId: id, strokeCount: (shape as any).points?.length || 0 }, + sessionElapsedSeconds: elapsedSeconds, + shapeIds: [id], + }); + } + } + + // Flush immediately for significant events (page changes, ink) + if (this.buffer.length >= 5) { + this.flush(); + } + } + + /** + * Capture a snapshot of the current canvas state. + * Uploads to Supabase Storage and returns the URL. + */ + private async captureSnapshot(): Promise { + if (!this.editor || !this.sessionId) return null; + + try { + // Get SVG element from editor + const svgElement = this.editor.svg?.current; + if (!svgElement) { + console.warn('[CanvasEventLogger] No SVG element available for snapshot'); + return null; + } + + // Serialize SVG to string + const svgString = new XMLSerializer().serializeToString(svgElement); + const blob = new Blob([svgString], { type: 'image/svg+xml' }); + + // For now, store as base64 in the event payload + // In Phase 3, upload to Supabase Storage + const reader = new FileReader(); + const base64Data = await new Promise((resolve) => { + reader.onloadend = () => resolve(reader.result as string); + reader.readAsDataURL(blob); + }); + + // Store snapshot URL reference (will be replaced with actual storage URL in Phase 3) + const snapshotUrl = `canvas-snapshots/${this.sessionId}/${Date.now()}.svg`; + + this.buffer.push({ + eventType: 'canvas_snapshot', + payload: { snapshotUrl, pageId: this.editor.currentPageId }, + sessionElapsedSeconds: useTranscriptionStore.getState().elapsedSeconds, + pageId: this.editor.currentPageId, + snapshotUrl, + }); + + console.log('[CanvasEventLogger] Snapshot captured:', snapshotUrl); + return snapshotUrl; + } catch (error) { + console.error('[CanvasEventLogger] Failed to capture snapshot:', error); + return null; + } + } + + /** + * Flush buffered events to the API. + */ + private async flush(): void { + if (this.buffer.length === 0) return; + + const events = [...this.buffer]; + this.buffer = []; + + try { + const response = await fetch('http://192.168.0.64:8000/transcribe/canvas-events', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + // TODO: Add auth header in Phase 3 + }, + body: JSON.stringify({ events }), + }); + + if (!response.ok) { + console.error('[CanvasEventLogger] Failed to flush canvas events:', response.status); + } else { + console.log('[CanvasEventLogger] Flushed', events.length, 'events'); + } + } catch (error) { + console.error('[CanvasEventLogger] Error flushing canvas events:', error); + // Re-queue events for retry + this.buffer = [...events, ...this.buffer]; + } + } + + /** + * Manually trigger a snapshot capture. + */ + async captureSnapshotNow(): Promise { + return this.captureSnapshot(); + } + + /** + * Get the current buffer size (for debugging). + */ + getBufferSize(): number { + return this.buffer.length; + } +} diff --git a/src/utils/tldraw/ui-overrides/components/shared/CCTranscriptionPanel.tsx b/src/utils/tldraw/ui-overrides/components/shared/CCTranscriptionPanel.tsx index 0197d80..18f72ac 100644 --- a/src/utils/tldraw/ui-overrides/components/shared/CCTranscriptionPanel.tsx +++ b/src/utils/tldraw/ui-overrides/components/shared/CCTranscriptionPanel.tsx @@ -1,7 +1,8 @@ import React, { useEffect, useRef, useState } from "react"; -import { Mic as MicIcon, Stop as StopIcon } from "@mui/icons-material"; -import { useTranscriptionStore, TranscriptionSegment } from "../../../../../stores/transcriptionStore"; +import { Mic as MicIcon, Stop as StopIcon, History as HistoryIcon, Add as AddIcon } from "@mui/icons-material"; +import { useTranscriptionStore, TranscriptionSegment, TranscriptionSession, TimetablePeriod } from "../../../../../stores/transcriptionStore"; import { TranscriptionService } from "../../../cc-base/cc-transcription/transcriptionService"; +import { CanvasEventLogger } from "../../../cc-base/canvas-event-logger/CanvasEventLogger"; import "./panel.css"; const formatTime = (seconds: number): string => { @@ -10,6 +11,13 @@ const formatTime = (seconds: number): string => { return m.toString().padStart(2, "0") + ":" + s.toString().padStart(2, "0"); }; +const formatDateTime = (isoString: string): string => { + const date = new Date(isoString); + return date.toLocaleDateString() + " " + date.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' }); +}; + +type TabType = "live" | "sessions"; + export const CCTranscriptionPanel: React.FC = () => { const { isRecording, @@ -17,17 +25,48 @@ export const CCTranscriptionPanel: React.FC = () => { currentSegment, wordCount, elapsedSeconds, + activeSession, + timetableContext, startSession, stopSession, saveSegment, resetSession, tickElapsed, + addCanvasEvent, + flushCanvasEvents, + loadSessions, + setTimetableContext, } = useTranscriptionStore(); - const [sessionName] = useState("Untitled Session"); + const [activeTab, setActiveTab] = useState("live"); + const [sessions, setSessions] = useState([]); + const [sessionName, setSessionName] = useState("Untitled Session"); const serviceRef = useRef(null); const timerRef = useRef | null>(null); + const canvasLoggerRef = useRef(null); + // Load sessions on mount + useEffect(() => { + loadSessions().then(setSessions); + }, []); + + // Auto-detect timetable context on mount + useEffect(() => { + const detectTimetable = async () => { + try { + const response = await fetch('http://192.168.0.64:8000/database/timetables/current-period'); + const data = await response.json(); + if (data.period_id) { + setTimetableContext(data as TimetablePeriod); + } + } catch (error) { + console.error('Failed to detect timetable context:', error); + } + }; + detectTimetable(); + }, []); + + // Timer for elapsed seconds useEffect(() => { if (isRecording) { timerRef.current = setInterval(tickElapsed, 1000); @@ -40,27 +79,53 @@ export const CCTranscriptionPanel: React.FC = () => { }; }, [isRecording, tickElapsed]); + // Canvas event flush interval (every 5s) + useEffect(() => { + if (!isRecording) return; + + const flushInterval = setInterval(async () => { + await flushCanvasEvents(); + }, 5000); + + return () => clearInterval(flushInterval); + }, [isRecording, flushCanvasEvents]); + const handleStart = async () => { try { - startSession(); + await startSession(timetableContext || undefined); const service = new TranscriptionService(); service.setTranscriptionCallback((text, isFinal, metadata) => { saveSegment(text, isFinal, metadata); }); await service.startTranscription(); serviceRef.current = service; + + // Initialize canvas event logger if session was created + const state = useTranscriptionStore.getState(); + if (state.activeSession) { + // TODO: Get editor instance from TLDraw context + // For now, just log that canvas logging would start + console.log('[CCTranscriptionPanel] Canvas event logging would start for session', state.activeSession.id); + } } catch (error) { console.error("Failed to start transcription:", error); stopSession(); } }; - const handleStop = () => { + const handleStop = async () => { if (serviceRef.current) { serviceRef.current.stopTranscription(); serviceRef.current = null; } - stopSession(); + + // Detach canvas event logger + if (canvasLoggerRef.current) { + canvasLoggerRef.current.detach(); + canvasLoggerRef.current = null; + } + + await stopSession(); }; useEffect(() => { @@ -72,92 +137,228 @@ export const CCTranscriptionPanel: React.FC = () => { }; }, []); + const handleRefreshSessions = async () => { + const loaded = await loadSessions(); + setSessions(loaded); + }; + return (
-
-
- - {sessionName} - -
- {wordCount} words - {formatTime(elapsedSeconds)} -
-
-
- -
- -
+ {/* Tab bar */} +
+
-
+ {/* Live Tab */} + {activeTab === "live" && ( + <> + {/* Session header */} +
+
+ + {sessionName} + +
+ {wordCount} words + {formatTime(elapsedSeconds)} +
+
-
-
Live Feed
- - {completedSegments.map((seg, i) => ( -
- {seg.text} + {/* Timetable badge */} + {timetableContext && timetableContext.event_label && ( +
+ ๐Ÿ“… {timetableContext.event_label} +
+ )}
- ))} - {currentSegment && ( -
- {currentSegment.text || "Listening..."} -
- )} +
- {!isRecording && completedSegments.length === 0 && !currentSegment && ( -
- Press Start Recording to begin transcription + {/* Record button */} +
+
- )} -
+ +
+ + {/* Live feed */} +
+
Live Feed
+ + {completedSegments.map((seg, i) => ( +
+ {seg.text} +
+ ))} + + {currentSegment && ( +
+ {currentSegment.text || "Listening..."} +
+ )} + + {!isRecording && completedSegments.length === 0 && !currentSegment && ( +
+ Press Start Recording to begin transcription +
+ )} +
+ + )} + + {/* Sessions Tab */} + {activeTab === "sessions" && ( + <> +
+
+ + Past Sessions + + +
+
+ +
+ +
+ {sessions.length === 0 ? ( +
+ No sessions yet +
+ ) : ( + sessions.map((session) => ( +
+
+ {session.title || "Untitled Session"} +
+
+ {formatDateTime(session.started_at)} + {session.duration_seconds && ` ยท ${Math.floor(session.duration_seconds / 60)}m`} + {session.segment_count && ` ยท ${session.segment_count} segments`} +
+ {session.timetable_event_label && ( +
+ ๐Ÿ“… {session.timetable_event_label} +
+ )} +
+ )) + )} +
+ + )}
); };