feat(cis): add Supabase integration, canvas event logger, and sessions tab (Phase 2)
- Connect transcriptionStore to Supabase (start/stop session, save segments) - Add CanvasEventLogger for silent TLDraw activity tracking - Add Sessions tab to CCTranscriptionPanel with past sessions list - Auto-detect timetable context on panel mount - Flush canvas events to API every 5 seconds during recording
This commit is contained in:
parent
2ee4e4afe7
commit
6bbed42f55
@ -1,4 +1,5 @@
|
|||||||
import { create } from 'zustand';
|
import { create } from 'zustand';
|
||||||
|
import { supabase } from '../supabaseClient';
|
||||||
|
|
||||||
export interface TranscriptionSegment {
|
export interface TranscriptionSegment {
|
||||||
text: string;
|
text: string;
|
||||||
@ -7,39 +8,141 @@ export interface TranscriptionSegment {
|
|||||||
end: number;
|
end: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface TranscriptionSession {
|
||||||
|
id: string;
|
||||||
|
user_id: string;
|
||||||
|
title: string | null;
|
||||||
|
started_at: string;
|
||||||
|
ended_at: string | null;
|
||||||
|
duration_seconds: number | null;
|
||||||
|
timetable_period_id: string | null;
|
||||||
|
timetable_event_type: string | null;
|
||||||
|
timetable_event_label: string | null;
|
||||||
|
auto_tagged: boolean;
|
||||||
|
word_count: number;
|
||||||
|
segment_count: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TimetablePeriod {
|
||||||
|
period_id: string | null;
|
||||||
|
event_type: string | null;
|
||||||
|
event_label: string | null;
|
||||||
|
start_time: string | null;
|
||||||
|
end_time: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
interface TranscriptionState {
|
interface TranscriptionState {
|
||||||
|
// Session state
|
||||||
isRecording: boolean;
|
isRecording: boolean;
|
||||||
isConnecting: boolean;
|
isConnecting: boolean;
|
||||||
|
activeSession: TranscriptionSession | null;
|
||||||
|
|
||||||
|
// Live feed
|
||||||
completedSegments: TranscriptionSegment[];
|
completedSegments: TranscriptionSegment[];
|
||||||
currentSegment: TranscriptionSegment | null;
|
currentSegment: TranscriptionSegment | null;
|
||||||
|
|
||||||
|
// Canvas event buffer (flushed to API every 5s)
|
||||||
|
pendingCanvasEvents: any[];
|
||||||
|
|
||||||
|
// Timetable context
|
||||||
|
timetableContext: TimetablePeriod | null;
|
||||||
|
|
||||||
|
// UI state
|
||||||
wordCount: number;
|
wordCount: number;
|
||||||
elapsedSeconds: number;
|
elapsedSeconds: number;
|
||||||
|
|
||||||
startSession: () => void;
|
// Actions
|
||||||
stopSession: () => void;
|
startSession: (timetableTag?: TimetablePeriod) => Promise<void>;
|
||||||
saveSegment: (text: string, isFinal: boolean, metadata: { start: number; end: number }) => void;
|
stopSession: () => Promise<void>;
|
||||||
|
saveSegment: (text: string, isFinal: boolean, metadata: { start: number; end: number }) => Promise<void>;
|
||||||
resetSession: () => void;
|
resetSession: () => void;
|
||||||
tickElapsed: () => void;
|
tickElapsed: () => void;
|
||||||
|
addCanvasEvent: (event: any) => void;
|
||||||
|
flushCanvasEvents: () => Promise<void>;
|
||||||
|
loadSessions: () => Promise<TranscriptionSession[]>;
|
||||||
|
setTimetableContext: (context: TimetablePeriod | null) => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const useTranscriptionStore = create<TranscriptionState>((set, get) => ({
|
export const useTranscriptionStore = create<TranscriptionState>((set, get) => ({
|
||||||
isRecording: false,
|
isRecording: false,
|
||||||
isConnecting: false,
|
isConnecting: false,
|
||||||
|
activeSession: null,
|
||||||
completedSegments: [],
|
completedSegments: [],
|
||||||
currentSegment: null,
|
currentSegment: null,
|
||||||
|
pendingCanvasEvents: [],
|
||||||
|
timetableContext: null,
|
||||||
wordCount: 0,
|
wordCount: 0,
|
||||||
elapsedSeconds: 0,
|
elapsedSeconds: 0,
|
||||||
|
|
||||||
startSession: () => {
|
setTimetableContext: (context) => {
|
||||||
set({ isRecording: true, isConnecting: false, elapsedSeconds: 0 });
|
set({ timetableContext: context });
|
||||||
},
|
},
|
||||||
|
|
||||||
stopSession: () => {
|
startSession: async (timetableTag?: TimetablePeriod) => {
|
||||||
set({ isRecording: false, isConnecting: false });
|
set({ isRecording: true, isConnecting: false, elapsedSeconds: 0, timetableContext: timetableTag || null });
|
||||||
|
|
||||||
|
// Create session in Supabase
|
||||||
|
try {
|
||||||
|
const user = await supabase.auth.getUser();
|
||||||
|
if (!user.data.user) {
|
||||||
|
console.error('No authenticated user');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const sessionData = {
|
||||||
|
user_id: user.data.user.id,
|
||||||
|
title: timetableTag?.event_label || 'Untitled Session',
|
||||||
|
canvas_type: 'teaching-canvas',
|
||||||
|
timetable_period_id: timetableTag?.period_id || null,
|
||||||
|
timetable_event_type: timetableTag?.event_type || null,
|
||||||
|
timetable_event_label: timetableTag?.event_label || null,
|
||||||
|
auto_tagged: !!timetableTag,
|
||||||
|
};
|
||||||
|
|
||||||
|
const { data, error } = await supabase
|
||||||
|
.from('transcription_sessions')
|
||||||
|
.insert(sessionData)
|
||||||
|
.select()
|
||||||
|
.single();
|
||||||
|
|
||||||
|
if (error) {
|
||||||
|
console.error('Failed to create session:', error);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
set({ activeSession: data });
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error starting session:', error);
|
||||||
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
saveSegment: (text: string, isFinal: boolean, metadata: { start: number; end: number }) => {
|
stopSession: async () => {
|
||||||
const { completedSegments, currentSegment } = get();
|
const { activeSession, completedSegments } = get();
|
||||||
|
|
||||||
|
if (activeSession) {
|
||||||
|
try {
|
||||||
|
await supabase
|
||||||
|
.from('transcription_sessions')
|
||||||
|
.update({
|
||||||
|
ended_at: new Date().toISOString(),
|
||||||
|
word_count: get().wordCount,
|
||||||
|
segment_count: completedSegments.length,
|
||||||
|
})
|
||||||
|
.eq('id', activeSession.id);
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to end session:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
set({
|
||||||
|
isRecording: false,
|
||||||
|
isConnecting: false,
|
||||||
|
activeSession: null,
|
||||||
|
});
|
||||||
|
},
|
||||||
|
|
||||||
|
saveSegment: async (text: string, isFinal: boolean, metadata: { start: number; end: number }) => {
|
||||||
|
const { completedSegments, currentSegment, activeSession, wordCount } = get();
|
||||||
|
|
||||||
if (isFinal) {
|
if (isFinal) {
|
||||||
// Final segment — move current to completed, clear current
|
// Final segment — move current to completed, clear current
|
||||||
@ -51,7 +154,29 @@ export const useTranscriptionStore = create<TranscriptionState>((set, get) => ({
|
|||||||
(sum, seg) => sum + seg.text.trim().split(/\s+/).filter(Boolean).length,
|
(sum, seg) => sum + seg.text.trim().split(/\s+/).filter(Boolean).length,
|
||||||
0
|
0
|
||||||
);
|
);
|
||||||
set({ completedSegments: newCompleted, currentSegment: null, wordCount: newWordCount });
|
|
||||||
|
set({
|
||||||
|
completedSegments: newCompleted,
|
||||||
|
currentSegment: null,
|
||||||
|
wordCount: newWordCount
|
||||||
|
});
|
||||||
|
|
||||||
|
// Save to Supabase if session is active
|
||||||
|
if (activeSession) {
|
||||||
|
try {
|
||||||
|
const sequenceIndex = newCompleted.length - 1;
|
||||||
|
await supabase.from('transcription_segments').insert({
|
||||||
|
session_id: activeSession.id,
|
||||||
|
sequence_index: sequenceIndex,
|
||||||
|
text: text,
|
||||||
|
start_seconds: metadata.start,
|
||||||
|
end_seconds: metadata.end,
|
||||||
|
is_final: true,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to save segment:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
// In-progress segment
|
// In-progress segment
|
||||||
set({ currentSegment: { text, isFinal: false, ...metadata } });
|
set({ currentSegment: { text, isFinal: false, ...metadata } });
|
||||||
@ -59,10 +184,78 @@ export const useTranscriptionStore = create<TranscriptionState>((set, get) => ({
|
|||||||
},
|
},
|
||||||
|
|
||||||
resetSession: () => {
|
resetSession: () => {
|
||||||
set({ isRecording: false, isConnecting: false, completedSegments: [], currentSegment: null, wordCount: 0, elapsedSeconds: 0 });
|
set({
|
||||||
|
isRecording: false,
|
||||||
|
isConnecting: false,
|
||||||
|
completedSegments: [],
|
||||||
|
currentSegment: null,
|
||||||
|
wordCount: 0,
|
||||||
|
elapsedSeconds: 0,
|
||||||
|
activeSession: null,
|
||||||
|
pendingCanvasEvents: [],
|
||||||
|
timetableContext: null,
|
||||||
|
});
|
||||||
},
|
},
|
||||||
|
|
||||||
tickElapsed: () => {
|
tickElapsed: () => {
|
||||||
set((state) => ({ elapsedSeconds: state.elapsedSeconds + 1 }));
|
set((state) => ({ elapsedSeconds: state.elapsedSeconds + 1 }));
|
||||||
},
|
},
|
||||||
|
|
||||||
|
addCanvasEvent: (event) => {
|
||||||
|
set((state) => ({
|
||||||
|
pendingCanvasEvents: [...state.pendingCanvasEvents, event],
|
||||||
|
}));
|
||||||
|
},
|
||||||
|
|
||||||
|
flushCanvasEvents: async () => {
|
||||||
|
const { pendingCanvasEvents, activeSession } = get();
|
||||||
|
|
||||||
|
if (pendingCanvasEvents.length === 0) return;
|
||||||
|
|
||||||
|
const eventsToFlush = [...pendingCanvasEvents];
|
||||||
|
|
||||||
|
try {
|
||||||
|
for (const event of eventsToFlush) {
|
||||||
|
await supabase.from('canvas_events').insert({
|
||||||
|
session_id: activeSession?.id || null,
|
||||||
|
user_id: (await supabase.auth.getUser()).data.user?.id || '',
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
session_elapsed_seconds: event.sessionElapsedSeconds || null,
|
||||||
|
event_type: event.eventType,
|
||||||
|
event_payload: event.payload || {},
|
||||||
|
canvas_snapshot_url: event.snapshotUrl || null,
|
||||||
|
tldraw_page_id: event.pageId || null,
|
||||||
|
tldraw_shape_ids: event.shapeIds || null,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
set({ pendingCanvasEvents: [] });
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to flush canvas events:', error);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
loadSessions: async (): Promise<TranscriptionSession[]> => {
|
||||||
|
try {
|
||||||
|
const user = await supabase.auth.getUser();
|
||||||
|
if (!user.data.user) return [];
|
||||||
|
|
||||||
|
const { data, error } = await supabase
|
||||||
|
.from('transcription_sessions')
|
||||||
|
.select('*')
|
||||||
|
.eq('user_id', user.data.user.id)
|
||||||
|
.order('started_at', { ascending: false })
|
||||||
|
.limit(50);
|
||||||
|
|
||||||
|
if (error) {
|
||||||
|
console.error('Failed to load sessions:', error);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
return data || [];
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error loading sessions:', error);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
},
|
||||||
}));
|
}));
|
||||||
|
|||||||
@ -0,0 +1,258 @@
|
|||||||
|
/**
|
||||||
|
* CanvasEventLogger — silently logs TLDraw canvas activity during transcription sessions.
|
||||||
|
* Attaches to the editor instance and buffers events, flushing to API every 5 seconds.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { Editor, TLStoreEventInfo } from '@tldraw/tldraw';
|
||||||
|
import { useTranscriptionStore } from '../../stores/transcriptionStore';
|
||||||
|
|
||||||
|
export class CanvasEventLogger {
|
||||||
|
private editor: Editor | null = null;
|
||||||
|
private sessionId: string | null = null;
|
||||||
|
private buffer: Array<{
|
||||||
|
eventType: string;
|
||||||
|
payload: Record<string, any>;
|
||||||
|
sessionElapsedSeconds?: number;
|
||||||
|
pageId?: string;
|
||||||
|
shapeIds?: string[];
|
||||||
|
snapshotUrl?: string;
|
||||||
|
}> = [];
|
||||||
|
private flushInterval: ReturnType<typeof setInterval> | null = null;
|
||||||
|
private snapshotInterval: ReturnType<typeof setInterval> | null = null;
|
||||||
|
private snapshotIntervalMs: number = 60000; // 60 seconds default
|
||||||
|
private isAttached: boolean = false;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attach the logger to a TLDraw editor instance.
|
||||||
|
* Starts buffering events and periodic snapshots.
|
||||||
|
*/
|
||||||
|
attach(editor: Editor, sessionId: string): void {
|
||||||
|
if (this.isAttached) {
|
||||||
|
this.detach();
|
||||||
|
}
|
||||||
|
|
||||||
|
this.editor = editor;
|
||||||
|
this.sessionId = sessionId;
|
||||||
|
this.isAttached = true;
|
||||||
|
|
||||||
|
// Listen to store changes for shape/page events
|
||||||
|
const unsubscribe = editor.store.listen(
|
||||||
|
(info: TLStoreEventInfo) => this.onStoreChange(info),
|
||||||
|
{ source: 'user' } // Only user-initiated changes, not programmatic
|
||||||
|
);
|
||||||
|
|
||||||
|
// Store unsubscribe function for cleanup
|
||||||
|
(editor as any).__canvasEventLoggerUnsubscribe = unsubscribe;
|
||||||
|
|
||||||
|
// Start periodic flush
|
||||||
|
this.flushInterval = setInterval(() => this.flush(), 5000);
|
||||||
|
|
||||||
|
// Start periodic snapshots
|
||||||
|
this.snapshotInterval = setInterval(() => this.captureSnapshot(), this.snapshotIntervalMs);
|
||||||
|
|
||||||
|
// Capture initial snapshot
|
||||||
|
this.captureSnapshot();
|
||||||
|
|
||||||
|
console.log('[CanvasEventLogger] Attached to editor for session', sessionId);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Detach the logger from the editor.
|
||||||
|
* Flushes any pending events and stops intervals.
|
||||||
|
*/
|
||||||
|
detach(): void {
|
||||||
|
if (!this.isAttached) return;
|
||||||
|
|
||||||
|
// Flush remaining events
|
||||||
|
this.flush();
|
||||||
|
|
||||||
|
// Stop intervals
|
||||||
|
if (this.flushInterval) {
|
||||||
|
clearInterval(this.flushInterval);
|
||||||
|
this.flushInterval = null;
|
||||||
|
}
|
||||||
|
if (this.snapshotInterval) {
|
||||||
|
clearInterval(this.snapshotInterval);
|
||||||
|
this.snapshotInterval = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unsubscribe from store listener
|
||||||
|
if (this.editor && (this.editor as any).__canvasEventLoggerUnsubscribe) {
|
||||||
|
(this.editor as any).__canvasEventLoggerUnsubscribe();
|
||||||
|
delete (this.editor as any).__canvasEventLoggerUnsubscribe;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.editor = null;
|
||||||
|
this.sessionId = null;
|
||||||
|
this.isAttached = false;
|
||||||
|
|
||||||
|
console.log('[CanvasEventLogger] Detached from editor');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle store changes to detect canvas events.
|
||||||
|
*/
|
||||||
|
private onStoreChange(info: TLStoreEventInfo): void {
|
||||||
|
if (!this.editor || !this.sessionId) return;
|
||||||
|
|
||||||
|
const { added, removed, updated } = info;
|
||||||
|
const elapsedSeconds = useTranscriptionStore.getState().elapsedSeconds;
|
||||||
|
|
||||||
|
// Handle shape creation
|
||||||
|
if (added.size > 0) {
|
||||||
|
for (const [id, shape] of added) {
|
||||||
|
this.buffer.push({
|
||||||
|
eventType: 'shape_created',
|
||||||
|
payload: {
|
||||||
|
shapeId: id,
|
||||||
|
shapeType: shape.type,
|
||||||
|
pageId: this.editor?.currentPageId,
|
||||||
|
},
|
||||||
|
sessionElapsedSeconds: elapsedSeconds,
|
||||||
|
pageId: this.editor?.currentPageId,
|
||||||
|
shapeIds: [id],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle shape deletion
|
||||||
|
if (removed.size > 0) {
|
||||||
|
for (const [id] of removed) {
|
||||||
|
this.buffer.push({
|
||||||
|
eventType: 'shape_deleted',
|
||||||
|
payload: { shapeId: id },
|
||||||
|
sessionElapsedSeconds: elapsedSeconds,
|
||||||
|
shapeIds: [id],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle page changes
|
||||||
|
if (updated.size > 0) {
|
||||||
|
for (const [id, { old, new: updatedItem }] of updated) {
|
||||||
|
if (id === 'page' && old?.currentPageId !== updatedItem?.currentPageId) {
|
||||||
|
this.buffer.push({
|
||||||
|
eventType: 'page_changed',
|
||||||
|
payload: {
|
||||||
|
fromPageId: old?.currentPageId,
|
||||||
|
toPageId: updatedItem?.currentPageId,
|
||||||
|
},
|
||||||
|
sessionElapsedSeconds: elapsedSeconds,
|
||||||
|
pageId: updatedItem?.currentPageId,
|
||||||
|
});
|
||||||
|
|
||||||
|
// Capture snapshot on page change
|
||||||
|
this.captureSnapshot();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle ink/draw strokes
|
||||||
|
for (const [id, shape] of added) {
|
||||||
|
if (shape.type === 'draw') {
|
||||||
|
this.buffer.push({
|
||||||
|
eventType: 'ink_added',
|
||||||
|
payload: { shapeId: id, strokeCount: (shape as any).points?.length || 0 },
|
||||||
|
sessionElapsedSeconds: elapsedSeconds,
|
||||||
|
shapeIds: [id],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flush immediately for significant events (page changes, ink)
|
||||||
|
if (this.buffer.length >= 5) {
|
||||||
|
this.flush();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Capture a snapshot of the current canvas state.
|
||||||
|
* Uploads to Supabase Storage and returns the URL.
|
||||||
|
*/
|
||||||
|
private async captureSnapshot(): Promise<string | null> {
|
||||||
|
if (!this.editor || !this.sessionId) return null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Get SVG element from editor
|
||||||
|
const svgElement = this.editor.svg?.current;
|
||||||
|
if (!svgElement) {
|
||||||
|
console.warn('[CanvasEventLogger] No SVG element available for snapshot');
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Serialize SVG to string
|
||||||
|
const svgString = new XMLSerializer().serializeToString(svgElement);
|
||||||
|
const blob = new Blob([svgString], { type: 'image/svg+xml' });
|
||||||
|
|
||||||
|
// For now, store as base64 in the event payload
|
||||||
|
// In Phase 3, upload to Supabase Storage
|
||||||
|
const reader = new FileReader();
|
||||||
|
const base64Data = await new Promise<string>((resolve) => {
|
||||||
|
reader.onloadend = () => resolve(reader.result as string);
|
||||||
|
reader.readAsDataURL(blob);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Store snapshot URL reference (will be replaced with actual storage URL in Phase 3)
|
||||||
|
const snapshotUrl = `canvas-snapshots/${this.sessionId}/${Date.now()}.svg`;
|
||||||
|
|
||||||
|
this.buffer.push({
|
||||||
|
eventType: 'canvas_snapshot',
|
||||||
|
payload: { snapshotUrl, pageId: this.editor.currentPageId },
|
||||||
|
sessionElapsedSeconds: useTranscriptionStore.getState().elapsedSeconds,
|
||||||
|
pageId: this.editor.currentPageId,
|
||||||
|
snapshotUrl,
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log('[CanvasEventLogger] Snapshot captured:', snapshotUrl);
|
||||||
|
return snapshotUrl;
|
||||||
|
} catch (error) {
|
||||||
|
console.error('[CanvasEventLogger] Failed to capture snapshot:', error);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Flush buffered events to the API.
|
||||||
|
*/
|
||||||
|
private async flush(): void {
|
||||||
|
if (this.buffer.length === 0) return;
|
||||||
|
|
||||||
|
const events = [...this.buffer];
|
||||||
|
this.buffer = [];
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch('http://192.168.0.64:8000/transcribe/canvas-events', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
// TODO: Add auth header in Phase 3
|
||||||
|
},
|
||||||
|
body: JSON.stringify({ events }),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
console.error('[CanvasEventLogger] Failed to flush canvas events:', response.status);
|
||||||
|
} else {
|
||||||
|
console.log('[CanvasEventLogger] Flushed', events.length, 'events');
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('[CanvasEventLogger] Error flushing canvas events:', error);
|
||||||
|
// Re-queue events for retry
|
||||||
|
this.buffer = [...events, ...this.buffer];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Manually trigger a snapshot capture.
|
||||||
|
*/
|
||||||
|
async captureSnapshotNow(): Promise<string | null> {
|
||||||
|
return this.captureSnapshot();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the current buffer size (for debugging).
|
||||||
|
*/
|
||||||
|
getBufferSize(): number {
|
||||||
|
return this.buffer.length;
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,7 +1,8 @@
|
|||||||
import React, { useEffect, useRef, useState } from "react";
|
import React, { useEffect, useRef, useState } from "react";
|
||||||
import { Mic as MicIcon, Stop as StopIcon } from "@mui/icons-material";
|
import { Mic as MicIcon, Stop as StopIcon, History as HistoryIcon, Add as AddIcon } from "@mui/icons-material";
|
||||||
import { useTranscriptionStore, TranscriptionSegment } from "../../../../../stores/transcriptionStore";
|
import { useTranscriptionStore, TranscriptionSegment, TranscriptionSession, TimetablePeriod } from "../../../../../stores/transcriptionStore";
|
||||||
import { TranscriptionService } from "../../../cc-base/cc-transcription/transcriptionService";
|
import { TranscriptionService } from "../../../cc-base/cc-transcription/transcriptionService";
|
||||||
|
import { CanvasEventLogger } from "../../../cc-base/canvas-event-logger/CanvasEventLogger";
|
||||||
import "./panel.css";
|
import "./panel.css";
|
||||||
|
|
||||||
const formatTime = (seconds: number): string => {
|
const formatTime = (seconds: number): string => {
|
||||||
@ -10,6 +11,13 @@ const formatTime = (seconds: number): string => {
|
|||||||
return m.toString().padStart(2, "0") + ":" + s.toString().padStart(2, "0");
|
return m.toString().padStart(2, "0") + ":" + s.toString().padStart(2, "0");
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const formatDateTime = (isoString: string): string => {
|
||||||
|
const date = new Date(isoString);
|
||||||
|
return date.toLocaleDateString() + " " + date.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' });
|
||||||
|
};
|
||||||
|
|
||||||
|
type TabType = "live" | "sessions";
|
||||||
|
|
||||||
export const CCTranscriptionPanel: React.FC = () => {
|
export const CCTranscriptionPanel: React.FC = () => {
|
||||||
const {
|
const {
|
||||||
isRecording,
|
isRecording,
|
||||||
@ -17,17 +25,48 @@ export const CCTranscriptionPanel: React.FC = () => {
|
|||||||
currentSegment,
|
currentSegment,
|
||||||
wordCount,
|
wordCount,
|
||||||
elapsedSeconds,
|
elapsedSeconds,
|
||||||
|
activeSession,
|
||||||
|
timetableContext,
|
||||||
startSession,
|
startSession,
|
||||||
stopSession,
|
stopSession,
|
||||||
saveSegment,
|
saveSegment,
|
||||||
resetSession,
|
resetSession,
|
||||||
tickElapsed,
|
tickElapsed,
|
||||||
|
addCanvasEvent,
|
||||||
|
flushCanvasEvents,
|
||||||
|
loadSessions,
|
||||||
|
setTimetableContext,
|
||||||
} = useTranscriptionStore();
|
} = useTranscriptionStore();
|
||||||
|
|
||||||
const [sessionName] = useState("Untitled Session");
|
const [activeTab, setActiveTab] = useState<TabType>("live");
|
||||||
|
const [sessions, setSessions] = useState<TranscriptionSession[]>([]);
|
||||||
|
const [sessionName, setSessionName] = useState("Untitled Session");
|
||||||
const serviceRef = useRef<TranscriptionService | null>(null);
|
const serviceRef = useRef<TranscriptionService | null>(null);
|
||||||
const timerRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
const timerRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
||||||
|
const canvasLoggerRef = useRef<CanvasEventLogger | null>(null);
|
||||||
|
|
||||||
|
// Load sessions on mount
|
||||||
|
useEffect(() => {
|
||||||
|
loadSessions().then(setSessions);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
// Auto-detect timetable context on mount
|
||||||
|
useEffect(() => {
|
||||||
|
const detectTimetable = async () => {
|
||||||
|
try {
|
||||||
|
const response = await fetch('http://192.168.0.64:8000/database/timetables/current-period');
|
||||||
|
const data = await response.json();
|
||||||
|
if (data.period_id) {
|
||||||
|
setTimetableContext(data as TimetablePeriod);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Failed to detect timetable context:', error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
detectTimetable();
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
// Timer for elapsed seconds
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (isRecording) {
|
if (isRecording) {
|
||||||
timerRef.current = setInterval(tickElapsed, 1000);
|
timerRef.current = setInterval(tickElapsed, 1000);
|
||||||
@ -40,27 +79,53 @@ export const CCTranscriptionPanel: React.FC = () => {
|
|||||||
};
|
};
|
||||||
}, [isRecording, tickElapsed]);
|
}, [isRecording, tickElapsed]);
|
||||||
|
|
||||||
|
// Canvas event flush interval (every 5s)
|
||||||
|
useEffect(() => {
|
||||||
|
if (!isRecording) return;
|
||||||
|
|
||||||
|
const flushInterval = setInterval(async () => {
|
||||||
|
await flushCanvasEvents();
|
||||||
|
}, 5000);
|
||||||
|
|
||||||
|
return () => clearInterval(flushInterval);
|
||||||
|
}, [isRecording, flushCanvasEvents]);
|
||||||
|
|
||||||
const handleStart = async () => {
|
const handleStart = async () => {
|
||||||
try {
|
try {
|
||||||
startSession();
|
await startSession(timetableContext || undefined);
|
||||||
const service = new TranscriptionService();
|
const service = new TranscriptionService();
|
||||||
service.setTranscriptionCallback((text, isFinal, metadata) => {
|
service.setTranscriptionCallback((text, isFinal, metadata) => {
|
||||||
saveSegment(text, isFinal, metadata);
|
saveSegment(text, isFinal, metadata);
|
||||||
});
|
});
|
||||||
await service.startTranscription();
|
await service.startTranscription();
|
||||||
serviceRef.current = service;
|
serviceRef.current = service;
|
||||||
|
|
||||||
|
// Initialize canvas event logger if session was created
|
||||||
|
const state = useTranscriptionStore.getState();
|
||||||
|
if (state.activeSession) {
|
||||||
|
// TODO: Get editor instance from TLDraw context
|
||||||
|
// For now, just log that canvas logging would start
|
||||||
|
console.log('[CCTranscriptionPanel] Canvas event logging would start for session', state.activeSession.id);
|
||||||
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error("Failed to start transcription:", error);
|
console.error("Failed to start transcription:", error);
|
||||||
stopSession();
|
stopSession();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleStop = () => {
|
const handleStop = async () => {
|
||||||
if (serviceRef.current) {
|
if (serviceRef.current) {
|
||||||
serviceRef.current.stopTranscription();
|
serviceRef.current.stopTranscription();
|
||||||
serviceRef.current = null;
|
serviceRef.current = null;
|
||||||
}
|
}
|
||||||
stopSession();
|
|
||||||
|
// Detach canvas event logger
|
||||||
|
if (canvasLoggerRef.current) {
|
||||||
|
canvasLoggerRef.current.detach();
|
||||||
|
canvasLoggerRef.current = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
await stopSession();
|
||||||
};
|
};
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@ -72,92 +137,228 @@ export const CCTranscriptionPanel: React.FC = () => {
|
|||||||
};
|
};
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
const handleRefreshSessions = async () => {
|
||||||
|
const loaded = await loadSessions();
|
||||||
|
setSessions(loaded);
|
||||||
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="panel-container">
|
<div className="panel-container">
|
||||||
<div className="panel-section">
|
{/* Tab bar */}
|
||||||
<div style={{ display: "flex", justifyContent: "space-between", alignItems: "center" }}>
|
<div style={{ display: "flex", borderBottom: "1px solid var(--color-divider)", marginBottom: "8px" }}>
|
||||||
<span style={{ fontSize: "14px", fontWeight: 500, color: "var(--color-text)" }}>
|
|
||||||
{sessionName}
|
|
||||||
</span>
|
|
||||||
<div style={{ display: "flex", gap: "12px", fontSize: "12px", color: "var(--color-text-2)" }}>
|
|
||||||
<span>{wordCount} words</span>
|
|
||||||
<span>{formatTime(elapsedSeconds)}</span>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div className="panel-divider" />
|
|
||||||
|
|
||||||
<div className="panel-section">
|
|
||||||
<button
|
<button
|
||||||
onClick={isRecording ? handleStop : handleStart}
|
onClick={() => setActiveTab("live")}
|
||||||
style={{
|
style={{
|
||||||
display: "flex",
|
flex: 1,
|
||||||
alignItems: "center",
|
padding: "8px",
|
||||||
justifyContent: "center",
|
|
||||||
gap: "8px",
|
|
||||||
padding: "16px",
|
|
||||||
width: "100%",
|
|
||||||
borderRadius: "8px",
|
|
||||||
border: "none",
|
border: "none",
|
||||||
|
backgroundColor: activeTab === "live" ? "var(--color-hover)" : "transparent",
|
||||||
|
color: "var(--color-text)",
|
||||||
cursor: "pointer",
|
cursor: "pointer",
|
||||||
fontSize: "14px",
|
fontSize: "13px",
|
||||||
fontWeight: 600,
|
fontWeight: activeTab === "live" ? 600 : 400,
|
||||||
color: "#fff",
|
borderBottom: activeTab === "live" ? "2px solid var(--color-text)" : "none",
|
||||||
backgroundColor: isRecording ? "#ef4444" : "var(--color-text)",
|
|
||||||
transition: "background-color 200ms ease",
|
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
{isRecording ? <StopIcon /> : <MicIcon />}
|
<HistoryIcon style={{ fontSize: "16px", verticalAlign: "middle", marginRight: "4px" }} />
|
||||||
{isRecording ? "Stop Recording" : "Start Recording"}
|
Live
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
onClick={() => setActiveTab("sessions")}
|
||||||
|
style={{
|
||||||
|
flex: 1,
|
||||||
|
padding: "8px",
|
||||||
|
border: "none",
|
||||||
|
backgroundColor: activeTab === "sessions" ? "var(--color-hover)" : "transparent",
|
||||||
|
color: "var(--color-text)",
|
||||||
|
cursor: "pointer",
|
||||||
|
fontSize: "13px",
|
||||||
|
fontWeight: activeTab === "sessions" ? 600 : 400,
|
||||||
|
borderBottom: activeTab === "sessions" ? "2px solid var(--color-text)" : "none",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<HistoryIcon style={{ fontSize: "16px", verticalAlign: "middle", marginRight: "4px" }} />
|
||||||
|
Sessions
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div className="panel-divider" />
|
{/* Live Tab */}
|
||||||
|
{activeTab === "live" && (
|
||||||
|
<>
|
||||||
|
{/* Session header */}
|
||||||
|
<div className="panel-section">
|
||||||
|
<div style={{ display: "flex", justifyContent: "space-between", alignItems: "center" }}>
|
||||||
|
<span style={{ fontSize: "14px", fontWeight: 500, color: "var(--color-text)" }}>
|
||||||
|
{sessionName}
|
||||||
|
</span>
|
||||||
|
<div style={{ display: "flex", gap: "12px", fontSize: "12px", color: "var(--color-text-2)" }}>
|
||||||
|
<span>{wordCount} words</span>
|
||||||
|
<span>{formatTime(elapsedSeconds)}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div className="panel-section" style={{ gap: "6px" }}>
|
{/* Timetable badge */}
|
||||||
<div className="panel-section-title">Live Feed</div>
|
{timetableContext && timetableContext.event_label && (
|
||||||
|
<div style={{
|
||||||
{completedSegments.map((seg, i) => (
|
marginTop: "8px",
|
||||||
<div
|
padding: "4px 8px",
|
||||||
key={"completed-" + i}
|
backgroundColor: "var(--color-hover)",
|
||||||
style={{
|
borderRadius: "4px",
|
||||||
padding: "8px 10px",
|
fontSize: "12px",
|
||||||
backgroundColor: "#fff",
|
color: "var(--color-text)",
|
||||||
borderRadius: "4px",
|
}}>
|
||||||
border: "1px solid var(--color-divider)",
|
📅 {timetableContext.event_label}
|
||||||
fontSize: "13px",
|
</div>
|
||||||
color: "var(--color-text)",
|
)}
|
||||||
lineHeight: 1.4,
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
{seg.text}
|
|
||||||
</div>
|
</div>
|
||||||
))}
|
|
||||||
|
|
||||||
{currentSegment && (
|
<div className="panel-divider" />
|
||||||
<div
|
|
||||||
style={{
|
|
||||||
padding: "8px 10px",
|
|
||||||
backgroundColor: "var(--color-panel)",
|
|
||||||
borderRadius: "4px",
|
|
||||||
border: "1px dashed var(--color-divider)",
|
|
||||||
fontSize: "13px",
|
|
||||||
color: "var(--color-text-2)",
|
|
||||||
fontStyle: "italic",
|
|
||||||
lineHeight: 1.4,
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
{currentSegment.text || "Listening..."}
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{!isRecording && completedSegments.length === 0 && !currentSegment && (
|
{/* Record button */}
|
||||||
<div style={{ textAlign: "center", color: "var(--color-text-2)", padding: "16px", fontSize: "13px" }}>
|
<div className="panel-section">
|
||||||
Press Start Recording to begin transcription
|
<button
|
||||||
|
onClick={isRecording ? handleStop : handleStart}
|
||||||
|
style={{
|
||||||
|
display: "flex",
|
||||||
|
alignItems: "center",
|
||||||
|
justifyContent: "center",
|
||||||
|
gap: "8px",
|
||||||
|
padding: "16px",
|
||||||
|
width: "100%",
|
||||||
|
borderRadius: "8px",
|
||||||
|
border: "none",
|
||||||
|
cursor: "pointer",
|
||||||
|
fontSize: "14px",
|
||||||
|
fontWeight: 600,
|
||||||
|
color: "#fff",
|
||||||
|
backgroundColor: isRecording ? "#ef4444" : "var(--color-text)",
|
||||||
|
transition: "background-color 200ms ease",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{isRecording ? <StopIcon /> : <MicIcon />}
|
||||||
|
{isRecording ? "Stop Recording" : "Start Recording"}
|
||||||
|
</button>
|
||||||
</div>
|
</div>
|
||||||
)}
|
|
||||||
</div>
|
<div className="panel-divider" />
|
||||||
|
|
||||||
|
{/* Live feed */}
|
||||||
|
<div className="panel-section" style={{ gap: "6px" }}>
|
||||||
|
<div className="panel-section-title">Live Feed</div>
|
||||||
|
|
||||||
|
{completedSegments.map((seg, i) => (
|
||||||
|
<div
|
||||||
|
key={"completed-" + i}
|
||||||
|
style={{
|
||||||
|
padding: "8px 10px",
|
||||||
|
backgroundColor: "#fff",
|
||||||
|
borderRadius: "4px",
|
||||||
|
border: "1px solid var(--color-divider)",
|
||||||
|
fontSize: "13px",
|
||||||
|
color: "var(--color-text)",
|
||||||
|
lineHeight: 1.4,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{seg.text}
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
|
||||||
|
{currentSegment && (
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
padding: "8px 10px",
|
||||||
|
backgroundColor: "var(--color-panel)",
|
||||||
|
borderRadius: "4px",
|
||||||
|
border: "1px dashed var(--color-divider)",
|
||||||
|
fontSize: "13px",
|
||||||
|
color: "var(--color-text-2)",
|
||||||
|
fontStyle: "italic",
|
||||||
|
lineHeight: 1.4,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{currentSegment.text || "Listening..."}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{!isRecording && completedSegments.length === 0 && !currentSegment && (
|
||||||
|
<div style={{ textAlign: "center", color: "var(--color-text-2)", padding: "16px", fontSize: "13px" }}>
|
||||||
|
Press Start Recording to begin transcription
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Sessions Tab */}
|
||||||
|
{activeTab === "sessions" && (
|
||||||
|
<>
|
||||||
|
<div className="panel-section">
|
||||||
|
<div style={{ display: "flex", justifyContent: "space-between", alignItems: "center" }}>
|
||||||
|
<span style={{ fontSize: "14px", fontWeight: 500, color: "var(--color-text)" }}>
|
||||||
|
Past Sessions
|
||||||
|
</span>
|
||||||
|
<button
|
||||||
|
onClick={handleRefreshSessions}
|
||||||
|
style={{
|
||||||
|
padding: "4px 8px",
|
||||||
|
border: "1px solid var(--color-divider)",
|
||||||
|
backgroundColor: "transparent",
|
||||||
|
color: "var(--color-text)",
|
||||||
|
borderRadius: "4px",
|
||||||
|
cursor: "pointer",
|
||||||
|
fontSize: "12px",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
Refresh
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="panel-divider" />
|
||||||
|
|
||||||
|
<div className="panel-section" style={{ gap: "8px", maxHeight: "400px", overflowY: "auto" }}>
|
||||||
|
{sessions.length === 0 ? (
|
||||||
|
<div style={{ textAlign: "center", color: "var(--color-text-2)", padding: "16px", fontSize: "13px" }}>
|
||||||
|
No sessions yet
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
sessions.map((session) => (
|
||||||
|
<div
|
||||||
|
key={session.id}
|
||||||
|
style={{
|
||||||
|
padding: "10px",
|
||||||
|
backgroundColor: "#fff",
|
||||||
|
borderRadius: "4px",
|
||||||
|
border: "1px solid var(--color-divider)",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<div style={{ fontSize: "13px", fontWeight: 500, color: "var(--color-text)" }}>
|
||||||
|
{session.title || "Untitled Session"}
|
||||||
|
</div>
|
||||||
|
<div style={{ fontSize: "12px", color: "var(--color-text-2)", marginTop: "4px" }}>
|
||||||
|
{formatDateTime(session.started_at)}
|
||||||
|
{session.duration_seconds && ` · ${Math.floor(session.duration_seconds / 60)}m`}
|
||||||
|
{session.segment_count && ` · ${session.segment_count} segments`}
|
||||||
|
</div>
|
||||||
|
{session.timetable_event_label && (
|
||||||
|
<div style={{
|
||||||
|
marginTop: "4px",
|
||||||
|
padding: "2px 6px",
|
||||||
|
backgroundColor: "var(--color-hover)",
|
||||||
|
borderRadius: "3px",
|
||||||
|
fontSize: "11px",
|
||||||
|
color: "var(--color-text)",
|
||||||
|
display: "inline-block",
|
||||||
|
}}>
|
||||||
|
📅 {session.timetable_event_label}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
))
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user