Initial commit
This commit is contained in:
@@ -0,0 +1,342 @@
|
||||
import React from 'react';
|
||||
import { CCBaseShapeUtil } from '../CCBaseShapeUtil';
|
||||
import { TLShapeId } from '@tldraw/tldraw';
|
||||
import { CCBaseShape } from '../cc-types';
|
||||
import { TranscriptionManager } from '../cc-transcription/TranscriptionManager';
|
||||
import { ccShapeProps, getDefaultCCLiveTranscriptionProps } from '../cc-props';
|
||||
import { ccShapeMigrations } from '../cc-migrations';
|
||||
import { CC_BASE_STYLE_CONSTANTS } from '../cc-styles';
|
||||
|
||||
export interface TranscriptionSegment {
|
||||
id: string
|
||||
text: string
|
||||
completed: boolean
|
||||
start: string
|
||||
end: string
|
||||
}
|
||||
|
||||
export interface CCLiveTranscriptionShape extends CCBaseShape {
|
||||
type: 'cc-live-transcription'
|
||||
props: {
|
||||
title: string
|
||||
w: number
|
||||
h: number
|
||||
headerColor: string
|
||||
backgroundColor: string
|
||||
isLocked: boolean
|
||||
isRecording: boolean
|
||||
segments: TranscriptionSegment[]
|
||||
currentSegment?: TranscriptionSegment
|
||||
lastProcessedSegment?: string // Track last processed segment to avoid duplicates
|
||||
}
|
||||
}
|
||||
|
||||
export class CCLiveTranscriptionShapeUtil extends CCBaseShapeUtil<CCLiveTranscriptionShape> {
|
||||
static override type = 'cc-live-transcription' as const;
|
||||
static override props = ccShapeProps.liveTranscription;
|
||||
static override migrations = ccShapeMigrations.liveTranscription;
|
||||
|
||||
override getDefaultProps(): CCLiveTranscriptionShape['props'] {
|
||||
return getDefaultCCLiveTranscriptionProps() as unknown as CCLiveTranscriptionShape['props'];
|
||||
}
|
||||
|
||||
override renderContent = (shape: CCLiveTranscriptionShape) => {
|
||||
return this.renderShapeContent(shape)
|
||||
}
|
||||
|
||||
renderShapeContent = (shape: CCLiveTranscriptionShape) => {
|
||||
const { isRecording, segments, currentSegment } = shape.props;
|
||||
const contentHeight = shape.props.h - CC_BASE_STYLE_CONSTANTS.HEADER.height - 2 * CC_BASE_STYLE_CONSTANTS.CONTENT.padding;
|
||||
const controlsHeight = 80;
|
||||
const transcriptHeight = contentHeight - controlsHeight;
|
||||
|
||||
return (
|
||||
<div style={{
|
||||
width: '100%',
|
||||
height: '100%',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
pointerEvents: 'all'
|
||||
}}>
|
||||
{/* Microphone Controls */}
|
||||
<div style={{
|
||||
height: controlsHeight,
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
gap: '16px',
|
||||
borderBottom: '1px solid #e0e0e0',
|
||||
padding: '8px'
|
||||
}}>
|
||||
<div
|
||||
role="button"
|
||||
tabIndex={0}
|
||||
onPointerDown={(e) => {
|
||||
e.stopPropagation();
|
||||
}}
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
this.toggleRecording(shape);
|
||||
}}
|
||||
style={{
|
||||
width: '48px',
|
||||
height: '48px',
|
||||
borderRadius: '50%',
|
||||
border: 'none',
|
||||
backgroundColor: isRecording ? '#f44336' : '#4CAF50',
|
||||
color: 'white',
|
||||
fontSize: '24px',
|
||||
cursor: 'pointer',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
justifyContent: 'center',
|
||||
boxShadow: '0 2px 4px rgba(0,0,0,0.2)',
|
||||
transition: 'all 0.3s ease',
|
||||
userSelect: 'none',
|
||||
}}
|
||||
>
|
||||
{isRecording ? '⏹' : '🎤'}
|
||||
</div>
|
||||
<div style={{
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
alignItems: 'flex-start'
|
||||
}}>
|
||||
<div style={{
|
||||
fontSize: '16px',
|
||||
fontWeight: 'bold',
|
||||
color: isRecording ? '#f44336' : '#4CAF50'
|
||||
}}>
|
||||
{isRecording ? 'Recording...' : 'Ready'}
|
||||
</div>
|
||||
<div style={{ fontSize: '12px', color: '#666' }}>
|
||||
{isRecording ? 'Click to stop' : 'Click to start recording'}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Transcription Content */}
|
||||
<AutoScrollContainer height={transcriptHeight}>
|
||||
{/* Completed Segments */}
|
||||
{segments.map((segment) => (
|
||||
<div
|
||||
key={segment.id}
|
||||
style={{
|
||||
padding: '8px',
|
||||
backgroundColor: '#FFF',
|
||||
borderRadius: '8px',
|
||||
fontSize: '14px',
|
||||
lineHeight: '1.4',
|
||||
color: '#000000',
|
||||
whiteSpace: 'pre-wrap',
|
||||
wordBreak: 'break-word',
|
||||
width: '100%',
|
||||
transition: 'color 0.3s ease',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '2px'
|
||||
}}
|
||||
>
|
||||
<div style={{
|
||||
fontSize: '11px',
|
||||
color: '#888',
|
||||
fontFamily: 'monospace'
|
||||
}}>
|
||||
{segment.start}s - {segment.end}s
|
||||
</div>
|
||||
<div>
|
||||
{segment.text}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
|
||||
{/* Current Segment */}
|
||||
{currentSegment && (
|
||||
<div
|
||||
style={{
|
||||
padding: '8px',
|
||||
backgroundColor: '#f5f5f5',
|
||||
borderRadius: '8px',
|
||||
fontSize: '14px',
|
||||
lineHeight: '1.4',
|
||||
color: '#666666',
|
||||
whiteSpace: 'pre-wrap',
|
||||
wordBreak: 'break-word',
|
||||
width: '100%',
|
||||
transition: 'color 0.3s ease',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
gap: '2px'
|
||||
}}
|
||||
>
|
||||
<div style={{
|
||||
fontSize: '11px',
|
||||
color: '#888',
|
||||
fontFamily: 'monospace'
|
||||
}}>
|
||||
{currentSegment.start}s - {currentSegment.end}s
|
||||
</div>
|
||||
<div>
|
||||
{currentSegment.text}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Initial State */}
|
||||
{!isRecording && segments.length === 0 && !currentSegment && (
|
||||
<div
|
||||
style={{
|
||||
padding: '8px',
|
||||
backgroundColor: '#FFF',
|
||||
borderRadius: '8px',
|
||||
fontSize: '18px',
|
||||
lineHeight: '1.5',
|
||||
color: '#666666',
|
||||
whiteSpace: 'pre-wrap',
|
||||
wordBreak: 'break-word',
|
||||
width: '100%',
|
||||
textAlign: 'center'
|
||||
}}
|
||||
>
|
||||
Click the microphone to start
|
||||
</div>
|
||||
)}
|
||||
</AutoScrollContainer>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
private toggleRecording(shape: CCLiveTranscriptionShape) {
|
||||
console.log('🎤 Toggle recording clicked');
|
||||
const { id } = shape;
|
||||
const { isRecording } = shape.props;
|
||||
|
||||
console.log('Current state:', { id, isRecording });
|
||||
|
||||
// When starting new recording, preserve existing props but reset segments
|
||||
const newProps = !isRecording ? {
|
||||
...shape.props,
|
||||
isRecording: true,
|
||||
segments: [],
|
||||
currentSegment: undefined,
|
||||
lastProcessedSegment: undefined,
|
||||
} : {
|
||||
...shape.props,
|
||||
isRecording: false,
|
||||
};
|
||||
|
||||
this.editor.updateShape<CCLiveTranscriptionShape>({
|
||||
id,
|
||||
type: 'cc-live-transcription',
|
||||
props: newProps,
|
||||
});
|
||||
|
||||
const manager = TranscriptionManager.getManager(this.editor);
|
||||
console.log('Got transcription manager');
|
||||
|
||||
if (!isRecording) {
|
||||
console.log('Starting transcription...');
|
||||
manager.startTranscription(id);
|
||||
} else {
|
||||
console.log('Stopping transcription...');
|
||||
manager.stopTranscription();
|
||||
}
|
||||
}
|
||||
|
||||
updateText(
|
||||
id: TLShapeId,
|
||||
text: string,
|
||||
isConfirmed: boolean,
|
||||
metadata?: { start: string | number, end: string | number }
|
||||
) {
|
||||
console.log('📝 Updating text:', { id, text, isConfirmed, metadata });
|
||||
const shape = this.editor.getShape<CCLiveTranscriptionShape>(id);
|
||||
if (!shape) {
|
||||
console.warn('❌ Shape not found for updating text:', id);
|
||||
return;
|
||||
}
|
||||
|
||||
// Format timestamps consistently
|
||||
const start = typeof metadata?.start === 'number' ? metadata.start.toFixed(3) : metadata?.start;
|
||||
const end = typeof metadata?.end === 'number' ? metadata.end.toFixed(3) : metadata?.end;
|
||||
|
||||
const segmentId = isConfirmed ? crypto.randomUUID() : 'current';
|
||||
const newSegment: TranscriptionSegment = {
|
||||
id: segmentId,
|
||||
text,
|
||||
completed: isConfirmed,
|
||||
start: start || '0.000',
|
||||
end: end || '0.000'
|
||||
};
|
||||
|
||||
// Handle current (incomplete) segment
|
||||
if (!isConfirmed) {
|
||||
// Only update if text has changed
|
||||
if (shape.props.currentSegment?.text !== text) {
|
||||
this.editor.updateShape<CCLiveTranscriptionShape>({
|
||||
id,
|
||||
type: 'cc-live-transcription',
|
||||
props: {
|
||||
...shape.props,
|
||||
currentSegment: newSegment
|
||||
},
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Handle completed segment
|
||||
let segments = [...shape.props.segments];
|
||||
|
||||
// Check if this segment is different from the last processed one
|
||||
// and not already in our segments list
|
||||
const isDuplicate = segments.some(s => s.text === text);
|
||||
if (shape.props.lastProcessedSegment !== text && !isDuplicate) {
|
||||
// Add new completed segment
|
||||
segments.push(newSegment);
|
||||
|
||||
this.editor.updateShape<CCLiveTranscriptionShape>({
|
||||
id,
|
||||
type: 'cc-live-transcription',
|
||||
props: {
|
||||
...shape.props,
|
||||
segments,
|
||||
lastProcessedSegment: text,
|
||||
// Clear current segment if it matches the completed one
|
||||
currentSegment: shape.props.currentSegment?.text === text ? undefined : shape.props.currentSegment
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
console.log('✅ Text updated');
|
||||
}
|
||||
}
|
||||
|
||||
// Auto-scrolling container component
|
||||
function AutoScrollContainer({ children, height }: { children: React.ReactNode, height: number }) {
|
||||
const containerRef = React.useRef<HTMLDivElement>(null);
|
||||
|
||||
React.useEffect(() => {
|
||||
if (containerRef.current) {
|
||||
containerRef.current.scrollTop = containerRef.current.scrollHeight;
|
||||
}
|
||||
}, [children]);
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={containerRef}
|
||||
style={{
|
||||
height,
|
||||
padding: '16px',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
justifyContent: 'flex-start',
|
||||
gap: '8px',
|
||||
overflow: 'auto',
|
||||
scrollBehavior: 'smooth'
|
||||
}}
|
||||
>
|
||||
{children}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
import type { Meta, StoryObj } from '@storybook/react';
|
||||
import { CCLiveTranscriptionShapeUtil } from './CCLiveTranscriptionShapeUtil';
|
||||
import { TranscriptionService } from './transcriptionService';
|
||||
import { getDefaultCCLiveTranscriptionProps } from '../cc-props';
|
||||
import { CCLiveTranscriptionShape } from './CCLiveTranscriptionShapeUtil';
|
||||
import { TLShapeId, IndexKey, TLParentId } from '@tldraw/tldraw';
|
||||
import { Editor } from '@tldraw/editor';
|
||||
|
||||
const meta: Meta = {
|
||||
title: 'Components/Transcription',
|
||||
parameters: {
|
||||
layout: 'centered',
|
||||
},
|
||||
tags: ['autodocs'],
|
||||
};
|
||||
|
||||
export default meta;
|
||||
type Story = StoryObj<typeof meta>;
|
||||
|
||||
// Mock editor
|
||||
const mockEditor = {
|
||||
updateShape: () => {},
|
||||
} as unknown as Editor;
|
||||
|
||||
// Mock shape data
|
||||
const mockShape: CCLiveTranscriptionShape = {
|
||||
id: 'shape:1' as TLShapeId,
|
||||
type: 'cc-live-transcription',
|
||||
x: 0,
|
||||
y: 0,
|
||||
rotation: 0,
|
||||
index: 'a1' as IndexKey,
|
||||
parentId: 'page:page' as TLParentId,
|
||||
isLocked: false,
|
||||
opacity: 1,
|
||||
meta: {},
|
||||
typeName: 'shape',
|
||||
props: getDefaultCCLiveTranscriptionProps(),
|
||||
};
|
||||
|
||||
// Basic story showing the transcription component
|
||||
export const Default: Story = {
|
||||
args: {},
|
||||
render: () => {
|
||||
const shapeUtil = new CCLiveTranscriptionShapeUtil(mockEditor);
|
||||
return (
|
||||
<div style={{ width: '800px', height: '600px', border: '1px solid #ccc' }}>
|
||||
{shapeUtil.renderShapeContent(mockShape)}
|
||||
</div>
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
// Story showing the transcription service in action
|
||||
export const WithService: Story = {
|
||||
args: {},
|
||||
render: () => {
|
||||
const service = new TranscriptionService('default');
|
||||
const shapeUtil = new CCLiveTranscriptionShapeUtil(mockEditor);
|
||||
|
||||
return (
|
||||
<div style={{ width: '800px', height: '600px', border: '1px solid #ccc' }}>
|
||||
<div style={{ padding: '20px' }}>
|
||||
<h3>Transcription Service Demo</h3>
|
||||
<button
|
||||
onClick={() => service.startTranscription()}
|
||||
style={{ marginRight: '10px', padding: '10px' }}
|
||||
>
|
||||
Start Transcription
|
||||
</button>
|
||||
<button
|
||||
onClick={() => service.stopTranscription()}
|
||||
style={{ padding: '10px' }}
|
||||
>
|
||||
Stop Transcription
|
||||
</button>
|
||||
</div>
|
||||
<div style={{ marginTop: '20px' }}>
|
||||
{shapeUtil.renderShapeContent(mockShape)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,73 @@
|
||||
import { Editor, TLShapeId } from '@tldraw/tldraw';
|
||||
import { TranscriptionService } from './transcriptionService';
|
||||
import { CCLiveTranscriptionShapeUtil } from './CCLiveTranscriptionShapeUtil';
|
||||
|
||||
export class TranscriptionManager {
|
||||
private static instances = new WeakMap<Editor, TranscriptionManager>();
|
||||
private transcriptionService?: TranscriptionService;
|
||||
private currentShapeId?: TLShapeId;
|
||||
private sameOutputCount = 0;
|
||||
private lastText = '';
|
||||
private readonly SAME_OUTPUT_THRESHOLD = 10;
|
||||
|
||||
constructor(private editor: Editor) {}
|
||||
|
||||
static getManager(editor: Editor): TranscriptionManager {
|
||||
let manager = TranscriptionManager.instances.get(editor);
|
||||
if (!manager) {
|
||||
manager = new TranscriptionManager(editor);
|
||||
TranscriptionManager.instances.set(editor, manager);
|
||||
}
|
||||
return manager;
|
||||
}
|
||||
|
||||
startTranscription(shapeId: TLShapeId) {
|
||||
console.log('Starting transcription...');
|
||||
this.currentShapeId = shapeId;
|
||||
this.transcriptionService = new TranscriptionService();
|
||||
|
||||
// Set up callback for transcription updates
|
||||
this.transcriptionService.setTranscriptionCallback((text: string, isFinal: boolean, metadata: { start: number, end: number }) => {
|
||||
console.log('📝 Transcription update received:', { text, metadata });
|
||||
const util = this.editor.getShapeUtil<CCLiveTranscriptionShapeUtil>('cc-live-transcription');
|
||||
if (!util) {
|
||||
console.warn('❌ Shape util not found');
|
||||
return;
|
||||
}
|
||||
console.log('Found transcription util:', !!util);
|
||||
|
||||
// Check if text is stable (same output multiple times)
|
||||
const isStable = text === this.lastText;
|
||||
if (isStable) {
|
||||
this.sameOutputCount++;
|
||||
} else {
|
||||
this.sameOutputCount = 0;
|
||||
this.lastText = text;
|
||||
}
|
||||
|
||||
// Mark as completed if we've seen the same output multiple times or if marked as final
|
||||
const isCompleted = isFinal || this.sameOutputCount >= this.SAME_OUTPUT_THRESHOLD;
|
||||
|
||||
util.updateText(
|
||||
this.currentShapeId!,
|
||||
text,
|
||||
isCompleted,
|
||||
metadata
|
||||
);
|
||||
});
|
||||
|
||||
// Start the transcription service
|
||||
this.transcriptionService.startTranscription();
|
||||
}
|
||||
|
||||
stopTranscription() {
|
||||
console.log('Stopping transcription...');
|
||||
if (this.transcriptionService) {
|
||||
this.transcriptionService.stopTranscription();
|
||||
this.transcriptionService = undefined;
|
||||
}
|
||||
this.currentShapeId = undefined;
|
||||
this.sameOutputCount = 0;
|
||||
this.lastText = '';
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
import logger from '../../../../debugConfig';
|
||||
|
||||
export interface TranscriptionConfig {
|
||||
language?: string;
|
||||
task?: string;
|
||||
modelSize?: string;
|
||||
useVad?: boolean;
|
||||
}
|
||||
|
||||
export class TranscriptionService {
|
||||
private socket: WebSocket | null = null;
|
||||
private stream: MediaStream | null = null;
|
||||
private audioContext: AudioContext | null = null;
|
||||
private mediaStreamSource: MediaStreamAudioSourceNode | null = null;
|
||||
private workletNode: AudioWorkletNode | null = null;
|
||||
private selectedDeviceId: string = '';
|
||||
private onTranscriptionUpdate: ((text: string, isFinal: boolean, metadata: { start: number, end: number }) => void) | null = null;
|
||||
|
||||
constructor(deviceId: string = '') {
|
||||
this.selectedDeviceId = deviceId;
|
||||
}
|
||||
|
||||
setTranscriptionCallback(callback: (text: string, isFinal: boolean, metadata: { start: number, end: number }) => void) {
|
||||
this.onTranscriptionUpdate = callback;
|
||||
}
|
||||
|
||||
async startTranscription(config: TranscriptionConfig = {}) {
|
||||
console.log('🎙️ Starting transcription service...');
|
||||
|
||||
try {
|
||||
// Get default audio device if none selected
|
||||
if (!this.selectedDeviceId) {
|
||||
console.log('No device selected, getting default device...');
|
||||
const devices = await navigator.mediaDevices.enumerateDevices();
|
||||
const audioDevice = devices.find(device => device.kind === 'audioinput');
|
||||
if (audioDevice) {
|
||||
this.selectedDeviceId = audioDevice.deviceId;
|
||||
console.log('Found default audio device:', audioDevice.label);
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.selectedDeviceId) {
|
||||
logger.error('transcription-service', '⚠️ No audio device available');
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info('transcription-service', '🔊 Accessing user media...');
|
||||
this.stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: { deviceId: this.selectedDeviceId },
|
||||
});
|
||||
console.log('Got audio stream');
|
||||
|
||||
const uuid = crypto.randomUUID();
|
||||
const wsUrl = import.meta.env.VITE_WHISPERLIVE_URL;
|
||||
console.log('Connecting to WebSocket:', wsUrl);
|
||||
|
||||
const ws = new WebSocket(wsUrl);
|
||||
this.socket = ws;
|
||||
|
||||
const connectionTimeout = setTimeout(() => {
|
||||
if (ws.readyState !== WebSocket.OPEN) {
|
||||
logger.error('transcription-service', '⏰ WebSocket connection timed out');
|
||||
ws.close();
|
||||
}
|
||||
}, 20000);
|
||||
|
||||
ws.onopen = () => {
|
||||
clearTimeout(connectionTimeout);
|
||||
logger.info('transcription-service', '✅ WebSocket connected');
|
||||
|
||||
// Send initial configuration message
|
||||
const message = JSON.stringify({
|
||||
uid: uuid,
|
||||
language: config.language || 'en',
|
||||
task: config.task || 'transcribe',
|
||||
model: config.modelSize || 'small',
|
||||
use_vad: config.useVad ?? true,
|
||||
});
|
||||
|
||||
ws.send(message);
|
||||
this.setupAudioProcessing();
|
||||
};
|
||||
|
||||
ws.onerror = (error) => {
|
||||
logger.error('transcription-service', '❌ WebSocket error:', error);
|
||||
};
|
||||
|
||||
ws.onclose = () => {
|
||||
logger.info('transcription-service', '🔌 WebSocket closed');
|
||||
this.cleanup();
|
||||
};
|
||||
|
||||
ws.onmessage = (event) => {
|
||||
const data = JSON.parse(event.data);
|
||||
if (data.uid !== uuid) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (data.status === 'WAIT') {
|
||||
logger.info('transcription-service', `⏳ Wait time: ${Math.round(data.message)} minutes`);
|
||||
this.cleanup();
|
||||
return;
|
||||
}
|
||||
|
||||
if (data.message === 'DISCONNECT') {
|
||||
logger.info('transcription-service', '🔕 Server requested disconnection');
|
||||
this.cleanup();
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.onTranscriptionUpdate && data.segments) {
|
||||
// Get the last segment which is the current one being updated
|
||||
const lastSegment = data.segments[data.segments.length - 1];
|
||||
|
||||
// Process completed segments
|
||||
let lastCompletedText = '';
|
||||
for (let i = 0; i < data.segments.length - 1; i++) {
|
||||
const segment = data.segments[i];
|
||||
// Only send update if this segment is different from the last one
|
||||
if (segment.text.trim() !== lastCompletedText.trim()) {
|
||||
this.onTranscriptionUpdate(
|
||||
segment.text,
|
||||
segment.completed ?? true, // Server marks completed segments
|
||||
{
|
||||
start: parseFloat(segment.start),
|
||||
end: parseFloat(segment.end)
|
||||
}
|
||||
);
|
||||
lastCompletedText = segment.text;
|
||||
}
|
||||
}
|
||||
|
||||
// Update the current (incomplete) segment only if it's different from the last completed one
|
||||
if (lastSegment && lastSegment.text.trim() !== lastCompletedText.trim()) {
|
||||
this.onTranscriptionUpdate(
|
||||
lastSegment.text,
|
||||
lastSegment.completed ?? false, // Last segment is typically incomplete unless marked otherwise
|
||||
{
|
||||
start: parseFloat(lastSegment.start),
|
||||
end: parseFloat(lastSegment.end)
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
} catch (error) {
|
||||
logger.error('transcription-service', '❌ Error starting transcription:', error);
|
||||
this.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
private async setupAudioProcessing() {
|
||||
if (!this.stream || !this.socket) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
this.audioContext = new AudioContext();
|
||||
|
||||
// Load and register the audio worklet
|
||||
await this.audioContext.audioWorklet.addModule('/audioWorklet.js');
|
||||
|
||||
this.mediaStreamSource = this.audioContext.createMediaStreamSource(this.stream);
|
||||
this.workletNode = new AudioWorkletNode(this.audioContext, 'audio-processor');
|
||||
|
||||
// Handle audio data from the worklet
|
||||
this.workletNode.port.onmessage = (event) => {
|
||||
if (this.socket?.readyState === WebSocket.OPEN) {
|
||||
const resampledData = this.resampleTo16kHZ(event.data, this.audioContext!.sampleRate);
|
||||
this.socket.send(resampledData);
|
||||
}
|
||||
};
|
||||
|
||||
this.mediaStreamSource.connect(this.workletNode);
|
||||
this.workletNode.connect(this.audioContext.destination);
|
||||
} catch (error) {
|
||||
console.error('Error setting up audio processing:', error);
|
||||
}
|
||||
}
|
||||
|
||||
private resampleTo16kHZ(audioData: Float32Array, origSampleRate: number): Float32Array {
|
||||
const ratio = origSampleRate / 16000;
|
||||
const newLength = Math.round(audioData.length / ratio);
|
||||
const result = new Float32Array(newLength);
|
||||
|
||||
for (let i = 0; i < newLength; i++) {
|
||||
const pos = i * ratio;
|
||||
const leftPos = Math.floor(pos);
|
||||
const rightPos = Math.ceil(pos);
|
||||
const weight = pos - leftPos;
|
||||
result[i] = audioData[leftPos] * (1 - weight) + (audioData[rightPos] || 0) * weight;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
stopTranscription() {
|
||||
this.cleanup();
|
||||
}
|
||||
|
||||
private cleanup() {
|
||||
if (this.workletNode) {
|
||||
this.workletNode.disconnect();
|
||||
this.workletNode = null;
|
||||
}
|
||||
|
||||
if (this.mediaStreamSource) {
|
||||
this.mediaStreamSource.disconnect();
|
||||
this.mediaStreamSource = null;
|
||||
}
|
||||
|
||||
if (this.audioContext) {
|
||||
this.audioContext.close();
|
||||
this.audioContext = null;
|
||||
}
|
||||
|
||||
if (this.stream) {
|
||||
this.stream.getTracks().forEach(track => track.stop());
|
||||
this.stream = null;
|
||||
}
|
||||
|
||||
if (this.socket) {
|
||||
this.socket.close();
|
||||
this.socket = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user