From 05d8bd284c688759f5c11a916f479bf458f14a23 Mon Sep 17 00:00:00 2001 From: kcar Date: Sun, 28 Jun 2026 08:26:27 +0000 Subject: [PATCH] fix: improve WhisperLive health, model, websocket defaults --- .env | 2 +- .gitignore | 1 + docker-compose.yaml | 1 + hybrid_server.py | 65 ++++++++++++++++++++++++++++++++++----------- 4 files changed, 52 insertions(+), 17 deletions(-) create mode 100644 .gitignore diff --git a/.env b/.env index ec83ddf..4fd0f0a 100644 --- a/.env +++ b/.env @@ -8,5 +8,5 @@ HTTP_PORT=8080 WHISPERLIVE_SSL=false WHISPL_USE_CUSTOM_MODEL=false -FASTERWHISPER_MODEL=faster-whisper-large-v3 +FASTERWHISPER_MODEL=large-v3-turbo WHISPERLIVE_URL=${APP_WS_PROTOCOL}://whisperlive.${APP_URL} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..effa83e --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +hf-cache/ diff --git a/docker-compose.yaml b/docker-compose.yaml index 9874e5a..5af5c01 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -21,6 +21,7 @@ services: - ./models:/app/models - ./ssl:/app/ssl - ./logs:/app/logs + - ./hf-cache:/root/.cache/huggingface deploy: resources: reservations: diff --git a/hybrid_server.py b/hybrid_server.py index c91057f..a785801 100644 --- a/hybrid_server.py +++ b/hybrid_server.py @@ -78,27 +78,49 @@ class HybridWhisperServer: from whisper_live.server import TranscriptionServer self.whisper_server = TranscriptionServer() - # Create a shared transcriber instance for HTTP requests + # Create a shared transcriber instance for HTTP requests. + # Prefer the configured production model over the previous hard-coded + # base default; faster-whisper accepts either a model size (for example + # large-v3-turbo) or a converted model path. + self.default_model = self.faster_whisper_custom_model_path or os.getenv("FASTERWHISPER_MODEL", "large-v3-turbo") self.shared_transcriber = None if self.backend == "faster_whisper": from whisper_live.transcriber import WhisperModel - # Use base model as default for HTTP requests - model_size = "base" - if self.faster_whisper_custom_model_path: - model_size = self.faster_whisper_custom_model_path - self.shared_transcriber = WhisperModel(model_size) + self.shared_transcriber = WhisperModel( + self.default_model, + device="cuda", + compute_type="int8", + ) def setup_routes(self): @self.app.route('/health', methods=['GET']) def health_check(): - # Get GPU memory from nvidia-smi (GPU 1) + # Query the GPUs visible inside the container. Docker CDI maps the + # assigned host GPU to container-local index 0, so hard-coding -i 1 + # reports 0.0 even while WhisperLive is using CUDA. import subprocess + + gpu_mem_used = 0.0 + gpu_mem_total = 0.0 try: - gpu_mem = float(subprocess.check_output( - 'nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i 1', shell=True - ).decode().strip()) / 1024.0 + output = subprocess.check_output( + [ + 'nvidia-smi', + '--query-gpu=memory.used,memory.total', + '--format=csv,noheader,nounits', + ], + text=True, + ) + for line in output.splitlines(): + if not line.strip(): + continue + used, total = [float(part.strip()) for part in line.split(',', 1)] + gpu_mem_used += used + gpu_mem_total += total + gpu_mem_used /= 1024.0 + gpu_mem_total /= 1024.0 except Exception: - gpu_mem = 0.0 + pass # Get active WS connections active = len(self.whisper_server.clients) if hasattr(self.whisper_server, 'clients') else 0 @@ -107,7 +129,9 @@ class HybridWhisperServer: 'status': 'healthy', 'service': 'WhisperLive Hybrid Server', 'model_loaded': self.shared_transcriber is not None, - 'gpu_memory_used_gb': round(gpu_mem, 1), + 'model': self.default_model, + 'gpu_memory_used_gb': round(gpu_mem_used, 1), + 'gpu_memory_total_gb': round(gpu_mem_total, 1), 'active_connections': active }) @@ -1086,9 +1110,8 @@ print(transcription.text) # Bridges browser WebSocket connections on the HTTP port (8080) # to the internal WhisperLive WebSocket server (port 5000). # This allows live transcription through a single HTTPS port via NPM. - @self.sock.route('/ws') - def ws_bridge(ws): - """Bridge WebSocket from HTTP port to internal WhisperLive WS server""" + def handle_ws_bridge(ws): + """Bridge WebSocket from HTTP port to internal WhisperLive WS server.""" internal_url = f"ws://127.0.0.1:{self.websocket_port}" logger.info(f"WebSocket bridge: new connection, proxying to {internal_url}") @@ -1143,6 +1166,16 @@ print(transcription.text) except Exception: pass logger.info("WebSocket bridge: connection closed") + + @self.sock.route('/ws') + def ws_bridge(ws): + """Canonical WebSocket bridge path for NPM/Cloudflare.""" + return handle_ws_bridge(ws) + + @self.sock.route('/') + def ws_bridge_root(ws): + """Compatibility bridge for clients configured with the bare WSS origin.""" + return handle_ws_bridge(ws) def run_websocket_server(self): """Run the WebSocket server in a separate thread""" @@ -1188,7 +1221,7 @@ if __name__ == "__main__": help='Backends from ["tensorrt", "faster_whisper"]') parser.add_argument('--faster_whisper_custom_model_path', '-fw', type=str, default=None, - help="Custom Faster Whisper Model") + help="Custom Faster Whisper converted model path") parser.add_argument('--trt_model_path', '-trt', type=str, default=None,