latest

clear_stuck_tasks.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""
+Clear stuck tasks from Redis queue after worker restart.
+
+This script identifies tasks that are marked as "active" but have no
+corresponding worker processing them, and returns them to the queue.
+"""
+
+import redis
+import json
+import os
+import sys
+from typing import List, Dict, Any
+
+def get_redis_client():
+    """Get Redis client using the same config as the main app."""
+    try:
+        from modules.redis_config import get_redis_url
+        return redis.from_url(get_redis_url())
+    except:
+        redis_url = os.getenv('REDIS_URL', 'redis://localhost:6379')
+        return redis.from_url(redis_url)
+
+def check_stuck_tasks():
+    """Check for stuck tasks in Redis."""
+    client = get_redis_client()
+    
+    print("=== REDIS QUEUE STATUS ===")
+    processing_count = client.llen("document_processing")
+    failed_count = client.llen("document_processing:failed") 
+    active_count = client.scard("active_tasks")
+    
+    print(f"Processing queue: {processing_count} tasks")
+    print(f"Failed queue: {failed_count} tasks") 
+    print(f"Active tasks: {active_count} tasks")
+    
+    if active_count > 0:
+        print(f"\n⚠️  WARNING: {active_count} tasks marked as 'active' but workers may have been restarted!")
+        return True
+    elif processing_count > 0:
+        print(f"\n✅ {processing_count} tasks waiting in queue (normal)")
+        return False
+    else:
+        print("\n✅ No stuck tasks found")
+        return False
+
+def clear_stuck_tasks():
+    """Move stuck 'active' tasks back to processing queue."""
+    client = get_redis_client()
+    
+    # Get all active task IDs
+    active_tasks = client.smembers("active_tasks")
+    if not active_tasks:
+        print("No active tasks to clear")
+        return
+    
+    cleared = 0
+    for task_id in active_tasks:
+        task_id_str = task_id.decode('utf-8')
+        try:
+            # Try to get the task data
+            task_data = client.get(f"task:{task_id_str}")
+            if task_data:
+                # Parse task to determine priority queue
+                task_obj = json.loads(task_data.decode('utf-8'))
+                priority = task_obj.get('priority', 'normal')
+                
+                queue_key = {
+                    'high': 'document_processing:high',
+                    'normal': 'document_processing', 
+                    'low': 'document_processing:low'
+                }.get(priority, 'document_processing')
+                
+                # Move task back to processing queue
+                client.lpush(queue_key, task_id_str)
+                client.srem("active_tasks", task_id_str)
+                cleared += 1
+                print(f"✅ Cleared stuck task: {task_id_str} ({task_obj.get('service', 'unknown')}/{task_obj.get('task_type', 'unknown')})")
+            else:
+                # Task data not found, just remove from active set
+                client.srem("active_tasks", task_id_str)
+                cleared += 1
+                print(f"🗑️  Removed orphaned active task: {task_id_str}")
+                
+        except Exception as e:
+            print(f"❌ Failed to clear task {task_id_str}: {e}")
+    
+    print(f"\n✅ Cleared {cleared} stuck tasks")
+
+def main():
+    """Main function."""
+    print("🔍 Checking for stuck tasks in Redis...")
+    
+    try:
+        has_stuck = check_stuck_tasks()
+        
+        if has_stuck:
+            response = input("\n❓ Clear stuck tasks? (y/N): ")
+            if response.lower() in ['y', 'yes']:
+                print("\n🧹 Clearing stuck tasks...")
+                clear_stuck_tasks()
+                print("\n✅ Done! Tasks should now be processed by workers.")
+            else:
+                print("Skipped clearing tasks.")
+        
+    except Exception as e:
+        print(f"❌ Error: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()