Update app.py

app.py

@@ -1,28 +1,25 @@
-import gradio as gr
 import torch
-from transformers import VideoMAEForVideoClassification, VideoMAEFeatureExtractor
+import gradio as gr
 import cv2
 import numpy as np
-import tempfile
-import os
-
-# Load the pre-trained model
-model_name = "Sokaina55/xclip-base-patch32-finetuned-ssl-sign-language-recognition"
-device = "cuda" if torch.cuda.is_available() else "cpu"
+from transformers import VideoMAEForVideoClassification, VideoMAEImageProcessor
 
-feature_extractor = VideoMAEFeatureExtractor.from_pretrained(model_name)
-model = VideoMAEForVideoClassification.from_pretrained(model_name).to(device)
+# Model name
+model_name = "MCG-NJU/videomae-base"  # Ensure this is a valid model on Hugging Face
 
-def process_video(video_path):
-    """Processes video and predicts sign language word."""
-    if not os.path.exists(video_path):
-        return "Error: Video file not found"
+# Load model and processor
+model = VideoMAEForVideoClassification.from_pretrained(model_name)
+processor = VideoMAEImageProcessor.from_pretrained(model_name)
 
-    # Read video
+# Function to extract frames from video
+def extract_frames(video_path, num_frames=16):
     cap = cv2.VideoCapture(video_path)
     frames = []
-
-    while cap.isOpened():
+    total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+    
+    # Select frames evenly spaced throughout the video
+    for i in np.linspace(0, total_frames - 1, num_frames, dtype=int):
+        cap.set(cv2.CAP_PROP_POS_FRAMES, i)
         ret, frame = cap.read()
         if not ret:
             break
@@ -30,30 +27,34 @@ def process_video(video_path):
         frames.append(frame)
     
     cap.release()
+    
+    # Ensure exactly `num_frames` frames are used
+    while len(frames) < num_frames:
+        frames.append(frames[-1])  # Duplicate last frame if needed
+    
+    return frames
 
-    if len(frames) == 0:
-        return "Error: No frames extracted from the video"
-
-    # Preprocess frames
-    inputs = feature_extractor(frames, return_tensors="pt")
-    inputs = {k: v.to(device) for k, v in inputs.items()}
+# Function to process video and make predictions
+def process_video(video):
+    frames = extract_frames(video)
 
-    # Get predictions
+    # Process video frames with correct resizing and normalization
+    inputs = processor(frames, return_tensors="pt", sampling_rate=30, do_resize=True, size={"shortest_edge": 224}, do_normalize=True)
+    
     with torch.no_grad():
         outputs = model(**inputs)
     
-    predicted_class = outputs.logits.argmax(-1).item()
-    class_labels = model.config.id2label  # Map predictions to words
+    logits = outputs.logits
+    predicted_class = torch.argmax(logits, dim=1).item()
 
-    return f"Predicted word: {class_labels.get(predicted_class, 'Unknown')}"
+    return f"Predicted Class: {predicted_class}"
 
 # Gradio UI
-with gr.Blocks() as demo:
-    gr.Markdown("## Sign Language to Text Recognition")
-    video_input = gr.Video(label="Upload a sign language video")
-    output_text = gr.Textbox(label="Predicted Word")
-    btn = gr.Button("Predict")
-
-    btn.click(fn=process_video, inputs=video_input, outputs=output_text)
-
-demo.launch()
+iface = gr.Interface(
+    fn=process_video,
+    inputs=gr.Video(label="Upload a video"),
+    outputs=gr.Textbox(label="Prediction"),
+)
+
+# Launch app
+iface.launch(server_name="0.0.0.0", server_port=7860, share=True)