# 🎬 Multilingual Video Classification (Beautiful + Voice Icon) import os, json, base64 from pathlib import Path import gradio as gr import torch, cv2, numpy as np from PIL import Image from gtts import gTTS from transformers import ( BlipProcessor, BlipForConditionalGeneration, AutoTokenizer, AutoModelForSequenceClassification, AutoModelForSeq2SeqLM ) # ---------- CONFIG ---------- MODEL_ID = "magedsar7an/caption-cls-en-small" # ← your HF model repo FRAMES_PER_VIDEO = 6 FRAME_SIZE = 384 device = "cuda" if torch.cuda.is_available() else "cpu" SUPPORTED_LANGS = { "en":"English","ar":"Arabic","fr":"French","tr":"Turkish", "es":"Spanish","de":"German","hi":"Hindi","id":"Indonesian" } MARIAN_TO_EN = { "ar":"Helsinki-NLP/opus-mt-ar-en", "fr":"Helsinki-NLP/opus-mt-fr-en", "tr":"Helsinki-NLP/opus-mt-tr-en", "es":"Helsinki-NLP/opus-mt-es-en", "de":"Helsinki-NLP/opus-mt-de-en", "hi":"Helsinki-NLP/opus-mt-hi-en", "id":"Helsinki-NLP/opus-mt-id-en", } LABEL_TRANSLATIONS = { "ar": {"clap":"تصفيق","drink":"يشرب","hug":"عناق","kick_ball":"ركل الكرة", "kiss":"قبلة","run":"يجري","sit":"يجلس","wave":"يلوح"}, "tr": {"clap":"alkış","drink":"içmek","hug":"sarılmak","kick_ball":"topa tekme", "kiss":"öpücük","run":"koşmak","sit":"oturmak","wave":"el sallamak"}, "fr": {"clap":"applaudir","drink":"boire","hug":"embrasser","kick_ball":"frapper le ballon", "kiss":"baiser","run":"courir","sit":"s’asseoir","wave":"saluer"}, "es": {"clap":"aplaudir","drink":"beber","hug":"abrazar","kick_ball":"patear la pelota", "kiss":"besar","run":"correr","sit":"sentarse","wave":"saludar"}, "de": {"clap":"klatschen","drink":"trinken","hug":"umarmen","kick_ball":"den Ball treten", "kiss":"küssen","run":"laufen","sit":"sitzen","wave":"winken"}, "hi": {"clap":"ताली बजाना","drink":"पीना","hug":"गले लगाना","kick_ball":"गेंद को मारना", "kiss":"चूमना","run":"दौड़ना","sit":"बैठना","wave":"हाथ हिलाना"}, "id": {"clap":"bertepuk tangan","drink":"minum","hug":"berpelukan","kick_ball":"menendang bola", "kiss":"cium","run":"berlari","sit":"duduk","wave":"melambaikan tangan"}, } # ---------- LOAD MODELS ---------- print("Loading BLIP captioner...") blip_proc = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") blip = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base").to(device).eval() print("Loading English classifier from HF Hub...") tok = AutoTokenizer.from_pretrained(MODEL_ID) cls = AutoModelForSequenceClassification.from_pretrained(MODEL_ID).to(device).eval() # id2label from model config (you embedded it during upload) cfg_map = getattr(cls.config, "id2label", None) if not cfg_map: raise RuntimeError("id2label not found in config.json; add it to your HF model.") # normalize keys to int id2label = {int(k): v for k, v in (cfg_map.items() if isinstance(cfg_map, dict) else enumerate(cfg_map))} print("✅ Models loaded successfully!") # ---------- HELPERS ---------- def _resolve_video_path(video): if isinstance(video, str): return video if os.path.exists(video) else None if isinstance(video, dict): p = video.get("path") or video.get("name") return p if (isinstance(p, str) and os.path.exists(p)) else None name = getattr(video, "name", None) if isinstance(name, str) and os.path.exists(name): return name return None def extract_frames(video_path, k=6, size=384): cap = cv2.VideoCapture(video_path) if not cap.isOpened(): return [] total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) or 0 idxs = np.linspace(0, max(total - 1, 0), num=k, dtype=int) if total > 0 else np.linspace(0, 240, num=k, dtype=int) frames = [] for i in idxs: cap.set(cv2.CAP_PROP_POS_FRAMES, int(i)) ok, frame = cap.read() if not ok or frame is None: continue h, w = frame.shape[:2] if h <= 0 or w <= 0: continue if h < w: new_h = size; new_w = int(w * (size / h)) else: new_w = size; new_h = int(h * (size / w)) frame = cv2.resize(frame, (new_w, new_h)) frames.append(Image.fromarray(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))) cap.release() return frames def blip_caption(img): inputs = blip_proc(images=img, return_tensors="pt").to(device) with torch.no_grad(): out = blip.generate(**inputs, max_new_tokens=30) return blip_proc.decode(out[0], skip_special_tokens=True).strip() def translate_to_en(texts, lang): if lang == "en": return texts model_name = MARIAN_TO_EN.get(lang) if not model_name: return texts try: tok_tr = AutoTokenizer.from_pretrained(model_name) mt = AutoModelForSeq2SeqLM.from_pretrained(model_name).to(device).eval() outs = [] for i in range(0, len(texts), 16): batch = texts[i:i + 16] enc = tok_tr(batch, return_tensors="pt", padding=True, truncation=True).to(device) with torch.no_grad(): gen = mt.generate(**enc, max_new_tokens=120) outs.extend(tok_tr.batch_decode(gen, skip_special_tokens=True)) return outs except Exception as e: print(f"⚠️ Translation failed: {e}") return texts def classify(texts): enc = tok(texts, return_tensors="pt", padding=True, truncation=True).to(device) with torch.no_grad(): logits = cls(**enc).logits probs = torch.softmax(logits, dim=-1).cpu().numpy() return probs # ---------- MAIN FN ---------- def classify_video(video, lang): try: if not video: return "
⚠️ Please upload a video first.
" video_path = _resolve_video_path(video) if not video_path: return "
❌ Could not find uploaded video path from Gradio input.
" frames = extract_frames(video_path, FRAMES_PER_VIDEO, FRAME_SIZE) if not frames: return "
❌ Could not extract frames. OpenCV could not decode the video.
" captions = [blip_caption(f) for f in frames] en_caps = translate_to_en(captions, lang) probs = classify(en_caps) pred = id2label[int(np.argmax(probs.mean(axis=0)))] localized = LABEL_TRANSLATIONS.get(lang, {}).get(pred, pred) # 🔊 TTS (fail-soft if blocked) audio_b64 = "" try: tts = gTTS(localized, lang=lang if lang in SUPPORTED_LANGS else "en") audio_path = "pred_voice.mp3" tts.save(audio_path) with open(audio_path, "rb") as f: audio_b64 = base64.b64encode(f.read()).decode() except Exception as e: print(f"⚠️ TTS failed: {e}") # 🎨 Card lang_name = SUPPORTED_LANGS.get(lang, "Unknown") btn = f"" if audio_b64 else "" html = f"""

🎬 Action Detected

{localized}

{btn}

({lang_name})

""" return html except Exception as e: import traceback; traceback.print_exc() return f"
❌ Error:
{e}
" # ---------- GRADIO UI ---------- custom_css = """ .gradio-container { background: linear-gradient(135deg,#0f2027,#203a43,#2c5364); color: white; } h1,h2,h3,label,p,.description {color: white !important;} footer {display:none !important;} """ title = "🎬 Multilingual Video Classification (Beautiful + Voice Icon)" description = """ 🌍Click the 🔊 icon to **hear the word pronounced** in that language. """ iface = gr.Interface( fn=classify_video, inputs=[ gr.Video(label="🎥 Upload Video", sources=["upload"], format="mp4"), gr.Radio(choices=list(SUPPORTED_LANGS.keys()), value="en", label="🌍 Choose Language"), ], outputs=gr.HTML(label="✨ Prediction Result"), title=title, description=description, theme="gradio/soft", css=custom_css, ) if __name__ == "__main__": iface.launch(server_name="0.0.0.0", server_port=int(os.environ.get("PORT", 7860)))