Create app.py

app.py

@@ -0,0 +1,86 @@
+import os
+import gradio as gr
+import torch
+import subprocess
+from PIL import Image
+from pathlib import Path
+
+# =========================================
+# 1. Define Hugging Face weights and paths
+# =========================================
+
+HF_DATASET_URL = "https://huggingface.co/datasets/roll-ai/FloVD-weights/resolve/main/ckpt"
+WEIGHT_FILES = {
+    "FVSM/FloVD_FVSM_Controlnet.pt": "FVSM/FloVD_FVSM_Controlnet.pt",
+    "OMSM/selected_blocks.safetensors": "OMSM/selected_blocks.safetensors",
+    "OMSM/pytorch_lora_weights.safetensors": "OMSM/pytorch_lora_weights.safetensors",
+    "others/depth_anything_v2_metric_hypersim_vitb.pth": "others/depth_anything_v2_metric_hypersim_vitb.pth"
+}
+
+def download_weights():
+    print("🔄 Downloading model weights...")
+    for rel_path in WEIGHT_FILES.values():
+        save_path = Path("ckpt") / rel_path
+        if not save_path.exists():
+            save_path.parent.mkdir(parents=True, exist_ok=True)
+            url = f"{HF_DATASET_URL}/{rel_path}"
+            print(f"📥 Downloading {url} → {save_path}")
+            subprocess.run(["wget", "-q", "-O", str(save_path), url], check=True)
+        else:
+            print(f"✅ Already exists: {save_path}")
+
+download_weights()
+
+# =========================================
+# 2. Import and load FloVD pipeline
+# =========================================
+
+from inference.flovd_demo import load_pipeline, generate_video
+
+pipeline = load_pipeline(
+    fvsm_path="ckpt/FVSM/FloVD_FVSM_Controlnet.pt",
+    omsm_path="ckpt/OMSM",
+    depth_path="ckpt/others/depth_anything_v2_metric_hypersim_vitb.pth",
+    device="cuda" if torch.cuda.is_available() else "cpu"
+)
+
+# =========================================
+# 3. Inference Function
+# =========================================
+
+def run_inference(image: Image.Image, prompt: str, cam_traj_path: str):
+    print("🚀 Running inference...")
+    output_path = generate_video(
+        image=image,
+        prompt=prompt,
+        cam_traj=cam_traj_path,
+        pipeline=pipeline,
+        num_frames=49,
+        fps=16,
+        controlnet_guidance_end=0.4,
+        flow_scale=(60, 36)
+    )
+    return output_path
+
+# =========================================
+# 4. Gradio UI
+# =========================================
+
+example_image = "assets/manual_poses/example_image.jpg"
+example_cam = "assets/cam_trajectory/dolly_zoom.txt"
+
+demo = gr.Interface(
+    fn=run_inference,
+    inputs=[
+        gr.Image(label="Input Image", type="pil"),
+        gr.Textbox(label="Text Prompt", value="A cinematic dolly zoom shot of a futuristic cityscape"),
+        gr.Textbox(label="Camera Trajectory File Path", value=example_cam),
+    ],
+    outputs=gr.Video(label="Generated Video"),
+    title="FloVD-CogVideoX 🌠",
+    description="Upload an image, enter a text prompt and a camera trajectory file path to generate a controlled video using CogVideoX + optical flow.",
+    examples=[[example_image, "A beautiful sunrise over a mountain range", example_cam]]
+)
+
+if __name__ == "__main__":
+    demo.launch()