Update app.py

app.py

@@ -9,6 +9,12 @@ from diffusers.utils import export_to_video
 from huggingface_hub import hf_hub_download
 from safetensors.torch import load_file
 from PIL import Image
+from gradio_client import Client, file
+from moviepy.editor import VideoFileClip, AudioFileClip, concatenate_videoclips
+
+
+# using tango2 via Gradio python client 
+client = Client("declare-lab/tango2")
 
 # Constants
 bases = {
@@ -77,8 +83,44 @@ def generate_image(prompt, base, motion, step, progress=gr.Progress()):
         return None
 
     name = str(uuid.uuid4()).replace("-", "")
-    path = f"/tmp/{name}.mp4"
+    video_path = f"/tmp/{name}.mp4"
     export_to_video(output.frames[0], path, fps=10)
+    
+    audio_path = tango2(prompt)
+    final_video_path = fuse_together(audio_path, video_path)
+    
+    return final_video_path
+
+
+def tango2(prompt):
+    results = client.predict(
+      prompt=prompt,
+      steps=100,
+      guidance=3,
+      api_name="/predict"
+    )
+    return results
+
+def fuse_together(audio, video):
+    
+    # Load your video and audio files
+    video_clip = VideoFileClip(video)
+    audio_clip = AudioFileClip(audio)
+    
+    # Loop the video twice
+    looped_video = concatenate_videoclips([video_clip, video_clip])
+    
+    # Cut the audio to match the duration of the looped video
+    looped_audio = audio_clip.subclip(0, looped_video.duration)
+    
+    # Set the audio of the looped video to the adjusted audio clip
+    final_video = looped_video.set_audio(looped_audio)
+    
+    # Write the result to a file (output will be twice the length of the original video)
+    name = str(uuid.uuid4()).replace("-", "")
+    path = f"/tmp/{name}.mp4"
+    final_video.write_videofile(path, codec="libx264", audio_codec="aac")
+
     return path