lora

app.py

@@ -13,7 +13,7 @@ import tempfile
 import numpy as np
 from PIL import Image
 import random
-from diffusers.hooks import apply_first_block_cache, FirstBlockCacheConfig
+
 from optimization import optimize_pipeline_
 
 
@@ -23,7 +23,7 @@ LANDSCAPE_WIDTH = 832
 LANDSCAPE_HEIGHT = 480
 MAX_SEED = np.iinfo(np.int32).max
 
-FIXED_FPS = 24
+FIXED_FPS = 16
 MIN_FRAMES_MODEL = 8
 MAX_FRAMES_MODEL = 81
 
@@ -45,8 +45,17 @@ pipe = WanImageToVideoPipeline.from_pretrained(MODEL_ID,
     torch_dtype=torch.bfloat16,
 ).to('cuda')
 
+# load, fuse, unload before compilation
+pipe.load_lora_weights(
+   "Kijai/WanVideo_comfy", 
+   weight_name="Lightx2v/lightx2v_I2V_14B_480p_cfg_step_distill_rank32_bf16.safetensors", 
+   adapter_name="lightx2v"
+)
+
+pipe.set_adapters(["lightx2v"], adapter_weights=[1.0])
+pipe.fuse_lora(adapter_names=["lightx2v"], lora_scale=1.0)
+pipe.unload_lora_weights()
 
-#apply_first_block_cache(pipe.transformer, FirstBlockCacheConfig(threshold=0.05))
 
 optimize_pipeline_(pipe,
     image=Image.new('RGB', (LANDSCAPE_WIDTH, LANDSCAPE_HEIGHT)),
@@ -89,6 +98,7 @@ def get_duration(
     negative_prompt,
     duration_seconds,
     guidance_scale,
+    guidance_scale_2,
     steps,
     seed,
     randomize_seed,
@@ -102,8 +112,9 @@ def generate_video(
     prompt,
     negative_prompt=default_negative_prompt,
     duration_seconds = MAX_DURATION,
-    guidance_scale = 1,
-    steps = 4,
+    guidance_scale = 2.5,
+    guidance_scale_2 = 3.5,
+    steps = 6,
     seed = 42,
     randomize_seed = False,
     progress=gr.Progress(track_tqdm=True),
@@ -124,6 +135,8 @@ def generate_video(
             Defaults to 2. Clamped between MIN_FRAMES_MODEL/FIXED_FPS and MAX_FRAMES_MODEL/FIXED_FPS.
         guidance_scale (float, optional): Controls adherence to the prompt. Higher values = more adherence.
             Defaults to 1.0. Range: 0.0-20.0.
+        guidance_scale_2 (float, optional): Controls adherence to the prompt. Higher values = more adherence.
+            Defaults to 1.0. Range: 0.0-20.0.
         steps (int, optional): Number of inference steps. More steps = higher quality but slower.
             Defaults to 4. Range: 1-30.
         seed (int, optional): Random seed for reproducible results. Defaults to 42.
@@ -162,6 +175,7 @@ def generate_video(
         width=resized_image.width,
         num_frames=num_frames,
         guidance_scale=float(guidance_scale),
+        guidance_scale_2=float(guidance_scale_2),
         num_inference_steps=int(steps),
         generator=torch.Generator(device="cuda").manual_seed(current_seed),
     ).frames[0]
@@ -186,8 +200,9 @@ with gr.Blocks() as demo:
                 negative_prompt_input = gr.Textbox(label="Negative Prompt", value=default_negative_prompt, lines=3)
                 seed_input = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=42, interactive=True)
                 randomize_seed_checkbox = gr.Checkbox(label="Randomize seed", value=True, interactive=True)
-                steps_slider = gr.Slider(minimum=1, maximum=40, step=1, value=35, label="Inference Steps") 
-                guidance_scale_input = gr.Slider(minimum=0.0, maximum=20.0, step=0.5, value=1.0, label="Guidance Scale", visible=False)
+                steps_slider = gr.Slider(minimum=1, maximum=30, step=1, value=6, label="Inference Steps") 
+                guidance_scale_input = gr.Slider(minimum=0.0, maximum=10.0, step=0.5, value=2.5, label="Guidance Scale - high noise stage", visible=False)
+                guidance_scale_2_input = gr.Slider(minimum=0.0, maximum=10.0, step=0.5, value=3.5, label="Guidance Scale 2 - low noise stage", visible=False)
 
             generate_button = gr.Button("Generate Video", variant="primary")
         with gr.Column():
@@ -196,7 +211,7 @@ with gr.Blocks() as demo:
     ui_inputs = [
         input_image_component, prompt_input,
         negative_prompt_input, duration_seconds_input,
-        guidance_scale_input, steps_slider, seed_input, randomize_seed_checkbox
+        guidance_scale_input, guidance_scale_2_input, steps_slider, seed_input, randomize_seed_checkbox
     ]
     generate_button.click(fn=generate_video, inputs=ui_inputs, outputs=[video_output, seed_input])