Spaces:
Running
on
Zero
Running
on
Zero
| ############################################# | |
| # from diffusers_helper.hf_login import login | |
| # ํ์์ HF ๋ก๊ทธ์ธ ์ฌ์ฉ (์ฃผ์ ํด์ ํ) | |
| ############################################# | |
| import os | |
| os.environ['HF_HOME'] = os.path.abspath( | |
| os.path.realpath(os.path.join(os.path.dirname(__file__), './hf_download')) | |
| ) | |
| import gradio as gr | |
| import torch | |
| import traceback | |
| import einops | |
| import safetensors.torch as sf | |
| import numpy as np | |
| import math | |
| import time | |
| # Hugging Face Spaces ํ๊ฒฝ ์ธ์ง ํ์ธ | |
| IN_HF_SPACE = os.environ.get('SPACE_ID') is not None | |
| # --------- ๋ฒ์ญ ๋์ ๋๋ฆฌ(์์ด ๊ณ ์ ) --------- | |
| translations = { | |
| "en": { | |
| "title": "FramePack - Image to Video Generation", | |
| "upload_image": "Upload Image", | |
| "prompt": "Prompt", | |
| "quick_prompts": "Quick Prompts", | |
| "start_generation": "Generate", | |
| "stop_generation": "Stop", | |
| "use_teacache": "Use TeaCache", | |
| "teacache_info": "Faster speed, but may result in slightly worse finger and hand generation.", | |
| "negative_prompt": "Negative Prompt", | |
| "seed": "Seed", | |
| # ์ต๋ 4์ด๋ก UI ํ๊ธฐ ์์ | |
| "video_length": "Video Length (max 4 seconds)", | |
| "latent_window": "Latent Window Size", | |
| "steps": "Inference Steps", | |
| "steps_info": "Changing this value is not recommended.", | |
| "cfg_scale": "CFG Scale", | |
| "distilled_cfg": "Distilled CFG Scale", | |
| "distilled_cfg_info": "Changing this value is not recommended.", | |
| "cfg_rescale": "CFG Rescale", | |
| "gpu_memory": "GPU Memory Preservation (GB) (larger means slower)", | |
| "gpu_memory_info": "Set this to a larger value if you encounter OOM errors. Larger values cause slower speed.", | |
| "next_latents": "Next Latents", | |
| "generated_video": "Generated Video", | |
| "sampling_note": "Note: The model predicts future frames from past frames. If the start action isn't immediately visible, please wait for more frames.", | |
| "error_message": "Error", | |
| "processing_error": "Processing error", | |
| "network_error": "Network connection is unstable, model download timed out. Please try again later.", | |
| "memory_error": "GPU memory insufficient, please try increasing GPU memory preservation value or reduce video length.", | |
| "model_error": "Failed to load model, possibly due to network issues or high server load. Please try again later.", | |
| "partial_video": "Processing error, but partial video has been generated", | |
| "processing_interrupt": "Processing was interrupted, but partial video has been generated" | |
| } | |
| } | |
| def get_translation(key): | |
| return translations["en"].get(key, key) | |
| ############################################# | |
| # diffusers_helper ๊ด๋ จ ์ํฌํธ | |
| ############################################# | |
| from diffusers_helper.thread_utils import AsyncStream, async_run | |
| from diffusers_helper.gradio.progress_bar import make_progress_bar_css, make_progress_bar_html | |
| from diffusers_helper.memory import ( | |
| cpu, | |
| gpu, | |
| get_cuda_free_memory_gb, | |
| move_model_to_device_with_memory_preservation, | |
| offload_model_from_device_for_memory_preservation, | |
| fake_diffusers_current_device, | |
| DynamicSwapInstaller, | |
| unload_complete_models, | |
| load_model_as_complete | |
| ) | |
| from diffusers_helper.utils import ( | |
| generate_timestamp, | |
| save_bcthw_as_mp4, | |
| resize_and_center_crop, | |
| crop_or_pad_yield_mask, | |
| soft_append_bcthw | |
| ) | |
| from diffusers_helper.bucket_tools import find_nearest_bucket | |
| from diffusers_helper.hunyuan import ( | |
| encode_prompt_conds, vae_encode, vae_decode, vae_decode_fake | |
| ) | |
| from diffusers_helper.clip_vision import hf_clip_vision_encode | |
| from diffusers_helper.models.hunyuan_video_packed import HunyuanVideoTransformer3DModelPacked | |
| from diffusers_helper.pipelines.k_diffusion_hunyuan import sample_hunyuan | |
| from diffusers import AutoencoderKLHunyuanVideo | |
| from transformers import ( | |
| LlamaModel, CLIPTextModel, | |
| LlamaTokenizerFast, CLIPTokenizer, | |
| SiglipVisionModel, SiglipImageProcessor | |
| ) | |
| ############################################# | |
| # GPU ์ฒดํฌ | |
| ############################################# | |
| GPU_AVAILABLE = torch.cuda.is_available() | |
| free_mem_gb = 0.0 | |
| high_vram = False | |
| if GPU_AVAILABLE: | |
| try: | |
| free_mem_gb = torch.cuda.get_device_properties(0).total_memory / 1e9 | |
| high_vram = (free_mem_gb > 60) | |
| except: | |
| pass | |
| print(f"GPU Available: {GPU_AVAILABLE}, free_mem_gb={free_mem_gb}, high_vram={high_vram}") | |
| cpu_fallback_mode = not GPU_AVAILABLE | |
| last_update_time = time.time() | |
| ############################################# | |
| # ๋ชจ๋ธ ๋ก๋ (์ ์ญ) | |
| ############################################# | |
| text_encoder = None | |
| text_encoder_2 = None | |
| tokenizer = None | |
| tokenizer_2 = None | |
| vae = None | |
| feature_extractor = None | |
| image_encoder = None | |
| transformer = None | |
| # ์๋ ๋ก์ง์ ์ง๋ฌธ์ ์ ์๋ '๋ ๋ฒ์งธ ์ฝ๋'์ ๋ชจ๋ธ ๋ก๋ ๋ถ๋ถ์ ๊ฑฐ์ ๊ทธ๋๋ก ์ฌ์ฉ | |
| def load_global_models(): | |
| global text_encoder, text_encoder_2, tokenizer, tokenizer_2 | |
| global vae, feature_extractor, image_encoder, transformer | |
| global cpu_fallback_mode | |
| # ์ด๋ฏธ ๋ก๋๋์์ผ๋ฉด ํจ์ค | |
| if transformer is not None: | |
| return | |
| # GPU ๋ฉ๋ชจ๋ฆฌ ์ ๋ณด | |
| device = gpu if GPU_AVAILABLE else cpu | |
| # diffusers_helper.memory.get_cuda_free_memory_gb(gpu)๋ก ๋ ์ ํํ ๊ตฌํด๋ ๋จ | |
| print("Loading models...") | |
| # ======== ์ค ์ฝ๋: ๋ ๋ฒ์งธ ์์ ๊ธฐ์ค ========= | |
| # (1) ํ์ด๋ธ๋ฆฌ๋ (if high_vram -> GPU๋ก ๋ก๋, ์๋๋ฉด CPU + DynamicSwap) | |
| # ๋ฐ๋์ float16, bfloat16๋ก ๋ก๋ | |
| text_encoder_local = LlamaModel.from_pretrained( | |
| "hunyuanvideo-community/HunyuanVideo", | |
| subfolder='text_encoder', | |
| torch_dtype=torch.float16 | |
| ).cpu() | |
| text_encoder_2_local = CLIPTextModel.from_pretrained( | |
| "hunyuanvideo-community/HunyuanVideo", | |
| subfolder='text_encoder_2', | |
| torch_dtype=torch.float16 | |
| ).cpu() | |
| tokenizer_local = LlamaTokenizerFast.from_pretrained( | |
| "hunyuanvideo-community/HunyuanVideo", | |
| subfolder='tokenizer' | |
| ) | |
| tokenizer_2_local = CLIPTokenizer.from_pretrained( | |
| "hunyuanvideo-community/HunyuanVideo", | |
| subfolder='tokenizer_2' | |
| ) | |
| vae_local = AutoencoderKLHunyuanVideo.from_pretrained( | |
| "hunyuanvideo-community/HunyuanVideo", | |
| subfolder='vae', | |
| torch_dtype=torch.float16 | |
| ).cpu() | |
| feature_extractor_local = SiglipImageProcessor.from_pretrained( | |
| "lllyasviel/flux_redux_bfl", subfolder='feature_extractor' | |
| ) | |
| image_encoder_local = SiglipVisionModel.from_pretrained( | |
| "lllyasviel/flux_redux_bfl", | |
| subfolder='image_encoder', | |
| torch_dtype=torch.float16 | |
| ).cpu() | |
| # FramePack_F1_I2V_HY_20250503 (bfloat16) | |
| transformer_local = HunyuanVideoTransformer3DModelPacked.from_pretrained( | |
| 'lllyasviel/FramePack_F1_I2V_HY_20250503', | |
| torch_dtype=torch.bfloat16 | |
| ).cpu() | |
| # eval & dtype | |
| vae_local.eval() | |
| text_encoder_local.eval() | |
| text_encoder_2_local.eval() | |
| image_encoder_local.eval() | |
| transformer_local.eval() | |
| # VAE slicing for low VRAM | |
| if not high_vram: | |
| vae_local.enable_slicing() | |
| vae_local.enable_tiling() | |
| # ์คํ๋ก๋์ฉ | |
| transformer_local.high_quality_fp32_output_for_inference = True | |
| transformer_local.to(dtype=torch.bfloat16) | |
| vae_local.to(dtype=torch.float16) | |
| image_encoder_local.to(dtype=torch.float16) | |
| text_encoder_local.to(dtype=torch.float16) | |
| text_encoder_2_local.to(dtype=torch.float16) | |
| # requires_grad_(False) | |
| for m in [vae_local, text_encoder_local, text_encoder_2_local, image_encoder_local, transformer_local]: | |
| m.requires_grad_(False) | |
| # GPU ๋ชจ๋ & VRAM ๋ง์ผ๋ฉด ์ ๋ถ GPU | |
| # ๊ทธ๋ ์ง ์์ผ๋ฉด DynamicSwap | |
| if GPU_AVAILABLE: | |
| if not high_vram: | |
| DynamicSwapInstaller.install_model(transformer_local, device=gpu) | |
| DynamicSwapInstaller.install_model(text_encoder_local, device=gpu) | |
| else: | |
| text_encoder_local.to(gpu) | |
| text_encoder_2_local.to(gpu) | |
| image_encoder_local.to(gpu) | |
| vae_local.to(gpu) | |
| transformer_local.to(gpu) | |
| else: | |
| cpu_fallback_mode = True | |
| # ๊ธ๋ก๋ฒ์ ํ ๋น | |
| print("Model loaded.") | |
| text_encoder = text_encoder_local | |
| text_encoder_2 = text_encoder_2_local | |
| tokenizer = tokenizer_local | |
| tokenizer_2 = tokenizer_2_local | |
| vae = vae_local | |
| feature_extractor = feature_extractor_local | |
| image_encoder = image_encoder_local | |
| transformer = transformer_local | |
| ############################################# | |
| # Worker ๋ก์ง (๋ ๋ฒ์งธ ์ฝ๋) ๊ทธ๋๋ก | |
| ############################################# | |
| stream = AsyncStream() | |
| outputs_folder = './outputs/' | |
| os.makedirs(outputs_folder, exist_ok=True) | |
| def worker( | |
| input_image, prompt, n_prompt, seed, | |
| total_second_length, latent_window_size, steps, | |
| cfg, gs, rs, gpu_memory_preservation, use_teacache | |
| ): | |
| """ | |
| ์ค์ ์ํ๋ง ๋ก์ง (๋ ๋ฒ์งธ ์ฝ๋ ๊ธฐ๋ฐ) | |
| """ | |
| load_global_models() # ๋ชจ๋ธ ๋ก๋ฉ | |
| global text_encoder, text_encoder_2, tokenizer, tokenizer_2 | |
| global vae, feature_extractor, image_encoder, transformer | |
| global last_update_time | |
| # ์ต๋ 4์ด๋ก ๊ณ ์ | |
| total_second_length = min(total_second_length, 4.0) | |
| total_latent_sections = (total_second_length * 30) / (latent_window_size * 4) | |
| total_latent_sections = int(max(round(total_latent_sections), 1)) | |
| job_id = generate_timestamp() | |
| stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Starting ...')))) | |
| try: | |
| # GPU ์ ์ ๊ฒฝ์ฐ Unload | |
| if not high_vram and GPU_AVAILABLE: | |
| unload_complete_models( | |
| text_encoder, text_encoder_2, image_encoder, vae, transformer | |
| ) | |
| # Text encoding | |
| stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Text encoding ...')))) | |
| if not high_vram and GPU_AVAILABLE: | |
| fake_diffusers_current_device(text_encoder, gpu) | |
| load_model_as_complete(text_encoder_2, target_device=gpu) | |
| llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2) | |
| if cfg == 1.0: | |
| llama_vec_n, clip_l_pooler_n = torch.zeros_like(llama_vec), torch.zeros_like(clip_l_pooler) | |
| else: | |
| llama_vec_n, clip_l_pooler_n = encode_prompt_conds(n_prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2) | |
| llama_vec, llama_mask = crop_or_pad_yield_mask(llama_vec, length=512) | |
| llama_vec_n, llama_mask_n = crop_or_pad_yield_mask(llama_vec_n, length=512) | |
| # Image processing | |
| stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Image processing ...')))) | |
| H, W, C = input_image.shape | |
| height, width = find_nearest_bucket(H, W, resolution=640) | |
| if cpu_fallback_mode: | |
| height = min(height, 320) | |
| width = min(width, 320) | |
| input_image_np = resize_and_center_crop(input_image, target_width=width, target_height=height) | |
| Image.fromarray(input_image_np).save(os.path.join(outputs_folder, f'{job_id}.png')) | |
| input_image_pt = torch.from_numpy(input_image_np).float() / 127.5 - 1 | |
| input_image_pt = input_image_pt.permute(2, 0, 1)[None, :, None] | |
| # VAE encode | |
| stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'VAE encoding ...')))) | |
| if not high_vram and GPU_AVAILABLE: | |
| load_model_as_complete(vae, target_device=gpu) | |
| start_latent = vae_encode(input_image_pt, vae) | |
| # CLIP Vision | |
| stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'CLIP Vision encoding ...')))) | |
| if not high_vram and GPU_AVAILABLE: | |
| load_model_as_complete(image_encoder, target_device=gpu) | |
| image_encoder_output = hf_clip_vision_encode(input_image_np, feature_extractor, image_encoder) | |
| image_encoder_last_hidden_state = image_encoder_output.last_hidden_state | |
| # dtype | |
| llama_vec = llama_vec.to(transformer.dtype) | |
| llama_vec_n = llama_vec_n.to(transformer.dtype) | |
| clip_l_pooler = clip_l_pooler.to(transformer.dtype) | |
| clip_l_pooler_n = clip_l_pooler_n.to(transformer.dtype) | |
| image_encoder_last_hidden_state = image_encoder_last_hidden_state.to(transformer.dtype) | |
| # Start sampling | |
| stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Start sampling ...')))) | |
| rnd = torch.Generator("cpu").manual_seed(seed) | |
| # ์ด๊ธฐ history latents | |
| history_latents = torch.zeros(size=(1, 16, 16 + 2 + 1, height // 8, width // 8), dtype=torch.float32).cpu() | |
| history_pixels = None | |
| # start_latent ๋ถ์ด๊ธฐ | |
| history_latents = torch.cat([history_latents, start_latent.to(history_latents)], dim=2) | |
| total_generated_latent_frames = 1 | |
| for section_index in range(total_latent_sections): | |
| if stream.input_queue.top() == 'end': | |
| stream.output_queue.push(('end', None)) | |
| return | |
| print(f'Section {section_index+1}/{total_latent_sections}') | |
| if not high_vram and GPU_AVAILABLE: | |
| unload_complete_models() | |
| move_model_to_device_with_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=gpu_memory_preservation) | |
| # teacache | |
| if use_teacache: | |
| transformer.initialize_teacache(enable_teacache=True, num_steps=steps) | |
| else: | |
| transformer.initialize_teacache(enable_teacache=False) | |
| def callback(d): | |
| preview = d['denoised'] | |
| preview = vae_decode_fake(preview) | |
| preview = (preview * 255.0).detach().cpu().numpy().clip(0, 255).astype(np.uint8) | |
| preview = einops.rearrange(preview, 'b c t h w -> (b h) (t w) c') | |
| if stream.input_queue.top() == 'end': | |
| stream.output_queue.push(('end', None)) | |
| raise KeyboardInterrupt('User stops generation.') | |
| current_step = d['i'] + 1 | |
| percentage = int(100.0 * current_step / steps) | |
| hint = f'Sampling {current_step}/{steps}' | |
| desc = f'Section {section_index+1}/{total_latent_sections}' | |
| stream.output_queue.push(('progress', (preview, desc, make_progress_bar_html(percentage, hint)))) | |
| return | |
| # indices | |
| frames_per_section = latent_window_size * 4 - 3 | |
| indices = torch.arange(0, sum([1, 16, 2, 1, latent_window_size])).unsqueeze(0) | |
| ( | |
| clean_latent_indices_start, | |
| clean_latent_4x_indices, | |
| clean_latent_2x_indices, | |
| clean_latent_1x_indices, | |
| latent_indices | |
| ) = indices.split([1, 16, 2, 1, latent_window_size], dim=1) | |
| clean_latent_indices = torch.cat([clean_latent_indices_start, clean_latent_1x_indices], dim=1) | |
| clean_latents_4x, clean_latents_2x, clean_latents_1x = history_latents[:, :, -19:, :, :].split([16, 2, 1], dim=2) | |
| clean_latents = torch.cat([start_latent.to(history_latents), clean_latents_1x], dim=2) | |
| try: | |
| generated_latents = sample_hunyuan( | |
| transformer=transformer, | |
| sampler='unipc', | |
| width=width, | |
| height=height, | |
| frames=frames_per_section, | |
| real_guidance_scale=cfg, | |
| distilled_guidance_scale=gs, | |
| guidance_rescale=rs, | |
| num_inference_steps=steps, | |
| generator=rnd, | |
| prompt_embeds=llama_vec, | |
| prompt_embeds_mask=llama_mask, | |
| prompt_poolers=clip_l_pooler, | |
| negative_prompt_embeds=llama_vec_n, | |
| negative_prompt_embeds_mask=llama_mask_n, | |
| negative_prompt_poolers=clip_l_pooler_n, | |
| device=gpu if GPU_AVAILABLE else cpu, | |
| dtype=torch.bfloat16, | |
| image_embeddings=image_encoder_last_hidden_state, | |
| latent_indices=latent_indices, | |
| clean_latents=clean_latents, | |
| clean_latent_indices=clean_latent_indices, | |
| clean_latents_2x=clean_latents_2x, | |
| clean_latent_2x_indices=clean_latent_2x_indices, | |
| clean_latents_4x=clean_latents_4x, | |
| clean_latent_4x_indices=clean_latent_4x_indices, | |
| callback=callback | |
| ) | |
| except KeyboardInterrupt: | |
| print("User cancelled.") | |
| stream.output_queue.push(('end', None)) | |
| return | |
| except Exception as e: | |
| traceback.print_exc() | |
| stream.output_queue.push(('end', None)) | |
| return | |
| total_generated_latent_frames += generated_latents.shape[2] | |
| history_latents = torch.cat([history_latents, generated_latents.to(history_latents)], dim=2) | |
| if not high_vram and GPU_AVAILABLE: | |
| offload_model_from_device_for_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=8) | |
| load_model_as_complete(vae, target_device=gpu) | |
| real_history_latents = history_latents[:, :, -total_generated_latent_frames:, :, :] | |
| if history_pixels is None: | |
| history_pixels = vae_decode(real_history_latents, vae).cpu() | |
| else: | |
| section_latent_frames = latent_window_size * 2 | |
| overlapped_frames = frames_per_section | |
| current_pixels = vae_decode(real_history_latents[:, :, -section_latent_frames:], vae).cpu() | |
| history_pixels = soft_append_bcthw(history_pixels, current_pixels, overlapped_frames) | |
| if not high_vram and GPU_AVAILABLE: | |
| unload_complete_models() | |
| output_filename = os.path.join(outputs_folder, f'{job_id}_{total_generated_latent_frames}.mp4') | |
| save_bcthw_as_mp4(history_pixels, output_filename, fps=30, crf=16) # CRF=16 | |
| stream.output_queue.push(('file', output_filename)) | |
| except: | |
| traceback.print_exc() | |
| if not high_vram and GPU_AVAILABLE: | |
| unload_complete_models(text_encoder, text_encoder_2, image_encoder, vae, transformer) | |
| stream.output_queue.push(('end', None)) | |
| return | |
| def end_process(): | |
| """ | |
| ์ค๋จ ์์ฒญ | |
| """ | |
| global stream | |
| stream.input_queue.push('end') | |
| # Gradio์์ ์ด worker ํจ์๋ฅผ ๋น๋๊ธฐ๋ก ํธ์ถ | |
| def process( | |
| input_image, prompt, n_prompt, seed, | |
| total_second_length, latent_window_size, steps, | |
| cfg, gs, rs, gpu_memory_preservation, use_teacache | |
| ): | |
| global stream | |
| if input_image is None: | |
| raise ValueError("No input image provided.") | |
| yield None, None, "", "", gr.update(interactive=False), gr.update(interactive=True) | |
| stream = AsyncStream() | |
| async_run( | |
| worker, | |
| input_image, prompt, n_prompt, seed, | |
| total_second_length, latent_window_size, steps, | |
| cfg, gs, rs, gpu_memory_preservation, use_teacache | |
| ) | |
| output_filename = None | |
| prev_filename = None | |
| error_message = None | |
| while True: | |
| flag, data = stream.output_queue.next() | |
| if flag == 'file': | |
| output_filename = data | |
| prev_filename = output_filename | |
| yield output_filename, gr.update(), gr.update(), "", gr.update(interactive=False), gr.update(interactive=True) | |
| elif flag == 'progress': | |
| preview, desc, html = data | |
| yield gr.update(), gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True) | |
| elif flag == 'error': | |
| error_message = data | |
| print(f"Error: {error_message}") | |
| elif flag == 'end': | |
| if output_filename is None and prev_filename: | |
| output_filename = prev_filename | |
| # ์๋ฌ๊ฐ ์์์ผ๋ฉด ์๋ฌ ํ์ | |
| if error_message: | |
| yield ( | |
| output_filename, # ๋ง์ง๋ง ํ์ผ (๋๋ None) | |
| gr.update(visible=False), | |
| gr.update(), | |
| f"<div style='color:red;'>{error_message}</div>", | |
| gr.update(interactive=True), | |
| gr.update(interactive=False) | |
| ) | |
| else: | |
| yield ( | |
| output_filename, gr.update(visible=False), gr.update(), "", gr.update(interactive=True), gr.update(interactive=False) | |
| ) | |
| break | |
| # UI CSS | |
| def make_custom_css(): | |
| base_progress_css = make_progress_bar_css() | |
| pastel_css = """ | |
| body { | |
| background: #faf9ff !important; | |
| font-family: "Noto Sans", sans-serif; | |
| } | |
| #app-container { | |
| max-width: 1200px; | |
| margin: 0 auto; | |
| padding: 1rem; | |
| position: relative; | |
| } | |
| #app-container h1 { | |
| color: #5F5AA2; | |
| margin-bottom: 1.2rem; | |
| font-weight: 700; | |
| text-shadow: 1px 1px 2px #bbb; | |
| } | |
| .gr-panel { | |
| background: #ffffffcc; | |
| border: 1px solid #e1dff0; | |
| border-radius: 8px; | |
| padding: 1rem; | |
| box-shadow: 0 1px 3px rgba(0,0,0,0.1); | |
| } | |
| .button-container button { | |
| min-height: 45px; | |
| font-size: 1rem; | |
| font-weight: 600; | |
| border-radius: 6px; | |
| } | |
| .button-container button#start-button { | |
| background-color: #A289E3 !important; | |
| color: #fff !important; | |
| border: 1px solid #a58de2; | |
| } | |
| .button-container button#stop-button { | |
| background-color: #F48A9B !important; | |
| color: #fff !important; | |
| border: 1px solid #f18fa0; | |
| } | |
| .button-container button:hover { | |
| filter: brightness(0.95); | |
| } | |
| .preview-container, .video-container { | |
| border: 1px solid #ded9f2; | |
| border-radius: 8px; | |
| overflow: hidden; | |
| } | |
| .progress-container { | |
| margin-top: 15px; | |
| margin-bottom: 15px; | |
| } | |
| .error-message { | |
| background-color: #FFF5F5; | |
| border: 1px solid #FED7D7; | |
| color: #E53E3E; | |
| padding: 10px; | |
| border-radius: 4px; | |
| margin-top: 10px; | |
| font-weight: 500; | |
| } | |
| @media (max-width: 768px) { | |
| #app-container { | |
| padding: 0.5rem; | |
| } | |
| .mobile-full-width { | |
| flex-direction: column !important; | |
| } | |
| .mobile-full-width > .gr-block { | |
| width: 100% !important; | |
| } | |
| } | |
| """ | |
| return base_progress_css + pastel_css | |
| css = make_custom_css() | |
| # ์ํ ํ๋กฌํํธ | |
| quick_prompts = [ | |
| ["The girl dances gracefully, with clear movements, full of charm."], | |
| ["A character doing some simple body movements."] | |
| ] | |
| # Gradio UI | |
| block = gr.Blocks(css=css).queue() | |
| with block: | |
| gr.HTML("<div id='app-container'><h1>FramePack - Image to Video Generation</h1></div>") | |
| with gr.Row(elem_classes="mobile-full-width"): | |
| # ์ผ์ชฝ | |
| with gr.Column(scale=1, elem_classes="gr-panel"): | |
| input_image = gr.Image( | |
| label=get_translation("upload_image"), | |
| type="numpy", | |
| height=320 | |
| ) | |
| prompt = gr.Textbox( | |
| label=get_translation("prompt"), | |
| value='' | |
| ) | |
| example_quick_prompts = gr.Dataset( | |
| samples=quick_prompts, | |
| label=get_translation("quick_prompts"), | |
| samples_per_page=1000, | |
| components=[prompt] | |
| ) | |
| example_quick_prompts.click( | |
| fn=lambda x: x[0], | |
| inputs=[example_quick_prompts], | |
| outputs=prompt, | |
| show_progress=False, | |
| queue=False | |
| ) | |
| # ์ค๋ฅธ์ชฝ | |
| with gr.Column(scale=1, elem_classes="gr-panel"): | |
| with gr.Row(elem_classes="button-container"): | |
| start_button = gr.Button( | |
| value=get_translation("start_generation"), | |
| elem_id="start-button", | |
| variant="primary" | |
| ) | |
| stop_button = gr.Button( | |
| value=get_translation("stop_generation"), | |
| elem_id="stop-button", | |
| interactive=False | |
| ) | |
| result_video = gr.Video( | |
| label=get_translation("generated_video"), | |
| autoplay=True, | |
| loop=True, | |
| height=320, | |
| elem_classes="video-container" | |
| ) | |
| preview_image = gr.Image( | |
| label=get_translation("next_latents"), | |
| visible=False, | |
| height=150, | |
| elem_classes="preview-container" | |
| ) | |
| gr.Markdown(get_translation("sampling_note")) | |
| with gr.Group(elem_classes="progress-container"): | |
| progress_desc = gr.Markdown('') | |
| progress_bar = gr.HTML('') | |
| error_message = gr.HTML('', visible=True) | |
| # Advanced | |
| with gr.Accordion("Advanced Settings", open=False, elem_classes="gr-panel"): | |
| use_teacache = gr.Checkbox( | |
| label=get_translation("use_teacache"), | |
| value=True, | |
| info=get_translation("teacache_info") | |
| ) | |
| n_prompt = gr.Textbox(label=get_translation("negative_prompt"), value="", visible=False) | |
| seed = gr.Number( | |
| label=get_translation("seed"), | |
| value=31337, | |
| precision=0 | |
| ) | |
| # ๊ธฐ๋ณธ 2์ด, ์ต๋ 4์ด | |
| total_second_length = gr.Slider( | |
| label=get_translation("video_length"), | |
| minimum=1, | |
| maximum=4, | |
| value=2, | |
| step=0.1 | |
| ) | |
| latent_window_size = gr.Slider( | |
| label=get_translation("latent_window"), | |
| minimum=1, | |
| maximum=33, | |
| value=9, | |
| step=1, | |
| visible=False | |
| ) | |
| steps = gr.Slider( | |
| label=get_translation("steps"), | |
| minimum=1, | |
| maximum=100, | |
| value=25, | |
| step=1, | |
| info=get_translation("steps_info") | |
| ) | |
| cfg = gr.Slider( | |
| label=get_translation("cfg_scale"), | |
| minimum=1.0, | |
| maximum=32.0, | |
| value=1.0, | |
| step=0.01, | |
| visible=False | |
| ) | |
| gs = gr.Slider( | |
| label=get_translation("distilled_cfg"), | |
| minimum=1.0, | |
| maximum=32.0, | |
| value=10.0, | |
| step=0.01, | |
| info=get_translation("distilled_cfg_info") | |
| ) | |
| rs = gr.Slider( | |
| label=get_translation("cfg_rescale"), | |
| minimum=0.0, | |
| maximum=1.0, | |
| value=0.0, | |
| step=0.01, | |
| visible=False | |
| ) | |
| gpu_memory_preservation = gr.Slider( | |
| label=get_translation("gpu_memory"), | |
| minimum=6, | |
| maximum=128, | |
| value=6, | |
| step=0.1, | |
| info=get_translation("gpu_memory_info") | |
| ) | |
| # ๋ฒํผ ์ฒ๋ฆฌ | |
| inputs_list = [ | |
| input_image, prompt, n_prompt, seed, | |
| total_second_length, latent_window_size, steps, | |
| cfg, gs, rs, gpu_memory_preservation, use_teacache | |
| ] | |
| start_button.click( | |
| fn=process, | |
| inputs=inputs_list, | |
| outputs=[result_video, preview_image, progress_desc, progress_bar, start_button, stop_button] | |
| ) | |
| stop_button.click(fn=end_process) | |
| block.launch() | |
| ############################################# | |
| # from diffusers_helper.hf_login import login | |
| # ํ์์ HF ๋ก๊ทธ์ธ ์ฌ์ฉ (์ฃผ์ ํด์ ํ) | |
| ############################################# | |
| import os | |
| os.environ['HF_HOME'] = os.path.abspath( | |
| os.path.realpath(os.path.join(os.path.dirname(__file__), './hf_download')) | |
| ) | |
| import gradio as gr | |
| import torch | |
| import traceback | |
| import einops | |
| import safetensors.torch as sf | |
| import numpy as np | |
| import math | |
| import time | |
| # Hugging Face Spaces ํ๊ฒฝ ์ธ์ง ํ์ธ | |
| IN_HF_SPACE = os.environ.get('SPACE_ID') is not None | |
| # --------- ๋ฒ์ญ ๋์ ๋๋ฆฌ(์์ด ๊ณ ์ ) --------- | |
| translations = { | |
| "en": { | |
| "title": "FramePack - Image to Video Generation", | |
| "upload_image": "Upload Image", | |
| "prompt": "Prompt", | |
| "quick_prompts": "Quick Prompts", | |
| "start_generation": "Generate", | |
| "stop_generation": "Stop", | |
| "use_teacache": "Use TeaCache", | |
| "teacache_info": "Faster speed, but may result in slightly worse finger and hand generation.", | |
| "negative_prompt": "Negative Prompt", | |
| "seed": "Seed", | |
| # ์ต๋ 4์ด๋ก UI ํ๊ธฐ ์์ | |
| "video_length": "Video Length (max 4 seconds)", | |
| "latent_window": "Latent Window Size", | |
| "steps": "Inference Steps", | |
| "steps_info": "Changing this value is not recommended.", | |
| "cfg_scale": "CFG Scale", | |
| "distilled_cfg": "Distilled CFG Scale", | |
| "distilled_cfg_info": "Changing this value is not recommended.", | |
| "cfg_rescale": "CFG Rescale", | |
| "gpu_memory": "GPU Memory Preservation (GB) (larger means slower)", | |
| "gpu_memory_info": "Set this to a larger value if you encounter OOM errors. Larger values cause slower speed.", | |
| "next_latents": "Next Latents", | |
| "generated_video": "Generated Video", | |
| "sampling_note": "Note: The model predicts future frames from past frames. If the start action isn't immediately visible, please wait for more frames.", | |
| "error_message": "Error", | |
| "processing_error": "Processing error", | |
| "network_error": "Network connection is unstable, model download timed out. Please try again later.", | |
| "memory_error": "GPU memory insufficient, please try increasing GPU memory preservation value or reduce video length.", | |
| "model_error": "Failed to load model, possibly due to network issues or high server load. Please try again later.", | |
| "partial_video": "Processing error, but partial video has been generated", | |
| "processing_interrupt": "Processing was interrupted, but partial video has been generated" | |
| } | |
| } | |
| def get_translation(key): | |
| return translations["en"].get(key, key) | |
| ############################################# | |
| # diffusers_helper ๊ด๋ จ ์ํฌํธ | |
| ############################################# | |
| from diffusers_helper.thread_utils import AsyncStream, async_run | |
| from diffusers_helper.gradio.progress_bar import make_progress_bar_css, make_progress_bar_html | |
| from diffusers_helper.memory import ( | |
| cpu, | |
| gpu, | |
| get_cuda_free_memory_gb, | |
| move_model_to_device_with_memory_preservation, | |
| offload_model_from_device_for_memory_preservation, | |
| fake_diffusers_current_device, | |
| DynamicSwapInstaller, | |
| unload_complete_models, | |
| load_model_as_complete | |
| ) | |
| from diffusers_helper.utils import ( | |
| generate_timestamp, | |
| save_bcthw_as_mp4, | |
| resize_and_center_crop, | |
| crop_or_pad_yield_mask, | |
| soft_append_bcthw | |
| ) | |
| from diffusers_helper.bucket_tools import find_nearest_bucket | |
| from diffusers_helper.hunyuan import ( | |
| encode_prompt_conds, vae_encode, vae_decode, vae_decode_fake | |
| ) | |
| from diffusers_helper.clip_vision import hf_clip_vision_encode | |
| from diffusers_helper.models.hunyuan_video_packed import HunyuanVideoTransformer3DModelPacked | |
| from diffusers_helper.pipelines.k_diffusion_hunyuan import sample_hunyuan | |
| from diffusers import AutoencoderKLHunyuanVideo | |
| from transformers import ( | |
| LlamaModel, CLIPTextModel, | |
| LlamaTokenizerFast, CLIPTokenizer, | |
| SiglipVisionModel, SiglipImageProcessor | |
| ) | |
| ############################################# | |
| # GPU ์ฒดํฌ | |
| ############################################# | |
| GPU_AVAILABLE = torch.cuda.is_available() | |
| free_mem_gb = 0.0 | |
| high_vram = False | |
| if GPU_AVAILABLE: | |
| try: | |
| free_mem_gb = torch.cuda.get_device_properties(0).total_memory / 1e9 | |
| high_vram = (free_mem_gb > 60) | |
| except: | |
| pass | |
| print(f"GPU Available: {GPU_AVAILABLE}, free_mem_gb={free_mem_gb}, high_vram={high_vram}") | |
| cpu_fallback_mode = not GPU_AVAILABLE | |
| last_update_time = time.time() | |
| ############################################# | |
| # ๋ชจ๋ธ ๋ก๋ (์ ์ญ) | |
| ############################################# | |
| text_encoder = None | |
| text_encoder_2 = None | |
| tokenizer = None | |
| tokenizer_2 = None | |
| vae = None | |
| feature_extractor = None | |
| image_encoder = None | |
| transformer = None | |
| # ์๋ ๋ก์ง์ ์ง๋ฌธ์ ์ ์๋ '๋ ๋ฒ์งธ ์ฝ๋'์ ๋ชจ๋ธ ๋ก๋ ๋ถ๋ถ์ ๊ฑฐ์ ๊ทธ๋๋ก ์ฌ์ฉ | |
| def load_global_models(): | |
| global text_encoder, text_encoder_2, tokenizer, tokenizer_2 | |
| global vae, feature_extractor, image_encoder, transformer | |
| global cpu_fallback_mode | |
| # ์ด๋ฏธ ๋ก๋๋์์ผ๋ฉด ํจ์ค | |
| if transformer is not None: | |
| return | |
| # GPU ๋ฉ๋ชจ๋ฆฌ ์ ๋ณด | |
| device = gpu if GPU_AVAILABLE else cpu | |
| # diffusers_helper.memory.get_cuda_free_memory_gb(gpu)๋ก ๋ ์ ํํ ๊ตฌํด๋ ๋จ | |
| print("Loading models...") | |
| # ======== ์ค ์ฝ๋: ๋ ๋ฒ์งธ ์์ ๊ธฐ์ค ========= | |
| # (1) ํ์ด๋ธ๋ฆฌ๋ (if high_vram -> GPU๋ก ๋ก๋, ์๋๋ฉด CPU + DynamicSwap) | |
| # ๋ฐ๋์ float16, bfloat16๋ก ๋ก๋ | |
| text_encoder_local = LlamaModel.from_pretrained( | |
| "hunyuanvideo-community/HunyuanVideo", | |
| subfolder='text_encoder', | |
| torch_dtype=torch.float16 | |
| ).cpu() | |
| text_encoder_2_local = CLIPTextModel.from_pretrained( | |
| "hunyuanvideo-community/HunyuanVideo", | |
| subfolder='text_encoder_2', | |
| torch_dtype=torch.float16 | |
| ).cpu() | |
| tokenizer_local = LlamaTokenizerFast.from_pretrained( | |
| "hunyuanvideo-community/HunyuanVideo", | |
| subfolder='tokenizer' | |
| ) | |
| tokenizer_2_local = CLIPTokenizer.from_pretrained( | |
| "hunyuanvideo-community/HunyuanVideo", | |
| subfolder='tokenizer_2' | |
| ) | |
| vae_local = AutoencoderKLHunyuanVideo.from_pretrained( | |
| "hunyuanvideo-community/HunyuanVideo", | |
| subfolder='vae', | |
| torch_dtype=torch.float16 | |
| ).cpu() | |
| feature_extractor_local = SiglipImageProcessor.from_pretrained( | |
| "lllyasviel/flux_redux_bfl", subfolder='feature_extractor' | |
| ) | |
| image_encoder_local = SiglipVisionModel.from_pretrained( | |
| "lllyasviel/flux_redux_bfl", | |
| subfolder='image_encoder', | |
| torch_dtype=torch.float16 | |
| ).cpu() | |
| # FramePack_F1_I2V_HY_20250503 (bfloat16) | |
| transformer_local = HunyuanVideoTransformer3DModelPacked.from_pretrained( | |
| 'lllyasviel/FramePack_F1_I2V_HY_20250503', | |
| torch_dtype=torch.bfloat16 | |
| ).cpu() | |
| # eval & dtype | |
| vae_local.eval() | |
| text_encoder_local.eval() | |
| text_encoder_2_local.eval() | |
| image_encoder_local.eval() | |
| transformer_local.eval() | |
| # VAE slicing for low VRAM | |
| if not high_vram: | |
| vae_local.enable_slicing() | |
| vae_local.enable_tiling() | |
| # ์คํ๋ก๋์ฉ | |
| transformer_local.high_quality_fp32_output_for_inference = True | |
| transformer_local.to(dtype=torch.bfloat16) | |
| vae_local.to(dtype=torch.float16) | |
| image_encoder_local.to(dtype=torch.float16) | |
| text_encoder_local.to(dtype=torch.float16) | |
| text_encoder_2_local.to(dtype=torch.float16) | |
| # requires_grad_(False) | |
| for m in [vae_local, text_encoder_local, text_encoder_2_local, image_encoder_local, transformer_local]: | |
| m.requires_grad_(False) | |
| # GPU ๋ชจ๋ & VRAM ๋ง์ผ๋ฉด ์ ๋ถ GPU | |
| # ๊ทธ๋ ์ง ์์ผ๋ฉด DynamicSwap | |
| if GPU_AVAILABLE: | |
| if not high_vram: | |
| DynamicSwapInstaller.install_model(transformer_local, device=gpu) | |
| DynamicSwapInstaller.install_model(text_encoder_local, device=gpu) | |
| else: | |
| text_encoder_local.to(gpu) | |
| text_encoder_2_local.to(gpu) | |
| image_encoder_local.to(gpu) | |
| vae_local.to(gpu) | |
| transformer_local.to(gpu) | |
| else: | |
| cpu_fallback_mode = True | |
| # ๊ธ๋ก๋ฒ์ ํ ๋น | |
| print("Model loaded.") | |
| text_encoder = text_encoder_local | |
| text_encoder_2 = text_encoder_2_local | |
| tokenizer = tokenizer_local | |
| tokenizer_2 = tokenizer_2_local | |
| vae = vae_local | |
| feature_extractor = feature_extractor_local | |
| image_encoder = image_encoder_local | |
| transformer = transformer_local | |
| ############################################# | |
| # Worker ๋ก์ง (๋ ๋ฒ์งธ ์ฝ๋) ๊ทธ๋๋ก | |
| ############################################# | |
| stream = AsyncStream() | |
| outputs_folder = './outputs/' | |
| os.makedirs(outputs_folder, exist_ok=True) | |
| def worker( | |
| input_image, prompt, n_prompt, seed, | |
| total_second_length, latent_window_size, steps, | |
| cfg, gs, rs, gpu_memory_preservation, use_teacache | |
| ): | |
| """ | |
| ์ค์ ์ํ๋ง ๋ก์ง (๋ ๋ฒ์งธ ์ฝ๋ ๊ธฐ๋ฐ) | |
| """ | |
| load_global_models() # ๋ชจ๋ธ ๋ก๋ฉ | |
| global text_encoder, text_encoder_2, tokenizer, tokenizer_2 | |
| global vae, feature_extractor, image_encoder, transformer | |
| global last_update_time | |
| # ์ต๋ 4์ด๋ก ๊ณ ์ | |
| total_second_length = min(total_second_length, 4.0) | |
| total_latent_sections = (total_second_length * 30) / (latent_window_size * 4) | |
| total_latent_sections = int(max(round(total_latent_sections), 1)) | |
| job_id = generate_timestamp() | |
| stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Starting ...')))) | |
| try: | |
| # GPU ์ ์ ๊ฒฝ์ฐ Unload | |
| if not high_vram and GPU_AVAILABLE: | |
| unload_complete_models( | |
| text_encoder, text_encoder_2, image_encoder, vae, transformer | |
| ) | |
| # Text encoding | |
| stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Text encoding ...')))) | |
| if not high_vram and GPU_AVAILABLE: | |
| fake_diffusers_current_device(text_encoder, gpu) | |
| load_model_as_complete(text_encoder_2, target_device=gpu) | |
| llama_vec, clip_l_pooler = encode_prompt_conds(prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2) | |
| if cfg == 1.0: | |
| llama_vec_n, clip_l_pooler_n = torch.zeros_like(llama_vec), torch.zeros_like(clip_l_pooler) | |
| else: | |
| llama_vec_n, clip_l_pooler_n = encode_prompt_conds(n_prompt, text_encoder, text_encoder_2, tokenizer, tokenizer_2) | |
| llama_vec, llama_mask = crop_or_pad_yield_mask(llama_vec, length=512) | |
| llama_vec_n, llama_mask_n = crop_or_pad_yield_mask(llama_vec_n, length=512) | |
| # Image processing | |
| stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Image processing ...')))) | |
| H, W, C = input_image.shape | |
| height, width = find_nearest_bucket(H, W, resolution=640) | |
| if cpu_fallback_mode: | |
| height = min(height, 320) | |
| width = min(width, 320) | |
| input_image_np = resize_and_center_crop(input_image, target_width=width, target_height=height) | |
| Image.fromarray(input_image_np).save(os.path.join(outputs_folder, f'{job_id}.png')) | |
| input_image_pt = torch.from_numpy(input_image_np).float() / 127.5 - 1 | |
| input_image_pt = input_image_pt.permute(2, 0, 1)[None, :, None] | |
| # VAE encode | |
| stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'VAE encoding ...')))) | |
| if not high_vram and GPU_AVAILABLE: | |
| load_model_as_complete(vae, target_device=gpu) | |
| start_latent = vae_encode(input_image_pt, vae) | |
| # CLIP Vision | |
| stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'CLIP Vision encoding ...')))) | |
| if not high_vram and GPU_AVAILABLE: | |
| load_model_as_complete(image_encoder, target_device=gpu) | |
| image_encoder_output = hf_clip_vision_encode(input_image_np, feature_extractor, image_encoder) | |
| image_encoder_last_hidden_state = image_encoder_output.last_hidden_state | |
| # dtype | |
| llama_vec = llama_vec.to(transformer.dtype) | |
| llama_vec_n = llama_vec_n.to(transformer.dtype) | |
| clip_l_pooler = clip_l_pooler.to(transformer.dtype) | |
| clip_l_pooler_n = clip_l_pooler_n.to(transformer.dtype) | |
| image_encoder_last_hidden_state = image_encoder_last_hidden_state.to(transformer.dtype) | |
| # Start sampling | |
| stream.output_queue.push(('progress', (None, '', make_progress_bar_html(0, 'Start sampling ...')))) | |
| rnd = torch.Generator("cpu").manual_seed(seed) | |
| # ์ด๊ธฐ history latents | |
| history_latents = torch.zeros(size=(1, 16, 16 + 2 + 1, height // 8, width // 8), dtype=torch.float32).cpu() | |
| history_pixels = None | |
| # start_latent ๋ถ์ด๊ธฐ | |
| history_latents = torch.cat([history_latents, start_latent.to(history_latents)], dim=2) | |
| total_generated_latent_frames = 1 | |
| for section_index in range(total_latent_sections): | |
| if stream.input_queue.top() == 'end': | |
| stream.output_queue.push(('end', None)) | |
| return | |
| print(f'Section {section_index+1}/{total_latent_sections}') | |
| if not high_vram and GPU_AVAILABLE: | |
| unload_complete_models() | |
| move_model_to_device_with_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=gpu_memory_preservation) | |
| # teacache | |
| if use_teacache: | |
| transformer.initialize_teacache(enable_teacache=True, num_steps=steps) | |
| else: | |
| transformer.initialize_teacache(enable_teacache=False) | |
| def callback(d): | |
| preview = d['denoised'] | |
| preview = vae_decode_fake(preview) | |
| preview = (preview * 255.0).detach().cpu().numpy().clip(0, 255).astype(np.uint8) | |
| preview = einops.rearrange(preview, 'b c t h w -> (b h) (t w) c') | |
| if stream.input_queue.top() == 'end': | |
| stream.output_queue.push(('end', None)) | |
| raise KeyboardInterrupt('User stops generation.') | |
| current_step = d['i'] + 1 | |
| percentage = int(100.0 * current_step / steps) | |
| hint = f'Sampling {current_step}/{steps}' | |
| desc = f'Section {section_index+1}/{total_latent_sections}' | |
| stream.output_queue.push(('progress', (preview, desc, make_progress_bar_html(percentage, hint)))) | |
| return | |
| # indices | |
| frames_per_section = latent_window_size * 4 - 3 | |
| indices = torch.arange(0, sum([1, 16, 2, 1, latent_window_size])).unsqueeze(0) | |
| ( | |
| clean_latent_indices_start, | |
| clean_latent_4x_indices, | |
| clean_latent_2x_indices, | |
| clean_latent_1x_indices, | |
| latent_indices | |
| ) = indices.split([1, 16, 2, 1, latent_window_size], dim=1) | |
| clean_latent_indices = torch.cat([clean_latent_indices_start, clean_latent_1x_indices], dim=1) | |
| clean_latents_4x, clean_latents_2x, clean_latents_1x = history_latents[:, :, -19:, :, :].split([16, 2, 1], dim=2) | |
| clean_latents = torch.cat([start_latent.to(history_latents), clean_latents_1x], dim=2) | |
| try: | |
| generated_latents = sample_hunyuan( | |
| transformer=transformer, | |
| sampler='unipc', | |
| width=width, | |
| height=height, | |
| frames=frames_per_section, | |
| real_guidance_scale=cfg, | |
| distilled_guidance_scale=gs, | |
| guidance_rescale=rs, | |
| num_inference_steps=steps, | |
| generator=rnd, | |
| prompt_embeds=llama_vec, | |
| prompt_embeds_mask=llama_mask, | |
| prompt_poolers=clip_l_pooler, | |
| negative_prompt_embeds=llama_vec_n, | |
| negative_prompt_embeds_mask=llama_mask_n, | |
| negative_prompt_poolers=clip_l_pooler_n, | |
| device=gpu if GPU_AVAILABLE else cpu, | |
| dtype=torch.bfloat16, | |
| image_embeddings=image_encoder_last_hidden_state, | |
| latent_indices=latent_indices, | |
| clean_latents=clean_latents, | |
| clean_latent_indices=clean_latent_indices, | |
| clean_latents_2x=clean_latents_2x, | |
| clean_latent_2x_indices=clean_latent_2x_indices, | |
| clean_latents_4x=clean_latents_4x, | |
| clean_latent_4x_indices=clean_latent_4x_indices, | |
| callback=callback | |
| ) | |
| except KeyboardInterrupt: | |
| print("User cancelled.") | |
| stream.output_queue.push(('end', None)) | |
| return | |
| except Exception as e: | |
| traceback.print_exc() | |
| stream.output_queue.push(('end', None)) | |
| return | |
| total_generated_latent_frames += generated_latents.shape[2] | |
| history_latents = torch.cat([history_latents, generated_latents.to(history_latents)], dim=2) | |
| if not high_vram and GPU_AVAILABLE: | |
| offload_model_from_device_for_memory_preservation(transformer, target_device=gpu, preserved_memory_gb=8) | |
| load_model_as_complete(vae, target_device=gpu) | |
| real_history_latents = history_latents[:, :, -total_generated_latent_frames:, :, :] | |
| if history_pixels is None: | |
| history_pixels = vae_decode(real_history_latents, vae).cpu() | |
| else: | |
| section_latent_frames = latent_window_size * 2 | |
| overlapped_frames = frames_per_section | |
| current_pixels = vae_decode(real_history_latents[:, :, -section_latent_frames:], vae).cpu() | |
| history_pixels = soft_append_bcthw(history_pixels, current_pixels, overlapped_frames) | |
| if not high_vram and GPU_AVAILABLE: | |
| unload_complete_models() | |
| output_filename = os.path.join(outputs_folder, f'{job_id}_{total_generated_latent_frames}.mp4') | |
| save_bcthw_as_mp4(history_pixels, output_filename, fps=30, crf=16) # CRF=16 | |
| stream.output_queue.push(('file', output_filename)) | |
| except: | |
| traceback.print_exc() | |
| if not high_vram and GPU_AVAILABLE: | |
| unload_complete_models(text_encoder, text_encoder_2, image_encoder, vae, transformer) | |
| stream.output_queue.push(('end', None)) | |
| return | |
| def end_process(): | |
| """ | |
| ์ค๋จ ์์ฒญ | |
| """ | |
| global stream | |
| stream.input_queue.push('end') | |
| # Gradio์์ ์ด worker ํจ์๋ฅผ ๋น๋๊ธฐ๋ก ํธ์ถ | |
| def process( | |
| input_image, prompt, n_prompt, seed, | |
| total_second_length, latent_window_size, steps, | |
| cfg, gs, rs, gpu_memory_preservation, use_teacache | |
| ): | |
| global stream | |
| if input_image is None: | |
| raise ValueError("No input image provided.") | |
| yield None, None, "", "", gr.update(interactive=False), gr.update(interactive=True) | |
| stream = AsyncStream() | |
| async_run( | |
| worker, | |
| input_image, prompt, n_prompt, seed, | |
| total_second_length, latent_window_size, steps, | |
| cfg, gs, rs, gpu_memory_preservation, use_teacache | |
| ) | |
| output_filename = None | |
| prev_filename = None | |
| error_message = None | |
| while True: | |
| flag, data = stream.output_queue.next() | |
| if flag == 'file': | |
| output_filename = data | |
| prev_filename = output_filename | |
| yield output_filename, gr.update(), gr.update(), "", gr.update(interactive=False), gr.update(interactive=True) | |
| elif flag == 'progress': | |
| preview, desc, html = data | |
| yield gr.update(), gr.update(visible=True, value=preview), desc, html, gr.update(interactive=False), gr.update(interactive=True) | |
| elif flag == 'error': | |
| error_message = data | |
| print(f"Error: {error_message}") | |
| elif flag == 'end': | |
| if output_filename is None and prev_filename: | |
| output_filename = prev_filename | |
| # ์๋ฌ๊ฐ ์์์ผ๋ฉด ์๋ฌ ํ์ | |
| if error_message: | |
| yield ( | |
| output_filename, # ๋ง์ง๋ง ํ์ผ (๋๋ None) | |
| gr.update(visible=False), | |
| gr.update(), | |
| f"<div style='color:red;'>{error_message}</div>", | |
| gr.update(interactive=True), | |
| gr.update(interactive=False) | |
| ) | |
| else: | |
| yield ( | |
| output_filename, gr.update(visible=False), gr.update(), "", gr.update(interactive=True), gr.update(interactive=False) | |
| ) | |
| break | |
| # UI CSS | |
| def make_custom_css(): | |
| base_progress_css = make_progress_bar_css() | |
| pastel_css = """ | |
| body { | |
| background: #faf9ff !important; | |
| font-family: "Noto Sans", sans-serif; | |
| } | |
| #app-container { | |
| max-width: 1200px; | |
| margin: 0 auto; | |
| padding: 1rem; | |
| position: relative; | |
| } | |
| #app-container h1 { | |
| color: #5F5AA2; | |
| margin-bottom: 1.2rem; | |
| font-weight: 700; | |
| text-shadow: 1px 1px 2px #bbb; | |
| } | |
| .gr-panel { | |
| background: #ffffffcc; | |
| border: 1px solid #e1dff0; | |
| border-radius: 8px; | |
| padding: 1rem; | |
| box-shadow: 0 1px 3px rgba(0,0,0,0.1); | |
| } | |
| .button-container button { | |
| min-height: 45px; | |
| font-size: 1rem; | |
| font-weight: 600; | |
| border-radius: 6px; | |
| } | |
| .button-container button#start-button { | |
| background-color: #A289E3 !important; | |
| color: #fff !important; | |
| border: 1px solid #a58de2; | |
| } | |
| .button-container button#stop-button { | |
| background-color: #F48A9B !important; | |
| color: #fff !important; | |
| border: 1px solid #f18fa0; | |
| } | |
| .button-container button:hover { | |
| filter: brightness(0.95); | |
| } | |
| .preview-container, .video-container { | |
| border: 1px solid #ded9f2; | |
| border-radius: 8px; | |
| overflow: hidden; | |
| } | |
| .progress-container { | |
| margin-top: 15px; | |
| margin-bottom: 15px; | |
| } | |
| .error-message { | |
| background-color: #FFF5F5; | |
| border: 1px solid #FED7D7; | |
| color: #E53E3E; | |
| padding: 10px; | |
| border-radius: 4px; | |
| margin-top: 10px; | |
| font-weight: 500; | |
| } | |
| @media (max-width: 768px) { | |
| #app-container { | |
| padding: 0.5rem; | |
| } | |
| .mobile-full-width { | |
| flex-direction: column !important; | |
| } | |
| .mobile-full-width > .gr-block { | |
| width: 100% !important; | |
| } | |
| } | |
| """ | |
| return base_progress_css + pastel_css | |
| css = make_custom_css() | |
| # ์ํ ํ๋กฌํํธ | |
| quick_prompts = [ | |
| ["The girl dances gracefully, with clear movements, full of charm."], | |
| ["A character doing some simple body movements."] | |
| ] | |
| # Gradio UI | |
| block = gr.Blocks(css=css).queue() | |
| with block: | |
| gr.HTML("<div id='app-container'><h1>FramePack - Image to Video Generation</h1></div>") | |
| with gr.Row(elem_classes="mobile-full-width"): | |
| # ์ผ์ชฝ | |
| with gr.Column(scale=1, elem_classes="gr-panel"): | |
| input_image = gr.Image( | |
| label=get_translation("upload_image"), | |
| type="numpy", | |
| height=320 | |
| ) | |
| prompt = gr.Textbox( | |
| label=get_translation("prompt"), | |
| value='' | |
| ) | |
| example_quick_prompts = gr.Dataset( | |
| samples=quick_prompts, | |
| label=get_translation("quick_prompts"), | |
| samples_per_page=1000, | |
| components=[prompt] | |
| ) | |
| example_quick_prompts.click( | |
| fn=lambda x: x[0], | |
| inputs=[example_quick_prompts], | |
| outputs=prompt, | |
| show_progress=False, | |
| queue=False | |
| ) | |
| # ์ค๋ฅธ์ชฝ | |
| with gr.Column(scale=1, elem_classes="gr-panel"): | |
| with gr.Row(elem_classes="button-container"): | |
| start_button = gr.Button( | |
| value=get_translation("start_generation"), | |
| elem_id="start-button", | |
| variant="primary" | |
| ) | |
| stop_button = gr.Button( | |
| value=get_translation("stop_generation"), | |
| elem_id="stop-button", | |
| interactive=False | |
| ) | |
| result_video = gr.Video( | |
| label=get_translation("generated_video"), | |
| autoplay=True, | |
| loop=True, | |
| height=320, | |
| elem_classes="video-container" | |
| ) | |
| preview_image = gr.Image( | |
| label=get_translation("next_latents"), | |
| visible=False, | |
| height=150, | |
| elem_classes="preview-container" | |
| ) | |
| gr.Markdown(get_translation("sampling_note")) | |
| with gr.Group(elem_classes="progress-container"): | |
| progress_desc = gr.Markdown('') | |
| progress_bar = gr.HTML('') | |
| error_message = gr.HTML('', visible=True) | |
| # Advanced | |
| with gr.Accordion("Advanced Settings", open=False, elem_classes="gr-panel"): | |
| use_teacache = gr.Checkbox( | |
| label=get_translation("use_teacache"), | |
| value=True, | |
| info=get_translation("teacache_info") | |
| ) | |
| n_prompt = gr.Textbox(label=get_translation("negative_prompt"), value="", visible=False) | |
| seed = gr.Number( | |
| label=get_translation("seed"), | |
| value=31337, | |
| precision=0 | |
| ) | |
| # ๊ธฐ๋ณธ 2์ด, ์ต๋ 4์ด | |
| total_second_length = gr.Slider( | |
| label=get_translation("video_length"), | |
| minimum=1, | |
| maximum=4, | |
| value=2, | |
| step=0.1 | |
| ) | |
| latent_window_size = gr.Slider( | |
| label=get_translation("latent_window"), | |
| minimum=1, | |
| maximum=33, | |
| value=9, | |
| step=1, | |
| visible=False | |
| ) | |
| steps = gr.Slider( | |
| label=get_translation("steps"), | |
| minimum=1, | |
| maximum=100, | |
| value=25, | |
| step=1, | |
| info=get_translation("steps_info") | |
| ) | |
| cfg = gr.Slider( | |
| label=get_translation("cfg_scale"), | |
| minimum=1.0, | |
| maximum=32.0, | |
| value=1.0, | |
| step=0.01, | |
| visible=False | |
| ) | |
| gs = gr.Slider( | |
| label=get_translation("distilled_cfg"), | |
| minimum=1.0, | |
| maximum=32.0, | |
| value=10.0, | |
| step=0.01, | |
| info=get_translation("distilled_cfg_info") | |
| ) | |
| rs = gr.Slider( | |
| label=get_translation("cfg_rescale"), | |
| minimum=0.0, | |
| maximum=1.0, | |
| value=0.0, | |
| step=0.01, | |
| visible=False | |
| ) | |
| gpu_memory_preservation = gr.Slider( | |
| label=get_translation("gpu_memory"), | |
| minimum=6, | |
| maximum=128, | |
| value=6, | |
| step=0.1, | |
| info=get_translation("gpu_memory_info") | |
| ) | |
| # ๋ฒํผ ์ฒ๋ฆฌ | |
| inputs_list = [ | |
| input_image, prompt, n_prompt, seed, | |
| total_second_length, latent_window_size, steps, | |
| cfg, gs, rs, gpu_memory_preservation, use_teacache | |
| ] | |
| start_button.click( | |
| fn=process, | |
| inputs=inputs_list, | |
| outputs=[result_video, preview_image, progress_desc, progress_bar, start_button, stop_button] | |
| ) | |
| stop_button.click(fn=end_process) | |
| block.launch() | |