File size: 4,543 Bytes
302920f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 |
# Copyright 2023-present the HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# The implementation is based on "Parameter-Efficient Orthogonal Finetuning
# via Butterfly Factorization" (https://huggingface.co/papers/2311.06243) in ICLR 2024.
import os
import sys
import time
from pathlib import Path
import numpy as np
import torch
import torch.utils.checkpoint
from accelerate import Accelerator
from diffusers import DDIMScheduler
from diffusers.utils import check_min_version
from safetensors.torch import load_file
from tqdm import tqdm
from transformers import AutoTokenizer
from utils.args_loader import parse_args
from utils.dataset import make_dataset
from utils.light_controlnet import ControlNetModel
from utils.pipeline_controlnet import LightControlNetPipeline
from utils.unet_2d_condition import UNet2DConditionNewModel
sys.path.append("../../src")
from peft import PeftModel # noqa: E402
# Will error if the minimal version of diffusers is not installed. Remove at your own risks.
check_min_version("0.10.0.dev0")
if torch.xpu.is_available():
device = "xpu:0"
elif torch.cuda.is_available():
device = "cuda:0"
else:
device = "cpu"
def main(args):
logging_dir = Path(args.output_dir, args.logging_dir)
accelerator = Accelerator(
gradient_accumulation_steps=args.gradient_accumulation_steps,
mixed_precision=args.mixed_precision,
log_with=args.report_to,
project_dir=logging_dir,
)
# Load the tokenizer
if args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name, revision=args.revision, use_fast=False)
elif args.pretrained_model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(
args.pretrained_model_name_or_path,
subfolder="tokenizer",
revision=args.revision,
use_fast=False,
)
val_dataset = make_dataset(args, tokenizer, accelerator, "test")
controlnet_path = args.controlnet_path
unet_path = args.unet_path
controlnet = ControlNetModel()
controlnet.load_state_dict(load_file(controlnet_path))
unet = UNet2DConditionNewModel.from_pretrained(args.pretrained_model_name_or_path, subfolder="unet")
unet = PeftModel.from_pretrained(unet, unet_path, adapter_name=args.adapter_name)
pipe = LightControlNetPipeline.from_pretrained(
args.pretrained_model_name_or_path,
controlnet=controlnet,
unet=unet.model,
torch_dtype=torch.float32,
requires_safety_checker=False,
).to(device)
pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config)
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir, exist_ok=True)
exist_lst = [int(img.split("_")[-1][:-4]) for img in os.listdir(args.output_dir)]
all_lst = np.arange(len(val_dataset))
idx_lst = [item for item in all_lst if item not in exist_lst]
print("Number of images to be processed: ", len(idx_lst))
np.random.seed(seed=int(time.time()))
np.random.shuffle(idx_lst)
for idx in tqdm(idx_lst):
output_path = os.path.join(args.output_dir, f"pred_img_{idx:04d}.png")
if not os.path.exists(output_path):
data = val_dataset[idx.item()]
negative_prompt = "low quality, blurry, unfinished"
with torch.no_grad():
pred_img = pipe(
data["text"],
[data["conditioning_pixel_values"]],
num_inference_steps=50,
guidance_scale=7,
negative_prompt=negative_prompt,
).images[0]
pred_img.save(output_path)
# control_img = Image.fromarray(
# (data["conditioning_pixel_value"] * 255).numpy().transpose(1, 2, 0).astype(np.uint8)
# )
# gt_img = Image.fromarray(
# ((data["pixel_value"] + 1.0) * 0.5 * 255).numpy().transpose(1, 2, 0).astype(np.uint8)
# )
if __name__ == "__main__":
args = parse_args()
main(args)
|