Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
|
@@ -17,15 +17,10 @@ from scepter.modules.transform.io import pillow_convert
|
|
| 17 |
from scepter.modules.utils.config import Config
|
| 18 |
from scepter.modules.utils.distribute import we
|
| 19 |
from scepter.modules.utils.file_system import FS
|
| 20 |
-
|
| 21 |
-
from inference.
|
| 22 |
from inference.utils import edit_preprocess
|
| 23 |
-
from examples.examples import all_examples
|
| 24 |
-
|
| 25 |
|
| 26 |
-
inference_dict = {
|
| 27 |
-
"ACE_DIFFUSER_PLUS": ACEPlusDiffuserInference
|
| 28 |
-
}
|
| 29 |
|
| 30 |
fs_list = [
|
| 31 |
Config(cfg_dict={"NAME": "HuggingfaceFs", "TEMP_DIR": "./cache"}, load=False),
|
|
@@ -38,15 +33,10 @@ for one_fs in fs_list:
|
|
| 38 |
FS.init_fs_client(one_fs)
|
| 39 |
|
| 40 |
os.environ["FLUX_FILL_PATH"]="hf://black-forest-labs/FLUX.1-Fill-dev"
|
| 41 |
-
os.environ["
|
| 42 |
-
os.environ["SUBJECT_MODEL_PATH"]="hf://ali-vilab/ACE_Plus@subject/comfyui_subject_lora16.safetensors"
|
| 43 |
-
os.environ["LOCAL_MODEL_PATH"]="hf://ali-vilab/ACE_Plus@local_editing/comfyui_local_lora16.safetensors"
|
| 44 |
|
| 45 |
FS.get_dir_to_local_dir(os.environ["FLUX_FILL_PATH"])
|
| 46 |
-
FS.get_from(os.environ["
|
| 47 |
-
FS.get_from(os.environ["SUBJECT_MODEL_PATH"])
|
| 48 |
-
FS.get_from(os.environ["LOCAL_MODEL_PATH"])
|
| 49 |
-
|
| 50 |
|
| 51 |
csv.field_size_limit(sys.maxsize)
|
| 52 |
refresh_sty = '\U0001f504' # 🔄
|
|
@@ -60,51 +50,39 @@ lock = threading.Lock()
|
|
| 60 |
class DemoUI(object):
|
| 61 |
#@spaces.GPU(duration=60)
|
| 62 |
def __init__(self,
|
| 63 |
-
infer_dir
|
| 64 |
-
model_list='./models/model_zoo.yaml'
|
| 65 |
):
|
| 66 |
-
self.model_yamls =
|
| 67 |
-
'*.yaml'))
|
| 68 |
self.model_choices = dict()
|
| 69 |
self.default_model_name = ''
|
|
|
|
|
|
|
|
|
|
| 70 |
for i in self.model_yamls:
|
| 71 |
model_cfg = Config(load=True, cfg_file=i)
|
| 72 |
-
model_name = model_cfg.
|
| 73 |
if model_cfg.IS_DEFAULT: self.default_model_name = model_name
|
| 74 |
self.model_choices[model_name] = model_cfg
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 75 |
print('Models: ', self.model_choices.keys())
|
| 76 |
assert len(self.model_choices) > 0
|
| 77 |
if self.default_model_name == "": self.default_model_name = list(self.model_choices.keys())[0]
|
| 78 |
self.model_name = self.default_model_name
|
| 79 |
pipe_cfg = self.model_choices[self.default_model_name]
|
| 80 |
-
|
| 81 |
-
self.pipe = inference_dict[infer_name]()
|
| 82 |
-
self.pipe.init_from_cfg(pipe_cfg)
|
| 83 |
-
|
| 84 |
-
# choose different model
|
| 85 |
-
self.task_model_cfg = Config(load=True, cfg_file=model_list)
|
| 86 |
-
self.task_model = {}
|
| 87 |
-
self.task_model_list = []
|
| 88 |
-
self.edit_type_dict = {"repainting": None}
|
| 89 |
-
self.edit_type_list = ["repainting"]
|
| 90 |
-
for task_name, task_model in self.task_model_cfg.MODEL.items():
|
| 91 |
-
self.task_model[task_name.lower()] = task_model
|
| 92 |
-
self.task_model_list.append(task_name.lower())
|
| 93 |
-
for preprocessor in task_model.get("PREPROCESSOR", []):
|
| 94 |
-
if preprocessor["TYPE"] in self.edit_type_dict:
|
| 95 |
-
continue
|
| 96 |
-
preprocessor["REPAINTING_SCALE"] = task_model.get("REPAINTING_SCALE", 1.0)
|
| 97 |
-
self.edit_type_dict[preprocessor["TYPE"]] = preprocessor
|
| 98 |
-
self.max_msgs = 20
|
| 99 |
# reformat examples
|
| 100 |
self.all_examples = [
|
| 101 |
[
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
for one_example in
|
| 108 |
]
|
| 109 |
|
| 110 |
def construct_edit_image(self, edit_image, edit_mask):
|
|
@@ -127,9 +105,6 @@ class DemoUI(object):
|
|
| 127 |
else:
|
| 128 |
return None
|
| 129 |
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
def create_ui(self):
|
| 134 |
with gr.Row(equal_height=True, visible=True):
|
| 135 |
with gr.Column(scale=2):
|
|
@@ -146,40 +121,102 @@ class DemoUI(object):
|
|
| 146 |
height=600,
|
| 147 |
interactive=False,
|
| 148 |
type='pil',
|
| 149 |
-
elem_id='preprocess_image'
|
|
|
|
| 150 |
)
|
| 151 |
|
| 152 |
self.edit_preprocess_mask_preview = gr.Image(
|
| 153 |
height=600,
|
| 154 |
interactive=False,
|
| 155 |
type='pil',
|
| 156 |
-
elem_id='preprocess_image_mask'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 157 |
)
|
| 158 |
with gr.Row():
|
| 159 |
instruction = """
|
| 160 |
**Instruction**:
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
color information, and more. The pre-processing information can be viewed in the 'related input image' tab.
|
| 167 |
-
4. More details can be found in [page](https://ali-vilab.github.io/ACE_plus_page).
|
| 168 |
"""
|
| 169 |
self.instruction = gr.Markdown(value=instruction)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 170 |
with gr.Row():
|
| 171 |
self.model_name_dd = gr.Dropdown(
|
| 172 |
choices=self.model_choices,
|
| 173 |
value=self.default_model_name,
|
| 174 |
label='Model Version')
|
| 175 |
-
self.
|
| 176 |
interactive=True,
|
| 177 |
-
value=self.
|
| 178 |
-
label='Task Type')
|
| 179 |
-
self.edit_type = gr.Dropdown(choices=self.edit_type_list,
|
| 180 |
-
interactive=True,
|
| 181 |
-
value=self.edit_type_list[0],
|
| 182 |
label='Edit Type')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 183 |
with gr.Row():
|
| 184 |
self.generation_info_preview = gr.Markdown(
|
| 185 |
label='System Log.',
|
|
@@ -192,11 +229,11 @@ class DemoUI(object):
|
|
| 192 |
placeholder='Input "@" find history of image',
|
| 193 |
label='Instruction',
|
| 194 |
container=False,
|
| 195 |
-
lines
|
| 196 |
with gr.Column(scale=2, min_width=100):
|
| 197 |
with gr.Row():
|
| 198 |
with gr.Column(scale=1, min_width=100):
|
| 199 |
-
self.chat_btn = gr.Button(value='Generate', variant
|
| 200 |
|
| 201 |
with gr.Accordion(label='Advance', open=True):
|
| 202 |
with gr.Row(visible=True):
|
|
@@ -223,45 +260,8 @@ class DemoUI(object):
|
|
| 223 |
format="png"
|
| 224 |
)
|
| 225 |
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
maximum=1000,
|
| 229 |
-
value=self.pipe.input.get("sample_steps", 20),
|
| 230 |
-
visible=self.pipe.input.get("sample_steps", None) is not None,
|
| 231 |
-
label='Sample Step')
|
| 232 |
-
self.cfg_scale = gr.Slider(
|
| 233 |
-
minimum=1.0,
|
| 234 |
-
maximum=100.0,
|
| 235 |
-
value=self.pipe.input.get("guide_scale", 4.5),
|
| 236 |
-
visible=self.pipe.input.get("guide_scale", None) is not None,
|
| 237 |
-
label='Guidance Scale')
|
| 238 |
-
self.seed = gr.Slider(minimum=-1,
|
| 239 |
-
maximum=10000000,
|
| 240 |
-
value=-1,
|
| 241 |
-
label='Seed')
|
| 242 |
-
self.output_height = gr.Slider(
|
| 243 |
-
minimum=256,
|
| 244 |
-
maximum=1440,
|
| 245 |
-
value=self.pipe.input.get("output_height", 1024),
|
| 246 |
-
visible=self.pipe.input.get("output_height", None) is not None,
|
| 247 |
-
label='Output Height')
|
| 248 |
-
self.output_width = gr.Slider(
|
| 249 |
-
minimum=256,
|
| 250 |
-
maximum=1440,
|
| 251 |
-
value=self.pipe.input.get("output_width", 1024),
|
| 252 |
-
visible=self.pipe.input.get("output_width", None) is not None,
|
| 253 |
-
label='Output Width')
|
| 254 |
-
|
| 255 |
-
self.repainting_scale = gr.Slider(
|
| 256 |
-
minimum=0.0,
|
| 257 |
-
maximum=1.0,
|
| 258 |
-
value=self.pipe.input.get("repainting_scale", 1.0),
|
| 259 |
-
visible=True,
|
| 260 |
-
label='Repainting Scale')
|
| 261 |
-
with gr.Row():
|
| 262 |
-
self.eg = gr.Column(visible=True)
|
| 263 |
-
|
| 264 |
-
|
| 265 |
|
| 266 |
def set_callbacks(self, *args, **kwargs):
|
| 267 |
########################################
|
|
@@ -276,25 +276,23 @@ class DemoUI(object):
|
|
| 276 |
torch.cuda.empty_cache()
|
| 277 |
torch.cuda.ipc_collect()
|
| 278 |
pipe_cfg = self.model_choices[model_name]
|
| 279 |
-
|
| 280 |
-
self.pipe = inference_dict[infer_name]()
|
| 281 |
-
self.pipe.init_from_cfg(pipe_cfg)
|
| 282 |
self.model_name = model_name
|
| 283 |
lock.release()
|
| 284 |
|
| 285 |
return (model_name, gr.update(),
|
| 286 |
gr.Slider(
|
| 287 |
-
|
| 288 |
-
|
| 289 |
gr.Slider(
|
| 290 |
value=self.pipe.input.get("guide_scale", 4.5),
|
| 291 |
visible=self.pipe.input.get("guide_scale", None) is not None),
|
| 292 |
gr.Slider(
|
| 293 |
-
value=self.pipe.input.get("
|
| 294 |
-
visible=self.pipe.input.get("
|
| 295 |
gr.Slider(
|
| 296 |
-
value=self.pipe.input.get("
|
| 297 |
-
visible=self.pipe.input.get("
|
| 298 |
gr.Slider(value=self.pipe.input.get("repainting_scale", 1.0))
|
| 299 |
)
|
| 300 |
|
|
@@ -309,31 +307,21 @@ class DemoUI(object):
|
|
| 309 |
self.output_width,
|
| 310 |
self.repainting_scale])
|
| 311 |
|
| 312 |
-
def change_task_type(task_type):
|
| 313 |
-
task_info = self.task_model[task_type]
|
| 314 |
-
edit_type_list = [self.edit_type_list[0]]
|
| 315 |
-
for preprocessor in task_info.get("PREPROCESSOR", []):
|
| 316 |
-
preprocessor["REPAINTING_SCALE"] = task_info.get("REPAINTING_SCALE", 1.0)
|
| 317 |
-
self.edit_type_dict[preprocessor["TYPE"]] = preprocessor
|
| 318 |
-
edit_type_list.append(preprocessor["TYPE"])
|
| 319 |
-
|
| 320 |
-
return gr.update(choices=edit_type_list, value=edit_type_list[0])
|
| 321 |
-
|
| 322 |
-
self.task_type.change(change_task_type, inputs=[self.task_type], outputs=[self.edit_type])
|
| 323 |
-
|
| 324 |
def change_edit_type(edit_type):
|
| 325 |
edit_info = self.edit_type_dict[edit_type]
|
| 326 |
edit_info = edit_info or {}
|
| 327 |
repainting_scale = edit_info.get("REPAINTING_SCALE", 1.0)
|
| 328 |
-
|
| 329 |
-
return gr.Slider(value=1.0)
|
| 330 |
-
else:
|
| 331 |
-
return gr.Slider(
|
| 332 |
-
value=repainting_scale)
|
| 333 |
|
| 334 |
self.edit_type.change(change_edit_type, inputs=[self.edit_type], outputs=[self.repainting_scale])
|
| 335 |
|
| 336 |
-
def
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 337 |
err_msg = ""
|
| 338 |
is_suc = True
|
| 339 |
if ref_image is not None:
|
|
@@ -349,8 +337,9 @@ class DemoUI(object):
|
|
| 349 |
edit_image = None
|
| 350 |
edit_mask = None
|
| 351 |
elif np.sum(np.array(edit_mask)) < 1:
|
| 352 |
-
|
| 353 |
-
|
|
|
|
| 354 |
else:
|
| 355 |
edit_image = pillow_convert(edit_image, "RGB")
|
| 356 |
edit_mask = Image.fromarray(edit_mask).convert('L')
|
|
@@ -358,43 +347,38 @@ class DemoUI(object):
|
|
| 358 |
err_msg = "Please provide the reference image or edited image."
|
| 359 |
return None, None, None, False, err_msg
|
| 360 |
return edit_image, edit_mask, ref_image, is_suc, err_msg
|
|
|
|
| 361 |
@spaces.GPU(duration=80)
|
| 362 |
def run_chat(
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
|
| 370 |
-
|
| 371 |
-
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
|
|
|
|
|
|
|
| 375 |
):
|
| 376 |
-
print(prompt)
|
| 377 |
-
model_path = self.task_model[task_type]["MODEL_PATH"]
|
| 378 |
edit_info = self.edit_type_dict[edit_type]
|
| 379 |
-
|
| 380 |
-
if task_type in ["portrait", "subject"] and ref_image is None:
|
| 381 |
-
err_msg = "<mark>Please provide the reference image.</mark>"
|
| 382 |
-
return (gr.Image(), gr.Column(visible=True),
|
| 383 |
-
gr.Image(),
|
| 384 |
-
gr.Image(),
|
| 385 |
-
gr.Text(value=err_msg))
|
| 386 |
-
|
| 387 |
pre_edit_image, pre_edit_mask, pre_ref_image, is_suc, err_msg = preprocess_input(ref_image, edit_image)
|
|
|
|
| 388 |
if not is_suc:
|
| 389 |
err_msg = f"<mark>{err_msg}</mark>"
|
| 390 |
return (gr.Image(), gr.Column(visible=True),
|
|
|
|
| 391 |
gr.Image(),
|
| 392 |
gr.Image(),
|
| 393 |
gr.Text(value=err_msg))
|
| 394 |
-
pre_edit_image = edit_preprocess(edit_info, we.device_id, pre_edit_image, pre_edit_mask)
|
| 395 |
# edit_image["background"] = pre_edit_image
|
| 396 |
st = time.time()
|
| 397 |
-
image, seed = self.pipe(
|
| 398 |
reference_image=pre_ref_image,
|
| 399 |
edit_image=pre_edit_image,
|
| 400 |
edit_mask=pre_edit_mask,
|
|
@@ -406,32 +390,43 @@ class DemoUI(object):
|
|
| 406 |
guide_scale=cfg_scale,
|
| 407 |
seed=seed,
|
| 408 |
repainting_scale=repainting_scale,
|
| 409 |
-
|
|
|
|
|
|
|
| 410 |
)
|
| 411 |
et = time.time()
|
| 412 |
msg = f"prompt: {prompt}; seed: {seed}; cost time: {et - st}s; repaiting scale: {repainting_scale}"
|
| 413 |
|
|
|
|
|
|
|
|
|
|
| 414 |
return (gr.Image(value=image), gr.Column(visible=True),
|
| 415 |
-
gr.Image(value=
|
|
|
|
| 416 |
gr.Image(value=pre_edit_mask if pre_edit_mask is not None else None),
|
| 417 |
-
gr.Text(value=msg)
|
|
|
|
| 418 |
|
| 419 |
chat_inputs = [
|
| 420 |
self.reference_image,
|
| 421 |
self.edit_image,
|
| 422 |
-
self.task_type,
|
| 423 |
self.edit_type,
|
| 424 |
self.cfg_scale,
|
| 425 |
self.step,
|
| 426 |
self.seed,
|
| 427 |
self.output_height,
|
| 428 |
self.output_width,
|
| 429 |
-
self.repainting_scale
|
|
|
|
|
|
|
|
|
|
| 430 |
]
|
| 431 |
|
| 432 |
chat_outputs = [
|
| 433 |
-
|
| 434 |
-
self.
|
|
|
|
|
|
|
| 435 |
]
|
| 436 |
|
| 437 |
self.chat_btn.click(run_chat,
|
|
@@ -445,23 +440,26 @@ class DemoUI(object):
|
|
| 445 |
queue=True)
|
| 446 |
|
| 447 |
@spaces.GPU(duration=80)
|
| 448 |
-
def run_example(
|
| 449 |
-
output_h, output_w, seed,
|
| 450 |
-
|
|
|
|
| 451 |
|
| 452 |
step = self.pipe.input.get("sample_steps", 20)
|
| 453 |
cfg_scale = self.pipe.input.get("guide_scale", 20)
|
| 454 |
-
|
| 455 |
edit_info = self.edit_type_dict[edit_type]
|
| 456 |
|
| 457 |
edit_image = self.construct_edit_image(edit_image, edit_mask)
|
| 458 |
|
| 459 |
-
pre_edit_image, pre_edit_mask, pre_ref_image,
|
| 460 |
-
|
|
|
|
|
|
|
|
|
|
| 461 |
edit_info = edit_info or {}
|
| 462 |
repainting_scale = edit_info.get("REPAINTING_SCALE", 1.0)
|
| 463 |
st = time.time()
|
| 464 |
-
image, seed = self.pipe(
|
| 465 |
reference_image=pre_ref_image,
|
| 466 |
edit_image=pre_edit_image,
|
| 467 |
edit_mask=pre_edit_mask,
|
|
@@ -473,40 +471,52 @@ class DemoUI(object):
|
|
| 473 |
guide_scale=cfg_scale,
|
| 474 |
seed=seed,
|
| 475 |
repainting_scale=repainting_scale,
|
| 476 |
-
|
|
|
|
|
|
|
| 477 |
)
|
| 478 |
et = time.time()
|
| 479 |
msg = f"prompt: {prompt}; seed: {seed}; cost time: {et - st}s; repaiting scale: {repainting_scale}"
|
| 480 |
if pre_edit_image is not None:
|
| 481 |
-
ret_image = Image.composite(Image.new("RGB", pre_edit_image.size, (0, 0, 0)), pre_edit_image,
|
|
|
|
| 482 |
else:
|
| 483 |
ret_image = None
|
|
|
|
|
|
|
|
|
|
| 484 |
return (gr.Image(value=image), gr.Column(visible=True),
|
| 485 |
-
gr.Image(value=
|
|
|
|
| 486 |
gr.Image(value=pre_edit_mask if pre_edit_mask is not None else None),
|
| 487 |
gr.Text(value=msg),
|
| 488 |
-
gr.update(value=ret_image)
|
|
|
|
| 489 |
|
| 490 |
with self.eg:
|
| 491 |
self.example_edit_image = gr.Image(label='Edit Image',
|
| 492 |
-
|
| 493 |
-
|
| 494 |
-
|
| 495 |
self.example_edit_mask = gr.Image(label='Edit Image Mask',
|
| 496 |
-
|
| 497 |
-
|
| 498 |
-
|
| 499 |
|
| 500 |
self.examples = gr.Examples(
|
| 501 |
fn=run_example,
|
| 502 |
examples=self.all_examples,
|
| 503 |
inputs=[
|
| 504 |
-
self.
|
| 505 |
-
self.example_edit_mask, self.output_height, self.output_width, self.seed
|
|
|
|
| 506 |
],
|
| 507 |
outputs=[self.gallery_image, self.edit_preprocess_panel, self.edit_preprocess_preview,
|
| 508 |
-
self.
|
| 509 |
-
|
|
|
|
|
|
|
|
|
|
| 510 |
cache_examples=False,
|
| 511 |
run_on_click=True)
|
| 512 |
|
|
|
|
| 17 |
from scepter.modules.utils.config import Config
|
| 18 |
from scepter.modules.utils.distribute import we
|
| 19 |
from scepter.modules.utils.file_system import FS
|
| 20 |
+
from examples.examples import fft_examples
|
| 21 |
+
from inference.registry import INFERENCES
|
| 22 |
from inference.utils import edit_preprocess
|
|
|
|
|
|
|
| 23 |
|
|
|
|
|
|
|
|
|
|
| 24 |
|
| 25 |
fs_list = [
|
| 26 |
Config(cfg_dict={"NAME": "HuggingfaceFs", "TEMP_DIR": "./cache"}, load=False),
|
|
|
|
| 33 |
FS.init_fs_client(one_fs)
|
| 34 |
|
| 35 |
os.environ["FLUX_FILL_PATH"]="hf://black-forest-labs/FLUX.1-Fill-dev"
|
| 36 |
+
os.environ["ACE_PLUS_FFT_MODEL"]="hf://ali-vilab/ACE_Plus@ace_plus_fft.safetensors"
|
|
|
|
|
|
|
| 37 |
|
| 38 |
FS.get_dir_to_local_dir(os.environ["FLUX_FILL_PATH"])
|
| 39 |
+
FS.get_from(os.environ["ACE_PLUS_FFT_MODEL"])
|
|
|
|
|
|
|
|
|
|
| 40 |
|
| 41 |
csv.field_size_limit(sys.maxsize)
|
| 42 |
refresh_sty = '\U0001f504' # 🔄
|
|
|
|
| 50 |
class DemoUI(object):
|
| 51 |
#@spaces.GPU(duration=60)
|
| 52 |
def __init__(self,
|
| 53 |
+
infer_dir="./config/ace_plus_fft.yaml"
|
|
|
|
| 54 |
):
|
| 55 |
+
self.model_yamls = [infer_dir]
|
|
|
|
| 56 |
self.model_choices = dict()
|
| 57 |
self.default_model_name = ''
|
| 58 |
+
self.edit_type_dict = {}
|
| 59 |
+
self.edit_type_list = []
|
| 60 |
+
self.default_type_list = []
|
| 61 |
for i in self.model_yamls:
|
| 62 |
model_cfg = Config(load=True, cfg_file=i)
|
| 63 |
+
model_name = model_cfg.VERSION
|
| 64 |
if model_cfg.IS_DEFAULT: self.default_model_name = model_name
|
| 65 |
self.model_choices[model_name] = model_cfg
|
| 66 |
+
for preprocessor in model_cfg.get("PREPROCESSOR", []):
|
| 67 |
+
if preprocessor["TYPE"] in self.edit_type_dict:
|
| 68 |
+
continue
|
| 69 |
+
self.edit_type_dict[preprocessor["TYPE"]] = preprocessor
|
| 70 |
+
self.default_type_list.append(preprocessor["TYPE"])
|
| 71 |
print('Models: ', self.model_choices.keys())
|
| 72 |
assert len(self.model_choices) > 0
|
| 73 |
if self.default_model_name == "": self.default_model_name = list(self.model_choices.keys())[0]
|
| 74 |
self.model_name = self.default_model_name
|
| 75 |
pipe_cfg = self.model_choices[self.default_model_name]
|
| 76 |
+
self.pipe = INFERENCES.build(pipe_cfg)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 77 |
# reformat examples
|
| 78 |
self.all_examples = [
|
| 79 |
[
|
| 80 |
+
one_example["edit_type"], one_example["instruction"],
|
| 81 |
+
one_example["input_reference_image"], one_example["input_image"],
|
| 82 |
+
one_example["input_mask"], one_example["output_h"],
|
| 83 |
+
one_example["output_w"], one_example["seed"]
|
| 84 |
+
]
|
| 85 |
+
for one_example in fft_examples
|
| 86 |
]
|
| 87 |
|
| 88 |
def construct_edit_image(self, edit_image, edit_mask):
|
|
|
|
| 105 |
else:
|
| 106 |
return None
|
| 107 |
|
|
|
|
|
|
|
|
|
|
| 108 |
def create_ui(self):
|
| 109 |
with gr.Row(equal_height=True, visible=True):
|
| 110 |
with gr.Column(scale=2):
|
|
|
|
| 121 |
height=600,
|
| 122 |
interactive=False,
|
| 123 |
type='pil',
|
| 124 |
+
elem_id='preprocess_image',
|
| 125 |
+
label='edit image'
|
| 126 |
)
|
| 127 |
|
| 128 |
self.edit_preprocess_mask_preview = gr.Image(
|
| 129 |
height=600,
|
| 130 |
interactive=False,
|
| 131 |
type='pil',
|
| 132 |
+
elem_id='preprocess_image_mask',
|
| 133 |
+
label='edit mask'
|
| 134 |
+
)
|
| 135 |
+
|
| 136 |
+
self.change_preprocess_preview = gr.Image(
|
| 137 |
+
height=600,
|
| 138 |
+
interactive=False,
|
| 139 |
+
type='pil',
|
| 140 |
+
elem_id='preprocess_change_image',
|
| 141 |
+
label='change image'
|
| 142 |
)
|
| 143 |
with gr.Row():
|
| 144 |
instruction = """
|
| 145 |
**Instruction**:
|
| 146 |
+
Users can perform reference generation or editing tasks by uploading reference images
|
| 147 |
+
and editing images. When uploading the editing image, various editing types are available
|
| 148 |
+
for selection. Users can choose different dimensions of information preservation,
|
| 149 |
+
such as edge information, color information, and more. Pre-processing information
|
| 150 |
+
can be viewed in the 'related input image' tab.
|
|
|
|
|
|
|
| 151 |
"""
|
| 152 |
self.instruction = gr.Markdown(value=instruction)
|
| 153 |
+
with gr.Row():
|
| 154 |
+
self.icon = gr.Image(
|
| 155 |
+
value=None,
|
| 156 |
+
interactive=False,
|
| 157 |
+
height=150,
|
| 158 |
+
type='pil',
|
| 159 |
+
elem_id='icon',
|
| 160 |
+
label='icon'
|
| 161 |
+
)
|
| 162 |
with gr.Row():
|
| 163 |
self.model_name_dd = gr.Dropdown(
|
| 164 |
choices=self.model_choices,
|
| 165 |
value=self.default_model_name,
|
| 166 |
label='Model Version')
|
| 167 |
+
self.edit_type = gr.Dropdown(choices=self.default_type_list,
|
| 168 |
interactive=True,
|
| 169 |
+
value=self.default_type_list[0],
|
|
|
|
|
|
|
|
|
|
|
|
|
| 170 |
label='Edit Type')
|
| 171 |
+
with gr.Row():
|
| 172 |
+
self.step = gr.Slider(minimum=1,
|
| 173 |
+
maximum=1000,
|
| 174 |
+
value=self.pipe.input.get("sample_steps", 20),
|
| 175 |
+
visible=self.pipe.input.get("sample_steps", None) is not None,
|
| 176 |
+
label='Sample Step')
|
| 177 |
+
self.cfg_scale = gr.Slider(
|
| 178 |
+
minimum=1.0,
|
| 179 |
+
maximum=100.0,
|
| 180 |
+
value=self.pipe.input.get("guide_scale", 4.5),
|
| 181 |
+
visible=self.pipe.input.get("guide_scale", None) is not None,
|
| 182 |
+
label='Guidance Scale')
|
| 183 |
+
self.seed = gr.Slider(minimum=-1,
|
| 184 |
+
maximum=1000000000000,
|
| 185 |
+
value=-1,
|
| 186 |
+
label='Seed')
|
| 187 |
+
self.output_height = gr.Slider(
|
| 188 |
+
minimum=256,
|
| 189 |
+
maximum=1440,
|
| 190 |
+
value=self.pipe.input.get("image_size", [1024, 1024])[0],
|
| 191 |
+
visible=self.pipe.input.get("image_size", None) is not None,
|
| 192 |
+
label='Output Height')
|
| 193 |
+
self.output_width = gr.Slider(
|
| 194 |
+
minimum=256,
|
| 195 |
+
maximum=1440,
|
| 196 |
+
value=self.pipe.input.get("image_size", [1024, 1024])[1],
|
| 197 |
+
visible=self.pipe.input.get("image_size", None) is not None,
|
| 198 |
+
label='Output Width')
|
| 199 |
+
|
| 200 |
+
self.repainting_scale = gr.Slider(
|
| 201 |
+
minimum=0.0,
|
| 202 |
+
maximum=1.0,
|
| 203 |
+
value=self.pipe.input.get("repainting_scale", 1.0),
|
| 204 |
+
visible=True,
|
| 205 |
+
label='Repainting Scale')
|
| 206 |
+
self.use_change = gr.Checkbox(
|
| 207 |
+
value=self.pipe.input.get("use_change", True),
|
| 208 |
+
visible=True,
|
| 209 |
+
label='Use Change')
|
| 210 |
+
self.keep_pixel = gr.Checkbox(
|
| 211 |
+
value=self.pipe.input.get("keep_pixel", True),
|
| 212 |
+
visible=True,
|
| 213 |
+
label='Keep Pixels')
|
| 214 |
+
self.keep_pixels_rate = gr.Slider(
|
| 215 |
+
minimum=0.5,
|
| 216 |
+
maximum=1.0,
|
| 217 |
+
value=0.8,
|
| 218 |
+
visible=True,
|
| 219 |
+
label='keep_pixel rate')
|
| 220 |
with gr.Row():
|
| 221 |
self.generation_info_preview = gr.Markdown(
|
| 222 |
label='System Log.',
|
|
|
|
| 229 |
placeholder='Input "@" find history of image',
|
| 230 |
label='Instruction',
|
| 231 |
container=False,
|
| 232 |
+
lines=1)
|
| 233 |
with gr.Column(scale=2, min_width=100):
|
| 234 |
with gr.Row():
|
| 235 |
with gr.Column(scale=1, min_width=100):
|
| 236 |
+
self.chat_btn = gr.Button(value='Generate', variant="primary")
|
| 237 |
|
| 238 |
with gr.Accordion(label='Advance', open=True):
|
| 239 |
with gr.Row(visible=True):
|
|
|
|
| 260 |
format="png"
|
| 261 |
)
|
| 262 |
|
| 263 |
+
with gr.Row():
|
| 264 |
+
self.eg = gr.Column(visible=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 265 |
|
| 266 |
def set_callbacks(self, *args, **kwargs):
|
| 267 |
########################################
|
|
|
|
| 276 |
torch.cuda.empty_cache()
|
| 277 |
torch.cuda.ipc_collect()
|
| 278 |
pipe_cfg = self.model_choices[model_name]
|
| 279 |
+
self.pipe = INFERENCES.build(pipe_cfg)
|
|
|
|
|
|
|
| 280 |
self.model_name = model_name
|
| 281 |
lock.release()
|
| 282 |
|
| 283 |
return (model_name, gr.update(),
|
| 284 |
gr.Slider(
|
| 285 |
+
value=self.pipe.input.get("sample_steps", 20),
|
| 286 |
+
visible=self.pipe.input.get("sample_steps", None) is not None),
|
| 287 |
gr.Slider(
|
| 288 |
value=self.pipe.input.get("guide_scale", 4.5),
|
| 289 |
visible=self.pipe.input.get("guide_scale", None) is not None),
|
| 290 |
gr.Slider(
|
| 291 |
+
value=self.pipe.input.get("image_size", [1024, 1024])[0],
|
| 292 |
+
visible=self.pipe.input.get("image_size", None) is not None),
|
| 293 |
gr.Slider(
|
| 294 |
+
value=self.pipe.input.get("image_size", [1024, 1024])[1],
|
| 295 |
+
visible=self.pipe.input.get("image_size", None) is not None),
|
| 296 |
gr.Slider(value=self.pipe.input.get("repainting_scale", 1.0))
|
| 297 |
)
|
| 298 |
|
|
|
|
| 307 |
self.output_width,
|
| 308 |
self.repainting_scale])
|
| 309 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 310 |
def change_edit_type(edit_type):
|
| 311 |
edit_info = self.edit_type_dict[edit_type]
|
| 312 |
edit_info = edit_info or {}
|
| 313 |
repainting_scale = edit_info.get("REPAINTING_SCALE", 1.0)
|
| 314 |
+
return gr.Slider(value=repainting_scale)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 315 |
|
| 316 |
self.edit_type.change(change_edit_type, inputs=[self.edit_type], outputs=[self.repainting_scale])
|
| 317 |
|
| 318 |
+
def resize_image(image, h):
|
| 319 |
+
ow, oh = image.size
|
| 320 |
+
w = int(h * ow / oh)
|
| 321 |
+
image = image.resize((w, h), Image.LANCZOS)
|
| 322 |
+
return image
|
| 323 |
+
|
| 324 |
+
def preprocess_input(ref_image, edit_image_dict, preprocess=None):
|
| 325 |
err_msg = ""
|
| 326 |
is_suc = True
|
| 327 |
if ref_image is not None:
|
|
|
|
| 337 |
edit_image = None
|
| 338 |
edit_mask = None
|
| 339 |
elif np.sum(np.array(edit_mask)) < 1:
|
| 340 |
+
edit_image = pillow_convert(edit_image, "RGB")
|
| 341 |
+
w, h = edit_image.size
|
| 342 |
+
edit_mask = Image.new("L", (w, h), 255)
|
| 343 |
else:
|
| 344 |
edit_image = pillow_convert(edit_image, "RGB")
|
| 345 |
edit_mask = Image.fromarray(edit_mask).convert('L')
|
|
|
|
| 347 |
err_msg = "Please provide the reference image or edited image."
|
| 348 |
return None, None, None, False, err_msg
|
| 349 |
return edit_image, edit_mask, ref_image, is_suc, err_msg
|
| 350 |
+
|
| 351 |
@spaces.GPU(duration=80)
|
| 352 |
def run_chat(
|
| 353 |
+
prompt,
|
| 354 |
+
ref_image,
|
| 355 |
+
edit_image,
|
| 356 |
+
edit_type,
|
| 357 |
+
cfg_scale,
|
| 358 |
+
step,
|
| 359 |
+
seed,
|
| 360 |
+
output_h,
|
| 361 |
+
output_w,
|
| 362 |
+
repainting_scale,
|
| 363 |
+
use_change,
|
| 364 |
+
keep_pixel,
|
| 365 |
+
keep_pixels_rate,
|
| 366 |
+
progress=gr.Progress(track_tqdm=True)
|
| 367 |
):
|
|
|
|
|
|
|
| 368 |
edit_info = self.edit_type_dict[edit_type]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 369 |
pre_edit_image, pre_edit_mask, pre_ref_image, is_suc, err_msg = preprocess_input(ref_image, edit_image)
|
| 370 |
+
icon = pre_edit_image or pre_ref_image
|
| 371 |
if not is_suc:
|
| 372 |
err_msg = f"<mark>{err_msg}</mark>"
|
| 373 |
return (gr.Image(), gr.Column(visible=True),
|
| 374 |
+
gr.Image(),
|
| 375 |
gr.Image(),
|
| 376 |
gr.Image(),
|
| 377 |
gr.Text(value=err_msg))
|
| 378 |
+
pre_edit_image = edit_preprocess(edit_info.ANNOTATOR, we.device_id, pre_edit_image, pre_edit_mask)
|
| 379 |
# edit_image["background"] = pre_edit_image
|
| 380 |
st = time.time()
|
| 381 |
+
image, edit_image, change_image, mask, seed = self.pipe(
|
| 382 |
reference_image=pre_ref_image,
|
| 383 |
edit_image=pre_edit_image,
|
| 384 |
edit_mask=pre_edit_mask,
|
|
|
|
| 390 |
guide_scale=cfg_scale,
|
| 391 |
seed=seed,
|
| 392 |
repainting_scale=repainting_scale,
|
| 393 |
+
use_change=use_change,
|
| 394 |
+
keep_pixels=keep_pixel,
|
| 395 |
+
keep_pixels_rate=keep_pixels_rate
|
| 396 |
)
|
| 397 |
et = time.time()
|
| 398 |
msg = f"prompt: {prompt}; seed: {seed}; cost time: {et - st}s; repaiting scale: {repainting_scale}"
|
| 399 |
|
| 400 |
+
if icon is not None:
|
| 401 |
+
icon = resize_image(icon, 150)
|
| 402 |
+
|
| 403 |
return (gr.Image(value=image), gr.Column(visible=True),
|
| 404 |
+
gr.Image(value=edit_image if edit_image is not None else edit_image),
|
| 405 |
+
gr.Image(value=change_image),
|
| 406 |
gr.Image(value=pre_edit_mask if pre_edit_mask is not None else None),
|
| 407 |
+
gr.Text(value=msg),
|
| 408 |
+
gr.Image(value=icon))
|
| 409 |
|
| 410 |
chat_inputs = [
|
| 411 |
self.reference_image,
|
| 412 |
self.edit_image,
|
|
|
|
| 413 |
self.edit_type,
|
| 414 |
self.cfg_scale,
|
| 415 |
self.step,
|
| 416 |
self.seed,
|
| 417 |
self.output_height,
|
| 418 |
self.output_width,
|
| 419 |
+
self.repainting_scale,
|
| 420 |
+
self.use_change,
|
| 421 |
+
self.keep_pixel,
|
| 422 |
+
self.keep_pixels_rate
|
| 423 |
]
|
| 424 |
|
| 425 |
chat_outputs = [
|
| 426 |
+
self.gallery_image, self.edit_preprocess_panel, self.edit_preprocess_preview,
|
| 427 |
+
self.change_preprocess_preview,
|
| 428 |
+
self.edit_preprocess_mask_preview, self.generation_info_preview,
|
| 429 |
+
self.icon
|
| 430 |
]
|
| 431 |
|
| 432 |
self.chat_btn.click(run_chat,
|
|
|
|
| 440 |
queue=True)
|
| 441 |
|
| 442 |
@spaces.GPU(duration=80)
|
| 443 |
+
def run_example(edit_type, prompt, ref_image, edit_image, edit_mask,
|
| 444 |
+
output_h, output_w, seed, use_change, keep_pixel,
|
| 445 |
+
keep_pixels_rate,
|
| 446 |
+
progress=gr.Progress(track_tqdm=True)):
|
| 447 |
|
| 448 |
step = self.pipe.input.get("sample_steps", 20)
|
| 449 |
cfg_scale = self.pipe.input.get("guide_scale", 20)
|
|
|
|
| 450 |
edit_info = self.edit_type_dict[edit_type]
|
| 451 |
|
| 452 |
edit_image = self.construct_edit_image(edit_image, edit_mask)
|
| 453 |
|
| 454 |
+
pre_edit_image, pre_edit_mask, pre_ref_image, _, _ = preprocess_input(ref_image, edit_image)
|
| 455 |
+
|
| 456 |
+
icon = pre_edit_image or pre_ref_image
|
| 457 |
+
|
| 458 |
+
pre_edit_image = edit_preprocess(edit_info.ANNOTATOR, we.device_id, pre_edit_image, pre_edit_mask)
|
| 459 |
edit_info = edit_info or {}
|
| 460 |
repainting_scale = edit_info.get("REPAINTING_SCALE", 1.0)
|
| 461 |
st = time.time()
|
| 462 |
+
image, edit_image, change_image, mask, seed = self.pipe(
|
| 463 |
reference_image=pre_ref_image,
|
| 464 |
edit_image=pre_edit_image,
|
| 465 |
edit_mask=pre_edit_mask,
|
|
|
|
| 471 |
guide_scale=cfg_scale,
|
| 472 |
seed=seed,
|
| 473 |
repainting_scale=repainting_scale,
|
| 474 |
+
use_change=use_change,
|
| 475 |
+
keep_pixels=keep_pixel,
|
| 476 |
+
keep_pixels_rate=keep_pixels_rate
|
| 477 |
)
|
| 478 |
et = time.time()
|
| 479 |
msg = f"prompt: {prompt}; seed: {seed}; cost time: {et - st}s; repaiting scale: {repainting_scale}"
|
| 480 |
if pre_edit_image is not None:
|
| 481 |
+
ret_image = Image.composite(Image.new("RGB", pre_edit_image.size, (0, 0, 0)), pre_edit_image,
|
| 482 |
+
pre_edit_mask)
|
| 483 |
else:
|
| 484 |
ret_image = None
|
| 485 |
+
|
| 486 |
+
if icon is not None:
|
| 487 |
+
icon = resize_image(icon, 150)
|
| 488 |
return (gr.Image(value=image), gr.Column(visible=True),
|
| 489 |
+
gr.Image(value=edit_image if edit_image is not None else edit_image),
|
| 490 |
+
gr.Image(value=change_image),
|
| 491 |
gr.Image(value=pre_edit_mask if pre_edit_mask is not None else None),
|
| 492 |
gr.Text(value=msg),
|
| 493 |
+
gr.update(value=ret_image),
|
| 494 |
+
gr.Image(value=icon))
|
| 495 |
|
| 496 |
with self.eg:
|
| 497 |
self.example_edit_image = gr.Image(label='Edit Image',
|
| 498 |
+
type='pil',
|
| 499 |
+
image_mode='RGB',
|
| 500 |
+
visible=False)
|
| 501 |
self.example_edit_mask = gr.Image(label='Edit Image Mask',
|
| 502 |
+
type='pil',
|
| 503 |
+
image_mode='L',
|
| 504 |
+
visible=False)
|
| 505 |
|
| 506 |
self.examples = gr.Examples(
|
| 507 |
fn=run_example,
|
| 508 |
examples=self.all_examples,
|
| 509 |
inputs=[
|
| 510 |
+
self.edit_type, self.text, self.reference_image, self.example_edit_image,
|
| 511 |
+
self.example_edit_mask, self.output_height, self.output_width, self.seed,
|
| 512 |
+
self.use_change, self.keep_pixel, self.keep_pixels_rate
|
| 513 |
],
|
| 514 |
outputs=[self.gallery_image, self.edit_preprocess_panel, self.edit_preprocess_preview,
|
| 515 |
+
self.change_preprocess_preview,
|
| 516 |
+
self.edit_preprocess_mask_preview, self.generation_info_preview,
|
| 517 |
+
self.edit_image,
|
| 518 |
+
self.icon],
|
| 519 |
+
examples_per_page=15,
|
| 520 |
cache_examples=False,
|
| 521 |
run_on_click=True)
|
| 522 |
|