Update: transparent image

app.py

@@ -8,13 +8,14 @@ from torch import nn
 from transformers import SegformerForSemanticSegmentation
 import sys
 import io 
-
+import pdb
 
 ###################
 # Setup label names
 target_list = ['Crack', 'ACrack', 'Wetspot', 'Efflorescence', 'Rust', 'Rockpocket', 'Hollowareas', 'Cavity',
                'Spalling', 'Graffiti', 'Weathering', 'Restformwork', 'ExposedRebars', 
                'Bearing', 'EJoint', 'Drainage', 'PEquipment', 'JTape', 'WConccor']
+target_list_all = ["All"] + target_list
 classes, nclasses = target_list, len(target_list)
 label2id = dict(zip(classes, range(nclasses)))
 id2label = dict(zip(range(nclasses), classes))
@@ -48,7 +49,9 @@ model.eval()
 ##################
 
 to_tensor = transforms.ToTensor()
+to_array = transforms.ToPILImage()
 resize = transforms.Resize((512, 512))
+resize_small = transforms.Resize((369,369))
 normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
                                      std=[0.229, 0.224, 0.225])
 
@@ -58,11 +61,50 @@ def process_pil(img):
     img = normalize(img)
     return img
 
+# the background of the image
+def resize_pil(img):
+    img = to_tensor(img)
+    img = resize_small(img)
+    img = to_array(img)
+    return img
+
+# combine the foreground (mask_all) and background (original image) to create one image
+def transparent(fg, bg, alpha_factor):
+
+    foreground = np.array(fg)
+    background = np.array(bg)
+
+    background = Image.fromarray(bg)
+    foreground = Image.fromarray(fg)
+    new_alpha_factor = int(255*alpha_factor)
+    foreground.putalpha(new_alpha_factor)
+    background.paste(foreground, (0, 0), foreground)
+
+    return background
+
+def show_img(all_imgs, dropdown, bg, alpha_factor):
+    idx = target_list_all.index(dropdown)
+    fg= all_imgs[idx]["name"]
+
+    foreground = Image.open(fg)
+    background = np.array(bg)
+
+    background = Image.fromarray(bg)
+    new_alpha_factor = int(255*alpha_factor)
+    foreground.putalpha(new_alpha_factor)
+    background.paste(foreground, (0, 0), foreground)
+
+    return background
+
 ###########
 # Inference
 
-def inference(img, name):
+
+def inference(img, alpha_factor):
+    background = resize_pil(img) 
+
     img = process_pil(img)
+
     mask = model(img.unsqueeze(0)) # we need a batch, hence we introduce an extra dimenation at position 0 (unsqueeze)
     mask = mask[0]
 
@@ -85,21 +127,39 @@ def inference(img, name):
     labs = ["ALL"] + target_list
 
     fig, axes = plt.subplots(5, 4, figsize = (10,10))
-            
+    
+    # save all mask_preds in all_mask
+    all_masks = []
+
     for i, ax in enumerate(axes.flat):
         label = labs[i]
+        
+        all_masks.append(mask_preds[i])
+
         ax.imshow(mask_preds[i])
         ax.set_title(label)
           
     plt.tight_layout()        
 
-
     # plt to PIL
     img_buf = io.BytesIO()
     fig.savefig(img_buf, format='png')
     im = Image.open(img_buf)
-    return im
 
+    # Saved all masks combined with unvisible xaxis und yaxis and without a white 
+    # background.
+    all_images = []
+    for i in range(len(all_masks)):
+        plt.figure()
+        fig = plt.imshow(all_masks[i])
+        plt.axis('off')
+        fig.axes.get_xaxis().set_visible(False)
+        fig.axes.get_yaxis().set_visible(False)
+        img_buf = io.BytesIO()
+        plt.savefig(img_buf, bbox_inches='tight', pad_inches = 0, format='png')
+        all_images.append(Image.open(img_buf))
+
+    return im, all_images, background
 
 
 title = "dacl-challenge @ WACV2024"
@@ -141,15 +201,42 @@ description = """
 """
 
 article = "<p style='text-align: center'><a href='https://github.com/phiyodr/dacl10k-toolkit' target='_blank'>Github Repo</a></p>"
-examples=[['assets/dacl10k_v2_validation_0037.jpg', 'dacl10k_v2_validation_0037.jpg'],['assets/dacl10k_v2_validation_0068.jpg','dacl10k_v2_validation_0068.jpg'], ['assets/dacl10k_v2_validation_0053.jpg', 'dacl10k_v2_validation_0053.jpg']]
-
-demo = gr.Interface(
-    fn=inference, 
-    inputs=gr.inputs.Image(type="pil"), 
-    outputs=gr.outputs.Image(type="pil"),
-    title=title,
-    description=description,
-    article=article,
-    examples=examples)
-
-demo.launch()
+examples=[
+["assets/dacl10k_v2_validation_0026.jpg", "dacl10k_v2_validation_0026.jpg"],
+["assets/dacl10k_v2_validation_0037.jpg", "dacl10k_v2_validation_0037.jpg"],
+["assets/dacl10k_v2_validation_0053.jpg", "dacl10k_v2_validation_0053.jpg"],
+["assets/dacl10k_v2_validation_0068.jpg", "dacl10k_v2_validation_0068.jpg"],
+["assets/dacl10k_v2_validation_0125.jpg", "dacl10k_v2_validation_0125.jpg"],
+["assets/dacl10k_v2_validation_0153.jpg", "dacl10k_v2_validation_0153.jpg"],
+["assets/dacl10k_v2_validation_0263.jpg", "dacl10k_v2_validation_0263.jpg"],
+["assets/dacl10k_v2_validation_0336.jpg", "dacl10k_v2_validation_0336.jpg"],
+["assets/dacl10k_v2_validation_0429.jpg", "dacl10k_v2_validation_0429.jpg"],
+["assets/dacl10k_v2_validation_0500.jpg", "dacl10k_v2_validation_0500.jpg"],
+["assets/dacl10k_v2_validation_0549.jpg", "dacl10k_v2_validation_0549.jpg"],
+["assets/dacl10k_v2_validation_0609.jpg", "dacl10k_v2_validation_0609.jpg"]
+]
+
+
+
+with gr.Blocks() as app:
+    with gr.Row():
+        input_img = gr.inputs.Image(type="pil", label="Original Image")
+        gr.Examples(examples=examples, inputs=[input_img])
+    with gr.Row():
+        img = gr.outputs.Image(type="pil", label="All Masks")
+        transparent_img = gr.outputs.Image(type="pil", label="Transparent Image")
+    with gr.Row():
+        slider = gr.Slider(minimum=0, maximum=1, value=0.5, label="Alpha Factor")
+        dropdown = gr.Dropdown(choices=target_list_all, label="Pick image", value="All")
+    
+    all_masks = gr.Gallery(visible=False)
+    background = gr.Image(visible=False)
+
+    generate_mask_slider = gr.Button("Generate Masks")
+    generate_mask_slider.click(inference, inputs=[input_img], outputs=[img, all_masks, background])
+
+    submit_transparent_img = gr.Button("Generate Transparent Mask (with Alpha Factor)")
+    submit_transparent_img.click(show_img, inputs=[all_masks, dropdown, background, slider], outputs=[transparent_img])
+
+
+app.launch()