update cap

app.py

@@ -2,8 +2,8 @@ import os
 import sys
 from pathlib import Path
 # os.system("cd transformers && pip install .")
-os.system("cd multimodal && pip install  .")
-os.system("cd multimodal/YOLOX && pip install .")
+# os.system("cd multimodal && pip install -e .")
+# os.system("cd multimodal/YOLOX && pip install .")
 import numpy as np
 import torch
 from PIL import Image

multimodal/open_flamingo/chat/conversation.py

@@ -324,7 +324,7 @@ class Chat:
                repetition_penalty=1.0, length_penalty=1, temperature=1, max_length=2000):
         # conv.append_message(conv.roles[1], None)
         # embs = self.get_context_emb(conv, img_list)
-        # 
+        #
         # # current_max_len = embs.shape[1] + max_new_tokens + 100
         # # begin_idx = max(0, current_max_len - max_length)
         # # embs = embs[:, begin_idx:]
@@ -494,7 +494,7 @@ class Chat:
         #     if len(image.shape) == 3:
         #         image = image.unsqueeze(0)
         #     # image = image.to(self.device)
-        # 
+        #
         # # image_emb, _ = self.model.encode_img(image)
         # img_list.append(image_emb)
         # conv.append_message(conv.roles[0], "<Img><ImageHere></Img>")
@@ -587,4 +587,3 @@ def evaluate_exp(
 
 
 
-

multimodal/open_flamingo/eval/task/caption_chat.py

@@ -51,7 +51,8 @@ def prepare_batch_images(batch, image_processor):
 
 
 def captioner(
-    model,tokenizer,image_ori,batch_images,input_ids,attention_mask,image_start_index_list,image_nums,added_bbox_list,debug=True):
+        model, tokenizer, image_ori, batch_images, input_ids, attention_mask, image_start_index_list, image_nums,
+        added_bbox_list, debug=True):
     """Evaluate a model on COCO dataset.
     Returns:
         float: CIDEr score
@@ -80,7 +81,6 @@ def captioner(
             input_ids = input_ids
             attention_mask = attention_mask
         else:
-
             encodings = tokenizer(
                 [prompt],
                 padding="longest",
@@ -93,7 +93,7 @@ def captioner(
         image_start_index_list = image_start_index_list
         image_nums = image_nums
         if debug:
-            print("input--->",tokenizer.decode(input_ids[0]))
+            print("input--->", tokenizer.decode(input_ids[0]))
         p1 = MinNewTokensLengthLogitsProcessor(
             prompt_length_to_skip=input_ids.shape[-1],
             min_new_tokens=5,
@@ -114,7 +114,7 @@ def captioner(
                 logits_processor_list=[p1, visual_logits_processor],
             )
         if debug:
-            print("outputs--->",tokenizer.decode(outputs[0]))
+            print("outputs--->", tokenizer.decode(outputs[0]))
         if outputs[0, -2] in [previsual_token_id, visual_token_id] and outputs[0, -1] == bos_token_id:
             prompt = tokenizer.decode(outputs.clone()[0])
             is_visual = (outputs[0, -2] == visual_token_id)
@@ -132,7 +132,7 @@ def captioner(
             image_start_index_list = [[x] for x in image_start_index_list]
             image_nums = [1] * len(input_ids)
             if debug:
-                print("get the visual bbox--->",tokenizer.decode(input_ids[0]))
+                print("get the visual bbox--->", tokenizer.decode(input_ids[0]))
             with torch.no_grad():
                 outputs = model(
                     vision_x=batch_images,
@@ -145,6 +145,8 @@ def captioner(
                 )
             boxes = outputs["boxes"]
             scores = outputs["scores"]
+            if debug:
+                print("box num---->", len(boxes))
             # if not model.valid:
             #     import pdb; pdb.set_trace()
             if boxes is not None:
@@ -168,7 +170,8 @@ def captioner(
                     open_cv_image = np.array(image_ori)
                     open_cv_image = open_cv_image[:, :, ::-1].copy()
                     for i, pre_box in enumerate(boxes):
-                        open_cv_image = cv2.rectangle(open_cv_image, pre_box[:2].astype(int), pre_box[2:].astype(int), (0, 255, 0), i+1)
+                        open_cv_image = cv2.rectangle(open_cv_image, pre_box[:2].astype(int), pre_box[2:].astype(int),
+                                                      (0, 255, 0), i + 1)
                     out_image = Image.fromarray(cv2.cvtColor(open_cv_image, cv2.COLOR_BGR2RGB))
                     # exit()
                     pre_box = boxes[scores.argmax()]
@@ -181,7 +184,14 @@ def captioner(
             else:
                 # if debug:
                 #     import pdb;pdb.set_trace()
+                prompt = tokenizer.decode(outputs.clone()[0])
+                if debug:
+                    print("before else---->", prompt)
                 prompt = tokenizer.decode(outputs[0, :-2].clone()[0])
+                if debug:
+                    print("after else---->", prompt)
+
+
         else:
             break
     outputs = outputs[:, ori_prompt_length:]
@@ -190,7 +200,8 @@ def captioner(
     #     postprocess_captioning_generation(out).replace('"', "")
     #     for out in tokenizer.batch_decode(outputs, skip_special_tokens=True)
     # ]
-        # import pdb; pdb.set_trace()
+    # import pdb; pdb.set_trace()
+    print("out----------------------------------------------------------------------------------------->")
     return outputs, out_image
 
 
@@ -428,5 +439,4 @@ def evaluate_coco_flickr(
         metrics = {}
         metrics["CIDEr"] = 0.0
 
-
     return metrics["CIDEr"]