support fused backbones and update MODEL_ID_TO_NAME

interactive_demo.py

@@ -152,7 +152,13 @@ class ModelWorker:
             # Assume `image_transform` is a HF ImageProcessor...
             pixel_values = self.image_processor(images[0].convert("RGB"), return_tensors="pt")["pixel_values"][0]
 
-        generated_text = self.vlm.generate_answer(torch.unsqueeze(pixel_values.cuda(), 0), question_prompt)[0]
+        if type(pixel_values) is dict:
+            for k in pixel_values.keys():
+                pixel_values[k] = torch.unsqueeze(pixel_values[k].cuda(), 0)
+        else:
+            pixel_values = torch.unsqueeze(pixel_values.cuda(), 0)
+
+        generated_text = self.vlm.generate_answer(pixel_values, question_prompt)[0]
         generated_text = generated_text.split("USER")[0].split("ASSISTANT")[0]
         yield json.dumps({"text": ori_prompt + generated_text, "error_code": 0}).encode() + b"\0"
 

serve/__init__.py

@@ -5,31 +5,31 @@ from collections import OrderedDict
 MODEL_ID_TO_NAME = OrderedDict(
     [
         (
-            "llava-lvis4v-lrv+lvis4v-lrv-resize-naive-clip-vit-l-14-336px-no-align-2-epochs-llama2pure+13b+stage-finetune+x7",
-            "Prism-CLIP 13B",
+            "llava-lvis4v-lrv+redux-lvis4v-lrv-resize-naive-dinosiglip-vit-so-14-384px-no-align+13b+stage-finetune+x7",
+            "PrismaticVLM 13B - Chat",
         ),
         (
-            "llava-lvis4v-lrv+lvis4v-lrv-resize-naive-clip-vit-l-14-336px-no-align-2-epochs-llama2pure+7b+stage-finetune+x7",
-            "Prism-CLIP 7B",
+            "llava-lvis4v-lrv+redux-lvis4v-lrv-resize-naive-dinosiglip-vit-so-14-384px-no-align+7b+stage-finetune+x7",
+            "PrismaticVLM 7B - Chat",
         ),
         (
-            "resize-naive-clip-vit-l-14-336px-no-align-llama2pure+13b+stage-finetune+x7",
-            "Prism-CLIP 13B (Controlled)",
+            "llava-lvis4v-lrv+redux-lvis4v-lrv-resize-naive-dinosiglip-vit-so-14-384px-no-align-llama2pure+13b+stage-finetune+x7",
+            "PrismaticVLM 13B",
         ),
         (
-            "resize-naive-clip-vit-l-14-336px-no-align-llama2pure+7b+stage-finetune+x7",
-            "Prism-CLIP 7B (Controlled)",
+            "llava-lvis4v-lrv+redux-lvis4v-lrv-resize-naive-dinosiglip-vit-so-14-384px-no-align-llama2pure+7b+stage-finetune+x7",
+            "PrismaticVLM 7B",
         ),
         (
-            "resize-naive-clip-vit-l-14-336px-no-align+13b+stage-finetune+x7",
-            "Prism-CLIP 13B (Controlled) - Chat",
+            "redux-resize-naive-dinosiglip-vit-so-14-384px-no-align-llama2pure+13b+stage-finetune+x7",
+            "PrismaticVLM 13B (Controlled)",
         ),
         (
-            "resize-naive-clip-vit-l-14-336px-no-align+7b+stage-finetune+x7",
-            "Prism-CLIP 7B (Controlled) - Chat",
+            "redux-resize-naive-dinosiglip-vit-so-14-384px-no-align-llama2pure+7b+stage-finetune+x7",
+            "PrismaticVLM 7B (Controlled)",
         ),
-        ("llava-v1.5-7b", "LLaVA 1.5: 7B"),
         ("llava-v1.5-13b", "LLaVA 1.5: 13B"),
+        ("llava-v1.5-7b", "LLaVA 1.5: 7B"),
     ]
 )