Update model_worker.py

model_worker.py

@@ -76,7 +76,8 @@ class ModelWorker:
     @torch.inference_mode()
     def generate_stream(self, params):
         tokenizer, model = self.tokenizer, self.model
-
+        # for adjust to zero environment of huggingface
+        model.to(self.device)
         prompt = params["prompt"]
         ori_prompt = prompt
         images = params.get("images", None)
@@ -90,9 +91,9 @@ class ModelWorker:
                 assert prompt.count(DEFAULT_IMAGE_TOKEN) == 1
 
                 images, patch_positions, prompt = self.doc_image_processor(images=image, query=prompt)
-                images = images.to(self.model.device, dtype=torch.float16)
-                # images = images.to(self.model.device, dtype=torch.bfloat16)
-                patch_positions = patch_positions.to(self.model.device)
+                images = images.to(self.device, dtype=torch.float16)
+                # images = images.to(self.device, dtype=torch.bfloat16)
+                patch_positions = patch_positions.to(self.device)
 
                 replace_token = DEFAULT_IMAGE_TOKEN
                 prompt = prompt.replace(DEFAULT_IMAGE_TOKEN, replace_token)