update

autoregressive/models/generate.py

@@ -57,6 +57,7 @@ def top_k_top_p_filtering(
 
 
 def sample(logits, temperature: float=1.0, top_k: int=2000, top_p: float=1.0, sample_logits=True):        
+    # print(logits, torch.any(torch.isnan(logits)))
     logits = logits[:, -1, :] / max(temperature, 1e-5)
     if top_k > 0 or top_p < 1.0:
         logits = top_k_top_p_filtering(logits, top_k=top_k, top_p=top_p)
@@ -137,15 +138,16 @@ def decode_n_tokens(
 
 @torch.no_grad()
 def generate(model, cond, max_new_tokens, emb_masks=None, cfg_scale=1.0, cfg_interval=-1, condition=None, condition_null=None, condition_token_nums=0, **sampling_kwargs):
+    print("cond", torch.any(torch.isnan(cond)))
     if condition is not None:
         with torch.no_grad():
-            print(f'nan: {torch.any(torch.isnan(model.adapter.model.embeddings.patch_embeddings.projection.weight))}')
-            print(model.adapter.model.embeddings.patch_embeddings.projection.weight)
+            # print(f'nan: {torch.any(torch.isnan(model.adapter.model.embeddings.patch_embeddings.projection.weight))}')
+            # print(model.adapter.model.embeddings.patch_embeddings.projection.weight)
             condition = model.adapter(condition)
-            print(condition)
-            condition = torch.ones_like(condition)
+            # print("before condition", condition)
+            # condition = torch.ones_like(condition)
             condition = model.adapter_mlp(condition)
-            #print(condition)
+            print("condition", torch.any(torch.isnan(condition)))
     if model.model_type == 'c2i':
         if cfg_scale > 1.0:
             cond_null = torch.ones_like(cond) * model.num_classes

language/t5.py

@@ -18,7 +18,7 @@ class T5Embedder:
 
     def __init__(self, device, dir_or_name='t5-v1_1-xxl', *, local_cache=False, cache_dir=None, hf_token=None, use_text_preprocessing=True,
                  t5_model_kwargs=None, torch_dtype=None, use_offload_folder=None, model_max_length=120):
-        self.device = torch.device('cuda:0')
+        self.device = torch.device('cpu')
         self.torch_dtype = torch_dtype or torch.bfloat16
         if t5_model_kwargs is None:
             t5_model_kwargs = {'low_cpu_mem_usage': True, 'torch_dtype': self.torch_dtype}
@@ -53,7 +53,7 @@ class T5Embedder:
         print(tokenizer_path)
         self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)
         self.model = T5EncoderModel.from_pretrained(path, **t5_model_kwargs).eval()
-        self.model.to('cuda')
+        # self.model.to('cuda')
         self.model_max_length = model_max_length
 
     def get_text_embeddings(self, texts):

model.py

@@ -123,9 +123,9 @@ class Model:
         image = resize_image_to_16_multiple(image, 'canny')
         W, H = image.size
         print(W, H)
-        print("before cuda", self.gpt_model_canny.adapter.model.embeddings.patch_embeddings.projection.weight)
         self.t5_model.model.to('cuda')
         self.gpt_model_canny.to('cuda')
+        print("after cuda", self.gpt_model_canny.adapter.model.embeddings.patch_embeddings.projection.weight)
 
         condition_img = self.get_control_canny(np.array(image), low_threshold,
                                                high_threshold)