Update human_text_detect.py

human_text_detect.py

@@ -102,17 +102,17 @@ def detect_human_text(model_name, topic, text):
     # Init model
     print('Init tokenizer')
     lm_name = 'gpt2-xl' if model_name == 'GPT2XL' else 'microsoft/phi-2'
-    tokenizer = AutoTokenizer.from_pretrained(lm_name, cache_dir=cache_dir_tokenizer) #
+    tokenizer = AutoTokenizer.from_pretrained(cache_dir_tokenizer) #lm_name, cache_dir=cache_dir_tokenizer
     
-    print("Save tokenizer")
-    tokenizer.save_pretrained(cache_dir_tokenizer)
+    # print("Save tokenizer")
+    # tokenizer.save_pretrained(cache_dir_tokenizer)
     
     print('Init model')
     cache_dir_model = f"/data/cacheHuggingface/{model_name}/model"
-    model = AutoModelForCausalLM.from_pretrained(lm_name, cache_dir=cache_dir_model)
+    model = AutoModelForCausalLM.from_pretrained(cache_dir_model) #lm_name, cache_dir=cache_dir_model
 
-    print("Save model")
-    model.save_pretrained(cache_dir_model)
+    # print("Save model")
+    # model.save_pretrained(cache_dir_model)
     
     print('Init PerplexityEvaluator')
     sentence_detector = PerplexityEvaluator(model, tokenizer)