Update human_text_detect.py

human_text_detect.py

@@ -93,21 +93,22 @@ def detect_human_text(model_name, topic, text):
     min_tokens_per_sentence = 10
     max_tokens_per_sentence = 100
 
-    cache_dir = f"/tmp/cacheHuggingface/{model_name}"
+    cache_dir_tokenizer = f"/data/cacheHuggingface/{model_name}/tokenizer"
     
     # Init model
     print('Init tokenizer')
     lm_name = 'gpt2-xl' if model_name == 'GPT2XL' else 'microsoft/phi-2'
-    tokenizer = AutoTokenizer.from_pretrained(lm_name) #, cache_dir=cache_dir
+    tokenizer = AutoTokenizer.from_pretrained(lm_name, cache_dir=cache_dir_tokenizer)
     
-    # print("Save tokenizer")
-    # tokenizer.save_pretrained(cache_dir)
+    print("Save tokenizer")
+    tokenizer.save_pretrained(cache_dir_tokenizer)
     
     print('Init model')
-    model = AutoModelForCausalLM.from_pretrained(lm_name) #, cache_dir=cache_dir
+    cache_dir_model = f"/data/cacheHuggingface/{model_name}/model"
+    model = AutoModelForCausalLM.from_pretrained(lm_name, cache_dir=cache_dir_model)
 
-    # print("Save model")
-    # model.save_pretrained(cache_dir)
+    print("Save model")
+    model.save_pretrained(cache_dir_model)
     
     print('Init PerplexityEvaluator')
     sentence_detector = PerplexityEvaluator(model, tokenizer)