Update human_text_detect.py

human_text_detect.py

@@ -6,7 +6,7 @@ import numpy as np
 import pickle
 from src.DetectLM import DetectLM
 from src.PerplexityEvaluator import PerplexityEvaluator
-from src.PrepareArticles import PrepareArticles #Idan 
+from src.PrepareArticles import PrepareArticles 
 from src.fit_survival_function import fit_per_length_survival_function
 from glob import glob
 import spacy
@@ -96,16 +96,7 @@ def detect_human_text(model_name, topic, text):
     min_tokens_per_sentence = 10
     max_tokens_per_sentence = 100
 
-    ####
-    cache_dir = "/cache/huggingface"
-    # Check if the directory exists and is writable
-    print(f"Cache directory exists: {os.path.exists(cache_dir)}")
-    print(f"Cache directory is writable: {os.access(cache_dir, os.W_OK)}")
-    
-    # List contents of the directory
-    print("Contents of cache directory before loading model:")
-    os.system(f"ls -lah {cache_dir}")
-    ###
+    cache_dir = f"/cache/huggingface/{model_name}"
     
     # Init model
     print('Init model')
@@ -114,17 +105,9 @@ def detect_human_text(model_name, topic, text):
     tokenizer = AutoTokenizer.from_pretrained(lm_name, cache_dir=cache_dir)
     model = AutoModelForCausalLM.from_pretrained(lm_name, cache_dir=cache_dir)
 
-    ###
-    print("Contents of cache directory after loading model:")
-    os.system(f"ls -lah {cache_dir}")
-
-    print(f"Current HF_HOME: {os.getenv('HF_HOME')}")
-    print(f"Current TRANSFORMERS_CACHE: {os.getenv('TRANSFORMERS_CACHE')}")
-    
-    # Check where the tokenizer and model are actually downloaded
-    print(f"Tokenizer saved at: {tokenizer.save_pretrained(cache_dir)}")
-    print(f"Model saved at: {model.save_pretrained(cache_dir)}")
-    ###
+    print("Save model")
+    tokenizer.save_pretrained(cache_dir)
+    model.save_pretrained(cache_dir)
     
     print('Init PerplexityEvaluator')
     sentence_detector = PerplexityEvaluator(model, tokenizer)