Update human_text_detect.py
Browse files- human_text_detect.py +4 -4
human_text_detect.py
CHANGED
|
@@ -15,7 +15,6 @@ import os
|
|
| 15 |
|
| 16 |
logging.basicConfig(level=logging.INFO)
|
| 17 |
|
| 18 |
-
|
| 19 |
def read_all_csv_files(pattern):
|
| 20 |
df = pd.DataFrame()
|
| 21 |
print(pattern)
|
|
@@ -23,7 +22,6 @@ def read_all_csv_files(pattern):
|
|
| 23 |
df = pd.concat([df, pd.read_csv(f)])
|
| 24 |
return df
|
| 25 |
|
| 26 |
-
|
| 27 |
def get_survival_function(df, G=101):
|
| 28 |
"""
|
| 29 |
Returns a survival function for every sentence length in tokens.
|
|
@@ -44,7 +42,6 @@ def get_survival_function(df, G=101):
|
|
| 44 |
xx1 = df1[value_name]
|
| 45 |
return fit_per_length_survival_function(ll, xx1, log_space=True, G=G)
|
| 46 |
|
| 47 |
-
|
| 48 |
def mark_edits_remove_tags(chunks, tag="edit"):
|
| 49 |
text_chunks = chunks['text']
|
| 50 |
edits = []
|
|
@@ -96,7 +93,10 @@ def detect_human_text(model_name, topic, text):
|
|
| 96 |
min_tokens_per_sentence = 10
|
| 97 |
max_tokens_per_sentence = 100
|
| 98 |
|
| 99 |
-
cache_dir = f"
|
|
|
|
|
|
|
|
|
|
| 100 |
|
| 101 |
# Init model
|
| 102 |
print('Init model')
|
|
|
|
| 15 |
|
| 16 |
logging.basicConfig(level=logging.INFO)
|
| 17 |
|
|
|
|
| 18 |
def read_all_csv_files(pattern):
|
| 19 |
df = pd.DataFrame()
|
| 20 |
print(pattern)
|
|
|
|
| 22 |
df = pd.concat([df, pd.read_csv(f)])
|
| 23 |
return df
|
| 24 |
|
|
|
|
| 25 |
def get_survival_function(df, G=101):
|
| 26 |
"""
|
| 27 |
Returns a survival function for every sentence length in tokens.
|
|
|
|
| 42 |
xx1 = df1[value_name]
|
| 43 |
return fit_per_length_survival_function(ll, xx1, log_space=True, G=G)
|
| 44 |
|
|
|
|
| 45 |
def mark_edits_remove_tags(chunks, tag="edit"):
|
| 46 |
text_chunks = chunks['text']
|
| 47 |
edits = []
|
|
|
|
| 93 |
min_tokens_per_sentence = 10
|
| 94 |
max_tokens_per_sentence = 100
|
| 95 |
|
| 96 |
+
cache_dir = f"/cacheHuggingface/{model_name}"
|
| 97 |
+
# Use a writable directory inside the Hugging Face Space
|
| 98 |
+
os.makedirs("/cacheHuggingface/PHI2", exist_ok=True)
|
| 99 |
+
os.makedirs("/cacheHuggingface/GPT2XL", exist_ok=True)
|
| 100 |
|
| 101 |
# Init model
|
| 102 |
print('Init model')
|