Update human_text_detect.py

human_text_detect.py

@@ -1,169 +1,169 @@
-import torch
-import pandas as pd
-from transformers import AutoTokenizer, AutoModelForCausalLM
-import logging
-import numpy as np
-import pickle
-from src.DetectLM import DetectLM
-from src.PerplexityEvaluator import PerplexityEvaluator
-from src.PrepareArticles import PrepareArticles #Idan 
-from src.fit_survival_function import fit_per_length_survival_function
-from glob import glob
-import spacy
-import re
-
-
-logging.basicConfig(level=logging.INFO)
-
-
-def read_all_csv_files(pattern):
-    df = pd.DataFrame()
-    print(pattern)
-    for f in glob(pattern):
-        df = pd.concat([df, pd.read_csv(f)])
-    return df
-
-
-def get_survival_function(df, G=101):
-    """
-    Returns a survival function for every sentence length in tokens.
-
-    Args:
-    :df:  data frame with columns 'response' and 'length'
-    :G:   number of interpolation points
-    
-    Return:
-        bivariate function (length, responce) -> (0,1)
-
-    """
-    assert not df.empty
-    value_name = "response" if "response" in df.columns else "logloss"
-
-    df1 = df[~df[value_name].isna()]
-    ll = df1['length']
-    xx1 = df1[value_name]
-    return fit_per_length_survival_function(ll, xx1, log_space=True, G=G)
-
-
-def mark_edits_remove_tags(chunks, tag="edit"):
-    text_chunks = chunks['text']
-    edits = []
-    for i,text in enumerate(text_chunks):
-        chunk_text = re.findall(rf"<{tag}>(.+)</{tag}>", text)
-        if len(chunk_text) > 0:
-            import pdb; pdb.set_trace()
-            chunks['text'][i] = chunk_text[0]
-            chunks['length'][i] -= 2
-            edits.append(True)
-        else:
-            edits.append(False)
-
-    return chunks, edits
-
-def get_null_data(model_name, topic):
-    data = None
-    try:
-        file = open(f'nullData/{model_name}_{topic}.pkl', 'rb')
-        data = pickle.load(file)
-    except:
-        pass
-
-    return data
-
-def get_threshold_obj(model_name, topic):
-    threshold = None
-    try:
-        file = open('threshold_obj.pkl', 'rb')
-        threshold_obj = pickle.load(file)
-        threshold = threshold_obj[model_name][topic]
-    except:
-        pass
-
-    return threshold
-
-def detect_human_text(model_name, topic, text):
-    
-    # Get null data
-    print('Get null data')
-    df_null = get_null_data(model_name, topic)
-    if 'num' in df_null.columns:
-        df_null = df_null[df_null.num > 1]
-    
-    # Get survival function
-        print('Get survival function')
-    pval_functions = get_survival_function(df_null, G=43)
-
-    min_tokens_per_sentence = 10
-    max_tokens_per_sentence = 100
-
-    # Init model
-    print('Init model')
-    lm_name = 'gpt2-xl' if model_name == 'GPT2XL' else 'microsoft/phi-2'
-    tokenizer = AutoTokenizer.from_pretrained(lm_name)
-    model = AutoModelForCausalLM.from_pretrained(lm_name)
-    
-    print('Init PerplexityEvaluator')
-    sentence_detector = PerplexityEvaluator(model, tokenizer)
-
-    if torch.backends.mps.is_available():
-        device = 'mps'
-    elif torch.cuda.is_available():
-        device = 'cuda'
-    else:
-        device = 'cpu'
-
-    print(f'device {device}')
-    model.to(device)
-
-    print('Init DetectLM')
-    detector = DetectLM(sentence_detector, pval_functions,
-                        min_len=min_tokens_per_sentence,
-                        max_len=max_tokens_per_sentence,
-                        length_limit_policy='truncate',
-                        HC_type='stbl',
-                        ignore_first_sentence= False
-                        )
-
-    # Convert text to object
-    print('Analyze text')
-    article_obj = get_article_obj(text)
-    parser = PrepareArticles(article_obj, min_tokens=min_tokens_per_sentence, max_tokens=max_tokens_per_sentence)
-    chunks = parser(combined=False)
-
-    # Go over all the document
-    for i in range(len(chunks['text'])):
-        print(chunks['text'][i])
-        # for p,v in enumerate(chunks['text'][i]):
-        #     print(f'{p}: {v}')
-        res = detector(chunks['text'][i], chunks['context'][i], dashboard=None)
-
-        # print(f"Num of Edits (rate) = {np.sum(df['tag'] == '<edit>')} ({edit_rate})")
-        # print(f"HC = {res['HC']}")
-        # print(f"Fisher = {res['fisher']}")
-        # print(f"Fisher (chisquared pvalue) = {res['fisher_pvalue']}")
-
-        results = res['HC']
-    
-    threshold = get_threshold_obj(model_name, topic)
-    print(f"threshold: {threshold}, results: {results}")
-    return '1' if results >= threshold else '0'
-
-# Convert article text into object
-def get_article_obj(text):
-    # Init article object
-    article_obj = {
-        'sub_titles': [{
-            'sentences': []
-        }]
-    }
-    
-    nlp = spacy.load("en_core_web_sm")  # Load model
-
-    for line in text.split('\n'):
-        doc = nlp(line) # Analyze text
-        sentences = [sent.text for sent in doc.sents if len(sent) >= 10] # Split it by sentence
-        for sentence in sentences:
-            sentence = re.sub(r' +', ' ', sentence) # Remove duplicate spaces
-            article_obj['sub_titles'][0]['sentences'].append({'sentence': sentence})
-
-    return article_obj
+import torch
+import pandas as pd
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import logging
+import numpy as np
+import pickle
+from src.DetectLM import DetectLM
+from src.PerplexityEvaluator import PerplexityEvaluator
+from src.PrepareArticles import PrepareArticles #Idan 
+from src.fit_survival_function import fit_per_length_survival_function
+from glob import glob
+import spacy
+import re
+
+
+logging.basicConfig(level=logging.INFO)
+
+
+def read_all_csv_files(pattern):
+    df = pd.DataFrame()
+    print(pattern)
+    for f in glob(pattern):
+        df = pd.concat([df, pd.read_csv(f)])
+    return df
+
+
+def get_survival_function(df, G=101):
+    """
+    Returns a survival function for every sentence length in tokens.
+
+    Args:
+    :df:  data frame with columns 'response' and 'length'
+    :G:   number of interpolation points
+    
+    Return:
+        bivariate function (length, responce) -> (0,1)
+
+    """
+    assert not df.empty
+    value_name = "response" if "response" in df.columns else "logloss"
+
+    df1 = df[~df[value_name].isna()]
+    ll = df1['length']
+    xx1 = df1[value_name]
+    return fit_per_length_survival_function(ll, xx1, log_space=True, G=G)
+
+
+def mark_edits_remove_tags(chunks, tag="edit"):
+    text_chunks = chunks['text']
+    edits = []
+    for i,text in enumerate(text_chunks):
+        chunk_text = re.findall(rf"<{tag}>(.+)</{tag}>", text)
+        if len(chunk_text) > 0:
+            import pdb; pdb.set_trace()
+            chunks['text'][i] = chunk_text[0]
+            chunks['length'][i] -= 2
+            edits.append(True)
+        else:
+            edits.append(False)
+
+    return chunks, edits
+
+def get_null_data(model_name, topic):
+    data = None
+    try:
+        file = open(f'nullData/{model_name}_{topic}.pkl', 'rb')
+        data = pickle.load(file)
+    except:
+        pass
+
+    return data
+
+def get_threshold_obj(model_name, topic):
+    threshold = None
+    try:
+        file = open('threshold_obj.pkl', 'rb')
+        threshold_obj = pickle.load(file)
+        threshold = threshold_obj[model_name][topic]
+    except:
+        pass
+
+    return threshold
+
+def detect_human_text(model_name, topic, text):
+    
+    # Get null data
+    print('Get null data')
+    df_null = get_null_data(model_name, topic)
+    if 'num' in df_null.columns:
+        df_null = df_null[df_null.num > 1]
+    
+    # Get survival function
+        print('Get survival function')
+    pval_functions = get_survival_function(df_null, G=43)
+
+    min_tokens_per_sentence = 10
+    max_tokens_per_sentence = 100
+
+    # Init model
+    print('Init model')
+    lm_name = 'gpt2-xl' if model_name == 'GPT2XL' else 'microsoft/phi-2'
+    tokenizer = AutoTokenizer.from_pretrained(lm_name)
+    model = AutoModelForCausalLM.from_pretrained(lm_name)
+    
+    print('Init PerplexityEvaluator')
+    sentence_detector = PerplexityEvaluator(model, tokenizer)
+
+    if torch.backends.mps.is_available():
+        device = 'mps'
+    elif torch.cuda.is_available():
+        device = 'cuda'
+    else:
+        device = 'cpu'
+
+    print(f'device {device}')
+    model.to(device)
+
+    print('Init DetectLM')
+    detector = DetectLM(sentence_detector, pval_functions,
+                        min_len=min_tokens_per_sentence,
+                        max_len=max_tokens_per_sentence,
+                        length_limit_policy='truncate',
+                        HC_type='stbl',
+                        ignore_first_sentence= False
+                        )
+
+    # Convert text to object
+    print('Analyze text')
+    article_obj = get_article_obj(text)
+    parser = PrepareArticles(article_obj, min_tokens=min_tokens_per_sentence, max_tokens=max_tokens_per_sentence)
+    chunks = parser(combined=False)
+
+    # Go over all the document
+    for i in range(len(chunks['text'])):
+        print(chunks['text'][i])
+        # for p,v in enumerate(chunks['text'][i]):
+        #     print(f'{p}: {v}')
+        res = detector(chunks['text'][i], chunks['context'][i], dashboard=None)
+
+        # print(f"Num of Edits (rate) = {np.sum(df['tag'] == '<edit>')} ({edit_rate})")
+        # print(f"HC = {res['HC']}")
+        # print(f"Fisher = {res['fisher']}")
+        # print(f"Fisher (chisquared pvalue) = {res['fisher_pvalue']}")
+
+        results = res['HC']
+    
+    threshold = get_threshold_obj(model_name, topic)
+    print(f"threshold: {threshold}, results: {results}")
+    return (results / threshold) - 1, res['sentences']
+
+# Convert article text into object
+def get_article_obj(text):
+    # Init article object
+    article_obj = {
+        'sub_titles': [{
+            'sentences': []
+        }]
+    }
+    
+    nlp = spacy.load("en_core_web_sm")  # Load model
+
+    for line in text.split('\n'):
+        doc = nlp(line) # Analyze text
+        sentences = [sent.text for sent in doc.sents if len(sent) >= 10] # Split it by sentence
+        for sentence in sentences:
+            sentence = re.sub(r' +', ' ', sentence) # Remove duplicate spaces
+            article_obj['sub_titles'][0]['sentences'].append({'sentence': sentence})
+
+    return article_obj