Update app.py

app.py

@@ -6,6 +6,26 @@ import PyPDF2
 import docx
 import io
 
+def chunk_text(text, chunk_size=128):
+    words = text.split()
+    chunks = []
+    current_chunk = []
+    current_length = 0
+
+    for word in words:
+        if current_length + len(word) + 1 > chunk_size:
+            chunks.append(' '.join(current_chunk))
+            current_chunk = [word]
+            current_length = len(word)
+        else:
+            current_chunk.append(word)
+            current_length += len(word) + 1
+
+    if current_chunk:
+        chunks.append(' '.join(current_chunk))
+
+    return chunks
+
 st.set_page_config(layout="wide")
 
 # Function to read text from uploaded file
@@ -22,12 +42,6 @@ def read_file(file):
         st.error("Unsupported file type")
         return None
 
-# Function to generate text chunks
-def chunk_text(text, max_length=128):
-    words = text.split()
-    for i in range(0, len(words), max_length):
-        yield " ".join(words[i:i + max_length])
-
 st.title("Turkish NER Models Testing")
 
 model_list = [
@@ -44,9 +58,7 @@ st.sidebar.write("For details of models: 'https://huggingface.co/akdeniz27/")
 st.sidebar.write("Only PDF, DOCX, and TXT files are supported.")
 
 # Determine aggregation strategy
-aggregation = "simple" if model_checkpoint in ["akdeniz27/xlm-roberta-base-turkish-ner", 
-                                                "xlm-roberta-large-finetuned-conll03-english", 
-                                                "asahi417/tner-xlm-roberta-base-ontonotes5"] else "first"
+aggregation = "simple" if model_checkpoint in ["akdeniz27/xlm-roberta-base-turkish-ner", "xlm-roberta-large-finetuned-conll03-english", "asahi417/tner-xlm-roberta-base-ontonotes5"] else "first"
 
 st.subheader("Select Text Input Method")
 input_method = st.radio("", ('Write or Paste New Text', 'Upload File'))
@@ -86,23 +98,37 @@ Run_Button = st.button("Run")
 if Run_Button and input_text:
     ner_pipeline = setModel(model_checkpoint, aggregation)
     
-    # Process the text in chunks
-    output_comb = []
-    for chunk in chunk_text(input_text):
+    # Chunk the input text
+    chunks = chunk_text(input_text)
+    
+    # Process each chunk
+    all_outputs = []
+    for i, chunk in enumerate(chunks):
         output = ner_pipeline(chunk)
-        output_comb.extend(entity_comb(output))
-
+        
+        # Adjust start and end positions for entities in chunks after the first
+        if i > 0:
+            offset = len(' '.join(chunks[:i])) + 1
+            for entity in output:
+                entity['start'] += offset
+                entity['end'] += offset
+        
+        all_outputs.extend(output)
+    
+    # Combine entities
+    output_comb = entity_comb(all_outputs)
+    
     df = pd.DataFrame.from_dict(output_comb)
     cols_to_keep = ['word', 'entity_group', 'score', 'start', 'end']
     df_final = df[cols_to_keep]
     
     st.subheader("Recognized Entities")
     st.dataframe(df_final)
-
+    
     # Spacy display logic
     spacy_display = {"ents": [], "text": input_text, "title": None}
     for entity in output_comb:
         spacy_display["ents"].append({"start": entity["start"], "end": entity["end"], "label": entity["entity_group"]})
-
+    
     html = spacy.displacy.render(spacy_display, style="ent", minify=True, manual=True)
     st.write(html, unsafe_allow_html=True)