Update app.py

app.py

@@ -22,6 +22,12 @@ def read_file(file):
         st.error("Unsupported file type")
         return None
 
+# Function to generate text chunks
+def chunk_text(text, max_length=128):
+    words = text.split()
+    for i in range(0, len(words), max_length):
+        yield " ".join(words[i:i + max_length])
+
 st.title("Turkish NER Models Testing")
 
 model_list = [
@@ -45,8 +51,6 @@ aggregation = "simple" if model_checkpoint in ["akdeniz27/xlm-roberta-base-turki
 st.subheader("Select Text Input Method")
 input_method = st.radio("", ('Write or Paste New Text', 'Upload File'))
 
-
-
 if input_method == "Write or Paste New Text":
     input_text = st.text_area('Write or Paste Text Below', value="", height=128)
 else:
@@ -81,10 +85,13 @@ Run_Button = st.button("Run")
 
 if Run_Button and input_text:
     ner_pipeline = setModel(model_checkpoint, aggregation)
-    output = ner_pipeline(input_text)
-    
-    output_comb = entity_comb(output)
     
+    # Process the text in chunks
+    output_comb = []
+    for chunk in chunk_text(input_text):
+        output = ner_pipeline(chunk)
+        output_comb.extend(entity_comb(output))
+
     df = pd.DataFrame.from_dict(output_comb)
     cols_to_keep = ['word', 'entity_group', 'score', 'start', 'end']
     df_final = df[cols_to_keep]
@@ -96,6 +103,6 @@ if Run_Button and input_text:
     spacy_display = {"ents": [], "text": input_text, "title": None}
     for entity in output_comb:
         spacy_display["ents"].append({"start": entity["start"], "end": entity["end"], "label": entity["entity_group"]})
-    
+
     html = spacy.displacy.render(spacy_display, style="ent", minify=True, manual=True)
-    st.write(html, unsafe_allow_html=True)
+    st.write(html, unsafe_allow_html=True)