Update functions.py

functions.py

@@ -136,6 +136,32 @@ def process_corpus(corpus, _tokenizer, title, embedding_model, chunk_size=200, o
 
     return docsearch
 
+@st.experimental_singleton(suppress_st_warning=True)
+def chunk_and_preprocess_text(text,thresh=500):
+    
+    """Chunk text longer than n tokens for summarization"""
+    
+    sentences = sent_tokenize(clean_text(text))
+    #sentences = [i.text for i in list(article.sents)]
+    
+    current_chunk = 0
+    chunks = []
+    
+    for sentence in sentences:
+        if len(chunks) == current_chunk + 1:
+            if len(chunks[current_chunk]) + len(sentence.split(" ")) <= thresh:
+                chunks[current_chunk].extend(sentence.split(" "))
+            else:
+                current_chunk += 1
+                chunks.append(sentence.split(" "))
+        else:
+            chunks.append(sentence.split(" "))
+
+    for chunk_id in range(len(chunks)):
+        chunks[chunk_id] = " ".join(chunks[chunk_id])
+    
+    return chunks
+    
 @st.experimental_singleton(suppress_st_warning=True)
 def gen_embeddings(embedding_model):