use sentence splitters from stopes

Signed-off-by: David Dale <daviddale@meta.com>

app.py

@@ -1,6 +1,7 @@
 import spaces
 import gradio as gr
 from sacremoses import MosesPunctNormalizer
+from stopes.pipelines.monolingual.utils.sentence_split import get_split_algo
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
 from flores import code_mapping
 import platform
@@ -35,6 +36,14 @@ tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
 
 punct_normalizer = MosesPunctNormalizer(lang="en")
 
+
+@lru_cache(maxsize=202)
+def get_language_specific_sentence_splitter(language_code):
+    short_code = language_code[:3]
+    splitter = get_split_algo(short_code, "default")
+    return splitter
+
+
 # cache function
 @lru_cache(maxsize=100)
 def translate(text: str, src_lang: str, tgt_lang: str):
@@ -60,7 +69,8 @@ def _translate(text: str, src_lang: str, tgt_lang: str):
     translated_paragraphs = []
 
     for paragraph in paragraphs:
-        sentences = nltk.sent_tokenize(paragraph)
+        splitter = get_language_specific_sentence_splitter(src_code)
+        sentences = list(splitter(paragraph))
         translated_sentences = []
 
         for sentence in sentences:

requirements.txt

@@ -5,3 +5,4 @@ gradio==4.32.2
 spaces
 nltk
 sacremoses
+stopes[mono] @ git+https://github.com/facebookresearch/stopes@better-sentence-splitters