Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -153,7 +153,7 @@ def save_bibtex(bibtex_content):
|
|
| 153 |
class CombinedProcessor:
|
| 154 |
def process(self, user_message):
|
| 155 |
editorial_text = re.sub("\n", " ¶ ", user_message)
|
| 156 |
-
editorial_text = re.sub(r'\s*([
|
| 157 |
num_tokens = len(tokenizer.tokenize(editorial_text))
|
| 158 |
|
| 159 |
batch_prompts = split_text(editorial_text, max_tokens=500) if num_tokens > 500 else [editorial_text]
|
|
|
|
| 153 |
class CombinedProcessor:
|
| 154 |
def process(self, user_message):
|
| 155 |
editorial_text = re.sub("\n", " ¶ ", user_message)
|
| 156 |
+
editorial_text = re.sub(r'\s*([;:,\.])\s*', r' \1 ', editorial_text)
|
| 157 |
num_tokens = len(tokenizer.tokenize(editorial_text))
|
| 158 |
|
| 159 |
batch_prompts = split_text(editorial_text, max_tokens=500) if num_tokens > 500 else [editorial_text]
|