Update tokenization_dart.py
Browse files- tokenization_dart.py +0 -9
tokenization_dart.py
CHANGED
|
@@ -37,20 +37,11 @@ PROMPT_TEMPLATE = (
|
|
| 37 |
"{{ '</character>' }}"
|
| 38 |
|
| 39 |
"{{ '<general>' }}"
|
| 40 |
-
# length token
|
| 41 |
-
"{% if 'length' not in messages or messages['length'] is none %}"
|
| 42 |
-
"{{ '<|long|>' }}"
|
| 43 |
-
"{% else %}"
|
| 44 |
-
"{{ messages['length'] }}"
|
| 45 |
-
"{% endif %}"
|
| 46 |
-
|
| 47 |
-
# general token
|
| 48 |
"{% if 'general' not in messages or messages['general'] is none %}"
|
| 49 |
"{{ '' }}"
|
| 50 |
"{% else %}"
|
| 51 |
"{{ messages['general'] }}"
|
| 52 |
"{% endif %}"
|
| 53 |
-
"{{ '<|input_end|>' }}"
|
| 54 |
).strip()
|
| 55 |
# fmt: on
|
| 56 |
|
|
|
|
| 37 |
"{{ '</character>' }}"
|
| 38 |
|
| 39 |
"{{ '<general>' }}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
"{% if 'general' not in messages or messages['general'] is none %}"
|
| 41 |
"{{ '' }}"
|
| 42 |
"{% else %}"
|
| 43 |
"{{ messages['general'] }}"
|
| 44 |
"{% endif %}"
|
|
|
|
| 45 |
).strip()
|
| 46 |
# fmt: on
|
| 47 |
|