Update tokenizer_config.json
#14
by
awni
- opened
- tokenizer_config.json +1 -1
tokenizer_config.json
CHANGED
|
@@ -490,6 +490,6 @@
|
|
| 490 |
"clean_up_tokenization_spaces": false,
|
| 491 |
"eos_token": "[e~[",
|
| 492 |
"model_max_length": 40960000,
|
| 493 |
-
"tokenizer_class": "
|
| 494 |
"unk_token": "]!d~["
|
| 495 |
}
|
|
|
|
| 490 |
"clean_up_tokenization_spaces": false,
|
| 491 |
"eos_token": "[e~[",
|
| 492 |
"model_max_length": 40960000,
|
| 493 |
+
"tokenizer_class": "TokenizersBackend",
|
| 494 |
"unk_token": "]!d~["
|
| 495 |
}
|