Update tokenizer

tokenizer.json

@@ -21,15 +21,6 @@
       "rstrip": false,
       "normalized": false
     },
-    {
-      "id": 2,
-      "special": true,
-      "content": "[SPACE]",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": true
-    },
     {
       "id": 255,
       "special": true,
@@ -443,69 +434,199 @@
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 6561,
+      "content": "[START_SPEECH]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 6562,
+      "content": "[STOP_SPEECH]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 6563,
+      "content": "[EXAGGERATION]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": {
-    "type": "Sequence",
-    "normalizers": [
-      {
-        "type": "Replace",
-        "pattern": { "String": " " },
-        "content": "[SPACE]"
-      }
-    ]
-  },
-  "pre_tokenizer": {
-    "type": "Whitespace"
+    "type": "Replace",
+    "pattern": {
+      "Regex": "\\s+"
+    },
+    "content": " "
   },
+  "pre_tokenizer": null,
   "post_processor": {
     "type": "TemplateProcessing",
     "single": [
-      { "SpecialToken": { "id": "EXAGGERATION", "type_id": 0 } },
-      { "SpecialToken": { "id": "BOS", "type_id": 0 } },
-      { "Sequence": { "id": "A", "type_id": 0 } },
-      { "SpecialToken": { "id": "EOS", "type_id": 0 } },
-      { "SpecialToken": { "id": "START_SPEECH", "type_id": 0 } },
-      { "SpecialToken": { "id": "START_SPEECH", "type_id": 0 } }
+      {
+        "SpecialToken": {
+          "id": "[EXAGGERATION]",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[START]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[STOP]",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[START_SPEECH]",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[START_SPEECH]",
+          "type_id": 0
+        }
+      }
     ],
     "pair": [
-      { "SpecialToken": { "id": "EXAGGERATION", "type_id": 0 } },
-      { "SpecialToken": { "id": "BOS", "type_id": 0 } },
-      { "Sequence": { "id": "A", "type_id": 0 } },
-      { "SpecialToken": { "id": "EOS", "type_id": 0 } },
-      { "SpecialToken": { "id": "START_SPEECH", "type_id": 0 } },
-      { "SpecialToken": { "id": "START_SPEECH", "type_id": 0 } },
-      { "SpecialToken": { "id": "EXAGGERATION", "type_id": 1 } },
-      { "SpecialToken": { "id": "BOS", "type_id": 1 } },
-      { "Sequence": { "id": "B", "type_id": 1 } },
-      { "SpecialToken": { "id": "EOS", "type_id": 1 } },
-      { "SpecialToken": { "id": "START_SPEECH", "type_id": 1 } },
-      { "SpecialToken": { "id": "START_SPEECH", "type_id": 1 } }
+      {
+        "SpecialToken": {
+          "id": "[EXAGGERATION]",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[START]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[STOP]",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[START_SPEECH]",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[START_SPEECH]",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[EXAGGERATION]",
+          "type_id": 1
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[START]",
+          "type_id": 1
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[STOP]",
+          "type_id": 1
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[START_SPEECH]",
+          "type_id": 1
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[START_SPEECH]",
+          "type_id": 1
+        }
+      }
     ],
     "special_tokens": {
-      "BOS": {
-        "id": "BOS",
-        "ids": [255],
-        "tokens": ["<s>"]
+      "[START]": {
+        "id": "[START]",
+        "ids": [
+          255
+        ],
+        "tokens": [
+          "[START]"
+        ]
       },
-      "EOS": {
-        "id": "EOS",
-        "ids": [0],
-        "tokens": ["</s>"]
+      "[STOP]": {
+        "id": "[STOP]",
+        "ids": [
+          0
+        ],
+        "tokens": [
+          "[STOP]"
+        ]
       },
-      "EXAGGERATION": {
-        "id": "EXAGGERATION",
-        "ids": [6563],
-        "tokens": ["<EXAGGERATION>"]
+      "[EXAGGERATION]": {
+        "id": "[EXAGGERATION]",
+        "ids": [
+          6563
+        ],
+        "tokens": [
+          "[EXAGGERATION]"
+        ]
       },
-      "START_SPEECH": {
-        "id": "START_SPEECH",
-        "ids": [6561],
-        "tokens": ["<START_SPEECH>"]
+      "[START_SPEECH]": {
+        "id": "[START_SPEECH]",
+        "ids": [
+          6561
+        ],
+        "tokens": [
+          "[START_SPEECH]"
+        ]
       }
     }
   },
-  "decoder": null,
+  "decoder": {
+    "type": "Fuse"
+  },
   "model": {
     "type": "BPE",
     "dropout": null,
@@ -516,7 +637,7 @@
     "vocab": {
       "[STOP]": 0,
       "[UNK]": 1,
-      "[SPACE]": 2,
+      " ": 2,
       "!": 3,
       "'": 4,
       "(": 5,