Upload tokenizer
#6
by
						
ArthurZ
	
							HF Staff
						- opened
							
					
- tokenizer_config.json +1 -0
 
    	
        tokenizer_config.json
    CHANGED
    
    | 
         @@ -6173,6 +6173,7 @@ 
     | 
|
| 6173 | 
         
             
                }
         
     | 
| 6174 | 
         
             
              },
         
     | 
| 6175 | 
         
             
              "bos_token": "<s>",
         
     | 
| 
         | 
|
| 6176 | 
         
             
              "clean_up_tokenization_spaces": false,
         
     | 
| 6177 | 
         
             
              "eos_token": "</s>",
         
     | 
| 6178 | 
         
             
              "legacy": false,
         
     | 
| 
         | 
|
| 6173 | 
         
             
                }
         
     | 
| 6174 | 
         
             
              },
         
     | 
| 6175 | 
         
             
              "bos_token": "<s>",
         
     | 
| 6176 | 
         
            +
              "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
         
     | 
| 6177 | 
         
             
              "clean_up_tokenization_spaces": false,
         
     | 
| 6178 | 
         
             
              "eos_token": "</s>",
         
     | 
| 6179 | 
         
             
              "legacy": false,
         
     |