amusktweewt commited on
Commit
db412cb
·
1 Parent(s): e3a31b9

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -1
tokenizer_config.json CHANGED
@@ -82,7 +82,7 @@
82
  }
83
  },
84
  "bos_token": "<sos>",
85
- "chat_template": "\n{%- for message in messages %}\n {%- if message['role'] == 'user' %}\n {{- bos_token + '<|instruction|>' + ' ' + message['content'].strip() }}\n {%- elif message['role'] == 'assistant' %}\n {{- '<|response|>' + ' ' + message['content'].strip() + eos_token }}\n {%- endif %}\n{%- endfor %}\n",
86
  "clean_up_tokenization_spaces": true,
87
  "eos_token": "<|endoftext|>",
88
  "extra_special_tokens": {},
 
82
  }
83
  },
84
  "bos_token": "<sos>",
85
+ "chat_template": "\n{%- for message in messages %}\n {%- if message['role'] == 'user' %}\n {{- '<|instruction|>' + ' ' + message['content'].strip() }}\n {%- elif message['role'] == 'assistant' %}\n {{- '<|response|>' + ' ' + message['content'].strip() }}\n {%- endif %}\n{%- endfor %}\n",
86
  "clean_up_tokenization_spaces": true,
87
  "eos_token": "<|endoftext|>",
88
  "extra_special_tokens": {},