Upload tokenizer
Browse files- tokenizer.json +2 -2
- tokenizer_config.json +17 -1
tokenizer.json
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
-
size
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:83dc0fce7d93d49a7363b834520a302d7f1144bd728792e5a39ca6f6f9bf3233
|
3 |
+
size 34363253
|
tokenizer_config.json
CHANGED
@@ -1993,6 +1993,22 @@
|
|
1993 |
"rstrip": false,
|
1994 |
"single_word": false,
|
1995 |
"special": false
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1996 |
}
|
1997 |
},
|
1998 |
"additional_special_tokens": [
|
@@ -2007,7 +2023,7 @@
|
|
2007 |
"pad_token": "<pad>",
|
2008 |
"sp_model_kwargs": {},
|
2009 |
"spaces_between_special_tokens": false,
|
2010 |
-
"tokenizer_class": "
|
2011 |
"unk_token": "<unk>",
|
2012 |
"use_default_system_prompt": false
|
2013 |
}
|
|
|
1993 |
"rstrip": false,
|
1994 |
"single_word": false,
|
1995 |
"special": false
|
1996 |
+
},
|
1997 |
+
"256000": {
|
1998 |
+
"content": "<start_of_sum>",
|
1999 |
+
"lstrip": false,
|
2000 |
+
"normalized": true,
|
2001 |
+
"rstrip": false,
|
2002 |
+
"single_word": false,
|
2003 |
+
"special": false
|
2004 |
+
},
|
2005 |
+
"256001": {
|
2006 |
+
"content": "<end_of_sum>",
|
2007 |
+
"lstrip": false,
|
2008 |
+
"normalized": true,
|
2009 |
+
"rstrip": false,
|
2010 |
+
"single_word": false,
|
2011 |
+
"special": false
|
2012 |
}
|
2013 |
},
|
2014 |
"additional_special_tokens": [
|
|
|
2023 |
"pad_token": "<pad>",
|
2024 |
"sp_model_kwargs": {},
|
2025 |
"spaces_between_special_tokens": false,
|
2026 |
+
"tokenizer_class": "GemmaTokenizer",
|
2027 |
"unk_token": "<unk>",
|
2028 |
"use_default_system_prompt": false
|
2029 |
}
|