itqop commited on
Commit
32ff187
·
verified ·
1 Parent(s): 3199fb8

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +2 -2
  2. tokenizer_config.json +17 -1
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5f7eee611703c5ce5d1eee32d9cdcfe465647b8aff0c1dfb3bed7ad7dbb05060
3
- size 34362873
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:83dc0fce7d93d49a7363b834520a302d7f1144bd728792e5a39ca6f6f9bf3233
3
+ size 34363253
tokenizer_config.json CHANGED
@@ -1993,6 +1993,22 @@
1993
  "rstrip": false,
1994
  "single_word": false,
1995
  "special": false
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1996
  }
1997
  },
1998
  "additional_special_tokens": [
@@ -2007,7 +2023,7 @@
2007
  "pad_token": "<pad>",
2008
  "sp_model_kwargs": {},
2009
  "spaces_between_special_tokens": false,
2010
- "tokenizer_class": "GemmaTokenizerFast",
2011
  "unk_token": "<unk>",
2012
  "use_default_system_prompt": false
2013
  }
 
1993
  "rstrip": false,
1994
  "single_word": false,
1995
  "special": false
1996
+ },
1997
+ "256000": {
1998
+ "content": "<start_of_sum>",
1999
+ "lstrip": false,
2000
+ "normalized": true,
2001
+ "rstrip": false,
2002
+ "single_word": false,
2003
+ "special": false
2004
+ },
2005
+ "256001": {
2006
+ "content": "<end_of_sum>",
2007
+ "lstrip": false,
2008
+ "normalized": true,
2009
+ "rstrip": false,
2010
+ "single_word": false,
2011
+ "special": false
2012
  }
2013
  },
2014
  "additional_special_tokens": [
 
2023
  "pad_token": "<pad>",
2024
  "sp_model_kwargs": {},
2025
  "spaces_between_special_tokens": false,
2026
+ "tokenizer_class": "GemmaTokenizer",
2027
  "unk_token": "<unk>",
2028
  "use_default_system_prompt": false
2029
  }