Upload tokenizer_input.json with huggingface_hub

tokenizer_input.json

@@ -0,0 +1 @@
+{"class_name": "Tokenizer", "config": {"num_words": null, "filters": "!\"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n", "lower": true, "split": " ", "char_level": true, "oov_token": "<unk>", "document_count": 438257, "word_counts": "{\"e\": 285715, \"n\": 216044, \"g\": 78592, \"d\": 166158, \"a\": 491993, \"r\": 186642, \"i\": 314828, \"j\": 32128, \"_\": 79, \"t\": 274409, \"'\": 7980, \"s\": 160264, \" \": 827685, \"o\": 213216, \"b\": 94849, \"v\": 15792, \"u\": 85628, \"h\": 238975, \"y\": 111452, \"k\": 131964, \"p\": 31049, \"c\": 93664, \"l\": 219258, \".\": 6834, \"\\u0628\": 70499, \"\\u0627\": 316267, \"\\u064a\": 203333, \"\\u0646\": 113768, \"\\u0647\": 56156, \"\\u0648\": 115397, \"\\u0645\": 95839, \"\\u062a\": 117606, \"\\u062d\": 28531, \"\\u0644\": 137966, \"\\u0625\": 8111, \"\\u0642\": 33636, \"\\u0631\": 89840, \"\\u0651\": 32470, \"\\u062f\": 88979, \"m\": 142479, \"7\": 28493, \"w\": 97847, \"9\": 37013, \"f\": 70372, \"\\u0637\": 9930, \"\\u0634\": 54234, \"\\u0641\": 43438, \"\\u0632\": 15756, \"z\": 20312, \"!\": 1576, \"\\u063a\": 29828, \"\\u062c\": 26256, \"\\u0639\": 58774, \"\\u062e\": 23238, \"\\u0629\": 16980, \"3\": 63470, \"\\u0643\": 86098, \"\\u0621\": 1633, \"2\": 6031, \"\\u0623\": 15433, \"\\u0633\": 48368, \"\\u0636\": 5709, \"x\": 7434, \"\\u06ad\": 5752, \"\\u0635\": 9188, \"?\": 10397, \"8\": 5792, \"\\u0649\": 2874, \"5\": 6529, \"0\": 832, \"-\": 313, \"\\u0626\": 539, \"\\u062b\": 414, \"\\u0624\": 270, \"q\": 2098, \"\\u067e\": 1215, \"1\": 490, \"\\u0638\": 248, \"\\u0622\": 466, \"\\u0630\": 263, \"/\": 30, \"4\": 1678, \":\": 43, \"6\": 116, \"\\u06a4\": 433, \";\": 30, \"\\u06af\": 72, \"\\u061f\": 2821, \"\\u0662\": 2, \"\\u064b\": 17, \"\\u064e\": 11, \"\\u0640\": 2, \"\\u00e9\": 359, \"\\u064f\": 2064, \"*\": 7, \"+\": 4, \"\\u0121\": 1, \"\\u00bd\": 1, \"\\u201f\": 1, \"(\": 4, \")\": 4, \"\\\\\": 2, \"\\u00fc\": 1, \"%\": 6, \"\\\"\": 47033, \"\\u0448\": 2, \"\\u045a\": 1, \"\\u0433\": 171, \"\\u00a9\": 149, \"\\u0451\": 5, \"\\u045e\": 3, \"\\u043f\": 2, \"\\u0457\": 2, \"\\u0455\": 1, \"\\u00a7\": 3, \"\\u2020\": 2, \"\\u0449\": 25, \"\\u0432\": 1, \"\\u045c\": 1, \"\\u00a4\": 1, \"\\t\": 14, \"\\u0435\": 4, \"\\u0441\": 2, \"\\u043b\": 9, \"\\u0444\": 1, \"\\u00a0\": 3, \"\\u00d7\": 1, \"\\u0439\": 4, \"\\u043a\": 2, \"\\ufefb\": 3, \"$\": 3, \"\\ufee1\": 1, \"\\u2116\": 1, \"\\u201c\": 1, \"\\u00ae\": 2, \"\\u0669\": 1, \"\\u00f4\": 22, \"\\u064c\": 1, \"\\u00e7\": 14, \"\\u00fa\": 3, \"\\u00e2\": 17, \"\\u2019\": 84, \"\\u00e0\": 24, \"\\u00e8\": 23, \"\\u0153\": 2, \"\\ufffd\": 6, \"\\u00fb\": 7, \"\\u0137\": 4, \"\\u00ab\": 2, \"#\": 2, \"\\u20ac\": 6, \"\\u00ed\": 6, \"\\u00f1\": 2, \"\\u060c\": 4, \"\\u00bb\": 2, \"\\u017a\": 2, \"\\u0763\": 1, \"\\u0686\": 3, \"\\u0652\": 1, \"\\u06a2\": 2, \"\\u0650\": 1}", "word_docs": "{\"n\": 143414, \"e\": 150579, \"g\": 69908, \"j\": 28357, \"a\": 211095, \"r\": 135127, \"_\": 79, \"d\": 118883, \"i\": 175377, \"v\": 14817, \"l\": 130859, \"c\": 75975, \" \": 262506, \"k\": 96118, \"'\": 7685, \"b\": 76458, \".\": 6226, \"p\": 26298, \"t\": 169324, \"u\": 67797, \"o\": 135358, \"h\": 148483, \"y\": 91205, \"s\": 113248, \"\\u0625\": 7844, \"\\u0642\": 30042, \"\\u0648\": 78803, \"\\u064a\": 100230, \"\\u062d\": 25554, \"\\u0644\": 79945, \"\\u0651\": 29775, \"\\u0627\": 121755, \"\\u062a\": 82307, \"\\u062f\": 62860, \"\\u0647\": 43234, \"\\u0646\": 74885, \"\\u0645\": 68472, \"\\u0628\": 55142, \"\\u0631\": 66744, \"m\": 105061, \"9\": 32407, \"w\": 78842, \"7\": 25427, \"f\": 59633, \"\\u0632\": 14587, \"\\u0634\": 43479, \"\\u0637\": 9549, \"\\u0641\": 37528, \"z\": 17230, \"!\": 1544, \"\\u062e\": 21629, \"\\u062c\": 23863, \"\\u063a\": 28439, \"\\u0629\": 14279, \"\\u0639\": 49063, \"3\": 53725, \"\\u0621\": 1630, \"\\u0643\": 62427, \"2\": 5819, \"\\u0623\": 14312, \"\\u0633\": 41361, \"\\u0636\": 5647, \"x\": 6864, \"\\u06ad\": 5590, \"\\u0635\": 8723, \"?\": 10281, \"8\": 5432, \"\\u0649\": 2804, \"5\": 5646, \"0\": 551, \"-\": 296, \"\\u0626\": 538, \"\\u062b\": 384, \"\\u0624\": 270, \"q\": 1914, \"\\u067e\": 1173, \"1\": 455, \"\\u0638\": 246, \"\\u0622\": 454, \"\\u0630\": 262, \"/\": 28, \"4\": 1588, \":\": 43, \"6\": 114, \"\\u06a4\": 429, \";\": 29, \"\\u06af\": 71, \"\\u061f\": 2820, \"\\u0662\": 2, \"\\u064b\": 17, \"\\u064e\": 5, \"\\u0640\": 2, \"\\u00e9\": 281, \"\\u064f\": 1982, \"*\": 7, \"+\": 4, \"\\u0121\": 1, \"\\u00bd\": 1, \"\\u201f\": 1, \")\": 4, \"(\": 4, \"\\\\\": 2, \"\\u00fc\": 1, \"%\": 6, \"\\\"\": 23514, \"\\u0448\": 2, \"\\u045a\": 1, \"\\u0433\": 116, \"\\u00a9\": 99, \"\\u0451\": 5, \"\\u045e\": 3, \"\\u043f\": 2, \"\\u0457\": 2, \"\\u0455\": 1, \"\\u00a7\": 3, \"\\u2020\": 2, \"\\u0449\": 25, \"\\u045c\": 1, \"\\u0432\": 1, \"\\u00a4\": 1, \"\\t\": 14, \"\\u0441\": 1, \"\\u0435\": 3, \"\\u043b\": 9, \"\\u0444\": 1, \"\\u00a0\": 3, \"\\u00d7\": 1, \"\\u0439\": 4, \"\\u043a\": 2, \"\\ufefb\": 3, \"$\": 3, \"\\ufee1\": 1, \"\\u2116\": 1, \"\\u201c\": 1, \"\\u00ae\": 2, \"\\u0669\": 1, \"\\u00f4\": 22, \"\\u064c\": 1, \"\\u00e7\": 14, \"\\u00fa\": 3, \"\\u00e2\": 17, \"\\u2019\": 81, \"\\u00e0\": 22, \"\\u00e8\": 23, \"\\u0153\": 2, \"\\ufffd\": 6, \"\\u00fb\": 7, \"\\u0137\": 4, \"\\u00ab\": 2, \"#\": 2, \"\\u20ac\": 6, \"\\u00ed\": 6, \"\\u00f1\": 2, \"\\u060c\": 4, \"\\u00bb\": 2, \"\\u017a\": 2, \"\\u0763\": 1, \"\\u0686\": 3, \"\\u0652\": 1, \"\\u06a2\": 2, \"\\u0650\": 1}", "index_docs": "{\"10\": 143414, \"6\": 150579, \"31\": 69908, \"44\": 28357, \"3\": 211095, \"13\": 135127, \"91\": 79, \"14\": 118883, \"5\": 175377, \"53\": 14817, \"9\": 130859, \"26\": 75975, \"2\": 262506, \"18\": 96118, \"60\": 7685, \"25\": 76458, \"62\": 6226, \"45\": 26298, \"7\": 169324, \"30\": 67797, \"11\": 135358, \"8\": 148483, \"22\": 91205, \"15\": 113248, \"59\": 7844, \"42\": 30042, \"20\": 78803, \"12\": 100230, \"47\": 25554, \"17\": 79945, \"43\": 29775, \"4\": 121755, \"19\": 82307, \"28\": 62860, \"36\": 43234, \"21\": 74885, \"24\": 68472, \"32\": 55142, \"27\": 66744, \"16\": 105061, \"41\": 32407, \"23\": 78842, \"48\": 25427, \"33\": 59633, \"54\": 14587, \"37\": 43479, \"57\": 9549, \"40\": 37528, \"51\": 17230, \"74\": 1544, \"50\": 21629, \"49\": 23863, \"46\": 28439, \"52\": 14279, \"35\": 49063, \"34\": 53725, \"73\": 1630, \"29\": 62427, \"64\": 5819, \"55\": 14312, \"38\": 41361, \"67\": 5647, \"61\": 6864, \"66\": 5590, \"58\": 8723, \"56\": 10281, \"65\": 5432, \"68\": 2804, \"63\": 5646, \"76\": 551, \"83\": 296, \"77\": 538, \"81\": 384, \"84\": 270, \"70\": 1914, \"75\": 1173, \"78\": 455, \"86\": 246, \"79\": 454, \"85\": 262, \"94\": 28, \"72\": 1588, \"93\": 43, \"89\": 114, \"80\": 429, \"95\": 29, \"92\": 71, \"69\": 2820, \"127\": 2, \"100\": 17, \"104\": 5, \"128\": 2, \"82\": 281, \"71\": 1982, \"106\": 7, \"113\": 4, \"144\": 1, \"145\": 1, \"146\": 1, \"115\": 4, \"114\": 4, \"129\": 2, \"147\": 1, \"108\": 6, \"39\": 23514, \"130\": 2, \"148\": 1, \"87\": 116, \"88\": 99, \"112\": 5, \"120\": 3, \"131\": 2, \"132\": 2, \"149\": 1, \"121\": 3, \"133\": 2, \"96\": 25, \"151\": 1, \"150\": 1, \"152\": 1, \"102\": 14, \"134\": 1, \"116\": 3, \"105\": 9, \"153\": 1, \"122\": 3, \"154\": 1, \"117\": 4, \"135\": 2, \"123\": 3, \"124\": 3, \"155\": 1, \"156\": 1, \"157\": 1, \"136\": 2, \"158\": 1, \"99\": 22, \"159\": 1, \"103\": 14, \"125\": 3, \"101\": 17, \"90\": 81, \"97\": 22, \"98\": 23, \"137\": 2, \"109\": 6, \"107\": 7, \"118\": 4, \"138\": 2, \"139\": 2, \"110\": 6, \"111\": 6, \"140\": 2, \"119\": 4, \"141\": 2, \"142\": 2, \"160\": 1, \"126\": 3, \"161\": 1, \"143\": 2, \"162\": 1}", "index_word": "{\"1\": \"<unk>\", \"2\": \" \", \"3\": \"a\", \"4\": \"\\u0627\", \"5\": \"i\", \"6\": \"e\", \"7\": \"t\", \"8\": \"h\", \"9\": \"l\", \"10\": \"n\", \"11\": \"o\", \"12\": \"\\u064a\", \"13\": \"r\", \"14\": \"d\", \"15\": \"s\", \"16\": \"m\", \"17\": \"\\u0644\", \"18\": \"k\", \"19\": \"\\u062a\", \"20\": \"\\u0648\", \"21\": \"\\u0646\", \"22\": \"y\", \"23\": \"w\", \"24\": \"\\u0645\", \"25\": \"b\", \"26\": \"c\", \"27\": \"\\u0631\", \"28\": \"\\u062f\", \"29\": \"\\u0643\", \"30\": \"u\", \"31\": \"g\", \"32\": \"\\u0628\", \"33\": \"f\", \"34\": \"3\", \"35\": \"\\u0639\", \"36\": \"\\u0647\", \"37\": \"\\u0634\", \"38\": \"\\u0633\", \"39\": \"\\\"\", \"40\": \"\\u0641\", \"41\": \"9\", \"42\": \"\\u0642\", \"43\": \"\\u0651\", \"44\": \"j\", \"45\": \"p\", \"46\": \"\\u063a\", \"47\": \"\\u062d\", \"48\": \"7\", \"49\": \"\\u062c\", \"50\": \"\\u062e\", \"51\": \"z\", \"52\": \"\\u0629\", \"53\": \"v\", \"54\": \"\\u0632\", \"55\": \"\\u0623\", \"56\": \"?\", \"57\": \"\\u0637\", \"58\": \"\\u0635\", \"59\": \"\\u0625\", \"60\": \"'\", \"61\": \"x\", \"62\": \".\", \"63\": \"5\", \"64\": \"2\", \"65\": \"8\", \"66\": \"\\u06ad\", \"67\": \"\\u0636\", \"68\": \"\\u0649\", \"69\": \"\\u061f\", \"70\": \"q\", \"71\": \"\\u064f\", \"72\": \"4\", \"73\": \"\\u0621\", \"74\": \"!\", \"75\": \"\\u067e\", \"76\": \"0\", \"77\": \"\\u0626\", \"78\": \"1\", \"79\": \"\\u0622\", \"80\": \"\\u06a4\", \"81\": \"\\u062b\", \"82\": \"\\u00e9\", \"83\": \"-\", \"84\": \"\\u0624\", \"85\": \"\\u0630\", \"86\": \"\\u0638\", \"87\": \"\\u0433\", \"88\": \"\\u00a9\", \"89\": \"6\", \"90\": \"\\u2019\", \"91\": \"_\", \"92\": \"\\u06af\", \"93\": \":\", \"94\": \"/\", \"95\": \";\", \"96\": \"\\u0449\", \"97\": \"\\u00e0\", \"98\": \"\\u00e8\", \"99\": \"\\u00f4\", \"100\": \"\\u064b\", \"101\": \"\\u00e2\", \"102\": \"\\t\", \"103\": \"\\u00e7\", \"104\": \"\\u064e\", \"105\": \"\\u043b\", \"106\": \"*\", \"107\": \"\\u00fb\", \"108\": \"%\", \"109\": \"\\ufffd\", \"110\": \"\\u20ac\", \"111\": \"\\u00ed\", \"112\": \"\\u0451\", \"113\": \"+\", \"114\": \"(\", \"115\": \")\", \"116\": \"\\u0435\", \"117\": \"\\u0439\", \"118\": \"\\u0137\", \"119\": \"\\u060c\", \"120\": \"\\u045e\", \"121\": \"\\u00a7\", \"122\": \"\\u00a0\", \"123\": \"\\ufefb\", \"124\": \"$\", \"125\": \"\\u00fa\", \"126\": \"\\u0686\", \"127\": \"\\u0662\", \"128\": \"\\u0640\", \"129\": \"\\\\\", \"130\": \"\\u0448\", \"131\": \"\\u043f\", \"132\": \"\\u0457\", \"133\": \"\\u2020\", \"134\": \"\\u0441\", \"135\": \"\\u043a\", \"136\": \"\\u00ae\", \"137\": \"\\u0153\", \"138\": \"\\u00ab\", \"139\": \"#\", \"140\": \"\\u00f1\", \"141\": \"\\u00bb\", \"142\": \"\\u017a\", \"143\": \"\\u06a2\", \"144\": \"\\u0121\", \"145\": \"\\u00bd\", \"146\": \"\\u201f\", \"147\": \"\\u00fc\", \"148\": \"\\u045a\", \"149\": \"\\u0455\", \"150\": \"\\u0432\", \"151\": \"\\u045c\", \"152\": \"\\u00a4\", \"153\": \"\\u0444\", \"154\": \"\\u00d7\", \"155\": \"\\ufee1\", \"156\": \"\\u2116\", \"157\": \"\\u201c\", \"158\": \"\\u0669\", \"159\": \"\\u064c\", \"160\": \"\\u0763\", \"161\": \"\\u0652\", \"162\": \"\\u0650\"}", "word_index": "{\"<unk>\": 1, \" \": 2, \"a\": 3, \"\\u0627\": 4, \"i\": 5, \"e\": 6, \"t\": 7, \"h\": 8, \"l\": 9, \"n\": 10, \"o\": 11, \"\\u064a\": 12, \"r\": 13, \"d\": 14, \"s\": 15, \"m\": 16, \"\\u0644\": 17, \"k\": 18, \"\\u062a\": 19, \"\\u0648\": 20, \"\\u0646\": 21, \"y\": 22, \"w\": 23, \"\\u0645\": 24, \"b\": 25, \"c\": 26, \"\\u0631\": 27, \"\\u062f\": 28, \"\\u0643\": 29, \"u\": 30, \"g\": 31, \"\\u0628\": 32, \"f\": 33, \"3\": 34, \"\\u0639\": 35, \"\\u0647\": 36, \"\\u0634\": 37, \"\\u0633\": 38, \"\\\"\": 39, \"\\u0641\": 40, \"9\": 41, \"\\u0642\": 42, \"\\u0651\": 43, \"j\": 44, \"p\": 45, \"\\u063a\": 46, \"\\u062d\": 47, \"7\": 48, \"\\u062c\": 49, \"\\u062e\": 50, \"z\": 51, \"\\u0629\": 52, \"v\": 53, \"\\u0632\": 54, \"\\u0623\": 55, \"?\": 56, \"\\u0637\": 57, \"\\u0635\": 58, \"\\u0625\": 59, \"'\": 60, \"x\": 61, \".\": 62, \"5\": 63, \"2\": 64, \"8\": 65, \"\\u06ad\": 66, \"\\u0636\": 67, \"\\u0649\": 68, \"\\u061f\": 69, \"q\": 70, \"\\u064f\": 71, \"4\": 72, \"\\u0621\": 73, \"!\": 74, \"\\u067e\": 75, \"0\": 76, \"\\u0626\": 77, \"1\": 78, \"\\u0622\": 79, \"\\u06a4\": 80, \"\\u062b\": 81, \"\\u00e9\": 82, \"-\": 83, \"\\u0624\": 84, \"\\u0630\": 85, \"\\u0638\": 86, \"\\u0433\": 87, \"\\u00a9\": 88, \"6\": 89, \"\\u2019\": 90, \"_\": 91, \"\\u06af\": 92, \":\": 93, \"/\": 94, \";\": 95, \"\\u0449\": 96, \"\\u00e0\": 97, \"\\u00e8\": 98, \"\\u00f4\": 99, \"\\u064b\": 100, \"\\u00e2\": 101, \"\\t\": 102, \"\\u00e7\": 103, \"\\u064e\": 104, \"\\u043b\": 105, \"*\": 106, \"\\u00fb\": 107, \"%\": 108, \"\\ufffd\": 109, \"\\u20ac\": 110, \"\\u00ed\": 111, \"\\u0451\": 112, \"+\": 113, \"(\": 114, \")\": 115, \"\\u0435\": 116, \"\\u0439\": 117, \"\\u0137\": 118, \"\\u060c\": 119, \"\\u045e\": 120, \"\\u00a7\": 121, \"\\u00a0\": 122, \"\\ufefb\": 123, \"$\": 124, \"\\u00fa\": 125, \"\\u0686\": 126, \"\\u0662\": 127, \"\\u0640\": 128, \"\\\\\": 129, \"\\u0448\": 130, \"\\u043f\": 131, \"\\u0457\": 132, \"\\u2020\": 133, \"\\u0441\": 134, \"\\u043a\": 135, \"\\u00ae\": 136, \"\\u0153\": 137, \"\\u00ab\": 138, \"#\": 139, \"\\u00f1\": 140, \"\\u00bb\": 141, \"\\u017a\": 142, \"\\u06a2\": 143, \"\\u0121\": 144, \"\\u00bd\": 145, \"\\u201f\": 146, \"\\u00fc\": 147, \"\\u045a\": 148, \"\\u0455\": 149, \"\\u0432\": 150, \"\\u045c\": 151, \"\\u00a4\": 152, \"\\u0444\": 153, \"\\u00d7\": 154, \"\\ufee1\": 155, \"\\u2116\": 156, \"\\u201c\": 157, \"\\u0669\": 158, \"\\u064c\": 159, \"\\u0763\": 160, \"\\u0652\": 161, \"\\u0650\": 162}"}}