Upload tokenizer_target.json with huggingface_hub

tokenizer_target.json

@@ -0,0 +1 @@
+{"class_name": "Tokenizer", "config": {"num_words": null, "filters": "!\"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n", "lower": true, "split": " ", "char_level": true, "oov_token": "<unk>", "document_count": 438257, "word_counts": "{\"d\": 273065, \"a\": 963323, \"r\": 255322, \"i\": 449963, \"j\": 74484, \"b\": 193971, \"y\": 138144, \"n\": 313627, \" \": 726498, \"h\": 409237, \"o\": 216471, \"m\": 264141, \"t\": 372935, \"7\": 73830, \"w\": 129943, \"l\": 364608, \"9\": 92877, \".\": 6485, \"c\": 138400, \"f\": 120038, \"z\": 48003, \"g\": 102278, \"3\": 159914, \"k\": 279287, \"!\": 1405, \"2\": 14609, \"e\": 248689, \"s\": 183836, \"u\": 64291, \"?\": 13486, \"8\": 14149, \"5\": 16819, \"0\": 787, \"p\": 5395, \"x\": 10002, \"1\": 611, \"'\": 378, \"/\": 34, \"4\": 2015, \"-\": 81, \":\": 42, \"6\": 116, \"v\": 1654, \";\": 12, \"q\": 2069, \"\\u00e9\": 393, \"+\": 6, \"\\u0121\": 3, \"\\\\\": 3, \"\\u0448\": 6, \"\\u045a\": 3, \"\\u0433\": 378, \"\\u00a9\": 333, \"\\u0451\": 12, \"\\u045e\": 9, \"\\u043f\": 3, \"\\u0457\": 3, \"\\u0455\": 3, \"\\u00a7\": 9, \"\\u0449\": 3, \"\\u2020\": 3, \"\\u0432\": 3, \"\\u045c\": 3, \"\\u00a4\": 3, \"\\t\": 29, \"\\u00a0\": 9, \"%\": 5, \"\\u043a\": 6, \"\\u2116\": 3, \"\\u0435\": 3, \"\\u201c\": 3, \"\\u00ae\": 6, \"\\u00f4\": 26, \"\\u00e7\": 18, \"\\u00fa\": 5, \"\\u00e2\": 19, \"\\\"\": 47478, \"_\": 4, \"\\u00e0\": 24, \"\\u00e8\": 26, \"\\u0153\": 2, \"\\ufffd\": 6, \"\\u00fb\": 9, \"\\u0137\": 4, \"\\u00ab\": 2, \"#\": 2, \"\\u20ac\": 6, \"\\u00ed\": 6, \"\\u00f1\": 2, \"\\u060c\": 4, \"\\u00bb\": 3, \"\\u017a\": 2, \"\\u0643\": 71, \"\\u0627\": 80, \"\\u064a\": 50, \"\\u0646\": 76, \"\\u0639\": 39, \"\\u062f\": 40, \"\\u0644\": 39, \"\\u0628\": 10, \"\\u0632\": 7, \"\\u0631\": 8, \"\\u061f\": 33, \"\\u0645\": 9, \"\\u0621\": 1, \"\\u0629\": 8, \"\\u0623\": 3, \"\\u062a\": 7, \"\\u0641\": 3, \"\\u0634\": 3, \"\\u0648\": 13, \"\\u0633\": 6, \"\\u062d\": 2, \"\\u06a4\": 1, \"\\u0647\": 1, \"\\u062e\": 3, \"\\u0763\": 1, \"\\u0637\": 1, \"\\u2019\": 18}", "word_docs": "{\"j\": 64938, \"a\": 349826, \"r\": 185367, \"d\": 183598, \"i\": 244735, \"t\": 249284, \"l\": 211595, \"h\": 238306, \" \": 219493, \"o\": 167043, \"b\": 150298, \"n\": 203917, \"y\": 111300, \"m\": 186763, \"9\": 81036, \".\": 5918, \"w\": 104138, \"7\": 65580, \"c\": 108996, \"f\": 101958, \"z\": 40026, \"3\": 134363, \"k\": 193478, \"g\": 94194, \"!\": 1375, \"2\": 14097, \"e\": 155758, \"s\": 136891, \"u\": 50704, \"?\": 13361, \"8\": 13224, \"5\": 14379, \"0\": 519, \"p\": 4880, \"x\": 8723, \"1\": 578, \"'\": 355, \"/\": 31, \"4\": 1889, \"-\": 75, \":\": 42, \"6\": 114, \"v\": 1639, \";\": 12, \"q\": 1588, \"\\u00e9\": 302, \"+\": 6, \"\\u0121\": 3, \"\\\\\": 3, \"\\u0448\": 6, \"\\u045a\": 3, \"\\u0433\": 273, \"\\u00a9\": 231, \"\\u0451\": 12, \"\\u045e\": 9, \"\\u043f\": 3, \"\\u0457\": 3, \"\\u0455\": 3, \"\\u00a7\": 9, \"\\u2020\": 3, \"\\u0449\": 3, \"\\u045c\": 3, \"\\u0432\": 3, \"\\u00a4\": 3, \"\\t\": 29, \"\\u00a0\": 9, \"%\": 5, \"\\u043a\": 6, \"\\u2116\": 3, \"\\u201c\": 3, \"\\u0435\": 3, \"\\u00ae\": 6, \"\\u00f4\": 26, \"\\u00e7\": 18, \"\\u00fa\": 5, \"\\u00e2\": 19, \"\\\"\": 23739, \"_\": 4, \"\\u00e0\": 22, \"\\u00e8\": 26, \"\\u0153\": 2, \"\\ufffd\": 6, \"\\u00fb\": 9, \"\\u0137\": 4, \"\\u00ab\": 2, \"#\": 2, \"\\u20ac\": 6, \"\\u00ed\": 6, \"\\u00f1\": 2, \"\\u060c\": 4, \"\\u00bb\": 3, \"\\u017a\": 2, \"\\u064a\": 33, \"\\u0644\": 31, \"\\u0627\": 33, \"\\u061f\": 33, \"\\u062f\": 33, \"\\u0628\": 9, \"\\u0643\": 33, \"\\u0639\": 33, \"\\u0646\": 33, \"\\u0631\": 8, \"\\u0632\": 7, \"\\u0645\": 8, \"\\u0621\": 1, \"\\u0629\": 6, \"\\u062a\": 7, \"\\u0623\": 3, \"\\u0641\": 3, \"\\u0648\": 11, \"\\u0634\": 2, \"\\u0633\": 6, \"\\u062d\": 2, \"\\u06a4\": 1, \"\\u0647\": 1, \"\\u062e\": 2, \"\\u0763\": 1, \"\\u0637\": 1, \"\\u2019\": 9}", "index_docs": "{\"24\": 64938, \"2\": 349826, \"12\": 185367, \"10\": 183598, \"4\": 244735, \"6\": 249284, \"7\": 211595, \"5\": 238306, \"3\": 219493, \"14\": 167043, \"15\": 150298, \"8\": 203917, \"19\": 111300, \"11\": 186763, \"23\": 81036, \"34\": 5918, \"20\": 104138, \"25\": 65580, \"18\": 108996, \"21\": 101958, \"27\": 40026, \"17\": 134363, \"9\": 193478, \"22\": 94194, \"39\": 1375, \"30\": 14097, \"13\": 155758, \"16\": 136891, \"26\": 50704, \"32\": 13361, \"31\": 13224, \"29\": 14379, \"40\": 519, \"35\": 4880, \"33\": 8723, \"41\": 578, \"43\": 355, \"56\": 31, \"37\": 1889, \"47\": 75, \"52\": 42, \"46\": 114, \"38\": 1639, \"66\": 12, \"36\": 1588, \"42\": 302, \"78\": 6, \"91\": 3, \"92\": 3, \"79\": 6, \"93\": 3, \"44\": 273, \"45\": 231, \"67\": 12, \"69\": 9, \"94\": 3, \"95\": 3, \"96\": 3, \"70\": 9, \"98\": 3, \"97\": 3, \"100\": 3, \"99\": 3, \"101\": 3, \"58\": 29, \"71\": 9, \"86\": 5, \"80\": 6, \"102\": 3, \"104\": 3, \"103\": 3, \"81\": 6, \"59\": 26, \"63\": 18, \"87\": 5, \"62\": 19, \"28\": 23739, \"88\": 4, \"61\": 22, \"60\": 26, \"110\": 2, \"82\": 6, \"72\": 9, \"89\": 4, \"111\": 2, \"112\": 2, \"83\": 6, \"84\": 6, \"113\": 2, \"90\": 4, \"105\": 3, \"114\": 2, \"51\": 33, \"55\": 31, \"48\": 33, \"57\": 33, \"53\": 33, \"68\": 9, \"50\": 33, \"54\": 33, \"49\": 33, \"74\": 8, \"76\": 7, \"73\": 8, \"116\": 1, \"75\": 6, \"77\": 7, \"106\": 3, \"107\": 3, \"65\": 11, \"108\": 2, \"85\": 6, \"115\": 2, \"117\": 1, \"118\": 1, \"109\": 2, \"119\": 1, \"120\": 1, \"64\": 9}", "index_word": "{\"1\": \"<unk>\", \"2\": \"a\", \"3\": \" \", \"4\": \"i\", \"5\": \"h\", \"6\": \"t\", \"7\": \"l\", \"8\": \"n\", \"9\": \"k\", \"10\": \"d\", \"11\": \"m\", \"12\": \"r\", \"13\": \"e\", \"14\": \"o\", \"15\": \"b\", \"16\": \"s\", \"17\": \"3\", \"18\": \"c\", \"19\": \"y\", \"20\": \"w\", \"21\": \"f\", \"22\": \"g\", \"23\": \"9\", \"24\": \"j\", \"25\": \"7\", \"26\": \"u\", \"27\": \"z\", \"28\": \"\\\"\", \"29\": \"5\", \"30\": \"2\", \"31\": \"8\", \"32\": \"?\", \"33\": \"x\", \"34\": \".\", \"35\": \"p\", \"36\": \"q\", \"37\": \"4\", \"38\": \"v\", \"39\": \"!\", \"40\": \"0\", \"41\": \"1\", \"42\": \"\\u00e9\", \"43\": \"'\", \"44\": \"\\u0433\", \"45\": \"\\u00a9\", \"46\": \"6\", \"47\": \"-\", \"48\": \"\\u0627\", \"49\": \"\\u0646\", \"50\": \"\\u0643\", \"51\": \"\\u064a\", \"52\": \":\", \"53\": \"\\u062f\", \"54\": \"\\u0639\", \"55\": \"\\u0644\", \"56\": \"/\", \"57\": \"\\u061f\", \"58\": \"\\t\", \"59\": \"\\u00f4\", \"60\": \"\\u00e8\", \"61\": \"\\u00e0\", \"62\": \"\\u00e2\", \"63\": \"\\u00e7\", \"64\": \"\\u2019\", \"65\": \"\\u0648\", \"66\": \";\", \"67\": \"\\u0451\", \"68\": \"\\u0628\", \"69\": \"\\u045e\", \"70\": \"\\u00a7\", \"71\": \"\\u00a0\", \"72\": \"\\u00fb\", \"73\": \"\\u0645\", \"74\": \"\\u0631\", \"75\": \"\\u0629\", \"76\": \"\\u0632\", \"77\": \"\\u062a\", \"78\": \"+\", \"79\": \"\\u0448\", \"80\": \"\\u043a\", \"81\": \"\\u00ae\", \"82\": \"\\ufffd\", \"83\": \"\\u20ac\", \"84\": \"\\u00ed\", \"85\": \"\\u0633\", \"86\": \"%\", \"87\": \"\\u00fa\", \"88\": \"_\", \"89\": \"\\u0137\", \"90\": \"\\u060c\", \"91\": \"\\u0121\", \"92\": \"\\\\\", \"93\": \"\\u045a\", \"94\": \"\\u043f\", \"95\": \"\\u0457\", \"96\": \"\\u0455\", \"97\": \"\\u0449\", \"98\": \"\\u2020\", \"99\": \"\\u0432\", \"100\": \"\\u045c\", \"101\": \"\\u00a4\", \"102\": \"\\u2116\", \"103\": \"\\u0435\", \"104\": \"\\u201c\", \"105\": \"\\u00bb\", \"106\": \"\\u0623\", \"107\": \"\\u0641\", \"108\": \"\\u0634\", \"109\": \"\\u062e\", \"110\": \"\\u0153\", \"111\": \"\\u00ab\", \"112\": \"#\", \"113\": \"\\u00f1\", \"114\": \"\\u017a\", \"115\": \"\\u062d\", \"116\": \"\\u0621\", \"117\": \"\\u06a4\", \"118\": \"\\u0647\", \"119\": \"\\u0763\", \"120\": \"\\u0637\"}", "word_index": "{\"<unk>\": 1, \"a\": 2, \" \": 3, \"i\": 4, \"h\": 5, \"t\": 6, \"l\": 7, \"n\": 8, \"k\": 9, \"d\": 10, \"m\": 11, \"r\": 12, \"e\": 13, \"o\": 14, \"b\": 15, \"s\": 16, \"3\": 17, \"c\": 18, \"y\": 19, \"w\": 20, \"f\": 21, \"g\": 22, \"9\": 23, \"j\": 24, \"7\": 25, \"u\": 26, \"z\": 27, \"\\\"\": 28, \"5\": 29, \"2\": 30, \"8\": 31, \"?\": 32, \"x\": 33, \".\": 34, \"p\": 35, \"q\": 36, \"4\": 37, \"v\": 38, \"!\": 39, \"0\": 40, \"1\": 41, \"\\u00e9\": 42, \"'\": 43, \"\\u0433\": 44, \"\\u00a9\": 45, \"6\": 46, \"-\": 47, \"\\u0627\": 48, \"\\u0646\": 49, \"\\u0643\": 50, \"\\u064a\": 51, \":\": 52, \"\\u062f\": 53, \"\\u0639\": 54, \"\\u0644\": 55, \"/\": 56, \"\\u061f\": 57, \"\\t\": 58, \"\\u00f4\": 59, \"\\u00e8\": 60, \"\\u00e0\": 61, \"\\u00e2\": 62, \"\\u00e7\": 63, \"\\u2019\": 64, \"\\u0648\": 65, \";\": 66, \"\\u0451\": 67, \"\\u0628\": 68, \"\\u045e\": 69, \"\\u00a7\": 70, \"\\u00a0\": 71, \"\\u00fb\": 72, \"\\u0645\": 73, \"\\u0631\": 74, \"\\u0629\": 75, \"\\u0632\": 76, \"\\u062a\": 77, \"+\": 78, \"\\u0448\": 79, \"\\u043a\": 80, \"\\u00ae\": 81, \"\\ufffd\": 82, \"\\u20ac\": 83, \"\\u00ed\": 84, \"\\u0633\": 85, \"%\": 86, \"\\u00fa\": 87, \"_\": 88, \"\\u0137\": 89, \"\\u060c\": 90, \"\\u0121\": 91, \"\\\\\": 92, \"\\u045a\": 93, \"\\u043f\": 94, \"\\u0457\": 95, \"\\u0455\": 96, \"\\u0449\": 97, \"\\u2020\": 98, \"\\u0432\": 99, \"\\u045c\": 100, \"\\u00a4\": 101, \"\\u2116\": 102, \"\\u0435\": 103, \"\\u201c\": 104, \"\\u00bb\": 105, \"\\u0623\": 106, \"\\u0641\": 107, \"\\u0634\": 108, \"\\u062e\": 109, \"\\u0153\": 110, \"\\u00ab\": 111, \"#\": 112, \"\\u00f1\": 113, \"\\u017a\": 114, \"\\u062d\": 115, \"\\u0621\": 116, \"\\u06a4\": 117, \"\\u0647\": 118, \"\\u0763\": 119, \"\\u0637\": 120}"}}