pretrain core model

scripts/prepare_core_datasets.py

@@ -8,6 +8,9 @@ from utils import tokenize_fn
 from core_base_datasets import core_base_datasets
 from core_instruct_datasets import core_instruct_datasets
 
+
+tokenizer_path = '../tokenizer'
+
 seqs = [
     (0, 1073741824, 4097, 4000),
 ]
@@ -24,8 +27,8 @@ for i, (min_len, max_len, block_size, subchunk_size) in enumerate(seqs):
             tokenize_fn,
             min_len=min_len,
             max_len=max_len,
-            hf_tokenizer=AutoTokenizer.from_pretrained('..', trust_remote_code=True, use_fast=True),
-            tokenizer=Tokenizer('..'),
+            hf_tokenizer=AutoTokenizer.from_pretrained(tokenizer_path, trust_remote_code=True, use_fast=True),
+            tokenizer=Tokenizer(tokenizer_path),
         ),
         inputs=core_base_datasets + core_instruct_datasets,
         output_dir=output_dir,