|
1
|
-
{"base_model": "microsoft/Phi-3-mini-4k-instruct", "class_name": "mttl.models.expert_model.MultiExpertModelConfig", "default_expert_name": null, "expert_infos": [{"expert_name": "edumunozsala_phi-3-mini-LoRA", "class_name": "mttl.models.library.expert_library.MetadataEntry", "expert_task_name": null, "parent_node": null, "expert_config": {"modify_modules": ".*", "class_name": "mttl.models.modifiers.lora.LoRAConfig", "modify_layers": "k_proj|up_proj|down_proj|o_proj|gate_proj|v_proj|q_proj", "tie_params": null, "lora_rank": 16, "lora_alpha": 16, "lora_dropout": 0.0, "lora_init_b_random": false}, "training_config": {"max_input_length": 1024, "class_name": "mttl.arguments.ExpertConfig", "tokenizer": null, "model_family": null, "modify_modules": ".*", "modify_layers": null, "tie_params": null, "lora_rank": 4, "lora_alpha": 16.0, "lora_dropout": 0.0, "lora_init_b_random": false, "n_skills": 1, "n_splits": 1, "n_embd": 2560, "n_heads": 8, "moe_num_experts": 100, "emb_dim": 128, "down_proj_layer": "fc1", "up_proj_layer": "fc2", "model": null, "soft_prompt_length": 10, "n_tasks": null, "patch_last_k_layers": -1, "prompt_placement": "prefix", "keep_ratio": 1.0, "block_size": 16, "selection_accumulation_steps": 1, "reselection_steps": 100, "sps_type": "block_sparse", "sps_impl": "sp_add+sp_mm", "selection_algorithm": "rigl", "reselection_rate_policy": "linear", "mask_updater": null, "dataset": null, "data_dir": "/tmp/", "train_batch_size": 4, "predict_batch_size": 4, "max_output_length": 128, "validation_portion": null, "padding_side": "right", "truncation_side": "right", "train_on_inputs": false, "add_eos_to_targets": true, "add_eos_to_downstream_targets": true, "finetune_task_name": null, "subsample_train": null, "subsample_dev": null, "subsample_test": null, "subsample_per_task": false, "subsample": -1, "pack_sequences": false, "pad_to_multiple_of": 8, "max_seq_per_pack": 4, "task_id_field": "task_id", "task_name_field": "task_name", "task_source_field": "task_source", "arc_type": "ARC-Easy", "few_shot": true, "augment_mmlu": false, "source_template": null, "augment_few_shot": 0, "include_template_type": "*", "include_task_source": "P3,Flan2021,CoT", "remove_phi_eval_tasks": false, "attn_implementation": null, "device_map": "cpu", "load_in_4bit": false, "load_in_8bit": false, "do_train": true, "cache_dir": "./cache", "output_dir": "./output", "exp_name": null, "expert_name": null, "micro_batch_size": 4, "compute_strategy": "auto", "scheduler": "linear_decay_with_warmup", "checkpoint": null, "checkpoint_step": null, "backbone_checkpoint": null, "learning_rate": 0.001, "warmup_proportion": 0.06, "trainable_param_names": ".*", "non_trainable_param_names": null, "weight_decay": 0.01, "adam_epsilon": 1e-08, "max_grad_norm": 0.1, "optimizer": "adamw", "adafactor_scale_parameter": true, "adafactor_warmup_init": false, "adafactor_relative_step": false, "num_train_epochs": -1, "warmup_steps": -1, "total_steps": -1, "num_tasks_per_batch": null, "save_every": null, "save_each_epoch": false, "eval_every": null, "eval_every_n_epoch": 1, "seed": 42, "debug": false, "precision": "32", "monitor_grad_alignment_on": null, "wandb_project": null, "wandb_run_name": null, "tensorboard": false, "remote_token": null, "library_id": null, "destination_library_id": null, "logging_prefix": "", "router_weight_decay": null, "router_learning_rate": null, "module_logits_relaxed_bernoulli": true, "module_logits_straight_through": false, "module_logits_learning_rate": 0.1, "adapters_learning_rate": null, "adapters_weight_decay": null, "module_logits_dropout": 0.0, "module_logits_l2_norm": false, "eval_mmlu_few_shot": true, "eval_mmlu_flag": false, "eval_rouge_flag": false, "eval_before_training": true, "create_transfer_matrix": false, "pipeline_eval_tasks": null, "save_if_loaded_from_ckpt": true, "dataset_type": null, "profile": false, "model_modifier": null}, "expert_model": "microsoft/Phi-3-mini-4k-instruct", "expert_deleted": false}], "selector_config": {"router_granularity": "*", "class_name": "mttl.models.containers.selectors.arrow_selector.ArrowSelectorConfig", "lora_merge_after": false, "selector_logging": true, "num_experts": 0, "library_id": null, "selector_data_id": null, "router_temp": 1.0, "top_k": 2, "proto_init": "arrow", "input_norm_fn": "id", "proto_norm_fn": "id"}}
|
