Upload model

README.md

@@ -0,0 +1,199 @@
+---
+library_name: transformers
+tags: []
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+This is the model card of a 🤗 transformers model that has been pushed on the Hub. This model card has been automatically generated.
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]

mttl_config.json

@@ -0,0 +1 @@
+{"base_model": "microsoft/Phi-3-mini-4k-instruct", "class_name": "mttl.models.expert_model.MultiExpertModelConfig", "default_expert_name": null, "expert_infos": [{"expert_name": "edumunozsala_phi-3-mini-LoRA", "class_name": "mttl.models.library.expert_library.MetadataEntry", "expert_task_name": null, "parent_node": null, "expert_config": {"modify_modules": ".*", "class_name": "mttl.models.modifiers.lora.LoRAConfig", "modify_layers": "k_proj|up_proj|down_proj|o_proj|gate_proj|v_proj|q_proj", "tie_params": null, "lora_rank": 16, "lora_alpha": 16, "lora_dropout": 0.0, "lora_init_b_random": false}, "training_config": {"max_input_length": 1024, "class_name": "mttl.arguments.ExpertConfig", "tokenizer": null, "model_family": null, "modify_modules": ".*", "modify_layers": null, "tie_params": null, "lora_rank": 4, "lora_alpha": 16.0, "lora_dropout": 0.0, "lora_init_b_random": false, "n_skills": 1, "n_splits": 1, "n_embd": 2560, "n_heads": 8, "moe_num_experts": 100, "emb_dim": 128, "down_proj_layer": "fc1", "up_proj_layer": "fc2", "model": null, "soft_prompt_length": 10, "n_tasks": null, "patch_last_k_layers": -1, "prompt_placement": "prefix", "keep_ratio": 1.0, "block_size": 16, "selection_accumulation_steps": 1, "reselection_steps": 100, "sps_type": "block_sparse", "sps_impl": "sp_add+sp_mm", "selection_algorithm": "rigl", "reselection_rate_policy": "linear", "mask_updater": null, "dataset": null, "data_dir": "/tmp/", "train_batch_size": 4, "predict_batch_size": 4, "max_output_length": 128, "validation_portion": null, "padding_side": "right", "truncation_side": "right", "train_on_inputs": false, "add_eos_to_targets": true, "add_eos_to_downstream_targets": true, "finetune_task_name": null, "subsample_train": null, "subsample_dev": null, "subsample_test": null, "subsample_per_task": false, "subsample": -1, "pack_sequences": false, "pad_to_multiple_of": 8, "max_seq_per_pack": 4, "task_id_field": "task_id", "task_name_field": "task_name", "task_source_field": "task_source", "arc_type": "ARC-Easy", "few_shot": true, "augment_mmlu": false, "source_template": null, "augment_few_shot": 0, "include_template_type": "*", "include_task_source": "P3,Flan2021,CoT", "remove_phi_eval_tasks": false, "attn_implementation": null, "device_map": "cpu", "load_in_4bit": false, "load_in_8bit": false, "do_train": true, "cache_dir": "./cache", "output_dir": "./output", "exp_name": null, "expert_name": null, "micro_batch_size": 4, "compute_strategy": "auto", "scheduler": "linear_decay_with_warmup", "checkpoint": null, "checkpoint_step": null, "backbone_checkpoint": null, "learning_rate": 0.001, "warmup_proportion": 0.06, "trainable_param_names": ".*", "non_trainable_param_names": null, "weight_decay": 0.01, "adam_epsilon": 1e-08, "max_grad_norm": 0.1, "optimizer": "adamw", "adafactor_scale_parameter": true, "adafactor_warmup_init": false, "adafactor_relative_step": false, "num_train_epochs": -1, "warmup_steps": -1, "total_steps": -1, "num_tasks_per_batch": null, "save_every": null, "save_each_epoch": false, "eval_every": null, "eval_every_n_epoch": 1, "seed": 42, "debug": false, "precision": "32", "monitor_grad_alignment_on": null, "wandb_project": null, "wandb_run_name": null, "tensorboard": false, "remote_token": null, "library_id": null, "destination_library_id": null, "logging_prefix": "", "router_weight_decay": null, "router_learning_rate": null, "module_logits_relaxed_bernoulli": true, "module_logits_straight_through": false, "module_logits_learning_rate": 0.1, "adapters_learning_rate": null, "adapters_weight_decay": null, "module_logits_dropout": 0.0, "module_logits_l2_norm": false, "eval_mmlu_few_shot": true, "eval_mmlu_flag": false, "eval_rouge_flag": false, "eval_before_training": true, "create_transfer_matrix": false, "pipeline_eval_tasks": null, "save_if_loaded_from_ckpt": true, "dataset_type": null, "profile": false, "model_modifier": null}, "expert_model": "microsoft/Phi-3-mini-4k-instruct", "expert_deleted": false}], "selector_config": {"router_granularity": "*", "class_name": "mttl.models.containers.selectors.arrow_selector.ArrowSelectorConfig", "lora_merge_after": false, "selector_logging": true, "num_experts": 0, "library_id": null, "selector_data_id": null, "router_temp": 1.0, "top_k": 2, "proto_init": "arrow", "input_norm_fn": "id", "proto_norm_fn": "id"}}

mttl_weights.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15a54808aa191e96ae9ce65d55f2de530484348eb3267d20348bb30d0be44277
+size 37223174