Upload folder using huggingface_hub
Browse files- .gitattributes +1 -0
- added_tokens.json +22 -0
- config.json +722 -0
- configuration_intern_vit.py +120 -0
- configuration_internvl_chat.py +105 -0
- configuration_phi3.py +211 -0
- configuration_radio.py +52 -0
- conversation.py +391 -0
- examples/image1.jpg +0 -0
- examples/image2.jpg +0 -0
- examples/match_case/FRAME00.jpg +0 -0
- examples/match_case/FRAME00.json +1 -0
- examples/match_case/FRAME00_ORI.jpg +0 -0
- examples/match_case/FRAME01_CAND.jpg +0 -0
- examples/match_case/FRAME01_CAND.json +1 -0
- examples/match_case/FRAME01_ORI.jpg +0 -0
- examples/match_case/anno.txt +3 -0
- examples/red-panda.mp4 +3 -0
- generation_config.json +4 -0
- model-00001-of-00002.safetensors +3 -0
- model-00002-of-00002.safetensors +3 -0
- model.safetensors.index.json +1010 -0
- modeling_intern_vit.py +430 -0
- modeling_internvl_chat.py +435 -0
- modeling_phi3.py +1610 -0
- modeling_radio.py +101 -0
- preprocessor_config.json +19 -0
- radio_adaptor_base.py +35 -0
- radio_adaptor_generic.py +29 -0
- radio_adaptor_mlp.py +150 -0
- radio_adaptor_registry.py +37 -0
- radio_cls_token.py +55 -0
- radio_common.py +51 -0
- radio_enable_cpe_support.py +67 -0
- radio_enable_spectral_reparam.py +227 -0
- radio_eradio_model.py +1392 -0
- radio_extra_timm_models.py +66 -0
- radio_input_conditioner.py +49 -0
- radio_model.py +204 -0
- radio_open_clip_adaptor.py +41 -0
- radio_vit_patch_generator.py +299 -0
- radio_vitdet.py +173 -0
- special_tokens_map.json +41 -0
- tokenizer.model +3 -0
- tokenizer_config.json +213 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
examples/red-panda.mp4 filter=lfs diff=lfs merge=lfs -text
|
added_tokens.json
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"</box>": 32019,
|
| 3 |
+
"</img>": 32012,
|
| 4 |
+
"</quad>": 32015,
|
| 5 |
+
"</ref>": 32017,
|
| 6 |
+
"<IMG_CONTEXT>": 32013,
|
| 7 |
+
"<box>": 32018,
|
| 8 |
+
"<img>": 32011,
|
| 9 |
+
"<quad>": 32014,
|
| 10 |
+
"<ref>": 32016,
|
| 11 |
+
"<|assistant|>": 32001,
|
| 12 |
+
"<|endoftext|>": 32000,
|
| 13 |
+
"<|end|>": 32007,
|
| 14 |
+
"<|placeholder1|>": 32002,
|
| 15 |
+
"<|placeholder2|>": 32003,
|
| 16 |
+
"<|placeholder3|>": 32004,
|
| 17 |
+
"<|placeholder4|>": 32005,
|
| 18 |
+
"<|placeholder5|>": 32008,
|
| 19 |
+
"<|placeholder6|>": 32009,
|
| 20 |
+
"<|system|>": 32006,
|
| 21 |
+
"<|user|>": 32010
|
| 22 |
+
}
|
config.json
ADDED
|
@@ -0,0 +1,722 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_commit_hash": null,
|
| 3 |
+
"_name_or_path": "./OpenGVLab/InternVL2-4B",
|
| 4 |
+
"architectures": [
|
| 5 |
+
"InternVLChatModel"
|
| 6 |
+
],
|
| 7 |
+
"auto_map": {
|
| 8 |
+
"AutoConfig": "configuration_internvl_chat.InternVLChatConfig",
|
| 9 |
+
"AutoModel": "modeling_internvl_chat.InternVLChatModel",
|
| 10 |
+
"AutoModelForCausalLM": "modeling_internvl_chat.InternVLChatModel"
|
| 11 |
+
},
|
| 12 |
+
"downsample_ratio": 0.5,
|
| 13 |
+
"dynamic_image_size": true,
|
| 14 |
+
"force_image_size": 448,
|
| 15 |
+
"llm_config": {
|
| 16 |
+
"_attn_implementation_autoset": false,
|
| 17 |
+
"_name_or_path": "microsoft/Phi-3-mini-128k-instruct",
|
| 18 |
+
"add_cross_attention": false,
|
| 19 |
+
"architectures": [
|
| 20 |
+
"Phi3ForCausalLM"
|
| 21 |
+
],
|
| 22 |
+
"attention_dropout": 0.0,
|
| 23 |
+
"auto_map": {
|
| 24 |
+
"AutoConfig": "configuration_phi3.Phi3Config",
|
| 25 |
+
"AutoModelForCausalLM": "modeling_phi3.Phi3ForCausalLM"
|
| 26 |
+
},
|
| 27 |
+
"bad_words_ids": null,
|
| 28 |
+
"begin_suppress_tokens": null,
|
| 29 |
+
"bos_token_id": 1,
|
| 30 |
+
"chunk_size_feed_forward": 0,
|
| 31 |
+
"cross_attention_hidden_size": null,
|
| 32 |
+
"decoder_start_token_id": null,
|
| 33 |
+
"diversity_penalty": 0.0,
|
| 34 |
+
"do_sample": false,
|
| 35 |
+
"early_stopping": false,
|
| 36 |
+
"embd_pdrop": 0.0,
|
| 37 |
+
"encoder_no_repeat_ngram_size": 0,
|
| 38 |
+
"eos_token_id": 32000,
|
| 39 |
+
"exponential_decay_length_penalty": null,
|
| 40 |
+
"finetuning_task": null,
|
| 41 |
+
"forced_bos_token_id": null,
|
| 42 |
+
"forced_eos_token_id": null,
|
| 43 |
+
"hidden_act": "silu",
|
| 44 |
+
"hidden_size": 3072,
|
| 45 |
+
"id2label": {
|
| 46 |
+
"0": "LABEL_0",
|
| 47 |
+
"1": "LABEL_1"
|
| 48 |
+
},
|
| 49 |
+
"initializer_range": 0.02,
|
| 50 |
+
"intermediate_size": 8192,
|
| 51 |
+
"is_decoder": false,
|
| 52 |
+
"is_encoder_decoder": false,
|
| 53 |
+
"label2id": {
|
| 54 |
+
"LABEL_0": 0,
|
| 55 |
+
"LABEL_1": 1
|
| 56 |
+
},
|
| 57 |
+
"length_penalty": 1.0,
|
| 58 |
+
"max_length": 20,
|
| 59 |
+
"max_position_embeddings": 131072,
|
| 60 |
+
"min_length": 0,
|
| 61 |
+
"model_type": "phi3",
|
| 62 |
+
"no_repeat_ngram_size": 0,
|
| 63 |
+
"num_attention_heads": 32,
|
| 64 |
+
"num_beam_groups": 1,
|
| 65 |
+
"num_beams": 1,
|
| 66 |
+
"num_hidden_layers": 32,
|
| 67 |
+
"num_key_value_heads": 32,
|
| 68 |
+
"num_return_sequences": 1,
|
| 69 |
+
"original_max_position_embeddings": 4096,
|
| 70 |
+
"output_attentions": false,
|
| 71 |
+
"output_hidden_states": false,
|
| 72 |
+
"output_scores": false,
|
| 73 |
+
"pad_token_id": 32000,
|
| 74 |
+
"prefix": null,
|
| 75 |
+
"problem_type": null,
|
| 76 |
+
"pruned_heads": {},
|
| 77 |
+
"remove_invalid_values": false,
|
| 78 |
+
"repetition_penalty": 1.0,
|
| 79 |
+
"resid_pdrop": 0.0,
|
| 80 |
+
"return_dict": true,
|
| 81 |
+
"return_dict_in_generate": false,
|
| 82 |
+
"rms_norm_eps": 1e-05,
|
| 83 |
+
"rope_scaling": {
|
| 84 |
+
"long_factor": [
|
| 85 |
+
1.0299999713897705,
|
| 86 |
+
1.0499999523162842,
|
| 87 |
+
1.0499999523162842,
|
| 88 |
+
1.0799999237060547,
|
| 89 |
+
1.2299998998641968,
|
| 90 |
+
1.2299998998641968,
|
| 91 |
+
1.2999999523162842,
|
| 92 |
+
1.4499999284744263,
|
| 93 |
+
1.5999999046325684,
|
| 94 |
+
1.6499998569488525,
|
| 95 |
+
1.8999998569488525,
|
| 96 |
+
2.859999895095825,
|
| 97 |
+
3.68999981880188,
|
| 98 |
+
5.419999599456787,
|
| 99 |
+
5.489999771118164,
|
| 100 |
+
5.489999771118164,
|
| 101 |
+
9.09000015258789,
|
| 102 |
+
11.579999923706055,
|
| 103 |
+
15.65999984741211,
|
| 104 |
+
15.769999504089355,
|
| 105 |
+
15.789999961853027,
|
| 106 |
+
18.360000610351562,
|
| 107 |
+
21.989999771118164,
|
| 108 |
+
23.079999923706055,
|
| 109 |
+
30.009998321533203,
|
| 110 |
+
32.35000228881836,
|
| 111 |
+
32.590003967285156,
|
| 112 |
+
35.56000518798828,
|
| 113 |
+
39.95000457763672,
|
| 114 |
+
53.840003967285156,
|
| 115 |
+
56.20000457763672,
|
| 116 |
+
57.95000457763672,
|
| 117 |
+
59.29000473022461,
|
| 118 |
+
59.77000427246094,
|
| 119 |
+
59.920005798339844,
|
| 120 |
+
61.190006256103516,
|
| 121 |
+
61.96000671386719,
|
| 122 |
+
62.50000762939453,
|
| 123 |
+
63.3700065612793,
|
| 124 |
+
63.48000717163086,
|
| 125 |
+
63.48000717163086,
|
| 126 |
+
63.66000747680664,
|
| 127 |
+
63.850006103515625,
|
| 128 |
+
64.08000946044922,
|
| 129 |
+
64.760009765625,
|
| 130 |
+
64.80001068115234,
|
| 131 |
+
64.81001281738281,
|
| 132 |
+
64.81001281738281
|
| 133 |
+
],
|
| 134 |
+
"short_factor": [
|
| 135 |
+
1.05,
|
| 136 |
+
1.05,
|
| 137 |
+
1.05,
|
| 138 |
+
1.1,
|
| 139 |
+
1.1,
|
| 140 |
+
1.1500000000000001,
|
| 141 |
+
1.2000000000000002,
|
| 142 |
+
1.2500000000000002,
|
| 143 |
+
1.3000000000000003,
|
| 144 |
+
1.3500000000000003,
|
| 145 |
+
1.5000000000000004,
|
| 146 |
+
2.000000000000001,
|
| 147 |
+
2.000000000000001,
|
| 148 |
+
2.000000000000001,
|
| 149 |
+
2.000000000000001,
|
| 150 |
+
2.000000000000001,
|
| 151 |
+
2.000000000000001,
|
| 152 |
+
2.000000000000001,
|
| 153 |
+
2.000000000000001,
|
| 154 |
+
2.000000000000001,
|
| 155 |
+
2.000000000000001,
|
| 156 |
+
2.000000000000001,
|
| 157 |
+
2.000000000000001,
|
| 158 |
+
2.000000000000001,
|
| 159 |
+
2.000000000000001,
|
| 160 |
+
2.000000000000001,
|
| 161 |
+
2.000000000000001,
|
| 162 |
+
2.000000000000001,
|
| 163 |
+
2.000000000000001,
|
| 164 |
+
2.000000000000001,
|
| 165 |
+
2.000000000000001,
|
| 166 |
+
2.000000000000001,
|
| 167 |
+
2.0500000000000007,
|
| 168 |
+
2.0500000000000007,
|
| 169 |
+
2.0500000000000007,
|
| 170 |
+
2.1000000000000005,
|
| 171 |
+
2.1000000000000005,
|
| 172 |
+
2.1000000000000005,
|
| 173 |
+
2.1500000000000004,
|
| 174 |
+
2.1500000000000004,
|
| 175 |
+
2.3499999999999996,
|
| 176 |
+
2.549999999999999,
|
| 177 |
+
2.5999999999999988,
|
| 178 |
+
2.5999999999999988,
|
| 179 |
+
2.7499999999999982,
|
| 180 |
+
2.849999999999998,
|
| 181 |
+
2.849999999999998,
|
| 182 |
+
2.9499999999999975
|
| 183 |
+
],
|
| 184 |
+
"type": "su"
|
| 185 |
+
},
|
| 186 |
+
"rope_theta": 10000.0,
|
| 187 |
+
"sep_token_id": null,
|
| 188 |
+
"sliding_window": 262144,
|
| 189 |
+
"suppress_tokens": null,
|
| 190 |
+
"task_specific_params": null,
|
| 191 |
+
"temperature": 1.0,
|
| 192 |
+
"tf_legacy_loss": false,
|
| 193 |
+
"tie_encoder_decoder": false,
|
| 194 |
+
"tie_word_embeddings": false,
|
| 195 |
+
"tokenizer_class": null,
|
| 196 |
+
"top_k": 50,
|
| 197 |
+
"top_p": 1.0,
|
| 198 |
+
"torch_dtype": "bfloat16",
|
| 199 |
+
"torchscript": false,
|
| 200 |
+
"transformers_version": "4.47.1",
|
| 201 |
+
"typical_p": 1.0,
|
| 202 |
+
"use_bfloat16": true,
|
| 203 |
+
"use_cache": true,
|
| 204 |
+
"vocab_size": 32020
|
| 205 |
+
},
|
| 206 |
+
"max_dynamic_patch": 12,
|
| 207 |
+
"min_dynamic_patch": 1,
|
| 208 |
+
"model_type": "internvl_chat",
|
| 209 |
+
"ps_version": "v2",
|
| 210 |
+
"radio_config": {
|
| 211 |
+
"_attn_implementation_autoset": true,
|
| 212 |
+
"_name_or_path": "./nvidia/RADIO",
|
| 213 |
+
"adaptor_names": null,
|
| 214 |
+
"add_cross_attention": false,
|
| 215 |
+
"architectures": [
|
| 216 |
+
"RADIOModel"
|
| 217 |
+
],
|
| 218 |
+
"args": {
|
| 219 |
+
"aa": null,
|
| 220 |
+
"amp": false,
|
| 221 |
+
"amp_dtype": "float16",
|
| 222 |
+
"amp_impl": "native",
|
| 223 |
+
"aug_repeats": 0,
|
| 224 |
+
"aug_splits": 0,
|
| 225 |
+
"auto_loss_balance_mode": "manual",
|
| 226 |
+
"batch_size": 32,
|
| 227 |
+
"bn_eps": null,
|
| 228 |
+
"bn_momentum": null,
|
| 229 |
+
"cache_dir": null,
|
| 230 |
+
"channels_last": false,
|
| 231 |
+
"checkpoint_hist": 10,
|
| 232 |
+
"chk_keep_forever": 10,
|
| 233 |
+
"class_map": "",
|
| 234 |
+
"clip_grad": null,
|
| 235 |
+
"clip_mode": "norm",
|
| 236 |
+
"cls_token_per_teacher": true,
|
| 237 |
+
"coco_annotations_file": "/datasets/coco2017-adlsa/annotations/captions_val2017.json",
|
| 238 |
+
"coco_image_dir": "/datasets/coco2017-adlsa/val2017",
|
| 239 |
+
"color_jitter": 0.4,
|
| 240 |
+
"cooldown_epochs": 0,
|
| 241 |
+
"cpe_max_size": 2048,
|
| 242 |
+
"crd_loss": false,
|
| 243 |
+
"crd_loss_weight": 0.8,
|
| 244 |
+
"crop_pct": null,
|
| 245 |
+
"cutmix": 0.0,
|
| 246 |
+
"cutmix_minmax": null,
|
| 247 |
+
"data_dir": [
|
| 248 |
+
[
|
| 249 |
+
"/lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/datasets/captioning/datacomp/dc1b/stage2",
|
| 250 |
+
0.95
|
| 251 |
+
],
|
| 252 |
+
[
|
| 253 |
+
"/lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/datasets/segmentation/sam/stage1",
|
| 254 |
+
0.05
|
| 255 |
+
]
|
| 256 |
+
],
|
| 257 |
+
"dataset": "nvgpt4",
|
| 258 |
+
"dataset_download": false,
|
| 259 |
+
"debug_full_knn": false,
|
| 260 |
+
"decay_epochs": 90,
|
| 261 |
+
"decay_milestones": [
|
| 262 |
+
90,
|
| 263 |
+
180,
|
| 264 |
+
270
|
| 265 |
+
],
|
| 266 |
+
"decay_rate": 0.1,
|
| 267 |
+
"device": "cuda:0",
|
| 268 |
+
"dist_bn": "reduce",
|
| 269 |
+
"distributed": true,
|
| 270 |
+
"drop": 0.0,
|
| 271 |
+
"drop_block": null,
|
| 272 |
+
"drop_connect": null,
|
| 273 |
+
"drop_path": null,
|
| 274 |
+
"dtype": "bfloat16",
|
| 275 |
+
"epoch_repeats": 0.0,
|
| 276 |
+
"epochs": 50,
|
| 277 |
+
"eval": false,
|
| 278 |
+
"eval_metric": "knn_top1",
|
| 279 |
+
"eval_teacher": false,
|
| 280 |
+
"eval_teacher_only": false,
|
| 281 |
+
"eval_throughput": false,
|
| 282 |
+
"experiment": "checkpoints",
|
| 283 |
+
"fast_norm": false,
|
| 284 |
+
"feature_summarizer": "cls_token",
|
| 285 |
+
"feature_upscale_factor": null,
|
| 286 |
+
"force_new_wandb_id": false,
|
| 287 |
+
"force_spectral_reparam": false,
|
| 288 |
+
"fuser": "",
|
| 289 |
+
"gp": null,
|
| 290 |
+
"grad_accum_steps": 1,
|
| 291 |
+
"grad_checkpointing": false,
|
| 292 |
+
"head_init_bias": null,
|
| 293 |
+
"head_init_scale": null,
|
| 294 |
+
"hflip": 0.5,
|
| 295 |
+
"img_size": null,
|
| 296 |
+
"in_chans": null,
|
| 297 |
+
"initial_checkpoint": "",
|
| 298 |
+
"input_size": null,
|
| 299 |
+
"interpolation": "",
|
| 300 |
+
"layer_decay": null,
|
| 301 |
+
"local_rank": 0,
|
| 302 |
+
"log_interval": 50,
|
| 303 |
+
"log_mlflow": false,
|
| 304 |
+
"log_wandb": true,
|
| 305 |
+
"loss": "cosine",
|
| 306 |
+
"loss_auto_balance": false,
|
| 307 |
+
"lr": 0.001,
|
| 308 |
+
"lr_base": 0.1,
|
| 309 |
+
"lr_base_scale": "",
|
| 310 |
+
"lr_base_size": 256,
|
| 311 |
+
"lr_cycle_decay": 0.5,
|
| 312 |
+
"lr_cycle_limit": 1,
|
| 313 |
+
"lr_cycle_mul": 1.0,
|
| 314 |
+
"lr_k_decay": 1.0,
|
| 315 |
+
"lr_noise": null,
|
| 316 |
+
"lr_noise_pct": 0.67,
|
| 317 |
+
"lr_noise_std": 1.0,
|
| 318 |
+
"mean": null,
|
| 319 |
+
"mesa": {
|
| 320 |
+
"gaussian_kl": false,
|
| 321 |
+
"start_epoch": 100
|
| 322 |
+
},
|
| 323 |
+
"min_lr": 0,
|
| 324 |
+
"mixup": 0.0,
|
| 325 |
+
"mixup_mode": "batch",
|
| 326 |
+
"mixup_off_epoch": 0,
|
| 327 |
+
"mixup_prob": 1.0,
|
| 328 |
+
"mixup_switch_prob": 0.5,
|
| 329 |
+
"mlp_hidden_size": 1520,
|
| 330 |
+
"mlp_num_inner": 3,
|
| 331 |
+
"mlp_version": "v2",
|
| 332 |
+
"model": "vit_huge_patch16_224_mlpnorm",
|
| 333 |
+
"model_ema": {
|
| 334 |
+
"decay": 0.9998,
|
| 335 |
+
"force_cpu": false,
|
| 336 |
+
"power": false,
|
| 337 |
+
"power_stds": [
|
| 338 |
+
0.05,
|
| 339 |
+
0.1
|
| 340 |
+
],
|
| 341 |
+
"start_epoch": 0
|
| 342 |
+
},
|
| 343 |
+
"model_kwargs": {},
|
| 344 |
+
"model_norm": true,
|
| 345 |
+
"momentum": 0.9,
|
| 346 |
+
"no_aug": false,
|
| 347 |
+
"no_ddp_bb": false,
|
| 348 |
+
"no_prefetcher": false,
|
| 349 |
+
"no_resume_opt": false,
|
| 350 |
+
"num_classes": null,
|
| 351 |
+
"opt": "lamb",
|
| 352 |
+
"opt_betas": null,
|
| 353 |
+
"opt_eps": null,
|
| 354 |
+
"opt_kwargs": {
|
| 355 |
+
"filter_bias_and_bn": false
|
| 356 |
+
},
|
| 357 |
+
"output": "/lustre/fs6/portfolios/llmservice/users/mranzinger/output/evfm/ohem/3-13-24_vit-h-16_bf16_ep50",
|
| 358 |
+
"patience_epochs": 10,
|
| 359 |
+
"pin_mem": false,
|
| 360 |
+
"prefetcher": true,
|
| 361 |
+
"pretrained": false,
|
| 362 |
+
"rank": 0,
|
| 363 |
+
"ratio": [
|
| 364 |
+
0.75,
|
| 365 |
+
1.3333333333333333
|
| 366 |
+
],
|
| 367 |
+
"recount": 1,
|
| 368 |
+
"recovery_interval": 0,
|
| 369 |
+
"register_multiple": 16,
|
| 370 |
+
"remode": "pixel",
|
| 371 |
+
"reprob": 0.0,
|
| 372 |
+
"resplit": false,
|
| 373 |
+
"resume": "/lustre/fs6/portfolios/llmservice/users/mranzinger/output/evfm/ohem/3-13-24_vit-h-16_bf16_ep50/checkpoints/checkpoint-48.pth.tar",
|
| 374 |
+
"save_images": false,
|
| 375 |
+
"scale": [
|
| 376 |
+
0.5,
|
| 377 |
+
1.0
|
| 378 |
+
],
|
| 379 |
+
"sched": "cosine",
|
| 380 |
+
"sched_on_updates": true,
|
| 381 |
+
"seed": 42,
|
| 382 |
+
"smoothing": 0.1,
|
| 383 |
+
"spectral_reparam": false,
|
| 384 |
+
"split_bn": false,
|
| 385 |
+
"start_epoch": null,
|
| 386 |
+
"std": null,
|
| 387 |
+
"steps_per_epoch": 2000,
|
| 388 |
+
"sync_bn": false,
|
| 389 |
+
"synchronize_step": true,
|
| 390 |
+
"teachers": [
|
| 391 |
+
{
|
| 392 |
+
"amp": true,
|
| 393 |
+
"amp_dtype": "bfloat16",
|
| 394 |
+
"batch_size": 16,
|
| 395 |
+
"data_dir": [
|
| 396 |
+
[
|
| 397 |
+
"/lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/datasets/captioning/datacomp/dc1b/stage2",
|
| 398 |
+
0.95
|
| 399 |
+
],
|
| 400 |
+
[
|
| 401 |
+
"/lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/datasets/segmentation/sam/stage1",
|
| 402 |
+
0.05
|
| 403 |
+
]
|
| 404 |
+
],
|
| 405 |
+
"fd_loss_weight": 1.0,
|
| 406 |
+
"fd_normalize": false,
|
| 407 |
+
"feature_distillation": true,
|
| 408 |
+
"input_size": 378,
|
| 409 |
+
"match_pre_proj": false,
|
| 410 |
+
"model": "ViT-H-14-378-quickgelu",
|
| 411 |
+
"name": "clip",
|
| 412 |
+
"pretrained": "dfn5b",
|
| 413 |
+
"sample_rate": 16,
|
| 414 |
+
"student_resolution": 432,
|
| 415 |
+
"summary_loss_weight": 1.0,
|
| 416 |
+
"torchcompile": true,
|
| 417 |
+
"type": "open_clip",
|
| 418 |
+
"vitdet_prob": 0.05,
|
| 419 |
+
"vitdet_window_sizes": [
|
| 420 |
+
3,
|
| 421 |
+
9,
|
| 422 |
+
9,
|
| 423 |
+
9
|
| 424 |
+
]
|
| 425 |
+
},
|
| 426 |
+
{
|
| 427 |
+
"amp": false,
|
| 428 |
+
"amp_dtype": "bfloat16",
|
| 429 |
+
"batch_size": 16,
|
| 430 |
+
"fd_loss_weight": 0.8,
|
| 431 |
+
"fd_normalize": false,
|
| 432 |
+
"feature_distillation": true,
|
| 433 |
+
"input_size": 336,
|
| 434 |
+
"match_pre_proj": false,
|
| 435 |
+
"model": "ViT-L/14@336px",
|
| 436 |
+
"name": "openai_clip",
|
| 437 |
+
"pretrained": "openai",
|
| 438 |
+
"sample_rate": 16,
|
| 439 |
+
"summary_loss_weight": 0.8,
|
| 440 |
+
"torchcompile": true,
|
| 441 |
+
"type": "openai_clip",
|
| 442 |
+
"use_summary": false
|
| 443 |
+
},
|
| 444 |
+
{
|
| 445 |
+
"amp": true,
|
| 446 |
+
"amp_dtype": "bfloat16",
|
| 447 |
+
"batch_size": 16,
|
| 448 |
+
"fd_loss_weight": 2.0,
|
| 449 |
+
"fd_normalize": false,
|
| 450 |
+
"feature_distillation": true,
|
| 451 |
+
"input_size": 378,
|
| 452 |
+
"model": "dinov2_vitg14_reg",
|
| 453 |
+
"name": "dino_v2",
|
| 454 |
+
"sample_rate": 16,
|
| 455 |
+
"summary_loss_weight": 1.0,
|
| 456 |
+
"torchcompile": true,
|
| 457 |
+
"type": "dino_v2"
|
| 458 |
+
},
|
| 459 |
+
{
|
| 460 |
+
"amp": false,
|
| 461 |
+
"batch_size": 2,
|
| 462 |
+
"data_dir": [
|
| 463 |
+
[
|
| 464 |
+
"/lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/datasets/segmentation/sam/stage1",
|
| 465 |
+
0.4
|
| 466 |
+
]
|
| 467 |
+
],
|
| 468 |
+
"fd_loss_fn": "MSE",
|
| 469 |
+
"fd_loss_weight": 0.25,
|
| 470 |
+
"fd_normalize": false,
|
| 471 |
+
"fd_ohem": true,
|
| 472 |
+
"feature_distillation": true,
|
| 473 |
+
"input_size": 1024,
|
| 474 |
+
"model": "vit-h",
|
| 475 |
+
"name": "sam",
|
| 476 |
+
"sample_rate": 2,
|
| 477 |
+
"student_resolution": 1024,
|
| 478 |
+
"summary_loss_weight": 1e-05,
|
| 479 |
+
"type": "sam",
|
| 480 |
+
"use_summary": false,
|
| 481 |
+
"vitdet_prob": 0.99,
|
| 482 |
+
"vitdet_window_sizes": [
|
| 483 |
+
8,
|
| 484 |
+
16,
|
| 485 |
+
16
|
| 486 |
+
]
|
| 487 |
+
},
|
| 488 |
+
{
|
| 489 |
+
"amp": true,
|
| 490 |
+
"batch_size": 2,
|
| 491 |
+
"data_dir": [
|
| 492 |
+
[
|
| 493 |
+
"/lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/datasets/ocr/publaynet/webdataset",
|
| 494 |
+
0.4
|
| 495 |
+
],
|
| 496 |
+
[
|
| 497 |
+
"/lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/datasets/ocr/staging/arxiv/hocr",
|
| 498 |
+
0.4
|
| 499 |
+
],
|
| 500 |
+
[
|
| 501 |
+
"/lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/datasets/ocr/scene-text/scene-text/text_ocr/webdataset",
|
| 502 |
+
0.15
|
| 503 |
+
],
|
| 504 |
+
[
|
| 505 |
+
"/lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/datasets/ocr/scene-text/scene-text/hiertext/webdataset",
|
| 506 |
+
0.05
|
| 507 |
+
]
|
| 508 |
+
],
|
| 509 |
+
"fd_loss_fn": "MSE",
|
| 510 |
+
"fd_loss_weight": 0.13,
|
| 511 |
+
"fd_normalize": false,
|
| 512 |
+
"fd_ohem": true,
|
| 513 |
+
"fd_upsample_factor": 4,
|
| 514 |
+
"feature_distillation": true,
|
| 515 |
+
"input_size": 1024,
|
| 516 |
+
"model": "quality",
|
| 517 |
+
"name": "rtx-translate",
|
| 518 |
+
"sample_rate": 2,
|
| 519 |
+
"student_resolution": 1024,
|
| 520 |
+
"summary_loss_weight": 1e-05,
|
| 521 |
+
"type": "rtx_translate",
|
| 522 |
+
"use_summary": false,
|
| 523 |
+
"vitdet_prob": 0.99,
|
| 524 |
+
"vitdet_window_sizes": [
|
| 525 |
+
8,
|
| 526 |
+
16,
|
| 527 |
+
16
|
| 528 |
+
]
|
| 529 |
+
}
|
| 530 |
+
],
|
| 531 |
+
"torchcompile": null,
|
| 532 |
+
"torchscript": false,
|
| 533 |
+
"train_interpolation": "random",
|
| 534 |
+
"train_split": "train",
|
| 535 |
+
"tta": 0,
|
| 536 |
+
"use_coco": false,
|
| 537 |
+
"use_multi_epochs_loader": false,
|
| 538 |
+
"val_data_dir": "/lustre/fsw/portfolios/llmservice/projects/llmservice_nlp_fm/datasets/classification/imagenet-1k/webdataset",
|
| 539 |
+
"val_ema_only": false,
|
| 540 |
+
"val_img_size": 432,
|
| 541 |
+
"val_jobs_script": "run_validation_jobs_vit-h-16.sh",
|
| 542 |
+
"val_split": "val",
|
| 543 |
+
"validation_batch_size": 64,
|
| 544 |
+
"vflip": 0.0,
|
| 545 |
+
"wandb_entity": "",
|
| 546 |
+
"wandb_group": "ohem",
|
| 547 |
+
"wandb_job_type": "",
|
| 548 |
+
"wandb_name": "",
|
| 549 |
+
"wandb_project": "",
|
| 550 |
+
"warmup_epochs": 0.5,
|
| 551 |
+
"warmup_lr": 1e-05,
|
| 552 |
+
"warmup_prefix": false,
|
| 553 |
+
"weight_decay": 0.02,
|
| 554 |
+
"worker_seeding": "all",
|
| 555 |
+
"workers": 8,
|
| 556 |
+
"world_size": 128
|
| 557 |
+
},
|
| 558 |
+
"auto_map": {
|
| 559 |
+
"AutoConfig": "configuraion_radio.RADIOConfig",
|
| 560 |
+
"AutoModel": "modeling_radio.RADIOModel"
|
| 561 |
+
},
|
| 562 |
+
"bad_words_ids": null,
|
| 563 |
+
"begin_suppress_tokens": null,
|
| 564 |
+
"bos_token_id": null,
|
| 565 |
+
"chunk_size_feed_forward": 0,
|
| 566 |
+
"cross_attention_hidden_size": null,
|
| 567 |
+
"decoder_start_token_id": null,
|
| 568 |
+
"diversity_penalty": 0.0,
|
| 569 |
+
"do_sample": false,
|
| 570 |
+
"early_stopping": false,
|
| 571 |
+
"encoder_no_repeat_ngram_size": 0,
|
| 572 |
+
"eos_token_id": null,
|
| 573 |
+
"exponential_decay_length_penalty": null,
|
| 574 |
+
"finetuning_task": null,
|
| 575 |
+
"forced_bos_token_id": null,
|
| 576 |
+
"forced_eos_token_id": null,
|
| 577 |
+
"id2label": {
|
| 578 |
+
"0": "LABEL_0",
|
| 579 |
+
"1": "LABEL_1"
|
| 580 |
+
},
|
| 581 |
+
"is_decoder": false,
|
| 582 |
+
"is_encoder_decoder": false,
|
| 583 |
+
"label2id": {
|
| 584 |
+
"LABEL_0": 0,
|
| 585 |
+
"LABEL_1": 1
|
| 586 |
+
},
|
| 587 |
+
"length_penalty": 1.0,
|
| 588 |
+
"max_length": 20,
|
| 589 |
+
"max_resolution": 2048,
|
| 590 |
+
"min_length": 0,
|
| 591 |
+
"model_type": "",
|
| 592 |
+
"no_repeat_ngram_size": 0,
|
| 593 |
+
"num_beam_groups": 1,
|
| 594 |
+
"num_beams": 1,
|
| 595 |
+
"num_return_sequences": 1,
|
| 596 |
+
"output_attentions": false,
|
| 597 |
+
"output_hidden_states": false,
|
| 598 |
+
"output_scores": false,
|
| 599 |
+
"pad_token_id": null,
|
| 600 |
+
"patch_size": 16,
|
| 601 |
+
"preferred_resolution": [
|
| 602 |
+
432,
|
| 603 |
+
432
|
| 604 |
+
],
|
| 605 |
+
"prefix": null,
|
| 606 |
+
"problem_type": null,
|
| 607 |
+
"pruned_heads": {},
|
| 608 |
+
"remove_invalid_values": false,
|
| 609 |
+
"repetition_penalty": 1.0,
|
| 610 |
+
"return_dict": true,
|
| 611 |
+
"return_dict_in_generate": false,
|
| 612 |
+
"sep_token_id": null,
|
| 613 |
+
"suppress_tokens": null,
|
| 614 |
+
"task_specific_params": null,
|
| 615 |
+
"temperature": 1.0,
|
| 616 |
+
"tf_legacy_loss": false,
|
| 617 |
+
"tie_encoder_decoder": false,
|
| 618 |
+
"tie_word_embeddings": true,
|
| 619 |
+
"tokenizer_class": null,
|
| 620 |
+
"top_k": 50,
|
| 621 |
+
"top_p": 1.0,
|
| 622 |
+
"torch_dtype": "bfloat16",
|
| 623 |
+
"torchscript": false,
|
| 624 |
+
"transformers_version": "4.47.1",
|
| 625 |
+
"typical_p": 1.0,
|
| 626 |
+
"use_bfloat16": false,
|
| 627 |
+
"version": "radio_v2.1",
|
| 628 |
+
"vitdet_window_size": null
|
| 629 |
+
},
|
| 630 |
+
"select_layer": -1,
|
| 631 |
+
"template": "phi3-chat",
|
| 632 |
+
"torch_dtype": "bfloat16",
|
| 633 |
+
"transformers_version": null,
|
| 634 |
+
"use_backbone_lora": 0,
|
| 635 |
+
"use_llm_lora": 0,
|
| 636 |
+
"use_thumbnail": true,
|
| 637 |
+
"vision_config": {
|
| 638 |
+
"_attn_implementation_autoset": false,
|
| 639 |
+
"_name_or_path": "",
|
| 640 |
+
"add_cross_attention": false,
|
| 641 |
+
"architectures": [
|
| 642 |
+
"InternVisionModel"
|
| 643 |
+
],
|
| 644 |
+
"attention_dropout": 0.0,
|
| 645 |
+
"bad_words_ids": null,
|
| 646 |
+
"begin_suppress_tokens": null,
|
| 647 |
+
"bos_token_id": null,
|
| 648 |
+
"chunk_size_feed_forward": 0,
|
| 649 |
+
"cross_attention_hidden_size": null,
|
| 650 |
+
"decoder_start_token_id": null,
|
| 651 |
+
"diversity_penalty": 0.0,
|
| 652 |
+
"do_sample": false,
|
| 653 |
+
"drop_path_rate": 0.0,
|
| 654 |
+
"dropout": 0.0,
|
| 655 |
+
"early_stopping": false,
|
| 656 |
+
"encoder_no_repeat_ngram_size": 0,
|
| 657 |
+
"eos_token_id": null,
|
| 658 |
+
"exponential_decay_length_penalty": null,
|
| 659 |
+
"finetuning_task": null,
|
| 660 |
+
"forced_bos_token_id": null,
|
| 661 |
+
"forced_eos_token_id": null,
|
| 662 |
+
"hidden_act": "gelu",
|
| 663 |
+
"hidden_size": 1024,
|
| 664 |
+
"id2label": {
|
| 665 |
+
"0": "LABEL_0",
|
| 666 |
+
"1": "LABEL_1"
|
| 667 |
+
},
|
| 668 |
+
"image_size": 448,
|
| 669 |
+
"initializer_factor": 1.0,
|
| 670 |
+
"initializer_range": 0.02,
|
| 671 |
+
"intermediate_size": 4096,
|
| 672 |
+
"is_decoder": false,
|
| 673 |
+
"is_encoder_decoder": false,
|
| 674 |
+
"label2id": {
|
| 675 |
+
"LABEL_0": 0,
|
| 676 |
+
"LABEL_1": 1
|
| 677 |
+
},
|
| 678 |
+
"layer_norm_eps": 1e-06,
|
| 679 |
+
"length_penalty": 1.0,
|
| 680 |
+
"max_length": 20,
|
| 681 |
+
"min_length": 0,
|
| 682 |
+
"model_type": "intern_vit_6b",
|
| 683 |
+
"no_repeat_ngram_size": 0,
|
| 684 |
+
"norm_type": "layer_norm",
|
| 685 |
+
"num_attention_heads": 16,
|
| 686 |
+
"num_beam_groups": 1,
|
| 687 |
+
"num_beams": 1,
|
| 688 |
+
"num_channels": 3,
|
| 689 |
+
"num_hidden_layers": 24,
|
| 690 |
+
"num_return_sequences": 1,
|
| 691 |
+
"output_attentions": false,
|
| 692 |
+
"output_hidden_states": false,
|
| 693 |
+
"output_scores": false,
|
| 694 |
+
"pad_token_id": null,
|
| 695 |
+
"patch_size": 14,
|
| 696 |
+
"prefix": null,
|
| 697 |
+
"problem_type": null,
|
| 698 |
+
"pruned_heads": {},
|
| 699 |
+
"qk_normalization": false,
|
| 700 |
+
"qkv_bias": true,
|
| 701 |
+
"remove_invalid_values": false,
|
| 702 |
+
"repetition_penalty": 1.0,
|
| 703 |
+
"return_dict": true,
|
| 704 |
+
"return_dict_in_generate": false,
|
| 705 |
+
"sep_token_id": null,
|
| 706 |
+
"suppress_tokens": null,
|
| 707 |
+
"task_specific_params": null,
|
| 708 |
+
"temperature": 1.0,
|
| 709 |
+
"tf_legacy_loss": false,
|
| 710 |
+
"tie_encoder_decoder": false,
|
| 711 |
+
"tie_word_embeddings": true,
|
| 712 |
+
"tokenizer_class": null,
|
| 713 |
+
"top_k": 50,
|
| 714 |
+
"top_p": 1.0,
|
| 715 |
+
"torch_dtype": "bfloat16",
|
| 716 |
+
"torchscript": false,
|
| 717 |
+
"transformers_version": "4.47.1",
|
| 718 |
+
"typical_p": 1.0,
|
| 719 |
+
"use_bfloat16": true,
|
| 720 |
+
"use_flash_attn": true
|
| 721 |
+
}
|
| 722 |
+
}
|
configuration_intern_vit.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# --------------------------------------------------------
|
| 2 |
+
# InternVL
|
| 3 |
+
# Copyright (c) 2024 OpenGVLab
|
| 4 |
+
# Licensed under The MIT License [see LICENSE for details]
|
| 5 |
+
# --------------------------------------------------------
|
| 6 |
+
|
| 7 |
+
import os
|
| 8 |
+
from typing import Union
|
| 9 |
+
|
| 10 |
+
from transformers.configuration_utils import PretrainedConfig
|
| 11 |
+
from transformers.utils import logging
|
| 12 |
+
|
| 13 |
+
logger = logging.get_logger(__name__)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class InternVisionConfig(PretrainedConfig):
|
| 17 |
+
r"""
|
| 18 |
+
This is the configuration class to store the configuration of a [`InternVisionModel`]. It is used to
|
| 19 |
+
instantiate a vision encoder according to the specified arguments, defining the model architecture.
|
| 20 |
+
|
| 21 |
+
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
| 22 |
+
documentation from [`PretrainedConfig`] for more information.
|
| 23 |
+
|
| 24 |
+
Args:
|
| 25 |
+
num_channels (`int`, *optional*, defaults to 3):
|
| 26 |
+
Number of color channels in the input images (e.g., 3 for RGB).
|
| 27 |
+
patch_size (`int`, *optional*, defaults to 14):
|
| 28 |
+
The size (resolution) of each patch.
|
| 29 |
+
image_size (`int`, *optional*, defaults to 224):
|
| 30 |
+
The size (resolution) of each image.
|
| 31 |
+
qkv_bias (`bool`, *optional*, defaults to `False`):
|
| 32 |
+
Whether to add a bias to the queries and values in the self-attention layers.
|
| 33 |
+
hidden_size (`int`, *optional*, defaults to 3200):
|
| 34 |
+
Dimensionality of the encoder layers and the pooler layer.
|
| 35 |
+
num_attention_heads (`int`, *optional*, defaults to 25):
|
| 36 |
+
Number of attention heads for each attention layer in the Transformer encoder.
|
| 37 |
+
intermediate_size (`int`, *optional*, defaults to 12800):
|
| 38 |
+
Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
|
| 39 |
+
qk_normalization (`bool`, *optional*, defaults to `True`):
|
| 40 |
+
Whether to normalize the queries and keys in the self-attention layers.
|
| 41 |
+
num_hidden_layers (`int`, *optional*, defaults to 48):
|
| 42 |
+
Number of hidden layers in the Transformer encoder.
|
| 43 |
+
use_flash_attn (`bool`, *optional*, defaults to `True`):
|
| 44 |
+
Whether to use flash attention mechanism.
|
| 45 |
+
hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
|
| 46 |
+
The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
|
| 47 |
+
`"relu"`, `"selu"` and `"gelu_new"` ``"gelu"` are supported.
|
| 48 |
+
layer_norm_eps (`float`, *optional*, defaults to 1e-6):
|
| 49 |
+
The epsilon used by the layer normalization layers.
|
| 50 |
+
dropout (`float`, *optional*, defaults to 0.0):
|
| 51 |
+
The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
|
| 52 |
+
drop_path_rate (`float`, *optional*, defaults to 0.0):
|
| 53 |
+
Dropout rate for stochastic depth.
|
| 54 |
+
attention_dropout (`float`, *optional*, defaults to 0.0):
|
| 55 |
+
The dropout ratio for the attention probabilities.
|
| 56 |
+
initializer_range (`float`, *optional*, defaults to 0.02):
|
| 57 |
+
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
| 58 |
+
initializer_factor (`float`, *optional*, defaults to 0.1):
|
| 59 |
+
A factor for layer scale.
|
| 60 |
+
"""
|
| 61 |
+
|
| 62 |
+
model_type = 'intern_vit_6b'
|
| 63 |
+
|
| 64 |
+
def __init__(
|
| 65 |
+
self,
|
| 66 |
+
num_channels=3,
|
| 67 |
+
patch_size=14,
|
| 68 |
+
image_size=224,
|
| 69 |
+
qkv_bias=False,
|
| 70 |
+
hidden_size=3200,
|
| 71 |
+
num_attention_heads=25,
|
| 72 |
+
intermediate_size=12800,
|
| 73 |
+
qk_normalization=True,
|
| 74 |
+
num_hidden_layers=48,
|
| 75 |
+
use_flash_attn=True,
|
| 76 |
+
hidden_act='gelu',
|
| 77 |
+
norm_type='rms_norm',
|
| 78 |
+
layer_norm_eps=1e-6,
|
| 79 |
+
dropout=0.0,
|
| 80 |
+
drop_path_rate=0.0,
|
| 81 |
+
attention_dropout=0.0,
|
| 82 |
+
initializer_range=0.02,
|
| 83 |
+
initializer_factor=0.1,
|
| 84 |
+
**kwargs,
|
| 85 |
+
):
|
| 86 |
+
super().__init__(**kwargs)
|
| 87 |
+
|
| 88 |
+
self.hidden_size = hidden_size
|
| 89 |
+
self.intermediate_size = intermediate_size
|
| 90 |
+
self.dropout = dropout
|
| 91 |
+
self.drop_path_rate = drop_path_rate
|
| 92 |
+
self.num_hidden_layers = num_hidden_layers
|
| 93 |
+
self.num_attention_heads = num_attention_heads
|
| 94 |
+
self.num_channels = num_channels
|
| 95 |
+
self.patch_size = patch_size
|
| 96 |
+
self.image_size = image_size
|
| 97 |
+
self.initializer_range = initializer_range
|
| 98 |
+
self.initializer_factor = initializer_factor
|
| 99 |
+
self.attention_dropout = attention_dropout
|
| 100 |
+
self.layer_norm_eps = layer_norm_eps
|
| 101 |
+
self.hidden_act = hidden_act
|
| 102 |
+
self.norm_type = norm_type
|
| 103 |
+
self.qkv_bias = qkv_bias
|
| 104 |
+
self.qk_normalization = qk_normalization
|
| 105 |
+
self.use_flash_attn = use_flash_attn
|
| 106 |
+
|
| 107 |
+
@classmethod
|
| 108 |
+
def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> 'PretrainedConfig':
|
| 109 |
+
config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
|
| 110 |
+
|
| 111 |
+
if 'vision_config' in config_dict:
|
| 112 |
+
config_dict = config_dict['vision_config']
|
| 113 |
+
|
| 114 |
+
if 'model_type' in config_dict and hasattr(cls, 'model_type') and config_dict['model_type'] != cls.model_type:
|
| 115 |
+
logger.warning(
|
| 116 |
+
f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
|
| 117 |
+
f'{cls.model_type}. This is not supported for all configurations of models and can yield errors.'
|
| 118 |
+
)
|
| 119 |
+
|
| 120 |
+
return cls.from_dict(config_dict, **kwargs)
|
configuration_internvl_chat.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# --------------------------------------------------------
|
| 2 |
+
# InternVL
|
| 3 |
+
# Copyright (c) 2024 OpenGVLab
|
| 4 |
+
# Licensed under The MIT License [see LICENSE for details]
|
| 5 |
+
# --------------------------------------------------------
|
| 6 |
+
|
| 7 |
+
import copy
|
| 8 |
+
|
| 9 |
+
from transformers import AutoConfig, LlamaConfig
|
| 10 |
+
from transformers.configuration_utils import PretrainedConfig
|
| 11 |
+
from transformers.utils import logging
|
| 12 |
+
|
| 13 |
+
from .configuration_intern_vit import InternVisionConfig
|
| 14 |
+
from .configuration_phi3 import Phi3Config
|
| 15 |
+
from .configuration_radio import RADIOConfig
|
| 16 |
+
|
| 17 |
+
logger = logging.get_logger(__name__)
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
class InternVLChatConfig(PretrainedConfig):
|
| 21 |
+
model_type = 'internvl_chat'
|
| 22 |
+
is_composition = True
|
| 23 |
+
|
| 24 |
+
def __init__(
|
| 25 |
+
self,
|
| 26 |
+
vision_config=None,
|
| 27 |
+
llm_config=None,
|
| 28 |
+
radio_config=None,
|
| 29 |
+
use_backbone_lora=0,
|
| 30 |
+
use_llm_lora=0,
|
| 31 |
+
select_layer=-1,
|
| 32 |
+
force_image_size=None,
|
| 33 |
+
downsample_ratio=0.5,
|
| 34 |
+
template=None,
|
| 35 |
+
dynamic_image_size=False,
|
| 36 |
+
use_thumbnail=False,
|
| 37 |
+
ps_version='v1',
|
| 38 |
+
min_dynamic_patch=1,
|
| 39 |
+
max_dynamic_patch=6,
|
| 40 |
+
**kwargs):
|
| 41 |
+
super().__init__(**kwargs)
|
| 42 |
+
|
| 43 |
+
if vision_config is None:
|
| 44 |
+
vision_config = {'architectures': ['InternVisionModel']}
|
| 45 |
+
logger.info('vision_config is None. Initializing the InternVisionConfig with default values.')
|
| 46 |
+
|
| 47 |
+
if llm_config is None:
|
| 48 |
+
llm_config = {'architectures': ['Phi3ForCausalLM']}
|
| 49 |
+
logger.info('llm_config is None. Initializing the LlamaConfig config with default values (`LlamaConfig`).')
|
| 50 |
+
|
| 51 |
+
if radio_config is None:
|
| 52 |
+
radio_config = {'architectures': ['RADIOModel']}
|
| 53 |
+
logger.info('radio_config is None. Initializing the RADIOConfig config with default values.')
|
| 54 |
+
|
| 55 |
+
self.vision_config = InternVisionConfig(**vision_config)
|
| 56 |
+
if llm_config.get('architectures')[0] == 'LlamaForCausalLM':
|
| 57 |
+
self.llm_config = LlamaConfig(**llm_config)
|
| 58 |
+
elif llm_config.get('architectures')[0] == 'Phi3ForCausalLM':
|
| 59 |
+
self.llm_config = Phi3Config(**llm_config)
|
| 60 |
+
else:
|
| 61 |
+
raise ValueError('Unsupported architecture: {}'.format(llm_config.get('architectures')[0]))
|
| 62 |
+
self.radio_config = RADIOConfig(**radio_config)
|
| 63 |
+
|
| 64 |
+
self.use_backbone_lora = use_backbone_lora
|
| 65 |
+
self.use_llm_lora = use_llm_lora
|
| 66 |
+
self.select_layer = select_layer
|
| 67 |
+
self.force_image_size = force_image_size
|
| 68 |
+
self.downsample_ratio = downsample_ratio
|
| 69 |
+
self.template = template
|
| 70 |
+
self.dynamic_image_size = dynamic_image_size
|
| 71 |
+
self.use_thumbnail = use_thumbnail
|
| 72 |
+
self.ps_version = ps_version # pixel shuffle version
|
| 73 |
+
self.min_dynamic_patch = min_dynamic_patch
|
| 74 |
+
self.max_dynamic_patch = max_dynamic_patch
|
| 75 |
+
|
| 76 |
+
logger.info(f'vision_select_layer: {self.select_layer}')
|
| 77 |
+
logger.info(f'ps_version: {self.ps_version}')
|
| 78 |
+
logger.info(f'min_dynamic_patch: {self.min_dynamic_patch}')
|
| 79 |
+
logger.info(f'max_dynamic_patch: {self.max_dynamic_patch}')
|
| 80 |
+
|
| 81 |
+
def to_dict(self):
|
| 82 |
+
"""
|
| 83 |
+
Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].
|
| 84 |
+
|
| 85 |
+
Returns:
|
| 86 |
+
`Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
|
| 87 |
+
"""
|
| 88 |
+
output = copy.deepcopy(self.__dict__)
|
| 89 |
+
output['vision_config'] = self.vision_config.to_dict()
|
| 90 |
+
output['llm_config'] = self.llm_config.to_dict()
|
| 91 |
+
output['radio_config'] = self.radio_config.to_dict()
|
| 92 |
+
output['model_type'] = self.__class__.model_type
|
| 93 |
+
output['use_backbone_lora'] = self.use_backbone_lora
|
| 94 |
+
output['use_llm_lora'] = self.use_llm_lora
|
| 95 |
+
output['select_layer'] = self.select_layer
|
| 96 |
+
output['force_image_size'] = self.force_image_size
|
| 97 |
+
output['downsample_ratio'] = self.downsample_ratio
|
| 98 |
+
output['template'] = self.template
|
| 99 |
+
output['dynamic_image_size'] = self.dynamic_image_size
|
| 100 |
+
output['use_thumbnail'] = self.use_thumbnail
|
| 101 |
+
output['ps_version'] = self.ps_version
|
| 102 |
+
output['min_dynamic_patch'] = self.min_dynamic_patch
|
| 103 |
+
output['max_dynamic_patch'] = self.max_dynamic_patch
|
| 104 |
+
|
| 105 |
+
return output
|
configuration_phi3.py
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License atd
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
""" Phi-3 model configuration"""
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
from transformers.configuration_utils import PretrainedConfig
|
| 19 |
+
from transformers.utils import logging
|
| 20 |
+
|
| 21 |
+
logger = logging.get_logger(__name__)
|
| 22 |
+
|
| 23 |
+
PHI3_PRETRAINED_CONFIG_ARCHIVE_MAP = {
|
| 24 |
+
'microsoft/Phi-3-mini-4k-instruct': 'https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/resolve/main/config.json',
|
| 25 |
+
'microsoft/Phi-3-mini-128k-instruct': 'https://huggingface.co/microsoft/Phi-3-mini-128k-instruct/resolve/main/config.json',
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class Phi3Config(PretrainedConfig):
|
| 30 |
+
r"""
|
| 31 |
+
This is the configuration class to store the configuration of a [`Phi3Model`]. It is used to instantiate a Phi-3
|
| 32 |
+
model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
|
| 33 |
+
defaults will yield a similar configuration to that of the
|
| 34 |
+
[microsoft/Phi-3-mini-4k-instruct](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct).
|
| 35 |
+
|
| 36 |
+
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
|
| 37 |
+
documentation from [`PretrainedConfig`] for more information.
|
| 38 |
+
|
| 39 |
+
Args:
|
| 40 |
+
vocab_size (`int`, *optional*, defaults to 32064):
|
| 41 |
+
Vocabulary size of the Phi-3 model. Defines the number of different tokens that can be represented by the
|
| 42 |
+
`inputs_ids` passed when calling [`Phi3Model`].
|
| 43 |
+
hidden_size (`int`, *optional*, defaults to 3072):
|
| 44 |
+
Dimension of the hidden representations.
|
| 45 |
+
intermediate_size (`int`, *optional*, defaults to 8192):
|
| 46 |
+
Dimension of the MLP representations.
|
| 47 |
+
num_hidden_layers (`int`, *optional*, defaults to 32):
|
| 48 |
+
Number of hidden layers in the Transformer decoder.
|
| 49 |
+
num_attention_heads (`int`, *optional*, defaults to 32):
|
| 50 |
+
Number of attention heads for each attention layer in the Transformer decoder.
|
| 51 |
+
num_key_value_heads (`int`, *optional*):
|
| 52 |
+
This is the number of key_value heads that should be used to implement Grouped Query Attention. If
|
| 53 |
+
`num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
|
| 54 |
+
`num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
|
| 55 |
+
converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
|
| 56 |
+
by meanpooling all the original heads within that group. For more details checkout [this
|
| 57 |
+
paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
|
| 58 |
+
`num_attention_heads`.
|
| 59 |
+
resid_pdrop (`float`, *optional*, defaults to 0.0):
|
| 60 |
+
Dropout probability for mlp outputs.
|
| 61 |
+
embd_pdrop (`int`, *optional*, defaults to 0.0):
|
| 62 |
+
The dropout ratio for the embeddings.
|
| 63 |
+
attention_dropout (`float`, *optional*, defaults to 0.0):
|
| 64 |
+
The dropout ratio after computing the attention scores.
|
| 65 |
+
hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
|
| 66 |
+
The non-linear activation function (function or string) in the decoder.
|
| 67 |
+
max_position_embeddings (`int`, *optional*, defaults to 4096):
|
| 68 |
+
The maximum sequence length that this model might ever be used with.
|
| 69 |
+
original_max_position_embeddings (`int`, *optional*, defaults to 4096):
|
| 70 |
+
The maximum sequence length that this model was trained with. This is used to determine the size of the
|
| 71 |
+
original RoPE embeddings when using long scaling.
|
| 72 |
+
initializer_range (`float`, *optional*, defaults to 0.02):
|
| 73 |
+
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
|
| 74 |
+
rms_norm_eps (`float`, *optional*, defaults to 1e-05):
|
| 75 |
+
The epsilon value used for the RMSNorm.
|
| 76 |
+
use_cache (`bool`, *optional*, defaults to `True`):
|
| 77 |
+
Whether or not the model should return the last key/values attentions (not used by all models). Only
|
| 78 |
+
relevant if `config.is_decoder=True`. Whether to tie weight embeddings or not.
|
| 79 |
+
tie_word_embeddings (`bool`, *optional*, defaults to `False`):
|
| 80 |
+
Whether to tie weight embeddings
|
| 81 |
+
rope_theta (`float`, *optional*, defaults to 10000.0):
|
| 82 |
+
The base period of the RoPE embeddings.
|
| 83 |
+
rope_scaling (`dict`, *optional*):
|
| 84 |
+
The scaling strategy for the RoPE embeddings. If `None`, no scaling is applied. If a dictionary, it must
|
| 85 |
+
contain the following keys: `type`, `short_factor` and `long_factor`. The `type` must be either `su` or `yarn` and
|
| 86 |
+
the `short_factor` and `long_factor` must be lists of numbers with the same length as the hidden size
|
| 87 |
+
divided by the number of attention heads divided by 2.
|
| 88 |
+
bos_token_id (`int`, *optional*, defaults to 1):
|
| 89 |
+
The id of the "beginning-of-sequence" token.
|
| 90 |
+
eos_token_id (`int`, *optional*, defaults to 32000):
|
| 91 |
+
The id of the "end-of-sequence" token.
|
| 92 |
+
pad_token_id (`int`, *optional*, defaults to 32000):
|
| 93 |
+
The id of the padding token.
|
| 94 |
+
sliding_window (`int`, *optional*):
|
| 95 |
+
Sliding window attention window size. If `None`, no sliding window is applied.
|
| 96 |
+
|
| 97 |
+
Example:
|
| 98 |
+
|
| 99 |
+
```python
|
| 100 |
+
>>> from transformers import Phi3Model, Phi3Config
|
| 101 |
+
|
| 102 |
+
>>> # Initializing a Phi-3 style configuration
|
| 103 |
+
>>> configuration = Phi3Config.from_pretrained("microsoft/Phi-3-mini-4k-instruct")
|
| 104 |
+
|
| 105 |
+
>>> # Initializing a model from the configuration
|
| 106 |
+
>>> model = Phi3Model(configuration)
|
| 107 |
+
|
| 108 |
+
>>> # Accessing the model configuration
|
| 109 |
+
>>> configuration = model.config
|
| 110 |
+
```"""
|
| 111 |
+
|
| 112 |
+
model_type = 'phi3'
|
| 113 |
+
keys_to_ignore_at_inference = ['past_key_values']
|
| 114 |
+
|
| 115 |
+
def __init__(
|
| 116 |
+
self,
|
| 117 |
+
vocab_size=32064,
|
| 118 |
+
hidden_size=3072,
|
| 119 |
+
intermediate_size=8192,
|
| 120 |
+
num_hidden_layers=32,
|
| 121 |
+
num_attention_heads=32,
|
| 122 |
+
num_key_value_heads=None,
|
| 123 |
+
resid_pdrop=0.0,
|
| 124 |
+
embd_pdrop=0.0,
|
| 125 |
+
attention_dropout=0.0,
|
| 126 |
+
hidden_act='silu',
|
| 127 |
+
max_position_embeddings=4096,
|
| 128 |
+
original_max_position_embeddings=4096,
|
| 129 |
+
initializer_range=0.02,
|
| 130 |
+
rms_norm_eps=1e-5,
|
| 131 |
+
use_cache=True,
|
| 132 |
+
tie_word_embeddings=False,
|
| 133 |
+
rope_theta=10000.0,
|
| 134 |
+
rope_scaling=None,
|
| 135 |
+
bos_token_id=1,
|
| 136 |
+
eos_token_id=32000,
|
| 137 |
+
pad_token_id=32000,
|
| 138 |
+
sliding_window=None,
|
| 139 |
+
**kwargs,
|
| 140 |
+
):
|
| 141 |
+
self.vocab_size = vocab_size
|
| 142 |
+
self.hidden_size = hidden_size
|
| 143 |
+
self.intermediate_size = intermediate_size
|
| 144 |
+
self.num_hidden_layers = num_hidden_layers
|
| 145 |
+
self.num_attention_heads = num_attention_heads
|
| 146 |
+
|
| 147 |
+
if num_key_value_heads is None:
|
| 148 |
+
num_key_value_heads = num_attention_heads
|
| 149 |
+
|
| 150 |
+
self.num_key_value_heads = num_key_value_heads
|
| 151 |
+
self.resid_pdrop = resid_pdrop
|
| 152 |
+
self.embd_pdrop = embd_pdrop
|
| 153 |
+
self.attention_dropout = attention_dropout
|
| 154 |
+
self.hidden_act = hidden_act
|
| 155 |
+
self.max_position_embeddings = max_position_embeddings
|
| 156 |
+
self.original_max_position_embeddings = original_max_position_embeddings
|
| 157 |
+
self.initializer_range = initializer_range
|
| 158 |
+
self.rms_norm_eps = rms_norm_eps
|
| 159 |
+
self.use_cache = use_cache
|
| 160 |
+
self.rope_theta = rope_theta
|
| 161 |
+
self.rope_scaling = rope_scaling
|
| 162 |
+
self._rope_scaling_validation()
|
| 163 |
+
self.sliding_window = sliding_window
|
| 164 |
+
|
| 165 |
+
super().__init__(
|
| 166 |
+
bos_token_id=bos_token_id,
|
| 167 |
+
eos_token_id=eos_token_id,
|
| 168 |
+
pad_token_id=pad_token_id,
|
| 169 |
+
tie_word_embeddings=tie_word_embeddings,
|
| 170 |
+
**kwargs,
|
| 171 |
+
)
|
| 172 |
+
|
| 173 |
+
def _rope_scaling_validation(self):
|
| 174 |
+
"""
|
| 175 |
+
Validate the `rope_scaling` configuration.
|
| 176 |
+
"""
|
| 177 |
+
if self.rope_scaling is None:
|
| 178 |
+
return
|
| 179 |
+
|
| 180 |
+
if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 3:
|
| 181 |
+
raise ValueError(
|
| 182 |
+
'`rope_scaling` must be a dictionary with three fields, `type`, `short_factor` and `long_factor`, '
|
| 183 |
+
f'got {self.rope_scaling}'
|
| 184 |
+
)
|
| 185 |
+
rope_scaling_type = self.rope_scaling.get('type', None)
|
| 186 |
+
rope_scaling_short_factor = self.rope_scaling.get('short_factor', None)
|
| 187 |
+
rope_scaling_long_factor = self.rope_scaling.get('long_factor', None)
|
| 188 |
+
if rope_scaling_type is None or rope_scaling_type not in ['su', 'yarn']:
|
| 189 |
+
raise ValueError(f"`rope_scaling`'s type field must be one of ['su', 'yarn'], got {rope_scaling_type}")
|
| 190 |
+
if not (
|
| 191 |
+
isinstance(rope_scaling_short_factor, list)
|
| 192 |
+
and all(isinstance(x, (int, float)) for x in rope_scaling_short_factor)
|
| 193 |
+
):
|
| 194 |
+
raise ValueError(
|
| 195 |
+
f"`rope_scaling`'s short_factor field must be a list of numbers, got {rope_scaling_short_factor}"
|
| 196 |
+
)
|
| 197 |
+
if not len(rope_scaling_short_factor) == self.hidden_size // self.num_attention_heads // 2:
|
| 198 |
+
raise ValueError(
|
| 199 |
+
f"`rope_scaling`'s short_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_short_factor)}"
|
| 200 |
+
)
|
| 201 |
+
if not (
|
| 202 |
+
isinstance(rope_scaling_long_factor, list)
|
| 203 |
+
and all(isinstance(x, (int, float)) for x in rope_scaling_long_factor)
|
| 204 |
+
):
|
| 205 |
+
raise ValueError(
|
| 206 |
+
f"`rope_scaling`'s long_factor field must be a list of numbers, got {rope_scaling_long_factor}"
|
| 207 |
+
)
|
| 208 |
+
if not len(rope_scaling_long_factor) == self.hidden_size // self.num_attention_heads // 2:
|
| 209 |
+
raise ValueError(
|
| 210 |
+
f"`rope_scaling`'s long_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_long_factor)}"
|
| 211 |
+
)
|
configuration_radio.py
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2023-2024, NVIDIA CORPORATION. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
from typing import Optional, List, Union, NamedTuple
|
| 15 |
+
import torch
|
| 16 |
+
from transformers import PretrainedConfig
|
| 17 |
+
|
| 18 |
+
from .radio_common import RESOURCE_MAP, DEFAULT_VERSION
|
| 19 |
+
|
| 20 |
+
from .radio_model import Resolution
|
| 21 |
+
|
| 22 |
+
class RADIOConfig(PretrainedConfig):
|
| 23 |
+
"""Pretrained Hugging Face configuration for RADIO models."""
|
| 24 |
+
|
| 25 |
+
def __init__(
|
| 26 |
+
self,
|
| 27 |
+
args: Optional[dict] = None,
|
| 28 |
+
version: Optional[str] = DEFAULT_VERSION,
|
| 29 |
+
patch_size: Optional[int] = None,
|
| 30 |
+
max_resolution: Optional[int] = None,
|
| 31 |
+
preferred_resolution: Optional[Resolution] = None,
|
| 32 |
+
adaptor_names: Union[str, List[str]] = None,
|
| 33 |
+
vitdet_window_size: Optional[int] = None,
|
| 34 |
+
**kwargs,
|
| 35 |
+
):
|
| 36 |
+
self.args = args
|
| 37 |
+
for field in ["dtype", "amp_dtype"]:
|
| 38 |
+
if self.args is not None and field in self.args:
|
| 39 |
+
# Convert to a string in order to make it serializable.
|
| 40 |
+
# For example for torch.float32 we will store "float32",
|
| 41 |
+
# for "bfloat16" we will store "bfloat16".
|
| 42 |
+
self.args[field] = str(args[field]).split(".")[-1]
|
| 43 |
+
self.version = version
|
| 44 |
+
resource = RESOURCE_MAP[version]
|
| 45 |
+
self.patch_size = patch_size or resource.patch_size
|
| 46 |
+
self.max_resolution = max_resolution or resource.max_resolution
|
| 47 |
+
self.preferred_resolution = (
|
| 48 |
+
preferred_resolution or resource.preferred_resolution
|
| 49 |
+
)
|
| 50 |
+
self.adaptor_names = adaptor_names
|
| 51 |
+
self.vitdet_window_size = vitdet_window_size
|
| 52 |
+
super().__init__(**kwargs)
|
conversation.py
ADDED
|
@@ -0,0 +1,391 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Conversation prompt templates.
|
| 3 |
+
|
| 4 |
+
We kindly request that you import fastchat instead of copying this file if you wish to use it.
|
| 5 |
+
If you have changes in mind, please contribute back so the community can benefit collectively and continue to maintain these valuable templates.
|
| 6 |
+
|
| 7 |
+
Modified from https://github.com/lm-sys/FastChat/blob/main/fastchat/conversation.py
|
| 8 |
+
"""
|
| 9 |
+
|
| 10 |
+
import dataclasses
|
| 11 |
+
from enum import IntEnum, auto
|
| 12 |
+
from typing import Dict, List, Tuple, Union
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class SeparatorStyle(IntEnum):
|
| 16 |
+
"""Separator styles."""
|
| 17 |
+
|
| 18 |
+
ADD_COLON_SINGLE = auto()
|
| 19 |
+
ADD_COLON_TWO = auto()
|
| 20 |
+
ADD_COLON_SPACE_SINGLE = auto()
|
| 21 |
+
NO_COLON_SINGLE = auto()
|
| 22 |
+
NO_COLON_TWO = auto()
|
| 23 |
+
ADD_NEW_LINE_SINGLE = auto()
|
| 24 |
+
LLAMA2 = auto()
|
| 25 |
+
CHATGLM = auto()
|
| 26 |
+
CHATML = auto()
|
| 27 |
+
CHATINTERN = auto()
|
| 28 |
+
DOLLY = auto()
|
| 29 |
+
RWKV = auto()
|
| 30 |
+
PHOENIX = auto()
|
| 31 |
+
ROBIN = auto()
|
| 32 |
+
FALCON_CHAT = auto()
|
| 33 |
+
CHATGLM3 = auto()
|
| 34 |
+
INTERNVL_ZH = auto()
|
| 35 |
+
MPT = auto()
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
@dataclasses.dataclass
|
| 39 |
+
class Conversation:
|
| 40 |
+
"""A class that manages prompt templates and keeps all conversation history."""
|
| 41 |
+
|
| 42 |
+
# The name of this template
|
| 43 |
+
name: str
|
| 44 |
+
# The template of the system prompt
|
| 45 |
+
system_template: str = '{system_message}'
|
| 46 |
+
# The system message
|
| 47 |
+
system_message: str = ''
|
| 48 |
+
# The names of two roles
|
| 49 |
+
roles: Tuple[str] = ('USER', 'ASSISTANT')
|
| 50 |
+
# All messages. Each item is (role, message).
|
| 51 |
+
messages: List[List[str]] = ()
|
| 52 |
+
# The number of few shot examples
|
| 53 |
+
offset: int = 0
|
| 54 |
+
# The separator style and configurations
|
| 55 |
+
sep_style: SeparatorStyle = SeparatorStyle.ADD_COLON_SINGLE
|
| 56 |
+
sep: str = '\n'
|
| 57 |
+
sep2: str = None
|
| 58 |
+
# Stop criteria (the default one is EOS token)
|
| 59 |
+
stop_str: Union[str, List[str]] = None
|
| 60 |
+
# Stops generation if meeting any token in this list
|
| 61 |
+
stop_token_ids: List[int] = None
|
| 62 |
+
|
| 63 |
+
def get_prompt(self) -> str:
|
| 64 |
+
"""Get the prompt for generation."""
|
| 65 |
+
system_prompt = self.system_template.format(system_message=self.system_message)
|
| 66 |
+
if self.sep_style == SeparatorStyle.ADD_COLON_SINGLE:
|
| 67 |
+
ret = system_prompt + self.sep
|
| 68 |
+
for role, message in self.messages:
|
| 69 |
+
if message:
|
| 70 |
+
ret += role + ': ' + message + self.sep
|
| 71 |
+
else:
|
| 72 |
+
ret += role + ':'
|
| 73 |
+
return ret
|
| 74 |
+
elif self.sep_style == SeparatorStyle.ADD_COLON_TWO:
|
| 75 |
+
seps = [self.sep, self.sep2]
|
| 76 |
+
ret = system_prompt + seps[0]
|
| 77 |
+
for i, (role, message) in enumerate(self.messages):
|
| 78 |
+
if message:
|
| 79 |
+
ret += role + ': ' + message + seps[i % 2]
|
| 80 |
+
else:
|
| 81 |
+
ret += role + ':'
|
| 82 |
+
return ret
|
| 83 |
+
elif self.sep_style == SeparatorStyle.ADD_COLON_SPACE_SINGLE:
|
| 84 |
+
ret = system_prompt + self.sep
|
| 85 |
+
for role, message in self.messages:
|
| 86 |
+
if message:
|
| 87 |
+
ret += role + ': ' + message + self.sep
|
| 88 |
+
else:
|
| 89 |
+
ret += role + ': ' # must be end with a space
|
| 90 |
+
return ret
|
| 91 |
+
elif self.sep_style == SeparatorStyle.ADD_NEW_LINE_SINGLE:
|
| 92 |
+
ret = '' if system_prompt == '' else system_prompt + self.sep
|
| 93 |
+
for role, message in self.messages:
|
| 94 |
+
if message:
|
| 95 |
+
ret += role + '\n' + message + self.sep
|
| 96 |
+
else:
|
| 97 |
+
ret += role + '\n'
|
| 98 |
+
return ret
|
| 99 |
+
elif self.sep_style == SeparatorStyle.NO_COLON_SINGLE:
|
| 100 |
+
ret = system_prompt
|
| 101 |
+
for role, message in self.messages:
|
| 102 |
+
if message:
|
| 103 |
+
ret += role + message + self.sep
|
| 104 |
+
else:
|
| 105 |
+
ret += role
|
| 106 |
+
return ret
|
| 107 |
+
elif self.sep_style == SeparatorStyle.NO_COLON_TWO:
|
| 108 |
+
seps = [self.sep, self.sep2]
|
| 109 |
+
ret = system_prompt
|
| 110 |
+
for i, (role, message) in enumerate(self.messages):
|
| 111 |
+
if message:
|
| 112 |
+
ret += role + message + seps[i % 2]
|
| 113 |
+
else:
|
| 114 |
+
ret += role
|
| 115 |
+
return ret
|
| 116 |
+
elif self.sep_style == SeparatorStyle.RWKV:
|
| 117 |
+
ret = system_prompt
|
| 118 |
+
for i, (role, message) in enumerate(self.messages):
|
| 119 |
+
if message:
|
| 120 |
+
ret += (
|
| 121 |
+
role
|
| 122 |
+
+ ': '
|
| 123 |
+
+ message.replace('\r\n', '\n').replace('\n\n', '\n')
|
| 124 |
+
)
|
| 125 |
+
ret += '\n\n'
|
| 126 |
+
else:
|
| 127 |
+
ret += role + ':'
|
| 128 |
+
return ret
|
| 129 |
+
elif self.sep_style == SeparatorStyle.LLAMA2:
|
| 130 |
+
seps = [self.sep, self.sep2]
|
| 131 |
+
if self.system_message:
|
| 132 |
+
ret = system_prompt
|
| 133 |
+
else:
|
| 134 |
+
ret = '[INST] '
|
| 135 |
+
for i, (role, message) in enumerate(self.messages):
|
| 136 |
+
tag = self.roles[i % 2]
|
| 137 |
+
if message:
|
| 138 |
+
if i == 0:
|
| 139 |
+
ret += message + ' '
|
| 140 |
+
else:
|
| 141 |
+
ret += tag + ' ' + message + seps[i % 2]
|
| 142 |
+
else:
|
| 143 |
+
ret += tag
|
| 144 |
+
return ret
|
| 145 |
+
elif self.sep_style == SeparatorStyle.CHATGLM:
|
| 146 |
+
# source: https://huggingface.co/THUDM/chatglm-6b/blob/1d240ba371910e9282298d4592532d7f0f3e9f3e/modeling_chatglm.py#L1302-L1308
|
| 147 |
+
# source2: https://huggingface.co/THUDM/chatglm2-6b/blob/e186c891cf64310ac66ef10a87e6635fa6c2a579/modeling_chatglm.py#L926
|
| 148 |
+
round_add_n = 1 if self.name == 'chatglm2' else 0
|
| 149 |
+
if system_prompt:
|
| 150 |
+
ret = system_prompt + self.sep
|
| 151 |
+
else:
|
| 152 |
+
ret = ''
|
| 153 |
+
|
| 154 |
+
for i, (role, message) in enumerate(self.messages):
|
| 155 |
+
if i % 2 == 0:
|
| 156 |
+
ret += f'[Round {i//2 + round_add_n}]{self.sep}'
|
| 157 |
+
|
| 158 |
+
if message:
|
| 159 |
+
ret += f'{role}：{message}{self.sep}'
|
| 160 |
+
else:
|
| 161 |
+
ret += f'{role}：'
|
| 162 |
+
return ret
|
| 163 |
+
elif self.sep_style == SeparatorStyle.CHATML:
|
| 164 |
+
ret = '' if system_prompt == '' else system_prompt + self.sep + '\n'
|
| 165 |
+
for role, message in self.messages:
|
| 166 |
+
if message:
|
| 167 |
+
ret += role + '\n' + message + self.sep + '\n'
|
| 168 |
+
else:
|
| 169 |
+
ret += role + '\n'
|
| 170 |
+
return ret
|
| 171 |
+
elif self.sep_style == SeparatorStyle.CHATGLM3:
|
| 172 |
+
ret = ''
|
| 173 |
+
if self.system_message:
|
| 174 |
+
ret += system_prompt
|
| 175 |
+
for role, message in self.messages:
|
| 176 |
+
if message:
|
| 177 |
+
ret += role + '\n' + ' ' + message
|
| 178 |
+
else:
|
| 179 |
+
ret += role
|
| 180 |
+
return ret
|
| 181 |
+
elif self.sep_style == SeparatorStyle.CHATINTERN:
|
| 182 |
+
# source: https://huggingface.co/internlm/internlm-chat-7b-8k/blob/bd546fa984b4b0b86958f56bf37f94aa75ab8831/modeling_internlm.py#L771
|
| 183 |
+
seps = [self.sep, self.sep2]
|
| 184 |
+
ret = system_prompt
|
| 185 |
+
for i, (role, message) in enumerate(self.messages):
|
| 186 |
+
# if i % 2 == 0:
|
| 187 |
+
# ret += "<s>"
|
| 188 |
+
if message:
|
| 189 |
+
ret += role + ':' + message + seps[i % 2] + '\n'
|
| 190 |
+
else:
|
| 191 |
+
ret += role + ':'
|
| 192 |
+
return ret
|
| 193 |
+
elif self.sep_style == SeparatorStyle.DOLLY:
|
| 194 |
+
seps = [self.sep, self.sep2]
|
| 195 |
+
ret = system_prompt
|
| 196 |
+
for i, (role, message) in enumerate(self.messages):
|
| 197 |
+
if message:
|
| 198 |
+
ret += role + ':\n' + message + seps[i % 2]
|
| 199 |
+
if i % 2 == 1:
|
| 200 |
+
ret += '\n\n'
|
| 201 |
+
else:
|
| 202 |
+
ret += role + ':\n'
|
| 203 |
+
return ret
|
| 204 |
+
elif self.sep_style == SeparatorStyle.PHOENIX:
|
| 205 |
+
ret = system_prompt
|
| 206 |
+
for role, message in self.messages:
|
| 207 |
+
if message:
|
| 208 |
+
ret += role + ': ' + '<s>' + message + '</s>'
|
| 209 |
+
else:
|
| 210 |
+
ret += role + ': ' + '<s>'
|
| 211 |
+
return ret
|
| 212 |
+
elif self.sep_style == SeparatorStyle.ROBIN:
|
| 213 |
+
ret = system_prompt + self.sep
|
| 214 |
+
for role, message in self.messages:
|
| 215 |
+
if message:
|
| 216 |
+
ret += role + ':\n' + message + self.sep
|
| 217 |
+
else:
|
| 218 |
+
ret += role + ':\n'
|
| 219 |
+
return ret
|
| 220 |
+
elif self.sep_style == SeparatorStyle.FALCON_CHAT:
|
| 221 |
+
ret = ''
|
| 222 |
+
if self.system_message:
|
| 223 |
+
ret += system_prompt + self.sep
|
| 224 |
+
for role, message in self.messages:
|
| 225 |
+
if message:
|
| 226 |
+
ret += role + ': ' + message + self.sep
|
| 227 |
+
else:
|
| 228 |
+
ret += role + ':'
|
| 229 |
+
|
| 230 |
+
return ret
|
| 231 |
+
elif self.sep_style == SeparatorStyle.INTERNVL_ZH:
|
| 232 |
+
seps = [self.sep, self.sep2]
|
| 233 |
+
ret = self.system_message + seps[0]
|
| 234 |
+
for i, (role, message) in enumerate(self.messages):
|
| 235 |
+
if message:
|
| 236 |
+
ret += role + ': ' + message + seps[i % 2]
|
| 237 |
+
else:
|
| 238 |
+
ret += role + ':'
|
| 239 |
+
return ret
|
| 240 |
+
elif self.sep_style == SeparatorStyle.MPT:
|
| 241 |
+
ret = system_prompt + self.sep
|
| 242 |
+
for role, message in self.messages:
|
| 243 |
+
if message:
|
| 244 |
+
if type(message) is tuple:
|
| 245 |
+
message, _, _ = message
|
| 246 |
+
ret += role + message + self.sep
|
| 247 |
+
else:
|
| 248 |
+
ret += role
|
| 249 |
+
return ret
|
| 250 |
+
else:
|
| 251 |
+
raise ValueError(f'Invalid style: {self.sep_style}')
|
| 252 |
+
|
| 253 |
+
def set_system_message(self, system_message: str):
|
| 254 |
+
"""Set the system message."""
|
| 255 |
+
self.system_message = system_message
|
| 256 |
+
|
| 257 |
+
def append_message(self, role: str, message: str):
|
| 258 |
+
"""Append a new message."""
|
| 259 |
+
self.messages.append([role, message])
|
| 260 |
+
|
| 261 |
+
def update_last_message(self, message: str):
|
| 262 |
+
"""Update the last output.
|
| 263 |
+
|
| 264 |
+
The last message is typically set to be None when constructing the prompt,
|
| 265 |
+
so we need to update it in-place after getting the response from a model.
|
| 266 |
+
"""
|
| 267 |
+
self.messages[-1][1] = message
|
| 268 |
+
|
| 269 |
+
def to_gradio_chatbot(self):
|
| 270 |
+
"""Convert the conversation to gradio chatbot format."""
|
| 271 |
+
ret = []
|
| 272 |
+
for i, (role, msg) in enumerate(self.messages[self.offset :]):
|
| 273 |
+
if i % 2 == 0:
|
| 274 |
+
ret.append([msg, None])
|
| 275 |
+
else:
|
| 276 |
+
ret[-1][-1] = msg
|
| 277 |
+
return ret
|
| 278 |
+
|
| 279 |
+
def to_openai_api_messages(self):
|
| 280 |
+
"""Convert the conversation to OpenAI chat completion format."""
|
| 281 |
+
ret = [{'role': 'system', 'content': self.system_message}]
|
| 282 |
+
|
| 283 |
+
for i, (_, msg) in enumerate(self.messages[self.offset :]):
|
| 284 |
+
if i % 2 == 0:
|
| 285 |
+
ret.append({'role': 'user', 'content': msg})
|
| 286 |
+
else:
|
| 287 |
+
if msg is not None:
|
| 288 |
+
ret.append({'role': 'assistant', 'content': msg})
|
| 289 |
+
return ret
|
| 290 |
+
|
| 291 |
+
def copy(self):
|
| 292 |
+
return Conversation(
|
| 293 |
+
name=self.name,
|
| 294 |
+
system_template=self.system_template,
|
| 295 |
+
system_message=self.system_message,
|
| 296 |
+
roles=self.roles,
|
| 297 |
+
messages=[[x, y] for x, y in self.messages],
|
| 298 |
+
offset=self.offset,
|
| 299 |
+
sep_style=self.sep_style,
|
| 300 |
+
sep=self.sep,
|
| 301 |
+
sep2=self.sep2,
|
| 302 |
+
stop_str=self.stop_str,
|
| 303 |
+
stop_token_ids=self.stop_token_ids,
|
| 304 |
+
)
|
| 305 |
+
|
| 306 |
+
def dict(self):
|
| 307 |
+
return {
|
| 308 |
+
'template_name': self.name,
|
| 309 |
+
'system_message': self.system_message,
|
| 310 |
+
'roles': self.roles,
|
| 311 |
+
'messages': self.messages,
|
| 312 |
+
'offset': self.offset,
|
| 313 |
+
}
|
| 314 |
+
|
| 315 |
+
|
| 316 |
+
# A global registry for all conversation templates
|
| 317 |
+
conv_templates: Dict[str, Conversation] = {}
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
def register_conv_template(template: Conversation, override: bool = False):
|
| 321 |
+
"""Register a new conversation template."""
|
| 322 |
+
if not override:
|
| 323 |
+
assert (
|
| 324 |
+
template.name not in conv_templates
|
| 325 |
+
), f'{template.name} has been registered.'
|
| 326 |
+
|
| 327 |
+
conv_templates[template.name] = template
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
def get_conv_template(name: str) -> Conversation:
|
| 331 |
+
"""Get a conversation template."""
|
| 332 |
+
return conv_templates[name].copy()
|
| 333 |
+
|
| 334 |
+
|
| 335 |
+
# Both Hermes-2 and internlm2-chat are chatml-format conversation templates. The difference
|
| 336 |
+
# is that during training, the preprocessing function for the Hermes-2 template doesn't add
|
| 337 |
+
# <s> at the beginning of the tokenized sequence, while the internlm2-chat template does.
|
| 338 |
+
# Therefore, they are completely equivalent during inference.
|
| 339 |
+
register_conv_template(
|
| 340 |
+
Conversation(
|
| 341 |
+
name='Hermes-2',
|
| 342 |
+
system_template='<|im_start|>system\n{system_message}',
|
| 343 |
+
# note: The new system prompt was not used here to avoid changes in benchmark performance.
|
| 344 |
+
# system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。',
|
| 345 |
+
system_message='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。',
|
| 346 |
+
roles=('<|im_start|>user\n', '<|im_start|>assistant\n'),
|
| 347 |
+
sep_style=SeparatorStyle.MPT,
|
| 348 |
+
sep='<|im_end|>',
|
| 349 |
+
stop_str='<|endoftext|>',
|
| 350 |
+
)
|
| 351 |
+
)
|
| 352 |
+
|
| 353 |
+
|
| 354 |
+
register_conv_template(
|
| 355 |
+
Conversation(
|
| 356 |
+
name='internlm2-chat',
|
| 357 |
+
system_template='<|im_start|>system\n{system_message}',
|
| 358 |
+
# note: The new system prompt was not used here to avoid changes in benchmark performance.
|
| 359 |
+
# system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。',
|
| 360 |
+
system_message='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。',
|
| 361 |
+
roles=('<|im_start|>user\n', '<|im_start|>assistant\n'),
|
| 362 |
+
sep_style=SeparatorStyle.MPT,
|
| 363 |
+
sep='<|im_end|>',
|
| 364 |
+
)
|
| 365 |
+
)
|
| 366 |
+
|
| 367 |
+
|
| 368 |
+
register_conv_template(
|
| 369 |
+
Conversation(
|
| 370 |
+
name='phi3-chat',
|
| 371 |
+
system_template='<|system|>\n{system_message}',
|
| 372 |
+
# note: The new system prompt was not used here to avoid changes in benchmark performance.
|
| 373 |
+
# system_message='我是书生·万象，英文名是InternVL，是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。',
|
| 374 |
+
system_message='你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。',
|
| 375 |
+
roles=('<|user|>\n', '<|assistant|>\n'),
|
| 376 |
+
sep_style=SeparatorStyle.MPT,
|
| 377 |
+
sep='<|end|>',
|
| 378 |
+
)
|
| 379 |
+
)
|
| 380 |
+
|
| 381 |
+
|
| 382 |
+
register_conv_template(
|
| 383 |
+
Conversation(
|
| 384 |
+
name='internvl2_5',
|
| 385 |
+
system_template='<|im_start|>system\n{system_message}',
|
| 386 |
+
system_message='你是书生·万象，英文名是InternVL，是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。',
|
| 387 |
+
roles=('<|im_start|>user\n', '<|im_start|>assistant\n'),
|
| 388 |
+
sep_style=SeparatorStyle.MPT,
|
| 389 |
+
sep='<|im_end|>\n',
|
| 390 |
+
)
|
| 391 |
+
)
|
examples/image1.jpg
ADDED
|
examples/image2.jpg
ADDED
|
examples/match_case/FRAME00.jpg
ADDED
|
examples/match_case/FRAME00.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
[{"height": 720, "width": 1280, "segmentation": [[891.0, 330.0, 886.0, 324.0, 879.0, 320.0, 870.0, 320.0, 847.0, 326.0, 816.0, 326.0, 788.0, 338.0, 751.0, 361.0, 734.0, 365.0, 713.0, 377.0, 685.0, 404.0, 656.0, 403.0, 652.0, 400.0, 635.0, 404.0, 626.0, 397.0, 626.0, 403.0, 630.0, 409.0, 639.0, 412.0, 646.0, 412.0, 654.0, 420.0, 651.0, 439.0, 655.0, 454.0, 655.0, 464.0, 648.0, 477.0, 650.0, 485.0, 660.0, 490.0, 668.0, 490.0, 672.0, 485.0, 682.0, 485.0, 691.0, 493.0, 702.0, 495.0, 711.0, 493.0, 721.0, 507.0, 721.0, 518.0, 716.0, 531.0, 716.0, 540.0, 722.0, 545.0, 729.0, 544.0, 734.0, 537.0, 732.0, 507.0, 734.0, 504.0, 771.0, 486.0, 776.0, 486.0, 784.0, 498.0, 786.0, 509.0, 796.0, 521.0, 801.0, 542.0, 809.0, 543.0, 815.0, 538.0, 816.0, 533.0, 813.0, 526.0, 804.0, 515.0, 802.0, 509.0, 801.0, 468.0, 804.0, 464.0, 838.0, 453.0, 842.0, 462.0, 838.0, 486.0, 838.0, 493.0, 842.0, 499.0, 841.0, 508.0, 846.0, 512.0, 854.0, 512.0, 857.0, 506.0, 859.0, 484.0, 869.0, 457.0, 870.0, 448.0, 876.0, 435.0, 882.0, 427.0, 886.0, 416.0, 888.0, 414.0, 893.0, 419.0, 897.0, 444.0, 901.0, 447.0, 908.0, 446.0, 909.0, 441.0, 905.0, 433.0, 903.0, 420.0, 895.0, 402.0, 898.0, 357.0, 894.0, 348.0]]}]
|
examples/match_case/FRAME00_ORI.jpg
ADDED
|
examples/match_case/FRAME01_CAND.jpg
ADDED
|
examples/match_case/FRAME01_CAND.json
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
[{"id": 2, "height": 720, "width": 1280, "segmentation": [[548.0, 243.0, 537.0, 258.0, 523.0, 259.0, 517.0, 255.0, 514.0, 246.0, 504.0, 238.0, 507.0, 226.0, 498.0, 240.0, 488.0, 243.0, 478.0, 254.0, 460.0, 254.0, 450.0, 249.0, 435.0, 246.0, 419.0, 247.0, 376.0, 261.0, 350.0, 277.0, 337.0, 279.0, 326.0, 284.0, 313.0, 285.0, 278.0, 297.0, 268.0, 307.0, 258.0, 331.0, 254.0, 376.0, 260.0, 423.0, 274.0, 484.0, 277.0, 506.0, 276.0, 535.0, 268.0, 557.0, 268.0, 563.0, 272.0, 569.0, 291.0, 566.0, 290.0, 525.0, 295.0, 514.0, 298.0, 498.0, 303.0, 494.0, 322.0, 501.0, 330.0, 510.0, 338.0, 538.0, 338.0, 565.0, 341.0, 570.0, 353.0, 576.0, 362.0, 573.0, 366.0, 569.0, 365.0, 562.0, 359.0, 554.0, 355.0, 539.0, 355.0, 504.0, 374.0, 455.0, 398.0, 440.0, 402.0, 439.0, 406.0, 441.0, 410.0, 448.0, 416.0, 471.0, 416.0, 498.0, 419.0, 505.0, 428.0, 509.0, 440.0, 504.0, 441.0, 501.0, 431.0, 479.0, 431.0, 467.0, 437.0, 432.0, 447.0, 406.0, 449.0, 380.0, 452.0, 377.0, 462.0, 379.0, 475.0, 373.0, 487.0, 360.0, 490.0, 350.0, 493.0, 347.0, 515.0, 341.0, 530.0, 342.0, 544.0, 332.0, 546.0, 320.0, 535.0, 303.0, 531.0, 280.0, 526.0, 271.0, 529.0, 267.0, 543.0, 260.0, 548.0, 252.0]]}, {"id": 3, "height": 720, "width": 1280, "segmentation": [[883.0, 226.0, 877.0, 226.0, 869.0, 230.0, 857.0, 241.0, 829.0, 255.0, 822.0, 263.0, 819.0, 272.0, 818.0, 293.0, 816.0, 297.0, 802.0, 308.0, 785.0, 316.0, 762.0, 332.0, 736.0, 336.0, 727.0, 329.0, 715.0, 328.0, 705.0, 331.0, 688.0, 331.0, 674.0, 325.0, 670.0, 327.0, 686.0, 337.0, 697.0, 339.0, 699.0, 341.0, 699.0, 351.0, 693.0, 361.0, 685.0, 360.0, 691.0, 371.0, 691.0, 379.0, 673.0, 425.0, 673.0, 433.0, 675.0, 437.0, 683.0, 443.0, 695.0, 444.0, 719.0, 436.0, 742.0, 436.0, 754.0, 428.0, 786.0, 428.0, 805.0, 458.0, 810.0, 478.0, 811.0, 495.0, 807.0, 517.0, 807.0, 528.0, 816.0, 525.0, 823.0, 526.0, 826.0, 523.0, 827.0, 512.0, 824.0, 492.0, 824.0, 477.0, 826.0, 470.0, 829.0, 467.0, 850.0, 458.0, 861.0, 445.0, 866.0, 443.0, 871.0, 444.0, 874.0, 448.0, 874.0, 479.0, 882.0, 492.0, 881.0, 519.0, 882.0, 521.0, 894.0, 525.0, 898.0, 523.0, 902.0, 517.0, 899.0, 501.0, 893.0, 485.0, 893.0, 461.0, 899.0, 439.0, 899.0, 415.0, 904.0, 405.0, 925.0, 383.0, 939.0, 353.0, 943.0, 338.0, 943.0, 314.0, 937.0, 299.0, 938.0, 265.0, 935.0, 255.0, 917.0, 243.0, 897.0, 239.0]]}, {"id": 1, "height": 720, "width": 1280, "segmentation": [[403.0, 136.0, 398.0, 134.0, 393.0, 134.0, 388.0, 135.0, 383.0, 138.0, 380.0, 141.0, 378.0, 145.0, 377.0, 151.0, 371.0, 157.0, 371.0, 159.0, 370.0, 161.0, 360.0, 171.0, 354.0, 175.0, 354.0, 176.0, 343.0, 186.0, 341.0, 189.0, 338.0, 196.0, 336.0, 206.0, 330.0, 222.0, 329.0, 226.0, 328.0, 237.0, 326.0, 245.0, 320.0, 259.0, 321.0, 274.0, 319.0, 279.0, 319.0, 282.0, 322.0, 283.0, 331.0, 281.0, 340.0, 277.0, 346.0, 277.0, 353.0, 274.0, 364.0, 266.0, 375.0, 260.0, 385.0, 257.0, 395.0, 255.0, 400.0, 253.0, 401.0, 245.0, 395.0, 235.0, 394.0, 223.0, 396.0, 208.0, 396.0, 192.0, 401.0, 187.0, 404.0, 181.0, 409.0, 176.0, 411.0, 173.0, 411.0, 171.0, 414.0, 168.0, 423.0, 166.0, 422.0, 162.0, 418.0, 158.0, 417.0, 153.0, 414.0, 146.0, 410.0, 141.0]]}]
|
examples/match_case/FRAME01_ORI.jpg
ADDED
|
examples/match_case/anno.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
ANSWER: 2
|
| 2 |
+
CAND: [2,3,1,]
|
| 3 |
+
TYPE: 3
|
examples/red-panda.mp4
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d921c07bb97224d65a37801541d246067f0d506f08723ffa1ad85c217907ccb8
|
| 3 |
+
size 1867237
|
generation_config.json
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"_from_model_config": true,
|
| 3 |
+
"transformers_version": "4.47.1"
|
| 4 |
+
}
|
model-00001-of-00002.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1cee5975c5d2d60f68e1c1fb19fb8526ac6eb5c87839f356160e7a8561579df7
|
| 3 |
+
size 4957392176
|
model-00002-of-00002.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:1728c8c67ca984991bff74d548c14f0d502e484455a078f5022a8e92e4dc7ac2
|
| 3 |
+
size 4737994808
|
model.safetensors.index.json
ADDED
|
@@ -0,0 +1,1010 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"metadata": {
|
| 3 |
+
"total_size": 9695259688
|
| 4 |
+
},
|
| 5 |
+
"weight_map": {
|
| 6 |
+
"language_model.lm_head.weight": "model-00002-of-00002.safetensors",
|
| 7 |
+
"language_model.model.embed_tokens.weight": "model-00001-of-00002.safetensors",
|
| 8 |
+
"language_model.model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 9 |
+
"language_model.model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 10 |
+
"language_model.model.layers.0.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
|
| 11 |
+
"language_model.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 12 |
+
"language_model.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 13 |
+
"language_model.model.layers.0.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
|
| 14 |
+
"language_model.model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 15 |
+
"language_model.model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 16 |
+
"language_model.model.layers.1.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
|
| 17 |
+
"language_model.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 18 |
+
"language_model.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 19 |
+
"language_model.model.layers.1.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
|
| 20 |
+
"language_model.model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 21 |
+
"language_model.model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 22 |
+
"language_model.model.layers.10.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
|
| 23 |
+
"language_model.model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 24 |
+
"language_model.model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 25 |
+
"language_model.model.layers.10.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
|
| 26 |
+
"language_model.model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 27 |
+
"language_model.model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 28 |
+
"language_model.model.layers.11.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
|
| 29 |
+
"language_model.model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 30 |
+
"language_model.model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 31 |
+
"language_model.model.layers.11.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
|
| 32 |
+
"language_model.model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 33 |
+
"language_model.model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 34 |
+
"language_model.model.layers.12.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
|
| 35 |
+
"language_model.model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 36 |
+
"language_model.model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 37 |
+
"language_model.model.layers.12.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
|
| 38 |
+
"language_model.model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 39 |
+
"language_model.model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 40 |
+
"language_model.model.layers.13.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
|
| 41 |
+
"language_model.model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 42 |
+
"language_model.model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 43 |
+
"language_model.model.layers.13.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
|
| 44 |
+
"language_model.model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 45 |
+
"language_model.model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 46 |
+
"language_model.model.layers.14.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
|
| 47 |
+
"language_model.model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 48 |
+
"language_model.model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 49 |
+
"language_model.model.layers.14.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
|
| 50 |
+
"language_model.model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 51 |
+
"language_model.model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 52 |
+
"language_model.model.layers.15.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
|
| 53 |
+
"language_model.model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 54 |
+
"language_model.model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 55 |
+
"language_model.model.layers.15.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
|
| 56 |
+
"language_model.model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 57 |
+
"language_model.model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 58 |
+
"language_model.model.layers.16.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
|
| 59 |
+
"language_model.model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 60 |
+
"language_model.model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 61 |
+
"language_model.model.layers.16.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
|
| 62 |
+
"language_model.model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 63 |
+
"language_model.model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 64 |
+
"language_model.model.layers.17.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
|
| 65 |
+
"language_model.model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 66 |
+
"language_model.model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 67 |
+
"language_model.model.layers.17.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
|
| 68 |
+
"language_model.model.layers.18.input_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 69 |
+
"language_model.model.layers.18.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
|
| 70 |
+
"language_model.model.layers.18.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
|
| 71 |
+
"language_model.model.layers.18.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 72 |
+
"language_model.model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 73 |
+
"language_model.model.layers.18.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
|
| 74 |
+
"language_model.model.layers.19.input_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 75 |
+
"language_model.model.layers.19.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
|
| 76 |
+
"language_model.model.layers.19.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
|
| 77 |
+
"language_model.model.layers.19.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 78 |
+
"language_model.model.layers.19.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
|
| 79 |
+
"language_model.model.layers.19.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
|
| 80 |
+
"language_model.model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 81 |
+
"language_model.model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 82 |
+
"language_model.model.layers.2.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
|
| 83 |
+
"language_model.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 84 |
+
"language_model.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 85 |
+
"language_model.model.layers.2.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
|
| 86 |
+
"language_model.model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 87 |
+
"language_model.model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
|
| 88 |
+
"language_model.model.layers.20.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
|
| 89 |
+
"language_model.model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 90 |
+
"language_model.model.layers.20.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
|
| 91 |
+
"language_model.model.layers.20.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
|
| 92 |
+
"language_model.model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 93 |
+
"language_model.model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
|
| 94 |
+
"language_model.model.layers.21.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
|
| 95 |
+
"language_model.model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 96 |
+
"language_model.model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
|
| 97 |
+
"language_model.model.layers.21.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
|
| 98 |
+
"language_model.model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 99 |
+
"language_model.model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
|
| 100 |
+
"language_model.model.layers.22.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
|
| 101 |
+
"language_model.model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 102 |
+
"language_model.model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
|
| 103 |
+
"language_model.model.layers.22.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
|
| 104 |
+
"language_model.model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 105 |
+
"language_model.model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
|
| 106 |
+
"language_model.model.layers.23.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
|
| 107 |
+
"language_model.model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 108 |
+
"language_model.model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
|
| 109 |
+
"language_model.model.layers.23.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
|
| 110 |
+
"language_model.model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 111 |
+
"language_model.model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
|
| 112 |
+
"language_model.model.layers.24.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
|
| 113 |
+
"language_model.model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 114 |
+
"language_model.model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
|
| 115 |
+
"language_model.model.layers.24.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
|
| 116 |
+
"language_model.model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 117 |
+
"language_model.model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
|
| 118 |
+
"language_model.model.layers.25.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
|
| 119 |
+
"language_model.model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 120 |
+
"language_model.model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
|
| 121 |
+
"language_model.model.layers.25.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
|
| 122 |
+
"language_model.model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 123 |
+
"language_model.model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
|
| 124 |
+
"language_model.model.layers.26.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
|
| 125 |
+
"language_model.model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 126 |
+
"language_model.model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
|
| 127 |
+
"language_model.model.layers.26.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
|
| 128 |
+
"language_model.model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 129 |
+
"language_model.model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
|
| 130 |
+
"language_model.model.layers.27.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
|
| 131 |
+
"language_model.model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 132 |
+
"language_model.model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
|
| 133 |
+
"language_model.model.layers.27.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
|
| 134 |
+
"language_model.model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 135 |
+
"language_model.model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
|
| 136 |
+
"language_model.model.layers.28.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
|
| 137 |
+
"language_model.model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 138 |
+
"language_model.model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
|
| 139 |
+
"language_model.model.layers.28.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
|
| 140 |
+
"language_model.model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 141 |
+
"language_model.model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
|
| 142 |
+
"language_model.model.layers.29.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
|
| 143 |
+
"language_model.model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 144 |
+
"language_model.model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
|
| 145 |
+
"language_model.model.layers.29.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
|
| 146 |
+
"language_model.model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 147 |
+
"language_model.model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 148 |
+
"language_model.model.layers.3.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
|
| 149 |
+
"language_model.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 150 |
+
"language_model.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 151 |
+
"language_model.model.layers.3.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
|
| 152 |
+
"language_model.model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 153 |
+
"language_model.model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
|
| 154 |
+
"language_model.model.layers.30.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
|
| 155 |
+
"language_model.model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 156 |
+
"language_model.model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
|
| 157 |
+
"language_model.model.layers.30.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
|
| 158 |
+
"language_model.model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 159 |
+
"language_model.model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
|
| 160 |
+
"language_model.model.layers.31.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
|
| 161 |
+
"language_model.model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
|
| 162 |
+
"language_model.model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
|
| 163 |
+
"language_model.model.layers.31.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
|
| 164 |
+
"language_model.model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 165 |
+
"language_model.model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 166 |
+
"language_model.model.layers.4.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
|
| 167 |
+
"language_model.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 168 |
+
"language_model.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 169 |
+
"language_model.model.layers.4.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
|
| 170 |
+
"language_model.model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 171 |
+
"language_model.model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 172 |
+
"language_model.model.layers.5.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
|
| 173 |
+
"language_model.model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 174 |
+
"language_model.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 175 |
+
"language_model.model.layers.5.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
|
| 176 |
+
"language_model.model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 177 |
+
"language_model.model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 178 |
+
"language_model.model.layers.6.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
|
| 179 |
+
"language_model.model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 180 |
+
"language_model.model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 181 |
+
"language_model.model.layers.6.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
|
| 182 |
+
"language_model.model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 183 |
+
"language_model.model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 184 |
+
"language_model.model.layers.7.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
|
| 185 |
+
"language_model.model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 186 |
+
"language_model.model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 187 |
+
"language_model.model.layers.7.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
|
| 188 |
+
"language_model.model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 189 |
+
"language_model.model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 190 |
+
"language_model.model.layers.8.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
|
| 191 |
+
"language_model.model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 192 |
+
"language_model.model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 193 |
+
"language_model.model.layers.8.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
|
| 194 |
+
"language_model.model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 195 |
+
"language_model.model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
|
| 196 |
+
"language_model.model.layers.9.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
|
| 197 |
+
"language_model.model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
| 198 |
+
"language_model.model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
| 199 |
+
"language_model.model.layers.9.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
|
| 200 |
+
"language_model.model.norm.weight": "model-00002-of-00002.safetensors",
|
| 201 |
+
"mlp1.0.bias": "model-00002-of-00002.safetensors",
|
| 202 |
+
"mlp1.0.weight": "model-00002-of-00002.safetensors",
|
| 203 |
+
"mlp1.1.bias": "model-00002-of-00002.safetensors",
|
| 204 |
+
"mlp1.1.weight": "model-00002-of-00002.safetensors",
|
| 205 |
+
"mlp1.3.bias": "model-00002-of-00002.safetensors",
|
| 206 |
+
"mlp1.3.weight": "model-00002-of-00002.safetensors",
|
| 207 |
+
"object_tokenizer.radio_model.input_conditioner.norm_mean": "model-00002-of-00002.safetensors",
|
| 208 |
+
"object_tokenizer.radio_model.input_conditioner.norm_std": "model-00002-of-00002.safetensors",
|
| 209 |
+
"object_tokenizer.radio_model.model.blocks.0.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 210 |
+
"object_tokenizer.radio_model.model.blocks.0.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 211 |
+
"object_tokenizer.radio_model.model.blocks.0.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 212 |
+
"object_tokenizer.radio_model.model.blocks.0.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 213 |
+
"object_tokenizer.radio_model.model.blocks.0.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 214 |
+
"object_tokenizer.radio_model.model.blocks.0.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 215 |
+
"object_tokenizer.radio_model.model.blocks.0.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 216 |
+
"object_tokenizer.radio_model.model.blocks.0.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 217 |
+
"object_tokenizer.radio_model.model.blocks.0.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 218 |
+
"object_tokenizer.radio_model.model.blocks.0.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 219 |
+
"object_tokenizer.radio_model.model.blocks.0.norm1.bias": "model-00002-of-00002.safetensors",
|
| 220 |
+
"object_tokenizer.radio_model.model.blocks.0.norm1.weight": "model-00002-of-00002.safetensors",
|
| 221 |
+
"object_tokenizer.radio_model.model.blocks.0.norm2.bias": "model-00002-of-00002.safetensors",
|
| 222 |
+
"object_tokenizer.radio_model.model.blocks.0.norm2.weight": "model-00002-of-00002.safetensors",
|
| 223 |
+
"object_tokenizer.radio_model.model.blocks.1.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 224 |
+
"object_tokenizer.radio_model.model.blocks.1.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 225 |
+
"object_tokenizer.radio_model.model.blocks.1.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 226 |
+
"object_tokenizer.radio_model.model.blocks.1.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 227 |
+
"object_tokenizer.radio_model.model.blocks.1.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 228 |
+
"object_tokenizer.radio_model.model.blocks.1.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 229 |
+
"object_tokenizer.radio_model.model.blocks.1.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 230 |
+
"object_tokenizer.radio_model.model.blocks.1.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 231 |
+
"object_tokenizer.radio_model.model.blocks.1.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 232 |
+
"object_tokenizer.radio_model.model.blocks.1.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 233 |
+
"object_tokenizer.radio_model.model.blocks.1.norm1.bias": "model-00002-of-00002.safetensors",
|
| 234 |
+
"object_tokenizer.radio_model.model.blocks.1.norm1.weight": "model-00002-of-00002.safetensors",
|
| 235 |
+
"object_tokenizer.radio_model.model.blocks.1.norm2.bias": "model-00002-of-00002.safetensors",
|
| 236 |
+
"object_tokenizer.radio_model.model.blocks.1.norm2.weight": "model-00002-of-00002.safetensors",
|
| 237 |
+
"object_tokenizer.radio_model.model.blocks.10.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 238 |
+
"object_tokenizer.radio_model.model.blocks.10.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 239 |
+
"object_tokenizer.radio_model.model.blocks.10.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 240 |
+
"object_tokenizer.radio_model.model.blocks.10.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 241 |
+
"object_tokenizer.radio_model.model.blocks.10.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 242 |
+
"object_tokenizer.radio_model.model.blocks.10.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 243 |
+
"object_tokenizer.radio_model.model.blocks.10.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 244 |
+
"object_tokenizer.radio_model.model.blocks.10.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 245 |
+
"object_tokenizer.radio_model.model.blocks.10.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 246 |
+
"object_tokenizer.radio_model.model.blocks.10.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 247 |
+
"object_tokenizer.radio_model.model.blocks.10.norm1.bias": "model-00002-of-00002.safetensors",
|
| 248 |
+
"object_tokenizer.radio_model.model.blocks.10.norm1.weight": "model-00002-of-00002.safetensors",
|
| 249 |
+
"object_tokenizer.radio_model.model.blocks.10.norm2.bias": "model-00002-of-00002.safetensors",
|
| 250 |
+
"object_tokenizer.radio_model.model.blocks.10.norm2.weight": "model-00002-of-00002.safetensors",
|
| 251 |
+
"object_tokenizer.radio_model.model.blocks.11.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 252 |
+
"object_tokenizer.radio_model.model.blocks.11.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 253 |
+
"object_tokenizer.radio_model.model.blocks.11.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 254 |
+
"object_tokenizer.radio_model.model.blocks.11.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 255 |
+
"object_tokenizer.radio_model.model.blocks.11.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 256 |
+
"object_tokenizer.radio_model.model.blocks.11.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 257 |
+
"object_tokenizer.radio_model.model.blocks.11.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 258 |
+
"object_tokenizer.radio_model.model.blocks.11.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 259 |
+
"object_tokenizer.radio_model.model.blocks.11.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 260 |
+
"object_tokenizer.radio_model.model.blocks.11.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 261 |
+
"object_tokenizer.radio_model.model.blocks.11.norm1.bias": "model-00002-of-00002.safetensors",
|
| 262 |
+
"object_tokenizer.radio_model.model.blocks.11.norm1.weight": "model-00002-of-00002.safetensors",
|
| 263 |
+
"object_tokenizer.radio_model.model.blocks.11.norm2.bias": "model-00002-of-00002.safetensors",
|
| 264 |
+
"object_tokenizer.radio_model.model.blocks.11.norm2.weight": "model-00002-of-00002.safetensors",
|
| 265 |
+
"object_tokenizer.radio_model.model.blocks.12.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 266 |
+
"object_tokenizer.radio_model.model.blocks.12.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 267 |
+
"object_tokenizer.radio_model.model.blocks.12.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 268 |
+
"object_tokenizer.radio_model.model.blocks.12.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 269 |
+
"object_tokenizer.radio_model.model.blocks.12.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 270 |
+
"object_tokenizer.radio_model.model.blocks.12.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 271 |
+
"object_tokenizer.radio_model.model.blocks.12.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 272 |
+
"object_tokenizer.radio_model.model.blocks.12.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 273 |
+
"object_tokenizer.radio_model.model.blocks.12.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 274 |
+
"object_tokenizer.radio_model.model.blocks.12.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 275 |
+
"object_tokenizer.radio_model.model.blocks.12.norm1.bias": "model-00002-of-00002.safetensors",
|
| 276 |
+
"object_tokenizer.radio_model.model.blocks.12.norm1.weight": "model-00002-of-00002.safetensors",
|
| 277 |
+
"object_tokenizer.radio_model.model.blocks.12.norm2.bias": "model-00002-of-00002.safetensors",
|
| 278 |
+
"object_tokenizer.radio_model.model.blocks.12.norm2.weight": "model-00002-of-00002.safetensors",
|
| 279 |
+
"object_tokenizer.radio_model.model.blocks.13.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 280 |
+
"object_tokenizer.radio_model.model.blocks.13.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 281 |
+
"object_tokenizer.radio_model.model.blocks.13.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 282 |
+
"object_tokenizer.radio_model.model.blocks.13.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 283 |
+
"object_tokenizer.radio_model.model.blocks.13.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 284 |
+
"object_tokenizer.radio_model.model.blocks.13.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 285 |
+
"object_tokenizer.radio_model.model.blocks.13.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 286 |
+
"object_tokenizer.radio_model.model.blocks.13.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 287 |
+
"object_tokenizer.radio_model.model.blocks.13.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 288 |
+
"object_tokenizer.radio_model.model.blocks.13.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 289 |
+
"object_tokenizer.radio_model.model.blocks.13.norm1.bias": "model-00002-of-00002.safetensors",
|
| 290 |
+
"object_tokenizer.radio_model.model.blocks.13.norm1.weight": "model-00002-of-00002.safetensors",
|
| 291 |
+
"object_tokenizer.radio_model.model.blocks.13.norm2.bias": "model-00002-of-00002.safetensors",
|
| 292 |
+
"object_tokenizer.radio_model.model.blocks.13.norm2.weight": "model-00002-of-00002.safetensors",
|
| 293 |
+
"object_tokenizer.radio_model.model.blocks.14.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 294 |
+
"object_tokenizer.radio_model.model.blocks.14.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 295 |
+
"object_tokenizer.radio_model.model.blocks.14.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 296 |
+
"object_tokenizer.radio_model.model.blocks.14.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 297 |
+
"object_tokenizer.radio_model.model.blocks.14.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 298 |
+
"object_tokenizer.radio_model.model.blocks.14.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 299 |
+
"object_tokenizer.radio_model.model.blocks.14.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 300 |
+
"object_tokenizer.radio_model.model.blocks.14.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 301 |
+
"object_tokenizer.radio_model.model.blocks.14.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 302 |
+
"object_tokenizer.radio_model.model.blocks.14.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 303 |
+
"object_tokenizer.radio_model.model.blocks.14.norm1.bias": "model-00002-of-00002.safetensors",
|
| 304 |
+
"object_tokenizer.radio_model.model.blocks.14.norm1.weight": "model-00002-of-00002.safetensors",
|
| 305 |
+
"object_tokenizer.radio_model.model.blocks.14.norm2.bias": "model-00002-of-00002.safetensors",
|
| 306 |
+
"object_tokenizer.radio_model.model.blocks.14.norm2.weight": "model-00002-of-00002.safetensors",
|
| 307 |
+
"object_tokenizer.radio_model.model.blocks.15.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 308 |
+
"object_tokenizer.radio_model.model.blocks.15.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 309 |
+
"object_tokenizer.radio_model.model.blocks.15.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 310 |
+
"object_tokenizer.radio_model.model.blocks.15.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 311 |
+
"object_tokenizer.radio_model.model.blocks.15.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 312 |
+
"object_tokenizer.radio_model.model.blocks.15.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 313 |
+
"object_tokenizer.radio_model.model.blocks.15.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 314 |
+
"object_tokenizer.radio_model.model.blocks.15.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 315 |
+
"object_tokenizer.radio_model.model.blocks.15.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 316 |
+
"object_tokenizer.radio_model.model.blocks.15.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 317 |
+
"object_tokenizer.radio_model.model.blocks.15.norm1.bias": "model-00002-of-00002.safetensors",
|
| 318 |
+
"object_tokenizer.radio_model.model.blocks.15.norm1.weight": "model-00002-of-00002.safetensors",
|
| 319 |
+
"object_tokenizer.radio_model.model.blocks.15.norm2.bias": "model-00002-of-00002.safetensors",
|
| 320 |
+
"object_tokenizer.radio_model.model.blocks.15.norm2.weight": "model-00002-of-00002.safetensors",
|
| 321 |
+
"object_tokenizer.radio_model.model.blocks.16.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 322 |
+
"object_tokenizer.radio_model.model.blocks.16.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 323 |
+
"object_tokenizer.radio_model.model.blocks.16.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 324 |
+
"object_tokenizer.radio_model.model.blocks.16.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 325 |
+
"object_tokenizer.radio_model.model.blocks.16.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 326 |
+
"object_tokenizer.radio_model.model.blocks.16.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 327 |
+
"object_tokenizer.radio_model.model.blocks.16.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 328 |
+
"object_tokenizer.radio_model.model.blocks.16.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 329 |
+
"object_tokenizer.radio_model.model.blocks.16.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 330 |
+
"object_tokenizer.radio_model.model.blocks.16.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 331 |
+
"object_tokenizer.radio_model.model.blocks.16.norm1.bias": "model-00002-of-00002.safetensors",
|
| 332 |
+
"object_tokenizer.radio_model.model.blocks.16.norm1.weight": "model-00002-of-00002.safetensors",
|
| 333 |
+
"object_tokenizer.radio_model.model.blocks.16.norm2.bias": "model-00002-of-00002.safetensors",
|
| 334 |
+
"object_tokenizer.radio_model.model.blocks.16.norm2.weight": "model-00002-of-00002.safetensors",
|
| 335 |
+
"object_tokenizer.radio_model.model.blocks.17.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 336 |
+
"object_tokenizer.radio_model.model.blocks.17.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 337 |
+
"object_tokenizer.radio_model.model.blocks.17.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 338 |
+
"object_tokenizer.radio_model.model.blocks.17.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 339 |
+
"object_tokenizer.radio_model.model.blocks.17.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 340 |
+
"object_tokenizer.radio_model.model.blocks.17.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 341 |
+
"object_tokenizer.radio_model.model.blocks.17.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 342 |
+
"object_tokenizer.radio_model.model.blocks.17.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 343 |
+
"object_tokenizer.radio_model.model.blocks.17.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 344 |
+
"object_tokenizer.radio_model.model.blocks.17.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 345 |
+
"object_tokenizer.radio_model.model.blocks.17.norm1.bias": "model-00002-of-00002.safetensors",
|
| 346 |
+
"object_tokenizer.radio_model.model.blocks.17.norm1.weight": "model-00002-of-00002.safetensors",
|
| 347 |
+
"object_tokenizer.radio_model.model.blocks.17.norm2.bias": "model-00002-of-00002.safetensors",
|
| 348 |
+
"object_tokenizer.radio_model.model.blocks.17.norm2.weight": "model-00002-of-00002.safetensors",
|
| 349 |
+
"object_tokenizer.radio_model.model.blocks.18.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 350 |
+
"object_tokenizer.radio_model.model.blocks.18.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 351 |
+
"object_tokenizer.radio_model.model.blocks.18.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 352 |
+
"object_tokenizer.radio_model.model.blocks.18.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 353 |
+
"object_tokenizer.radio_model.model.blocks.18.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 354 |
+
"object_tokenizer.radio_model.model.blocks.18.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 355 |
+
"object_tokenizer.radio_model.model.blocks.18.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 356 |
+
"object_tokenizer.radio_model.model.blocks.18.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 357 |
+
"object_tokenizer.radio_model.model.blocks.18.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 358 |
+
"object_tokenizer.radio_model.model.blocks.18.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 359 |
+
"object_tokenizer.radio_model.model.blocks.18.norm1.bias": "model-00002-of-00002.safetensors",
|
| 360 |
+
"object_tokenizer.radio_model.model.blocks.18.norm1.weight": "model-00002-of-00002.safetensors",
|
| 361 |
+
"object_tokenizer.radio_model.model.blocks.18.norm2.bias": "model-00002-of-00002.safetensors",
|
| 362 |
+
"object_tokenizer.radio_model.model.blocks.18.norm2.weight": "model-00002-of-00002.safetensors",
|
| 363 |
+
"object_tokenizer.radio_model.model.blocks.19.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 364 |
+
"object_tokenizer.radio_model.model.blocks.19.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 365 |
+
"object_tokenizer.radio_model.model.blocks.19.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 366 |
+
"object_tokenizer.radio_model.model.blocks.19.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 367 |
+
"object_tokenizer.radio_model.model.blocks.19.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 368 |
+
"object_tokenizer.radio_model.model.blocks.19.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 369 |
+
"object_tokenizer.radio_model.model.blocks.19.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 370 |
+
"object_tokenizer.radio_model.model.blocks.19.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 371 |
+
"object_tokenizer.radio_model.model.blocks.19.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 372 |
+
"object_tokenizer.radio_model.model.blocks.19.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 373 |
+
"object_tokenizer.radio_model.model.blocks.19.norm1.bias": "model-00002-of-00002.safetensors",
|
| 374 |
+
"object_tokenizer.radio_model.model.blocks.19.norm1.weight": "model-00002-of-00002.safetensors",
|
| 375 |
+
"object_tokenizer.radio_model.model.blocks.19.norm2.bias": "model-00002-of-00002.safetensors",
|
| 376 |
+
"object_tokenizer.radio_model.model.blocks.19.norm2.weight": "model-00002-of-00002.safetensors",
|
| 377 |
+
"object_tokenizer.radio_model.model.blocks.2.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 378 |
+
"object_tokenizer.radio_model.model.blocks.2.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 379 |
+
"object_tokenizer.radio_model.model.blocks.2.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 380 |
+
"object_tokenizer.radio_model.model.blocks.2.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 381 |
+
"object_tokenizer.radio_model.model.blocks.2.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 382 |
+
"object_tokenizer.radio_model.model.blocks.2.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 383 |
+
"object_tokenizer.radio_model.model.blocks.2.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 384 |
+
"object_tokenizer.radio_model.model.blocks.2.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 385 |
+
"object_tokenizer.radio_model.model.blocks.2.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 386 |
+
"object_tokenizer.radio_model.model.blocks.2.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 387 |
+
"object_tokenizer.radio_model.model.blocks.2.norm1.bias": "model-00002-of-00002.safetensors",
|
| 388 |
+
"object_tokenizer.radio_model.model.blocks.2.norm1.weight": "model-00002-of-00002.safetensors",
|
| 389 |
+
"object_tokenizer.radio_model.model.blocks.2.norm2.bias": "model-00002-of-00002.safetensors",
|
| 390 |
+
"object_tokenizer.radio_model.model.blocks.2.norm2.weight": "model-00002-of-00002.safetensors",
|
| 391 |
+
"object_tokenizer.radio_model.model.blocks.20.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 392 |
+
"object_tokenizer.radio_model.model.blocks.20.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 393 |
+
"object_tokenizer.radio_model.model.blocks.20.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 394 |
+
"object_tokenizer.radio_model.model.blocks.20.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 395 |
+
"object_tokenizer.radio_model.model.blocks.20.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 396 |
+
"object_tokenizer.radio_model.model.blocks.20.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 397 |
+
"object_tokenizer.radio_model.model.blocks.20.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 398 |
+
"object_tokenizer.radio_model.model.blocks.20.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 399 |
+
"object_tokenizer.radio_model.model.blocks.20.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 400 |
+
"object_tokenizer.radio_model.model.blocks.20.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 401 |
+
"object_tokenizer.radio_model.model.blocks.20.norm1.bias": "model-00002-of-00002.safetensors",
|
| 402 |
+
"object_tokenizer.radio_model.model.blocks.20.norm1.weight": "model-00002-of-00002.safetensors",
|
| 403 |
+
"object_tokenizer.radio_model.model.blocks.20.norm2.bias": "model-00002-of-00002.safetensors",
|
| 404 |
+
"object_tokenizer.radio_model.model.blocks.20.norm2.weight": "model-00002-of-00002.safetensors",
|
| 405 |
+
"object_tokenizer.radio_model.model.blocks.21.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 406 |
+
"object_tokenizer.radio_model.model.blocks.21.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 407 |
+
"object_tokenizer.radio_model.model.blocks.21.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 408 |
+
"object_tokenizer.radio_model.model.blocks.21.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 409 |
+
"object_tokenizer.radio_model.model.blocks.21.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 410 |
+
"object_tokenizer.radio_model.model.blocks.21.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 411 |
+
"object_tokenizer.radio_model.model.blocks.21.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 412 |
+
"object_tokenizer.radio_model.model.blocks.21.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 413 |
+
"object_tokenizer.radio_model.model.blocks.21.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 414 |
+
"object_tokenizer.radio_model.model.blocks.21.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 415 |
+
"object_tokenizer.radio_model.model.blocks.21.norm1.bias": "model-00002-of-00002.safetensors",
|
| 416 |
+
"object_tokenizer.radio_model.model.blocks.21.norm1.weight": "model-00002-of-00002.safetensors",
|
| 417 |
+
"object_tokenizer.radio_model.model.blocks.21.norm2.bias": "model-00002-of-00002.safetensors",
|
| 418 |
+
"object_tokenizer.radio_model.model.blocks.21.norm2.weight": "model-00002-of-00002.safetensors",
|
| 419 |
+
"object_tokenizer.radio_model.model.blocks.22.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 420 |
+
"object_tokenizer.radio_model.model.blocks.22.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 421 |
+
"object_tokenizer.radio_model.model.blocks.22.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 422 |
+
"object_tokenizer.radio_model.model.blocks.22.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 423 |
+
"object_tokenizer.radio_model.model.blocks.22.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 424 |
+
"object_tokenizer.radio_model.model.blocks.22.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 425 |
+
"object_tokenizer.radio_model.model.blocks.22.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 426 |
+
"object_tokenizer.radio_model.model.blocks.22.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 427 |
+
"object_tokenizer.radio_model.model.blocks.22.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 428 |
+
"object_tokenizer.radio_model.model.blocks.22.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 429 |
+
"object_tokenizer.radio_model.model.blocks.22.norm1.bias": "model-00002-of-00002.safetensors",
|
| 430 |
+
"object_tokenizer.radio_model.model.blocks.22.norm1.weight": "model-00002-of-00002.safetensors",
|
| 431 |
+
"object_tokenizer.radio_model.model.blocks.22.norm2.bias": "model-00002-of-00002.safetensors",
|
| 432 |
+
"object_tokenizer.radio_model.model.blocks.22.norm2.weight": "model-00002-of-00002.safetensors",
|
| 433 |
+
"object_tokenizer.radio_model.model.blocks.23.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 434 |
+
"object_tokenizer.radio_model.model.blocks.23.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 435 |
+
"object_tokenizer.radio_model.model.blocks.23.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 436 |
+
"object_tokenizer.radio_model.model.blocks.23.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 437 |
+
"object_tokenizer.radio_model.model.blocks.23.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 438 |
+
"object_tokenizer.radio_model.model.blocks.23.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 439 |
+
"object_tokenizer.radio_model.model.blocks.23.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 440 |
+
"object_tokenizer.radio_model.model.blocks.23.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 441 |
+
"object_tokenizer.radio_model.model.blocks.23.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 442 |
+
"object_tokenizer.radio_model.model.blocks.23.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 443 |
+
"object_tokenizer.radio_model.model.blocks.23.norm1.bias": "model-00002-of-00002.safetensors",
|
| 444 |
+
"object_tokenizer.radio_model.model.blocks.23.norm1.weight": "model-00002-of-00002.safetensors",
|
| 445 |
+
"object_tokenizer.radio_model.model.blocks.23.norm2.bias": "model-00002-of-00002.safetensors",
|
| 446 |
+
"object_tokenizer.radio_model.model.blocks.23.norm2.weight": "model-00002-of-00002.safetensors",
|
| 447 |
+
"object_tokenizer.radio_model.model.blocks.24.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 448 |
+
"object_tokenizer.radio_model.model.blocks.24.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 449 |
+
"object_tokenizer.radio_model.model.blocks.24.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 450 |
+
"object_tokenizer.radio_model.model.blocks.24.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 451 |
+
"object_tokenizer.radio_model.model.blocks.24.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 452 |
+
"object_tokenizer.radio_model.model.blocks.24.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 453 |
+
"object_tokenizer.radio_model.model.blocks.24.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 454 |
+
"object_tokenizer.radio_model.model.blocks.24.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 455 |
+
"object_tokenizer.radio_model.model.blocks.24.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 456 |
+
"object_tokenizer.radio_model.model.blocks.24.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 457 |
+
"object_tokenizer.radio_model.model.blocks.24.norm1.bias": "model-00002-of-00002.safetensors",
|
| 458 |
+
"object_tokenizer.radio_model.model.blocks.24.norm1.weight": "model-00002-of-00002.safetensors",
|
| 459 |
+
"object_tokenizer.radio_model.model.blocks.24.norm2.bias": "model-00002-of-00002.safetensors",
|
| 460 |
+
"object_tokenizer.radio_model.model.blocks.24.norm2.weight": "model-00002-of-00002.safetensors",
|
| 461 |
+
"object_tokenizer.radio_model.model.blocks.25.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 462 |
+
"object_tokenizer.radio_model.model.blocks.25.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 463 |
+
"object_tokenizer.radio_model.model.blocks.25.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 464 |
+
"object_tokenizer.radio_model.model.blocks.25.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 465 |
+
"object_tokenizer.radio_model.model.blocks.25.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 466 |
+
"object_tokenizer.radio_model.model.blocks.25.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 467 |
+
"object_tokenizer.radio_model.model.blocks.25.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 468 |
+
"object_tokenizer.radio_model.model.blocks.25.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 469 |
+
"object_tokenizer.radio_model.model.blocks.25.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 470 |
+
"object_tokenizer.radio_model.model.blocks.25.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 471 |
+
"object_tokenizer.radio_model.model.blocks.25.norm1.bias": "model-00002-of-00002.safetensors",
|
| 472 |
+
"object_tokenizer.radio_model.model.blocks.25.norm1.weight": "model-00002-of-00002.safetensors",
|
| 473 |
+
"object_tokenizer.radio_model.model.blocks.25.norm2.bias": "model-00002-of-00002.safetensors",
|
| 474 |
+
"object_tokenizer.radio_model.model.blocks.25.norm2.weight": "model-00002-of-00002.safetensors",
|
| 475 |
+
"object_tokenizer.radio_model.model.blocks.26.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 476 |
+
"object_tokenizer.radio_model.model.blocks.26.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 477 |
+
"object_tokenizer.radio_model.model.blocks.26.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 478 |
+
"object_tokenizer.radio_model.model.blocks.26.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 479 |
+
"object_tokenizer.radio_model.model.blocks.26.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 480 |
+
"object_tokenizer.radio_model.model.blocks.26.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 481 |
+
"object_tokenizer.radio_model.model.blocks.26.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 482 |
+
"object_tokenizer.radio_model.model.blocks.26.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 483 |
+
"object_tokenizer.radio_model.model.blocks.26.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 484 |
+
"object_tokenizer.radio_model.model.blocks.26.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 485 |
+
"object_tokenizer.radio_model.model.blocks.26.norm1.bias": "model-00002-of-00002.safetensors",
|
| 486 |
+
"object_tokenizer.radio_model.model.blocks.26.norm1.weight": "model-00002-of-00002.safetensors",
|
| 487 |
+
"object_tokenizer.radio_model.model.blocks.26.norm2.bias": "model-00002-of-00002.safetensors",
|
| 488 |
+
"object_tokenizer.radio_model.model.blocks.26.norm2.weight": "model-00002-of-00002.safetensors",
|
| 489 |
+
"object_tokenizer.radio_model.model.blocks.27.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 490 |
+
"object_tokenizer.radio_model.model.blocks.27.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 491 |
+
"object_tokenizer.radio_model.model.blocks.27.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 492 |
+
"object_tokenizer.radio_model.model.blocks.27.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 493 |
+
"object_tokenizer.radio_model.model.blocks.27.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 494 |
+
"object_tokenizer.radio_model.model.blocks.27.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 495 |
+
"object_tokenizer.radio_model.model.blocks.27.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 496 |
+
"object_tokenizer.radio_model.model.blocks.27.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 497 |
+
"object_tokenizer.radio_model.model.blocks.27.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 498 |
+
"object_tokenizer.radio_model.model.blocks.27.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 499 |
+
"object_tokenizer.radio_model.model.blocks.27.norm1.bias": "model-00002-of-00002.safetensors",
|
| 500 |
+
"object_tokenizer.radio_model.model.blocks.27.norm1.weight": "model-00002-of-00002.safetensors",
|
| 501 |
+
"object_tokenizer.radio_model.model.blocks.27.norm2.bias": "model-00002-of-00002.safetensors",
|
| 502 |
+
"object_tokenizer.radio_model.model.blocks.27.norm2.weight": "model-00002-of-00002.safetensors",
|
| 503 |
+
"object_tokenizer.radio_model.model.blocks.28.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 504 |
+
"object_tokenizer.radio_model.model.blocks.28.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 505 |
+
"object_tokenizer.radio_model.model.blocks.28.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 506 |
+
"object_tokenizer.radio_model.model.blocks.28.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 507 |
+
"object_tokenizer.radio_model.model.blocks.28.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 508 |
+
"object_tokenizer.radio_model.model.blocks.28.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 509 |
+
"object_tokenizer.radio_model.model.blocks.28.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 510 |
+
"object_tokenizer.radio_model.model.blocks.28.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 511 |
+
"object_tokenizer.radio_model.model.blocks.28.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 512 |
+
"object_tokenizer.radio_model.model.blocks.28.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 513 |
+
"object_tokenizer.radio_model.model.blocks.28.norm1.bias": "model-00002-of-00002.safetensors",
|
| 514 |
+
"object_tokenizer.radio_model.model.blocks.28.norm1.weight": "model-00002-of-00002.safetensors",
|
| 515 |
+
"object_tokenizer.radio_model.model.blocks.28.norm2.bias": "model-00002-of-00002.safetensors",
|
| 516 |
+
"object_tokenizer.radio_model.model.blocks.28.norm2.weight": "model-00002-of-00002.safetensors",
|
| 517 |
+
"object_tokenizer.radio_model.model.blocks.29.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 518 |
+
"object_tokenizer.radio_model.model.blocks.29.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 519 |
+
"object_tokenizer.radio_model.model.blocks.29.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 520 |
+
"object_tokenizer.radio_model.model.blocks.29.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 521 |
+
"object_tokenizer.radio_model.model.blocks.29.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 522 |
+
"object_tokenizer.radio_model.model.blocks.29.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 523 |
+
"object_tokenizer.radio_model.model.blocks.29.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 524 |
+
"object_tokenizer.radio_model.model.blocks.29.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 525 |
+
"object_tokenizer.radio_model.model.blocks.29.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 526 |
+
"object_tokenizer.radio_model.model.blocks.29.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 527 |
+
"object_tokenizer.radio_model.model.blocks.29.norm1.bias": "model-00002-of-00002.safetensors",
|
| 528 |
+
"object_tokenizer.radio_model.model.blocks.29.norm1.weight": "model-00002-of-00002.safetensors",
|
| 529 |
+
"object_tokenizer.radio_model.model.blocks.29.norm2.bias": "model-00002-of-00002.safetensors",
|
| 530 |
+
"object_tokenizer.radio_model.model.blocks.29.norm2.weight": "model-00002-of-00002.safetensors",
|
| 531 |
+
"object_tokenizer.radio_model.model.blocks.3.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 532 |
+
"object_tokenizer.radio_model.model.blocks.3.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 533 |
+
"object_tokenizer.radio_model.model.blocks.3.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 534 |
+
"object_tokenizer.radio_model.model.blocks.3.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 535 |
+
"object_tokenizer.radio_model.model.blocks.3.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 536 |
+
"object_tokenizer.radio_model.model.blocks.3.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 537 |
+
"object_tokenizer.radio_model.model.blocks.3.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 538 |
+
"object_tokenizer.radio_model.model.blocks.3.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 539 |
+
"object_tokenizer.radio_model.model.blocks.3.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 540 |
+
"object_tokenizer.radio_model.model.blocks.3.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 541 |
+
"object_tokenizer.radio_model.model.blocks.3.norm1.bias": "model-00002-of-00002.safetensors",
|
| 542 |
+
"object_tokenizer.radio_model.model.blocks.3.norm1.weight": "model-00002-of-00002.safetensors",
|
| 543 |
+
"object_tokenizer.radio_model.model.blocks.3.norm2.bias": "model-00002-of-00002.safetensors",
|
| 544 |
+
"object_tokenizer.radio_model.model.blocks.3.norm2.weight": "model-00002-of-00002.safetensors",
|
| 545 |
+
"object_tokenizer.radio_model.model.blocks.30.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 546 |
+
"object_tokenizer.radio_model.model.blocks.30.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 547 |
+
"object_tokenizer.radio_model.model.blocks.30.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 548 |
+
"object_tokenizer.radio_model.model.blocks.30.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 549 |
+
"object_tokenizer.radio_model.model.blocks.30.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 550 |
+
"object_tokenizer.radio_model.model.blocks.30.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 551 |
+
"object_tokenizer.radio_model.model.blocks.30.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 552 |
+
"object_tokenizer.radio_model.model.blocks.30.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 553 |
+
"object_tokenizer.radio_model.model.blocks.30.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 554 |
+
"object_tokenizer.radio_model.model.blocks.30.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 555 |
+
"object_tokenizer.radio_model.model.blocks.30.norm1.bias": "model-00002-of-00002.safetensors",
|
| 556 |
+
"object_tokenizer.radio_model.model.blocks.30.norm1.weight": "model-00002-of-00002.safetensors",
|
| 557 |
+
"object_tokenizer.radio_model.model.blocks.30.norm2.bias": "model-00002-of-00002.safetensors",
|
| 558 |
+
"object_tokenizer.radio_model.model.blocks.30.norm2.weight": "model-00002-of-00002.safetensors",
|
| 559 |
+
"object_tokenizer.radio_model.model.blocks.31.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 560 |
+
"object_tokenizer.radio_model.model.blocks.31.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 561 |
+
"object_tokenizer.radio_model.model.blocks.31.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 562 |
+
"object_tokenizer.radio_model.model.blocks.31.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 563 |
+
"object_tokenizer.radio_model.model.blocks.31.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 564 |
+
"object_tokenizer.radio_model.model.blocks.31.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 565 |
+
"object_tokenizer.radio_model.model.blocks.31.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 566 |
+
"object_tokenizer.radio_model.model.blocks.31.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 567 |
+
"object_tokenizer.radio_model.model.blocks.31.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 568 |
+
"object_tokenizer.radio_model.model.blocks.31.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 569 |
+
"object_tokenizer.radio_model.model.blocks.31.norm1.bias": "model-00002-of-00002.safetensors",
|
| 570 |
+
"object_tokenizer.radio_model.model.blocks.31.norm1.weight": "model-00002-of-00002.safetensors",
|
| 571 |
+
"object_tokenizer.radio_model.model.blocks.31.norm2.bias": "model-00002-of-00002.safetensors",
|
| 572 |
+
"object_tokenizer.radio_model.model.blocks.31.norm2.weight": "model-00002-of-00002.safetensors",
|
| 573 |
+
"object_tokenizer.radio_model.model.blocks.4.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 574 |
+
"object_tokenizer.radio_model.model.blocks.4.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 575 |
+
"object_tokenizer.radio_model.model.blocks.4.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 576 |
+
"object_tokenizer.radio_model.model.blocks.4.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 577 |
+
"object_tokenizer.radio_model.model.blocks.4.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 578 |
+
"object_tokenizer.radio_model.model.blocks.4.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 579 |
+
"object_tokenizer.radio_model.model.blocks.4.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 580 |
+
"object_tokenizer.radio_model.model.blocks.4.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 581 |
+
"object_tokenizer.radio_model.model.blocks.4.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 582 |
+
"object_tokenizer.radio_model.model.blocks.4.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 583 |
+
"object_tokenizer.radio_model.model.blocks.4.norm1.bias": "model-00002-of-00002.safetensors",
|
| 584 |
+
"object_tokenizer.radio_model.model.blocks.4.norm1.weight": "model-00002-of-00002.safetensors",
|
| 585 |
+
"object_tokenizer.radio_model.model.blocks.4.norm2.bias": "model-00002-of-00002.safetensors",
|
| 586 |
+
"object_tokenizer.radio_model.model.blocks.4.norm2.weight": "model-00002-of-00002.safetensors",
|
| 587 |
+
"object_tokenizer.radio_model.model.blocks.5.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 588 |
+
"object_tokenizer.radio_model.model.blocks.5.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 589 |
+
"object_tokenizer.radio_model.model.blocks.5.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 590 |
+
"object_tokenizer.radio_model.model.blocks.5.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 591 |
+
"object_tokenizer.radio_model.model.blocks.5.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 592 |
+
"object_tokenizer.radio_model.model.blocks.5.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 593 |
+
"object_tokenizer.radio_model.model.blocks.5.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 594 |
+
"object_tokenizer.radio_model.model.blocks.5.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 595 |
+
"object_tokenizer.radio_model.model.blocks.5.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 596 |
+
"object_tokenizer.radio_model.model.blocks.5.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 597 |
+
"object_tokenizer.radio_model.model.blocks.5.norm1.bias": "model-00002-of-00002.safetensors",
|
| 598 |
+
"object_tokenizer.radio_model.model.blocks.5.norm1.weight": "model-00002-of-00002.safetensors",
|
| 599 |
+
"object_tokenizer.radio_model.model.blocks.5.norm2.bias": "model-00002-of-00002.safetensors",
|
| 600 |
+
"object_tokenizer.radio_model.model.blocks.5.norm2.weight": "model-00002-of-00002.safetensors",
|
| 601 |
+
"object_tokenizer.radio_model.model.blocks.6.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 602 |
+
"object_tokenizer.radio_model.model.blocks.6.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 603 |
+
"object_tokenizer.radio_model.model.blocks.6.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 604 |
+
"object_tokenizer.radio_model.model.blocks.6.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 605 |
+
"object_tokenizer.radio_model.model.blocks.6.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 606 |
+
"object_tokenizer.radio_model.model.blocks.6.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 607 |
+
"object_tokenizer.radio_model.model.blocks.6.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 608 |
+
"object_tokenizer.radio_model.model.blocks.6.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 609 |
+
"object_tokenizer.radio_model.model.blocks.6.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 610 |
+
"object_tokenizer.radio_model.model.blocks.6.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 611 |
+
"object_tokenizer.radio_model.model.blocks.6.norm1.bias": "model-00002-of-00002.safetensors",
|
| 612 |
+
"object_tokenizer.radio_model.model.blocks.6.norm1.weight": "model-00002-of-00002.safetensors",
|
| 613 |
+
"object_tokenizer.radio_model.model.blocks.6.norm2.bias": "model-00002-of-00002.safetensors",
|
| 614 |
+
"object_tokenizer.radio_model.model.blocks.6.norm2.weight": "model-00002-of-00002.safetensors",
|
| 615 |
+
"object_tokenizer.radio_model.model.blocks.7.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 616 |
+
"object_tokenizer.radio_model.model.blocks.7.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 617 |
+
"object_tokenizer.radio_model.model.blocks.7.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 618 |
+
"object_tokenizer.radio_model.model.blocks.7.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 619 |
+
"object_tokenizer.radio_model.model.blocks.7.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 620 |
+
"object_tokenizer.radio_model.model.blocks.7.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 621 |
+
"object_tokenizer.radio_model.model.blocks.7.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 622 |
+
"object_tokenizer.radio_model.model.blocks.7.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 623 |
+
"object_tokenizer.radio_model.model.blocks.7.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 624 |
+
"object_tokenizer.radio_model.model.blocks.7.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 625 |
+
"object_tokenizer.radio_model.model.blocks.7.norm1.bias": "model-00002-of-00002.safetensors",
|
| 626 |
+
"object_tokenizer.radio_model.model.blocks.7.norm1.weight": "model-00002-of-00002.safetensors",
|
| 627 |
+
"object_tokenizer.radio_model.model.blocks.7.norm2.bias": "model-00002-of-00002.safetensors",
|
| 628 |
+
"object_tokenizer.radio_model.model.blocks.7.norm2.weight": "model-00002-of-00002.safetensors",
|
| 629 |
+
"object_tokenizer.radio_model.model.blocks.8.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 630 |
+
"object_tokenizer.radio_model.model.blocks.8.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 631 |
+
"object_tokenizer.radio_model.model.blocks.8.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 632 |
+
"object_tokenizer.radio_model.model.blocks.8.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 633 |
+
"object_tokenizer.radio_model.model.blocks.8.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 634 |
+
"object_tokenizer.radio_model.model.blocks.8.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 635 |
+
"object_tokenizer.radio_model.model.blocks.8.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 636 |
+
"object_tokenizer.radio_model.model.blocks.8.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 637 |
+
"object_tokenizer.radio_model.model.blocks.8.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 638 |
+
"object_tokenizer.radio_model.model.blocks.8.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 639 |
+
"object_tokenizer.radio_model.model.blocks.8.norm1.bias": "model-00002-of-00002.safetensors",
|
| 640 |
+
"object_tokenizer.radio_model.model.blocks.8.norm1.weight": "model-00002-of-00002.safetensors",
|
| 641 |
+
"object_tokenizer.radio_model.model.blocks.8.norm2.bias": "model-00002-of-00002.safetensors",
|
| 642 |
+
"object_tokenizer.radio_model.model.blocks.8.norm2.weight": "model-00002-of-00002.safetensors",
|
| 643 |
+
"object_tokenizer.radio_model.model.blocks.9.attn.proj.bias": "model-00002-of-00002.safetensors",
|
| 644 |
+
"object_tokenizer.radio_model.model.blocks.9.attn.proj.weight": "model-00002-of-00002.safetensors",
|
| 645 |
+
"object_tokenizer.radio_model.model.blocks.9.attn.qkv.bias": "model-00002-of-00002.safetensors",
|
| 646 |
+
"object_tokenizer.radio_model.model.blocks.9.attn.qkv.weight": "model-00002-of-00002.safetensors",
|
| 647 |
+
"object_tokenizer.radio_model.model.blocks.9.mlp.fc1.bias": "model-00002-of-00002.safetensors",
|
| 648 |
+
"object_tokenizer.radio_model.model.blocks.9.mlp.fc1.weight": "model-00002-of-00002.safetensors",
|
| 649 |
+
"object_tokenizer.radio_model.model.blocks.9.mlp.fc2.bias": "model-00002-of-00002.safetensors",
|
| 650 |
+
"object_tokenizer.radio_model.model.blocks.9.mlp.fc2.weight": "model-00002-of-00002.safetensors",
|
| 651 |
+
"object_tokenizer.radio_model.model.blocks.9.mlp.norm.bias": "model-00002-of-00002.safetensors",
|
| 652 |
+
"object_tokenizer.radio_model.model.blocks.9.mlp.norm.weight": "model-00002-of-00002.safetensors",
|
| 653 |
+
"object_tokenizer.radio_model.model.blocks.9.norm1.bias": "model-00002-of-00002.safetensors",
|
| 654 |
+
"object_tokenizer.radio_model.model.blocks.9.norm1.weight": "model-00002-of-00002.safetensors",
|
| 655 |
+
"object_tokenizer.radio_model.model.blocks.9.norm2.bias": "model-00002-of-00002.safetensors",
|
| 656 |
+
"object_tokenizer.radio_model.model.blocks.9.norm2.weight": "model-00002-of-00002.safetensors",
|
| 657 |
+
"object_tokenizer.radio_model.model.norm.bias": "model-00002-of-00002.safetensors",
|
| 658 |
+
"object_tokenizer.radio_model.model.norm.weight": "model-00002-of-00002.safetensors",
|
| 659 |
+
"object_tokenizer.radio_model.model.patch_generator.cls_token.token": "model-00002-of-00002.safetensors",
|
| 660 |
+
"object_tokenizer.radio_model.model.patch_generator.embedder.weight": "model-00002-of-00002.safetensors",
|
| 661 |
+
"object_tokenizer.radio_model.model.patch_generator.pos_embed": "model-00002-of-00002.safetensors",
|
| 662 |
+
"object_tokenizer.radio_model.summary_idxs": "model-00002-of-00002.safetensors",
|
| 663 |
+
"ot_mlp1.0.bias": "model-00002-of-00002.safetensors",
|
| 664 |
+
"ot_mlp1.0.weight": "model-00002-of-00002.safetensors",
|
| 665 |
+
"ot_mlp1.1.bias": "model-00002-of-00002.safetensors",
|
| 666 |
+
"ot_mlp1.1.weight": "model-00002-of-00002.safetensors",
|
| 667 |
+
"ot_mlp1.3.bias": "model-00002-of-00002.safetensors",
|
| 668 |
+
"ot_mlp1.3.weight": "model-00002-of-00002.safetensors",
|
| 669 |
+
"vision_model.embeddings.class_embedding": "model-00001-of-00002.safetensors",
|
| 670 |
+
"vision_model.embeddings.patch_embedding.bias": "model-00001-of-00002.safetensors",
|
| 671 |
+
"vision_model.embeddings.patch_embedding.weight": "model-00001-of-00002.safetensors",
|
| 672 |
+
"vision_model.embeddings.position_embedding": "model-00001-of-00002.safetensors",
|
| 673 |
+
"vision_model.encoder.layers.0.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 674 |
+
"vision_model.encoder.layers.0.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 675 |
+
"vision_model.encoder.layers.0.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 676 |
+
"vision_model.encoder.layers.0.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 677 |
+
"vision_model.encoder.layers.0.ls1": "model-00001-of-00002.safetensors",
|
| 678 |
+
"vision_model.encoder.layers.0.ls2": "model-00001-of-00002.safetensors",
|
| 679 |
+
"vision_model.encoder.layers.0.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 680 |
+
"vision_model.encoder.layers.0.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 681 |
+
"vision_model.encoder.layers.0.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 682 |
+
"vision_model.encoder.layers.0.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 683 |
+
"vision_model.encoder.layers.0.norm1.bias": "model-00001-of-00002.safetensors",
|
| 684 |
+
"vision_model.encoder.layers.0.norm1.weight": "model-00001-of-00002.safetensors",
|
| 685 |
+
"vision_model.encoder.layers.0.norm2.bias": "model-00001-of-00002.safetensors",
|
| 686 |
+
"vision_model.encoder.layers.0.norm2.weight": "model-00001-of-00002.safetensors",
|
| 687 |
+
"vision_model.encoder.layers.1.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 688 |
+
"vision_model.encoder.layers.1.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 689 |
+
"vision_model.encoder.layers.1.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 690 |
+
"vision_model.encoder.layers.1.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 691 |
+
"vision_model.encoder.layers.1.ls1": "model-00001-of-00002.safetensors",
|
| 692 |
+
"vision_model.encoder.layers.1.ls2": "model-00001-of-00002.safetensors",
|
| 693 |
+
"vision_model.encoder.layers.1.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 694 |
+
"vision_model.encoder.layers.1.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 695 |
+
"vision_model.encoder.layers.1.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 696 |
+
"vision_model.encoder.layers.1.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 697 |
+
"vision_model.encoder.layers.1.norm1.bias": "model-00001-of-00002.safetensors",
|
| 698 |
+
"vision_model.encoder.layers.1.norm1.weight": "model-00001-of-00002.safetensors",
|
| 699 |
+
"vision_model.encoder.layers.1.norm2.bias": "model-00001-of-00002.safetensors",
|
| 700 |
+
"vision_model.encoder.layers.1.norm2.weight": "model-00001-of-00002.safetensors",
|
| 701 |
+
"vision_model.encoder.layers.10.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 702 |
+
"vision_model.encoder.layers.10.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 703 |
+
"vision_model.encoder.layers.10.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 704 |
+
"vision_model.encoder.layers.10.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 705 |
+
"vision_model.encoder.layers.10.ls1": "model-00001-of-00002.safetensors",
|
| 706 |
+
"vision_model.encoder.layers.10.ls2": "model-00001-of-00002.safetensors",
|
| 707 |
+
"vision_model.encoder.layers.10.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 708 |
+
"vision_model.encoder.layers.10.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 709 |
+
"vision_model.encoder.layers.10.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 710 |
+
"vision_model.encoder.layers.10.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 711 |
+
"vision_model.encoder.layers.10.norm1.bias": "model-00001-of-00002.safetensors",
|
| 712 |
+
"vision_model.encoder.layers.10.norm1.weight": "model-00001-of-00002.safetensors",
|
| 713 |
+
"vision_model.encoder.layers.10.norm2.bias": "model-00001-of-00002.safetensors",
|
| 714 |
+
"vision_model.encoder.layers.10.norm2.weight": "model-00001-of-00002.safetensors",
|
| 715 |
+
"vision_model.encoder.layers.11.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 716 |
+
"vision_model.encoder.layers.11.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 717 |
+
"vision_model.encoder.layers.11.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 718 |
+
"vision_model.encoder.layers.11.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 719 |
+
"vision_model.encoder.layers.11.ls1": "model-00001-of-00002.safetensors",
|
| 720 |
+
"vision_model.encoder.layers.11.ls2": "model-00001-of-00002.safetensors",
|
| 721 |
+
"vision_model.encoder.layers.11.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 722 |
+
"vision_model.encoder.layers.11.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 723 |
+
"vision_model.encoder.layers.11.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 724 |
+
"vision_model.encoder.layers.11.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 725 |
+
"vision_model.encoder.layers.11.norm1.bias": "model-00001-of-00002.safetensors",
|
| 726 |
+
"vision_model.encoder.layers.11.norm1.weight": "model-00001-of-00002.safetensors",
|
| 727 |
+
"vision_model.encoder.layers.11.norm2.bias": "model-00001-of-00002.safetensors",
|
| 728 |
+
"vision_model.encoder.layers.11.norm2.weight": "model-00001-of-00002.safetensors",
|
| 729 |
+
"vision_model.encoder.layers.12.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 730 |
+
"vision_model.encoder.layers.12.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 731 |
+
"vision_model.encoder.layers.12.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 732 |
+
"vision_model.encoder.layers.12.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 733 |
+
"vision_model.encoder.layers.12.ls1": "model-00001-of-00002.safetensors",
|
| 734 |
+
"vision_model.encoder.layers.12.ls2": "model-00001-of-00002.safetensors",
|
| 735 |
+
"vision_model.encoder.layers.12.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 736 |
+
"vision_model.encoder.layers.12.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 737 |
+
"vision_model.encoder.layers.12.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 738 |
+
"vision_model.encoder.layers.12.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 739 |
+
"vision_model.encoder.layers.12.norm1.bias": "model-00001-of-00002.safetensors",
|
| 740 |
+
"vision_model.encoder.layers.12.norm1.weight": "model-00001-of-00002.safetensors",
|
| 741 |
+
"vision_model.encoder.layers.12.norm2.bias": "model-00001-of-00002.safetensors",
|
| 742 |
+
"vision_model.encoder.layers.12.norm2.weight": "model-00001-of-00002.safetensors",
|
| 743 |
+
"vision_model.encoder.layers.13.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 744 |
+
"vision_model.encoder.layers.13.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 745 |
+
"vision_model.encoder.layers.13.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 746 |
+
"vision_model.encoder.layers.13.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 747 |
+
"vision_model.encoder.layers.13.ls1": "model-00001-of-00002.safetensors",
|
| 748 |
+
"vision_model.encoder.layers.13.ls2": "model-00001-of-00002.safetensors",
|
| 749 |
+
"vision_model.encoder.layers.13.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 750 |
+
"vision_model.encoder.layers.13.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 751 |
+
"vision_model.encoder.layers.13.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 752 |
+
"vision_model.encoder.layers.13.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 753 |
+
"vision_model.encoder.layers.13.norm1.bias": "model-00001-of-00002.safetensors",
|
| 754 |
+
"vision_model.encoder.layers.13.norm1.weight": "model-00001-of-00002.safetensors",
|
| 755 |
+
"vision_model.encoder.layers.13.norm2.bias": "model-00001-of-00002.safetensors",
|
| 756 |
+
"vision_model.encoder.layers.13.norm2.weight": "model-00001-of-00002.safetensors",
|
| 757 |
+
"vision_model.encoder.layers.14.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 758 |
+
"vision_model.encoder.layers.14.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 759 |
+
"vision_model.encoder.layers.14.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 760 |
+
"vision_model.encoder.layers.14.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 761 |
+
"vision_model.encoder.layers.14.ls1": "model-00001-of-00002.safetensors",
|
| 762 |
+
"vision_model.encoder.layers.14.ls2": "model-00001-of-00002.safetensors",
|
| 763 |
+
"vision_model.encoder.layers.14.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 764 |
+
"vision_model.encoder.layers.14.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 765 |
+
"vision_model.encoder.layers.14.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 766 |
+
"vision_model.encoder.layers.14.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 767 |
+
"vision_model.encoder.layers.14.norm1.bias": "model-00001-of-00002.safetensors",
|
| 768 |
+
"vision_model.encoder.layers.14.norm1.weight": "model-00001-of-00002.safetensors",
|
| 769 |
+
"vision_model.encoder.layers.14.norm2.bias": "model-00001-of-00002.safetensors",
|
| 770 |
+
"vision_model.encoder.layers.14.norm2.weight": "model-00001-of-00002.safetensors",
|
| 771 |
+
"vision_model.encoder.layers.15.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 772 |
+
"vision_model.encoder.layers.15.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 773 |
+
"vision_model.encoder.layers.15.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 774 |
+
"vision_model.encoder.layers.15.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 775 |
+
"vision_model.encoder.layers.15.ls1": "model-00001-of-00002.safetensors",
|
| 776 |
+
"vision_model.encoder.layers.15.ls2": "model-00001-of-00002.safetensors",
|
| 777 |
+
"vision_model.encoder.layers.15.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 778 |
+
"vision_model.encoder.layers.15.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 779 |
+
"vision_model.encoder.layers.15.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 780 |
+
"vision_model.encoder.layers.15.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 781 |
+
"vision_model.encoder.layers.15.norm1.bias": "model-00001-of-00002.safetensors",
|
| 782 |
+
"vision_model.encoder.layers.15.norm1.weight": "model-00001-of-00002.safetensors",
|
| 783 |
+
"vision_model.encoder.layers.15.norm2.bias": "model-00001-of-00002.safetensors",
|
| 784 |
+
"vision_model.encoder.layers.15.norm2.weight": "model-00001-of-00002.safetensors",
|
| 785 |
+
"vision_model.encoder.layers.16.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 786 |
+
"vision_model.encoder.layers.16.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 787 |
+
"vision_model.encoder.layers.16.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 788 |
+
"vision_model.encoder.layers.16.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 789 |
+
"vision_model.encoder.layers.16.ls1": "model-00001-of-00002.safetensors",
|
| 790 |
+
"vision_model.encoder.layers.16.ls2": "model-00001-of-00002.safetensors",
|
| 791 |
+
"vision_model.encoder.layers.16.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 792 |
+
"vision_model.encoder.layers.16.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 793 |
+
"vision_model.encoder.layers.16.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 794 |
+
"vision_model.encoder.layers.16.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 795 |
+
"vision_model.encoder.layers.16.norm1.bias": "model-00001-of-00002.safetensors",
|
| 796 |
+
"vision_model.encoder.layers.16.norm1.weight": "model-00001-of-00002.safetensors",
|
| 797 |
+
"vision_model.encoder.layers.16.norm2.bias": "model-00001-of-00002.safetensors",
|
| 798 |
+
"vision_model.encoder.layers.16.norm2.weight": "model-00001-of-00002.safetensors",
|
| 799 |
+
"vision_model.encoder.layers.17.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 800 |
+
"vision_model.encoder.layers.17.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 801 |
+
"vision_model.encoder.layers.17.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 802 |
+
"vision_model.encoder.layers.17.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 803 |
+
"vision_model.encoder.layers.17.ls1": "model-00001-of-00002.safetensors",
|
| 804 |
+
"vision_model.encoder.layers.17.ls2": "model-00001-of-00002.safetensors",
|
| 805 |
+
"vision_model.encoder.layers.17.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 806 |
+
"vision_model.encoder.layers.17.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 807 |
+
"vision_model.encoder.layers.17.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 808 |
+
"vision_model.encoder.layers.17.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 809 |
+
"vision_model.encoder.layers.17.norm1.bias": "model-00001-of-00002.safetensors",
|
| 810 |
+
"vision_model.encoder.layers.17.norm1.weight": "model-00001-of-00002.safetensors",
|
| 811 |
+
"vision_model.encoder.layers.17.norm2.bias": "model-00001-of-00002.safetensors",
|
| 812 |
+
"vision_model.encoder.layers.17.norm2.weight": "model-00001-of-00002.safetensors",
|
| 813 |
+
"vision_model.encoder.layers.18.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 814 |
+
"vision_model.encoder.layers.18.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 815 |
+
"vision_model.encoder.layers.18.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 816 |
+
"vision_model.encoder.layers.18.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 817 |
+
"vision_model.encoder.layers.18.ls1": "model-00001-of-00002.safetensors",
|
| 818 |
+
"vision_model.encoder.layers.18.ls2": "model-00001-of-00002.safetensors",
|
| 819 |
+
"vision_model.encoder.layers.18.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 820 |
+
"vision_model.encoder.layers.18.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 821 |
+
"vision_model.encoder.layers.18.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 822 |
+
"vision_model.encoder.layers.18.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 823 |
+
"vision_model.encoder.layers.18.norm1.bias": "model-00001-of-00002.safetensors",
|
| 824 |
+
"vision_model.encoder.layers.18.norm1.weight": "model-00001-of-00002.safetensors",
|
| 825 |
+
"vision_model.encoder.layers.18.norm2.bias": "model-00001-of-00002.safetensors",
|
| 826 |
+
"vision_model.encoder.layers.18.norm2.weight": "model-00001-of-00002.safetensors",
|
| 827 |
+
"vision_model.encoder.layers.19.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 828 |
+
"vision_model.encoder.layers.19.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 829 |
+
"vision_model.encoder.layers.19.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 830 |
+
"vision_model.encoder.layers.19.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 831 |
+
"vision_model.encoder.layers.19.ls1": "model-00001-of-00002.safetensors",
|
| 832 |
+
"vision_model.encoder.layers.19.ls2": "model-00001-of-00002.safetensors",
|
| 833 |
+
"vision_model.encoder.layers.19.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 834 |
+
"vision_model.encoder.layers.19.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 835 |
+
"vision_model.encoder.layers.19.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 836 |
+
"vision_model.encoder.layers.19.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 837 |
+
"vision_model.encoder.layers.19.norm1.bias": "model-00001-of-00002.safetensors",
|
| 838 |
+
"vision_model.encoder.layers.19.norm1.weight": "model-00001-of-00002.safetensors",
|
| 839 |
+
"vision_model.encoder.layers.19.norm2.bias": "model-00001-of-00002.safetensors",
|
| 840 |
+
"vision_model.encoder.layers.19.norm2.weight": "model-00001-of-00002.safetensors",
|
| 841 |
+
"vision_model.encoder.layers.2.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 842 |
+
"vision_model.encoder.layers.2.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 843 |
+
"vision_model.encoder.layers.2.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 844 |
+
"vision_model.encoder.layers.2.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 845 |
+
"vision_model.encoder.layers.2.ls1": "model-00001-of-00002.safetensors",
|
| 846 |
+
"vision_model.encoder.layers.2.ls2": "model-00001-of-00002.safetensors",
|
| 847 |
+
"vision_model.encoder.layers.2.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 848 |
+
"vision_model.encoder.layers.2.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 849 |
+
"vision_model.encoder.layers.2.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 850 |
+
"vision_model.encoder.layers.2.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 851 |
+
"vision_model.encoder.layers.2.norm1.bias": "model-00001-of-00002.safetensors",
|
| 852 |
+
"vision_model.encoder.layers.2.norm1.weight": "model-00001-of-00002.safetensors",
|
| 853 |
+
"vision_model.encoder.layers.2.norm2.bias": "model-00001-of-00002.safetensors",
|
| 854 |
+
"vision_model.encoder.layers.2.norm2.weight": "model-00001-of-00002.safetensors",
|
| 855 |
+
"vision_model.encoder.layers.20.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 856 |
+
"vision_model.encoder.layers.20.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 857 |
+
"vision_model.encoder.layers.20.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 858 |
+
"vision_model.encoder.layers.20.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 859 |
+
"vision_model.encoder.layers.20.ls1": "model-00001-of-00002.safetensors",
|
| 860 |
+
"vision_model.encoder.layers.20.ls2": "model-00001-of-00002.safetensors",
|
| 861 |
+
"vision_model.encoder.layers.20.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 862 |
+
"vision_model.encoder.layers.20.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 863 |
+
"vision_model.encoder.layers.20.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 864 |
+
"vision_model.encoder.layers.20.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 865 |
+
"vision_model.encoder.layers.20.norm1.bias": "model-00001-of-00002.safetensors",
|
| 866 |
+
"vision_model.encoder.layers.20.norm1.weight": "model-00001-of-00002.safetensors",
|
| 867 |
+
"vision_model.encoder.layers.20.norm2.bias": "model-00001-of-00002.safetensors",
|
| 868 |
+
"vision_model.encoder.layers.20.norm2.weight": "model-00001-of-00002.safetensors",
|
| 869 |
+
"vision_model.encoder.layers.21.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 870 |
+
"vision_model.encoder.layers.21.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 871 |
+
"vision_model.encoder.layers.21.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 872 |
+
"vision_model.encoder.layers.21.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 873 |
+
"vision_model.encoder.layers.21.ls1": "model-00001-of-00002.safetensors",
|
| 874 |
+
"vision_model.encoder.layers.21.ls2": "model-00001-of-00002.safetensors",
|
| 875 |
+
"vision_model.encoder.layers.21.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 876 |
+
"vision_model.encoder.layers.21.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 877 |
+
"vision_model.encoder.layers.21.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 878 |
+
"vision_model.encoder.layers.21.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 879 |
+
"vision_model.encoder.layers.21.norm1.bias": "model-00001-of-00002.safetensors",
|
| 880 |
+
"vision_model.encoder.layers.21.norm1.weight": "model-00001-of-00002.safetensors",
|
| 881 |
+
"vision_model.encoder.layers.21.norm2.bias": "model-00001-of-00002.safetensors",
|
| 882 |
+
"vision_model.encoder.layers.21.norm2.weight": "model-00001-of-00002.safetensors",
|
| 883 |
+
"vision_model.encoder.layers.22.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 884 |
+
"vision_model.encoder.layers.22.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 885 |
+
"vision_model.encoder.layers.22.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 886 |
+
"vision_model.encoder.layers.22.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 887 |
+
"vision_model.encoder.layers.22.ls1": "model-00001-of-00002.safetensors",
|
| 888 |
+
"vision_model.encoder.layers.22.ls2": "model-00001-of-00002.safetensors",
|
| 889 |
+
"vision_model.encoder.layers.22.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 890 |
+
"vision_model.encoder.layers.22.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 891 |
+
"vision_model.encoder.layers.22.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 892 |
+
"vision_model.encoder.layers.22.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 893 |
+
"vision_model.encoder.layers.22.norm1.bias": "model-00001-of-00002.safetensors",
|
| 894 |
+
"vision_model.encoder.layers.22.norm1.weight": "model-00001-of-00002.safetensors",
|
| 895 |
+
"vision_model.encoder.layers.22.norm2.bias": "model-00001-of-00002.safetensors",
|
| 896 |
+
"vision_model.encoder.layers.22.norm2.weight": "model-00001-of-00002.safetensors",
|
| 897 |
+
"vision_model.encoder.layers.23.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 898 |
+
"vision_model.encoder.layers.23.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 899 |
+
"vision_model.encoder.layers.23.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 900 |
+
"vision_model.encoder.layers.23.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 901 |
+
"vision_model.encoder.layers.23.ls1": "model-00001-of-00002.safetensors",
|
| 902 |
+
"vision_model.encoder.layers.23.ls2": "model-00001-of-00002.safetensors",
|
| 903 |
+
"vision_model.encoder.layers.23.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 904 |
+
"vision_model.encoder.layers.23.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 905 |
+
"vision_model.encoder.layers.23.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 906 |
+
"vision_model.encoder.layers.23.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 907 |
+
"vision_model.encoder.layers.23.norm1.bias": "model-00001-of-00002.safetensors",
|
| 908 |
+
"vision_model.encoder.layers.23.norm1.weight": "model-00001-of-00002.safetensors",
|
| 909 |
+
"vision_model.encoder.layers.23.norm2.bias": "model-00001-of-00002.safetensors",
|
| 910 |
+
"vision_model.encoder.layers.23.norm2.weight": "model-00001-of-00002.safetensors",
|
| 911 |
+
"vision_model.encoder.layers.3.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 912 |
+
"vision_model.encoder.layers.3.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 913 |
+
"vision_model.encoder.layers.3.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 914 |
+
"vision_model.encoder.layers.3.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 915 |
+
"vision_model.encoder.layers.3.ls1": "model-00001-of-00002.safetensors",
|
| 916 |
+
"vision_model.encoder.layers.3.ls2": "model-00001-of-00002.safetensors",
|
| 917 |
+
"vision_model.encoder.layers.3.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 918 |
+
"vision_model.encoder.layers.3.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 919 |
+
"vision_model.encoder.layers.3.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 920 |
+
"vision_model.encoder.layers.3.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 921 |
+
"vision_model.encoder.layers.3.norm1.bias": "model-00001-of-00002.safetensors",
|
| 922 |
+
"vision_model.encoder.layers.3.norm1.weight": "model-00001-of-00002.safetensors",
|
| 923 |
+
"vision_model.encoder.layers.3.norm2.bias": "model-00001-of-00002.safetensors",
|
| 924 |
+
"vision_model.encoder.layers.3.norm2.weight": "model-00001-of-00002.safetensors",
|
| 925 |
+
"vision_model.encoder.layers.4.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 926 |
+
"vision_model.encoder.layers.4.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 927 |
+
"vision_model.encoder.layers.4.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 928 |
+
"vision_model.encoder.layers.4.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 929 |
+
"vision_model.encoder.layers.4.ls1": "model-00001-of-00002.safetensors",
|
| 930 |
+
"vision_model.encoder.layers.4.ls2": "model-00001-of-00002.safetensors",
|
| 931 |
+
"vision_model.encoder.layers.4.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 932 |
+
"vision_model.encoder.layers.4.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 933 |
+
"vision_model.encoder.layers.4.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 934 |
+
"vision_model.encoder.layers.4.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 935 |
+
"vision_model.encoder.layers.4.norm1.bias": "model-00001-of-00002.safetensors",
|
| 936 |
+
"vision_model.encoder.layers.4.norm1.weight": "model-00001-of-00002.safetensors",
|
| 937 |
+
"vision_model.encoder.layers.4.norm2.bias": "model-00001-of-00002.safetensors",
|
| 938 |
+
"vision_model.encoder.layers.4.norm2.weight": "model-00001-of-00002.safetensors",
|
| 939 |
+
"vision_model.encoder.layers.5.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 940 |
+
"vision_model.encoder.layers.5.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 941 |
+
"vision_model.encoder.layers.5.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 942 |
+
"vision_model.encoder.layers.5.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 943 |
+
"vision_model.encoder.layers.5.ls1": "model-00001-of-00002.safetensors",
|
| 944 |
+
"vision_model.encoder.layers.5.ls2": "model-00001-of-00002.safetensors",
|
| 945 |
+
"vision_model.encoder.layers.5.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 946 |
+
"vision_model.encoder.layers.5.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 947 |
+
"vision_model.encoder.layers.5.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 948 |
+
"vision_model.encoder.layers.5.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 949 |
+
"vision_model.encoder.layers.5.norm1.bias": "model-00001-of-00002.safetensors",
|
| 950 |
+
"vision_model.encoder.layers.5.norm1.weight": "model-00001-of-00002.safetensors",
|
| 951 |
+
"vision_model.encoder.layers.5.norm2.bias": "model-00001-of-00002.safetensors",
|
| 952 |
+
"vision_model.encoder.layers.5.norm2.weight": "model-00001-of-00002.safetensors",
|
| 953 |
+
"vision_model.encoder.layers.6.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 954 |
+
"vision_model.encoder.layers.6.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 955 |
+
"vision_model.encoder.layers.6.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 956 |
+
"vision_model.encoder.layers.6.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 957 |
+
"vision_model.encoder.layers.6.ls1": "model-00001-of-00002.safetensors",
|
| 958 |
+
"vision_model.encoder.layers.6.ls2": "model-00001-of-00002.safetensors",
|
| 959 |
+
"vision_model.encoder.layers.6.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 960 |
+
"vision_model.encoder.layers.6.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 961 |
+
"vision_model.encoder.layers.6.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 962 |
+
"vision_model.encoder.layers.6.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 963 |
+
"vision_model.encoder.layers.6.norm1.bias": "model-00001-of-00002.safetensors",
|
| 964 |
+
"vision_model.encoder.layers.6.norm1.weight": "model-00001-of-00002.safetensors",
|
| 965 |
+
"vision_model.encoder.layers.6.norm2.bias": "model-00001-of-00002.safetensors",
|
| 966 |
+
"vision_model.encoder.layers.6.norm2.weight": "model-00001-of-00002.safetensors",
|
| 967 |
+
"vision_model.encoder.layers.7.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 968 |
+
"vision_model.encoder.layers.7.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 969 |
+
"vision_model.encoder.layers.7.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 970 |
+
"vision_model.encoder.layers.7.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 971 |
+
"vision_model.encoder.layers.7.ls1": "model-00001-of-00002.safetensors",
|
| 972 |
+
"vision_model.encoder.layers.7.ls2": "model-00001-of-00002.safetensors",
|
| 973 |
+
"vision_model.encoder.layers.7.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 974 |
+
"vision_model.encoder.layers.7.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 975 |
+
"vision_model.encoder.layers.7.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 976 |
+
"vision_model.encoder.layers.7.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 977 |
+
"vision_model.encoder.layers.7.norm1.bias": "model-00001-of-00002.safetensors",
|
| 978 |
+
"vision_model.encoder.layers.7.norm1.weight": "model-00001-of-00002.safetensors",
|
| 979 |
+
"vision_model.encoder.layers.7.norm2.bias": "model-00001-of-00002.safetensors",
|
| 980 |
+
"vision_model.encoder.layers.7.norm2.weight": "model-00001-of-00002.safetensors",
|
| 981 |
+
"vision_model.encoder.layers.8.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 982 |
+
"vision_model.encoder.layers.8.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 983 |
+
"vision_model.encoder.layers.8.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 984 |
+
"vision_model.encoder.layers.8.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 985 |
+
"vision_model.encoder.layers.8.ls1": "model-00001-of-00002.safetensors",
|
| 986 |
+
"vision_model.encoder.layers.8.ls2": "model-00001-of-00002.safetensors",
|
| 987 |
+
"vision_model.encoder.layers.8.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 988 |
+
"vision_model.encoder.layers.8.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 989 |
+
"vision_model.encoder.layers.8.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 990 |
+
"vision_model.encoder.layers.8.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 991 |
+
"vision_model.encoder.layers.8.norm1.bias": "model-00001-of-00002.safetensors",
|
| 992 |
+
"vision_model.encoder.layers.8.norm1.weight": "model-00001-of-00002.safetensors",
|
| 993 |
+
"vision_model.encoder.layers.8.norm2.bias": "model-00001-of-00002.safetensors",
|
| 994 |
+
"vision_model.encoder.layers.8.norm2.weight": "model-00001-of-00002.safetensors",
|
| 995 |
+
"vision_model.encoder.layers.9.attn.proj.bias": "model-00001-of-00002.safetensors",
|
| 996 |
+
"vision_model.encoder.layers.9.attn.proj.weight": "model-00001-of-00002.safetensors",
|
| 997 |
+
"vision_model.encoder.layers.9.attn.qkv.bias": "model-00001-of-00002.safetensors",
|
| 998 |
+
"vision_model.encoder.layers.9.attn.qkv.weight": "model-00001-of-00002.safetensors",
|
| 999 |
+
"vision_model.encoder.layers.9.ls1": "model-00001-of-00002.safetensors",
|
| 1000 |
+
"vision_model.encoder.layers.9.ls2": "model-00001-of-00002.safetensors",
|
| 1001 |
+
"vision_model.encoder.layers.9.mlp.fc1.bias": "model-00001-of-00002.safetensors",
|
| 1002 |
+
"vision_model.encoder.layers.9.mlp.fc1.weight": "model-00001-of-00002.safetensors",
|
| 1003 |
+
"vision_model.encoder.layers.9.mlp.fc2.bias": "model-00001-of-00002.safetensors",
|
| 1004 |
+
"vision_model.encoder.layers.9.mlp.fc2.weight": "model-00001-of-00002.safetensors",
|
| 1005 |
+
"vision_model.encoder.layers.9.norm1.bias": "model-00001-of-00002.safetensors",
|
| 1006 |
+
"vision_model.encoder.layers.9.norm1.weight": "model-00001-of-00002.safetensors",
|
| 1007 |
+
"vision_model.encoder.layers.9.norm2.bias": "model-00001-of-00002.safetensors",
|
| 1008 |
+
"vision_model.encoder.layers.9.norm2.weight": "model-00001-of-00002.safetensors"
|
| 1009 |
+
}
|
| 1010 |
+
}
|
modeling_intern_vit.py
ADDED
|
@@ -0,0 +1,430 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# --------------------------------------------------------
|
| 2 |
+
# InternVL
|
| 3 |
+
# Copyright (c) 2024 OpenGVLab
|
| 4 |
+
# Licensed under The MIT License [see LICENSE for details]
|
| 5 |
+
# --------------------------------------------------------
|
| 6 |
+
|
| 7 |
+
from typing import Optional, Tuple, Union
|
| 8 |
+
|
| 9 |
+
import torch
|
| 10 |
+
import torch.nn.functional as F
|
| 11 |
+
import torch.utils.checkpoint
|
| 12 |
+
from einops import rearrange
|
| 13 |
+
from timm.models.layers import DropPath
|
| 14 |
+
from torch import nn
|
| 15 |
+
from transformers.activations import ACT2FN
|
| 16 |
+
from transformers.modeling_outputs import (BaseModelOutput,
|
| 17 |
+
BaseModelOutputWithPooling)
|
| 18 |
+
from transformers.modeling_utils import PreTrainedModel
|
| 19 |
+
from transformers.utils import logging
|
| 20 |
+
|
| 21 |
+
from .configuration_intern_vit import InternVisionConfig
|
| 22 |
+
|
| 23 |
+
try:
|
| 24 |
+
from flash_attn.bert_padding import pad_input, unpad_input
|
| 25 |
+
from flash_attn.flash_attn_interface import \
|
| 26 |
+
flash_attn_varlen_qkvpacked_func
|
| 27 |
+
has_flash_attn = True
|
| 28 |
+
except:
|
| 29 |
+
print('FlashAttention2 is not installed.')
|
| 30 |
+
has_flash_attn = False
|
| 31 |
+
|
| 32 |
+
logger = logging.get_logger(__name__)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
class FlashAttention(nn.Module):
|
| 36 |
+
"""Implement the scaled dot product attention with softmax.
|
| 37 |
+
Arguments
|
| 38 |
+
---------
|
| 39 |
+
softmax_scale: The temperature to use for the softmax attention.
|
| 40 |
+
(default: 1/sqrt(d_keys) where d_keys is computed at
|
| 41 |
+
runtime)
|
| 42 |
+
attention_dropout: The dropout rate to apply to the attention
|
| 43 |
+
(default: 0.0)
|
| 44 |
+
"""
|
| 45 |
+
|
| 46 |
+
def __init__(self, softmax_scale=None, attention_dropout=0.0, device=None, dtype=None):
|
| 47 |
+
super().__init__()
|
| 48 |
+
self.softmax_scale = softmax_scale
|
| 49 |
+
self.dropout_p = attention_dropout
|
| 50 |
+
|
| 51 |
+
def forward(self, qkv, key_padding_mask=None, causal=False, cu_seqlens=None,
|
| 52 |
+
max_s=None, need_weights=False):
|
| 53 |
+
"""Implements the multihead softmax attention.
|
| 54 |
+
Arguments
|
| 55 |
+
---------
|
| 56 |
+
qkv: The tensor containing the query, key, and value. (B, S, 3, H, D) if key_padding_mask is None
|
| 57 |
+
if unpadded: (nnz, 3, h, d)
|
| 58 |
+
key_padding_mask: a bool tensor of shape (B, S)
|
| 59 |
+
"""
|
| 60 |
+
assert not need_weights
|
| 61 |
+
assert qkv.dtype in [torch.float16, torch.bfloat16]
|
| 62 |
+
assert qkv.is_cuda
|
| 63 |
+
|
| 64 |
+
if cu_seqlens is None:
|
| 65 |
+
batch_size = qkv.shape[0]
|
| 66 |
+
seqlen = qkv.shape[1]
|
| 67 |
+
if key_padding_mask is None:
|
| 68 |
+
qkv = rearrange(qkv, 'b s ... -> (b s) ...')
|
| 69 |
+
max_s = seqlen
|
| 70 |
+
cu_seqlens = torch.arange(0, (batch_size + 1) * seqlen, step=seqlen, dtype=torch.int32,
|
| 71 |
+
device=qkv.device)
|
| 72 |
+
output = flash_attn_varlen_qkvpacked_func(
|
| 73 |
+
qkv, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
|
| 74 |
+
softmax_scale=self.softmax_scale, causal=causal
|
| 75 |
+
)
|
| 76 |
+
output = rearrange(output, '(b s) ... -> b s ...', b=batch_size)
|
| 77 |
+
else:
|
| 78 |
+
nheads = qkv.shape[-2]
|
| 79 |
+
x = rearrange(qkv, 'b s three h d -> b s (three h d)')
|
| 80 |
+
x_unpad, indices, cu_seqlens, max_s = unpad_input(x, key_padding_mask)
|
| 81 |
+
x_unpad = rearrange(x_unpad, 'nnz (three h d) -> nnz three h d', three=3, h=nheads)
|
| 82 |
+
output_unpad = flash_attn_varlen_qkvpacked_func(
|
| 83 |
+
x_unpad, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
|
| 84 |
+
softmax_scale=self.softmax_scale, causal=causal
|
| 85 |
+
)
|
| 86 |
+
output = rearrange(pad_input(rearrange(output_unpad, 'nnz h d -> nnz (h d)'),
|
| 87 |
+
indices, batch_size, seqlen),
|
| 88 |
+
'b s (h d) -> b s h d', h=nheads)
|
| 89 |
+
else:
|
| 90 |
+
assert max_s is not None
|
| 91 |
+
output = flash_attn_varlen_qkvpacked_func(
|
| 92 |
+
qkv, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
|
| 93 |
+
softmax_scale=self.softmax_scale, causal=causal
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
return output, None
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
class InternRMSNorm(nn.Module):
|
| 100 |
+
def __init__(self, hidden_size, eps=1e-6):
|
| 101 |
+
super().__init__()
|
| 102 |
+
self.weight = nn.Parameter(torch.ones(hidden_size))
|
| 103 |
+
self.variance_epsilon = eps
|
| 104 |
+
|
| 105 |
+
def forward(self, hidden_states):
|
| 106 |
+
input_dtype = hidden_states.dtype
|
| 107 |
+
hidden_states = hidden_states.to(torch.float32)
|
| 108 |
+
variance = hidden_states.pow(2).mean(-1, keepdim=True)
|
| 109 |
+
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
|
| 110 |
+
return self.weight * hidden_states.to(input_dtype)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
try:
|
| 114 |
+
from apex.normalization import FusedRMSNorm
|
| 115 |
+
|
| 116 |
+
InternRMSNorm = FusedRMSNorm # noqa
|
| 117 |
+
|
| 118 |
+
logger.info('Discovered apex.normalization.FusedRMSNorm - will use it instead of InternRMSNorm')
|
| 119 |
+
except ImportError:
|
| 120 |
+
# using the normal InternRMSNorm
|
| 121 |
+
pass
|
| 122 |
+
except Exception:
|
| 123 |
+
logger.warning('discovered apex but it failed to load, falling back to InternRMSNorm')
|
| 124 |
+
pass
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
NORM2FN = {
|
| 128 |
+
'rms_norm': InternRMSNorm,
|
| 129 |
+
'layer_norm': nn.LayerNorm,
|
| 130 |
+
}
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
class InternVisionEmbeddings(nn.Module):
|
| 134 |
+
def __init__(self, config: InternVisionConfig):
|
| 135 |
+
super().__init__()
|
| 136 |
+
self.config = config
|
| 137 |
+
self.embed_dim = config.hidden_size
|
| 138 |
+
self.image_size = config.image_size
|
| 139 |
+
self.patch_size = config.patch_size
|
| 140 |
+
|
| 141 |
+
self.class_embedding = nn.Parameter(
|
| 142 |
+
torch.randn(1, 1, self.embed_dim),
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
self.patch_embedding = nn.Conv2d(
|
| 146 |
+
in_channels=3, out_channels=self.embed_dim, kernel_size=self.patch_size, stride=self.patch_size
|
| 147 |
+
)
|
| 148 |
+
|
| 149 |
+
self.num_patches = (self.image_size // self.patch_size) ** 2
|
| 150 |
+
self.num_positions = self.num_patches + 1
|
| 151 |
+
|
| 152 |
+
self.position_embedding = nn.Parameter(torch.randn(1, self.num_positions, self.embed_dim))
|
| 153 |
+
|
| 154 |
+
def _get_pos_embed(self, pos_embed, H, W):
|
| 155 |
+
target_dtype = pos_embed.dtype
|
| 156 |
+
pos_embed = pos_embed.float().reshape(
|
| 157 |
+
1, self.image_size // self.patch_size, self.image_size // self.patch_size, -1).permute(0, 3, 1, 2)
|
| 158 |
+
pos_embed = F.interpolate(pos_embed, size=(H, W), mode='bicubic', align_corners=False). \
|
| 159 |
+
reshape(1, -1, H * W).permute(0, 2, 1).to(target_dtype)
|
| 160 |
+
return pos_embed
|
| 161 |
+
|
| 162 |
+
def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor:
|
| 163 |
+
target_dtype = self.patch_embedding.weight.dtype
|
| 164 |
+
patch_embeds = self.patch_embedding(pixel_values) # shape = [*, channel, width, height]
|
| 165 |
+
batch_size, _, height, width = patch_embeds.shape
|
| 166 |
+
patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
|
| 167 |
+
class_embeds = self.class_embedding.expand(batch_size, 1, -1).to(target_dtype)
|
| 168 |
+
embeddings = torch.cat([class_embeds, patch_embeds], dim=1)
|
| 169 |
+
position_embedding = torch.cat([
|
| 170 |
+
self.position_embedding[:, :1, :],
|
| 171 |
+
self._get_pos_embed(self.position_embedding[:, 1:, :], height, width)
|
| 172 |
+
], dim=1)
|
| 173 |
+
embeddings = embeddings + position_embedding.to(target_dtype)
|
| 174 |
+
return embeddings
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
class InternAttention(nn.Module):
|
| 178 |
+
"""Multi-headed attention from 'Attention Is All You Need' paper"""
|
| 179 |
+
|
| 180 |
+
def __init__(self, config: InternVisionConfig):
|
| 181 |
+
super().__init__()
|
| 182 |
+
self.config = config
|
| 183 |
+
self.embed_dim = config.hidden_size
|
| 184 |
+
self.num_heads = config.num_attention_heads
|
| 185 |
+
self.use_flash_attn = config.use_flash_attn and has_flash_attn
|
| 186 |
+
if config.use_flash_attn and not has_flash_attn:
|
| 187 |
+
print('Warning: Flash Attention is not available, use_flash_attn is set to False.')
|
| 188 |
+
self.head_dim = self.embed_dim // self.num_heads
|
| 189 |
+
if self.head_dim * self.num_heads != self.embed_dim:
|
| 190 |
+
raise ValueError(
|
| 191 |
+
f'embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:'
|
| 192 |
+
f' {self.num_heads}).'
|
| 193 |
+
)
|
| 194 |
+
|
| 195 |
+
self.scale = self.head_dim ** -0.5
|
| 196 |
+
self.qkv = nn.Linear(self.embed_dim, 3 * self.embed_dim, bias=config.qkv_bias)
|
| 197 |
+
self.attn_drop = nn.Dropout(config.attention_dropout)
|
| 198 |
+
self.proj_drop = nn.Dropout(config.dropout)
|
| 199 |
+
|
| 200 |
+
self.qk_normalization = config.qk_normalization
|
| 201 |
+
|
| 202 |
+
if self.qk_normalization:
|
| 203 |
+
self.q_norm = InternRMSNorm(self.embed_dim, eps=config.layer_norm_eps)
|
| 204 |
+
self.k_norm = InternRMSNorm(self.embed_dim, eps=config.layer_norm_eps)
|
| 205 |
+
|
| 206 |
+
if self.use_flash_attn:
|
| 207 |
+
self.inner_attn = FlashAttention(attention_dropout=config.attention_dropout)
|
| 208 |
+
self.proj = nn.Linear(self.embed_dim, self.embed_dim)
|
| 209 |
+
|
| 210 |
+
def _naive_attn(self, x):
|
| 211 |
+
B, N, C = x.shape
|
| 212 |
+
qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
|
| 213 |
+
q, k, v = qkv.unbind(0) # make torchscript happy (cannot use tensor as tuple)
|
| 214 |
+
|
| 215 |
+
if self.qk_normalization:
|
| 216 |
+
B_, H_, N_, D_ = q.shape
|
| 217 |
+
q = self.q_norm(q.transpose(1, 2).flatten(-2, -1)).view(B_, N_, H_, D_).transpose(1, 2)
|
| 218 |
+
k = self.k_norm(k.transpose(1, 2).flatten(-2, -1)).view(B_, N_, H_, D_).transpose(1, 2)
|
| 219 |
+
|
| 220 |
+
attn = ((q * self.scale) @ k.transpose(-2, -1))
|
| 221 |
+
attn = attn.softmax(dim=-1)
|
| 222 |
+
attn = self.attn_drop(attn)
|
| 223 |
+
|
| 224 |
+
x = (attn @ v).transpose(1, 2).reshape(B, N, C)
|
| 225 |
+
x = self.proj(x)
|
| 226 |
+
x = self.proj_drop(x)
|
| 227 |
+
return x
|
| 228 |
+
|
| 229 |
+
def _flash_attn(self, x, key_padding_mask=None, need_weights=False):
|
| 230 |
+
qkv = self.qkv(x)
|
| 231 |
+
qkv = rearrange(qkv, 'b s (three h d) -> b s three h d', three=3, h=self.num_heads)
|
| 232 |
+
|
| 233 |
+
if self.qk_normalization:
|
| 234 |
+
q, k, v = qkv.unbind(2)
|
| 235 |
+
q = self.q_norm(q.flatten(-2, -1)).view(q.shape)
|
| 236 |
+
k = self.k_norm(k.flatten(-2, -1)).view(k.shape)
|
| 237 |
+
qkv = torch.stack([q, k, v], dim=2)
|
| 238 |
+
|
| 239 |
+
context, _ = self.inner_attn(
|
| 240 |
+
qkv, key_padding_mask=key_padding_mask, need_weights=need_weights, causal=False
|
| 241 |
+
)
|
| 242 |
+
outs = self.proj(rearrange(context, 'b s h d -> b s (h d)'))
|
| 243 |
+
outs = self.proj_drop(outs)
|
| 244 |
+
return outs
|
| 245 |
+
|
| 246 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 247 |
+
x = self._naive_attn(hidden_states) if not self.use_flash_attn else self._flash_attn(hidden_states)
|
| 248 |
+
return x
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
class InternMLP(nn.Module):
|
| 252 |
+
def __init__(self, config: InternVisionConfig):
|
| 253 |
+
super().__init__()
|
| 254 |
+
self.config = config
|
| 255 |
+
self.act = ACT2FN[config.hidden_act]
|
| 256 |
+
self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
|
| 257 |
+
self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
|
| 258 |
+
|
| 259 |
+
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
| 260 |
+
hidden_states = self.fc1(hidden_states)
|
| 261 |
+
hidden_states = self.act(hidden_states)
|
| 262 |
+
hidden_states = self.fc2(hidden_states)
|
| 263 |
+
return hidden_states
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
class InternVisionEncoderLayer(nn.Module):
|
| 267 |
+
def __init__(self, config: InternVisionConfig, drop_path_rate: float):
|
| 268 |
+
super().__init__()
|
| 269 |
+
self.embed_dim = config.hidden_size
|
| 270 |
+
self.intermediate_size = config.intermediate_size
|
| 271 |
+
self.norm_type = config.norm_type
|
| 272 |
+
|
| 273 |
+
self.attn = InternAttention(config)
|
| 274 |
+
self.mlp = InternMLP(config)
|
| 275 |
+
self.norm1 = NORM2FN[self.norm_type](self.embed_dim, eps=config.layer_norm_eps)
|
| 276 |
+
self.norm2 = NORM2FN[self.norm_type](self.embed_dim, eps=config.layer_norm_eps)
|
| 277 |
+
|
| 278 |
+
self.ls1 = nn.Parameter(config.initializer_factor * torch.ones(self.embed_dim))
|
| 279 |
+
self.ls2 = nn.Parameter(config.initializer_factor * torch.ones(self.embed_dim))
|
| 280 |
+
self.drop_path1 = DropPath(drop_path_rate) if drop_path_rate > 0. else nn.Identity()
|
| 281 |
+
self.drop_path2 = DropPath(drop_path_rate) if drop_path_rate > 0. else nn.Identity()
|
| 282 |
+
|
| 283 |
+
def forward(
|
| 284 |
+
self,
|
| 285 |
+
hidden_states: torch.Tensor,
|
| 286 |
+
) -> Tuple[torch.FloatTensor, Optional[torch.FloatTensor], Optional[Tuple[torch.FloatTensor]]]:
|
| 287 |
+
"""
|
| 288 |
+
Args:
|
| 289 |
+
hidden_states (`Tuple[torch.FloatTensor, Optional[torch.FloatTensor]]`): input to the layer of shape `(batch, seq_len, embed_dim)`
|
| 290 |
+
"""
|
| 291 |
+
hidden_states = hidden_states + self.drop_path1(self.attn(self.norm1(hidden_states).to(hidden_states.dtype)) * self.ls1)
|
| 292 |
+
|
| 293 |
+
hidden_states = hidden_states + self.drop_path2(self.mlp(self.norm2(hidden_states).to(hidden_states.dtype)) * self.ls2)
|
| 294 |
+
|
| 295 |
+
return hidden_states
|
| 296 |
+
|
| 297 |
+
|
| 298 |
+
class InternVisionEncoder(nn.Module):
|
| 299 |
+
"""
|
| 300 |
+
Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
|
| 301 |
+
[`InternEncoderLayer`].
|
| 302 |
+
|
| 303 |
+
Args:
|
| 304 |
+
config (`InternConfig`):
|
| 305 |
+
The corresponding vision configuration for the `InternEncoder`.
|
| 306 |
+
"""
|
| 307 |
+
|
| 308 |
+
def __init__(self, config: InternVisionConfig):
|
| 309 |
+
super().__init__()
|
| 310 |
+
self.config = config
|
| 311 |
+
# stochastic depth decay rule
|
| 312 |
+
dpr = [x.item() for x in torch.linspace(0, config.drop_path_rate, config.num_hidden_layers)]
|
| 313 |
+
self.layers = nn.ModuleList([
|
| 314 |
+
InternVisionEncoderLayer(config, dpr[idx]) for idx in range(config.num_hidden_layers)])
|
| 315 |
+
self.gradient_checkpointing = True
|
| 316 |
+
|
| 317 |
+
def forward(
|
| 318 |
+
self,
|
| 319 |
+
inputs_embeds,
|
| 320 |
+
output_hidden_states: Optional[bool] = None,
|
| 321 |
+
return_dict: Optional[bool] = None,
|
| 322 |
+
) -> Union[Tuple, BaseModelOutput]:
|
| 323 |
+
r"""
|
| 324 |
+
Args:
|
| 325 |
+
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
|
| 326 |
+
Embedded representation of the inputs. Should be float, not int tokens.
|
| 327 |
+
output_hidden_states (`bool`, *optional*):
|
| 328 |
+
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
|
| 329 |
+
for more detail.
|
| 330 |
+
return_dict (`bool`, *optional*):
|
| 331 |
+
Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
|
| 332 |
+
"""
|
| 333 |
+
output_hidden_states = (
|
| 334 |
+
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
| 335 |
+
)
|
| 336 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 337 |
+
|
| 338 |
+
encoder_states = () if output_hidden_states else None
|
| 339 |
+
hidden_states = inputs_embeds
|
| 340 |
+
|
| 341 |
+
for idx, encoder_layer in enumerate(self.layers):
|
| 342 |
+
if output_hidden_states:
|
| 343 |
+
encoder_states = encoder_states + (hidden_states,)
|
| 344 |
+
if self.gradient_checkpointing and self.training:
|
| 345 |
+
layer_outputs = torch.utils.checkpoint.checkpoint(
|
| 346 |
+
encoder_layer,
|
| 347 |
+
hidden_states)
|
| 348 |
+
else:
|
| 349 |
+
layer_outputs = encoder_layer(
|
| 350 |
+
hidden_states,
|
| 351 |
+
)
|
| 352 |
+
hidden_states = layer_outputs
|
| 353 |
+
|
| 354 |
+
if output_hidden_states:
|
| 355 |
+
encoder_states = encoder_states + (hidden_states,)
|
| 356 |
+
|
| 357 |
+
if not return_dict:
|
| 358 |
+
return tuple(v for v in [hidden_states, encoder_states] if v is not None)
|
| 359 |
+
return BaseModelOutput(
|
| 360 |
+
last_hidden_state=hidden_states, hidden_states=encoder_states
|
| 361 |
+
)
|
| 362 |
+
|
| 363 |
+
|
| 364 |
+
class InternVisionModel(PreTrainedModel):
|
| 365 |
+
main_input_name = 'pixel_values'
|
| 366 |
+
_supports_flash_attn_2 = True
|
| 367 |
+
config_class = InternVisionConfig
|
| 368 |
+
_no_split_modules = ['InternVisionEncoderLayer']
|
| 369 |
+
|
| 370 |
+
def __init__(self, config: InternVisionConfig):
|
| 371 |
+
super().__init__(config)
|
| 372 |
+
self.config = config
|
| 373 |
+
|
| 374 |
+
self.embeddings = InternVisionEmbeddings(config)
|
| 375 |
+
self.encoder = InternVisionEncoder(config)
|
| 376 |
+
|
| 377 |
+
def resize_pos_embeddings(self, old_size, new_size, patch_size):
|
| 378 |
+
pos_emb = self.embeddings.position_embedding
|
| 379 |
+
_, num_positions, embed_dim = pos_emb.shape
|
| 380 |
+
cls_emb = pos_emb[:, :1, :]
|
| 381 |
+
pos_emb = pos_emb[:, 1:, :].reshape(1, old_size // patch_size, old_size // patch_size, -1).permute(0, 3, 1, 2)
|
| 382 |
+
pos_emb = F.interpolate(pos_emb.float(), size=new_size // patch_size, mode='bicubic', align_corners=False)
|
| 383 |
+
pos_emb = pos_emb.to(cls_emb.dtype).reshape(1, embed_dim, -1).permute(0, 2, 1)
|
| 384 |
+
pos_emb = torch.cat([cls_emb, pos_emb], dim=1)
|
| 385 |
+
self.embeddings.position_embedding = nn.Parameter(pos_emb)
|
| 386 |
+
self.embeddings.image_size = new_size
|
| 387 |
+
logger.info('Resized position embeddings from {} to {}'.format(old_size, new_size))
|
| 388 |
+
|
| 389 |
+
def get_input_embeddings(self):
|
| 390 |
+
return self.embeddings
|
| 391 |
+
|
| 392 |
+
def forward(
|
| 393 |
+
self,
|
| 394 |
+
pixel_values: Optional[torch.FloatTensor] = None,
|
| 395 |
+
output_hidden_states: Optional[bool] = None,
|
| 396 |
+
return_dict: Optional[bool] = None,
|
| 397 |
+
pixel_embeds: Optional[torch.FloatTensor] = None,
|
| 398 |
+
) -> Union[Tuple, BaseModelOutputWithPooling]:
|
| 399 |
+
output_hidden_states = (
|
| 400 |
+
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
| 401 |
+
)
|
| 402 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 403 |
+
|
| 404 |
+
if pixel_values is None and pixel_embeds is None:
|
| 405 |
+
raise ValueError('You have to specify pixel_values or pixel_embeds')
|
| 406 |
+
|
| 407 |
+
if pixel_embeds is not None:
|
| 408 |
+
hidden_states = pixel_embeds
|
| 409 |
+
else:
|
| 410 |
+
if len(pixel_values.shape) == 4:
|
| 411 |
+
hidden_states = self.embeddings(pixel_values)
|
| 412 |
+
else:
|
| 413 |
+
raise ValueError(f'wrong pixel_values size: {pixel_values.shape}')
|
| 414 |
+
encoder_outputs = self.encoder(
|
| 415 |
+
inputs_embeds=hidden_states,
|
| 416 |
+
output_hidden_states=output_hidden_states,
|
| 417 |
+
return_dict=return_dict,
|
| 418 |
+
)
|
| 419 |
+
last_hidden_state = encoder_outputs.last_hidden_state
|
| 420 |
+
pooled_output = last_hidden_state[:, 0, :]
|
| 421 |
+
|
| 422 |
+
if not return_dict:
|
| 423 |
+
return (last_hidden_state, pooled_output) + encoder_outputs[1:]
|
| 424 |
+
|
| 425 |
+
return BaseModelOutputWithPooling(
|
| 426 |
+
last_hidden_state=last_hidden_state,
|
| 427 |
+
pooler_output=pooled_output,
|
| 428 |
+
hidden_states=encoder_outputs.hidden_states,
|
| 429 |
+
attentions=encoder_outputs.attentions,
|
| 430 |
+
)
|
modeling_internvl_chat.py
ADDED
|
@@ -0,0 +1,435 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# --------------------------------------------------------
|
| 2 |
+
# InternVL
|
| 3 |
+
# Copyright (c) 2024 OpenGVLab
|
| 4 |
+
# Licensed under The MIT License [see LICENSE for details]
|
| 5 |
+
# --------------------------------------------------------
|
| 6 |
+
|
| 7 |
+
import warnings
|
| 8 |
+
from typing import List, Optional, Tuple, Union
|
| 9 |
+
|
| 10 |
+
import torch.utils.checkpoint
|
| 11 |
+
import transformers
|
| 12 |
+
from torch import nn
|
| 13 |
+
from torch.nn import CrossEntropyLoss
|
| 14 |
+
import torch.nn.functional as F
|
| 15 |
+
from transformers import (AutoModel, GenerationConfig, LlamaForCausalLM,
|
| 16 |
+
LlamaTokenizer)
|
| 17 |
+
from transformers.modeling_outputs import CausalLMOutputWithPast
|
| 18 |
+
from transformers.modeling_utils import PreTrainedModel
|
| 19 |
+
from transformers.utils import ModelOutput, logging
|
| 20 |
+
|
| 21 |
+
from .configuration_internvl_chat import InternVLChatConfig
|
| 22 |
+
from .conversation import get_conv_template
|
| 23 |
+
from .modeling_intern_vit import InternVisionModel, has_flash_attn
|
| 24 |
+
from .modeling_phi3 import Phi3ForCausalLM
|
| 25 |
+
from .modeling_radio import RADIOModel
|
| 26 |
+
|
| 27 |
+
# Import all required modules.
|
| 28 |
+
from .radio_adaptor_base import AdaptorBase, RadioOutput, AdaptorInput
|
| 29 |
+
from .radio_adaptor_generic import GenericAdaptor, AdaptorBase
|
| 30 |
+
from .radio_adaptor_mlp import create_mlp_from_state
|
| 31 |
+
from .radio_adaptor_registry import adaptor_registry
|
| 32 |
+
from .radio_cls_token import ClsToken
|
| 33 |
+
from .radio_enable_cpe_support import enable_cpe
|
| 34 |
+
from .radio_enable_spectral_reparam import configure_spectral_reparam_from_args
|
| 35 |
+
from .radio_eradio_model import eradio
|
| 36 |
+
from .radio_model import create_model_from_args
|
| 37 |
+
from .radio_model import RADIOModel as RADIOModelBase, Resolution
|
| 38 |
+
from .radio_input_conditioner import get_default_conditioner, InputConditioner
|
| 39 |
+
from .radio_open_clip_adaptor import OpenCLIP_RADIO
|
| 40 |
+
from .radio_vit_patch_generator import ViTPatchGenerator
|
| 41 |
+
from .radio_vitdet import apply_vitdet_arch, VitDetArgs
|
| 42 |
+
|
| 43 |
+
# Register extra models
|
| 44 |
+
from .radio_extra_timm_models import *
|
| 45 |
+
|
| 46 |
+
from .configuration_radio import RADIOConfig
|
| 47 |
+
|
| 48 |
+
logger = logging.get_logger(__name__)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def version_cmp(v1, v2, op='eq'):
|
| 52 |
+
import operator
|
| 53 |
+
|
| 54 |
+
from packaging import version
|
| 55 |
+
op_func = getattr(operator, op)
|
| 56 |
+
return op_func(version.parse(v1), version.parse(v2))
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
class InternVLChatModel(PreTrainedModel):
|
| 60 |
+
config_class = InternVLChatConfig
|
| 61 |
+
main_input_name = 'pixel_values'
|
| 62 |
+
base_model_prefix = 'language_model'
|
| 63 |
+
_supports_flash_attn_2 = True
|
| 64 |
+
_no_split_modules = ['InternVisionModel', 'LlamaDecoderLayer', 'Phi3DecoderLayer']
|
| 65 |
+
|
| 66 |
+
def __init__(self, config: InternVLChatConfig, vision_model=None, language_model=None, radio_model=None, use_flash_attn=True):
|
| 67 |
+
super().__init__(config)
|
| 68 |
+
|
| 69 |
+
assert version_cmp(transformers.__version__, '4.37.0', 'ge')
|
| 70 |
+
image_size = config.force_image_size or config.vision_config.image_size
|
| 71 |
+
patch_size = config.vision_config.patch_size
|
| 72 |
+
self.patch_size = patch_size
|
| 73 |
+
self.select_layer = config.select_layer
|
| 74 |
+
self.template = config.template
|
| 75 |
+
self.num_image_token = int((image_size // patch_size) ** 2 * (config.downsample_ratio ** 2))
|
| 76 |
+
self.downsample_ratio = config.downsample_ratio
|
| 77 |
+
self.ps_version = config.ps_version
|
| 78 |
+
use_flash_attn = use_flash_attn if has_flash_attn else False
|
| 79 |
+
config.vision_config.use_flash_attn = True if use_flash_attn else False
|
| 80 |
+
config.llm_config._attn_implementation = 'flash_attention_2' if use_flash_attn else 'eager'
|
| 81 |
+
|
| 82 |
+
logger.info(f'num_image_token: {self.num_image_token}')
|
| 83 |
+
logger.info(f'ps_version: {self.ps_version}')
|
| 84 |
+
if vision_model is not None:
|
| 85 |
+
self.vision_model = vision_model
|
| 86 |
+
else:
|
| 87 |
+
self.vision_model = InternVisionModel(config.vision_config)
|
| 88 |
+
if language_model is not None:
|
| 89 |
+
self.language_model = language_model
|
| 90 |
+
else:
|
| 91 |
+
if config.llm_config.architectures[0] == 'LlamaForCausalLM':
|
| 92 |
+
self.language_model = LlamaForCausalLM(config.llm_config)
|
| 93 |
+
elif config.llm_config.architectures[0] == 'Phi3ForCausalLM':
|
| 94 |
+
self.language_model = Phi3ForCausalLM(config.llm_config)
|
| 95 |
+
else:
|
| 96 |
+
raise NotImplementedError(f'{config.llm_config.architectures[0]} is not implemented.')
|
| 97 |
+
if radio_model is not None:
|
| 98 |
+
self.object_tokenizer = radio_model
|
| 99 |
+
else:
|
| 100 |
+
self.object_tokenizer = RADIOModel(config.radio_config)
|
| 101 |
+
|
| 102 |
+
vit_hidden_size = config.vision_config.hidden_size
|
| 103 |
+
llm_hidden_size = config.llm_config.hidden_size
|
| 104 |
+
|
| 105 |
+
self.mlp1 = nn.Sequential(
|
| 106 |
+
nn.LayerNorm(vit_hidden_size * int(1 / self.downsample_ratio) ** 2),
|
| 107 |
+
nn.Linear(vit_hidden_size * int(1 / self.downsample_ratio) ** 2, llm_hidden_size),
|
| 108 |
+
nn.GELU(),
|
| 109 |
+
nn.Linear(llm_hidden_size, llm_hidden_size)
|
| 110 |
+
)
|
| 111 |
+
|
| 112 |
+
# additional modules
|
| 113 |
+
ot_hidden_size = self.object_tokenizer.model.num_features
|
| 114 |
+
self.ot_mlp1 = nn.Sequential(
|
| 115 |
+
nn.LayerNorm(ot_hidden_size,),
|
| 116 |
+
nn.Linear(ot_hidden_size, config.llm_config.hidden_size,),
|
| 117 |
+
nn.GELU(),
|
| 118 |
+
nn.Linear(config.llm_config.hidden_size, config.llm_config.hidden_size)
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
self.ot_config = config.radio_config
|
| 122 |
+
self.img_context_token_id = None
|
| 123 |
+
self.conv_template = get_conv_template(self.template)
|
| 124 |
+
self.system_message = self.conv_template.system_message
|
| 125 |
+
|
| 126 |
+
def _add_special_tokens(self, tokenizer):
|
| 127 |
+
special_tokens = ['<VPT_CONTEXT>', ]
|
| 128 |
+
num_new_tokens = tokenizer.add_tokens(special_tokens, special_tokens=True)
|
| 129 |
+
return tokenizer
|
| 130 |
+
|
| 131 |
+
def forward(
|
| 132 |
+
self,
|
| 133 |
+
pixel_values: torch.FloatTensor,
|
| 134 |
+
input_ids: torch.LongTensor = None,
|
| 135 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 136 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 137 |
+
image_flags: Optional[torch.LongTensor] = None,
|
| 138 |
+
past_key_values: Optional[List[torch.FloatTensor]] = None,
|
| 139 |
+
labels: Optional[torch.LongTensor] = None,
|
| 140 |
+
use_cache: Optional[bool] = None,
|
| 141 |
+
output_attentions: Optional[bool] = None,
|
| 142 |
+
output_hidden_states: Optional[bool] = None,
|
| 143 |
+
return_dict: Optional[bool] = None,
|
| 144 |
+
) -> Union[Tuple, CausalLMOutputWithPast]:
|
| 145 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 146 |
+
|
| 147 |
+
image_flags = image_flags.squeeze(-1)
|
| 148 |
+
input_embeds = self.language_model.get_input_embeddings()(input_ids).clone()
|
| 149 |
+
|
| 150 |
+
vit_embeds = self.extract_feature(pixel_values)
|
| 151 |
+
vit_embeds = vit_embeds[image_flags == 1]
|
| 152 |
+
vit_batch_size = pixel_values.shape[0]
|
| 153 |
+
|
| 154 |
+
B, N, C = input_embeds.shape
|
| 155 |
+
input_embeds = input_embeds.reshape(B * N, C)
|
| 156 |
+
|
| 157 |
+
if torch.distributed.is_initialized() and torch.distributed.get_rank() == 0:
|
| 158 |
+
print(f'dynamic ViT batch size: {vit_batch_size}, images per sample: {vit_batch_size / B}, dynamic token length: {N}')
|
| 159 |
+
|
| 160 |
+
input_ids = input_ids.reshape(B * N)
|
| 161 |
+
selected = (input_ids == self.img_context_token_id)
|
| 162 |
+
try:
|
| 163 |
+
input_embeds[selected] = input_embeds[selected] * 0.0 + vit_embeds.reshape(-1, C)
|
| 164 |
+
except Exception as e:
|
| 165 |
+
vit_embeds = vit_embeds.reshape(-1, C)
|
| 166 |
+
print(f'warning: {e}, input_embeds[selected].shape={input_embeds[selected].shape}, '
|
| 167 |
+
f'vit_embeds.shape={vit_embeds.shape}')
|
| 168 |
+
n_token = selected.sum()
|
| 169 |
+
input_embeds[selected] = input_embeds[selected] * 0.0 + vit_embeds[:n_token]
|
| 170 |
+
|
| 171 |
+
input_embeds = input_embeds.reshape(B, N, C)
|
| 172 |
+
|
| 173 |
+
outputs = self.language_model(
|
| 174 |
+
inputs_embeds=input_embeds,
|
| 175 |
+
attention_mask=attention_mask,
|
| 176 |
+
position_ids=position_ids,
|
| 177 |
+
past_key_values=past_key_values,
|
| 178 |
+
use_cache=use_cache,
|
| 179 |
+
output_attentions=output_attentions,
|
| 180 |
+
output_hidden_states=output_hidden_states,
|
| 181 |
+
return_dict=return_dict,
|
| 182 |
+
)
|
| 183 |
+
logits = outputs.logits
|
| 184 |
+
|
| 185 |
+
loss = None
|
| 186 |
+
if labels is not None:
|
| 187 |
+
# Shift so that tokens < n predict n
|
| 188 |
+
shift_logits = logits[..., :-1, :].contiguous()
|
| 189 |
+
shift_labels = labels[..., 1:].contiguous()
|
| 190 |
+
# Flatten the tokens
|
| 191 |
+
loss_fct = CrossEntropyLoss()
|
| 192 |
+
shift_logits = shift_logits.view(-1, self.language_model.config.vocab_size)
|
| 193 |
+
shift_labels = shift_labels.view(-1)
|
| 194 |
+
# Enable model parallelism
|
| 195 |
+
shift_labels = shift_labels.to(shift_logits.device)
|
| 196 |
+
loss = loss_fct(shift_logits, shift_labels)
|
| 197 |
+
|
| 198 |
+
if not return_dict:
|
| 199 |
+
output = (logits,) + outputs[1:]
|
| 200 |
+
return (loss,) + output if loss is not None else output
|
| 201 |
+
|
| 202 |
+
return CausalLMOutputWithPast(
|
| 203 |
+
loss=loss,
|
| 204 |
+
logits=logits,
|
| 205 |
+
past_key_values=outputs.past_key_values,
|
| 206 |
+
hidden_states=outputs.hidden_states,
|
| 207 |
+
attentions=outputs.attentions,
|
| 208 |
+
)
|
| 209 |
+
|
| 210 |
+
def pixel_shuffle(self, x, scale_factor=0.5):
|
| 211 |
+
n, w, h, c = x.size()
|
| 212 |
+
# N, W, H, C --> N, W, H * scale, C // scale
|
| 213 |
+
x = x.view(n, w, int(h * scale_factor), int(c / scale_factor))
|
| 214 |
+
# N, W, H * scale, C // scale --> N, H * scale, W, C // scale
|
| 215 |
+
x = x.permute(0, 2, 1, 3).contiguous()
|
| 216 |
+
# N, H * scale, W, C // scale --> N, H * scale, W * scale, C // (scale ** 2)
|
| 217 |
+
x = x.view(n, int(h * scale_factor), int(w * scale_factor),
|
| 218 |
+
int(c / (scale_factor * scale_factor)))
|
| 219 |
+
if self.ps_version == 'v1':
|
| 220 |
+
warnings.warn("In ps_version 'v1', the height and width have not been swapped back, "
|
| 221 |
+
'which results in a transposed image.')
|
| 222 |
+
else:
|
| 223 |
+
x = x.permute(0, 2, 1, 3).contiguous()
|
| 224 |
+
return x
|
| 225 |
+
|
| 226 |
+
def extract_feature(self, pixel_values):
|
| 227 |
+
if self.select_layer == -1:
|
| 228 |
+
vit_embeds = self.vision_model(
|
| 229 |
+
pixel_values=pixel_values,
|
| 230 |
+
output_hidden_states=False,
|
| 231 |
+
return_dict=True).last_hidden_state
|
| 232 |
+
else:
|
| 233 |
+
vit_embeds = self.vision_model(
|
| 234 |
+
pixel_values=pixel_values,
|
| 235 |
+
output_hidden_states=True,
|
| 236 |
+
return_dict=True).hidden_states[self.select_layer]
|
| 237 |
+
vit_embeds = vit_embeds[:, 1:, :]
|
| 238 |
+
|
| 239 |
+
h = w = int(vit_embeds.shape[1] ** 0.5)
|
| 240 |
+
vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
|
| 241 |
+
vit_embeds = self.pixel_shuffle(vit_embeds, scale_factor=self.downsample_ratio)
|
| 242 |
+
vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], -1, vit_embeds.shape[-1])
|
| 243 |
+
vit_embeds = self.mlp1(vit_embeds)
|
| 244 |
+
return vit_embeds
|
| 245 |
+
|
| 246 |
+
def batch_chat(self, tokenizer, pixel_values, questions, generation_config, num_patches_list=None,
|
| 247 |
+
history=None, return_history=False, IMG_START_TOKEN='<img>', IMG_END_TOKEN='</img>',
|
| 248 |
+
IMG_CONTEXT_TOKEN='<IMG_CONTEXT>', verbose=False, image_counts=None):
|
| 249 |
+
raise NotImplementedError
|
| 250 |
+
# if history is not None or return_history:
|
| 251 |
+
# print('Now multi-turn chat is not supported in batch_chat.')
|
| 252 |
+
# raise NotImplementedError
|
| 253 |
+
|
| 254 |
+
# if image_counts is not None:
|
| 255 |
+
# num_patches_list = image_counts
|
| 256 |
+
# print('Warning: `image_counts` is deprecated. Please use `num_patches_list` instead.')
|
| 257 |
+
|
| 258 |
+
# img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
|
| 259 |
+
# self.img_context_token_id = img_context_token_id
|
| 260 |
+
|
| 261 |
+
# if verbose and pixel_values is not None:
|
| 262 |
+
# image_bs = pixel_values.shape[0]
|
| 263 |
+
# print(f'dynamic ViT batch size: {image_bs}')
|
| 264 |
+
|
| 265 |
+
# queries = []
|
| 266 |
+
# for idx, num_patches in enumerate(num_patches_list):
|
| 267 |
+
# question = questions[idx]
|
| 268 |
+
# if pixel_values is not None and '<image>' not in question:
|
| 269 |
+
# question = '<image>\n' + question
|
| 270 |
+
# template = get_conv_template(self.template)
|
| 271 |
+
# template.system_message = self.system_message
|
| 272 |
+
# template.append_message(template.roles[0], question)
|
| 273 |
+
# template.append_message(template.roles[1], None)
|
| 274 |
+
# query = template.get_prompt()
|
| 275 |
+
|
| 276 |
+
# image_tokens = IMG_START_TOKEN + IMG_CONTEXT_TOKEN * self.num_image_token * num_patches + IMG_END_TOKEN
|
| 277 |
+
# query = query.replace('<image>', image_tokens, 1)
|
| 278 |
+
# queries.append(query)
|
| 279 |
+
|
| 280 |
+
# tokenizer.padding_side = 'left'
|
| 281 |
+
# model_inputs = tokenizer(queries, return_tensors='pt', padding=True)
|
| 282 |
+
# input_ids = model_inputs['input_ids'].to(self.device)
|
| 283 |
+
# attention_mask = model_inputs['attention_mask'].to(self.device)
|
| 284 |
+
# eos_token_id = tokenizer.convert_tokens_to_ids(template.sep.strip())
|
| 285 |
+
# generation_config['eos_token_id'] = eos_token_id
|
| 286 |
+
# generation_output = self.generate(
|
| 287 |
+
# pixel_values=pixel_values,
|
| 288 |
+
# input_ids=input_ids,
|
| 289 |
+
# attention_mask=attention_mask,
|
| 290 |
+
# **generation_config
|
| 291 |
+
# )
|
| 292 |
+
# responses = tokenizer.batch_decode(generation_output, skip_special_tokens=True)
|
| 293 |
+
# responses = [response.split(template.sep.strip())[0].strip() for response in responses]
|
| 294 |
+
# return responses
|
| 295 |
+
|
| 296 |
+
def chat(self, tokenizer, pixel_values, question, generation_config, history=None, return_history=False,
|
| 297 |
+
num_patches_list=None, IMG_START_TOKEN='<img>', IMG_END_TOKEN='</img>', IMG_CONTEXT_TOKEN='<IMG_CONTEXT>',
|
| 298 |
+
verbose=False, ot_pixel_values=None, ot_visual_prompts=None):
|
| 299 |
+
|
| 300 |
+
tokenizer = self._add_special_tokens(tokenizer)
|
| 301 |
+
self.vpt_content_token_idx = tokenizer('<VPT_CONTEXT>', add_special_tokens=False).input_ids[0]
|
| 302 |
+
|
| 303 |
+
if history is None and pixel_values is not None and '<image>' not in question:
|
| 304 |
+
question = '<image>\n' + question
|
| 305 |
+
|
| 306 |
+
if num_patches_list is None:
|
| 307 |
+
num_patches_list = [pixel_values.shape[0]] if pixel_values is not None else []
|
| 308 |
+
assert pixel_values is None or len(pixel_values) == sum(num_patches_list)
|
| 309 |
+
|
| 310 |
+
img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
|
| 311 |
+
self.img_context_token_id = img_context_token_id
|
| 312 |
+
|
| 313 |
+
template = get_conv_template(self.template)
|
| 314 |
+
template.system_message = self.system_message
|
| 315 |
+
eos_token_id = tokenizer.convert_tokens_to_ids(template.sep.strip())
|
| 316 |
+
|
| 317 |
+
history = [] if history is None else history
|
| 318 |
+
for (old_question, old_answer) in history:
|
| 319 |
+
template.append_message(template.roles[0], old_question)
|
| 320 |
+
template.append_message(template.roles[1], old_answer)
|
| 321 |
+
template.append_message(template.roles[0], question)
|
| 322 |
+
template.append_message(template.roles[1], None)
|
| 323 |
+
query = template.get_prompt()
|
| 324 |
+
|
| 325 |
+
if verbose and pixel_values is not None:
|
| 326 |
+
image_bs = pixel_values.shape[0]
|
| 327 |
+
print(f'dynamic ViT batch size: {image_bs}')
|
| 328 |
+
|
| 329 |
+
# object tokenizer
|
| 330 |
+
if ot_visual_prompts is not None and len(ot_visual_prompts) > 0:
|
| 331 |
+
ot_pixel_values = ot_pixel_values.to(self.object_tokenizer.dtype)
|
| 332 |
+
ot_h, ot_w = ot_pixel_values.shape[-2:]
|
| 333 |
+
ot_num_tokens_h, ot_num_tokens_w = ot_h // self.ot_config.patch_size, ot_w // self.ot_config.patch_size
|
| 334 |
+
summary, ot_embeds = self.object_tokenizer(ot_pixel_values)
|
| 335 |
+
# for param in self.ot_mlp1.parameters():
|
| 336 |
+
# if param.dtype != ot_embeds.dtype:
|
| 337 |
+
# param.data = param.data.to(ot_embeds)
|
| 338 |
+
with torch.autocast(device_type='cuda', dtype=torch.bfloat16):
|
| 339 |
+
ot_embeds = self.ot_mlp1(ot_embeds)
|
| 340 |
+
|
| 341 |
+
ot_object_embeds_list = []
|
| 342 |
+
for fidx, ot_visual_prompts_fi in enumerate(ot_visual_prompts):
|
| 343 |
+
nvp, h, w = ot_visual_prompts_fi.shape
|
| 344 |
+
ot_visual_prompts_fi = ot_visual_prompts_fi[:, None, :, :].to("cuda").to(self.object_tokenizer.dtype)
|
| 345 |
+
ot_visual_prompts_fi = F.interpolate(ot_visual_prompts_fi.to(ot_embeds.dtype), (ot_num_tokens_h, ot_num_tokens_w), mode="bilinear")
|
| 346 |
+
ot_visual_prompts_fi = (ot_visual_prompts_fi > 0.55).to(ot_embeds.dtype)
|
| 347 |
+
ot_visual_prompts_fi = ot_visual_prompts_fi.reshape(nvp, -1)
|
| 348 |
+
|
| 349 |
+
num_vp_tokens = torch.sum(ot_visual_prompts_fi, dim=-1, keepdim=False)
|
| 350 |
+
ot_embeds_fi = ot_embeds[fidx]
|
| 351 |
+
object_embeds = (ot_visual_prompts_fi[:, :, None] / (num_vp_tokens[:, None, None] + 1e-4) * ot_embeds_fi[None, :, :])
|
| 352 |
+
object_embeds = torch.sum(object_embeds, dim=1)
|
| 353 |
+
ot_object_embeds_list.append(object_embeds)
|
| 354 |
+
ot_object_embeds = torch.cat(ot_object_embeds_list)
|
| 355 |
+
else:
|
| 356 |
+
ot_object_embeds = None
|
| 357 |
+
|
| 358 |
+
for num_patches in num_patches_list:
|
| 359 |
+
image_tokens = IMG_START_TOKEN + IMG_CONTEXT_TOKEN * self.num_image_token * num_patches + IMG_END_TOKEN
|
| 360 |
+
query = query.replace('<image>', image_tokens, 1)
|
| 361 |
+
|
| 362 |
+
model_inputs = tokenizer(query, return_tensors='pt')
|
| 363 |
+
input_ids = model_inputs['input_ids'].to(self.device)
|
| 364 |
+
attention_mask = model_inputs['attention_mask'].to(self.device)
|
| 365 |
+
generation_config['eos_token_id'] = eos_token_id
|
| 366 |
+
generation_output = self.generate(
|
| 367 |
+
pixel_values=pixel_values,
|
| 368 |
+
input_ids=input_ids,
|
| 369 |
+
attention_mask=attention_mask,
|
| 370 |
+
ot_object_embeds=ot_object_embeds,
|
| 371 |
+
**generation_config
|
| 372 |
+
)
|
| 373 |
+
response = tokenizer.batch_decode(generation_output, skip_special_tokens=True)[0]
|
| 374 |
+
response = response.split(template.sep.strip())[0].strip()
|
| 375 |
+
history.append((question, response))
|
| 376 |
+
if return_history:
|
| 377 |
+
return response, history
|
| 378 |
+
else:
|
| 379 |
+
query_to_print = query.replace(IMG_CONTEXT_TOKEN, '')
|
| 380 |
+
query_to_print = query_to_print.replace(f'{IMG_START_TOKEN}{IMG_END_TOKEN}', '<image>')
|
| 381 |
+
if verbose:
|
| 382 |
+
print(query_to_print, response)
|
| 383 |
+
return response
|
| 384 |
+
|
| 385 |
+
@torch.no_grad()
|
| 386 |
+
def generate(
|
| 387 |
+
self,
|
| 388 |
+
pixel_values: Optional[torch.FloatTensor] = None,
|
| 389 |
+
input_ids: Optional[torch.FloatTensor] = None,
|
| 390 |
+
attention_mask: Optional[torch.LongTensor] = None,
|
| 391 |
+
visual_features: Optional[torch.FloatTensor] = None,
|
| 392 |
+
generation_config: Optional[GenerationConfig] = None,
|
| 393 |
+
output_hidden_states: Optional[bool] = None,
|
| 394 |
+
ot_object_embeds: Optional[torch.FloatTensor] = None,
|
| 395 |
+
**generate_kwargs,
|
| 396 |
+
) -> torch.LongTensor:
|
| 397 |
+
|
| 398 |
+
assert self.img_context_token_id is not None
|
| 399 |
+
if pixel_values is not None:
|
| 400 |
+
B, N = input_ids.shape
|
| 401 |
+
temp_input_ids = input_ids.clone().flatten()
|
| 402 |
+
temp_input_ids[temp_input_ids == self.vpt_content_token_idx] = self.img_context_token_id
|
| 403 |
+
|
| 404 |
+
if visual_features is not None:
|
| 405 |
+
vit_embeds = visual_features
|
| 406 |
+
else:
|
| 407 |
+
vit_embeds = self.extract_feature(pixel_values)
|
| 408 |
+
input_embeds = self.language_model.get_input_embeddings()(temp_input_ids.reshape(B, N))
|
| 409 |
+
B, N, C = input_embeds.shape
|
| 410 |
+
input_embeds = input_embeds.reshape(B * N, C)
|
| 411 |
+
|
| 412 |
+
input_ids = input_ids.reshape(B * N)
|
| 413 |
+
|
| 414 |
+
if ot_object_embeds is not None:
|
| 415 |
+
selected = (input_ids == self.vpt_content_token_idx)
|
| 416 |
+
input_embeds[selected] = input_embeds[selected] * 0.0 + ot_object_embeds.to(input_embeds.dtype)
|
| 417 |
+
|
| 418 |
+
selected = (input_ids == self.img_context_token_id)
|
| 419 |
+
assert selected.sum() != 0
|
| 420 |
+
input_embeds[selected] = vit_embeds.reshape(-1, C).to(input_embeds.device)
|
| 421 |
+
|
| 422 |
+
input_embeds = input_embeds.reshape(B, N, C)
|
| 423 |
+
else:
|
| 424 |
+
input_embeds = self.language_model.get_input_embeddings()(input_ids)
|
| 425 |
+
|
| 426 |
+
outputs = self.language_model.generate(
|
| 427 |
+
inputs_embeds=input_embeds,
|
| 428 |
+
attention_mask=attention_mask,
|
| 429 |
+
generation_config=generation_config,
|
| 430 |
+
output_hidden_states=output_hidden_states,
|
| 431 |
+
use_cache=True,
|
| 432 |
+
**generate_kwargs,
|
| 433 |
+
)
|
| 434 |
+
|
| 435 |
+
return outputs
|
modeling_phi3.py
ADDED
|
@@ -0,0 +1,1610 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
|
| 15 |
+
""" PyTorch Phi-3 model."""
|
| 16 |
+
|
| 17 |
+
import inspect
|
| 18 |
+
import math
|
| 19 |
+
import warnings
|
| 20 |
+
from typing import List, Optional, Tuple, Union
|
| 21 |
+
|
| 22 |
+
import torch
|
| 23 |
+
import torch.nn.functional as F
|
| 24 |
+
import torch.utils.checkpoint
|
| 25 |
+
from torch import nn
|
| 26 |
+
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
|
| 27 |
+
from transformers.activations import ACT2FN
|
| 28 |
+
from transformers.cache_utils import Cache, DynamicCache
|
| 29 |
+
from transformers.modeling_attn_mask_utils import \
|
| 30 |
+
_prepare_4d_causal_attention_mask
|
| 31 |
+
from transformers.modeling_outputs import (BaseModelOutputWithPast,
|
| 32 |
+
CausalLMOutputWithPast,
|
| 33 |
+
SequenceClassifierOutputWithPast,
|
| 34 |
+
TokenClassifierOutput)
|
| 35 |
+
from transformers.modeling_utils import PreTrainedModel
|
| 36 |
+
from transformers.utils import (add_code_sample_docstrings,
|
| 37 |
+
add_start_docstrings,
|
| 38 |
+
add_start_docstrings_to_model_forward,
|
| 39 |
+
is_flash_attn_2_available,
|
| 40 |
+
is_flash_attn_greater_or_equal_2_10, logging,
|
| 41 |
+
replace_return_docstrings)
|
| 42 |
+
|
| 43 |
+
from .configuration_phi3 import Phi3Config
|
| 44 |
+
|
| 45 |
+
logger = logging.get_logger(__name__)
|
| 46 |
+
|
| 47 |
+
# Transformers scans dependencies in the modeling file, causing issues on conditional loading. The regex only ignores try/catch blocks, but not if statements
|
| 48 |
+
# if is_flash_attn_2_available():
|
| 49 |
+
_flash_supports_window_size = False
|
| 50 |
+
try:
|
| 51 |
+
from flash_attn import flash_attn_func, flash_attn_varlen_func
|
| 52 |
+
from flash_attn.bert_padding import (index_first_axis, pad_input, # noqa
|
| 53 |
+
unpad_input)
|
| 54 |
+
|
| 55 |
+
_flash_supports_window_size = 'window_size' in list(inspect.signature(flash_attn_func).parameters)
|
| 56 |
+
has_flash_attn = True
|
| 57 |
+
except ImportError as error:
|
| 58 |
+
logger.warning(
|
| 59 |
+
f'`flash-attention` package not found, consider installing for better performance: {error}.'
|
| 60 |
+
)
|
| 61 |
+
if not _flash_supports_window_size:
|
| 62 |
+
logger.warning(
|
| 63 |
+
"Current `flash-attenton` does not support `window_size`. Either upgrade or use `attn_implementation='eager'`."
|
| 64 |
+
)
|
| 65 |
+
has_flash_attn = False
|
| 66 |
+
|
| 67 |
+
_CHECKPOINT_FOR_DOC = 'microsoft/Phi-3-mini-4k-instruct'
|
| 68 |
+
_CONFIG_FOR_DOC = 'Phi3Config'
|
| 69 |
+
|
| 70 |
+
PHI3_PRETRAINED_MODEL_ARCHIVE_LIST = [
|
| 71 |
+
'microsoft/Phi-3-mini-4k-instruct',
|
| 72 |
+
'microsoft/Phi-3-mini-128k-instruct',
|
| 73 |
+
# See all Phi-3 models at https://huggingface.co/models?filter=Phi-3
|
| 74 |
+
]
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
# Copied from transformers.models.llama.modeling_llama.LlamaRMSNorm with Llama->Phi3
|
| 78 |
+
class Phi3RMSNorm(nn.Module):
|
| 79 |
+
def __init__(self, hidden_size, eps=1e-6):
|
| 80 |
+
"""
|
| 81 |
+
Phi3RMSNorm is equivalent to T5LayerNorm
|
| 82 |
+
"""
|
| 83 |
+
super().__init__()
|
| 84 |
+
self.weight = nn.Parameter(torch.ones(hidden_size))
|
| 85 |
+
self.variance_epsilon = eps
|
| 86 |
+
|
| 87 |
+
def forward(self, hidden_states):
|
| 88 |
+
input_dtype = hidden_states.dtype
|
| 89 |
+
hidden_states = hidden_states.to(torch.float32)
|
| 90 |
+
variance = hidden_states.pow(2).mean(-1, keepdim=True)
|
| 91 |
+
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
|
| 92 |
+
return self.weight * hidden_states.to(input_dtype)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
# Copied from transformers.models.llama.modeling_llama._get_unpad_data
|
| 96 |
+
def _get_unpad_data(attention_mask):
|
| 97 |
+
seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
|
| 98 |
+
indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
|
| 99 |
+
max_seqlen_in_batch = seqlens_in_batch.max().item()
|
| 100 |
+
cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.int32), (1, 0))
|
| 101 |
+
return (
|
| 102 |
+
indices,
|
| 103 |
+
cu_seqlens,
|
| 104 |
+
max_seqlen_in_batch,
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
# Copied from transformers.models.gemma.modeling_gemma.GemmaRotaryEmbedding with gemma->phi3, Gemma->Phi3
|
| 109 |
+
class Phi3RotaryEmbedding(nn.Module):
|
| 110 |
+
def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
|
| 111 |
+
super().__init__()
|
| 112 |
+
|
| 113 |
+
self.dim = dim
|
| 114 |
+
self.max_position_embeddings = max_position_embeddings
|
| 115 |
+
self.base = base
|
| 116 |
+
self.register_buffer('inv_freq', None, persistent=False)
|
| 117 |
+
|
| 118 |
+
@torch.no_grad()
|
| 119 |
+
def forward(self, x, position_ids, seq_len=None):
|
| 120 |
+
# x: [bs, num_attention_heads, seq_len, head_size]
|
| 121 |
+
if self.inv_freq is None:
|
| 122 |
+
self.inv_freq = 1.0 / (
|
| 123 |
+
self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64, device=x.device).float() / self.dim)
|
| 124 |
+
)
|
| 125 |
+
inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
|
| 126 |
+
position_ids_expanded = position_ids[:, None, :].float()
|
| 127 |
+
# Force float32 since bfloat16 loses precision on long contexts
|
| 128 |
+
# See https://github.com/huggingface/transformers/pull/29285
|
| 129 |
+
device_type = x.device.type
|
| 130 |
+
device_type = device_type if isinstance(device_type, str) and device_type != 'mps' else 'cpu'
|
| 131 |
+
with torch.autocast(device_type=device_type, enabled=False):
|
| 132 |
+
freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
|
| 133 |
+
emb = torch.cat((freqs, freqs), dim=-1)
|
| 134 |
+
cos = emb.cos()
|
| 135 |
+
sin = emb.sin()
|
| 136 |
+
return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
class Phi3SuScaledRotaryEmbedding(Phi3RotaryEmbedding):
|
| 140 |
+
def __init__(self, dim, config, device=None):
|
| 141 |
+
super().__init__(dim, config.max_position_embeddings, config.rope_theta, device)
|
| 142 |
+
|
| 143 |
+
self.short_factor = config.rope_scaling['short_factor']
|
| 144 |
+
self.long_factor = config.rope_scaling['long_factor']
|
| 145 |
+
self.original_max_position_embeddings = config.original_max_position_embeddings
|
| 146 |
+
|
| 147 |
+
@torch.no_grad()
|
| 148 |
+
def forward(self, x, position_ids, seq_len=None):
|
| 149 |
+
seq_len = torch.max(position_ids) + 1
|
| 150 |
+
if seq_len > self.original_max_position_embeddings:
|
| 151 |
+
ext_factors = torch.tensor(self.long_factor, dtype=torch.float32, device=x.device)
|
| 152 |
+
else:
|
| 153 |
+
ext_factors = torch.tensor(self.short_factor, dtype=torch.float32, device=x.device)
|
| 154 |
+
|
| 155 |
+
inv_freq_shape = torch.arange(0, self.dim, 2, dtype=torch.int64, device=x.device).float() / self.dim
|
| 156 |
+
self.inv_freq = 1.0 / (ext_factors * self.base**inv_freq_shape)
|
| 157 |
+
|
| 158 |
+
inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
|
| 159 |
+
position_ids_expanded = position_ids[:, None, :].float()
|
| 160 |
+
|
| 161 |
+
# Force float32 since bfloat16 loses precision on long contexts
|
| 162 |
+
# See https://github.com/huggingface/transformers/pull/29285
|
| 163 |
+
device_type = x.device.type
|
| 164 |
+
device_type = device_type if isinstance(device_type, str) and device_type != 'mps' else 'cpu'
|
| 165 |
+
with torch.autocast(device_type=device_type, enabled=False):
|
| 166 |
+
freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
|
| 167 |
+
emb = torch.cat((freqs, freqs), dim=-1)
|
| 168 |
+
|
| 169 |
+
scale = self.max_position_embeddings / self.original_max_position_embeddings
|
| 170 |
+
if scale <= 1.0:
|
| 171 |
+
scaling_factor = 1.0
|
| 172 |
+
else:
|
| 173 |
+
scaling_factor = math.sqrt(1 + math.log(scale) / math.log(self.original_max_position_embeddings))
|
| 174 |
+
|
| 175 |
+
cos = emb.cos() * scaling_factor
|
| 176 |
+
sin = emb.sin() * scaling_factor
|
| 177 |
+
return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
class Phi3YarnScaledRotaryEmbedding(Phi3RotaryEmbedding):
|
| 181 |
+
def __init__(self, dim, config, device=None):
|
| 182 |
+
super().__init__(dim, config.max_position_embeddings, config.rope_theta, device)
|
| 183 |
+
|
| 184 |
+
self.short_factor = config.rope_scaling['short_factor']
|
| 185 |
+
self.long_factor = config.rope_scaling['long_factor']
|
| 186 |
+
self.original_max_position_embeddings = config.original_max_position_embeddings
|
| 187 |
+
|
| 188 |
+
@torch.no_grad()
|
| 189 |
+
def forward(self, x, position_ids, seq_len=None):
|
| 190 |
+
seq_len = torch.max(position_ids) + 1
|
| 191 |
+
if seq_len > self.original_max_position_embeddings:
|
| 192 |
+
ext_factors = torch.tensor(self.long_factor, dtype=torch.float32, device=x.device)
|
| 193 |
+
else:
|
| 194 |
+
ext_factors = torch.tensor(self.short_factor, dtype=torch.float32, device=x.device)
|
| 195 |
+
|
| 196 |
+
inv_freq_shape = torch.arange(0, self.dim, 2, dtype=torch.int64, device=x.device).float() / self.dim
|
| 197 |
+
self.inv_freq = 1.0 / (ext_factors * self.base**inv_freq_shape)
|
| 198 |
+
|
| 199 |
+
inv_freq_expanded = self.inv_freq[None, :, None].float().expand(position_ids.shape[0], -1, 1)
|
| 200 |
+
position_ids_expanded = position_ids[:, None, :].float()
|
| 201 |
+
|
| 202 |
+
# Force float32 since bfloat16 loses precision on long contexts
|
| 203 |
+
# See https://github.com/huggingface/transformers/pull/29285
|
| 204 |
+
device_type = x.device.type
|
| 205 |
+
device_type = device_type if isinstance(device_type, str) and device_type != 'mps' else 'cpu'
|
| 206 |
+
with torch.autocast(device_type=device_type, enabled=False):
|
| 207 |
+
freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(1, 2)
|
| 208 |
+
emb = torch.cat((freqs, freqs), dim=-1)
|
| 209 |
+
|
| 210 |
+
scale = self.max_position_embeddings / self.original_max_position_embeddings
|
| 211 |
+
if scale <= 1.0:
|
| 212 |
+
scaling_factor = 1.0
|
| 213 |
+
else:
|
| 214 |
+
scaling_factor = 0.1 * math.log(scale) + 1.0
|
| 215 |
+
|
| 216 |
+
cos = emb.cos() * scaling_factor
|
| 217 |
+
sin = emb.sin() * scaling_factor
|
| 218 |
+
return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
# Copied from transformers.models.llama.modeling_llama.rotate_half
|
| 222 |
+
def rotate_half(x):
|
| 223 |
+
"""Rotates half the hidden dims of the input."""
|
| 224 |
+
x1 = x[..., : x.shape[-1] // 2]
|
| 225 |
+
x2 = x[..., x.shape[-1] // 2 :]
|
| 226 |
+
return torch.cat((-x2, x1), dim=-1)
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
# Copied from transformers.models.llama.modeling_llama.apply_rotary_pos_emb
|
| 230 |
+
def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1):
|
| 231 |
+
"""Applies Rotary Position Embedding to the query and key tensors.
|
| 232 |
+
|
| 233 |
+
Args:
|
| 234 |
+
q (`torch.Tensor`): The query tensor.
|
| 235 |
+
k (`torch.Tensor`): The key tensor.
|
| 236 |
+
cos (`torch.Tensor`): The cosine part of the rotary embedding.
|
| 237 |
+
sin (`torch.Tensor`): The sine part of the rotary embedding.
|
| 238 |
+
position_ids (`torch.Tensor`, *optional*):
|
| 239 |
+
Deprecated and unused.
|
| 240 |
+
unsqueeze_dim (`int`, *optional*, defaults to 1):
|
| 241 |
+
The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
|
| 242 |
+
sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
|
| 243 |
+
that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
|
| 244 |
+
k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
|
| 245 |
+
cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
|
| 246 |
+
the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
|
| 247 |
+
Returns:
|
| 248 |
+
`tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
|
| 249 |
+
"""
|
| 250 |
+
cos = cos.unsqueeze(unsqueeze_dim)
|
| 251 |
+
sin = sin.unsqueeze(unsqueeze_dim)
|
| 252 |
+
q_embed = (q * cos) + (rotate_half(q) * sin)
|
| 253 |
+
k_embed = (k * cos) + (rotate_half(k) * sin)
|
| 254 |
+
return q_embed, k_embed
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
class Phi3MLP(nn.Module):
|
| 258 |
+
def __init__(self, config):
|
| 259 |
+
super().__init__()
|
| 260 |
+
|
| 261 |
+
self.config = config
|
| 262 |
+
self.gate_up_proj = nn.Linear(config.hidden_size, 2 * config.intermediate_size, bias=False)
|
| 263 |
+
self.down_proj = nn.Linear(config.intermediate_size, config.hidden_size, bias=False)
|
| 264 |
+
|
| 265 |
+
self.activation_fn = ACT2FN[config.hidden_act]
|
| 266 |
+
|
| 267 |
+
def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor:
|
| 268 |
+
up_states = self.gate_up_proj(hidden_states)
|
| 269 |
+
|
| 270 |
+
gate, up_states = up_states.chunk(2, dim=-1)
|
| 271 |
+
up_states = up_states * self.activation_fn(gate)
|
| 272 |
+
|
| 273 |
+
return self.down_proj(up_states)
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
# Copied from transformers.models.llama.modeling_llama.repeat_kv with llama->phi
|
| 277 |
+
def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
|
| 278 |
+
"""
|
| 279 |
+
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
|
| 280 |
+
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
|
| 281 |
+
"""
|
| 282 |
+
batch, num_key_value_heads, slen, head_dim = hidden_states.shape
|
| 283 |
+
if n_rep == 1:
|
| 284 |
+
return hidden_states
|
| 285 |
+
hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
|
| 286 |
+
return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
class Phi3Attention(nn.Module):
|
| 290 |
+
"""Multi-headed attention from 'Attention Is All You Need' paper"""
|
| 291 |
+
|
| 292 |
+
def __init__(self, config: Phi3Config, layer_idx: Optional[int] = None):
|
| 293 |
+
super().__init__()
|
| 294 |
+
self.config = config
|
| 295 |
+
self.layer_idx = layer_idx
|
| 296 |
+
if layer_idx is None:
|
| 297 |
+
logger.warning_once(
|
| 298 |
+
f'Instantiating {self.__class__.__name__} without passing a `layer_idx` is not recommended and will '
|
| 299 |
+
'lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` '
|
| 300 |
+
'when creating this class.'
|
| 301 |
+
)
|
| 302 |
+
|
| 303 |
+
self.attention_dropout = config.attention_dropout
|
| 304 |
+
self.hidden_size = config.hidden_size
|
| 305 |
+
self.num_heads = config.num_attention_heads
|
| 306 |
+
self.head_dim = self.hidden_size // self.num_heads
|
| 307 |
+
self.num_key_value_heads = config.num_key_value_heads
|
| 308 |
+
self.num_key_value_groups = self.num_heads // self.num_key_value_heads
|
| 309 |
+
self.max_position_embeddings = config.max_position_embeddings
|
| 310 |
+
self.original_max_position_embeddings = config.original_max_position_embeddings
|
| 311 |
+
self.rope_theta = config.rope_theta
|
| 312 |
+
self.rope_scaling = config.rope_scaling
|
| 313 |
+
self.is_causal = True
|
| 314 |
+
|
| 315 |
+
if (self.head_dim * self.num_heads) != self.hidden_size:
|
| 316 |
+
raise ValueError(
|
| 317 |
+
f'hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}'
|
| 318 |
+
f' and `num_heads`: {self.num_heads}).'
|
| 319 |
+
)
|
| 320 |
+
|
| 321 |
+
op_size = self.num_heads * self.head_dim + 2 * (self.num_key_value_heads * self.head_dim)
|
| 322 |
+
self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
|
| 323 |
+
self.qkv_proj = nn.Linear(self.hidden_size, op_size, bias=False)
|
| 324 |
+
self._init_rope()
|
| 325 |
+
|
| 326 |
+
def _init_rope(self):
|
| 327 |
+
if self.rope_scaling is None:
|
| 328 |
+
self.rotary_emb = Phi3RotaryEmbedding(
|
| 329 |
+
self.head_dim,
|
| 330 |
+
max_position_embeddings=self.max_position_embeddings,
|
| 331 |
+
base=self.rope_theta,
|
| 332 |
+
)
|
| 333 |
+
else:
|
| 334 |
+
scaling_type = self.config.rope_scaling['type']
|
| 335 |
+
if scaling_type == 'su':
|
| 336 |
+
self.rotary_emb = Phi3SuScaledRotaryEmbedding(self.head_dim, self.config)
|
| 337 |
+
elif scaling_type == 'yarn':
|
| 338 |
+
self.rotary_emb = Phi3YarnScaledRotaryEmbedding(self.head_dim, self.config)
|
| 339 |
+
else:
|
| 340 |
+
raise ValueError(f'Unknown RoPE scaling type {scaling_type}')
|
| 341 |
+
|
| 342 |
+
def forward(
|
| 343 |
+
self,
|
| 344 |
+
hidden_states: torch.Tensor,
|
| 345 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 346 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 347 |
+
past_key_value: Optional[Cache] = None,
|
| 348 |
+
output_attentions: bool = False,
|
| 349 |
+
use_cache: bool = False,
|
| 350 |
+
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
|
| 351 |
+
logger.warning_once('You are not running the flash-attention implementation, expect numerical differences.')
|
| 352 |
+
|
| 353 |
+
bsz, q_len, _ = hidden_states.size()
|
| 354 |
+
|
| 355 |
+
qkv = self.qkv_proj(hidden_states)
|
| 356 |
+
query_pos = self.num_heads * self.head_dim
|
| 357 |
+
query_states = qkv[..., :query_pos]
|
| 358 |
+
key_states = qkv[..., query_pos : query_pos + self.num_key_value_heads * self.head_dim]
|
| 359 |
+
value_states = qkv[..., query_pos + self.num_key_value_heads * self.head_dim :]
|
| 360 |
+
|
| 361 |
+
query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
|
| 362 |
+
key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
|
| 363 |
+
value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
|
| 364 |
+
|
| 365 |
+
kv_seq_len = key_states.shape[-2]
|
| 366 |
+
if past_key_value is not None:
|
| 367 |
+
if self.layer_idx is None:
|
| 368 |
+
raise ValueError(
|
| 369 |
+
f'The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} '
|
| 370 |
+
'for auto-regressive decoding with k/v caching, please make sure to initialize the attention class '
|
| 371 |
+
'with a layer index.'
|
| 372 |
+
)
|
| 373 |
+
kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
|
| 374 |
+
cos, sin = self.rotary_emb(value_states, position_ids, seq_len=kv_seq_len)
|
| 375 |
+
|
| 376 |
+
query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
|
| 377 |
+
|
| 378 |
+
if past_key_value is not None:
|
| 379 |
+
cache_kwargs = {'sin': sin, 'cos': cos} # Specific to RoPE models
|
| 380 |
+
key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
|
| 381 |
+
|
| 382 |
+
# repeat k/v heads if n_kv_heads < n_heads
|
| 383 |
+
key_states = repeat_kv(key_states, self.num_key_value_groups)
|
| 384 |
+
value_states = repeat_kv(value_states, self.num_key_value_groups)
|
| 385 |
+
|
| 386 |
+
attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
|
| 387 |
+
|
| 388 |
+
if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
|
| 389 |
+
raise ValueError(
|
| 390 |
+
f'Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is'
|
| 391 |
+
f' {attn_weights.size()}'
|
| 392 |
+
)
|
| 393 |
+
|
| 394 |
+
if attention_mask is not None:
|
| 395 |
+
if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
|
| 396 |
+
raise ValueError(
|
| 397 |
+
f'Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}'
|
| 398 |
+
)
|
| 399 |
+
attn_weights = attn_weights + attention_mask
|
| 400 |
+
|
| 401 |
+
# upcast attention to fp32
|
| 402 |
+
attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(value_states.dtype)
|
| 403 |
+
attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training)
|
| 404 |
+
|
| 405 |
+
attn_output = torch.matmul(attn_weights, value_states)
|
| 406 |
+
|
| 407 |
+
if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
|
| 408 |
+
raise ValueError(
|
| 409 |
+
f'`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is'
|
| 410 |
+
f' {attn_output.size()}'
|
| 411 |
+
)
|
| 412 |
+
|
| 413 |
+
attn_output = attn_output.transpose(1, 2).contiguous()
|
| 414 |
+
attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
|
| 415 |
+
|
| 416 |
+
attn_output = self.o_proj(attn_output)
|
| 417 |
+
|
| 418 |
+
if not output_attentions:
|
| 419 |
+
attn_weights = None
|
| 420 |
+
|
| 421 |
+
return attn_output, attn_weights, past_key_value
|
| 422 |
+
|
| 423 |
+
|
| 424 |
+
class Phi3FlashAttention2(Phi3Attention):
|
| 425 |
+
"""
|
| 426 |
+
Phi-3 flash attention module. This module inherits from `Phi3Attention` as the weights of the module stays
|
| 427 |
+
untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
|
| 428 |
+
flash attention and deal with padding tokens in case the input contains any of them.
|
| 429 |
+
"""
|
| 430 |
+
|
| 431 |
+
# Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2.__init__
|
| 432 |
+
def __init__(self, *args, **kwargs):
|
| 433 |
+
super().__init__(*args, **kwargs)
|
| 434 |
+
|
| 435 |
+
# TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1.
|
| 436 |
+
# flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0.
|
| 437 |
+
# Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left).
|
| 438 |
+
self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10()
|
| 439 |
+
|
| 440 |
+
def forward(
|
| 441 |
+
self,
|
| 442 |
+
hidden_states: torch.Tensor,
|
| 443 |
+
attention_mask: Optional[torch.LongTensor] = None,
|
| 444 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 445 |
+
past_key_value: Optional[Cache] = None,
|
| 446 |
+
output_attentions: bool = False,
|
| 447 |
+
use_cache: bool = False,
|
| 448 |
+
**kwargs,
|
| 449 |
+
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
|
| 450 |
+
# Phi3FlashAttention2 attention does not support output_attentions
|
| 451 |
+
|
| 452 |
+
if not _flash_supports_window_size:
|
| 453 |
+
logger.warning_once(
|
| 454 |
+
"The current flash attention version does not support sliding window attention. Please use `attn_implementation='eager'` or upgrade flash-attn library."
|
| 455 |
+
)
|
| 456 |
+
raise ValueError('The current flash attention version does not support sliding window attention.')
|
| 457 |
+
|
| 458 |
+
output_attentions = False
|
| 459 |
+
|
| 460 |
+
if 'padding_mask' in kwargs:
|
| 461 |
+
warnings.warn(
|
| 462 |
+
'Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`'
|
| 463 |
+
)
|
| 464 |
+
|
| 465 |
+
# overwrite attention_mask with padding_mask
|
| 466 |
+
attention_mask = kwargs.pop('padding_mask')
|
| 467 |
+
|
| 468 |
+
bsz, q_len, _ = hidden_states.size()
|
| 469 |
+
|
| 470 |
+
qkv = self.qkv_proj(hidden_states)
|
| 471 |
+
query_pos = self.num_heads * self.head_dim
|
| 472 |
+
query_states = qkv[..., :query_pos]
|
| 473 |
+
key_states = qkv[..., query_pos : query_pos + self.num_key_value_heads * self.head_dim]
|
| 474 |
+
value_states = qkv[..., query_pos + self.num_key_value_heads * self.head_dim :]
|
| 475 |
+
|
| 476 |
+
# Flash attention requires the input to have the shape
|
| 477 |
+
# batch_size x seq_length x head_dim x hidden_dim
|
| 478 |
+
# therefore we just need to keep the original shape
|
| 479 |
+
query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
|
| 480 |
+
key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
|
| 481 |
+
value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
|
| 482 |
+
|
| 483 |
+
kv_seq_len = key_states.shape[-2]
|
| 484 |
+
if past_key_value is not None:
|
| 485 |
+
if self.layer_idx is None:
|
| 486 |
+
raise ValueError(
|
| 487 |
+
f'The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} '
|
| 488 |
+
'for auto-regressive decoding with k/v caching, please make sure to initialize the attention class '
|
| 489 |
+
'with a layer index.'
|
| 490 |
+
)
|
| 491 |
+
kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
|
| 492 |
+
|
| 493 |
+
# Because the input can be padded, the absolute sequence length depends on the max position id.
|
| 494 |
+
rotary_seq_len = max(kv_seq_len, position_ids[:, -1].max().item()) + 1
|
| 495 |
+
cos, sin = self.rotary_emb(value_states, position_ids, seq_len=rotary_seq_len)
|
| 496 |
+
|
| 497 |
+
query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
|
| 498 |
+
|
| 499 |
+
use_sliding_windows = (
|
| 500 |
+
_flash_supports_window_size
|
| 501 |
+
and getattr(self.config, 'sliding_window', None) is not None
|
| 502 |
+
and kv_seq_len > self.config.sliding_window
|
| 503 |
+
)
|
| 504 |
+
|
| 505 |
+
if past_key_value is not None:
|
| 506 |
+
# Activate slicing cache only if the config has a value `sliding_windows` attribute
|
| 507 |
+
cache_has_contents = past_key_value.get_seq_length(self.layer_idx) > 0
|
| 508 |
+
if (
|
| 509 |
+
getattr(self.config, 'sliding_window', None) is not None
|
| 510 |
+
and kv_seq_len > self.config.sliding_window
|
| 511 |
+
and cache_has_contents
|
| 512 |
+
):
|
| 513 |
+
slicing_tokens = 1 - self.config.sliding_window
|
| 514 |
+
|
| 515 |
+
past_key = past_key_value[self.layer_idx][0]
|
| 516 |
+
past_value = past_key_value[self.layer_idx][1]
|
| 517 |
+
|
| 518 |
+
past_key = past_key[:, :, slicing_tokens:, :].contiguous()
|
| 519 |
+
past_value = past_value[:, :, slicing_tokens:, :].contiguous()
|
| 520 |
+
|
| 521 |
+
if past_key.shape[-2] != self.config.sliding_window - 1:
|
| 522 |
+
raise ValueError(
|
| 523 |
+
f'past key must have a shape of (`batch_size, num_heads, self.config.sliding_window-1, head_dim`), got'
|
| 524 |
+
f' {past_key.shape}'
|
| 525 |
+
)
|
| 526 |
+
|
| 527 |
+
if attention_mask is not None:
|
| 528 |
+
attention_mask = attention_mask[:, slicing_tokens:]
|
| 529 |
+
attention_mask = torch.cat([attention_mask, torch.ones_like(attention_mask[:, -1:])], dim=-1)
|
| 530 |
+
|
| 531 |
+
cache_kwargs = {'sin': sin, 'cos': cos} # Specific to RoPE models
|
| 532 |
+
key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
|
| 533 |
+
|
| 534 |
+
# repeat k/v heads if n_kv_heads < n_heads
|
| 535 |
+
key_states = repeat_kv(key_states, self.num_key_value_groups)
|
| 536 |
+
value_states = repeat_kv(value_states, self.num_key_value_groups)
|
| 537 |
+
|
| 538 |
+
attn_dropout = self.attention_dropout if self.training else 0.0
|
| 539 |
+
|
| 540 |
+
# In PEFT, usually we cast the layer norms in float32 for training stability reasons
|
| 541 |
+
# therefore the input hidden states gets silently casted in float32. Hence, we need
|
| 542 |
+
# cast them back in the correct dtype just to be sure everything works as expected.
|
| 543 |
+
# This might slowdown training & inference so it is recommended to not cast the LayerNorms
|
| 544 |
+
# in fp32.
|
| 545 |
+
|
| 546 |
+
if query_states.dtype == torch.float32:
|
| 547 |
+
if torch.is_autocast_enabled():
|
| 548 |
+
target_dtype = torch.get_autocast_gpu_dtype()
|
| 549 |
+
# Handle the case where the model is quantized
|
| 550 |
+
elif hasattr(self.config, '_pre_quantization_dtype'):
|
| 551 |
+
target_dtype = self.config._pre_quantization_dtype
|
| 552 |
+
else:
|
| 553 |
+
target_dtype = self.qkv_proj.weight.dtype
|
| 554 |
+
|
| 555 |
+
logger.warning_once(
|
| 556 |
+
f'The input hidden states seems to be silently casted in float32, this might be related to'
|
| 557 |
+
f' the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in'
|
| 558 |
+
f' {target_dtype}.'
|
| 559 |
+
)
|
| 560 |
+
|
| 561 |
+
query_states = query_states.to(target_dtype)
|
| 562 |
+
key_states = key_states.to(target_dtype)
|
| 563 |
+
value_states = value_states.to(target_dtype)
|
| 564 |
+
|
| 565 |
+
# Reashape to the expected shape for Flash Attention
|
| 566 |
+
query_states = query_states.transpose(1, 2)
|
| 567 |
+
key_states = key_states.transpose(1, 2)
|
| 568 |
+
value_states = value_states.transpose(1, 2)
|
| 569 |
+
|
| 570 |
+
attn_output = self._flash_attention_forward(
|
| 571 |
+
query_states,
|
| 572 |
+
key_states,
|
| 573 |
+
value_states,
|
| 574 |
+
attention_mask,
|
| 575 |
+
q_len,
|
| 576 |
+
dropout=attn_dropout,
|
| 577 |
+
use_sliding_windows=use_sliding_windows,
|
| 578 |
+
)
|
| 579 |
+
|
| 580 |
+
attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous()
|
| 581 |
+
attn_output = self.o_proj(attn_output)
|
| 582 |
+
|
| 583 |
+
if not output_attentions:
|
| 584 |
+
attn_weights = None
|
| 585 |
+
|
| 586 |
+
return attn_output, attn_weights, past_key_value
|
| 587 |
+
|
| 588 |
+
# Copied from transformers.models.mistral.modeling_mistral.MistralFlashAttention2._flash_attention_forward
|
| 589 |
+
def _flash_attention_forward(
|
| 590 |
+
self,
|
| 591 |
+
query_states,
|
| 592 |
+
key_states,
|
| 593 |
+
value_states,
|
| 594 |
+
attention_mask,
|
| 595 |
+
query_length,
|
| 596 |
+
dropout=0.0,
|
| 597 |
+
softmax_scale=None,
|
| 598 |
+
use_sliding_windows=False,
|
| 599 |
+
):
|
| 600 |
+
"""
|
| 601 |
+
Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
|
| 602 |
+
first unpad the input, then computes the attention scores and pad the final attention scores.
|
| 603 |
+
|
| 604 |
+
Args:
|
| 605 |
+
query_states (`torch.Tensor`):
|
| 606 |
+
Input query states to be passed to Flash Attention API
|
| 607 |
+
key_states (`torch.Tensor`):
|
| 608 |
+
Input key states to be passed to Flash Attention API
|
| 609 |
+
value_states (`torch.Tensor`):
|
| 610 |
+
Input value states to be passed to Flash Attention API
|
| 611 |
+
attention_mask (`torch.Tensor`):
|
| 612 |
+
The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the
|
| 613 |
+
position of padding tokens and 1 for the position of non-padding tokens.
|
| 614 |
+
dropout (`float`):
|
| 615 |
+
Attention dropout
|
| 616 |
+
softmax_scale (`float`, *optional*):
|
| 617 |
+
The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)
|
| 618 |
+
use_sliding_windows (`bool`, *optional*):
|
| 619 |
+
Whether to activate sliding window attention.
|
| 620 |
+
"""
|
| 621 |
+
if not self._flash_attn_uses_top_left_mask:
|
| 622 |
+
causal = self.is_causal
|
| 623 |
+
else:
|
| 624 |
+
# TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__.
|
| 625 |
+
causal = self.is_causal and query_length != 1
|
| 626 |
+
|
| 627 |
+
# Contains at least one padding token in the sequence
|
| 628 |
+
if attention_mask is not None:
|
| 629 |
+
batch_size = query_states.shape[0]
|
| 630 |
+
query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input(
|
| 631 |
+
query_states, key_states, value_states, attention_mask, query_length
|
| 632 |
+
)
|
| 633 |
+
|
| 634 |
+
cu_seqlens_q, cu_seqlens_k = cu_seq_lens
|
| 635 |
+
max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
|
| 636 |
+
|
| 637 |
+
if not use_sliding_windows:
|
| 638 |
+
attn_output_unpad = flash_attn_varlen_func(
|
| 639 |
+
query_states,
|
| 640 |
+
key_states,
|
| 641 |
+
value_states,
|
| 642 |
+
cu_seqlens_q=cu_seqlens_q,
|
| 643 |
+
cu_seqlens_k=cu_seqlens_k,
|
| 644 |
+
max_seqlen_q=max_seqlen_in_batch_q,
|
| 645 |
+
max_seqlen_k=max_seqlen_in_batch_k,
|
| 646 |
+
dropout_p=dropout,
|
| 647 |
+
softmax_scale=softmax_scale,
|
| 648 |
+
causal=causal,
|
| 649 |
+
)
|
| 650 |
+
else:
|
| 651 |
+
attn_output_unpad = flash_attn_varlen_func(
|
| 652 |
+
query_states,
|
| 653 |
+
key_states,
|
| 654 |
+
value_states,
|
| 655 |
+
cu_seqlens_q=cu_seqlens_q,
|
| 656 |
+
cu_seqlens_k=cu_seqlens_k,
|
| 657 |
+
max_seqlen_q=max_seqlen_in_batch_q,
|
| 658 |
+
max_seqlen_k=max_seqlen_in_batch_k,
|
| 659 |
+
dropout_p=dropout,
|
| 660 |
+
softmax_scale=softmax_scale,
|
| 661 |
+
causal=causal,
|
| 662 |
+
window_size=(self.config.sliding_window, self.config.sliding_window),
|
| 663 |
+
)
|
| 664 |
+
|
| 665 |
+
attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)
|
| 666 |
+
else:
|
| 667 |
+
if not use_sliding_windows:
|
| 668 |
+
attn_output = flash_attn_func(
|
| 669 |
+
query_states,
|
| 670 |
+
key_states,
|
| 671 |
+
value_states,
|
| 672 |
+
dropout,
|
| 673 |
+
softmax_scale=softmax_scale,
|
| 674 |
+
causal=causal,
|
| 675 |
+
)
|
| 676 |
+
else:
|
| 677 |
+
attn_output = flash_attn_func(
|
| 678 |
+
query_states,
|
| 679 |
+
key_states,
|
| 680 |
+
value_states,
|
| 681 |
+
dropout,
|
| 682 |
+
softmax_scale=softmax_scale,
|
| 683 |
+
causal=causal,
|
| 684 |
+
window_size=(self.config.sliding_window, self.config.sliding_window),
|
| 685 |
+
)
|
| 686 |
+
|
| 687 |
+
return attn_output
|
| 688 |
+
|
| 689 |
+
# Copied from transformers.models.mistral.modeling_mistral.MistralFlashAttention2._upad_input
|
| 690 |
+
def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length):
|
| 691 |
+
batch_size, kv_seq_len, num_heads, head_dim = key_layer.shape
|
| 692 |
+
|
| 693 |
+
# On the first iteration we need to properly re-create the padding mask
|
| 694 |
+
# by slicing it on the proper place
|
| 695 |
+
if kv_seq_len != attention_mask.shape[-1]:
|
| 696 |
+
attention_mask_num_tokens = attention_mask.shape[-1]
|
| 697 |
+
attention_mask = attention_mask[:, attention_mask_num_tokens - kv_seq_len :]
|
| 698 |
+
|
| 699 |
+
indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)
|
| 700 |
+
|
| 701 |
+
key_layer = index_first_axis(key_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k)
|
| 702 |
+
value_layer = index_first_axis(value_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k)
|
| 703 |
+
|
| 704 |
+
if query_length == kv_seq_len:
|
| 705 |
+
query_layer = index_first_axis(
|
| 706 |
+
query_layer.reshape(batch_size * kv_seq_len, num_heads, head_dim), indices_k
|
| 707 |
+
)
|
| 708 |
+
cu_seqlens_q = cu_seqlens_k
|
| 709 |
+
max_seqlen_in_batch_q = max_seqlen_in_batch_k
|
| 710 |
+
indices_q = indices_k
|
| 711 |
+
elif query_length == 1:
|
| 712 |
+
max_seqlen_in_batch_q = 1
|
| 713 |
+
cu_seqlens_q = torch.arange(
|
| 714 |
+
batch_size + 1, dtype=torch.int32, device=query_layer.device
|
| 715 |
+
) # There is a memcpy here, that is very bad.
|
| 716 |
+
indices_q = cu_seqlens_q[:-1]
|
| 717 |
+
query_layer = query_layer.squeeze(1)
|
| 718 |
+
else:
|
| 719 |
+
# The -q_len: slice assumes left padding.
|
| 720 |
+
attention_mask = attention_mask[:, -query_length:]
|
| 721 |
+
query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask)
|
| 722 |
+
|
| 723 |
+
return (
|
| 724 |
+
query_layer,
|
| 725 |
+
key_layer,
|
| 726 |
+
value_layer,
|
| 727 |
+
indices_q,
|
| 728 |
+
(cu_seqlens_q, cu_seqlens_k),
|
| 729 |
+
(max_seqlen_in_batch_q, max_seqlen_in_batch_k),
|
| 730 |
+
)
|
| 731 |
+
|
| 732 |
+
|
| 733 |
+
# copied from transformers.models.llama.modeling_llama.LlamaSdpaAttention with Llama->Phi3
|
| 734 |
+
# TODO @Arthur no longer copied from LLama after static cache
|
| 735 |
+
class Phi3SdpaAttention(Phi3Attention):
|
| 736 |
+
"""
|
| 737 |
+
Phi3 attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from
|
| 738 |
+
`Phi3Attention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to
|
| 739 |
+
SDPA API.
|
| 740 |
+
"""
|
| 741 |
+
|
| 742 |
+
# Adapted from Phi3Attention.forward
|
| 743 |
+
def forward(
|
| 744 |
+
self,
|
| 745 |
+
hidden_states: torch.Tensor,
|
| 746 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 747 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 748 |
+
past_key_value: Optional[Cache] = None,
|
| 749 |
+
output_attentions: bool = False,
|
| 750 |
+
use_cache: bool = False,
|
| 751 |
+
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
|
| 752 |
+
if output_attentions:
|
| 753 |
+
# TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented.
|
| 754 |
+
logger.warning_once(
|
| 755 |
+
'Phi3Model is using Phi3SdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, '
|
| 756 |
+
'but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.'
|
| 757 |
+
)
|
| 758 |
+
return super().forward(
|
| 759 |
+
hidden_states=hidden_states,
|
| 760 |
+
attention_mask=attention_mask,
|
| 761 |
+
position_ids=position_ids,
|
| 762 |
+
past_key_value=past_key_value,
|
| 763 |
+
output_attentions=output_attentions,
|
| 764 |
+
use_cache=use_cache,
|
| 765 |
+
)
|
| 766 |
+
|
| 767 |
+
bsz, q_len, _ = hidden_states.size()
|
| 768 |
+
|
| 769 |
+
qkv = self.qkv_proj(hidden_states)
|
| 770 |
+
query_pos = self.num_heads * self.head_dim
|
| 771 |
+
query_states = qkv[..., :query_pos]
|
| 772 |
+
key_states = qkv[..., query_pos : query_pos + self.num_key_value_heads * self.head_dim]
|
| 773 |
+
value_states = qkv[..., query_pos + self.num_key_value_heads * self.head_dim :]
|
| 774 |
+
|
| 775 |
+
query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
|
| 776 |
+
key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
|
| 777 |
+
value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
|
| 778 |
+
|
| 779 |
+
kv_seq_len = key_states.shape[-2]
|
| 780 |
+
if past_key_value is not None:
|
| 781 |
+
kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
|
| 782 |
+
cos, sin = self.rotary_emb(value_states, position_ids, seq_len=kv_seq_len)
|
| 783 |
+
|
| 784 |
+
query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
|
| 785 |
+
|
| 786 |
+
if past_key_value is not None:
|
| 787 |
+
cache_kwargs = {'sin': sin, 'cos': cos} # Specific to RoPE models
|
| 788 |
+
key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
|
| 789 |
+
|
| 790 |
+
key_states = repeat_kv(key_states, self.num_key_value_groups)
|
| 791 |
+
value_states = repeat_kv(value_states, self.num_key_value_groups)
|
| 792 |
+
|
| 793 |
+
if attention_mask is not None:
|
| 794 |
+
if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
|
| 795 |
+
raise ValueError(
|
| 796 |
+
f'Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}'
|
| 797 |
+
)
|
| 798 |
+
|
| 799 |
+
# SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask,
|
| 800 |
+
# Reference: https://github.com/pytorch/pytorch/issues/112577.
|
| 801 |
+
if query_states.device.type == 'cuda' and attention_mask is not None:
|
| 802 |
+
query_states = query_states.contiguous()
|
| 803 |
+
key_states = key_states.contiguous()
|
| 804 |
+
value_states = value_states.contiguous()
|
| 805 |
+
|
| 806 |
+
attn_output = torch.nn.functional.scaled_dot_product_attention(
|
| 807 |
+
query_states,
|
| 808 |
+
key_states,
|
| 809 |
+
value_states,
|
| 810 |
+
attn_mask=attention_mask,
|
| 811 |
+
dropout_p=self.attention_dropout if self.training else 0.0,
|
| 812 |
+
# The q_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d that does not create a causal mask in case q_len == 1.
|
| 813 |
+
is_causal=self.is_causal and attention_mask is None and q_len > 1,
|
| 814 |
+
)
|
| 815 |
+
|
| 816 |
+
attn_output = attn_output.transpose(1, 2).contiguous()
|
| 817 |
+
attn_output = attn_output.view(bsz, q_len, self.hidden_size)
|
| 818 |
+
|
| 819 |
+
attn_output = self.o_proj(attn_output)
|
| 820 |
+
|
| 821 |
+
return attn_output, None, past_key_value
|
| 822 |
+
|
| 823 |
+
|
| 824 |
+
PHI3_ATTENTION_CLASSES = {
|
| 825 |
+
'eager': Phi3Attention,
|
| 826 |
+
'flash_attention_2': Phi3FlashAttention2,
|
| 827 |
+
'sdpa': Phi3SdpaAttention,
|
| 828 |
+
}
|
| 829 |
+
|
| 830 |
+
|
| 831 |
+
class Phi3DecoderLayer(nn.Module):
|
| 832 |
+
def __init__(self, config: Phi3Config, layer_idx: int):
|
| 833 |
+
super().__init__()
|
| 834 |
+
|
| 835 |
+
self.config = config
|
| 836 |
+
self.self_attn = PHI3_ATTENTION_CLASSES[config._attn_implementation](config, layer_idx=layer_idx)
|
| 837 |
+
|
| 838 |
+
self.mlp = Phi3MLP(config)
|
| 839 |
+
self.input_layernorm = Phi3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 840 |
+
|
| 841 |
+
self.resid_attn_dropout = nn.Dropout(config.resid_pdrop)
|
| 842 |
+
self.resid_mlp_dropout = nn.Dropout(config.resid_pdrop)
|
| 843 |
+
self.post_attention_layernorm = Phi3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 844 |
+
|
| 845 |
+
def forward(
|
| 846 |
+
self,
|
| 847 |
+
hidden_states: torch.Tensor,
|
| 848 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 849 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 850 |
+
past_key_value: Optional[Tuple[torch.Tensor]] = None,
|
| 851 |
+
output_attentions: Optional[bool] = False,
|
| 852 |
+
use_cache: Optional[bool] = False,
|
| 853 |
+
**kwargs,
|
| 854 |
+
) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
|
| 855 |
+
if 'padding_mask' in kwargs:
|
| 856 |
+
warnings.warn(
|
| 857 |
+
'Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`'
|
| 858 |
+
)
|
| 859 |
+
"""
|
| 860 |
+
Args:
|
| 861 |
+
hidden_states (`torch.FloatTensor`):
|
| 862 |
+
input to the layer of shape `(batch, seq_len, embed_dim)`
|
| 863 |
+
attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
|
| 864 |
+
`(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
|
| 865 |
+
position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
|
| 866 |
+
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range
|
| 867 |
+
`[0, config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
|
| 868 |
+
output_attentions (`bool`, *optional*):
|
| 869 |
+
Whether or not to return the attentions tensors of all attention layers. See `attentions` under
|
| 870 |
+
returned tensors for more detail.
|
| 871 |
+
use_cache (`bool`, *optional*):
|
| 872 |
+
If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
|
| 873 |
+
(see `past_key_values`).
|
| 874 |
+
past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
|
| 875 |
+
"""
|
| 876 |
+
|
| 877 |
+
residual = hidden_states
|
| 878 |
+
|
| 879 |
+
hidden_states = self.input_layernorm(hidden_states)
|
| 880 |
+
|
| 881 |
+
# Self Attention
|
| 882 |
+
attn_outputs, self_attn_weights, present_key_value = self.self_attn(
|
| 883 |
+
hidden_states=hidden_states,
|
| 884 |
+
attention_mask=attention_mask,
|
| 885 |
+
position_ids=position_ids,
|
| 886 |
+
past_key_value=past_key_value,
|
| 887 |
+
output_attentions=output_attentions,
|
| 888 |
+
use_cache=use_cache,
|
| 889 |
+
)
|
| 890 |
+
|
| 891 |
+
hidden_states = residual + self.resid_attn_dropout(attn_outputs)
|
| 892 |
+
|
| 893 |
+
residual = hidden_states
|
| 894 |
+
hidden_states = self.post_attention_layernorm(hidden_states)
|
| 895 |
+
hidden_states = self.mlp(hidden_states)
|
| 896 |
+
hidden_states = residual + self.resid_mlp_dropout(hidden_states)
|
| 897 |
+
|
| 898 |
+
outputs = (hidden_states,)
|
| 899 |
+
|
| 900 |
+
if output_attentions:
|
| 901 |
+
outputs += (self_attn_weights,)
|
| 902 |
+
|
| 903 |
+
if use_cache:
|
| 904 |
+
outputs += (present_key_value,)
|
| 905 |
+
|
| 906 |
+
return outputs
|
| 907 |
+
|
| 908 |
+
|
| 909 |
+
PHI3_START_DOCSTRING = r"""
|
| 910 |
+
This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
|
| 911 |
+
library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
|
| 912 |
+
etc.)
|
| 913 |
+
|
| 914 |
+
This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
|
| 915 |
+
Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
|
| 916 |
+
and behavior.
|
| 917 |
+
|
| 918 |
+
Parameters:
|
| 919 |
+
config ([`Phi3Config`]):
|
| 920 |
+
Model configuration class with all the parameters of the model. Initializing with a config file does not
|
| 921 |
+
load the weights associated with the model, only the configuration. Check out the
|
| 922 |
+
[`~PreTrainedModel.from_pretrained`] method to load the model weights.
|
| 923 |
+
"""
|
| 924 |
+
|
| 925 |
+
|
| 926 |
+
@add_start_docstrings(
|
| 927 |
+
'The bare Phi-3 model outputting raw hidden-states without any specific head on top.',
|
| 928 |
+
PHI3_START_DOCSTRING,
|
| 929 |
+
)
|
| 930 |
+
class Phi3PreTrainedModel(PreTrainedModel):
|
| 931 |
+
config_class = Phi3Config
|
| 932 |
+
base_model_prefix = 'model'
|
| 933 |
+
supports_gradient_checkpointing = True
|
| 934 |
+
_no_split_modules = ['Phi3DecoderLayer']
|
| 935 |
+
_skip_keys_device_placement = 'past_key_values'
|
| 936 |
+
_supports_flash_attn_2 = True
|
| 937 |
+
_supports_sdpa = False
|
| 938 |
+
_supports_cache_class = True
|
| 939 |
+
|
| 940 |
+
_version = '0.0.5'
|
| 941 |
+
|
| 942 |
+
def __init__(self, config: Phi3Config):
|
| 943 |
+
if not has_flash_attn:
|
| 944 |
+
config._attn_implementation = 'eager'
|
| 945 |
+
print('Warning: Flash attention is not available, using eager attention instead.')
|
| 946 |
+
super().__init__(config)
|
| 947 |
+
|
| 948 |
+
def _init_weights(self, module):
|
| 949 |
+
std = self.config.initializer_range
|
| 950 |
+
if isinstance(module, nn.Linear):
|
| 951 |
+
module.weight.data.normal_(mean=0.0, std=std)
|
| 952 |
+
if module.bias is not None:
|
| 953 |
+
module.bias.data.zero_()
|
| 954 |
+
elif isinstance(module, nn.Embedding):
|
| 955 |
+
module.weight.data.normal_(mean=0.0, std=std)
|
| 956 |
+
if module.padding_idx is not None:
|
| 957 |
+
module.weight.data[module.padding_idx].zero_()
|
| 958 |
+
|
| 959 |
+
|
| 960 |
+
PHI3_INPUTS_DOCSTRING = r"""
|
| 961 |
+
Args:
|
| 962 |
+
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
|
| 963 |
+
Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
|
| 964 |
+
it.
|
| 965 |
+
|
| 966 |
+
Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
|
| 967 |
+
[`PreTrainedTokenizer.__call__`] for details.
|
| 968 |
+
|
| 969 |
+
[What are input IDs?](../glossary#input-ids)
|
| 970 |
+
attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
|
| 971 |
+
Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
|
| 972 |
+
|
| 973 |
+
- 1 for tokens that are **not masked**,
|
| 974 |
+
- 0 for tokens that are **masked**.
|
| 975 |
+
|
| 976 |
+
[What are attention masks?](../glossary#attention-mask)
|
| 977 |
+
|
| 978 |
+
Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
|
| 979 |
+
[`PreTrainedTokenizer.__call__`] for details.
|
| 980 |
+
|
| 981 |
+
If `past_key_values` is used, optionally only the last `input_ids` have to be input (see
|
| 982 |
+
`past_key_values`).
|
| 983 |
+
|
| 984 |
+
If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
|
| 985 |
+
and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
|
| 986 |
+
information on the default strategy.
|
| 987 |
+
|
| 988 |
+
- 1 indicates the head is **not masked**,
|
| 989 |
+
- 0 indicates the head is **masked**.
|
| 990 |
+
position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
|
| 991 |
+
Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
|
| 992 |
+
config.n_positions - 1]`.
|
| 993 |
+
|
| 994 |
+
[What are position IDs?](../glossary#position-ids)
|
| 995 |
+
past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):
|
| 996 |
+
Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
|
| 997 |
+
blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`
|
| 998 |
+
returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.
|
| 999 |
+
|
| 1000 |
+
Two formats are allowed:
|
| 1001 |
+
- a [`~cache_utils.Cache`] instance;
|
| 1002 |
+
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
|
| 1003 |
+
shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy
|
| 1004 |
+
cache format.
|
| 1005 |
+
|
| 1006 |
+
The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the
|
| 1007 |
+
legacy cache format will be returned.
|
| 1008 |
+
|
| 1009 |
+
If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
|
| 1010 |
+
have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
|
| 1011 |
+
of shape `(batch_size, sequence_length)`.
|
| 1012 |
+
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
|
| 1013 |
+
Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
|
| 1014 |
+
is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
|
| 1015 |
+
model's internal embedding lookup matrix.
|
| 1016 |
+
use_cache (`bool`, *optional*):
|
| 1017 |
+
If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
|
| 1018 |
+
`past_key_values`).
|
| 1019 |
+
output_attentions (`bool`, *optional*):
|
| 1020 |
+
Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
|
| 1021 |
+
tensors for more detail.
|
| 1022 |
+
output_hidden_states (`bool`, *optional*):
|
| 1023 |
+
Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
|
| 1024 |
+
more detail.
|
| 1025 |
+
return_dict (`bool`, *optional*):
|
| 1026 |
+
Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
|
| 1027 |
+
"""
|
| 1028 |
+
|
| 1029 |
+
|
| 1030 |
+
@add_start_docstrings(
|
| 1031 |
+
'The bare Phi-3 model outputting raw hidden-states without any specific head on top.',
|
| 1032 |
+
PHI3_START_DOCSTRING,
|
| 1033 |
+
)
|
| 1034 |
+
class Phi3Model(Phi3PreTrainedModel):
|
| 1035 |
+
"""
|
| 1036 |
+
Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Phi3DecoderLayer`]
|
| 1037 |
+
|
| 1038 |
+
Args:
|
| 1039 |
+
config: Phi3Config
|
| 1040 |
+
"""
|
| 1041 |
+
|
| 1042 |
+
def __init__(self, config: Phi3Config):
|
| 1043 |
+
super().__init__(config)
|
| 1044 |
+
self.padding_idx = config.pad_token_id
|
| 1045 |
+
self.vocab_size = config.vocab_size
|
| 1046 |
+
|
| 1047 |
+
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
|
| 1048 |
+
self.embed_dropout = nn.Dropout(config.embd_pdrop)
|
| 1049 |
+
self.layers = nn.ModuleList(
|
| 1050 |
+
[Phi3DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
|
| 1051 |
+
)
|
| 1052 |
+
self._attn_implementation = config._attn_implementation
|
| 1053 |
+
|
| 1054 |
+
self.norm = Phi3RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
| 1055 |
+
|
| 1056 |
+
self.gradient_checkpointing = False
|
| 1057 |
+
# Initialize weights and apply final processing
|
| 1058 |
+
self.post_init()
|
| 1059 |
+
|
| 1060 |
+
def get_input_embeddings(self):
|
| 1061 |
+
return self.embed_tokens
|
| 1062 |
+
|
| 1063 |
+
def set_input_embeddings(self, value):
|
| 1064 |
+
self.embed_tokens = value
|
| 1065 |
+
|
| 1066 |
+
@add_start_docstrings_to_model_forward(PHI3_INPUTS_DOCSTRING)
|
| 1067 |
+
def forward(
|
| 1068 |
+
self,
|
| 1069 |
+
input_ids: torch.LongTensor = None,
|
| 1070 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 1071 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 1072 |
+
past_key_values: Optional[List[torch.FloatTensor]] = None,
|
| 1073 |
+
inputs_embeds: Optional[torch.FloatTensor] = None,
|
| 1074 |
+
use_cache: Optional[bool] = None,
|
| 1075 |
+
output_attentions: Optional[bool] = None,
|
| 1076 |
+
output_hidden_states: Optional[bool] = None,
|
| 1077 |
+
return_dict: Optional[bool] = None,
|
| 1078 |
+
) -> Union[Tuple, BaseModelOutputWithPast]:
|
| 1079 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
| 1080 |
+
output_hidden_states = (
|
| 1081 |
+
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
| 1082 |
+
)
|
| 1083 |
+
use_cache = use_cache if use_cache is not None else self.config.use_cache
|
| 1084 |
+
|
| 1085 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 1086 |
+
|
| 1087 |
+
# retrieve input_ids and inputs_embeds
|
| 1088 |
+
if input_ids is not None and inputs_embeds is not None:
|
| 1089 |
+
raise ValueError('You cannot specify both input_ids and inputs_embeds at the same time')
|
| 1090 |
+
elif input_ids is not None:
|
| 1091 |
+
batch_size, seq_length = input_ids.shape[:2]
|
| 1092 |
+
elif inputs_embeds is not None:
|
| 1093 |
+
batch_size, seq_length = inputs_embeds.shape[:2]
|
| 1094 |
+
else:
|
| 1095 |
+
raise ValueError('You have to specify either input_ids or inputs_embeds')
|
| 1096 |
+
|
| 1097 |
+
past_key_values_length = 0
|
| 1098 |
+
|
| 1099 |
+
if self.gradient_checkpointing and self.training:
|
| 1100 |
+
if use_cache:
|
| 1101 |
+
logger.warning_once(
|
| 1102 |
+
'`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...'
|
| 1103 |
+
)
|
| 1104 |
+
use_cache = False
|
| 1105 |
+
|
| 1106 |
+
if use_cache:
|
| 1107 |
+
use_legacy_cache = not isinstance(past_key_values, Cache)
|
| 1108 |
+
if use_legacy_cache:
|
| 1109 |
+
past_key_values = DynamicCache.from_legacy_cache(past_key_values)
|
| 1110 |
+
past_key_values_length = past_key_values.get_usable_length(seq_length)
|
| 1111 |
+
|
| 1112 |
+
if position_ids is None:
|
| 1113 |
+
device = input_ids.device if input_ids is not None else inputs_embeds.device
|
| 1114 |
+
position_ids = torch.arange(
|
| 1115 |
+
past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device
|
| 1116 |
+
)
|
| 1117 |
+
position_ids = position_ids.unsqueeze(0).view(-1, seq_length)
|
| 1118 |
+
else:
|
| 1119 |
+
position_ids = position_ids.view(-1, seq_length).long()
|
| 1120 |
+
|
| 1121 |
+
if inputs_embeds is None:
|
| 1122 |
+
inputs_embeds = self.embed_tokens(input_ids)
|
| 1123 |
+
|
| 1124 |
+
if attention_mask is not None and self._attn_implementation == 'flash_attention_2' and use_cache:
|
| 1125 |
+
is_padding_right = attention_mask[:, -1].sum().item() != batch_size
|
| 1126 |
+
if is_padding_right:
|
| 1127 |
+
raise ValueError(
|
| 1128 |
+
"You are attempting to perform batched generation with padding_side='right'"
|
| 1129 |
+
' this may lead to unexpected behaviour for Flash Attention version of Phi3. Make sure to '
|
| 1130 |
+
" call `tokenizer.padding_side = 'left'` before tokenizing the input. "
|
| 1131 |
+
)
|
| 1132 |
+
|
| 1133 |
+
if self._attn_implementation == 'flash_attention_2':
|
| 1134 |
+
# 2d mask is passed through the layers
|
| 1135 |
+
attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None
|
| 1136 |
+
else:
|
| 1137 |
+
# 4d mask is passed through the layers
|
| 1138 |
+
attention_mask = _prepare_4d_causal_attention_mask(
|
| 1139 |
+
attention_mask,
|
| 1140 |
+
(batch_size, seq_length),
|
| 1141 |
+
inputs_embeds,
|
| 1142 |
+
past_key_values_length,
|
| 1143 |
+
sliding_window=self.config.sliding_window,
|
| 1144 |
+
)
|
| 1145 |
+
|
| 1146 |
+
hidden_states = inputs_embeds
|
| 1147 |
+
|
| 1148 |
+
# decoder layers
|
| 1149 |
+
all_hidden_states = () if output_hidden_states else None
|
| 1150 |
+
all_self_attns = () if output_attentions else None
|
| 1151 |
+
next_decoder_cache = None
|
| 1152 |
+
|
| 1153 |
+
for decoder_layer in self.layers:
|
| 1154 |
+
if output_hidden_states:
|
| 1155 |
+
all_hidden_states += (hidden_states,)
|
| 1156 |
+
|
| 1157 |
+
if self.gradient_checkpointing and self.training:
|
| 1158 |
+
layer_outputs = self._gradient_checkpointing_func(
|
| 1159 |
+
decoder_layer.__call__,
|
| 1160 |
+
hidden_states,
|
| 1161 |
+
attention_mask,
|
| 1162 |
+
position_ids,
|
| 1163 |
+
past_key_values,
|
| 1164 |
+
output_attentions,
|
| 1165 |
+
use_cache,
|
| 1166 |
+
)
|
| 1167 |
+
else:
|
| 1168 |
+
layer_outputs = decoder_layer(
|
| 1169 |
+
hidden_states,
|
| 1170 |
+
attention_mask=attention_mask,
|
| 1171 |
+
position_ids=position_ids,
|
| 1172 |
+
past_key_value=past_key_values,
|
| 1173 |
+
output_attentions=output_attentions,
|
| 1174 |
+
use_cache=use_cache,
|
| 1175 |
+
)
|
| 1176 |
+
|
| 1177 |
+
hidden_states = layer_outputs[0]
|
| 1178 |
+
|
| 1179 |
+
if use_cache:
|
| 1180 |
+
next_decoder_cache = layer_outputs[2 if output_attentions else 1]
|
| 1181 |
+
|
| 1182 |
+
if output_attentions:
|
| 1183 |
+
all_self_attns += (layer_outputs[1],)
|
| 1184 |
+
|
| 1185 |
+
hidden_states = self.norm(hidden_states)
|
| 1186 |
+
|
| 1187 |
+
# add hidden states from the last decoder layer
|
| 1188 |
+
if output_hidden_states:
|
| 1189 |
+
all_hidden_states += (hidden_states,)
|
| 1190 |
+
|
| 1191 |
+
next_cache = None
|
| 1192 |
+
if use_cache:
|
| 1193 |
+
next_cache = next_decoder_cache.to_legacy_cache() if use_legacy_cache else next_decoder_cache
|
| 1194 |
+
if not return_dict:
|
| 1195 |
+
return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)
|
| 1196 |
+
return BaseModelOutputWithPast(
|
| 1197 |
+
last_hidden_state=hidden_states,
|
| 1198 |
+
past_key_values=next_cache,
|
| 1199 |
+
hidden_states=all_hidden_states,
|
| 1200 |
+
attentions=all_self_attns,
|
| 1201 |
+
)
|
| 1202 |
+
|
| 1203 |
+
|
| 1204 |
+
class Phi3ForCausalLM(Phi3PreTrainedModel):
|
| 1205 |
+
_tied_weights_keys = ['lm_head.weight']
|
| 1206 |
+
|
| 1207 |
+
# Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.__init__ with Llama->Phi3
|
| 1208 |
+
def __init__(self, config):
|
| 1209 |
+
super().__init__(config)
|
| 1210 |
+
self.model = Phi3Model(config)
|
| 1211 |
+
self.vocab_size = config.vocab_size
|
| 1212 |
+
self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
|
| 1213 |
+
|
| 1214 |
+
# Initialize weights and apply final processing
|
| 1215 |
+
self.post_init()
|
| 1216 |
+
|
| 1217 |
+
# Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.get_input_embeddings
|
| 1218 |
+
def get_input_embeddings(self):
|
| 1219 |
+
return self.model.embed_tokens
|
| 1220 |
+
|
| 1221 |
+
# Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.set_input_embeddings
|
| 1222 |
+
def set_input_embeddings(self, value):
|
| 1223 |
+
self.model.embed_tokens = value
|
| 1224 |
+
|
| 1225 |
+
# Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.get_output_embeddings
|
| 1226 |
+
def get_output_embeddings(self):
|
| 1227 |
+
return self.lm_head
|
| 1228 |
+
|
| 1229 |
+
# Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.set_output_embeddings
|
| 1230 |
+
def set_output_embeddings(self, new_embeddings):
|
| 1231 |
+
self.lm_head = new_embeddings
|
| 1232 |
+
|
| 1233 |
+
# Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.set_decoder
|
| 1234 |
+
def set_decoder(self, decoder):
|
| 1235 |
+
self.model = decoder
|
| 1236 |
+
|
| 1237 |
+
# Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.get_decoder
|
| 1238 |
+
def get_decoder(self):
|
| 1239 |
+
return self.model
|
| 1240 |
+
|
| 1241 |
+
# Ignore copy
|
| 1242 |
+
@add_start_docstrings_to_model_forward(PHI3_INPUTS_DOCSTRING)
|
| 1243 |
+
@replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)
|
| 1244 |
+
def forward(
|
| 1245 |
+
self,
|
| 1246 |
+
input_ids: torch.LongTensor = None,
|
| 1247 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 1248 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 1249 |
+
past_key_values: Optional[List[torch.FloatTensor]] = None,
|
| 1250 |
+
inputs_embeds: Optional[torch.FloatTensor] = None,
|
| 1251 |
+
labels: Optional[torch.LongTensor] = None,
|
| 1252 |
+
use_cache: Optional[bool] = None,
|
| 1253 |
+
output_attentions: Optional[bool] = None,
|
| 1254 |
+
output_hidden_states: Optional[bool] = None,
|
| 1255 |
+
return_dict: Optional[bool] = None,
|
| 1256 |
+
) -> Union[Tuple, CausalLMOutputWithPast]:
|
| 1257 |
+
r"""
|
| 1258 |
+
Args:
|
| 1259 |
+
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
|
| 1260 |
+
Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
|
| 1261 |
+
config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
|
| 1262 |
+
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
|
| 1263 |
+
|
| 1264 |
+
Returns:
|
| 1265 |
+
|
| 1266 |
+
Example:
|
| 1267 |
+
|
| 1268 |
+
```python
|
| 1269 |
+
>>> from transformers import AutoTokenizer, Phi3ForCausalLM
|
| 1270 |
+
|
| 1271 |
+
>>> model = Phi3ForCausalLM.from_pretrained("microsoft/phi-3-mini-4k-instruct")
|
| 1272 |
+
>>> tokenizer = AutoTokenizer.from_pretrained("microsoft/phi-3-mini-4k-instruct")
|
| 1273 |
+
|
| 1274 |
+
>>> prompt = "This is an example script ."
|
| 1275 |
+
>>> inputs = tokenizer(prompt, return_tensors="pt")
|
| 1276 |
+
|
| 1277 |
+
>>> # Generate
|
| 1278 |
+
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
|
| 1279 |
+
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
|
| 1280 |
+
'This is an example script .\n Certainly! Below is a sample script that demonstrates a simple task, such as calculating the sum'
|
| 1281 |
+
```"""
|
| 1282 |
+
|
| 1283 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
| 1284 |
+
output_hidden_states = (
|
| 1285 |
+
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
| 1286 |
+
)
|
| 1287 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 1288 |
+
|
| 1289 |
+
# decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
|
| 1290 |
+
outputs = self.model(
|
| 1291 |
+
input_ids=input_ids,
|
| 1292 |
+
attention_mask=attention_mask,
|
| 1293 |
+
position_ids=position_ids,
|
| 1294 |
+
past_key_values=past_key_values,
|
| 1295 |
+
inputs_embeds=inputs_embeds,
|
| 1296 |
+
use_cache=use_cache,
|
| 1297 |
+
output_attentions=output_attentions,
|
| 1298 |
+
output_hidden_states=output_hidden_states,
|
| 1299 |
+
return_dict=return_dict,
|
| 1300 |
+
)
|
| 1301 |
+
|
| 1302 |
+
hidden_states = outputs[0]
|
| 1303 |
+
logits = self.lm_head(hidden_states)
|
| 1304 |
+
logits = logits.float()
|
| 1305 |
+
|
| 1306 |
+
loss = None
|
| 1307 |
+
if labels is not None:
|
| 1308 |
+
# Shift so that tokens < n predict n
|
| 1309 |
+
shift_logits = logits[..., :-1, :].contiguous()
|
| 1310 |
+
shift_labels = labels[..., 1:].contiguous()
|
| 1311 |
+
# Flatten the tokens
|
| 1312 |
+
loss_fct = CrossEntropyLoss()
|
| 1313 |
+
shift_logits = shift_logits.view(-1, self.config.vocab_size)
|
| 1314 |
+
shift_labels = shift_labels.view(-1)
|
| 1315 |
+
# Enable model parallelism
|
| 1316 |
+
shift_labels = shift_labels.to(shift_logits.device)
|
| 1317 |
+
loss = loss_fct(shift_logits, shift_labels)
|
| 1318 |
+
|
| 1319 |
+
if not return_dict:
|
| 1320 |
+
output = (logits,) + outputs[1:]
|
| 1321 |
+
return (loss,) + output if loss is not None else output
|
| 1322 |
+
|
| 1323 |
+
return CausalLMOutputWithPast(
|
| 1324 |
+
loss=loss,
|
| 1325 |
+
logits=logits,
|
| 1326 |
+
past_key_values=outputs.past_key_values,
|
| 1327 |
+
hidden_states=outputs.hidden_states,
|
| 1328 |
+
attentions=outputs.attentions,
|
| 1329 |
+
)
|
| 1330 |
+
|
| 1331 |
+
# Copied from transformers.models.persimmon.modeling_persimmon.PersimmonForCausalLM.prepare_inputs_for_generation
|
| 1332 |
+
def prepare_inputs_for_generation(
|
| 1333 |
+
self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
|
| 1334 |
+
):
|
| 1335 |
+
if past_key_values is not None:
|
| 1336 |
+
if isinstance(past_key_values, Cache):
|
| 1337 |
+
cache_length = past_key_values.get_seq_length()
|
| 1338 |
+
past_length = past_key_values.seen_tokens
|
| 1339 |
+
max_cache_length = past_key_values.get_max_length()
|
| 1340 |
+
else:
|
| 1341 |
+
cache_length = past_length = past_key_values[0][0].shape[2]
|
| 1342 |
+
max_cache_length = None
|
| 1343 |
+
|
| 1344 |
+
# Keep only the unprocessed tokens:
|
| 1345 |
+
# 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
|
| 1346 |
+
# some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as
|
| 1347 |
+
# input)
|
| 1348 |
+
if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
|
| 1349 |
+
input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
|
| 1350 |
+
# 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard
|
| 1351 |
+
# input_ids based on the past_length.
|
| 1352 |
+
elif past_length < input_ids.shape[1]:
|
| 1353 |
+
input_ids = input_ids[:, past_length:]
|
| 1354 |
+
# 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.
|
| 1355 |
+
|
| 1356 |
+
# If we are about to go beyond the maximum cache length, we need to crop the input attention mask.
|
| 1357 |
+
if (
|
| 1358 |
+
max_cache_length is not None
|
| 1359 |
+
and attention_mask is not None
|
| 1360 |
+
and cache_length + input_ids.shape[1] > max_cache_length
|
| 1361 |
+
):
|
| 1362 |
+
attention_mask = attention_mask[:, -max_cache_length:]
|
| 1363 |
+
|
| 1364 |
+
position_ids = kwargs.get('position_ids', None)
|
| 1365 |
+
if attention_mask is not None and position_ids is None:
|
| 1366 |
+
# create position_ids on the fly for batch generation
|
| 1367 |
+
position_ids = attention_mask.long().cumsum(-1) - 1
|
| 1368 |
+
position_ids.masked_fill_(attention_mask == 0, 1)
|
| 1369 |
+
if past_key_values:
|
| 1370 |
+
position_ids = position_ids[:, -input_ids.shape[1] :]
|
| 1371 |
+
|
| 1372 |
+
# if `inputs_embeds` are passed, we only want to use them in the 1st generation step
|
| 1373 |
+
if (inputs_embeds is not None and past_key_values is None) or (inputs_embeds is not None and len(past_key_values) == 0):
|
| 1374 |
+
model_inputs = {'inputs_embeds': inputs_embeds}
|
| 1375 |
+
else:
|
| 1376 |
+
model_inputs = {'input_ids': input_ids}
|
| 1377 |
+
|
| 1378 |
+
model_inputs.update(
|
| 1379 |
+
{
|
| 1380 |
+
'position_ids': position_ids,
|
| 1381 |
+
'past_key_values': past_key_values,
|
| 1382 |
+
'use_cache': kwargs.get('use_cache'),
|
| 1383 |
+
'attention_mask': attention_mask,
|
| 1384 |
+
}
|
| 1385 |
+
)
|
| 1386 |
+
return model_inputs
|
| 1387 |
+
|
| 1388 |
+
@staticmethod
|
| 1389 |
+
# Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM._reorder_cache
|
| 1390 |
+
def _reorder_cache(past_key_values, beam_idx):
|
| 1391 |
+
reordered_past = ()
|
| 1392 |
+
for layer_past in past_key_values:
|
| 1393 |
+
reordered_past += (
|
| 1394 |
+
tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past),
|
| 1395 |
+
)
|
| 1396 |
+
return reordered_past
|
| 1397 |
+
|
| 1398 |
+
|
| 1399 |
+
@add_start_docstrings(
|
| 1400 |
+
"""
|
| 1401 |
+
The [`Phi3Model`] with a sequence classification head on top (linear layer).
|
| 1402 |
+
|
| 1403 |
+
[`Phi3ForSequenceClassification`] uses the last token in order to do the classification, as other causal models
|
| 1404 |
+
(e.g. GPT-2) do.
|
| 1405 |
+
|
| 1406 |
+
Since it does classification on the last token, it requires to know the position of the last token. If a
|
| 1407 |
+
`pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
|
| 1408 |
+
no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
|
| 1409 |
+
padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
|
| 1410 |
+
each row of the batch).
|
| 1411 |
+
""",
|
| 1412 |
+
PHI3_START_DOCSTRING,
|
| 1413 |
+
)
|
| 1414 |
+
# Copied from transformers.models.llama.modeling_llama.LlamaForSequenceClassification with Llama->Phi3, LLAMA->PHI3, self.transformer->self.model, transformer_outputs->model_outputs
|
| 1415 |
+
class Phi3ForSequenceClassification(Phi3PreTrainedModel):
|
| 1416 |
+
def __init__(self, config):
|
| 1417 |
+
super().__init__(config)
|
| 1418 |
+
self.num_labels = config.num_labels
|
| 1419 |
+
self.model = Phi3Model(config)
|
| 1420 |
+
self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)
|
| 1421 |
+
|
| 1422 |
+
# Initialize weights and apply final processing
|
| 1423 |
+
self.post_init()
|
| 1424 |
+
|
| 1425 |
+
def get_input_embeddings(self):
|
| 1426 |
+
return self.model.embed_tokens
|
| 1427 |
+
|
| 1428 |
+
def set_input_embeddings(self, value):
|
| 1429 |
+
self.model.embed_tokens = value
|
| 1430 |
+
|
| 1431 |
+
@add_start_docstrings_to_model_forward(PHI3_INPUTS_DOCSTRING)
|
| 1432 |
+
def forward(
|
| 1433 |
+
self,
|
| 1434 |
+
input_ids: torch.LongTensor = None,
|
| 1435 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 1436 |
+
position_ids: Optional[torch.LongTensor] = None,
|
| 1437 |
+
past_key_values: Optional[List[torch.FloatTensor]] = None,
|
| 1438 |
+
inputs_embeds: Optional[torch.FloatTensor] = None,
|
| 1439 |
+
labels: Optional[torch.LongTensor] = None,
|
| 1440 |
+
use_cache: Optional[bool] = None,
|
| 1441 |
+
output_attentions: Optional[bool] = None,
|
| 1442 |
+
output_hidden_states: Optional[bool] = None,
|
| 1443 |
+
return_dict: Optional[bool] = None,
|
| 1444 |
+
) -> Union[Tuple, SequenceClassifierOutputWithPast]:
|
| 1445 |
+
r"""
|
| 1446 |
+
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
|
| 1447 |
+
Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
|
| 1448 |
+
config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
|
| 1449 |
+
`config.num_labels > 1` a classification loss is computed (Cross-Entropy).
|
| 1450 |
+
"""
|
| 1451 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 1452 |
+
|
| 1453 |
+
model_outputs = self.model(
|
| 1454 |
+
input_ids,
|
| 1455 |
+
attention_mask=attention_mask,
|
| 1456 |
+
position_ids=position_ids,
|
| 1457 |
+
past_key_values=past_key_values,
|
| 1458 |
+
inputs_embeds=inputs_embeds,
|
| 1459 |
+
use_cache=use_cache,
|
| 1460 |
+
output_attentions=output_attentions,
|
| 1461 |
+
output_hidden_states=output_hidden_states,
|
| 1462 |
+
return_dict=return_dict,
|
| 1463 |
+
)
|
| 1464 |
+
hidden_states = model_outputs[0]
|
| 1465 |
+
logits = self.score(hidden_states)
|
| 1466 |
+
|
| 1467 |
+
if input_ids is not None:
|
| 1468 |
+
batch_size = input_ids.shape[0]
|
| 1469 |
+
else:
|
| 1470 |
+
batch_size = inputs_embeds.shape[0]
|
| 1471 |
+
|
| 1472 |
+
if self.config.pad_token_id is None and batch_size != 1:
|
| 1473 |
+
raise ValueError('Cannot handle batch sizes > 1 if no padding token is defined.')
|
| 1474 |
+
if self.config.pad_token_id is None:
|
| 1475 |
+
sequence_lengths = -1
|
| 1476 |
+
else:
|
| 1477 |
+
if input_ids is not None:
|
| 1478 |
+
# if no pad token found, use modulo instead of reverse indexing for ONNX compatibility
|
| 1479 |
+
sequence_lengths = torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1
|
| 1480 |
+
sequence_lengths = sequence_lengths % input_ids.shape[-1]
|
| 1481 |
+
sequence_lengths = sequence_lengths.to(logits.device)
|
| 1482 |
+
else:
|
| 1483 |
+
sequence_lengths = -1
|
| 1484 |
+
|
| 1485 |
+
pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths]
|
| 1486 |
+
|
| 1487 |
+
loss = None
|
| 1488 |
+
if labels is not None:
|
| 1489 |
+
labels = labels.to(logits.device)
|
| 1490 |
+
if self.config.problem_type is None:
|
| 1491 |
+
if self.num_labels == 1:
|
| 1492 |
+
self.config.problem_type = 'regression'
|
| 1493 |
+
elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
|
| 1494 |
+
self.config.problem_type = 'single_label_classification'
|
| 1495 |
+
else:
|
| 1496 |
+
self.config.problem_type = 'multi_label_classification'
|
| 1497 |
+
|
| 1498 |
+
if self.config.problem_type == 'regression':
|
| 1499 |
+
loss_fct = MSELoss()
|
| 1500 |
+
if self.num_labels == 1:
|
| 1501 |
+
loss = loss_fct(pooled_logits.squeeze(), labels.squeeze())
|
| 1502 |
+
else:
|
| 1503 |
+
loss = loss_fct(pooled_logits, labels)
|
| 1504 |
+
elif self.config.problem_type == 'single_label_classification':
|
| 1505 |
+
loss_fct = CrossEntropyLoss()
|
| 1506 |
+
loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1))
|
| 1507 |
+
elif self.config.problem_type == 'multi_label_classification':
|
| 1508 |
+
loss_fct = BCEWithLogitsLoss()
|
| 1509 |
+
loss = loss_fct(pooled_logits, labels)
|
| 1510 |
+
if not return_dict:
|
| 1511 |
+
output = (pooled_logits,) + model_outputs[1:]
|
| 1512 |
+
return ((loss,) + output) if loss is not None else output
|
| 1513 |
+
|
| 1514 |
+
return SequenceClassifierOutputWithPast(
|
| 1515 |
+
loss=loss,
|
| 1516 |
+
logits=pooled_logits,
|
| 1517 |
+
past_key_values=model_outputs.past_key_values,
|
| 1518 |
+
hidden_states=model_outputs.hidden_states,
|
| 1519 |
+
attentions=model_outputs.attentions,
|
| 1520 |
+
)
|
| 1521 |
+
|
| 1522 |
+
|
| 1523 |
+
@add_start_docstrings(
|
| 1524 |
+
"""
|
| 1525 |
+
[`Phi3Model`] with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for
|
| 1526 |
+
Named-Entity-Recognition (NER) tasks.
|
| 1527 |
+
""",
|
| 1528 |
+
PHI3_START_DOCSTRING,
|
| 1529 |
+
)
|
| 1530 |
+
# Copied from transformers.models.mpt.modeling_mpt.MptForTokenClassification with Mpt->Phi3,MPT->PHI3,self.transformer->self.model,transformer_outputs->model_outputs
|
| 1531 |
+
class Phi3ForTokenClassification(Phi3PreTrainedModel):
|
| 1532 |
+
def __init__(self, config: Phi3Config):
|
| 1533 |
+
super().__init__(config)
|
| 1534 |
+
self.num_labels = config.num_labels
|
| 1535 |
+
|
| 1536 |
+
self.model = Phi3Model(config)
|
| 1537 |
+
if hasattr(config, 'classifier_dropout') and config.classifier_dropout is not None:
|
| 1538 |
+
classifier_dropout = config.classifier_dropout
|
| 1539 |
+
elif hasattr(config, 'hidden_dropout') and config.hidden_dropout is not None:
|
| 1540 |
+
classifier_dropout = config.hidden_dropout
|
| 1541 |
+
else:
|
| 1542 |
+
classifier_dropout = 0.1
|
| 1543 |
+
self.dropout = nn.Dropout(classifier_dropout)
|
| 1544 |
+
self.classifier = nn.Linear(config.hidden_size, config.num_labels)
|
| 1545 |
+
|
| 1546 |
+
# Initialize weights and apply final processing
|
| 1547 |
+
self.post_init()
|
| 1548 |
+
|
| 1549 |
+
@add_start_docstrings_to_model_forward(PHI3_INPUTS_DOCSTRING)
|
| 1550 |
+
@add_code_sample_docstrings(
|
| 1551 |
+
checkpoint=_CHECKPOINT_FOR_DOC,
|
| 1552 |
+
output_type=TokenClassifierOutput,
|
| 1553 |
+
config_class=_CONFIG_FOR_DOC,
|
| 1554 |
+
)
|
| 1555 |
+
def forward(
|
| 1556 |
+
self,
|
| 1557 |
+
input_ids: Optional[torch.LongTensor] = None,
|
| 1558 |
+
past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None,
|
| 1559 |
+
attention_mask: Optional[torch.Tensor] = None,
|
| 1560 |
+
inputs_embeds: Optional[torch.Tensor] = None,
|
| 1561 |
+
labels: Optional[torch.Tensor] = None,
|
| 1562 |
+
use_cache: Optional[bool] = None,
|
| 1563 |
+
output_attentions: Optional[bool] = None,
|
| 1564 |
+
output_hidden_states: Optional[bool] = None,
|
| 1565 |
+
return_dict: Optional[bool] = None,
|
| 1566 |
+
**deprecated_arguments,
|
| 1567 |
+
) -> Union[Tuple[torch.Tensor], TokenClassifierOutput]:
|
| 1568 |
+
r"""
|
| 1569 |
+
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
|
| 1570 |
+
Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
|
| 1571 |
+
config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
|
| 1572 |
+
`config.num_labels > 1` a classification loss is computed (Cross-Entropy).
|
| 1573 |
+
"""
|
| 1574 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
| 1575 |
+
|
| 1576 |
+
model_outputs = self.model(
|
| 1577 |
+
input_ids,
|
| 1578 |
+
past_key_values=past_key_values,
|
| 1579 |
+
attention_mask=attention_mask,
|
| 1580 |
+
inputs_embeds=inputs_embeds,
|
| 1581 |
+
use_cache=use_cache,
|
| 1582 |
+
output_attentions=output_attentions,
|
| 1583 |
+
output_hidden_states=output_hidden_states,
|
| 1584 |
+
return_dict=return_dict,
|
| 1585 |
+
)
|
| 1586 |
+
|
| 1587 |
+
hidden_states = model_outputs[0]
|
| 1588 |
+
hidden_states = self.dropout(hidden_states)
|
| 1589 |
+
logits = self.classifier(hidden_states)
|
| 1590 |
+
|
| 1591 |
+
loss = None
|
| 1592 |
+
if labels is not None:
|
| 1593 |
+
# move labels to correct device to enable model parallelism
|
| 1594 |
+
labels = labels.to(logits.device)
|
| 1595 |
+
batch_size, seq_length = labels.shape
|
| 1596 |
+
loss_fct = CrossEntropyLoss()
|
| 1597 |
+
loss = loss_fct(
|
| 1598 |
+
logits.view(batch_size * seq_length, self.num_labels), labels.view(batch_size * seq_length)
|
| 1599 |
+
)
|
| 1600 |
+
|
| 1601 |
+
if not return_dict:
|
| 1602 |
+
output = (logits,) + model_outputs[2:]
|
| 1603 |
+
return ((loss,) + output) if loss is not None else output
|
| 1604 |
+
|
| 1605 |
+
return TokenClassifierOutput(
|
| 1606 |
+
loss=loss,
|
| 1607 |
+
logits=logits,
|
| 1608 |
+
hidden_states=model_outputs.hidden_states,
|
| 1609 |
+
attentions=model_outputs.attentions,
|
| 1610 |
+
)
|
modeling_radio.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2023-2024, NVIDIA CORPORATION. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 |
+
# you may not use this file except in compliance with the License.
|
| 5 |
+
# You may obtain a copy of the License at
|
| 6 |
+
#
|
| 7 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
| 8 |
+
#
|
| 9 |
+
# Unless required by applicable law or agreed to in writing, software
|
| 10 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
| 11 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 12 |
+
# See the License for the specific language governing permissions and
|
| 13 |
+
# limitations under the License.
|
| 14 |
+
from collections import namedtuple
|
| 15 |
+
from typing import Optional, List, Union
|
| 16 |
+
|
| 17 |
+
from timm.models import VisionTransformer
|
| 18 |
+
import torch
|
| 19 |
+
from transformers import PreTrainedModel
|
| 20 |
+
|
| 21 |
+
# Import all required modules.
|
| 22 |
+
from .radio_adaptor_base import AdaptorBase, RadioOutput, AdaptorInput
|
| 23 |
+
from .radio_adaptor_generic import GenericAdaptor, AdaptorBase
|
| 24 |
+
from .radio_adaptor_mlp import create_mlp_from_state
|
| 25 |
+
from .radio_adaptor_registry import adaptor_registry
|
| 26 |
+
from .radio_cls_token import ClsToken
|
| 27 |
+
from .radio_enable_cpe_support import enable_cpe
|
| 28 |
+
from .radio_enable_spectral_reparam import configure_spectral_reparam_from_args
|
| 29 |
+
from .radio_eradio_model import eradio
|
| 30 |
+
from .radio_model import create_model_from_args
|
| 31 |
+
from .radio_model import RADIOModel as RADIOModelBase, Resolution
|
| 32 |
+
from .radio_input_conditioner import get_default_conditioner, InputConditioner
|
| 33 |
+
from .radio_open_clip_adaptor import OpenCLIP_RADIO
|
| 34 |
+
from .radio_vit_patch_generator import ViTPatchGenerator
|
| 35 |
+
from .radio_vitdet import apply_vitdet_arch, VitDetArgs
|
| 36 |
+
|
| 37 |
+
# Register extra models
|
| 38 |
+
from .radio_extra_timm_models import *
|
| 39 |
+
|
| 40 |
+
from .configuration_radio import RADIOConfig
|
| 41 |
+
|
| 42 |
+
class RADIOModel(PreTrainedModel):
|
| 43 |
+
"""Pretrained Hugging Face model for RADIO.
|
| 44 |
+
|
| 45 |
+
This class inherits from PreTrainedModel, which provides
|
| 46 |
+
HuggingFace's functionality for loading and saving models.
|
| 47 |
+
"""
|
| 48 |
+
|
| 49 |
+
config_class = RADIOConfig
|
| 50 |
+
|
| 51 |
+
def __init__(self, config):
|
| 52 |
+
super().__init__(config)
|
| 53 |
+
|
| 54 |
+
RADIOArgs = namedtuple("RADIOArgs", config.args.keys())
|
| 55 |
+
args = RADIOArgs(**config.args)
|
| 56 |
+
self.config = config
|
| 57 |
+
|
| 58 |
+
model = create_model_from_args(args)
|
| 59 |
+
input_conditioner: InputConditioner = get_default_conditioner()
|
| 60 |
+
|
| 61 |
+
dtype = getattr(args, "dtype", torch.float32)
|
| 62 |
+
if isinstance(dtype, str):
|
| 63 |
+
# Convert the dtype's string representation back to a dtype.
|
| 64 |
+
dtype = getattr(torch, dtype)
|
| 65 |
+
model.to(dtype=dtype)
|
| 66 |
+
input_conditioner.dtype = dtype
|
| 67 |
+
|
| 68 |
+
summary_idxs = torch.tensor(
|
| 69 |
+
[i for i, t in enumerate(args.teachers) if t.get("use_summary", True)],
|
| 70 |
+
dtype=torch.int64,
|
| 71 |
+
)
|
| 72 |
+
|
| 73 |
+
adaptor_names = config.adaptor_names
|
| 74 |
+
if adaptor_names is not None:
|
| 75 |
+
raise NotImplementedError(
|
| 76 |
+
f"Adaptors are not yet supported in Hugging Face models. Adaptor names: {adaptor_names}"
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
adaptors = dict()
|
| 80 |
+
|
| 81 |
+
self.radio_model = RADIOModelBase(
|
| 82 |
+
model,
|
| 83 |
+
input_conditioner,
|
| 84 |
+
summary_idxs=summary_idxs,
|
| 85 |
+
patch_size=config.patch_size,
|
| 86 |
+
max_resolution=config.max_resolution,
|
| 87 |
+
window_size=config.vitdet_window_size,
|
| 88 |
+
preferred_resolution=config.preferred_resolution,
|
| 89 |
+
adaptors=adaptors,
|
| 90 |
+
)
|
| 91 |
+
|
| 92 |
+
@property
|
| 93 |
+
def model(self) -> VisionTransformer:
|
| 94 |
+
return self.radio_model.model
|
| 95 |
+
|
| 96 |
+
@property
|
| 97 |
+
def input_conditioner(self) -> InputConditioner:
|
| 98 |
+
return self.radio_model.input_conditioner
|
| 99 |
+
|
| 100 |
+
def forward(self, x: torch.Tensor):
|
| 101 |
+
return self.radio_model.forward(x)
|
preprocessor_config.json
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"crop_size": 448,
|
| 3 |
+
"do_center_crop": true,
|
| 4 |
+
"do_normalize": true,
|
| 5 |
+
"do_resize": true,
|
| 6 |
+
"feature_extractor_type": "CLIPFeatureExtractor",
|
| 7 |
+
"image_mean": [
|
| 8 |
+
0.485,
|
| 9 |
+
0.456,
|
| 10 |
+
0.406
|
| 11 |
+
],
|
| 12 |
+
"image_std": [
|
| 13 |
+
0.229,
|
| 14 |
+
0.224,
|
| 15 |
+
0.225
|
| 16 |
+
],
|
| 17 |
+
"resample": 3,
|
| 18 |
+
"size": 448
|
| 19 |
+
}
|
radio_adaptor_base.py
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# NVIDIA CORPORATION and its licensors retain all intellectual property
|
| 4 |
+
# and proprietary rights in and to this software, related documentation
|
| 5 |
+
# and any modifications thereto. Any use, reproduction, disclosure or
|
| 6 |
+
# distribution of this software and related documentation without an express
|
| 7 |
+
# license agreement from NVIDIA CORPORATION is strictly prohibited.
|
| 8 |
+
from argparse import Namespace
|
| 9 |
+
from typing import NamedTuple
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
from torch import nn
|
| 13 |
+
import torch.nn.functional as F
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class AdaptorInput(NamedTuple):
|
| 17 |
+
images: torch.Tensor
|
| 18 |
+
summary: torch.Tensor
|
| 19 |
+
features: torch.Tensor
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class RadioOutput(NamedTuple):
|
| 23 |
+
summary: torch.Tensor
|
| 24 |
+
features: torch.Tensor
|
| 25 |
+
|
| 26 |
+
def to(self, *args, **kwargs):
|
| 27 |
+
return RadioOutput(
|
| 28 |
+
self.summary.to(*args, **kwargs) if self.summary is not None else None,
|
| 29 |
+
self.features.to(*args, **kwargs) if self.features is not None else None,
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class AdaptorBase(nn.Module):
|
| 34 |
+
def forward(self, input: AdaptorInput) -> RadioOutput:
|
| 35 |
+
raise NotImplementedError("Subclasses must implement this!")
|
radio_adaptor_generic.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# NVIDIA CORPORATION and its licensors retain all intellectual property
|
| 4 |
+
# and proprietary rights in and to this software, related documentation
|
| 5 |
+
# and any modifications thereto. Any use, reproduction, disclosure or
|
| 6 |
+
# distribution of this software and related documentation without an express
|
| 7 |
+
# license agreement from NVIDIA CORPORATION is strictly prohibited.
|
| 8 |
+
from argparse import Namespace
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
from torch import nn
|
| 12 |
+
import torch.nn.functional as F
|
| 13 |
+
|
| 14 |
+
from .radio_adaptor_base import AdaptorBase, AdaptorInput, RadioOutput
|
| 15 |
+
from .radio_adaptor_mlp import create_mlp_from_state
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class GenericAdaptor(AdaptorBase):
|
| 19 |
+
def __init__(self, main_config: Namespace, adaptor_config, state):
|
| 20 |
+
super().__init__()
|
| 21 |
+
|
| 22 |
+
self.head_mlp = create_mlp_from_state(main_config.mlp_version, state, 'summary.')
|
| 23 |
+
self.feat_mlp = create_mlp_from_state(main_config.mlp_version, state, 'feature.')
|
| 24 |
+
|
| 25 |
+
def forward(self, input: AdaptorInput) -> RadioOutput:
|
| 26 |
+
summary = self.head_mlp(input.summary)
|
| 27 |
+
feat = self.feat_mlp(input.features)
|
| 28 |
+
|
| 29 |
+
return RadioOutput(summary, feat)
|
radio_adaptor_mlp.py
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# NVIDIA CORPORATION and its licensors retain all intellectual property
|
| 4 |
+
# and proprietary rights in and to this software, related documentation
|
| 5 |
+
# and any modifications thereto. Any use, reproduction, disclosure or
|
| 6 |
+
# distribution of this software and related documentation without an express
|
| 7 |
+
# license agreement from NVIDIA CORPORATION is strictly prohibited.
|
| 8 |
+
import math
|
| 9 |
+
from typing import Dict
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
from torch import nn
|
| 13 |
+
|
| 14 |
+
from einops import rearrange
|
| 15 |
+
from timm.models.vision_transformer import Block
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class MLP(nn.Module):
|
| 19 |
+
def __init__(self, input_size: int, hidden_size: int, output_size: int,
|
| 20 |
+
num_inner: int = 0, device: torch.device = None, **kwargs):
|
| 21 |
+
super(MLP, self).__init__()
|
| 22 |
+
self.fc1 = nn.Linear(input_size, hidden_size, device=device)
|
| 23 |
+
self.norm = nn.LayerNorm(hidden_size, device=device)
|
| 24 |
+
self.relu = nn.ReLU()
|
| 25 |
+
|
| 26 |
+
inner = []
|
| 27 |
+
for _ in range(num_inner):
|
| 28 |
+
inner.extend([
|
| 29 |
+
nn.Linear(hidden_size, hidden_size, device=device),
|
| 30 |
+
nn.LayerNorm(hidden_size, device=device),
|
| 31 |
+
nn.ReLU(),
|
| 32 |
+
])
|
| 33 |
+
if inner:
|
| 34 |
+
self.inner = nn.Sequential(*inner)
|
| 35 |
+
else:
|
| 36 |
+
self.inner = nn.Identity()
|
| 37 |
+
|
| 38 |
+
self.fc2 = nn.Linear(hidden_size, output_size, device=device)
|
| 39 |
+
|
| 40 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 41 |
+
x = self.fc1(x)
|
| 42 |
+
x = self.norm(x)
|
| 43 |
+
x = self.relu(x)
|
| 44 |
+
x = self.inner(x)
|
| 45 |
+
x = self.fc2(x)
|
| 46 |
+
return x
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
class MLP2(nn.Module):
|
| 50 |
+
def __init__(self, input_size: int, hidden_size: int, output_size: int,
|
| 51 |
+
num_inner: int = 0,
|
| 52 |
+
pre_norm: bool = False, device: torch.device = None,
|
| 53 |
+
upsample_factor: int = 1,
|
| 54 |
+
**kwargs):
|
| 55 |
+
super().__init__()
|
| 56 |
+
|
| 57 |
+
self.pre_norm = nn.Sequential(
|
| 58 |
+
nn.LayerNorm(input_size),
|
| 59 |
+
nn.GELU(),
|
| 60 |
+
) if pre_norm else nn.Identity()
|
| 61 |
+
|
| 62 |
+
self.upsample_factor = upsample_factor
|
| 63 |
+
self._real_output_dim = output_size
|
| 64 |
+
|
| 65 |
+
hidden_size *= upsample_factor
|
| 66 |
+
output_size *= (upsample_factor ** 2)
|
| 67 |
+
|
| 68 |
+
self.fc1 = nn.Linear(input_size, hidden_size, device=device)
|
| 69 |
+
|
| 70 |
+
blocks = []
|
| 71 |
+
for _ in range(num_inner):
|
| 72 |
+
blocks.append(nn.Sequential(
|
| 73 |
+
nn.LayerNorm(hidden_size, device=device),
|
| 74 |
+
nn.GELU(),
|
| 75 |
+
nn.Linear(hidden_size, hidden_size, device=device),
|
| 76 |
+
))
|
| 77 |
+
self.blocks = nn.ModuleList(blocks)
|
| 78 |
+
|
| 79 |
+
self.final = nn.Sequential(
|
| 80 |
+
nn.LayerNorm(hidden_size, device=device),
|
| 81 |
+
nn.GELU(),
|
| 82 |
+
nn.Linear(hidden_size, output_size, device=device),
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 86 |
+
x = self.pre_norm(x)
|
| 87 |
+
x = self.fc1(x)
|
| 88 |
+
for block in self.blocks:
|
| 89 |
+
x = x + block(x)
|
| 90 |
+
x = self.final(x)
|
| 91 |
+
|
| 92 |
+
if self.upsample_factor > 1:
|
| 93 |
+
h = w = int(math.sqrt(x.shape[1]))
|
| 94 |
+
x = rearrange(x, 'b (h w) (u1 u2 c) -> b (u1 h u2 w) c',
|
| 95 |
+
h=h, w=w, u1=self.upsample_factor, u2=self.upsample_factor,
|
| 96 |
+
c=self._real_output_dim)
|
| 97 |
+
|
| 98 |
+
return x
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
MLP_FACTORY = {
|
| 102 |
+
'v1': MLP,
|
| 103 |
+
'v2': MLP2,
|
| 104 |
+
}
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def strip_prefix(state: Dict[str, torch.Tensor], prefix: str):
|
| 108 |
+
state = {
|
| 109 |
+
k[len(prefix):]: v
|
| 110 |
+
for k, v in state.items()
|
| 111 |
+
if k.startswith(prefix)
|
| 112 |
+
}
|
| 113 |
+
return state
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def get_mlp_info_from_state(version: str, state: Dict[str, torch.Tensor], prefix: str = ''):
|
| 117 |
+
state = strip_prefix(state, prefix)
|
| 118 |
+
|
| 119 |
+
if version == 'v1':
|
| 120 |
+
hidden_dim, input_dim = state['fc1.weight'].shape
|
| 121 |
+
output_dim = state['fc2.weight'].shape[0]
|
| 122 |
+
|
| 123 |
+
for num_inner in range(1000):
|
| 124 |
+
k = f'inner.{num_inner}.0.weight'
|
| 125 |
+
if k not in state:
|
| 126 |
+
break
|
| 127 |
+
elif version == 'v2':
|
| 128 |
+
hidden_dim, input_dim = state['fc1.weight'].shape
|
| 129 |
+
output_dim = state['final.2.weight'].shape[0]
|
| 130 |
+
|
| 131 |
+
for num_inner in range(1000):
|
| 132 |
+
k = f'blocks.{num_inner}.0.weight'
|
| 133 |
+
if k not in state:
|
| 134 |
+
break
|
| 135 |
+
else:
|
| 136 |
+
raise ValueError(f'Unsupported MLP version: {version}')
|
| 137 |
+
|
| 138 |
+
return input_dim, hidden_dim, output_dim, num_inner
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def create_mlp_from_state(version: str, state: Dict[str, torch.Tensor], prefix: str = ''):
|
| 142 |
+
state = strip_prefix(state, prefix)
|
| 143 |
+
|
| 144 |
+
input_dim, hidden_dim, output_dim, num_inner = get_mlp_info_from_state(version, state)
|
| 145 |
+
|
| 146 |
+
ret: nn.Module = MLP_FACTORY[version](input_dim, hidden_dim, output_dim, num_inner)
|
| 147 |
+
|
| 148 |
+
ret.load_state_dict(state)
|
| 149 |
+
|
| 150 |
+
return ret
|
radio_adaptor_registry.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# NVIDIA CORPORATION and its licensors retain all intellectual property
|
| 4 |
+
# and proprietary rights in and to this software, related documentation
|
| 5 |
+
# and any modifications thereto. Any use, reproduction, disclosure or
|
| 6 |
+
# distribution of this software and related documentation without an express
|
| 7 |
+
# license agreement from NVIDIA CORPORATION is strictly prohibited.
|
| 8 |
+
from argparse import Namespace
|
| 9 |
+
from typing import Dict, Any
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
|
| 13 |
+
from .radio_adaptor_generic import GenericAdaptor, AdaptorBase
|
| 14 |
+
|
| 15 |
+
dict_t = Dict[str, Any]
|
| 16 |
+
state_t = Dict[str, torch.Tensor]
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class AdaptorRegistry:
|
| 20 |
+
def __init__(self):
|
| 21 |
+
self._registry = {}
|
| 22 |
+
|
| 23 |
+
def register_adaptor(self, name):
|
| 24 |
+
def decorator(factory_function):
|
| 25 |
+
if name in self._registry:
|
| 26 |
+
raise ValueError(f"Model '{name}' already registered")
|
| 27 |
+
self._registry[name] = factory_function
|
| 28 |
+
return factory_function
|
| 29 |
+
return decorator
|
| 30 |
+
|
| 31 |
+
def create_adaptor(self, name, main_config: Namespace, adaptor_config: dict_t, state: state_t) -> AdaptorBase:
|
| 32 |
+
if name not in self._registry:
|
| 33 |
+
return GenericAdaptor(main_config, adaptor_config, state)
|
| 34 |
+
return self._registry[name](main_config, adaptor_config, state)
|
| 35 |
+
|
| 36 |
+
# Creating an instance of the registry
|
| 37 |
+
adaptor_registry = AdaptorRegistry()
|
radio_cls_token.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2023-2024, NVIDIA CORPORATION. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# NVIDIA CORPORATION and its licensors retain all intellectual property
|
| 4 |
+
# and proprietary rights in and to this software, related documentation
|
| 5 |
+
# and any modifications thereto. Any use, reproduction, disclosure or
|
| 6 |
+
# distribution of this software and related documentation without an express
|
| 7 |
+
# license agreement from NVIDIA CORPORATION is strictly prohibited.
|
| 8 |
+
|
| 9 |
+
import torch
|
| 10 |
+
from torch import nn
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class ClsToken(nn.Module):
|
| 14 |
+
def __init__(self, ndim: int,
|
| 15 |
+
num_tokens: int = 1,
|
| 16 |
+
enabled: bool = True,
|
| 17 |
+
register_multiple: int = 0,
|
| 18 |
+
):
|
| 19 |
+
super().__init__()
|
| 20 |
+
|
| 21 |
+
self.ndim = ndim
|
| 22 |
+
self.enabled = enabled
|
| 23 |
+
self.num_registers = 0
|
| 24 |
+
self.num_tokens = num_tokens
|
| 25 |
+
if enabled:
|
| 26 |
+
if register_multiple > 0:
|
| 27 |
+
self.num_registers = register_multiple - (num_tokens % register_multiple)
|
| 28 |
+
|
| 29 |
+
scale = ndim ** -0.5
|
| 30 |
+
self.token = nn.Parameter(torch.randn(num_tokens + self.num_registers, ndim) * scale)
|
| 31 |
+
else:
|
| 32 |
+
self.token = None
|
| 33 |
+
|
| 34 |
+
self.num_patches = self.num_tokens + self.num_registers
|
| 35 |
+
|
| 36 |
+
def disable(self):
|
| 37 |
+
self.token = None
|
| 38 |
+
self.enabled = False
|
| 39 |
+
|
| 40 |
+
def forward(self, x: torch.Tensor):
|
| 41 |
+
if self.token is None:
|
| 42 |
+
return x
|
| 43 |
+
|
| 44 |
+
token = self.token.unsqueeze(0).expand(x.shape[0], -1, -1)
|
| 45 |
+
x = torch.cat([
|
| 46 |
+
token,
|
| 47 |
+
x,
|
| 48 |
+
], dim=1)
|
| 49 |
+
|
| 50 |
+
return x
|
| 51 |
+
|
| 52 |
+
def no_weight_decay(self):
|
| 53 |
+
return [
|
| 54 |
+
'token',
|
| 55 |
+
]
|
radio_common.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# NVIDIA CORPORATION and its licensors retain all intellectual property
|
| 4 |
+
# and proprietary rights in and to this software, related documentation
|
| 5 |
+
# and any modifications thereto. Any use, reproduction, disclosure or
|
| 6 |
+
# distribution of this software and related documentation without an express
|
| 7 |
+
# license agreement from NVIDIA CORPORATION is strictly prohibited.
|
| 8 |
+
|
| 9 |
+
from dataclasses import dataclass
|
| 10 |
+
|
| 11 |
+
from .radio_model import Resolution
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
@dataclass
|
| 15 |
+
class RadioResource:
|
| 16 |
+
url: str
|
| 17 |
+
patch_size: int
|
| 18 |
+
max_resolution: int
|
| 19 |
+
preferred_resolution: Resolution
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
RESOURCE_MAP = {
|
| 23 |
+
# RADIO
|
| 24 |
+
"radio_v2.1": RadioResource(
|
| 25 |
+
"https://huggingface.co/nvidia/RADIO/resolve/main/radio_v2.1_bf16.pth.tar?download=true",
|
| 26 |
+
patch_size=16,
|
| 27 |
+
max_resolution=2048,
|
| 28 |
+
preferred_resolution=Resolution(432, 432),
|
| 29 |
+
),
|
| 30 |
+
"radio_v2": RadioResource(
|
| 31 |
+
"https://huggingface.co/nvidia/RADIO/resolve/main/radio_v2.pth.tar?download=true",
|
| 32 |
+
patch_size=16,
|
| 33 |
+
max_resolution=2048,
|
| 34 |
+
preferred_resolution=Resolution(432, 432),
|
| 35 |
+
),
|
| 36 |
+
"radio_v1": RadioResource(
|
| 37 |
+
"https://huggingface.co/nvidia/RADIO/resolve/main/radio_v1.pth.tar?download=true",
|
| 38 |
+
patch_size=14,
|
| 39 |
+
max_resolution=1050,
|
| 40 |
+
preferred_resolution=Resolution(378, 378),
|
| 41 |
+
),
|
| 42 |
+
# E-RADIO
|
| 43 |
+
"e-radio_v2": RadioResource(
|
| 44 |
+
"https://huggingface.co/nvidia/RADIO/resolve/main/eradio_v2.pth.tar?download=true",
|
| 45 |
+
patch_size=16,
|
| 46 |
+
max_resolution=2048,
|
| 47 |
+
preferred_resolution=Resolution(512, 512),
|
| 48 |
+
),
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
DEFAULT_VERSION = "radio_v2.1"
|
radio_enable_cpe_support.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2023-2024, NVIDIA CORPORATION. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# NVIDIA CORPORATION and its licensors retain all intellectual property
|
| 4 |
+
# and proprietary rights in and to this software, related documentation
|
| 5 |
+
# and any modifications thereto. Any use, reproduction, disclosure or
|
| 6 |
+
# distribution of this software and related documentation without an express
|
| 7 |
+
# license agreement from NVIDIA CORPORATION is strictly prohibited.
|
| 8 |
+
|
| 9 |
+
from typing import Union, Tuple
|
| 10 |
+
from types import MethodType
|
| 11 |
+
|
| 12 |
+
import torch
|
| 13 |
+
from torch import nn
|
| 14 |
+
|
| 15 |
+
from timm.models import VisionTransformer, checkpoint_seq
|
| 16 |
+
|
| 17 |
+
from .radio_vit_patch_generator import ViTPatchGenerator
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def _forward_cpe(self: VisionTransformer, x: torch.Tensor) -> torch.Tensor:
|
| 21 |
+
x = self.patch_generator(x)
|
| 22 |
+
if self.grad_checkpointing and not torch.jit.is_scripting():
|
| 23 |
+
x = checkpoint_seq(self.blocks, x)
|
| 24 |
+
else:
|
| 25 |
+
x = self.blocks(x)
|
| 26 |
+
x = self.norm(x)
|
| 27 |
+
return x
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def enable_cpe(model: nn.Module,
|
| 31 |
+
max_img_size: Union[int, Tuple[int, int]] = 1024,
|
| 32 |
+
num_cls_tokens: int = 1,
|
| 33 |
+
pos_dropout: float = 0.1,
|
| 34 |
+
register_multiple: int = 0,
|
| 35 |
+
):
|
| 36 |
+
if not isinstance(model, VisionTransformer):
|
| 37 |
+
raise ValueError("CPE only support for VisionTransformer models!")
|
| 38 |
+
|
| 39 |
+
patch_size = model.patch_embed.patch_size[0]
|
| 40 |
+
embed_dim = model.embed_dim
|
| 41 |
+
input_dims = model.patch_embed.img_size
|
| 42 |
+
normalize_patches = not isinstance(model.patch_embed.norm, nn.Identity)
|
| 43 |
+
cls_token = model.cls_token is not None
|
| 44 |
+
|
| 45 |
+
max_img_size = int(round(max_img_size / patch_size) * patch_size)
|
| 46 |
+
|
| 47 |
+
patch_generator = ViTPatchGenerator(
|
| 48 |
+
patch_size=patch_size,
|
| 49 |
+
embed_dim=embed_dim,
|
| 50 |
+
input_dims=input_dims,
|
| 51 |
+
normalize_patches=normalize_patches,
|
| 52 |
+
cls_token=cls_token,
|
| 53 |
+
max_input_dims=max_img_size,
|
| 54 |
+
pos_dropout=pos_dropout,
|
| 55 |
+
num_cls_tokens=num_cls_tokens,
|
| 56 |
+
register_multiple=register_multiple,
|
| 57 |
+
)
|
| 58 |
+
|
| 59 |
+
model.patch_generator = patch_generator
|
| 60 |
+
model.patch_embed = None
|
| 61 |
+
model.cls_token = None
|
| 62 |
+
model.pos_embed = None
|
| 63 |
+
model.pos_drop = None
|
| 64 |
+
model.num_cls_tokens = num_cls_tokens
|
| 65 |
+
model.num_registers = patch_generator.num_registers
|
| 66 |
+
|
| 67 |
+
model.forward_features = MethodType(_forward_cpe, model)
|
radio_enable_spectral_reparam.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from logging import getLogger
|
| 2 |
+
import math
|
| 3 |
+
import os
|
| 4 |
+
from typing import Union, Tuple
|
| 5 |
+
from types import MethodType
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
from torch import nn
|
| 9 |
+
from torch.nn import functional as F
|
| 10 |
+
from torch.nn.utils import parametrize
|
| 11 |
+
from torch.nn.utils.parametrizations import _SpectralNorm
|
| 12 |
+
|
| 13 |
+
from timm.models.vision_transformer import Attention, Mlp
|
| 14 |
+
|
| 15 |
+
_EPS = 1e-5
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class _SNReweight(_SpectralNorm):
|
| 19 |
+
def __init__(self, weight: torch.Tensor, *args, init_norm_to_current: bool = False, alpha: float = 0.05, version: int = 2, **kwargs):
|
| 20 |
+
super().__init__(weight, *args, **kwargs)
|
| 21 |
+
|
| 22 |
+
self.alpha = alpha
|
| 23 |
+
self.version = version
|
| 24 |
+
self.register_buffer('_sn_version', torch.tensor(version))
|
| 25 |
+
|
| 26 |
+
if init_norm_to_current:
|
| 27 |
+
# This will set the numerator to match the denominator, which should preserve the original values
|
| 28 |
+
init_scale = self._get_sigma(weight).item()
|
| 29 |
+
else:
|
| 30 |
+
init_scale = 1.0
|
| 31 |
+
|
| 32 |
+
if version == 1:
|
| 33 |
+
init_value = init_scale
|
| 34 |
+
elif version == 2:
|
| 35 |
+
t = init_scale - alpha
|
| 36 |
+
if t < _EPS:
|
| 37 |
+
getLogger("spectral_reparam").warn(f'The initialized spectral norm {init_scale} is too small to be represented. Setting to {_EPS} instead.')
|
| 38 |
+
t = _EPS
|
| 39 |
+
|
| 40 |
+
init_value = math.log(math.exp(t) - 1)
|
| 41 |
+
else:
|
| 42 |
+
raise ValueError(f'Unsupported version: {version}')
|
| 43 |
+
|
| 44 |
+
# Make 2D so that weight decay gets applied
|
| 45 |
+
self.scale = nn.Parameter(torch.tensor([[init_value]], dtype=torch.float32, device=weight.device))
|
| 46 |
+
|
| 47 |
+
# Re-implementing this because we need to make division by sigma safe
|
| 48 |
+
def _get_sigma(self, weight: torch.Tensor) -> torch.Tensor:
|
| 49 |
+
if weight.ndim == 1:
|
| 50 |
+
# Faster and more exact path, no need to approximate anything
|
| 51 |
+
sigma = weight.norm()
|
| 52 |
+
else:
|
| 53 |
+
weight_mat = self._reshape_weight_to_matrix(weight)
|
| 54 |
+
if self.training:
|
| 55 |
+
self._power_method(weight_mat, self.n_power_iterations)
|
| 56 |
+
# See above on why we need to clone
|
| 57 |
+
u = self._u.clone(memory_format=torch.contiguous_format)
|
| 58 |
+
v = self._v.clone(memory_format=torch.contiguous_format)
|
| 59 |
+
# The proper way of computing this should be through F.bilinear, but
|
| 60 |
+
# it seems to have some efficiency issues:
|
| 61 |
+
# https://github.com/pytorch/pytorch/issues/58093
|
| 62 |
+
sigma = torch.dot(u, torch.mv(weight_mat, v))
|
| 63 |
+
|
| 64 |
+
return sigma + self.eps
|
| 65 |
+
|
| 66 |
+
def forward(self, weight: torch.Tensor, *args, **kwargs):
|
| 67 |
+
dtype = weight.dtype
|
| 68 |
+
sigma = self._get_sigma(weight, *args, **kwargs)
|
| 69 |
+
|
| 70 |
+
if self.version == 1:
|
| 71 |
+
scale = self.scale
|
| 72 |
+
elif self.version == 2:
|
| 73 |
+
scale = F.softplus(self.scale) + self.alpha
|
| 74 |
+
else:
|
| 75 |
+
raise ValueError(f'Unsupported version: {self.version}')
|
| 76 |
+
|
| 77 |
+
scale = scale.float() / sigma.float()
|
| 78 |
+
|
| 79 |
+
y = weight * scale
|
| 80 |
+
|
| 81 |
+
if dtype in (torch.float16, torch.bfloat16):
|
| 82 |
+
y = y.to(dtype)
|
| 83 |
+
return y
|
| 84 |
+
|
| 85 |
+
def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs):
|
| 86 |
+
version_key = f'{prefix}_sn_version'
|
| 87 |
+
if version_key not in state_dict:
|
| 88 |
+
self.version = 1
|
| 89 |
+
state_dict[version_key] = torch.tensor(1)
|
| 90 |
+
return super()._load_from_state_dict(state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs)
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
class _AttnSNReweight(nn.Module):
|
| 94 |
+
def __init__(self, weight: torch.Tensor, *args, init_norm_to_current: bool = False, renorm_values: bool = False, **kwargs):
|
| 95 |
+
super().__init__()
|
| 96 |
+
|
| 97 |
+
parts = weight.split(weight.shape[0] // 3, dim=0)
|
| 98 |
+
|
| 99 |
+
ct = 2 if not renorm_values else 3
|
| 100 |
+
|
| 101 |
+
self.parts = nn.ModuleList([
|
| 102 |
+
_SNReweight(p, *args, init_norm_to_current=init_norm_to_current, **kwargs) if i < ct else nn.Identity()
|
| 103 |
+
for i, p in enumerate(parts)
|
| 104 |
+
])
|
| 105 |
+
|
| 106 |
+
def forward(self, weight: torch.Tensor, *args, **kwargs):
|
| 107 |
+
parts = weight.split(weight.shape[0] // 3, dim=0)
|
| 108 |
+
|
| 109 |
+
parts = [
|
| 110 |
+
fn(p)
|
| 111 |
+
for fn, p in zip(self.parts, parts)
|
| 112 |
+
]
|
| 113 |
+
|
| 114 |
+
return torch.cat(parts, dim=0)
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def enable_spectral_reparam(model: nn.Module,
|
| 118 |
+
n_power_iterations: int = 1,
|
| 119 |
+
eps: float = 1e-6,
|
| 120 |
+
init_norm_to_current: bool = False,
|
| 121 |
+
renorm_values: bool = True,
|
| 122 |
+
renorm_mlp: bool = True):
|
| 123 |
+
# print('Enabling spectral reparametrization')
|
| 124 |
+
for mod in model.modules():
|
| 125 |
+
if isinstance(mod, Attention):
|
| 126 |
+
parametrize.register_parametrization(
|
| 127 |
+
mod.qkv,
|
| 128 |
+
'weight',
|
| 129 |
+
_AttnSNReweight(mod.qkv.weight, n_power_iterations, dim=0, eps=eps, init_norm_to_current=init_norm_to_current, renorm_values=renorm_values),
|
| 130 |
+
)
|
| 131 |
+
pass
|
| 132 |
+
elif isinstance(mod, Mlp) and renorm_mlp:
|
| 133 |
+
parametrize.register_parametrization(
|
| 134 |
+
mod.fc1,
|
| 135 |
+
'weight',
|
| 136 |
+
_SNReweight(mod.fc1.weight, n_power_iterations, dim=0, eps=eps, init_norm_to_current=init_norm_to_current),
|
| 137 |
+
)
|
| 138 |
+
parametrize.register_parametrization(
|
| 139 |
+
mod.fc2,
|
| 140 |
+
'weight',
|
| 141 |
+
_SNReweight(mod.fc2.weight, n_power_iterations, dim=0, eps=eps, init_norm_to_current=init_norm_to_current),
|
| 142 |
+
)
|
| 143 |
+
pass
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def configure_spectral_reparam_from_args(model: nn.Module, args):
|
| 147 |
+
spectral_reparam = getattr(args, 'spectral_reparam', False)
|
| 148 |
+
if isinstance(spectral_reparam, bool) and spectral_reparam:
|
| 149 |
+
enable_spectral_reparam(model, init_norm_to_current=args.pretrained)
|
| 150 |
+
elif isinstance(spectral_reparam, dict):
|
| 151 |
+
enable_spectral_reparam(
|
| 152 |
+
model,
|
| 153 |
+
n_power_iterations=spectral_reparam.get('n_power_iterations', 1),
|
| 154 |
+
eps=spectral_reparam.get('eps', 1e-12),
|
| 155 |
+
init_norm_to_current=args.pretrained,
|
| 156 |
+
)
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def disable_spectral_reparam(model: nn.Module):
|
| 160 |
+
for mod in model.modules():
|
| 161 |
+
if isinstance(mod, Attention):
|
| 162 |
+
parametrize.remove_parametrizations(mod.qkv, 'weight')
|
| 163 |
+
pass
|
| 164 |
+
elif isinstance(mod, Mlp):
|
| 165 |
+
parametrize.remove_parametrizations(mod.fc1, 'weight')
|
| 166 |
+
parametrize.remove_parametrizations(mod.fc2, 'weight')
|
| 167 |
+
pass
|
| 168 |
+
|
| 169 |
+
|
| 170 |
+
if __name__ == '__main__':
|
| 171 |
+
import argparse
|
| 172 |
+
from . import radio_model as create_model
|
| 173 |
+
|
| 174 |
+
parser = argparse.ArgumentParser(description='Remove parametrization from state dict')
|
| 175 |
+
parser.add_argument('--checkpoint', type=str, required=True, help='The checkpoint to load')
|
| 176 |
+
parser.add_argument('--output', type=str, default='', help='Where to store the checkpoint')
|
| 177 |
+
parser.add_argument('--release', default=False, action='store_true', help='Prune extraneous checkpoint fields')
|
| 178 |
+
parser.add_argument('--strict', default=False, action='store_true', help='Strictly load the state dict')
|
| 179 |
+
|
| 180 |
+
args = parser.parse_args()
|
| 181 |
+
|
| 182 |
+
if not args.output:
|
| 183 |
+
chk_dir, chk_name = os.path.split(args.checkpoint)
|
| 184 |
+
args.output = os.path.join(chk_dir, f'clean_{chk_name}')
|
| 185 |
+
print(f'Set output to "{args.output}"')
|
| 186 |
+
|
| 187 |
+
chk = torch.load(args.checkpoint, map_location='cpu', mmap=True)
|
| 188 |
+
|
| 189 |
+
model = create_model.create_model_from_args(chk['args'])
|
| 190 |
+
|
| 191 |
+
key = 'base_model.'
|
| 192 |
+
mod_state = dict()
|
| 193 |
+
extra_state = dict()
|
| 194 |
+
for k, v in chk['state_dict'].items():
|
| 195 |
+
if k.startswith(key):
|
| 196 |
+
mod_state[k[len(key):]] = v
|
| 197 |
+
else:
|
| 198 |
+
extra_state[k] = v
|
| 199 |
+
|
| 200 |
+
chk_load_info = model.load_state_dict(mod_state, strict=args.strict)
|
| 201 |
+
if chk_load_info.unexpected_keys or chk_load_info.missing_keys:
|
| 202 |
+
print(chk_load_info)
|
| 203 |
+
|
| 204 |
+
if chk['args'].spectral_reparam:
|
| 205 |
+
disable_spectral_reparam(model)
|
| 206 |
+
|
| 207 |
+
if hasattr(chk['args'], 'dtype'):
|
| 208 |
+
model.to(dtype=chk['args'].dtype)
|
| 209 |
+
|
| 210 |
+
mod_state = model.state_dict()
|
| 211 |
+
final_state = dict()
|
| 212 |
+
final_state.update({f'{key}{k}': v for k, v in mod_state.items()})
|
| 213 |
+
final_state.update(extra_state)
|
| 214 |
+
|
| 215 |
+
chk['state_dict'] = final_state
|
| 216 |
+
chk['args'].spectral_reparam = False
|
| 217 |
+
|
| 218 |
+
if args.release:
|
| 219 |
+
chk = {
|
| 220 |
+
'arch': chk['arch'],
|
| 221 |
+
'epoch': chk['epoch'],
|
| 222 |
+
'state_dict': chk['state_dict'],
|
| 223 |
+
'args': chk['args'],
|
| 224 |
+
}
|
| 225 |
+
|
| 226 |
+
torch.save(chk, args.output)
|
| 227 |
+
pass
|
radio_eradio_model.py
ADDED
|
@@ -0,0 +1,1392 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
|
| 3 |
+
# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
|
| 4 |
+
#
|
| 5 |
+
# NVIDIA CORPORATION and its licensors retain all intellectual property
|
| 6 |
+
# and proprietary rights in and to this software, related documentation
|
| 7 |
+
# and any modifications thereto. Any use, reproduction, disclosure or
|
| 8 |
+
# distribution of this software and related documentation without an express
|
| 9 |
+
# license agreement from NVIDIA CORPORATION is strictly prohibited.
|
| 10 |
+
|
| 11 |
+
# E-RADIO model from
|
| 12 |
+
# Mike Ranzinger, Greg Heinrich, Jan Kautz, and Pavlo Molchanov. "AM-RADIO: Agglomerative Model--Reduce All Domains Into One." arXiv preprint arXiv:2312.06709 (2023).
|
| 13 |
+
|
| 14 |
+
# based on FasterViT, Swin Transformer, YOLOv8
|
| 15 |
+
|
| 16 |
+
# FasterViT:
|
| 17 |
+
# Ali Hatamizadeh, Greg Heinrich, Hongxu Yin, Andrew Tao, Jose M. Alvarez, Jan Kautz, and Pavlo Molchanov. "FasterViT: Fast Vision Transformers with Hierarchical Attention." arXiv preprint arXiv:2306.06189 (2023).
|
| 18 |
+
|
| 19 |
+
import timm
|
| 20 |
+
import torch
|
| 21 |
+
import torch.nn as nn
|
| 22 |
+
from timm.models.registry import register_model
|
| 23 |
+
|
| 24 |
+
from timm.models.layers import trunc_normal_, DropPath, LayerNorm2d
|
| 25 |
+
import numpy as np
|
| 26 |
+
import torch.nn.functional as F
|
| 27 |
+
import math
|
| 28 |
+
import warnings
|
| 29 |
+
|
| 30 |
+
#######################
|
| 31 |
+
## Codebase from YOLOv8
|
| 32 |
+
## BEGINNING
|
| 33 |
+
#######################
|
| 34 |
+
|
| 35 |
+
class C2f(nn.Module):
|
| 36 |
+
"""Faster Implementation of CSP Bottleneck with 2 convolutions."""
|
| 37 |
+
"""From YOLOv8 codebase"""
|
| 38 |
+
def __init__(self, c1, c2, n=1, shortcut=False, g=1, e=0.5, drop_path=None): # ch_in, ch_out, number, shortcut, groups, expansion
|
| 39 |
+
super().__init__()
|
| 40 |
+
if drop_path is None:
|
| 41 |
+
drop_path = [0.0] * n
|
| 42 |
+
|
| 43 |
+
self.c = int(c2 * e) # hidden channels
|
| 44 |
+
self.cv1 = Conv(c1, 2 * self.c, 1, 1)
|
| 45 |
+
self.cv2 = Conv((2 + n) * self.c, c2, 1) # optional act=FReLU(c2)
|
| 46 |
+
self.m = nn.ModuleList(Bottleneck(self.c, self.c, shortcut, g, k=((3, 3), (3, 3)), e=1.0, drop_path=drop_path[i]) for i in range(n))
|
| 47 |
+
|
| 48 |
+
def forward(self, x):
|
| 49 |
+
"""Forward pass through C2f layer."""
|
| 50 |
+
y = list(self.cv1(x).chunk(2, 1))
|
| 51 |
+
y.extend(m(y[-1]) for m in self.m)
|
| 52 |
+
return self.cv2(torch.cat(y, 1))
|
| 53 |
+
|
| 54 |
+
def forward_split(self, x):
|
| 55 |
+
"""Forward pass using split() instead of chunk()."""
|
| 56 |
+
y = list(self.cv1(x).split((self.c, self.c), 1))
|
| 57 |
+
y.extend(m(y[-1]) for m in self.m)
|
| 58 |
+
return self.cv2(torch.cat(y, 1))
|
| 59 |
+
|
| 60 |
+
class Bottleneck(nn.Module):
|
| 61 |
+
"""Standard bottleneck."""
|
| 62 |
+
|
| 63 |
+
def __init__(self, c1, c2, shortcut=True, g=1, k=(3, 3), e=0.5, drop_path=0.0): # ch_in, ch_out, shortcut, groups, kernels, expand
|
| 64 |
+
super().__init__()
|
| 65 |
+
c_ = int(c2 * e) # hidden channels
|
| 66 |
+
self.cv1 = Conv(c1, c_, k[0], 1)
|
| 67 |
+
self.cv2 = Conv(c_, c2, k[1], 1, g=g)
|
| 68 |
+
self.add = shortcut and c1 == c2
|
| 69 |
+
self.drop_path1 = DropPath(drop_path) if drop_path > 0. else nn.Identity()
|
| 70 |
+
|
| 71 |
+
def forward(self, x):
|
| 72 |
+
"""'forward()' applies the YOLOv5 FPN to input data."""
|
| 73 |
+
return x + self.drop_path1(self.cv2(self.cv1(x))) if self.add else self.cv2(self.cv1(x))
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
class Conv(nn.Module):
|
| 77 |
+
"""Modified to support layer fusion"""
|
| 78 |
+
default_act = nn.SiLU() # default activation
|
| 79 |
+
|
| 80 |
+
def __init__(self, a, b, kernel_size=1, stride=1, padding=None, g=1, dilation=1, bn_weight_init=1, bias=False, act=True):
|
| 81 |
+
super().__init__()
|
| 82 |
+
|
| 83 |
+
self.conv = torch.nn.Conv2d(a, b, kernel_size, stride, autopad(kernel_size, padding, dilation), dilation, g, bias=False)
|
| 84 |
+
if 1:
|
| 85 |
+
self.bn = torch.nn.BatchNorm2d(b)
|
| 86 |
+
torch.nn.init.constant_(self.bn.weight, bn_weight_init)
|
| 87 |
+
torch.nn.init.constant_(self.bn.bias, 0)
|
| 88 |
+
self.act = self.default_act if act is True else act if isinstance(act, nn.Module) else nn.Identity()
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def forward(self,x):
|
| 92 |
+
x = self.conv(x)
|
| 93 |
+
x = self.bn(x)
|
| 94 |
+
x = self.act(x)
|
| 95 |
+
return x
|
| 96 |
+
|
| 97 |
+
@torch.no_grad()
|
| 98 |
+
def switch_to_deploy(self):
|
| 99 |
+
# return 1
|
| 100 |
+
if not isinstance(self.bn, nn.Identity):
|
| 101 |
+
c, bn = self.conv, self.bn
|
| 102 |
+
w = bn.weight / (bn.running_var + bn.eps) ** 0.5
|
| 103 |
+
w = c.weight * w[:, None, None, None]
|
| 104 |
+
b = bn.bias - bn.running_mean * bn.weight / \
|
| 105 |
+
(bn.running_var + bn.eps)**0.5
|
| 106 |
+
|
| 107 |
+
self.conv.weight.data.copy_(w)
|
| 108 |
+
self.conv.bias = nn.Parameter(b)
|
| 109 |
+
|
| 110 |
+
self.bn = nn.Identity()
|
| 111 |
+
|
| 112 |
+
def autopad(k, p=None, d=1): # kernel, padding, dilation
|
| 113 |
+
"""Pad to 'same' shape outputs."""
|
| 114 |
+
if d > 1:
|
| 115 |
+
k = d * (k - 1) + 1 if isinstance(k, int) else [d * (x - 1) + 1 for x in k] # actual kernel-size
|
| 116 |
+
if p is None:
|
| 117 |
+
p = k // 2 if isinstance(k, int) else [x // 2 for x in k] # auto-pad
|
| 118 |
+
return p
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
#######################
|
| 122 |
+
## Codebase from YOLOv8
|
| 123 |
+
## END
|
| 124 |
+
#######################
|
| 125 |
+
|
| 126 |
+
def pixel_unshuffle(data, factor=2):
|
| 127 |
+
# performs nn.PixelShuffle(factor) in reverse, torch has some bug for ONNX and TRT, so doing it manually
|
| 128 |
+
B, C, H, W = data.shape
|
| 129 |
+
return data.view(B, C, factor, H//factor, factor, W//factor).permute(0,1,2,4,3,5).reshape(B, -1, H//factor, W//factor)
|
| 130 |
+
|
| 131 |
+
class SwiGLU(nn.Module):
|
| 132 |
+
# should be more advanced, but doesnt improve results so far
|
| 133 |
+
def forward(self, x):
|
| 134 |
+
x, gate = x.chunk(2, dim=-1)
|
| 135 |
+
return F.silu(gate) * x
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def window_partition(x, window_size):
|
| 139 |
+
"""
|
| 140 |
+
Function for partitioning image into windows and later do windowed attention
|
| 141 |
+
Args:
|
| 142 |
+
x: (B, C, H, W)
|
| 143 |
+
window_size: window size
|
| 144 |
+
Returns:
|
| 145 |
+
windows - local window features (num_windows*B, window_size*window_size, C)
|
| 146 |
+
(Hp, Wp) - the size of the padded image
|
| 147 |
+
"""
|
| 148 |
+
B, C, H, W = x.shape
|
| 149 |
+
|
| 150 |
+
if window_size == 0 or (window_size==H and window_size==W):
|
| 151 |
+
windows = x.flatten(2).transpose(1, 2)
|
| 152 |
+
Hp, Wp = H, W
|
| 153 |
+
else:
|
| 154 |
+
pad_h = (window_size - H % window_size) % window_size
|
| 155 |
+
pad_w = (window_size - W % window_size) % window_size
|
| 156 |
+
if pad_h > 0 or pad_w > 0:
|
| 157 |
+
x = F.pad(x, (0, pad_w, 0, pad_h), mode="reflect")
|
| 158 |
+
Hp, Wp = H + pad_h, W + pad_w
|
| 159 |
+
|
| 160 |
+
x = x.view(B, C, Hp // window_size, window_size, Wp // window_size, window_size)
|
| 161 |
+
windows = x.permute(0, 2, 4, 3, 5, 1).reshape(-1, window_size*window_size, C)
|
| 162 |
+
|
| 163 |
+
return windows, (Hp, Wp)
|
| 164 |
+
|
| 165 |
+
class Conv2d_BN(nn.Module):
|
| 166 |
+
'''
|
| 167 |
+
Conv2d + BN layer with folding capability to speed up inference
|
| 168 |
+
Can be merged with Conv() function with additional arguments
|
| 169 |
+
'''
|
| 170 |
+
def __init__(self, a, b, kernel_size=1, stride=1, padding=0, dilation=1, groups=1, bn_weight_init=1, bias=False):
|
| 171 |
+
super().__init__()
|
| 172 |
+
self.conv = torch.nn.Conv2d(a, b, kernel_size, stride, padding, dilation, groups, bias=False)
|
| 173 |
+
if 1:
|
| 174 |
+
self.bn = torch.nn.BatchNorm2d(b)
|
| 175 |
+
torch.nn.init.constant_(self.bn.weight, bn_weight_init)
|
| 176 |
+
torch.nn.init.constant_(self.bn.bias, 0)
|
| 177 |
+
|
| 178 |
+
def forward(self,x):
|
| 179 |
+
x = self.conv(x)
|
| 180 |
+
x = self.bn(x)
|
| 181 |
+
return x
|
| 182 |
+
|
| 183 |
+
@torch.no_grad()
|
| 184 |
+
def switch_to_deploy(self):
|
| 185 |
+
if not isinstance(self.bn, nn.Identity):
|
| 186 |
+
c, bn = self.conv, self.bn
|
| 187 |
+
w = bn.weight / (bn.running_var + bn.eps) ** 0.5
|
| 188 |
+
w = c.weight * w[:, None, None, None]
|
| 189 |
+
b = bn.bias - bn.running_mean * bn.weight / \
|
| 190 |
+
(bn.running_var + bn.eps)**0.5
|
| 191 |
+
self.conv.weight.data.copy_(w)
|
| 192 |
+
self.conv.bias = nn.Parameter(b)
|
| 193 |
+
self.bn = nn.Identity()
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
def window_reverse(windows, window_size, H, W, pad_hw):
|
| 198 |
+
"""
|
| 199 |
+
Windows to the full feature map
|
| 200 |
+
Args:
|
| 201 |
+
windows: local window features (num_windows*B, window_size, window_size, C)
|
| 202 |
+
window_size: Window size
|
| 203 |
+
H: Height of image
|
| 204 |
+
W: Width of image
|
| 205 |
+
pad_w - a tuple of image passing used in windowing step
|
| 206 |
+
Returns:
|
| 207 |
+
x: (B, C, H, W)
|
| 208 |
+
|
| 209 |
+
"""
|
| 210 |
+
# print(f"window_reverse, windows.shape {windows.shape}")
|
| 211 |
+
Hp, Wp = pad_hw
|
| 212 |
+
if window_size == 0 or (window_size==H and window_size==W):
|
| 213 |
+
B = int(windows.shape[0] / (Hp * Wp / window_size / window_size))
|
| 214 |
+
x = windows.transpose(1, 2).view(B, -1, H, W)
|
| 215 |
+
else:
|
| 216 |
+
B = int(windows.shape[0] / (Hp * Wp / window_size / window_size))
|
| 217 |
+
x = windows.view(B, Hp // window_size, Wp // window_size, window_size, window_size, -1)
|
| 218 |
+
x = x.permute(0, 5, 1, 3, 2, 4).reshape(B,windows.shape[2], Hp, Wp)
|
| 219 |
+
|
| 220 |
+
if Hp > H or Wp > W:
|
| 221 |
+
x = x[:, :, :H, :W, ].contiguous()
|
| 222 |
+
|
| 223 |
+
return x
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
class PosEmbMLPSwinv2D(nn.Module):
|
| 228 |
+
"""
|
| 229 |
+
2D positional embedding from Swin Transformer v2
|
| 230 |
+
Added functionality to store the positional embedding in the model and not recompute it every time
|
| 231 |
+
"""
|
| 232 |
+
def __init__(
|
| 233 |
+
self, window_size, pretrained_window_size, num_heads, seq_length, no_log=False, cpb_mlp_hidden=512,
|
| 234 |
+
):
|
| 235 |
+
super().__init__()
|
| 236 |
+
self.window_size = window_size
|
| 237 |
+
self.num_heads = num_heads
|
| 238 |
+
# mlp to generate continuous relative position bias
|
| 239 |
+
self.cpb_mlp = nn.Sequential(
|
| 240 |
+
nn.Linear(2, cpb_mlp_hidden, bias=True),
|
| 241 |
+
nn.ReLU(inplace=True),
|
| 242 |
+
nn.Linear(cpb_mlp_hidden, num_heads, bias=False),
|
| 243 |
+
)
|
| 244 |
+
|
| 245 |
+
self.grid_exists = False
|
| 246 |
+
self.seq_length = seq_length
|
| 247 |
+
self.deploy = False
|
| 248 |
+
self.num_heads = num_heads
|
| 249 |
+
self.no_log = no_log
|
| 250 |
+
self.pretrained_window_size = pretrained_window_size
|
| 251 |
+
self.relative_bias_window_size = window_size
|
| 252 |
+
|
| 253 |
+
relative_coords_table, relative_position_index, relative_bias = self.relative_bias_initialization(window_size, num_heads,
|
| 254 |
+
pretrained_window_size, seq_length,
|
| 255 |
+
no_log)
|
| 256 |
+
|
| 257 |
+
self.register_buffer("relative_coords_table", relative_coords_table)
|
| 258 |
+
self.register_buffer("relative_position_index", relative_position_index)
|
| 259 |
+
self.register_buffer("relative_bias", relative_bias) # for EMA
|
| 260 |
+
|
| 261 |
+
def relative_bias_initialization(self, window_size, num_heads, pretrained_window_size, seq_length, no_log):
|
| 262 |
+
# as in separate function to support window size chage after model weights loading
|
| 263 |
+
relative_coords_h = torch.arange(
|
| 264 |
+
-(window_size[0] - 1), window_size[0], dtype=torch.float32
|
| 265 |
+
)
|
| 266 |
+
relative_coords_w = torch.arange(
|
| 267 |
+
-(window_size[1] - 1), window_size[1], dtype=torch.float32
|
| 268 |
+
)
|
| 269 |
+
relative_coords_table = (
|
| 270 |
+
torch.stack(torch.meshgrid([relative_coords_h, relative_coords_w]))
|
| 271 |
+
.permute(1, 2, 0)
|
| 272 |
+
.contiguous()
|
| 273 |
+
.unsqueeze(0)
|
| 274 |
+
) # 1, 2*Wh-1, 2*Ww-1, 2
|
| 275 |
+
if pretrained_window_size[0] > 0:
|
| 276 |
+
relative_coords_table[:, :, :, 0] /= pretrained_window_size[0] - 1
|
| 277 |
+
relative_coords_table[:, :, :, 1] /= pretrained_window_size[1] - 1
|
| 278 |
+
else:
|
| 279 |
+
relative_coords_table[:, :, :, 0] /= self.window_size[0] - 1
|
| 280 |
+
relative_coords_table[:, :, :, 1] /= self.window_size[1] - 1
|
| 281 |
+
|
| 282 |
+
if not no_log:
|
| 283 |
+
relative_coords_table *= 8 # normalize to -8, 8
|
| 284 |
+
relative_coords_table = (
|
| 285 |
+
torch.sign(relative_coords_table)
|
| 286 |
+
* torch.log2(torch.abs(relative_coords_table) + 1.0)
|
| 287 |
+
/ np.log2(8)
|
| 288 |
+
)
|
| 289 |
+
|
| 290 |
+
# get pair-wise relative position index for each token inside the window
|
| 291 |
+
coords_h = torch.arange(self.window_size[0])
|
| 292 |
+
coords_w = torch.arange(self.window_size[1])
|
| 293 |
+
coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww
|
| 294 |
+
coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww
|
| 295 |
+
relative_coords = (
|
| 296 |
+
coords_flatten[:, :, None] - coords_flatten[:, None, :]
|
| 297 |
+
) # 2, Wh*Ww, Wh*Ww
|
| 298 |
+
relative_coords = relative_coords.permute(
|
| 299 |
+
1, 2, 0
|
| 300 |
+
).contiguous() # Wh*Ww, Wh*Ww, 2
|
| 301 |
+
relative_coords[:, :, 0] += self.window_size[0] - 1 # shift to start from 0
|
| 302 |
+
relative_coords[:, :, 1] += self.window_size[1] - 1
|
| 303 |
+
relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1
|
| 304 |
+
relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww
|
| 305 |
+
|
| 306 |
+
relative_bias = torch.zeros(1, num_heads, seq_length, seq_length)
|
| 307 |
+
|
| 308 |
+
self.relative_bias_window_size = window_size
|
| 309 |
+
|
| 310 |
+
return relative_coords_table, relative_position_index, relative_bias
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
def switch_to_deploy(self):
|
| 314 |
+
self.deploy = True
|
| 315 |
+
self.grid_exists = True
|
| 316 |
+
|
| 317 |
+
def forward(self, input_tensor):
|
| 318 |
+
# for efficiency, we want this forward to be folded into a single operation (sum)
|
| 319 |
+
# if resolution stays the same, then we dont need to recompute MLP layers
|
| 320 |
+
|
| 321 |
+
if not self.deploy or self.training:
|
| 322 |
+
self.grid_exists = False
|
| 323 |
+
|
| 324 |
+
#compare if all elements in self.window_size list match those in self.relative_bias_window_size
|
| 325 |
+
if not all([self.window_size[i] == self.relative_bias_window_size[i] for i in range(len(self.window_size))]):
|
| 326 |
+
relative_coords_table, relative_position_index, relative_bias = self.relative_bias_initialization(self.window_size, self.num_heads,
|
| 327 |
+
self.pretrained_window_size, self.seq_length,
|
| 328 |
+
self.no_log)
|
| 329 |
+
|
| 330 |
+
self.relative_coords_table = relative_coords_table.to(self.relative_coords_table.device)
|
| 331 |
+
self.relative_position_index = relative_position_index.to(self.relative_position_index.device)
|
| 332 |
+
self.relative_bias = relative_bias.to(self.relative_bias.device)
|
| 333 |
+
|
| 334 |
+
if self.deploy and self.grid_exists:
|
| 335 |
+
input_tensor = input_tensor + self.relative_bias
|
| 336 |
+
return input_tensor
|
| 337 |
+
|
| 338 |
+
if 1:
|
| 339 |
+
self.grid_exists = True
|
| 340 |
+
|
| 341 |
+
relative_position_bias_table = self.cpb_mlp(
|
| 342 |
+
self.relative_coords_table
|
| 343 |
+
).view(-1, self.num_heads)
|
| 344 |
+
relative_position_bias = relative_position_bias_table[
|
| 345 |
+
self.relative_position_index.view(-1)
|
| 346 |
+
].view(
|
| 347 |
+
self.window_size[0] * self.window_size[1],
|
| 348 |
+
self.window_size[0] * self.window_size[1],
|
| 349 |
+
-1,
|
| 350 |
+
) # Wh*Ww,Wh*Ww,nH
|
| 351 |
+
|
| 352 |
+
relative_position_bias = relative_position_bias.permute(
|
| 353 |
+
2, 0, 1
|
| 354 |
+
).contiguous() # nH, Wh*Ww, Wh*Ww
|
| 355 |
+
relative_position_bias = 16 * torch.sigmoid(relative_position_bias)
|
| 356 |
+
|
| 357 |
+
self.relative_bias = relative_position_bias.unsqueeze(0)
|
| 358 |
+
|
| 359 |
+
input_tensor = input_tensor + self.relative_bias
|
| 360 |
+
return input_tensor
|
| 361 |
+
|
| 362 |
+
|
| 363 |
+
class GRAAttentionBlock(nn.Module):
|
| 364 |
+
def __init__(self, window_size, dim_in, dim_out,
|
| 365 |
+
num_heads, drop_path=0., qk_scale=None, qkv_bias=False,
|
| 366 |
+
norm_layer=nn.LayerNorm, layer_scale=None,
|
| 367 |
+
use_swiglu=True,
|
| 368 |
+
subsample_ratio=1, dim_ratio=1, conv_base=False,
|
| 369 |
+
do_windowing=True, multi_query=False, use_shift=0,
|
| 370 |
+
cpb_mlp_hidden=512, conv_groups_ratio=0):
|
| 371 |
+
'''
|
| 372 |
+
Global Resolution Attention Block , see README for details
|
| 373 |
+
Attention with subsampling to get a bigger receptive field for attention
|
| 374 |
+
conv_base - use conv2d instead of avgpool2d for downsample / upsample
|
| 375 |
+
|
| 376 |
+
|
| 377 |
+
'''
|
| 378 |
+
super().__init__()
|
| 379 |
+
|
| 380 |
+
self.shift_size=window_size//2 if use_shift else 0
|
| 381 |
+
|
| 382 |
+
self.do_windowing = do_windowing
|
| 383 |
+
self.subsample_ratio = subsample_ratio
|
| 384 |
+
|
| 385 |
+
|
| 386 |
+
|
| 387 |
+
if do_windowing:
|
| 388 |
+
if conv_base:
|
| 389 |
+
self.downsample_op = nn.Conv2d(dim_in, dim_out, kernel_size=subsample_ratio, stride=subsample_ratio) if subsample_ratio > 1 else nn.Identity()
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
self.downsample_mixer = nn.Identity()
|
| 393 |
+
self.upsample_mixer = nn.Identity()
|
| 394 |
+
self.upsample_op = nn.ConvTranspose2d(dim_in, dim_out, kernel_size=subsample_ratio, stride=subsample_ratio) if subsample_ratio > 1 else nn.Identity()
|
| 395 |
+
else:
|
| 396 |
+
self.downsample_op = nn.AvgPool2d(kernel_size=subsample_ratio, stride=subsample_ratio) if subsample_ratio > 1 else nn.Identity()
|
| 397 |
+
self.downsample_mixer = Conv2d_BN(dim_in, dim_out, kernel_size=1, stride=1) if subsample_ratio > 1 else nn.Identity()
|
| 398 |
+
self.upsample_mixer = nn.Upsample(scale_factor=subsample_ratio, mode='nearest') if subsample_ratio > 1 else nn.Identity()
|
| 399 |
+
self.upsample_op = Conv2d_BN(dim_in, dim_out, kernel_size=1, stride=1, padding=0, bias=False) if subsample_ratio > 1 else nn.Identity()
|
| 400 |
+
|
| 401 |
+
|
| 402 |
+
# in case there is no downsampling conv we want to have it separately
|
| 403 |
+
# will help with information propagation between windows
|
| 404 |
+
if subsample_ratio == 1:
|
| 405 |
+
# conv_groups_ratio=0
|
| 406 |
+
self.pre_conv = Conv2d_BN(dim_in, dim_in, kernel_size=3, stride=1, padding=1, groups=max(1,int(conv_groups_ratio*dim_in)), bias=False)
|
| 407 |
+
# self.pre_conv = nn.Conv2d(dim_in, dim_in, kernel_size=3, stride=1, padding=1, groups=max(1,int(conv_groups_ratio*dim_in)), bias=False)
|
| 408 |
+
# self.pre_conv_act = nn.ReLU6()
|
| 409 |
+
#for simplicity:
|
| 410 |
+
self.pre_conv_act = nn.Identity()
|
| 411 |
+
if conv_groups_ratio == -1:
|
| 412 |
+
self.pre_conv = nn.Identity()
|
| 413 |
+
self.pre_conv_act = nn.Identity()
|
| 414 |
+
|
| 415 |
+
self.window_size = window_size
|
| 416 |
+
|
| 417 |
+
self.norm1 = norm_layer(dim_in)
|
| 418 |
+
|
| 419 |
+
self.attn = WindowAttention(
|
| 420 |
+
dim_in,
|
| 421 |
+
num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale,
|
| 422 |
+
resolution=window_size,
|
| 423 |
+
seq_length=window_size**2, dim_out=dim_in, multi_query=multi_query,
|
| 424 |
+
shift_size=self.shift_size, cpb_mlp_hidden=cpb_mlp_hidden)
|
| 425 |
+
|
| 426 |
+
self.drop_path1 = DropPath(drop_path) if drop_path > 0. else nn.Identity()
|
| 427 |
+
|
| 428 |
+
use_layer_scale = layer_scale is not None and type(layer_scale) in [int, float]
|
| 429 |
+
self.gamma1 = nn.Parameter(layer_scale * torch.ones(dim_in)) if use_layer_scale else 1
|
| 430 |
+
|
| 431 |
+
### mlp layer
|
| 432 |
+
mlp_ratio = 4
|
| 433 |
+
self.norm2 = norm_layer(dim_in)
|
| 434 |
+
mlp_hidden_dim = int(dim_in * mlp_ratio)
|
| 435 |
+
|
| 436 |
+
activation = nn.GELU if not use_swiglu else SwiGLU
|
| 437 |
+
mlp_hidden_dim = int((4 * dim_in * 1 / 2) / 64) * 64 if use_swiglu else mlp_hidden_dim
|
| 438 |
+
|
| 439 |
+
self.mlp = Mlp(in_features=dim_in, hidden_features=mlp_hidden_dim, act_layer=activation, use_swiglu=use_swiglu)
|
| 440 |
+
|
| 441 |
+
self.gamma2 = nn.Parameter(layer_scale * torch.ones(dim_in)) if layer_scale else 1
|
| 442 |
+
self.drop_path2=DropPath(drop_path) if drop_path > 0. else nn.Identity()
|
| 443 |
+
|
| 444 |
+
|
| 445 |
+
def forward(self, x):
|
| 446 |
+
skip_connection = x
|
| 447 |
+
attn_mask = None
|
| 448 |
+
|
| 449 |
+
# in case there is no downsampling conv we want to have it separately
|
| 450 |
+
# will help with information propagation
|
| 451 |
+
if self.subsample_ratio == 1:
|
| 452 |
+
x = self.pre_conv_act(self.pre_conv(x)) + skip_connection
|
| 453 |
+
|
| 454 |
+
if self.do_windowing:
|
| 455 |
+
# performing windowing if required
|
| 456 |
+
x = self.downsample_op(x)
|
| 457 |
+
x = self.downsample_mixer(x)
|
| 458 |
+
|
| 459 |
+
if self.window_size>0:
|
| 460 |
+
H, W = x.shape[2], x.shape[3]
|
| 461 |
+
|
| 462 |
+
if self.shift_size > 0 and H>self.window_size and W>self.window_size:
|
| 463 |
+
# @swin like cyclic shift, doesnt show better performance
|
| 464 |
+
x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(2, 3))
|
| 465 |
+
|
| 466 |
+
x, pad_hw = window_partition(x, self.window_size)
|
| 467 |
+
|
| 468 |
+
if self.shift_size > 0 and H>self.window_size and W>self.window_size:
|
| 469 |
+
# set atten matrix to have -100 and the top right square
|
| 470 |
+
# attn[:, :, :-self.shift_size, -self.shift_size:] = -100.0
|
| 471 |
+
# calculate attention mask for SW-MSA
|
| 472 |
+
# not used in final version, can be useful for some cases especially for high res
|
| 473 |
+
H, W = pad_hw
|
| 474 |
+
img_mask = torch.zeros((1, H, W, 1), device=x.device) # 1 H W 1
|
| 475 |
+
h_slices = (slice(0, -self.window_size),
|
| 476 |
+
slice(-self.window_size, -self.shift_size),
|
| 477 |
+
slice(-self.shift_size, None))
|
| 478 |
+
w_slices = (slice(0, -self.window_size),
|
| 479 |
+
slice(-self.window_size, -self.shift_size),
|
| 480 |
+
slice(-self.shift_size, None))
|
| 481 |
+
cnt = 0
|
| 482 |
+
for h in h_slices:
|
| 483 |
+
for w in w_slices:
|
| 484 |
+
img_mask[:, h, w, :] = cnt
|
| 485 |
+
cnt += 1
|
| 486 |
+
img_mask = img_mask.transpose(1,2).transpose(1,3)
|
| 487 |
+
mask_windows = window_partition(img_mask, self.window_size) # nW, window_size, window_size, 1
|
| 488 |
+
|
| 489 |
+
mask_windows = mask_windows[0].view(-1, self.window_size * self.window_size)
|
| 490 |
+
attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2)
|
| 491 |
+
attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0))
|
| 492 |
+
|
| 493 |
+
# window attention
|
| 494 |
+
x = x + self.drop_path1(self.gamma1*self.attn(self.norm1(x), attn_mask=attn_mask)) # or pass H,W
|
| 495 |
+
# mlp layer
|
| 496 |
+
x = x + self.drop_path2(self.gamma2*self.mlp(self.norm2(x)))
|
| 497 |
+
|
| 498 |
+
if self.do_windowing:
|
| 499 |
+
if self.window_size > 0:
|
| 500 |
+
x = window_reverse(x, self.window_size, H, W, pad_hw)
|
| 501 |
+
|
| 502 |
+
# reverse cyclic shift
|
| 503 |
+
if self.shift_size > 0 and H>self.window_size and W>self.window_size:
|
| 504 |
+
# @swin like cyclic shift, not tested
|
| 505 |
+
x = torch.roll(x, shifts=(self.shift_size, self.shift_size), dims=(2, 3))
|
| 506 |
+
|
| 507 |
+
x = self.upsample_mixer(x)
|
| 508 |
+
x = self.upsample_op(x)
|
| 509 |
+
|
| 510 |
+
|
| 511 |
+
if x.shape[2] != skip_connection.shape[2] or x.shape[3] != skip_connection.shape[3]:
|
| 512 |
+
x = torch.nn.functional.pad(x, ( 0, -x.shape[3] + skip_connection.shape[3], 0, -x.shape[2] + skip_connection.shape[2]), mode="reflect")
|
| 513 |
+
# need to add skip connection because downsampling and upsampling will break residual connection
|
| 514 |
+
# 0.5 is needed to make sure that the skip connection is not too strong
|
| 515 |
+
# in case of no downsample / upsample we can show that 0.5 compensates for the residual connection
|
| 516 |
+
x = 0.5 * x + 0.5 * skip_connection
|
| 517 |
+
return x
|
| 518 |
+
|
| 519 |
+
|
| 520 |
+
|
| 521 |
+
|
| 522 |
+
class MultiResolutionAttention(nn.Module):
|
| 523 |
+
"""
|
| 524 |
+
MultiResolutionAttention (MRA) module
|
| 525 |
+
The idea is to use multiple attention blocks with different resolution
|
| 526 |
+
Feature maps are downsampled / upsampled for each attention block on different blocks
|
| 527 |
+
Every attention block supports windowing
|
| 528 |
+
"""
|
| 529 |
+
|
| 530 |
+
def __init__(self, window_size, sr_ratio,
|
| 531 |
+
dim, dim_ratio, num_heads,
|
| 532 |
+
do_windowing=True,
|
| 533 |
+
layer_scale=1e-5, norm_layer=nn.LayerNorm,
|
| 534 |
+
drop_path = 0, qkv_bias=False, qk_scale=1.0,
|
| 535 |
+
use_swiglu=True, multi_query=False, conv_base=False,
|
| 536 |
+
use_shift=0, cpb_mlp_hidden=512, conv_groups_ratio=0) -> None:
|
| 537 |
+
"""
|
| 538 |
+
Args:
|
| 539 |
+
input_resolution: input image resolution
|
| 540 |
+
window_size: window size
|
| 541 |
+
compression_ratio: compression ratio
|
| 542 |
+
max_depth: maximum depth of the GRA module
|
| 543 |
+
use_shift: do window shifting
|
| 544 |
+
"""
|
| 545 |
+
super().__init__()
|
| 546 |
+
|
| 547 |
+
depth = len(sr_ratio)
|
| 548 |
+
|
| 549 |
+
self.attention_blocks = nn.ModuleList()
|
| 550 |
+
|
| 551 |
+
|
| 552 |
+
for i in range(depth):
|
| 553 |
+
subsample_ratio = sr_ratio[i]
|
| 554 |
+
if len(window_size) > i:
|
| 555 |
+
window_size_local = window_size[i]
|
| 556 |
+
else:
|
| 557 |
+
window_size_local = window_size[0]
|
| 558 |
+
|
| 559 |
+
self.attention_blocks.append(GRAAttentionBlock(window_size=window_size_local,
|
| 560 |
+
dim_in=dim, dim_out=dim, num_heads=num_heads,
|
| 561 |
+
qkv_bias=qkv_bias, qk_scale=qk_scale, norm_layer=norm_layer,
|
| 562 |
+
layer_scale=layer_scale, drop_path=drop_path,
|
| 563 |
+
use_swiglu=use_swiglu, subsample_ratio=subsample_ratio, dim_ratio=dim_ratio,
|
| 564 |
+
do_windowing=do_windowing, multi_query=multi_query, conv_base=conv_base,
|
| 565 |
+
use_shift=use_shift, cpb_mlp_hidden=cpb_mlp_hidden, conv_groups_ratio=conv_groups_ratio),
|
| 566 |
+
)
|
| 567 |
+
|
| 568 |
+
def forward(self, x):
|
| 569 |
+
|
| 570 |
+
for attention_block in self.attention_blocks:
|
| 571 |
+
x = attention_block(x)
|
| 572 |
+
|
| 573 |
+
return x
|
| 574 |
+
|
| 575 |
+
|
| 576 |
+
|
| 577 |
+
class Mlp(nn.Module):
|
| 578 |
+
"""
|
| 579 |
+
Multi-Layer Perceptron (MLP) block
|
| 580 |
+
"""
|
| 581 |
+
|
| 582 |
+
def __init__(self,
|
| 583 |
+
in_features,
|
| 584 |
+
hidden_features=None,
|
| 585 |
+
out_features=None,
|
| 586 |
+
act_layer=nn.GELU,
|
| 587 |
+
use_swiglu=True,
|
| 588 |
+
drop=0.):
|
| 589 |
+
"""
|
| 590 |
+
Args:
|
| 591 |
+
in_features: input features dimension.
|
| 592 |
+
hidden_features: hidden features dimension.
|
| 593 |
+
out_features: output features dimension.
|
| 594 |
+
act_layer: activation function.
|
| 595 |
+
drop: dropout rate.
|
| 596 |
+
"""
|
| 597 |
+
|
| 598 |
+
super().__init__()
|
| 599 |
+
out_features = out_features or in_features
|
| 600 |
+
hidden_features = hidden_features or in_features
|
| 601 |
+
self.fc1 = nn.Linear(in_features, hidden_features * (2 if use_swiglu else 1), bias=False)
|
| 602 |
+
self.act = act_layer()
|
| 603 |
+
self.fc2 = nn.Linear(hidden_features, out_features, bias=False)
|
| 604 |
+
|
| 605 |
+
def forward(self, x):
|
| 606 |
+
x_size = x.size()
|
| 607 |
+
x = x.view(-1, x_size[-1])
|
| 608 |
+
x = self.fc1(x)
|
| 609 |
+
x = self.act(x)
|
| 610 |
+
x = self.fc2(x)
|
| 611 |
+
x = x.view(x_size)
|
| 612 |
+
return x
|
| 613 |
+
|
| 614 |
+
class Downsample(nn.Module):
|
| 615 |
+
"""
|
| 616 |
+
Down-sampling block
|
| 617 |
+
Pixel Unshuffle is used for down-sampling, works great accuracy - wise but takes 10% more TRT time
|
| 618 |
+
"""
|
| 619 |
+
|
| 620 |
+
def __init__(self,
|
| 621 |
+
dim,
|
| 622 |
+
shuffle = False,
|
| 623 |
+
):
|
| 624 |
+
"""
|
| 625 |
+
Args:
|
| 626 |
+
dim: feature size dimension.
|
| 627 |
+
shuffle: idea with
|
| 628 |
+
keep_dim: bool argument for maintaining the resolution.
|
| 629 |
+
"""
|
| 630 |
+
|
| 631 |
+
super().__init__()
|
| 632 |
+
dim_out = 2 * dim
|
| 633 |
+
|
| 634 |
+
if shuffle:
|
| 635 |
+
self.norm = lambda x: pixel_unshuffle(x, factor=2)
|
| 636 |
+
self.reduction = Conv2d_BN(dim*4, dim_out, 1, 1, 0, bias=False)
|
| 637 |
+
# pixel unshuffleging works well but doesnt provide any speedup
|
| 638 |
+
else:
|
| 639 |
+
# removed layer norm for better, in this formulation we are getting 10% better speed
|
| 640 |
+
# LayerNorm for high resolution inputs will be a pain as it pools over the entire spatial dimension
|
| 641 |
+
# therefore we remove it compared to the original implementation in FasterViT
|
| 642 |
+
self.norm = nn.Identity()
|
| 643 |
+
self.reduction = Conv2d_BN(dim, dim_out, 3, 2, 1, bias=False)
|
| 644 |
+
|
| 645 |
+
|
| 646 |
+
def forward(self, x):
|
| 647 |
+
x = self.norm(x)
|
| 648 |
+
x = self.reduction(x)
|
| 649 |
+
return x
|
| 650 |
+
|
| 651 |
+
|
| 652 |
+
class PatchEmbed(nn.Module):
|
| 653 |
+
"""
|
| 654 |
+
Patch embedding block
|
| 655 |
+
Used to convert image into an initial set of feature maps with lower resolution
|
| 656 |
+
"""
|
| 657 |
+
|
| 658 |
+
def __init__(self, in_chans=3, in_dim=64, dim=96, shuffle_down=False):
|
| 659 |
+
"""
|
| 660 |
+
Args:
|
| 661 |
+
in_chans: number of input channels.
|
| 662 |
+
in_dim: intermediate feature size dimension to speed up stem.
|
| 663 |
+
dim: final stem channel number
|
| 664 |
+
shuffle_down: use PixelUnshuffle for down-sampling, effectively increases the receptive field
|
| 665 |
+
"""
|
| 666 |
+
|
| 667 |
+
super().__init__()
|
| 668 |
+
# shuffle_down = False
|
| 669 |
+
if not shuffle_down:
|
| 670 |
+
self.proj = nn.Identity()
|
| 671 |
+
self.conv_down = nn.Sequential(
|
| 672 |
+
Conv2d_BN(in_chans, in_dim, 3, 2, 1, bias=False),
|
| 673 |
+
nn.ReLU(),
|
| 674 |
+
Conv2d_BN(in_dim, dim, 3, 2, 1, bias=False),
|
| 675 |
+
nn.ReLU()
|
| 676 |
+
)
|
| 677 |
+
else:
|
| 678 |
+
self.proj = lambda x: pixel_unshuffle(x, factor=4)
|
| 679 |
+
self.conv_down = nn.Sequential(Conv2d_BN(in_chans*16, dim, 3, 1, 1),
|
| 680 |
+
nn.ReLU(),
|
| 681 |
+
)
|
| 682 |
+
|
| 683 |
+
def forward(self, x):
|
| 684 |
+
x = self.proj(x)
|
| 685 |
+
x = self.conv_down(x)
|
| 686 |
+
return x
|
| 687 |
+
|
| 688 |
+
|
| 689 |
+
|
| 690 |
+
class ConvBlock(nn.Module):
|
| 691 |
+
"""
|
| 692 |
+
Convolutional block, used in first couple of stages
|
| 693 |
+
Experimented with plan resnet-18 like modules, they are the best in terms of throughput
|
| 694 |
+
Finally, YOLOv8 idea seem to work fine (resnet-18 like block with squeezed feature dimension, and feature concatendation at the end)
|
| 695 |
+
"""
|
| 696 |
+
def __init__(self, dim,
|
| 697 |
+
drop_path=0.,
|
| 698 |
+
layer_scale=None,
|
| 699 |
+
kernel_size=3,
|
| 700 |
+
):
|
| 701 |
+
super().__init__()
|
| 702 |
+
|
| 703 |
+
self.conv1 = Conv2d_BN(dim, dim, kernel_size=kernel_size, stride=1, padding=1)
|
| 704 |
+
self.act1 = nn.GELU()
|
| 705 |
+
|
| 706 |
+
self.conv2 = Conv2d_BN(dim, dim, kernel_size=kernel_size, stride=1, padding=1)
|
| 707 |
+
|
| 708 |
+
self.layer_scale = layer_scale
|
| 709 |
+
if layer_scale is not None and type(layer_scale) in [int, float]:
|
| 710 |
+
self.gamma = nn.Parameter(layer_scale * torch.ones(dim))
|
| 711 |
+
self.layer_scale = True
|
| 712 |
+
else:
|
| 713 |
+
self.layer_scale = False
|
| 714 |
+
self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity()
|
| 715 |
+
|
| 716 |
+
def forward(self, x):
|
| 717 |
+
input = x
|
| 718 |
+
|
| 719 |
+
x = self.conv1(x)
|
| 720 |
+
x = self.act1(x)
|
| 721 |
+
x = self.conv2(x)
|
| 722 |
+
|
| 723 |
+
if self.layer_scale:
|
| 724 |
+
x = x * self.gamma.view(1, -1, 1, 1)
|
| 725 |
+
x = input + self.drop_path(x)
|
| 726 |
+
return x
|
| 727 |
+
|
| 728 |
+
|
| 729 |
+
class WindowAttention(nn.Module):
|
| 730 |
+
# Windowed Attention from SwinV2
|
| 731 |
+
# use a MLP trick to deal with various input image resolutions, then fold it to improve speed
|
| 732 |
+
|
| 733 |
+
def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, resolution=0,
|
| 734 |
+
seq_length=0, dim_out=None, multi_query=False, shift_size=0, cpb_mlp_hidden=512):
|
| 735 |
+
# taken from EdgeViT and tweaked with attention bias.
|
| 736 |
+
super().__init__()
|
| 737 |
+
if not dim_out: dim_out = dim
|
| 738 |
+
self.shift_size = shift_size
|
| 739 |
+
self.multi_query = multi_query
|
| 740 |
+
self.num_heads = num_heads
|
| 741 |
+
head_dim = dim // num_heads
|
| 742 |
+
self.head_dim = dim // num_heads
|
| 743 |
+
|
| 744 |
+
self.dim_internal = dim
|
| 745 |
+
|
| 746 |
+
self.scale = qk_scale or head_dim ** -0.5
|
| 747 |
+
if not multi_query:
|
| 748 |
+
self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
|
| 749 |
+
else:
|
| 750 |
+
self.qkv = nn.Linear(dim, dim + 2*self.head_dim, bias=qkv_bias)
|
| 751 |
+
|
| 752 |
+
self.proj = nn.Linear(dim, dim_out, bias=False)
|
| 753 |
+
# attention positional bias
|
| 754 |
+
self.pos_emb_funct = PosEmbMLPSwinv2D(window_size=[resolution, resolution],
|
| 755 |
+
pretrained_window_size=[resolution, resolution],
|
| 756 |
+
num_heads=num_heads,
|
| 757 |
+
seq_length=seq_length,
|
| 758 |
+
cpb_mlp_hidden=cpb_mlp_hidden)
|
| 759 |
+
|
| 760 |
+
self.resolution = resolution
|
| 761 |
+
|
| 762 |
+
def forward(self, x, attn_mask = None):
|
| 763 |
+
B, N, C = x.shape
|
| 764 |
+
|
| 765 |
+
if not self.multi_query:
|
| 766 |
+
qkv = self.qkv(x).reshape(B, -1, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
|
| 767 |
+
q, k, v = qkv[0], qkv[1], qkv[2]
|
| 768 |
+
else:
|
| 769 |
+
qkv = self.qkv(x)
|
| 770 |
+
(q, k, v) = qkv.split([self.dim_internal, self.head_dim, self.head_dim], dim=2)
|
| 771 |
+
|
| 772 |
+
q = q.reshape(B, -1, self.num_heads, C // self.num_heads).permute(0, 2, 1, 3)
|
| 773 |
+
k = k.reshape(B, -1, 1, C // self.num_heads).permute(0, 2, 1, 3)
|
| 774 |
+
v = v.reshape(B, -1, 1, C // self.num_heads).permute(0, 2, 1, 3)
|
| 775 |
+
|
| 776 |
+
attn = (q @ k.transpose(-2, -1)) * self.scale
|
| 777 |
+
|
| 778 |
+
attn = self.pos_emb_funct(attn)
|
| 779 |
+
|
| 780 |
+
#add window shift
|
| 781 |
+
if attn_mask is not None:
|
| 782 |
+
nW = attn_mask.shape[0]
|
| 783 |
+
attn = attn.view(B // nW, nW, self.num_heads, N, N) + attn_mask.unsqueeze(1).unsqueeze(0)
|
| 784 |
+
attn = attn.view(-1, self.num_heads, N, N)
|
| 785 |
+
|
| 786 |
+
attn = attn.softmax(dim=-1)
|
| 787 |
+
x = (attn @ v).transpose(1, 2).reshape(B, -1, C)
|
| 788 |
+
x = self.proj(x)
|
| 789 |
+
return x
|
| 790 |
+
|
| 791 |
+
|
| 792 |
+
|
| 793 |
+
class ERADIOLayer(nn.Module):
|
| 794 |
+
"""
|
| 795 |
+
E-RADIO Layer
|
| 796 |
+
"""
|
| 797 |
+
|
| 798 |
+
def __init__(self,
|
| 799 |
+
dim,
|
| 800 |
+
depth,
|
| 801 |
+
num_heads,
|
| 802 |
+
window_size,
|
| 803 |
+
conv=False,
|
| 804 |
+
downsample=True,
|
| 805 |
+
mlp_ratio=4.,
|
| 806 |
+
qkv_bias=False,
|
| 807 |
+
qk_scale=None,
|
| 808 |
+
norm_layer=nn.LayerNorm,
|
| 809 |
+
drop_path=0.,
|
| 810 |
+
layer_scale=None,
|
| 811 |
+
layer_scale_conv=None,
|
| 812 |
+
sr_dim_ratio=1,
|
| 813 |
+
sr_ratio=1,
|
| 814 |
+
multi_query=False,
|
| 815 |
+
use_swiglu=True,
|
| 816 |
+
yolo_arch=False,
|
| 817 |
+
downsample_shuffle=False,
|
| 818 |
+
conv_base=False,
|
| 819 |
+
use_shift=False,
|
| 820 |
+
cpb_mlp_hidden=512,
|
| 821 |
+
conv_groups_ratio=0,
|
| 822 |
+
verbose: bool = True,
|
| 823 |
+
|
| 824 |
+
):
|
| 825 |
+
"""
|
| 826 |
+
Args:
|
| 827 |
+
dim: feature size dimension.
|
| 828 |
+
depth: number of layers in each stage.
|
| 829 |
+
input_resolution: input image resolution.
|
| 830 |
+
window_size: window size in each stage.
|
| 831 |
+
downsample: bool argument for down-sampling.
|
| 832 |
+
mlp_ratio: MLP ratio.
|
| 833 |
+
num_heads: number of heads in each stage.
|
| 834 |
+
qkv_bias: bool argument for query, key, value learnable bias.
|
| 835 |
+
qk_scale: bool argument to scaling query, key.
|
| 836 |
+
drop: dropout rate.
|
| 837 |
+
attn_drop: attention dropout rate.
|
| 838 |
+
drop_path: drop path rate.
|
| 839 |
+
norm_layer: normalization layer.
|
| 840 |
+
layer_scale: layer scaling coefficient.
|
| 841 |
+
use_shift: SWIN like window shifting for half the window size for every alternating layer (considering multi-resolution)
|
| 842 |
+
conv_groups_ratio: group ratio for conv when no subsampling in multi-res attention
|
| 843 |
+
"""
|
| 844 |
+
|
| 845 |
+
super().__init__()
|
| 846 |
+
self.conv = conv
|
| 847 |
+
self.yolo_arch=False
|
| 848 |
+
self.verbose = verbose
|
| 849 |
+
if conv:
|
| 850 |
+
if not yolo_arch:
|
| 851 |
+
self.blocks = nn.ModuleList([
|
| 852 |
+
ConvBlock(dim=dim,
|
| 853 |
+
drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path,
|
| 854 |
+
layer_scale=layer_scale_conv)
|
| 855 |
+
for i in range(depth)])
|
| 856 |
+
self.blocks = nn.Sequential(*self.blocks)
|
| 857 |
+
else:
|
| 858 |
+
self.blocks = C2f(dim,dim,n=depth,shortcut=True,e=0.5)
|
| 859 |
+
self.yolo_arch=True
|
| 860 |
+
else:
|
| 861 |
+
if not isinstance(window_size, list): window_size = [window_size]
|
| 862 |
+
self.window_size = window_size[0]
|
| 863 |
+
self.do_single_windowing = True
|
| 864 |
+
if not isinstance(sr_ratio, list): sr_ratio = [sr_ratio]
|
| 865 |
+
self.sr_ratio = sr_ratio
|
| 866 |
+
if any([sr!=1 for sr in sr_ratio]) or len(set(window_size))>1:
|
| 867 |
+
self.do_single_windowing = False
|
| 868 |
+
do_windowing = True
|
| 869 |
+
else:
|
| 870 |
+
self.do_single_windowing = True
|
| 871 |
+
do_windowing = False
|
| 872 |
+
|
| 873 |
+
#for v2_2
|
| 874 |
+
if conv_groups_ratio != -1:
|
| 875 |
+
self.do_single_windowing = False
|
| 876 |
+
do_windowing = True
|
| 877 |
+
|
| 878 |
+
self.blocks = nn.ModuleList()
|
| 879 |
+
for i in range(depth):
|
| 880 |
+
self.blocks.append(
|
| 881 |
+
MultiResolutionAttention(window_size=window_size,
|
| 882 |
+
sr_ratio=sr_ratio,
|
| 883 |
+
dim=dim,
|
| 884 |
+
dim_ratio = sr_dim_ratio,
|
| 885 |
+
num_heads=num_heads,
|
| 886 |
+
norm_layer=norm_layer,
|
| 887 |
+
drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path,
|
| 888 |
+
layer_scale=layer_scale,
|
| 889 |
+
qkv_bias=qkv_bias,
|
| 890 |
+
qk_scale=qk_scale,
|
| 891 |
+
use_swiglu=use_swiglu,
|
| 892 |
+
do_windowing=do_windowing,
|
| 893 |
+
multi_query=multi_query,
|
| 894 |
+
conv_base=conv_base,
|
| 895 |
+
cpb_mlp_hidden=cpb_mlp_hidden,
|
| 896 |
+
use_shift =0 if ((not use_shift) or ((i) % 2 == 0)) else True ,
|
| 897 |
+
conv_groups_ratio=conv_groups_ratio,
|
| 898 |
+
))
|
| 899 |
+
self.blocks = nn.Sequential(*self.blocks)
|
| 900 |
+
|
| 901 |
+
self.transformer = not conv
|
| 902 |
+
self.downsample = None if not downsample else Downsample(dim=dim, shuffle=downsample_shuffle)
|
| 903 |
+
|
| 904 |
+
|
| 905 |
+
def forward(self, x):
|
| 906 |
+
B, C, H, W = x.shape
|
| 907 |
+
|
| 908 |
+
# do padding for transforemr
|
| 909 |
+
interpolate = True
|
| 910 |
+
if self.transformer and interpolate:
|
| 911 |
+
# Windowed Attention will split feature map into windows with the size of window_size x window_size
|
| 912 |
+
# if the resolution is not divisible by window_size, we need to interpolate the feature map
|
| 913 |
+
# can be done via padding, but doing so after training hurts the model performance.
|
| 914 |
+
# interpolation affects the performance as well, but not as much as padding
|
| 915 |
+
if isinstance(self.window_size, list) or isinstance(self.window_size, tuple):
|
| 916 |
+
current_max_window_size = max(self.window_size)
|
| 917 |
+
else:
|
| 918 |
+
current_max_window_size = self.window_size
|
| 919 |
+
|
| 920 |
+
max_window_size = max([res_upsample*current_max_window_size for res_upsample in self.sr_ratio])
|
| 921 |
+
if H % max_window_size != 0 or W % max_window_size != 0:
|
| 922 |
+
new_h = int(np.ceil(H/max_window_size)*max_window_size)
|
| 923 |
+
new_w = int(np.ceil(W/max_window_size)*max_window_size)
|
| 924 |
+
x = F.interpolate(x, size=(new_h, new_w), mode='nearest')
|
| 925 |
+
if self.verbose:
|
| 926 |
+
warnings.warn(f"Choosen window size is not optimal for given resolution. Interpolation of features maps will be done and it can affect the performance. Max window size is {max_window_size}, feature map size is {H}x{W}, interpolated feature map size is {new_h}x{new_w}.")
|
| 927 |
+
|
| 928 |
+
|
| 929 |
+
if self.transformer and self.do_single_windowing:
|
| 930 |
+
H, W = x.shape[2], x.shape[3]
|
| 931 |
+
x, pad_hw = window_partition(x, self.window_size)
|
| 932 |
+
|
| 933 |
+
#run main blocks
|
| 934 |
+
x = self.blocks(x)
|
| 935 |
+
|
| 936 |
+
if self.transformer and self.do_single_windowing:
|
| 937 |
+
x = window_reverse(x, self.window_size, H, W, pad_hw)
|
| 938 |
+
|
| 939 |
+
if self.transformer and interpolate:
|
| 940 |
+
#lets keep original resolution, might be not ideal, but for the upsampling tower we need to keep the expected resolution.
|
| 941 |
+
x = F.interpolate(x, size=(H, W), mode='nearest')
|
| 942 |
+
|
| 943 |
+
if self.downsample is None:
|
| 944 |
+
return x, x
|
| 945 |
+
|
| 946 |
+
return self.downsample(x), x # changing to output pre downsampled features
|
| 947 |
+
|
| 948 |
+
|
| 949 |
+
class InterpolateLayer(nn.Module):
|
| 950 |
+
def __init__(self, size=None, scale_factor=None, mode='nearest'):
|
| 951 |
+
super(InterpolateLayer, self).__init__()
|
| 952 |
+
self.size = size
|
| 953 |
+
self.scale_factor = scale_factor
|
| 954 |
+
self.mode = mode
|
| 955 |
+
|
| 956 |
+
def forward(self, x):
|
| 957 |
+
return F.interpolate(x, size=self.size, scale_factor=self.scale_factor, mode=self.mode)
|
| 958 |
+
|
| 959 |
+
|
| 960 |
+
class HiResNeck(nn.Module):
|
| 961 |
+
"""
|
| 962 |
+
The block is used to output dense features from all stages
|
| 963 |
+
Otherwise, by default, only the last stage features are returned with E-RADIO
|
| 964 |
+
"""
|
| 965 |
+
def __init__(self, dim, depths, neck_start_stage, full_features_head_dim, downsample_enabled):
|
| 966 |
+
|
| 967 |
+
'''
|
| 968 |
+
Hi Resolution neck to support output of high res features that are useful for dense tasks.
|
| 969 |
+
depths - total number of layers in the base model
|
| 970 |
+
neck_start_stage - when to start the neck, 0 - start from the first stage, 1 - start from the second stage etc.
|
| 971 |
+
earlier layers result in higher resolution features at the cost of compute
|
| 972 |
+
full_features_head_dim - number of channels in the dense features head
|
| 973 |
+
'''
|
| 974 |
+
super().__init__()
|
| 975 |
+
# create feature projection layers for segmentation output
|
| 976 |
+
self.neck_features_proj = nn.ModuleList()
|
| 977 |
+
self.neck_start_stage = neck_start_stage
|
| 978 |
+
upsample_ratio = 1
|
| 979 |
+
for i in range(len(depths)):
|
| 980 |
+
level_n_features_output = int(dim * 2 ** i)
|
| 981 |
+
|
| 982 |
+
if self.neck_start_stage > i: continue
|
| 983 |
+
|
| 984 |
+
if (upsample_ratio > 1) or full_features_head_dim!=level_n_features_output:
|
| 985 |
+
feature_projection = nn.Sequential()
|
| 986 |
+
if False:
|
| 987 |
+
feature_projection.add_module("norm",nn.BatchNorm2d(level_n_features_output)) #fast, but worse
|
| 988 |
+
feature_projection.add_module("dconv", nn.ConvTranspose2d(level_n_features_output,
|
| 989 |
+
full_features_head_dim, kernel_size=upsample_ratio, stride=upsample_ratio))
|
| 990 |
+
else:
|
| 991 |
+
# B, in_channels, H, W -> B, in_channels, H*upsample_ratio, W*upsample_ratio
|
| 992 |
+
# print("upsample ratio", upsample_ratio, level_n_features_output, level_n_features_output)
|
| 993 |
+
feature_projection.add_module("upsample", InterpolateLayer(scale_factor=upsample_ratio, mode='nearest'))
|
| 994 |
+
feature_projection.add_module("conv1", nn.Conv2d(level_n_features_output, level_n_features_output, kernel_size=3, stride=1, padding=1, groups=level_n_features_output))
|
| 995 |
+
feature_projection.add_module("norm",nn.BatchNorm2d(level_n_features_output))
|
| 996 |
+
# B, in_channels, H*upsample_ratio, W*upsample_ratio -> B, full_features_head_dim, H*upsample_ratio, W*upsample_ratio
|
| 997 |
+
feature_projection.add_module("conv2", nn.Conv2d(level_n_features_output, full_features_head_dim, kernel_size=1, stride=1, padding=0))
|
| 998 |
+
else:
|
| 999 |
+
feature_projection = nn.Sequential()
|
| 1000 |
+
|
| 1001 |
+
self.neck_features_proj.append(feature_projection)
|
| 1002 |
+
|
| 1003 |
+
if i>0 and downsample_enabled[i]:
|
| 1004 |
+
upsample_ratio *= 2
|
| 1005 |
+
|
| 1006 |
+
def forward(self, x, il_level=-1, full_features=None):
|
| 1007 |
+
if self.neck_start_stage > il_level:
|
| 1008 |
+
return full_features
|
| 1009 |
+
|
| 1010 |
+
if full_features is None:
|
| 1011 |
+
full_features = self.neck_features_proj[il_level - self.neck_start_stage](x)
|
| 1012 |
+
else:
|
| 1013 |
+
#upsample torch tensor x to match full_features size, and add to full_features
|
| 1014 |
+
feature_projection = self.neck_features_proj[il_level - self.neck_start_stage](x)
|
| 1015 |
+
if feature_projection.shape[2] != full_features.shape[2] or feature_projection.shape[3] != full_features.shape[3]:
|
| 1016 |
+
feature_projection = torch.nn.functional.pad(feature_projection, ( 0, -feature_projection.shape[3] + full_features.shape[3], 0, -feature_projection.shape[2] + full_features.shape[2]))
|
| 1017 |
+
full_features = full_features + feature_projection
|
| 1018 |
+
return full_features
|
| 1019 |
+
|
| 1020 |
+
class ERADIO(nn.Module):
|
| 1021 |
+
"""
|
| 1022 |
+
Efficient RADIO
|
| 1023 |
+
"""
|
| 1024 |
+
|
| 1025 |
+
def __init__(self,
|
| 1026 |
+
dim,
|
| 1027 |
+
in_dim,
|
| 1028 |
+
depths,
|
| 1029 |
+
window_size,
|
| 1030 |
+
mlp_ratio,
|
| 1031 |
+
num_heads,
|
| 1032 |
+
drop_path_rate=0.2,
|
| 1033 |
+
in_chans=3,
|
| 1034 |
+
num_classes=1000,
|
| 1035 |
+
qkv_bias=False,
|
| 1036 |
+
qk_scale=None,
|
| 1037 |
+
layer_scale=None,
|
| 1038 |
+
layer_scale_conv=None,
|
| 1039 |
+
layer_norm_last=False,
|
| 1040 |
+
sr_ratio = [1, 1, 1, 1],
|
| 1041 |
+
max_depth = -1,
|
| 1042 |
+
conv_base=False,
|
| 1043 |
+
use_swiglu=False,
|
| 1044 |
+
multi_query=False,
|
| 1045 |
+
norm_layer=nn.LayerNorm,
|
| 1046 |
+
drop_uniform=False,
|
| 1047 |
+
yolo_arch=False,
|
| 1048 |
+
shuffle_down=False,
|
| 1049 |
+
downsample_shuffle=False,
|
| 1050 |
+
return_full_features=False,
|
| 1051 |
+
full_features_head_dim=128,
|
| 1052 |
+
neck_start_stage=1,
|
| 1053 |
+
use_neck=False,
|
| 1054 |
+
use_shift=False,
|
| 1055 |
+
cpb_mlp_hidden=512,
|
| 1056 |
+
conv_groups_ratio=0,
|
| 1057 |
+
verbose: bool = False,
|
| 1058 |
+
**kwargs):
|
| 1059 |
+
"""
|
| 1060 |
+
Args:
|
| 1061 |
+
dim: feature size dimension.
|
| 1062 |
+
depths: number of layers in each stage.
|
| 1063 |
+
window_size: window size in each stage.
|
| 1064 |
+
mlp_ratio: MLP ratio.
|
| 1065 |
+
num_heads: number of heads in each stage.
|
| 1066 |
+
drop_path_rate: drop path rate.
|
| 1067 |
+
in_chans: number of input channels.
|
| 1068 |
+
num_classes: number of classes.
|
| 1069 |
+
qkv_bias: bool argument for query, key, value learnable bias.
|
| 1070 |
+
qk_scale: bool argument to scaling query, key.
|
| 1071 |
+
drop_rate: dropout rate.
|
| 1072 |
+
attn_drop_rate: attention dropout rate.
|
| 1073 |
+
norm_layer: normalization layer.
|
| 1074 |
+
layer_scale: layer scaling coefficient.
|
| 1075 |
+
return_full_features: output dense features as well as logits
|
| 1076 |
+
full_features_head_dim: number of channels in the dense features head
|
| 1077 |
+
neck_start_stage: a stage id to start full feature neck. Model has 4 stages, indix starts with 0
|
| 1078 |
+
for 224 resolution, the output of the stage before downsample:
|
| 1079 |
+
stage 0: 56x56, stage 1: 28x28, stage 2: 14x14, stage 3: 7x7
|
| 1080 |
+
use_neck: even for summarization embedding use neck
|
| 1081 |
+
use_shift: SWIN like window shifting but without masking attention
|
| 1082 |
+
conv_groups_ratio: will be used for conv blocks where there is no multires attention,
|
| 1083 |
+
if 0 then normal conv,
|
| 1084 |
+
if 1 then channels are independent,
|
| 1085 |
+
if -1 then no conv at all
|
| 1086 |
+
|
| 1087 |
+
"""
|
| 1088 |
+
super().__init__()
|
| 1089 |
+
|
| 1090 |
+
num_features = int(dim * 2 ** (len(depths) - 1))
|
| 1091 |
+
self.num_classes = num_classes
|
| 1092 |
+
self.patch_embed = PatchEmbed(in_chans=in_chans, in_dim=in_dim, dim=dim, shuffle_down=shuffle_down)
|
| 1093 |
+
# set return_full_features true if we want to return full features from all stages
|
| 1094 |
+
self.return_full_features = return_full_features
|
| 1095 |
+
self.use_neck = use_neck
|
| 1096 |
+
|
| 1097 |
+
dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))]
|
| 1098 |
+
if drop_uniform:
|
| 1099 |
+
dpr = [drop_path_rate for x in range(sum(depths))]
|
| 1100 |
+
|
| 1101 |
+
if not isinstance(max_depth, list): max_depth = [max_depth] * len(depths)
|
| 1102 |
+
|
| 1103 |
+
self.levels = nn.ModuleList()
|
| 1104 |
+
for i in range(len(depths)):
|
| 1105 |
+
conv = True if (i == 0 or i == 1) else False
|
| 1106 |
+
|
| 1107 |
+
level = ERADIOLayer(dim=int(dim * 2 ** i),
|
| 1108 |
+
depth=depths[i],
|
| 1109 |
+
num_heads=num_heads[i],
|
| 1110 |
+
window_size=window_size[i],
|
| 1111 |
+
mlp_ratio=mlp_ratio,
|
| 1112 |
+
qkv_bias=qkv_bias,
|
| 1113 |
+
qk_scale=qk_scale,
|
| 1114 |
+
conv=conv,
|
| 1115 |
+
drop_path=dpr[sum(depths[:i]):sum(depths[:i + 1])],
|
| 1116 |
+
downsample=(i < len(depths) - 1),
|
| 1117 |
+
layer_scale=layer_scale,
|
| 1118 |
+
layer_scale_conv=layer_scale_conv,
|
| 1119 |
+
sr_ratio=sr_ratio[i],
|
| 1120 |
+
use_swiglu=use_swiglu,
|
| 1121 |
+
multi_query=multi_query,
|
| 1122 |
+
norm_layer=norm_layer,
|
| 1123 |
+
yolo_arch=yolo_arch,
|
| 1124 |
+
downsample_shuffle=downsample_shuffle,
|
| 1125 |
+
conv_base=conv_base,
|
| 1126 |
+
cpb_mlp_hidden=cpb_mlp_hidden,
|
| 1127 |
+
use_shift=use_shift,
|
| 1128 |
+
conv_groups_ratio=conv_groups_ratio,
|
| 1129 |
+
verbose=verbose)
|
| 1130 |
+
|
| 1131 |
+
self.levels.append(level)
|
| 1132 |
+
|
| 1133 |
+
if self.return_full_features or self.use_neck:
|
| 1134 |
+
#num_heads
|
| 1135 |
+
downsample_enabled = [self.levels[i-1].downsample is not None for i in range(len(self.levels))]
|
| 1136 |
+
self.high_res_neck = HiResNeck(dim, depths, neck_start_stage, full_features_head_dim, downsample_enabled)
|
| 1137 |
+
|
| 1138 |
+
self.switched_to_deploy = False
|
| 1139 |
+
|
| 1140 |
+
self.norm = LayerNorm2d(num_features) if layer_norm_last else nn.BatchNorm2d(num_features)
|
| 1141 |
+
self.avgpool = nn.AdaptiveAvgPool2d(1)
|
| 1142 |
+
self.head = nn.Linear(num_features, num_classes) if num_classes > 0 else nn.Identity()
|
| 1143 |
+
self.apply(self._init_weights)
|
| 1144 |
+
|
| 1145 |
+
def _init_weights(self, m):
|
| 1146 |
+
if isinstance(m, nn.Linear):
|
| 1147 |
+
trunc_normal_(m.weight, std=.02)
|
| 1148 |
+
if isinstance(m, nn.Linear) and m.bias is not None:
|
| 1149 |
+
nn.init.constant_(m.bias, 0)
|
| 1150 |
+
elif isinstance(m, nn.LayerNorm):
|
| 1151 |
+
nn.init.constant_(m.bias, 0)
|
| 1152 |
+
nn.init.constant_(m.weight, 1.0)
|
| 1153 |
+
elif isinstance(m, LayerNorm2d):
|
| 1154 |
+
nn.init.constant_(m.bias, 0)
|
| 1155 |
+
nn.init.constant_(m.weight, 1.0)
|
| 1156 |
+
elif isinstance(m, nn.BatchNorm2d):
|
| 1157 |
+
nn.init.ones_(m.weight)
|
| 1158 |
+
nn.init.zeros_(m.bias)
|
| 1159 |
+
|
| 1160 |
+
@torch.jit.ignore
|
| 1161 |
+
def no_weight_decay_keywords(self):
|
| 1162 |
+
return {'rpb'}
|
| 1163 |
+
|
| 1164 |
+
def forward_features(self, x):
|
| 1165 |
+
_, _, H, W = x.shape
|
| 1166 |
+
if H % 32 != 0 or W % 32 != 0:
|
| 1167 |
+
raise ValueError(f"E-RADIO requires input dimensions to be divisible by 32 but got H x W: {H} x {W}")
|
| 1168 |
+
x = self.patch_embed(x)
|
| 1169 |
+
full_features = None
|
| 1170 |
+
for il, level in enumerate(self.levels):
|
| 1171 |
+
x, pre_downsample_x = level(x)
|
| 1172 |
+
|
| 1173 |
+
if self.return_full_features or self.use_neck:
|
| 1174 |
+
full_features = self.high_res_neck(pre_downsample_x, il, full_features)
|
| 1175 |
+
|
| 1176 |
+
# x = self.norm(full_features if (self.return_full_features or self.use_neck) else x)
|
| 1177 |
+
x = self.norm(x) # new version for
|
| 1178 |
+
|
| 1179 |
+
if not self.return_full_features:
|
| 1180 |
+
return x, None
|
| 1181 |
+
|
| 1182 |
+
return x, full_features
|
| 1183 |
+
|
| 1184 |
+
def forward(self, x):
|
| 1185 |
+
x, full_features = self.forward_features(x)
|
| 1186 |
+
|
| 1187 |
+
x = self.avgpool(x)
|
| 1188 |
+
x = torch.flatten(x, 1)
|
| 1189 |
+
|
| 1190 |
+
x = self.head(x)
|
| 1191 |
+
if full_features is not None:
|
| 1192 |
+
return x, full_features
|
| 1193 |
+
return x
|
| 1194 |
+
|
| 1195 |
+
def switch_to_deploy(self):
|
| 1196 |
+
'''
|
| 1197 |
+
A method to perform model self-compression
|
| 1198 |
+
merges BN into conv layers
|
| 1199 |
+
converts MLP relative positional bias into precomputed buffers
|
| 1200 |
+
'''
|
| 1201 |
+
if not self.switched_to_deploy:
|
| 1202 |
+
for level in [self.patch_embed, self.levels, self.head]:
|
| 1203 |
+
for module in level.modules():
|
| 1204 |
+
if hasattr(module, 'switch_to_deploy'):
|
| 1205 |
+
module.switch_to_deploy()
|
| 1206 |
+
self.switched_to_deploy = True
|
| 1207 |
+
|
| 1208 |
+
|
| 1209 |
+
def change_window_size(self, new_window_size):
|
| 1210 |
+
"""
|
| 1211 |
+
E-RADIO employs windowed attention, which may be sensitive to the choice of this parameter,
|
| 1212 |
+
especially in cases of uneven partitioning of the feature maps.
|
| 1213 |
+
E-RADIO allows for the adjustment of the window size after training,
|
| 1214 |
+
making it adaptable to different input image resolutions.
|
| 1215 |
+
The recommended values for window size based on input resolution are as follows:
|
| 1216 |
+
|
| 1217 |
+
Input Resolution | Window Size
|
| 1218 |
+
224 | 7
|
| 1219 |
+
256 | 8
|
| 1220 |
+
386 | 12
|
| 1221 |
+
512 | 16
|
| 1222 |
+
Ideally, the window size should be a factor of the input resolution. In the third stage, we divide the resolution by 16, so the window size should be
|
| 1223 |
+
img_res/16/2
|
| 1224 |
+
for the third stage and img_res/32 for the last stage. While this can be applied in a brute-force manner, a better way is to do model.change_window_size.
|
| 1225 |
+
Manual way to change resolution -> model.change_window_size(resolution)
|
| 1226 |
+
"""
|
| 1227 |
+
window_size = new_window_size
|
| 1228 |
+
print(f"Setting window size to {window_size}")
|
| 1229 |
+
for module in self.modules():
|
| 1230 |
+
if hasattr(module, "window_size"):
|
| 1231 |
+
# check if tuple or a number
|
| 1232 |
+
if isinstance(module.window_size, tuple):
|
| 1233 |
+
if module.window_size[0] != window_size:
|
| 1234 |
+
module.window_size = (window_size, window_size)
|
| 1235 |
+
elif isinstance(module.window_size, list):
|
| 1236 |
+
if module.window_size[0] != window_size:
|
| 1237 |
+
module.window_size = [window_size, window_size]
|
| 1238 |
+
else:
|
| 1239 |
+
module.window_size = window_size
|
| 1240 |
+
|
| 1241 |
+
|
| 1242 |
+
def set_optimal_window_size(self, image_dim, max_window_size = 16):
|
| 1243 |
+
"""
|
| 1244 |
+
Using hand picked window size for various resolutions.
|
| 1245 |
+
|
| 1246 |
+
E-RADIO employs windowed attention, which may be sensitive to the choice of this parameter,
|
| 1247 |
+
especially in cases of uneven partitioning of the feature maps.
|
| 1248 |
+
E-RADIO allows for the adjustment of the window size after training,
|
| 1249 |
+
making it adaptable to different input image resolutions.
|
| 1250 |
+
The recommended values for window size based on input resolution are as follows:
|
| 1251 |
+
|
| 1252 |
+
Input Resolution | Window Size
|
| 1253 |
+
224 | 7
|
| 1254 |
+
256 | 8
|
| 1255 |
+
386 | 12
|
| 1256 |
+
512 | 16
|
| 1257 |
+
Ideally, the window size should be a factor of the input resolution. In the third stage, we divide the resolution by 16, so the window size should be
|
| 1258 |
+
img_res/16/2
|
| 1259 |
+
for the third stage and img_res/32 for the last stage. While this can be applied in a brute-force manner, a better way is to do model.change_window_size.
|
| 1260 |
+
Manual way to change resolution -> model.change_window_size(resolution)
|
| 1261 |
+
|
| 1262 |
+
"""
|
| 1263 |
+
# import math
|
| 1264 |
+
|
| 1265 |
+
def divisorGenerator(n):
|
| 1266 |
+
large_divisors = []
|
| 1267 |
+
for i in range(1, int(math.sqrt(n) + 1)):
|
| 1268 |
+
if n % i == 0:
|
| 1269 |
+
yield i
|
| 1270 |
+
if i*i != n:
|
| 1271 |
+
large_divisors.append(n / i)
|
| 1272 |
+
for divisor in reversed(large_divisors):
|
| 1273 |
+
yield divisor
|
| 1274 |
+
|
| 1275 |
+
if isinstance(image_dim, list) or isinstance(image_dim, tuple):
|
| 1276 |
+
image_dim = min(image_dim)
|
| 1277 |
+
|
| 1278 |
+
# we do windowed attention in the 3rd stage for the first time, therefore //16,
|
| 1279 |
+
# we do subsampled attention with downsample by 2 so need to get //32 actually
|
| 1280 |
+
# ideally we should rewrite this to be dependent on the structure of the model like what if subsampled is removed etc
|
| 1281 |
+
all_divisors = np.array(list(divisorGenerator(image_dim//32)))
|
| 1282 |
+
new_window_size = int(min(all_divisors[all_divisors <= max_window_size][-1], max_window_size))
|
| 1283 |
+
|
| 1284 |
+
# for image_dim in [128, 224, 256, 384, 512, 768, 1024]:
|
| 1285 |
+
# all_divisors = np.array(list(divisorGenerator(image_dim//32)))
|
| 1286 |
+
# new_window_size = int(min(all_divisors[all_divisors <= max_window_size][-1], max_window_size))
|
| 1287 |
+
# print(f"Setting window size to {new_window_size} for image resolution {image_dim}")
|
| 1288 |
+
|
| 1289 |
+
self.change_window_size(new_window_size = new_window_size)
|
| 1290 |
+
|
| 1291 |
+
|
| 1292 |
+
@register_model
|
| 1293 |
+
def eradio_large_fullres_ws16(pretrained=False, **kwargs):
|
| 1294 |
+
model = ERADIO(
|
| 1295 |
+
depths=[3, 3, 5, 5],
|
| 1296 |
+
num_heads=[2, 4, 8, 16],
|
| 1297 |
+
window_size=[None, None, [16, 16], 16],
|
| 1298 |
+
dim=192,
|
| 1299 |
+
in_dim=64,
|
| 1300 |
+
mlp_ratio=4,
|
| 1301 |
+
drop_path_rate=0.0,
|
| 1302 |
+
sr_ratio=[1, 1, [2, 1], 1],
|
| 1303 |
+
use_swiglu=False,
|
| 1304 |
+
yolo_arch=True,
|
| 1305 |
+
shuffle_down=False,
|
| 1306 |
+
conv_base=True,
|
| 1307 |
+
use_neck=True,
|
| 1308 |
+
full_features_head_dim=1536,
|
| 1309 |
+
neck_start_stage=2,
|
| 1310 |
+
**kwargs,
|
| 1311 |
+
)
|
| 1312 |
+
if pretrained:
|
| 1313 |
+
model.load_state_dict(torch.load(pretrained)["state_dict"])
|
| 1314 |
+
return model
|
| 1315 |
+
|
| 1316 |
+
|
| 1317 |
+
@register_model
|
| 1318 |
+
def eradio_xxxtiny(pretrained=False, **kwargs): # ,
|
| 1319 |
+
model = ERADIO(
|
| 1320 |
+
depths=[1, 3, 4, 5],
|
| 1321 |
+
num_heads=[2, 4, 8, 16],
|
| 1322 |
+
window_size=[None, None, [16, 16], 16],
|
| 1323 |
+
dim=32,
|
| 1324 |
+
in_dim=32,
|
| 1325 |
+
mlp_ratio=4,
|
| 1326 |
+
drop_path_rate=0.0,
|
| 1327 |
+
sr_ratio=[1, 1, [2, 1], 1],
|
| 1328 |
+
use_swiglu=False,
|
| 1329 |
+
yolo_arch=True,
|
| 1330 |
+
shuffle_down=False,
|
| 1331 |
+
conv_base=True,
|
| 1332 |
+
use_neck=True,
|
| 1333 |
+
full_features_head_dim=256,
|
| 1334 |
+
neck_start_stage=2,
|
| 1335 |
+
**kwargs,
|
| 1336 |
+
)
|
| 1337 |
+
if pretrained:
|
| 1338 |
+
model.load_state_dict(torch.load(pretrained))
|
| 1339 |
+
return model
|
| 1340 |
+
|
| 1341 |
+
@register_model
|
| 1342 |
+
def eradio_xxxtiny_8x_ws12(pretrained=False, **kwargs):
|
| 1343 |
+
model = ERADIO(depths=[1, 3, 4, 5],
|
| 1344 |
+
num_heads=[2, 4, 8, 16],
|
| 1345 |
+
window_size=[None, None, [12, 12], 12],
|
| 1346 |
+
dim=32,
|
| 1347 |
+
in_dim=32,
|
| 1348 |
+
mlp_ratio=4,
|
| 1349 |
+
drop_path_rate=0.0,
|
| 1350 |
+
sr_ratio=[1, 1, [2, 1], 1],
|
| 1351 |
+
use_swiglu=False,
|
| 1352 |
+
downsample_shuffle=False,
|
| 1353 |
+
yolo_arch=True,
|
| 1354 |
+
shuffle_down=False,
|
| 1355 |
+
cpb_mlp_hidden=64,
|
| 1356 |
+
use_neck=True,
|
| 1357 |
+
full_features_head_dim=256,
|
| 1358 |
+
neck_start_stage=2,
|
| 1359 |
+
conv_groups_ratio = 1,
|
| 1360 |
+
**kwargs)
|
| 1361 |
+
if pretrained:
|
| 1362 |
+
model.load_state_dict(torch.load(pretrained)["state_dict"])
|
| 1363 |
+
return model
|
| 1364 |
+
|
| 1365 |
+
|
| 1366 |
+
@register_model
|
| 1367 |
+
def eradio_xxxtiny_8x_ws16(pretrained=False, **kwargs):
|
| 1368 |
+
model = ERADIO(depths=[1, 3, 4, 5],
|
| 1369 |
+
num_heads=[2, 4, 8, 16],
|
| 1370 |
+
window_size=[None, None, [16, 16], 16],
|
| 1371 |
+
dim=32,
|
| 1372 |
+
in_dim=32,
|
| 1373 |
+
mlp_ratio=4,
|
| 1374 |
+
drop_path_rate=0.0,
|
| 1375 |
+
sr_ratio=[1, 1, [2, 1], 1],
|
| 1376 |
+
use_swiglu=False,
|
| 1377 |
+
downsample_shuffle=False,
|
| 1378 |
+
yolo_arch=True,
|
| 1379 |
+
shuffle_down=False,
|
| 1380 |
+
cpb_mlp_hidden=64,
|
| 1381 |
+
use_neck=True,
|
| 1382 |
+
full_features_head_dim=256,
|
| 1383 |
+
neck_start_stage=1,
|
| 1384 |
+
conv_groups_ratio = 1,
|
| 1385 |
+
**kwargs)
|
| 1386 |
+
if pretrained:
|
| 1387 |
+
model.load_state_dict(torch.load(pretrained)["state_dict"])
|
| 1388 |
+
return model
|
| 1389 |
+
|
| 1390 |
+
@register_model
|
| 1391 |
+
def eradio(pretrained=False, **kwargs):
|
| 1392 |
+
return eradio_large_fullres_ws16(pretrained=pretrained, **kwargs)
|
radio_extra_timm_models.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2023-2024, NVIDIA CORPORATION. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# NVIDIA CORPORATION and its licensors retain all intellectual property
|
| 4 |
+
# and proprietary rights in and to this software, related documentation
|
| 5 |
+
# and any modifications thereto. Any use, reproduction, disclosure or
|
| 6 |
+
# distribution of this software and related documentation without an express
|
| 7 |
+
# license agreement from NVIDIA CORPORATION is strictly prohibited.
|
| 8 |
+
|
| 9 |
+
from torch import nn
|
| 10 |
+
|
| 11 |
+
from timm.models import register_model
|
| 12 |
+
from timm.models.vision_transformer import VisionTransformer, _create_vision_transformer, Mlp
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
@register_model
|
| 16 |
+
def vit_tiny_patch14_224(pretrained=False, **kwargs) -> VisionTransformer:
|
| 17 |
+
""" ViT-Tiny (Vit-Ti/16)
|
| 18 |
+
"""
|
| 19 |
+
model_args = dict(patch_size=14, embed_dim=192, depth=12, num_heads=3)
|
| 20 |
+
model = _create_vision_transformer('vit_tiny_patch14_224', pretrained=pretrained, **dict(model_args, **kwargs))
|
| 21 |
+
return model
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
@register_model
|
| 25 |
+
def vit_small_patch14_224(pretrained=False, **kwargs) -> VisionTransformer:
|
| 26 |
+
""" ViT-Small (ViT-S/16)
|
| 27 |
+
"""
|
| 28 |
+
model_args = dict(patch_size=14, embed_dim=384, depth=12, num_heads=6)
|
| 29 |
+
model = _create_vision_transformer('vit_small_patch16_224', pretrained=pretrained, **dict(model_args, **kwargs))
|
| 30 |
+
return model
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
@register_model
|
| 34 |
+
def vit_base_patch14_224(pretrained=False, **kwargs) -> VisionTransformer:
|
| 35 |
+
""" ViT-Base (ViT-B/14) from original paper (https://arxiv.org/abs/2010.11929).
|
| 36 |
+
ImageNet-1k weights fine-tuned from in21k @ 224x224, source https://github.com/google-research/vision_transformer.
|
| 37 |
+
"""
|
| 38 |
+
model_args = dict(patch_size=14, embed_dim=768, depth=12, num_heads=12)
|
| 39 |
+
model = _create_vision_transformer('vit_base_patch14_224', pretrained=pretrained, **dict(model_args, **kwargs))
|
| 40 |
+
return model
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
@register_model
|
| 44 |
+
def vit_huge_patch16_224(pretrained=False, **kwargs) -> VisionTransformer:
|
| 45 |
+
""" ViT-Huge model (ViT-H/16) from original paper (https://arxiv.org/abs/2010.11929).
|
| 46 |
+
"""
|
| 47 |
+
model_args = dict(patch_size=16, embed_dim=1280, depth=32, num_heads=16)
|
| 48 |
+
if pretrained:
|
| 49 |
+
# There is no pretrained version of ViT-H/16, but we can adapt a ViT-H/14 for this purpose
|
| 50 |
+
model = _create_vision_transformer('vit_huge_patch14_clip_336', pretrained=True, **dict(model_args, pre_norm=True, **kwargs))
|
| 51 |
+
else:
|
| 52 |
+
model = _create_vision_transformer('vit_huge_patch16_224', pretrained=False, **dict(model_args, **kwargs))
|
| 53 |
+
return model
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
@register_model
|
| 57 |
+
def vit_huge_patch16_224_mlpnorm(pretrained=False, **kwargs) -> VisionTransformer:
|
| 58 |
+
""" ViT-Huge model (ViT-H/16) from original paper (https://arxiv.org/abs/2010.11929).
|
| 59 |
+
"""
|
| 60 |
+
model = vit_huge_patch16_224(pretrained=pretrained, **kwargs)
|
| 61 |
+
|
| 62 |
+
for m in model.modules():
|
| 63 |
+
if isinstance(m, Mlp) and not isinstance(m.norm, nn.LayerNorm):
|
| 64 |
+
m.norm = nn.LayerNorm(m.fc1.out_features)
|
| 65 |
+
|
| 66 |
+
return model
|
radio_input_conditioner.py
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2023-2024, NVIDIA CORPORATION. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# NVIDIA CORPORATION and its licensors retain all intellectual property
|
| 4 |
+
# and proprietary rights in and to this software, related documentation
|
| 5 |
+
# and any modifications thereto. Any use, reproduction, disclosure or
|
| 6 |
+
# distribution of this software and related documentation without an express
|
| 7 |
+
# license agreement from NVIDIA CORPORATION is strictly prohibited.
|
| 8 |
+
|
| 9 |
+
from typing import Union, Tuple
|
| 10 |
+
|
| 11 |
+
import torch
|
| 12 |
+
from torch import nn
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
norm_t = Union[Tuple[float, float, float], torch.Tensor]
|
| 16 |
+
|
| 17 |
+
class InputConditioner(nn.Module):
|
| 18 |
+
def __init__(self,
|
| 19 |
+
input_scale: float,
|
| 20 |
+
norm_mean: norm_t,
|
| 21 |
+
norm_std: norm_t,
|
| 22 |
+
dtype: torch.dtype = None,
|
| 23 |
+
):
|
| 24 |
+
super().__init__()
|
| 25 |
+
|
| 26 |
+
self.dtype = dtype
|
| 27 |
+
|
| 28 |
+
self.register_buffer("norm_mean", _to_tensor(norm_mean) / input_scale)
|
| 29 |
+
self.register_buffer("norm_std", _to_tensor(norm_std) / input_scale)
|
| 30 |
+
|
| 31 |
+
def forward(self, x: torch.Tensor):
|
| 32 |
+
y = (x - self.norm_mean) / self.norm_std
|
| 33 |
+
if self.dtype is not None:
|
| 34 |
+
y = y.to(self.dtype)
|
| 35 |
+
return y
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def get_default_conditioner():
|
| 39 |
+
from timm.data.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD
|
| 40 |
+
|
| 41 |
+
return InputConditioner(
|
| 42 |
+
input_scale=1.0,
|
| 43 |
+
norm_mean=OPENAI_CLIP_MEAN,
|
| 44 |
+
norm_std=OPENAI_CLIP_STD,
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def _to_tensor(v: norm_t):
|
| 49 |
+
return torch.as_tensor(v, dtype=torch.float32).view(-1, 1, 1)
|
radio_model.py
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2023-2024, NVIDIA CORPORATION. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# NVIDIA CORPORATION and its licensors retain all intellectual property
|
| 4 |
+
# and proprietary rights in and to this software, related documentation
|
| 5 |
+
# and any modifications thereto. Any use, reproduction, disclosure or
|
| 6 |
+
# distribution of this software and related documentation without an express
|
| 7 |
+
# license agreement from NVIDIA CORPORATION is strictly prohibited.
|
| 8 |
+
from typing import Optional, Callable, Union, Tuple, Any, Dict, NamedTuple
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
from torch import nn
|
| 12 |
+
|
| 13 |
+
from timm.models import create_model, VisionTransformer
|
| 14 |
+
|
| 15 |
+
from .radio_enable_cpe_support import enable_cpe
|
| 16 |
+
from .radio_input_conditioner import InputConditioner
|
| 17 |
+
# Register extra models
|
| 18 |
+
from . import radio_extra_timm_models
|
| 19 |
+
from .radio_adaptor_base import AdaptorBase, RadioOutput, AdaptorInput
|
| 20 |
+
from . import radio_eradio_model
|
| 21 |
+
from .radio_enable_spectral_reparam import configure_spectral_reparam_from_args
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
class Resolution(NamedTuple):
|
| 25 |
+
height: int
|
| 26 |
+
width: int
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
class RADIOModel(nn.Module):
|
| 30 |
+
def __init__(
|
| 31 |
+
self,
|
| 32 |
+
model: nn.Module,
|
| 33 |
+
input_conditioner: InputConditioner,
|
| 34 |
+
patch_size: int,
|
| 35 |
+
max_resolution: int,
|
| 36 |
+
preferred_resolution: Resolution,
|
| 37 |
+
summary_idxs: Optional[torch.Tensor] = None,
|
| 38 |
+
window_size: int = None,
|
| 39 |
+
adaptors: Dict[str, AdaptorBase] = None,
|
| 40 |
+
):
|
| 41 |
+
super().__init__()
|
| 42 |
+
|
| 43 |
+
self.model = model
|
| 44 |
+
self.input_conditioner = input_conditioner
|
| 45 |
+
if summary_idxs is not None:
|
| 46 |
+
self.register_buffer('summary_idxs', summary_idxs)
|
| 47 |
+
else:
|
| 48 |
+
self.summary_idxs = None
|
| 49 |
+
|
| 50 |
+
self._preferred_resolution = preferred_resolution
|
| 51 |
+
self._patch_size = patch_size
|
| 52 |
+
self._max_resolution = max_resolution
|
| 53 |
+
self._window_size = window_size
|
| 54 |
+
|
| 55 |
+
adaptors = adaptors or dict()
|
| 56 |
+
self.adaptors = nn.ModuleDict(adaptors)
|
| 57 |
+
|
| 58 |
+
@property
|
| 59 |
+
def num_summary_tokens(self) -> int:
|
| 60 |
+
patch_gen = getattr(self.model, "patch_generator", None)
|
| 61 |
+
if patch_gen is not None:
|
| 62 |
+
return patch_gen.num_skip
|
| 63 |
+
elif self.model.global_pool == 'avg':
|
| 64 |
+
return 0
|
| 65 |
+
return 1
|
| 66 |
+
|
| 67 |
+
@property
|
| 68 |
+
def patch_size(self) -> int:
|
| 69 |
+
return self._patch_size
|
| 70 |
+
|
| 71 |
+
@property
|
| 72 |
+
def max_resolution(self) -> int:
|
| 73 |
+
return self._max_resolution
|
| 74 |
+
|
| 75 |
+
@property
|
| 76 |
+
def preferred_resolution(self) -> Resolution:
|
| 77 |
+
return self._preferred_resolution
|
| 78 |
+
|
| 79 |
+
@property
|
| 80 |
+
def window_size(self) -> int:
|
| 81 |
+
return self._window_size
|
| 82 |
+
|
| 83 |
+
@property
|
| 84 |
+
def min_resolution_step(self) -> int:
|
| 85 |
+
res = self.patch_size
|
| 86 |
+
if self.window_size is not None:
|
| 87 |
+
res *= self.window_size
|
| 88 |
+
return res
|
| 89 |
+
|
| 90 |
+
def make_preprocessor_external(self) -> Callable[[torch.Tensor], torch.Tensor]:
|
| 91 |
+
ret = self.input_conditioner
|
| 92 |
+
self.input_conditioner = nn.Identity()
|
| 93 |
+
return ret
|
| 94 |
+
|
| 95 |
+
def get_nearest_supported_resolution(self, height: int, width: int) -> Resolution:
|
| 96 |
+
height = int(round(height / self.min_resolution_step) * self.min_resolution_step)
|
| 97 |
+
width = int(round(width / self.min_resolution_step) * self.min_resolution_step)
|
| 98 |
+
|
| 99 |
+
height = max(height, self.min_resolution_step)
|
| 100 |
+
width = max(width, self.min_resolution_step)
|
| 101 |
+
|
| 102 |
+
return Resolution(height=height, width=width)
|
| 103 |
+
|
| 104 |
+
def switch_to_deploy(self):
|
| 105 |
+
fn = getattr(self.model, 'switch_to_deploy', None)
|
| 106 |
+
if fn is not None:
|
| 107 |
+
fn()
|
| 108 |
+
|
| 109 |
+
def forward(self, x: torch.Tensor) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
|
| 110 |
+
x = self.input_conditioner(x)
|
| 111 |
+
y = self.model.forward_features(x)
|
| 112 |
+
|
| 113 |
+
if isinstance(self.model, VisionTransformer):
|
| 114 |
+
patch_gen = getattr(self.model, "patch_generator", None)
|
| 115 |
+
if patch_gen is not None:
|
| 116 |
+
all_summary = y[:, : patch_gen.num_cls_tokens]
|
| 117 |
+
if self.summary_idxs is not None:
|
| 118 |
+
bb_summary = all_summary[:, self.summary_idxs]
|
| 119 |
+
else:
|
| 120 |
+
bb_summary = all_summary
|
| 121 |
+
all_feat = y[:, patch_gen.num_skip :]
|
| 122 |
+
elif self.model.global_pool == "avg":
|
| 123 |
+
all_summary = y[:, self.model.num_prefix_tokens :].mean(dim=1)
|
| 124 |
+
bb_summary = all_summary
|
| 125 |
+
all_feat = y
|
| 126 |
+
else:
|
| 127 |
+
all_summary = y[:, 0]
|
| 128 |
+
bb_summary = all_summary
|
| 129 |
+
all_feat = y[:, 1:]
|
| 130 |
+
elif isinstance(self.model, radio_eradio_model.ERADIO):
|
| 131 |
+
_, f = y
|
| 132 |
+
all_feat = f.flatten(2).transpose(1, 2)
|
| 133 |
+
all_summary = all_feat.mean(dim=1)
|
| 134 |
+
bb_summary = all_summary
|
| 135 |
+
elif isinstance(y, (list, tuple)):
|
| 136 |
+
all_summary, all_feat = y
|
| 137 |
+
bb_summary = all_summary
|
| 138 |
+
else:
|
| 139 |
+
raise ValueError("Unsupported model type")
|
| 140 |
+
|
| 141 |
+
all_feat = all_feat.float()
|
| 142 |
+
ret = RadioOutput(bb_summary.flatten(1), all_feat).to(torch.float32)
|
| 143 |
+
if self.adaptors:
|
| 144 |
+
ret = dict(backbone=ret)
|
| 145 |
+
for name, adaptor in self.adaptors.items():
|
| 146 |
+
if all_summary.ndim == 3:
|
| 147 |
+
summary = all_summary[:, adaptor.head_idx]
|
| 148 |
+
else:
|
| 149 |
+
summary = all_summary
|
| 150 |
+
ada_input = AdaptorInput(images=x, summary=summary.float(), features=all_feat)
|
| 151 |
+
v = adaptor(ada_input).to(torch.float32)
|
| 152 |
+
ret[name] = v
|
| 153 |
+
|
| 154 |
+
return ret
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
def create_model_from_args(args) -> nn.Module:
|
| 158 |
+
in_chans = 3
|
| 159 |
+
if args.in_chans is not None:
|
| 160 |
+
in_chans = args.in_chans
|
| 161 |
+
elif args.input_size is not None:
|
| 162 |
+
in_chans = args.input_size[0]
|
| 163 |
+
|
| 164 |
+
# Skip weight initialization unless it's explicitly requested.
|
| 165 |
+
weight_init = args.model_kwargs.pop("weight_init", "skip")
|
| 166 |
+
|
| 167 |
+
model = create_model(
|
| 168 |
+
args.model,
|
| 169 |
+
pretrained=args.pretrained,
|
| 170 |
+
in_chans=in_chans,
|
| 171 |
+
num_classes=args.num_classes,
|
| 172 |
+
drop_rate=args.drop,
|
| 173 |
+
drop_path_rate=args.drop_path,
|
| 174 |
+
drop_block_rate=args.drop_block,
|
| 175 |
+
global_pool=args.gp,
|
| 176 |
+
bn_momentum=args.bn_momentum,
|
| 177 |
+
bn_eps=args.bn_eps,
|
| 178 |
+
scriptable=args.torchscript,
|
| 179 |
+
checkpoint_path=args.initial_checkpoint,
|
| 180 |
+
weight_init=weight_init,
|
| 181 |
+
**args.model_kwargs,
|
| 182 |
+
)
|
| 183 |
+
|
| 184 |
+
if hasattr(model, 'norm') and not getattr(args, 'model_norm', False):
|
| 185 |
+
model.norm = nn.Identity()
|
| 186 |
+
|
| 187 |
+
model.head = nn.Identity()
|
| 188 |
+
|
| 189 |
+
assert (
|
| 190 |
+
not args.cls_token_per_teacher or args.cpe_max_size is not None
|
| 191 |
+
), "CPE must be enabled for multiple CLS tokens!"
|
| 192 |
+
|
| 193 |
+
if args.cpe_max_size is not None:
|
| 194 |
+
enable_cpe(
|
| 195 |
+
model,
|
| 196 |
+
args.cpe_max_size,
|
| 197 |
+
num_cls_tokens=len(args.teachers) if args.cls_token_per_teacher else 1,
|
| 198 |
+
register_multiple=args.register_multiple,
|
| 199 |
+
)
|
| 200 |
+
|
| 201 |
+
if args.spectral_reparam:
|
| 202 |
+
configure_spectral_reparam_from_args(model, args)
|
| 203 |
+
|
| 204 |
+
return model
|
radio_open_clip_adaptor.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# NVIDIA CORPORATION and its licensors retain all intellectual property
|
| 4 |
+
# and proprietary rights in and to this software, related documentation
|
| 5 |
+
# and any modifications thereto. Any use, reproduction, disclosure or
|
| 6 |
+
# distribution of this software and related documentation without an express
|
| 7 |
+
# license agreement from NVIDIA CORPORATION is strictly prohibited.
|
| 8 |
+
from argparse import Namespace
|
| 9 |
+
|
| 10 |
+
import torch
|
| 11 |
+
from torch import nn
|
| 12 |
+
import torch.nn.functional as F
|
| 13 |
+
|
| 14 |
+
from .radio_adaptor_registry import adaptor_registry, dict_t, state_t
|
| 15 |
+
|
| 16 |
+
from .radio_adaptor_generic import GenericAdaptor
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
class OpenCLIP_RADIO(GenericAdaptor):
|
| 20 |
+
def __init__(self, main_config: Namespace, adaptor_config: dict_t, state: state_t):
|
| 21 |
+
super().__init__(main_config, adaptor_config, state)
|
| 22 |
+
|
| 23 |
+
import open_clip
|
| 24 |
+
|
| 25 |
+
self.oc_model = open_clip.create_model_from_pretrained(
|
| 26 |
+
model_name=adaptor_config['model'],
|
| 27 |
+
pretrained=adaptor_config['pretrained'],
|
| 28 |
+
return_transform=False,
|
| 29 |
+
)
|
| 30 |
+
# Unload these parameters
|
| 31 |
+
self.oc_model.visual = None
|
| 32 |
+
|
| 33 |
+
self.tokenizer = open_clip.get_tokenizer(model_name=adaptor_config['model'])
|
| 34 |
+
|
| 35 |
+
def encode_text(self, text, normalize: bool = False):
|
| 36 |
+
return self.oc_model.encode_text(text, normalize=normalize)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
@adaptor_registry.register_adaptor("open_clip")
|
| 40 |
+
def create_open_clip_adaptor(main_config: Namespace, adaptor_config: dict_t, state: state_t):
|
| 41 |
+
return OpenCLIP_RADIO(main_config, adaptor_config, state)
|
radio_vit_patch_generator.py
ADDED
|
@@ -0,0 +1,299 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (c) 2023-2024, NVIDIA CORPORATION. All rights reserved.
|
| 2 |
+
#
|
| 3 |
+
# NVIDIA CORPORATION and its licensors retain all intellectual property
|
| 4 |
+
# and proprietary rights in and to this software, related documentation
|
| 5 |
+
# and any modifications thereto. Any use, reproduction, disclosure or
|
| 6 |
+
# distribution of this software and related documentation without an express
|
| 7 |
+
# license agreement from NVIDIA CORPORATION is strictly prohibited.
|
| 8 |
+
|
| 9 |
+
import math
|
| 10 |
+
from typing import Union, Tuple, Optional
|
| 11 |
+
|
| 12 |
+
import torch
|
| 13 |
+
import torch.nn.functional as F
|
| 14 |
+
from torch import nn
|
| 15 |
+
from einops import rearrange
|
| 16 |
+
|
| 17 |
+
from .radio_cls_token import ClsToken
|
| 18 |
+
|
| 19 |
+
input_dim_t = Union[int, Tuple[int, int]]
|
| 20 |
+
|
| 21 |
+
try:
|
| 22 |
+
# raise ImportError()
|
| 23 |
+
from indirect_grid_sample import indirect_grid_sample
|
| 24 |
+
except ImportError:
|
| 25 |
+
indirect_grid_sample = None
|
| 26 |
+
|
| 27 |
+
class ViTPatchGenerator(nn.Module):
|
| 28 |
+
def __init__(self,
|
| 29 |
+
patch_size: int,
|
| 30 |
+
embed_dim: int,
|
| 31 |
+
input_dims: input_dim_t,
|
| 32 |
+
abs_pos: bool = True,
|
| 33 |
+
normalize_patches: bool = False,
|
| 34 |
+
cls_token: bool = False,
|
| 35 |
+
max_input_dims: Optional[input_dim_t] = None,
|
| 36 |
+
pos_dropout: float = 0.0,
|
| 37 |
+
return_pos_enc: bool = False,
|
| 38 |
+
num_cls_tokens: int = 1,
|
| 39 |
+
register_multiple: int = 0,
|
| 40 |
+
device=None, dtype=None,
|
| 41 |
+
):
|
| 42 |
+
super().__init__()
|
| 43 |
+
|
| 44 |
+
if isinstance(input_dims, int):
|
| 45 |
+
input_dims = (input_dims, input_dims)
|
| 46 |
+
|
| 47 |
+
if max_input_dims is None:
|
| 48 |
+
max_input_dims = input_dims
|
| 49 |
+
if isinstance(max_input_dims, int):
|
| 50 |
+
max_input_dims = (max_input_dims, max_input_dims)
|
| 51 |
+
|
| 52 |
+
max_input_dims = tuple(
|
| 53 |
+
int(math.ceil(d / patch_size) * patch_size)
|
| 54 |
+
for d in max_input_dims
|
| 55 |
+
)
|
| 56 |
+
|
| 57 |
+
self.cpe_mode = max_input_dims != input_dims
|
| 58 |
+
self.pos_dropout = pos_dropout
|
| 59 |
+
self.return_pos_enc = return_pos_enc
|
| 60 |
+
|
| 61 |
+
factory = dict(device=device, dtype=dtype)
|
| 62 |
+
|
| 63 |
+
self.patch_size = patch_size
|
| 64 |
+
self.abs_pos = abs_pos
|
| 65 |
+
self.embed_dim = embed_dim
|
| 66 |
+
|
| 67 |
+
self.num_rows = max_input_dims[0] // patch_size
|
| 68 |
+
self.num_cols = max_input_dims[1] // patch_size
|
| 69 |
+
self.input_dims = tuple(d // patch_size for d in input_dims)
|
| 70 |
+
self.num_patches = self.num_rows * self.num_cols
|
| 71 |
+
self.max_input_dims = max_input_dims
|
| 72 |
+
|
| 73 |
+
self.im_to_patches = Im2Patches(patch_size)
|
| 74 |
+
self.embedder = ViTPatchLinear(patch_size, embed_dim, **factory)
|
| 75 |
+
|
| 76 |
+
if abs_pos:
|
| 77 |
+
scale = embed_dim ** -0.5
|
| 78 |
+
self.pos_embed = nn.Parameter(torch.randn(1, self.num_patches, embed_dim, **factory) * scale)
|
| 79 |
+
|
| 80 |
+
self.cls_token = ClsToken(
|
| 81 |
+
embed_dim,
|
| 82 |
+
num_tokens=num_cls_tokens,
|
| 83 |
+
enabled=cls_token,
|
| 84 |
+
register_multiple=register_multiple,
|
| 85 |
+
)
|
| 86 |
+
|
| 87 |
+
self.patch_normalizer = nn.LayerNorm(embed_dim) if normalize_patches else nn.Identity()
|
| 88 |
+
|
| 89 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 90 |
+
patches = self.embed_patches(x)
|
| 91 |
+
patches, pos_enc = self.apply_pos_enc(patches, input_size=x.shape[2:])
|
| 92 |
+
patches = self.cls_token(patches)
|
| 93 |
+
patches = self.patch_normalizer(patches)
|
| 94 |
+
if self.return_pos_enc:
|
| 95 |
+
return patches, pos_enc
|
| 96 |
+
return patches
|
| 97 |
+
|
| 98 |
+
@property
|
| 99 |
+
def apply_cls_token(self):
|
| 100 |
+
return self.cls_token.enabled
|
| 101 |
+
|
| 102 |
+
@property
|
| 103 |
+
def num_cls_tokens(self):
|
| 104 |
+
return self.cls_token.num_tokens
|
| 105 |
+
|
| 106 |
+
@property
|
| 107 |
+
def num_registers(self):
|
| 108 |
+
return self.cls_token.num_registers
|
| 109 |
+
|
| 110 |
+
@property
|
| 111 |
+
def num_skip(self):
|
| 112 |
+
return self.num_cls_tokens + self.num_registers
|
| 113 |
+
|
| 114 |
+
def no_weight_decay(self):
|
| 115 |
+
return [
|
| 116 |
+
'pos_embed',
|
| 117 |
+
]
|
| 118 |
+
|
| 119 |
+
def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs):
|
| 120 |
+
if self.abs_pos:
|
| 121 |
+
self._load_embed(state_dict[f'{prefix}pos_embed'], self.pos_embed)
|
| 122 |
+
|
| 123 |
+
def _load_embed(self, src_embed: torch.Tensor, targ_embed: nn.Parameter):
|
| 124 |
+
if src_embed.shape != targ_embed.shape:
|
| 125 |
+
src_size = int(math.sqrt(src_embed.shape[1]))
|
| 126 |
+
|
| 127 |
+
assert src_size ** 2 == src_embed.shape[1], 'Unable to interpolate non-square embedding'
|
| 128 |
+
|
| 129 |
+
src_embed = rearrange(src_embed, 'b (h w) c -> b c h w', h=src_size, w=src_size)
|
| 130 |
+
src_embed = F.interpolate(src_embed, size=(self.num_rows, self.num_cols), mode='bicubic', align_corners=True, antialias=False)
|
| 131 |
+
src_embed = rearrange(src_embed, 'b c h w -> b (h w) c')
|
| 132 |
+
targ_embed.data.copy_(src_embed)
|
| 133 |
+
|
| 134 |
+
def _load_projection(self, src_proj_weight: torch.Tensor, targ_proj_weight: torch.Tensor):
|
| 135 |
+
if src_proj_weight.shape != targ_proj_weight.shape:
|
| 136 |
+
src_patch_size = int(math.sqrt(src_proj_weight.shape[1] // 3))
|
| 137 |
+
|
| 138 |
+
assert (src_patch_size ** 2) * 3 == src_proj_weight.shape[1], 'Unable to interpolate non-square patch size'
|
| 139 |
+
|
| 140 |
+
src_proj_weight = rearrange(src_proj_weight, 'b (c h w) -> b c h w', c=3, h=src_patch_size, w=src_patch_size)
|
| 141 |
+
src_proj_weight = F.interpolate(src_proj_weight, size=(self.patch_size, self.patch_size), mode='bicubic', align_corners=True, antialias=False)
|
| 142 |
+
src_proj_weight = rearrange(src_proj_weight, 'b c h w -> b (c h w)')
|
| 143 |
+
targ_proj_weight.data.copy_(src_proj_weight)
|
| 144 |
+
|
| 145 |
+
def embed_patches(self, x: torch.Tensor) -> torch.Tensor:
|
| 146 |
+
patches = self.im_to_patches(x)
|
| 147 |
+
patches = self.embedder(patches)
|
| 148 |
+
return patches
|
| 149 |
+
|
| 150 |
+
def apply_pos_enc(self,
|
| 151 |
+
patches: torch.Tensor,
|
| 152 |
+
patch_idxs: Optional[torch.Tensor] = None,
|
| 153 |
+
input_size: Optional[Tuple[int, int]] = None,
|
| 154 |
+
) -> torch.Tensor:
|
| 155 |
+
if not self.abs_pos:
|
| 156 |
+
return patches
|
| 157 |
+
|
| 158 |
+
pos_enc = self.get_pos_enc(patches.shape[0], patch_idxs, input_size)
|
| 159 |
+
|
| 160 |
+
if self.training and self.pos_dropout > 0:
|
| 161 |
+
keeps = torch.rand(patches.shape[0], 1, 1, dtype=pos_enc.dtype, device=pos_enc.device) > self.pos_dropout
|
| 162 |
+
pos_enc_drop = torch.where(keeps, pos_enc, 0)
|
| 163 |
+
else:
|
| 164 |
+
pos_enc_drop = pos_enc
|
| 165 |
+
|
| 166 |
+
return patches + pos_enc_drop, pos_enc
|
| 167 |
+
|
| 168 |
+
def get_pos_enc(self,
|
| 169 |
+
batch_size: int,
|
| 170 |
+
patch_idxs: Optional[torch.Tensor] = None,
|
| 171 |
+
input_size: Optional[Tuple[int, int]] = None,
|
| 172 |
+
) -> torch.Tensor:
|
| 173 |
+
if input_size is None:
|
| 174 |
+
input_dims = self.input_dims
|
| 175 |
+
else:
|
| 176 |
+
input_dims = tuple(d // self.patch_size for d in input_size)
|
| 177 |
+
|
| 178 |
+
pos_embed = self._get_pos_embeddings(batch_size, input_dims)
|
| 179 |
+
|
| 180 |
+
if patch_idxs is None:
|
| 181 |
+
return pos_embed
|
| 182 |
+
|
| 183 |
+
exp_patch_idxs = patch_idxs.unsqueeze(-1).expand(-1, -1, pos_embed.shape[-1])
|
| 184 |
+
|
| 185 |
+
pos_embed = torch.gather(pos_embed.expand(patch_idxs.shape[0], -1, -1), dim=1, index=exp_patch_idxs)
|
| 186 |
+
return pos_embed
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def _get_pos_embeddings(self, batch_size: int, input_dims: Tuple[int, int]):
|
| 190 |
+
if (self.num_rows, self.num_cols) == input_dims:
|
| 191 |
+
return self.pos_embed
|
| 192 |
+
|
| 193 |
+
pos_embed = self.pos_embed.reshape(1, self.num_rows, self.num_cols, -1).permute(0, 3, 1, 2)
|
| 194 |
+
|
| 195 |
+
def window_select(pos_embed):
|
| 196 |
+
if input_dims[0] < pos_embed.shape[-2]:
|
| 197 |
+
pos_embed = pos_embed[..., :input_dims[0], :]
|
| 198 |
+
if input_dims[1] < pos_embed.shape[-1]:
|
| 199 |
+
pos_embed = pos_embed[..., :, :input_dims[1]]
|
| 200 |
+
return pos_embed
|
| 201 |
+
|
| 202 |
+
if self.cpe_mode:
|
| 203 |
+
if self.training:
|
| 204 |
+
min_scale = math.sqrt(0.1)
|
| 205 |
+
scale = torch.rand(batch_size, 1, 1, device=pos_embed.device) * (1 - min_scale) + min_scale
|
| 206 |
+
aspect_min = math.log(3 / 4)
|
| 207 |
+
aspect_max = -aspect_min
|
| 208 |
+
aspect = torch.exp(torch.rand(batch_size, 1, 1, device=pos_embed.device) * (aspect_max - aspect_min) + aspect_min)
|
| 209 |
+
|
| 210 |
+
scale_x = scale * aspect
|
| 211 |
+
scale_y = scale * (1 / aspect)
|
| 212 |
+
scale_xy = torch.stack([scale_x, scale_y], dim=-1).clamp_(0, 1)
|
| 213 |
+
|
| 214 |
+
pos_xy = torch.rand(batch_size, 1, 1, 2, device=pos_embed.device) * (1 - scale_xy)
|
| 215 |
+
|
| 216 |
+
lin_x = torch.linspace(0, 1, steps=input_dims[1], device=pos_embed.device)[None, None].expand(batch_size, input_dims[0], -1)
|
| 217 |
+
lin_y = torch.linspace(0, 1, steps=input_dims[0], device=pos_embed.device)[None, :, None].expand(batch_size, -1, input_dims[1])
|
| 218 |
+
|
| 219 |
+
lin_xy = torch.stack([lin_x, lin_y], dim=-1)
|
| 220 |
+
|
| 221 |
+
grid_xy = lin_xy * scale_xy + pos_xy
|
| 222 |
+
|
| 223 |
+
# Convert to [-1, 1] range
|
| 224 |
+
grid_xy.mul_(2).sub_(1)
|
| 225 |
+
|
| 226 |
+
pos_embed = F.grid_sample(
|
| 227 |
+
pos_embed.float().expand(batch_size, -1, -1, -1),
|
| 228 |
+
grid=grid_xy,
|
| 229 |
+
mode='bilinear',
|
| 230 |
+
padding_mode='zeros',
|
| 231 |
+
align_corners=True,
|
| 232 |
+
).to(pos_embed.dtype)
|
| 233 |
+
else:
|
| 234 |
+
# i_rows, i_cols = input_dims
|
| 235 |
+
# p_rows, p_cols = pos_embed.shape[2:]
|
| 236 |
+
# if i_rows <= p_rows and i_cols <= p_cols:
|
| 237 |
+
# left = (p_cols - i_cols) // 2
|
| 238 |
+
# top = (p_rows - i_rows) // 2
|
| 239 |
+
# pos_embed = pos_embed[..., top:top+i_rows, left:left+i_cols]
|
| 240 |
+
# else:
|
| 241 |
+
max_dim = max(input_dims)
|
| 242 |
+
pos_embed = F.interpolate(pos_embed.float(), size=(max_dim, max_dim), align_corners=True, mode='bilinear').to(pos_embed.dtype)
|
| 243 |
+
|
| 244 |
+
pos_embed = window_select(pos_embed)
|
| 245 |
+
else:
|
| 246 |
+
pos_embed = window_select(pos_embed)
|
| 247 |
+
|
| 248 |
+
if pos_embed.shape[-2:] != input_dims:
|
| 249 |
+
pos_embed = F.interpolate(pos_embed.float(), size=input_dims, align_corners=True, mode='bilinear').to(pos_embed.dtype)
|
| 250 |
+
|
| 251 |
+
pos_embed = pos_embed.flatten(2).permute(0, 2, 1)
|
| 252 |
+
|
| 253 |
+
return pos_embed
|
| 254 |
+
|
| 255 |
+
|
| 256 |
+
class Im2Patches(nn.Module):
|
| 257 |
+
def __init__(self, patch_size: int):
|
| 258 |
+
super().__init__()
|
| 259 |
+
self.patch_size = patch_size
|
| 260 |
+
|
| 261 |
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
| 262 |
+
if self.patch_size == 1:
|
| 263 |
+
patches = x.flatten(2)
|
| 264 |
+
patches = patches.permute(0, 2, 1)
|
| 265 |
+
return patches
|
| 266 |
+
|
| 267 |
+
py = x.shape[-2] // self.patch_size
|
| 268 |
+
px = x.shape[-1] // self.patch_size
|
| 269 |
+
patches = rearrange(x, 'b c (py yy) (px xx) -> b (py px) (c yy xx)',
|
| 270 |
+
py=py, yy=self.patch_size,
|
| 271 |
+
px=px, xx=self.patch_size,
|
| 272 |
+
)
|
| 273 |
+
return patches
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
class ViTPatchLinear(nn.Linear):
|
| 277 |
+
def __init__(self, patch_size: int, embed_dim: int, **factory):
|
| 278 |
+
super().__init__(
|
| 279 |
+
3 * (patch_size ** 2),
|
| 280 |
+
embed_dim,
|
| 281 |
+
bias=False,
|
| 282 |
+
**factory
|
| 283 |
+
)
|
| 284 |
+
self.patch_size = patch_size
|
| 285 |
+
|
| 286 |
+
def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs):
|
| 287 |
+
if self.bias is not None:
|
| 288 |
+
self.bias.data.copy_(state_dict[f'{prefix}bias'])
|
| 289 |
+
|
| 290 |
+
chk_weight = state_dict[f'{prefix}weight']
|
| 291 |
+
if chk_weight.shape != self.weight.shape:
|
| 292 |
+
src_patch_size = int(math.sqrt(chk_weight.shape[1] // 3))
|
| 293 |
+
|
| 294 |
+
assert (src_patch_size ** 2) * 3 == chk_weight.shape[1], 'Unable to interpolate non-square patch size'
|
| 295 |
+
|
| 296 |
+
chk_weight = rearrange(chk_weight, 'b (c h w) -> b c h w', c=3, h=src_patch_size, w=src_patch_size)
|
| 297 |
+
chk_weight = F.interpolate(chk_weight, size=(self.patch_size, self.patch_size), mode='bicubic', align_corners=True, antialias=False)
|
| 298 |
+
chk_weight = rearrange(chk_weight, 'b c h w -> b (c h w)')
|
| 299 |
+
self.weight.data.copy_(chk_weight)
|
radio_vitdet.py
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from collections import defaultdict
|
| 2 |
+
from contextlib import contextmanager
|
| 3 |
+
from logging import getLogger
|
| 4 |
+
import math
|
| 5 |
+
import sys
|
| 6 |
+
from typing import List, Union, Iterable
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
import torch
|
| 10 |
+
from torch import nn
|
| 11 |
+
|
| 12 |
+
from timm.models import VisionTransformer
|
| 13 |
+
from einops import rearrange
|
| 14 |
+
|
| 15 |
+
DEFAULT_NUM_WINDOWED = 5
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class VitDetArgs:
|
| 19 |
+
def __init__(self,
|
| 20 |
+
window_size: int,
|
| 21 |
+
num_summary_tokens: int,
|
| 22 |
+
num_windowed: int = DEFAULT_NUM_WINDOWED,
|
| 23 |
+
):
|
| 24 |
+
self.window_size = window_size
|
| 25 |
+
self.num_summary_tokens = num_summary_tokens
|
| 26 |
+
self.num_windowed = num_windowed
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def apply_vitdet_arch(model: VisionTransformer, args: VitDetArgs):
|
| 30 |
+
if isinstance(model, VisionTransformer):
|
| 31 |
+
patch_embed = getattr(model, 'patch_generator', model.patch_embed)
|
| 32 |
+
|
| 33 |
+
return ViTDetHook(patch_embed, model.blocks, args)
|
| 34 |
+
else:
|
| 35 |
+
print(f'Warning: Unable to apply VitDet aug!', file=sys.stderr)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class ViTDetHook:
|
| 39 |
+
def __init__(self,
|
| 40 |
+
embedder: nn.Module,
|
| 41 |
+
blocks: nn.Sequential,
|
| 42 |
+
args: VitDetArgs,
|
| 43 |
+
):
|
| 44 |
+
self.blocks = blocks
|
| 45 |
+
self.num_summary_tokens = args.num_summary_tokens
|
| 46 |
+
self.window_size = args.window_size
|
| 47 |
+
|
| 48 |
+
self._input_resolution = None
|
| 49 |
+
self._num_windows = None
|
| 50 |
+
self._cls_patch = None
|
| 51 |
+
self._order_cache = dict()
|
| 52 |
+
|
| 53 |
+
embedder.register_forward_pre_hook(self._enter_model)
|
| 54 |
+
|
| 55 |
+
# This will decide if we window-fy the patches
|
| 56 |
+
# and enable vit-det for this iteration, and if so,
|
| 57 |
+
# rearrange the patches for efficient mode switching
|
| 58 |
+
blocks.register_forward_pre_hook(self._enter_blocks)
|
| 59 |
+
|
| 60 |
+
is_global = True
|
| 61 |
+
period = args.num_windowed + 1
|
| 62 |
+
for i, layer in enumerate(blocks[:-1]):
|
| 63 |
+
ctr = i % period
|
| 64 |
+
if ctr == 0:
|
| 65 |
+
layer.register_forward_pre_hook(self._to_windows)
|
| 66 |
+
is_global = False
|
| 67 |
+
elif ctr == args.num_windowed:
|
| 68 |
+
layer.register_forward_pre_hook(self._to_global)
|
| 69 |
+
is_global = True
|
| 70 |
+
|
| 71 |
+
# Always ensure the final layer is a global layer
|
| 72 |
+
if not is_global:
|
| 73 |
+
blocks[-1].register_forward_pre_hook(self._to_global)
|
| 74 |
+
|
| 75 |
+
blocks.register_forward_hook(self._exit_model)
|
| 76 |
+
|
| 77 |
+
def _enter_model(self, _, input: List[torch.Tensor]):
|
| 78 |
+
self._input_resolution = input[0].shape[-2:]
|
| 79 |
+
|
| 80 |
+
def _enter_blocks(self, _, input: List[torch.Tensor]):
|
| 81 |
+
# print(f'{get_rank()} - ViTDet Window Size: {self._window_size}', file=sys.stderr)
|
| 82 |
+
|
| 83 |
+
patches = input[0]
|
| 84 |
+
patches = self._rearrange_patches(patches)
|
| 85 |
+
|
| 86 |
+
return (patches,) + input[1:]
|
| 87 |
+
|
| 88 |
+
def _to_windows(self, _, input: List[torch.Tensor]):
|
| 89 |
+
patches = input[0]
|
| 90 |
+
|
| 91 |
+
if self.num_summary_tokens:
|
| 92 |
+
self._cls_patch = patches[:, :self.num_summary_tokens]
|
| 93 |
+
patches = patches[:, self.num_summary_tokens:]
|
| 94 |
+
|
| 95 |
+
patches = rearrange(
|
| 96 |
+
patches, 'b (p t) c -> (b p) t c',
|
| 97 |
+
p=self._num_windows, t=self.window_size ** 2,
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
return (patches,) + input[1:]
|
| 101 |
+
|
| 102 |
+
def _to_global(self, _, input: List[torch.Tensor]):
|
| 103 |
+
patches = input[0]
|
| 104 |
+
|
| 105 |
+
patches = rearrange(
|
| 106 |
+
patches, '(b p) t c -> b (p t) c',
|
| 107 |
+
p=self._num_windows, t=self.window_size ** 2,
|
| 108 |
+
b=patches.shape[0] // self._num_windows,
|
| 109 |
+
)
|
| 110 |
+
|
| 111 |
+
if self.num_summary_tokens:
|
| 112 |
+
patches = torch.cat([
|
| 113 |
+
self._cls_patch,
|
| 114 |
+
patches,
|
| 115 |
+
], dim=1)
|
| 116 |
+
|
| 117 |
+
return (patches,) + input[1:]
|
| 118 |
+
|
| 119 |
+
def _exit_model(self, _, inputs: List[torch.Tensor], patches: torch.Tensor):
|
| 120 |
+
# Return patches to their original order
|
| 121 |
+
patch_order = self._order_cache[self._input_resolution][0]
|
| 122 |
+
patch_order = patch_order.reshape(1, -1, 1).expand_as(patches)
|
| 123 |
+
|
| 124 |
+
ret_patches = torch.empty_like(patches)
|
| 125 |
+
ret_patches = torch.scatter(
|
| 126 |
+
ret_patches,
|
| 127 |
+
dim=1,
|
| 128 |
+
index=patch_order,
|
| 129 |
+
src=patches,
|
| 130 |
+
)
|
| 131 |
+
|
| 132 |
+
return ret_patches
|
| 133 |
+
|
| 134 |
+
def _rearrange_patches(self, patches: torch.Tensor):
|
| 135 |
+
# We rearrange the patches so that we can efficiently
|
| 136 |
+
# switch between windowed and global mode by just
|
| 137 |
+
# reshaping the tensor
|
| 138 |
+
|
| 139 |
+
patch_order, self._num_windows = self._order_cache.get(self._input_resolution, (None, None))
|
| 140 |
+
if patch_order is None:
|
| 141 |
+
num_feat_patches = patches.shape[1] - self.num_summary_tokens
|
| 142 |
+
num_pixels = self._input_resolution[0] * self._input_resolution[1]
|
| 143 |
+
|
| 144 |
+
patch_size = int(round(math.sqrt(num_pixels / num_feat_patches)))
|
| 145 |
+
rows = self._input_resolution[-2] // patch_size
|
| 146 |
+
cols = self._input_resolution[-1] // patch_size
|
| 147 |
+
|
| 148 |
+
w_rows = rows // self.window_size
|
| 149 |
+
w_cols = cols // self.window_size
|
| 150 |
+
|
| 151 |
+
patch_order = torch.arange(0, num_feat_patches, device=patches.device)
|
| 152 |
+
|
| 153 |
+
patch_order = rearrange(
|
| 154 |
+
patch_order, '(wy py wx px) -> (wy wx py px)',
|
| 155 |
+
wy=w_rows, wx=w_cols,
|
| 156 |
+
py=self.window_size, px=self.window_size,
|
| 157 |
+
)
|
| 158 |
+
|
| 159 |
+
if self.num_summary_tokens:
|
| 160 |
+
patch_order = torch.cat([
|
| 161 |
+
torch.arange(self.num_summary_tokens, dtype=patch_order.dtype, device=patch_order.device),
|
| 162 |
+
patch_order + self.num_summary_tokens,
|
| 163 |
+
])
|
| 164 |
+
|
| 165 |
+
self._num_windows = w_rows * w_cols
|
| 166 |
+
self._order_cache[self._input_resolution] = (
|
| 167 |
+
patch_order,
|
| 168 |
+
self._num_windows,
|
| 169 |
+
)
|
| 170 |
+
|
| 171 |
+
patch_order = patch_order.reshape(1, -1, 1).expand_as(patches)
|
| 172 |
+
patches = torch.gather(patches, dim=1, index=patch_order)
|
| 173 |
+
return patches
|
special_tokens_map.json
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"additional_special_tokens": [
|
| 3 |
+
"<img>",
|
| 4 |
+
"</img>",
|
| 5 |
+
"<IMG_CONTEXT>",
|
| 6 |
+
"<quad>",
|
| 7 |
+
"</quad>",
|
| 8 |
+
"<ref>",
|
| 9 |
+
"</ref>",
|
| 10 |
+
"<box>",
|
| 11 |
+
"</box>"
|
| 12 |
+
],
|
| 13 |
+
"bos_token": {
|
| 14 |
+
"content": "<s>",
|
| 15 |
+
"lstrip": false,
|
| 16 |
+
"normalized": false,
|
| 17 |
+
"rstrip": false,
|
| 18 |
+
"single_word": false
|
| 19 |
+
},
|
| 20 |
+
"eos_token": {
|
| 21 |
+
"content": "</s>",
|
| 22 |
+
"lstrip": false,
|
| 23 |
+
"normalized": false,
|
| 24 |
+
"rstrip": true,
|
| 25 |
+
"single_word": false
|
| 26 |
+
},
|
| 27 |
+
"pad_token": {
|
| 28 |
+
"content": "</s>",
|
| 29 |
+
"lstrip": false,
|
| 30 |
+
"normalized": false,
|
| 31 |
+
"rstrip": true,
|
| 32 |
+
"single_word": false
|
| 33 |
+
},
|
| 34 |
+
"unk_token": {
|
| 35 |
+
"content": "<unk>",
|
| 36 |
+
"lstrip": false,
|
| 37 |
+
"normalized": false,
|
| 38 |
+
"rstrip": false,
|
| 39 |
+
"single_word": false
|
| 40 |
+
}
|
| 41 |
+
}
|
tokenizer.model
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
|
| 3 |
+
size 499723
|
tokenizer_config.json
ADDED
|
@@ -0,0 +1,213 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"add_bos_token": true,
|
| 3 |
+
"add_eos_token": false,
|
| 4 |
+
"added_tokens_decoder": {
|
| 5 |
+
"0": {
|
| 6 |
+
"content": "<unk>",
|
| 7 |
+
"lstrip": false,
|
| 8 |
+
"normalized": false,
|
| 9 |
+
"rstrip": false,
|
| 10 |
+
"single_word": false,
|
| 11 |
+
"special": true
|
| 12 |
+
},
|
| 13 |
+
"1": {
|
| 14 |
+
"content": "<s>",
|
| 15 |
+
"lstrip": false,
|
| 16 |
+
"normalized": false,
|
| 17 |
+
"rstrip": false,
|
| 18 |
+
"single_word": false,
|
| 19 |
+
"special": true
|
| 20 |
+
},
|
| 21 |
+
"2": {
|
| 22 |
+
"content": "</s>",
|
| 23 |
+
"lstrip": false,
|
| 24 |
+
"normalized": false,
|
| 25 |
+
"rstrip": true,
|
| 26 |
+
"single_word": false,
|
| 27 |
+
"special": true
|
| 28 |
+
},
|
| 29 |
+
"32000": {
|
| 30 |
+
"content": "<|endoftext|>",
|
| 31 |
+
"lstrip": false,
|
| 32 |
+
"normalized": false,
|
| 33 |
+
"rstrip": false,
|
| 34 |
+
"single_word": false,
|
| 35 |
+
"special": true
|
| 36 |
+
},
|
| 37 |
+
"32001": {
|
| 38 |
+
"content": "<|assistant|>",
|
| 39 |
+
"lstrip": false,
|
| 40 |
+
"normalized": false,
|
| 41 |
+
"rstrip": true,
|
| 42 |
+
"single_word": false,
|
| 43 |
+
"special": true
|
| 44 |
+
},
|
| 45 |
+
"32002": {
|
| 46 |
+
"content": "<|placeholder1|>",
|
| 47 |
+
"lstrip": false,
|
| 48 |
+
"normalized": false,
|
| 49 |
+
"rstrip": true,
|
| 50 |
+
"single_word": false,
|
| 51 |
+
"special": true
|
| 52 |
+
},
|
| 53 |
+
"32003": {
|
| 54 |
+
"content": "<|placeholder2|>",
|
| 55 |
+
"lstrip": false,
|
| 56 |
+
"normalized": false,
|
| 57 |
+
"rstrip": true,
|
| 58 |
+
"single_word": false,
|
| 59 |
+
"special": true
|
| 60 |
+
},
|
| 61 |
+
"32004": {
|
| 62 |
+
"content": "<|placeholder3|>",
|
| 63 |
+
"lstrip": false,
|
| 64 |
+
"normalized": false,
|
| 65 |
+
"rstrip": true,
|
| 66 |
+
"single_word": false,
|
| 67 |
+
"special": true
|
| 68 |
+
},
|
| 69 |
+
"32005": {
|
| 70 |
+
"content": "<|placeholder4|>",
|
| 71 |
+
"lstrip": false,
|
| 72 |
+
"normalized": false,
|
| 73 |
+
"rstrip": true,
|
| 74 |
+
"single_word": false,
|
| 75 |
+
"special": true
|
| 76 |
+
},
|
| 77 |
+
"32006": {
|
| 78 |
+
"content": "<|system|>",
|
| 79 |
+
"lstrip": false,
|
| 80 |
+
"normalized": false,
|
| 81 |
+
"rstrip": true,
|
| 82 |
+
"single_word": false,
|
| 83 |
+
"special": true
|
| 84 |
+
},
|
| 85 |
+
"32007": {
|
| 86 |
+
"content": "<|end|>",
|
| 87 |
+
"lstrip": false,
|
| 88 |
+
"normalized": false,
|
| 89 |
+
"rstrip": true,
|
| 90 |
+
"single_word": false,
|
| 91 |
+
"special": true
|
| 92 |
+
},
|
| 93 |
+
"32008": {
|
| 94 |
+
"content": "<|placeholder5|>",
|
| 95 |
+
"lstrip": false,
|
| 96 |
+
"normalized": false,
|
| 97 |
+
"rstrip": true,
|
| 98 |
+
"single_word": false,
|
| 99 |
+
"special": true
|
| 100 |
+
},
|
| 101 |
+
"32009": {
|
| 102 |
+
"content": "<|placeholder6|>",
|
| 103 |
+
"lstrip": false,
|
| 104 |
+
"normalized": false,
|
| 105 |
+
"rstrip": true,
|
| 106 |
+
"single_word": false,
|
| 107 |
+
"special": true
|
| 108 |
+
},
|
| 109 |
+
"32010": {
|
| 110 |
+
"content": "<|user|>",
|
| 111 |
+
"lstrip": false,
|
| 112 |
+
"normalized": false,
|
| 113 |
+
"rstrip": true,
|
| 114 |
+
"single_word": false,
|
| 115 |
+
"special": true
|
| 116 |
+
},
|
| 117 |
+
"32011": {
|
| 118 |
+
"content": "<img>",
|
| 119 |
+
"lstrip": false,
|
| 120 |
+
"normalized": false,
|
| 121 |
+
"rstrip": false,
|
| 122 |
+
"single_word": false,
|
| 123 |
+
"special": true
|
| 124 |
+
},
|
| 125 |
+
"32012": {
|
| 126 |
+
"content": "</img>",
|
| 127 |
+
"lstrip": false,
|
| 128 |
+
"normalized": false,
|
| 129 |
+
"rstrip": false,
|
| 130 |
+
"single_word": false,
|
| 131 |
+
"special": true
|
| 132 |
+
},
|
| 133 |
+
"32013": {
|
| 134 |
+
"content": "<IMG_CONTEXT>",
|
| 135 |
+
"lstrip": false,
|
| 136 |
+
"normalized": false,
|
| 137 |
+
"rstrip": false,
|
| 138 |
+
"single_word": false,
|
| 139 |
+
"special": true
|
| 140 |
+
},
|
| 141 |
+
"32014": {
|
| 142 |
+
"content": "<quad>",
|
| 143 |
+
"lstrip": false,
|
| 144 |
+
"normalized": false,
|
| 145 |
+
"rstrip": false,
|
| 146 |
+
"single_word": false,
|
| 147 |
+
"special": true
|
| 148 |
+
},
|
| 149 |
+
"32015": {
|
| 150 |
+
"content": "</quad>",
|
| 151 |
+
"lstrip": false,
|
| 152 |
+
"normalized": false,
|
| 153 |
+
"rstrip": false,
|
| 154 |
+
"single_word": false,
|
| 155 |
+
"special": true
|
| 156 |
+
},
|
| 157 |
+
"32016": {
|
| 158 |
+
"content": "<ref>",
|
| 159 |
+
"lstrip": false,
|
| 160 |
+
"normalized": false,
|
| 161 |
+
"rstrip": false,
|
| 162 |
+
"single_word": false,
|
| 163 |
+
"special": true
|
| 164 |
+
},
|
| 165 |
+
"32017": {
|
| 166 |
+
"content": "</ref>",
|
| 167 |
+
"lstrip": false,
|
| 168 |
+
"normalized": false,
|
| 169 |
+
"rstrip": false,
|
| 170 |
+
"single_word": false,
|
| 171 |
+
"special": true
|
| 172 |
+
},
|
| 173 |
+
"32018": {
|
| 174 |
+
"content": "<box>",
|
| 175 |
+
"lstrip": false,
|
| 176 |
+
"normalized": false,
|
| 177 |
+
"rstrip": false,
|
| 178 |
+
"single_word": false,
|
| 179 |
+
"special": true
|
| 180 |
+
},
|
| 181 |
+
"32019": {
|
| 182 |
+
"content": "</box>",
|
| 183 |
+
"lstrip": false,
|
| 184 |
+
"normalized": false,
|
| 185 |
+
"rstrip": false,
|
| 186 |
+
"single_word": false,
|
| 187 |
+
"special": true
|
| 188 |
+
}
|
| 189 |
+
},
|
| 190 |
+
"additional_special_tokens": [
|
| 191 |
+
"<img>",
|
| 192 |
+
"</img>",
|
| 193 |
+
"<IMG_CONTEXT>",
|
| 194 |
+
"<quad>",
|
| 195 |
+
"</quad>",
|
| 196 |
+
"<ref>",
|
| 197 |
+
"</ref>",
|
| 198 |
+
"<box>",
|
| 199 |
+
"</box>"
|
| 200 |
+
],
|
| 201 |
+
"bos_token": "<s>",
|
| 202 |
+
"chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}",
|
| 203 |
+
"clean_up_tokenization_spaces": false,
|
| 204 |
+
"eos_token": "</s>",
|
| 205 |
+
"legacy": false,
|
| 206 |
+
"model_max_length": 8192,
|
| 207 |
+
"pad_token": "</s>",
|
| 208 |
+
"sp_model_kwargs": {},
|
| 209 |
+
"spaces_between_special_tokens": false,
|
| 210 |
+
"tokenizer_class": "LlamaTokenizer",
|
| 211 |
+
"unk_token": "<unk>",
|
| 212 |
+
"use_default_system_prompt": false
|
| 213 |
+
}
|