Upload Phi4MMForCausalLM

Updated weights (4 epochs)

README.md

@@ -8,45 +8,45 @@ metrics:
 - cer
 base_model:
 - microsoft/Phi-4-multimodal-instruct
+language:
+- ko
+license: mit
+tags:
+- korean
+- stt
+- custom_code
+- phi
+- phi-4-multimodal
 model-index:
 - name: Phi-4-mm-inst-zeroth-kor
   results:
   - task:
       type: speech-to-text-translation
     dataset:
-      type: seastar105/fleurs_ko_en_test
       name: fleurs (ko-en test intersection)
+      type: seastar105/fleurs_ko_en_test
     metrics:
     - type: bleu
-      name: ko2en
       value: 7.03
+      name: ko2en
     - type: bleu
-      name: ko2en-cot
       value: 7.04
+      name: ko2en-cot
     - type: bleu
+      value: 12.5
       name: en2ko (ko-mecab)
-      value: 12.50
     - type: bleu
-      name: en2ko-cot (ko-mecab)
       value: 9.54
+      name: en2ko-cot (ko-mecab)
   - task:
       type: automatic-speech-recognition
     dataset:
-      type: kresnik/zeroth_korean
       name: zeroth_korean test
+      type: kresnik/zeroth_korean
     metrics:
     - type: cer
-      name: test CER
       value: 7.02
-language:
-- ko
-license: mit
-tags:
-- korean
-- stt
-- custom_code
-- phi
-- phi-4-multimodal
+      name: test CER
 ---
 
 # Phi-4-multimodal-finetune-ko-speech

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "./output",
+  "_name_or_path": "./Phi-4-multimodal-finetune-ko-speech",
   "architectures": [
     "Phi4MMForCausalLM"
   ],

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5dd159a69ab84ceb84ae8495bc2c5b186bbdd666786e6fa95d8f3860119f3484
-size 4998420448
+oid sha256:0a1e57956624d5a96c5dcaab437261353efbcfaf5eff49d58d92329311020802
+size 4997504848

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:402ce19eefd8c071927dd399bf6a84ce7a1ed1c400ed96cfe0f5b4e7872050f4
-size 4983891952
+oid sha256:a7ae17c45b5ab41c7659784e8cb2d6a79e6f886783b3c2af0297448a1f240c1c
+size 4952333128

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2dc9ce4edc2596f5d9439b5762f9b31b7e19342ea1305769388907d7bc015fc8
-size 1905111704
+oid sha256:7277ccd89f1b88a99dd7ce37095510f2036e18e6289b732164704ee0d2934b92
+size 1199389232

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 11887118272
+    "total_size": 11148920768
   },
   "weight_map": {
     "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
@@ -1741,50 +1741,50 @@
     "model.layers.24.self_attn.qkv_proj.lora_A.vision.weight": "model-00002-of-00003.safetensors",
     "model.layers.24.self_attn.qkv_proj.lora_B.speech.weight": "model-00002-of-00003.safetensors",
     "model.layers.24.self_attn.qkv_proj.lora_B.vision.weight": "model-00002-of-00003.safetensors",
-    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
-    "model.layers.25.mlp.down_proj.base_layer.weight": "model-00003-of-00003.safetensors",
-    "model.layers.25.mlp.down_proj.lora_A.speech.weight": "model-00003-of-00003.safetensors",
-    "model.layers.25.mlp.down_proj.lora_A.vision.weight": "model-00003-of-00003.safetensors",
-    "model.layers.25.mlp.down_proj.lora_B.speech.weight": "model-00003-of-00003.safetensors",
-    "model.layers.25.mlp.down_proj.lora_B.vision.weight": "model-00003-of-00003.safetensors",
-    "model.layers.25.mlp.gate_up_proj.base_layer.weight": "model-00003-of-00003.safetensors",
-    "model.layers.25.mlp.gate_up_proj.lora_A.speech.weight": "model-00003-of-00003.safetensors",
-    "model.layers.25.mlp.gate_up_proj.lora_A.vision.weight": "model-00003-of-00003.safetensors",
-    "model.layers.25.mlp.gate_up_proj.lora_B.speech.weight": "model-00003-of-00003.safetensors",
-    "model.layers.25.mlp.gate_up_proj.lora_B.vision.weight": "model-00003-of-00003.safetensors",
-    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.base_layer.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.lora_A.speech.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.lora_A.vision.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.lora_B.speech.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.lora_B.vision.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.mlp.gate_up_proj.base_layer.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.mlp.gate_up_proj.lora_A.speech.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.mlp.gate_up_proj.lora_A.vision.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.mlp.gate_up_proj.lora_B.speech.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.mlp.gate_up_proj.lora_B.vision.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
     "model.layers.25.self_attn.o_proj.base_layer.weight": "model-00002-of-00003.safetensors",
     "model.layers.25.self_attn.o_proj.lora_A.speech.weight": "model-00002-of-00003.safetensors",
     "model.layers.25.self_attn.o_proj.lora_A.vision.weight": "model-00002-of-00003.safetensors",
     "model.layers.25.self_attn.o_proj.lora_B.speech.weight": "model-00002-of-00003.safetensors",
     "model.layers.25.self_attn.o_proj.lora_B.vision.weight": "model-00002-of-00003.safetensors",
-    "model.layers.25.self_attn.qkv_proj.base_layer.weight": "model-00003-of-00003.safetensors",
-    "model.layers.25.self_attn.qkv_proj.lora_A.speech.weight": "model-00003-of-00003.safetensors",
-    "model.layers.25.self_attn.qkv_proj.lora_A.vision.weight": "model-00003-of-00003.safetensors",
-    "model.layers.25.self_attn.qkv_proj.lora_B.speech.weight": "model-00003-of-00003.safetensors",
-    "model.layers.25.self_attn.qkv_proj.lora_B.vision.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.mlp.down_proj.base_layer.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.mlp.down_proj.lora_A.speech.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.mlp.down_proj.lora_A.vision.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.mlp.down_proj.lora_B.speech.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.mlp.down_proj.lora_B.vision.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.mlp.gate_up_proj.base_layer.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.mlp.gate_up_proj.lora_A.speech.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.mlp.gate_up_proj.lora_A.vision.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.mlp.gate_up_proj.lora_B.speech.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.mlp.gate_up_proj.lora_B.vision.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.self_attn.o_proj.base_layer.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.self_attn.o_proj.lora_A.speech.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.self_attn.o_proj.lora_A.vision.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.self_attn.o_proj.lora_B.speech.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.self_attn.o_proj.lora_B.vision.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.self_attn.qkv_proj.base_layer.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.self_attn.qkv_proj.lora_A.speech.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.self_attn.qkv_proj.lora_A.vision.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.self_attn.qkv_proj.lora_B.speech.weight": "model-00003-of-00003.safetensors",
-    "model.layers.26.self_attn.qkv_proj.lora_B.vision.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.qkv_proj.base_layer.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.qkv_proj.lora_A.speech.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.qkv_proj.lora_A.vision.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.qkv_proj.lora_B.speech.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.qkv_proj.lora_B.vision.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.base_layer.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.lora_A.speech.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.lora_A.vision.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.lora_B.speech.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.lora_B.vision.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.mlp.gate_up_proj.base_layer.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.mlp.gate_up_proj.lora_A.speech.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.mlp.gate_up_proj.lora_A.vision.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.mlp.gate_up_proj.lora_B.speech.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.mlp.gate_up_proj.lora_B.vision.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.base_layer.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.lora_A.speech.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.lora_A.vision.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.lora_B.speech.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.lora_B.vision.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.qkv_proj.base_layer.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.qkv_proj.lora_A.speech.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.qkv_proj.lora_A.vision.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.qkv_proj.lora_B.speech.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.qkv_proj.lora_B.vision.weight": "model-00002-of-00003.safetensors",
     "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
     "model.layers.27.mlp.down_proj.base_layer.weight": "model-00003-of-00003.safetensors",
     "model.layers.27.mlp.down_proj.lora_A.speech.weight": "model-00003-of-00003.safetensors",
@@ -1797,16 +1797,16 @@
     "model.layers.27.mlp.gate_up_proj.lora_B.speech.weight": "model-00003-of-00003.safetensors",
     "model.layers.27.mlp.gate_up_proj.lora_B.vision.weight": "model-00003-of-00003.safetensors",
     "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
-    "model.layers.27.self_attn.o_proj.base_layer.weight": "model-00003-of-00003.safetensors",
-    "model.layers.27.self_attn.o_proj.lora_A.speech.weight": "model-00003-of-00003.safetensors",
-    "model.layers.27.self_attn.o_proj.lora_A.vision.weight": "model-00003-of-00003.safetensors",
-    "model.layers.27.self_attn.o_proj.lora_B.speech.weight": "model-00003-of-00003.safetensors",
-    "model.layers.27.self_attn.o_proj.lora_B.vision.weight": "model-00003-of-00003.safetensors",
-    "model.layers.27.self_attn.qkv_proj.base_layer.weight": "model-00003-of-00003.safetensors",
-    "model.layers.27.self_attn.qkv_proj.lora_A.speech.weight": "model-00003-of-00003.safetensors",
-    "model.layers.27.self_attn.qkv_proj.lora_A.vision.weight": "model-00003-of-00003.safetensors",
-    "model.layers.27.self_attn.qkv_proj.lora_B.speech.weight": "model-00003-of-00003.safetensors",
-    "model.layers.27.self_attn.qkv_proj.lora_B.vision.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.base_layer.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.lora_A.speech.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.lora_A.vision.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.lora_B.speech.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.lora_B.vision.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.qkv_proj.base_layer.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.qkv_proj.lora_A.speech.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.qkv_proj.lora_A.vision.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.qkv_proj.lora_B.speech.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.qkv_proj.lora_B.vision.weight": "model-00002-of-00003.safetensors",
     "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
     "model.layers.28.mlp.down_proj.base_layer.weight": "model-00003-of-00003.safetensors",
     "model.layers.28.mlp.down_proj.lora_A.speech.weight": "model-00003-of-00003.safetensors",
@@ -1989,22 +1989,22 @@
     "model.layers.7.mlp.down_proj.lora_A.vision.weight": "model-00002-of-00003.safetensors",
     "model.layers.7.mlp.down_proj.lora_B.speech.weight": "model-00002-of-00003.safetensors",
     "model.layers.7.mlp.down_proj.lora_B.vision.weight": "model-00002-of-00003.safetensors",
-    "model.layers.7.mlp.gate_up_proj.base_layer.weight": "model-00002-of-00003.safetensors",
-    "model.layers.7.mlp.gate_up_proj.lora_A.speech.weight": "model-00002-of-00003.safetensors",
-    "model.layers.7.mlp.gate_up_proj.lora_A.vision.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.mlp.gate_up_proj.base_layer.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_up_proj.lora_A.speech.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_up_proj.lora_A.vision.weight": "model-00001-of-00003.safetensors",
     "model.layers.7.mlp.gate_up_proj.lora_B.speech.weight": "model-00002-of-00003.safetensors",
-    "model.layers.7.mlp.gate_up_proj.lora_B.vision.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.mlp.gate_up_proj.lora_B.vision.weight": "model-00001-of-00003.safetensors",
     "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
     "model.layers.7.self_attn.o_proj.base_layer.weight": "model-00001-of-00003.safetensors",
-    "model.layers.7.self_attn.o_proj.lora_A.speech.weight": "model-00002-of-00003.safetensors",
-    "model.layers.7.self_attn.o_proj.lora_A.vision.weight": "model-00002-of-00003.safetensors",
-    "model.layers.7.self_attn.o_proj.lora_B.speech.weight": "model-00002-of-00003.safetensors",
-    "model.layers.7.self_attn.o_proj.lora_B.vision.weight": "model-00002-of-00003.safetensors",
-    "model.layers.7.self_attn.qkv_proj.base_layer.weight": "model-00002-of-00003.safetensors",
-    "model.layers.7.self_attn.qkv_proj.lora_A.speech.weight": "model-00002-of-00003.safetensors",
-    "model.layers.7.self_attn.qkv_proj.lora_A.vision.weight": "model-00002-of-00003.safetensors",
-    "model.layers.7.self_attn.qkv_proj.lora_B.speech.weight": "model-00002-of-00003.safetensors",
-    "model.layers.7.self_attn.qkv_proj.lora_B.vision.weight": "model-00002-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.lora_A.speech.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.lora_A.vision.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.lora_B.speech.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.lora_B.vision.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.qkv_proj.base_layer.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.qkv_proj.lora_A.speech.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.qkv_proj.lora_A.vision.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.qkv_proj.lora_B.speech.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.qkv_proj.lora_B.vision.weight": "model-00001-of-00003.safetensors",
     "model.layers.8.input_layernorm.weight": "model-00002-of-00003.safetensors",
     "model.layers.8.mlp.down_proj.base_layer.weight": "model-00002-of-00003.safetensors",
     "model.layers.8.mlp.down_proj.lora_A.speech.weight": "model-00002-of-00003.safetensors",