Add new SentenceTransformer model.

README.md

@@ -45,34 +45,34 @@ tags:
 - sentence-similarity
 - feature-extraction
 - generated_from_trainer
-- dataset_size:171
+- dataset_size:267
 - loss:MultipleNegativesRankingLoss
 widget:
-- source_sentence: ナイトスタンドにある？
+- source_sentence: 昨日夕飯にチキンヌードル食べた？
   sentences:
-  - なんで話せるの？
-  - やっぱり、タイマツがいい
-  - スカーフはナイトスタンドにある？
-- source_sentence: 夕飯が辛かったから
+  - ナイトスタンドにスカーフはある？
+  - 夕飯はチキンヌードルだった？
+  - スカーフがキャンプファイヤーで燃えてる
+- source_sentence: テーブル
   sentences:
-  - 夕飯に辛いスープを飲んだから
-  - 村人について教えて
-  - 昨日なに作ったの？
-- source_sentence: じぶん
-  sentences:
-  - 窓が開いていたから
+  - はじめにどこをさがせばいい？
   - 自分がやった
-  - タイマツ要らない
-- source_sentence: 夜ごはんの時
+  - テーブルを調べよう
+- source_sentence: 欲しくない
+  sentences:
+  - 物の姿を変える魔法が使える村人を知っている？
+  - 誰かが魔法を使った
+  - 家の中を探してみよう
+- source_sentence: 家の外
   sentences:
-  - キャンドルがいいな
-  - 晩ご飯のとき
-  - 赤い染みが皿にあった
-- source_sentence: あなた
+  - キャンドル要らない
+  - どこでもいいよ
+  - 魔法使い
+- source_sentence: キャンドル頂戴
   sentences:
-  - 賢者の木について教えて
-  - どっちも欲しくない
-  - 長老
+  - 物の姿を変える魔法が使える村人を知っている？
+  - 魔女
+  - やっぱり、キャンドルがいい
 model-index:
 - name: SentenceTransformer based on colorfulscoop/sbert-base-ja
   results:
@@ -84,109 +84,109 @@ model-index:
       type: custom-arc-semantics-data
     metrics:
     - type: cosine_accuracy
-      value: 0.991304347826087
+      value: 0.8258426966292135
       name: Cosine Accuracy
     - type: cosine_accuracy_threshold
-      value: 0.3721364140510559
+      value: 0.530483067035675
       name: Cosine Accuracy Threshold
     - type: cosine_f1
-      value: 0.9956331877729258
+      value: 0.8571428571428571
       name: Cosine F1
     - type: cosine_f1_threshold
-      value: 0.3721364140510559
+      value: 0.530483067035675
       name: Cosine F1 Threshold
     - type: cosine_precision
-      value: 1.0
+      value: 0.8532110091743119
       name: Cosine Precision
     - type: cosine_recall
-      value: 0.991304347826087
+      value: 0.8611111111111112
       name: Cosine Recall
     - type: cosine_ap
-      value: 1.0
+      value: 0.9302395955607082
       name: Cosine Ap
     - type: dot_accuracy
-      value: 0.991304347826087
+      value: 0.8202247191011236
       name: Dot Accuracy
     - type: dot_accuracy_threshold
-      value: 196.46392822265625
+      value: 286.6033630371094
       name: Dot Accuracy Threshold
     - type: dot_f1
-      value: 0.9956331877729258
+      value: 0.8518518518518519
       name: Dot F1
     - type: dot_f1_threshold
-      value: 196.46392822265625
+      value: 286.6033630371094
       name: Dot F1 Threshold
     - type: dot_precision
-      value: 1.0
+      value: 0.8518518518518519
       name: Dot Precision
     - type: dot_recall
-      value: 0.991304347826087
+      value: 0.8518518518518519
       name: Dot Recall
     - type: dot_ap
-      value: 1.0
+      value: 0.9269146593596983
       name: Dot Ap
     - type: manhattan_accuracy
-      value: 0.991304347826087
+      value: 0.8258426966292135
       name: Manhattan Accuracy
     - type: manhattan_accuracy_threshold
-      value: 580.8839111328125
+      value: 500.2329406738281
       name: Manhattan Accuracy Threshold
     - type: manhattan_f1
-      value: 0.9956331877729258
+      value: 0.8597285067873304
       name: Manhattan F1
     - type: manhattan_f1_threshold
-      value: 580.8839111328125
+      value: 500.2329406738281
       name: Manhattan F1 Threshold
     - type: manhattan_precision
-      value: 1.0
+      value: 0.8407079646017699
       name: Manhattan Precision
     - type: manhattan_recall
-      value: 0.991304347826087
+      value: 0.8796296296296297
       name: Manhattan Recall
     - type: manhattan_ap
-      value: 1.0
+      value: 0.9284651287730749
       name: Manhattan Ap
     - type: euclidean_accuracy
-      value: 0.991304347826087
+      value: 0.8202247191011236
       name: Euclidean Accuracy
     - type: euclidean_accuracy_threshold
-      value: 26.66432762145996
+      value: 21.535140991210938
       name: Euclidean Accuracy Threshold
     - type: euclidean_f1
-      value: 0.9956331877729258
+      value: 0.8571428571428572
       name: Euclidean F1
     - type: euclidean_f1_threshold
-      value: 26.66432762145996
+      value: 23.045635223388672
       name: Euclidean F1 Threshold
     - type: euclidean_precision
-      value: 1.0
+      value: 0.8275862068965517
       name: Euclidean Precision
     - type: euclidean_recall
-      value: 0.991304347826087
+      value: 0.8888888888888888
       name: Euclidean Recall
     - type: euclidean_ap
-      value: 1.0
+      value: 0.9285413234296498
       name: Euclidean Ap
     - type: max_accuracy
-      value: 0.991304347826087
+      value: 0.8258426966292135
       name: Max Accuracy
     - type: max_accuracy_threshold
-      value: 580.8839111328125
+      value: 500.2329406738281
       name: Max Accuracy Threshold
     - type: max_f1
-      value: 0.9956331877729258
+      value: 0.8597285067873304
       name: Max F1
     - type: max_f1_threshold
-      value: 580.8839111328125
+      value: 500.2329406738281
       name: Max F1 Threshold
     - type: max_precision
-      value: 1.0
+      value: 0.8532110091743119
       name: Max Precision
     - type: max_recall
-      value: 0.991304347826087
+      value: 0.8888888888888888
       name: Max Recall
     - type: max_ap
-      value: 1.0
+      value: 0.9302395955607082
       name: Max Ap
 ---
 
@@ -239,9 +239,9 @@ from sentence_transformers import SentenceTransformer
 model = SentenceTransformer("LeoChiuu/sbert-base-ja-arc-temp")
 # Run inference
 sentences = [
-    'あなた',
-    '長老',
-    '賢者の木について教えて',
+    'キャンドル頂戴',
+    'やっぱり、キャンドルがいい',
+    '物の姿を変える魔法が使える村人を知っている？',
 ]
 embeddings = model.encode(sentences)
 print(embeddings.shape)
@@ -285,43 +285,43 @@ You can finetune this model on your own dataset.
 * Dataset: `custom-arc-semantics-data`
 * Evaluated with [<code>BinaryClassificationEvaluator</code>](https://sbert.net/docs/package_reference/sentence_transformer/evaluation.html#sentence_transformers.evaluation.BinaryClassificationEvaluator)
 
-| Metric                       | Value    |
-|:-----------------------------|:---------|
-| cosine_accuracy              | 0.9913   |
-| cosine_accuracy_threshold    | 0.3721   |
-| cosine_f1                    | 0.9956   |
-| cosine_f1_threshold          | 0.3721   |
-| cosine_precision             | 1.0      |
-| cosine_recall                | 0.9913   |
-| cosine_ap                    | 1.0      |
-| dot_accuracy                 | 0.9913   |
-| dot_accuracy_threshold       | 196.4639 |
-| dot_f1                       | 0.9956   |
-| dot_f1_threshold             | 196.4639 |
-| dot_precision                | 1.0      |
-| dot_recall                   | 0.9913   |
-| dot_ap                       | 1.0      |
-| manhattan_accuracy           | 0.9913   |
-| manhattan_accuracy_threshold | 580.8839 |
-| manhattan_f1                 | 0.9956   |
-| manhattan_f1_threshold       | 580.8839 |
-| manhattan_precision          | 1.0      |
-| manhattan_recall             | 0.9913   |
-| manhattan_ap                 | 1.0      |
-| euclidean_accuracy           | 0.9913   |
-| euclidean_accuracy_threshold | 26.6643  |
-| euclidean_f1                 | 0.9956   |
-| euclidean_f1_threshold       | 26.6643  |
-| euclidean_precision          | 1.0      |
-| euclidean_recall             | 0.9913   |
-| euclidean_ap                 | 1.0      |
-| max_accuracy                 | 0.9913   |
-| max_accuracy_threshold       | 580.8839 |
-| max_f1                       | 0.9956   |
-| max_f1_threshold             | 580.8839 |
-| max_precision                | 1.0      |
-| max_recall                   | 0.9913   |
-| **max_ap**                   | **1.0**  |
+| Metric                       | Value      |
+|:-----------------------------|:-----------|
+| cosine_accuracy              | 0.8258     |
+| cosine_accuracy_threshold    | 0.5305     |
+| cosine_f1                    | 0.8571     |
+| cosine_f1_threshold          | 0.5305     |
+| cosine_precision             | 0.8532     |
+| cosine_recall                | 0.8611     |
+| cosine_ap                    | 0.9302     |
+| dot_accuracy                 | 0.8202     |
+| dot_accuracy_threshold       | 286.6034   |
+| dot_f1                       | 0.8519     |
+| dot_f1_threshold             | 286.6034   |
+| dot_precision                | 0.8519     |
+| dot_recall                   | 0.8519     |
+| dot_ap                       | 0.9269     |
+| manhattan_accuracy           | 0.8258     |
+| manhattan_accuracy_threshold | 500.2329   |
+| manhattan_f1                 | 0.8597     |
+| manhattan_f1_threshold       | 500.2329   |
+| manhattan_precision          | 0.8407     |
+| manhattan_recall             | 0.8796     |
+| manhattan_ap                 | 0.9285     |
+| euclidean_accuracy           | 0.8202     |
+| euclidean_accuracy_threshold | 21.5351    |
+| euclidean_f1                 | 0.8571     |
+| euclidean_f1_threshold       | 23.0456    |
+| euclidean_precision          | 0.8276     |
+| euclidean_recall             | 0.8889     |
+| euclidean_ap                 | 0.9285     |
+| max_accuracy                 | 0.8258     |
+| max_accuracy_threshold       | 500.2329   |
+| max_f1                       | 0.8597     |
+| max_f1_threshold             | 500.2329   |
+| max_precision                | 0.8532     |
+| max_recall                   | 0.8889     |
+| **max_ap**                   | **0.9302** |
 
 <!--
 ## Bias, Risks and Limitations
@@ -342,19 +342,19 @@ You can finetune this model on your own dataset.
 #### Unnamed Dataset
 
 
-* Size: 171 training samples
+* Size: 267 training samples
 * Columns: <code>text1</code>, <code>text2</code>, and <code>label</code>
 * Approximate statistics based on the first 1000 samples:
-  |         | text1                                                                            | text2                                                                            | label                        |
-  |:--------|:---------------------------------------------------------------------------------|:---------------------------------------------------------------------------------|:-----------------------------|
-  | type    | string                                                                           | string                                                                           | int                          |
-  | details | <ul><li>min: 4 tokens</li><li>mean: 8.22 tokens</li><li>max: 15 tokens</li></ul> | <ul><li>min: 4 tokens</li><li>mean: 8.67 tokens</li><li>max: 14 tokens</li></ul> | <ul><li>1: 100.00%</li></ul> |
+  |         | text1                                                                            | text2                                                                            | label                                           |
+  |:--------|:---------------------------------------------------------------------------------|:---------------------------------------------------------------------------------|:------------------------------------------------|
+  | type    | string                                                                           | string                                                                           | int                                             |
+  | details | <ul><li>min: 4 tokens</li><li>mean: 8.36 tokens</li><li>max: 15 tokens</li></ul> | <ul><li>min: 4 tokens</li><li>mean: 8.35 tokens</li><li>max: 14 tokens</li></ul> | <ul><li>0: ~33.33%</li><li>1: ~66.67%</li></ul> |
 * Samples:
-  | text1                       | text2                   | label          |
-  |:----------------------------|:------------------------|:---------------|
-  | <code>キャンドルを用意して</code>     | <code>ロウソク</code>       | <code>1</code> |
-  | <code>なんで話せるの？</code>       | <code>なんでしゃべれるの？</code> | <code>1</code> |
-  | <code>それは物の見た目を変える魔法</code> | <code>物の見た目を変える</code>  | <code>1</code> |
+  | text1                        | text2                        | label          |
+  |:-----------------------------|:-----------------------------|:---------------|
+  | <code>ジャックはどんな魔法を使うの？</code> | <code>見た目を変える魔法</code>       | <code>0</code> |
+  | <code>魔法使い</code>            | <code>魔法をかけられる人</code>       | <code>1</code> |
+  | <code>ぬいぐるみが花</code>         | <code>花がぬいぐるみに変えられている</code> | <code>1</code> |
 * Loss: [<code>MultipleNegativesRankingLoss</code>](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#multiplenegativesrankingloss) with these parameters:
   ```json
   {
@@ -368,19 +368,19 @@ You can finetune this model on your own dataset.
 #### Unnamed Dataset
 
 
-* Size: 115 evaluation samples
+* Size: 178 evaluation samples
 * Columns: <code>text1</code>, <code>text2</code>, and <code>label</code>
 * Approximate statistics based on the first 1000 samples:
-  |         | text1                                                                            | text2                                                                            | label                        |
-  |:--------|:---------------------------------------------------------------------------------|:---------------------------------------------------------------------------------|:-----------------------------|
-  | type    | string                                                                           | string                                                                           | int                          |
-  | details | <ul><li>min: 4 tokens</li><li>mean: 8.39 tokens</li><li>max: 14 tokens</li></ul> | <ul><li>min: 4 tokens</li><li>mean: 8.45 tokens</li><li>max: 14 tokens</li></ul> | <ul><li>1: 100.00%</li></ul> |
+  |         | text1                                                                           | text2                                                                            | label                                           |
+  |:--------|:--------------------------------------------------------------------------------|:---------------------------------------------------------------------------------|:------------------------------------------------|
+  | type    | string                                                                          | string                                                                           | int                                             |
+  | details | <ul><li>min: 4 tokens</li><li>mean: 8.2 tokens</li><li>max: 15 tokens</li></ul> | <ul><li>min: 4 tokens</li><li>mean: 8.17 tokens</li><li>max: 14 tokens</li></ul> | <ul><li>0: ~39.33%</li><li>1: ~60.67%</li></ul> |
 * Samples:
-  | text1                    | text2                       | label          |
-  |:-------------------------|:----------------------------|:---------------|
-  | <code>あの木の上の布はなに？</code> | <code>あの木の上にあるやつはなに？</code> | <code>1</code> |
-  | <code>物の姿を変えられる人</code>  | <code>物の形を変えられる人</code>     | <code>1</code> |
-  | <code>夕飯の時</code>        | <code>夜ご飯を作る前</code>        | <code>1</code> |
+  | text1                        | text2                           | label          |
+  |:-----------------------------|:--------------------------------|:---------------|
+  | <code>巻き割をした？</code>         | <code>家の中を調べよう</code>           | <code>0</code> |
+  | <code>花がぬいぐるみに変えられている</code> | <code>だれかが魔法で花をぬいぐるみに変えた</code> | <code>1</code> |
+  | <code>カミーユ</code>            | <code>試すため</code>               | <code>0</code> |
 * Loss: [<code>MultipleNegativesRankingLoss</code>](https://sbert.net/docs/package_reference/sentence_transformer/losses.html#multiplenegativesrankingloss) with these parameters:
   ```json
   {
@@ -393,8 +393,8 @@ You can finetune this model on your own dataset.
 #### Non-Default Hyperparameters
 
 - `eval_strategy`: epoch
-- `learning_rate`: 1e-05
-- `num_train_epochs`: 8
+- `learning_rate`: 2e-05
+- `num_train_epochs`: 13
 - `warmup_ratio`: 0.1
 - `fp16`: True
 - `batch_sampler`: no_duplicates
@@ -413,13 +413,13 @@ You can finetune this model on your own dataset.
 - `gradient_accumulation_steps`: 1
 - `eval_accumulation_steps`: None
 - `torch_empty_cache_steps`: None
-- `learning_rate`: 1e-05
+- `learning_rate`: 2e-05
 - `weight_decay`: 0.0
 - `adam_beta1`: 0.9
 - `adam_beta2`: 0.999
 - `adam_epsilon`: 1e-08
 - `max_grad_norm`: 1.0
-- `num_train_epochs`: 8
+- `num_train_epochs`: 13
 - `max_steps`: -1
 - `lr_scheduler_type`: linear
 - `lr_scheduler_kwargs`: {}
@@ -519,15 +519,20 @@ You can finetune this model on your own dataset.
 ### Training Logs
 | Epoch | Step | Training Loss | loss   | custom-arc-semantics-data_max_ap |
 |:-----:|:----:|:-------------:|:------:|:--------------------------------:|
-| None  | 0    | -             | -      | 1.0                              |
-| 1.0   | 22   | 0.641         | 0.4950 | 1.0                              |
-| 2.0   | 44   | 0.2924        | 0.4102 | 1.0                              |
-| 3.0   | 66   | 0.1517        | 0.3816 | 1.0                              |
-| 4.0   | 88   | 0.0913        | 0.3794 | 1.0                              |
-| 5.0   | 110  | 0.0444        | 0.3810 | 1.0                              |
-| 6.0   | 132  | 0.0259        | 0.3880 | 1.0                              |
-| 7.0   | 154  | 0.0214        | 0.3931 | 1.0                              |
-| 8.0   | 176  | 0.0245        | 0.3960 | 1.0                              |
+| None  | 0    | -             | -      | 0.9463                           |
+| 1.0   | 34   | 1.4241        | 1.3327 | 0.9563                           |
+| 2.0   | 68   | 0.8143        | 1.1203 | 0.9564                           |
+| 3.0   | 102  | 0.4052        | 1.0773 | 0.9507                           |
+| 4.0   | 136  | 0.2227        | 1.0795 | 0.9459                           |
+| 5.0   | 170  | 0.1109        | 1.1310 | 0.9377                           |
+| 6.0   | 204  | 0.079         | 1.1382 | 0.9410                           |
+| 7.0   | 238  | 0.0513        | 1.1439 | 0.9369                           |
+| 8.0   | 272  | 0.0369        | 1.1683 | 0.9369                           |
+| 9.0   | 306  | 0.0277        | 1.1558 | 0.9339                           |
+| 10.0  | 340  | 0.0215        | 1.1511 | 0.9338                           |
+| 11.0  | 374  | 0.0156        | 1.1560 | 0.9310                           |
+| 12.0  | 408  | 0.0191        | 1.1661 | 0.9307                           |
+| 13.0  | 442  | 0.0113        | 1.1681 | 0.9302                           |
 
 
 ### Framework Versions

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bfe55f84834bdc680c3f241db31c9119fd94418ea7a99bcd88250bd28c7c8536
+oid sha256:af049657c02958ab57b8c8cd2b82d3b0165733d92e6db76037000fa3437cfa7d
 size 442491744