Fix bug with safe suffix removal (#34)
Browse files- Safe suffix removal (507cf490804a20826b1defcf7f4dbf9af57a0f78)
Co-authored-by: Junkun Chen <[email protected]>
sample_finetune_speech.py
CHANGED
@@ -284,7 +284,7 @@ def evaluate(
|
|
284 |
for _pred_ids, _stop_tokens_idx in zip(generated_ids, stop_tokens_idx)
|
285 |
]
|
286 |
all_generated_texts.extend(generated_text)
|
287 |
-
labels = [processor.decode(_label_ids[_label_ids != 0]).
|
288 |
all_labels.extend(labels)
|
289 |
|
290 |
all_generated_texts = gather_object(all_generated_texts)
|
|
|
284 |
for _pred_ids, _stop_tokens_idx in zip(generated_ids, stop_tokens_idx)
|
285 |
]
|
286 |
all_generated_texts.extend(generated_text)
|
287 |
+
labels = [processor.decode(_label_ids[_label_ids != 0]).removesuffix(ANSWER_SUFFIX) for _label_ids in inputs["labels"]]
|
288 |
all_labels.extend(labels)
|
289 |
|
290 |
all_generated_texts = gather_object(all_generated_texts)
|