From d6de81643928d653435c431bae19945d41d32520 Mon Sep 17 00:00:00 2001 From: Ruilin Gao Date: Mon, 14 Sep 2026 06:51:05 -0400 Subject: [PATCH] fix(basqueglue): load the f1 metric with evaluate (#4076) datasets.load_metric was removed in datasets 3.0 and the unit-test job resolves datasets==5.0.1, so the module-level from datasets import load_metric makes lm_eval/tasks/basqueglue/utils.py unimportable. Five of the six BasqueGLUE tasks resolve a !function from that module, so they now fail to load at all, not just at scoring time: bec.yaml bec2016eu utils.micro_f1_score bhtc.yaml bhtc_v2 utils.micro_f1_score coref.yaml epec_koref_bin utils.coref_doc_to_text vaxx.yaml vaxx_stance utils.vaxx_f1_score wic.yaml wiceu utils.process_wic_docs Load the metric with evaluate instead, the same move #2351 made for squadv2, with the import inside the function as that fix and kobest/utils.py do. The metric and the compute() arguments are unchanged. --- lm_eval/tasks/basqueglue/utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lm_eval/tasks/basqueglue/utils.py b/lm_eval/tasks/basqueglue/utils.py index 401375f70..3a13927f0 100644 --- a/lm_eval/tasks/basqueglue/utils.py +++ b/lm_eval/tasks/basqueglue/utils.py @@ -1,8 +1,6 @@ import html import re -from datasets import load_metric - def general_detokenize(string): string = re.sub(r"\s+([.,;:!?)])", r"\1", string) @@ -60,7 +58,9 @@ def coref_doc_to_text(x): def micro_f1_score(items): - f1_metric = load_metric("f1") + import evaluate + + f1_metric = evaluate.load("f1") golds, preds = list(zip(*items)) f1_score = f1_metric.compute(references=golds, predictions=preds, average="micro")[ "f1" @@ -69,7 +69,9 @@ def micro_f1_score(items): def vaxx_f1_score(items): - f1_metric = load_metric("f1") + import evaluate + + f1_metric = evaluate.load("f1") golds, preds = list(zip(*items)) f1_class = f1_metric.compute( references=golds, predictions=preds, labels=[0, 2], average=None