mirror of
https://github.com/EleutherAI/lm-evaluation-harness.git
synced 2026-10-11 21:59:53 +00:00
fix(basqueglue): load the f1 metric with evaluate (#4076)
datasets.load_metric was removed in datasets 3.0 and the unit-test job
resolves datasets==5.0.1, so the module-level
from datasets import load_metric
makes lm_eval/tasks/basqueglue/utils.py unimportable. Five of the six
BasqueGLUE tasks resolve a !function from that module, so they now fail to
load at all, not just at scoring time:
bec.yaml bec2016eu utils.micro_f1_score
bhtc.yaml bhtc_v2 utils.micro_f1_score
coref.yaml epec_koref_bin utils.coref_doc_to_text
vaxx.yaml vaxx_stance utils.vaxx_f1_score
wic.yaml wiceu utils.process_wic_docs
Load the metric with evaluate instead, the same move #2351 made for
squadv2, with the import inside the function as that fix and
kobest/utils.py do. The metric and the compute() arguments are unchanged.
This commit is contained in:
@@ -1,8 +1,6 @@
|
||||
import html
|
||||
import re
|
||||
|
||||
from datasets import load_metric
|
||||
|
||||
|
||||
def general_detokenize(string):
|
||||
string = re.sub(r"\s+([.,;:!?)])", r"\1", string)
|
||||
@@ -60,7 +58,9 @@ def coref_doc_to_text(x):
|
||||
|
||||
|
||||
def micro_f1_score(items):
|
||||
f1_metric = load_metric("f1")
|
||||
import evaluate
|
||||
|
||||
f1_metric = evaluate.load("f1")
|
||||
golds, preds = list(zip(*items))
|
||||
f1_score = f1_metric.compute(references=golds, predictions=preds, average="micro")[
|
||||
"f1"
|
||||
@@ -69,7 +69,9 @@ def micro_f1_score(items):
|
||||
|
||||
|
||||
def vaxx_f1_score(items):
|
||||
f1_metric = load_metric("f1")
|
||||
import evaluate
|
||||
|
||||
f1_metric = evaluate.load("f1")
|
||||
golds, preds = list(zip(*items))
|
||||
f1_class = f1_metric.compute(
|
||||
references=golds, predictions=preds, labels=[0, 2], average=None
|
||||
|
||||
Reference in New Issue
Block a user