From 868f8a2f2b774f73869264c050415e4d83dc7b9b Mon Sep 17 00:00:00 2001 From: Jaehun Kim <42717042+wogns3623@users.noreply.github.com> Date: Tue, 27 Jan 2026 19:11:03 +0900 Subject: [PATCH] fix: super_glue is moved to aps/super_glue (#3523) --- examples/lm-eval-overview.ipynb | 2 +- lm_eval/tasks/benchmarks/t0_eval.yaml | 10 +++++----- lm_eval/tasks/super_glue/boolq/default.yaml | 2 +- lm_eval/tasks/super_glue/boolq/seq2seq.yaml | 2 +- lm_eval/tasks/super_glue/boolq/t5-prompt.yaml | 2 +- lm_eval/tasks/super_glue/cb/default.yaml | 2 +- lm_eval/tasks/super_glue/cb/t5-prompt.yaml | 2 +- lm_eval/tasks/super_glue/copa/default.yaml | 2 +- lm_eval/tasks/super_glue/copa/t5-prompt.yaml | 2 +- lm_eval/tasks/super_glue/multirc/default.yaml | 2 +- lm_eval/tasks/super_glue/multirc/t5-prompt.yaml | 2 +- lm_eval/tasks/super_glue/record/default.yaml | 2 +- lm_eval/tasks/super_glue/record/t5-prompt.yaml | 2 +- lm_eval/tasks/super_glue/rte/default.yaml | 2 +- lm_eval/tasks/super_glue/rte/t5-prompt.yaml | 2 +- lm_eval/tasks/super_glue/wic/default.yaml | 2 +- lm_eval/tasks/super_glue/wic/t5-prompt.yaml | 2 +- lm_eval/tasks/super_glue/wsc/default.yaml | 2 +- lm_eval/tasks/super_glue/wsc/t5-prompt.yaml | 2 +- 19 files changed, 23 insertions(+), 23 deletions(-) diff --git a/examples/lm-eval-overview.ipynb b/examples/lm-eval-overview.ipynb index 6beb34d71..ac7ff4b5e 100644 --- a/examples/lm-eval-overview.ipynb +++ b/examples/lm-eval-overview.ipynb @@ -288,7 +288,7 @@ "source": [ "YAML_boolq_string = \"\"\"\n", "task: demo_boolq\n", - "dataset_path: super_glue\n", + "dataset_path: aps/super_glue\n", "dataset_name: boolq\n", "output_type: multiple_choice\n", "training_split: train\n", diff --git a/lm_eval/tasks/benchmarks/t0_eval.yaml b/lm_eval/tasks/benchmarks/t0_eval.yaml index 27e7adc41..843fcd05f 100644 --- a/lm_eval/tasks/benchmarks/t0_eval.yaml +++ b/lm_eval/tasks/benchmarks/t0_eval.yaml @@ -1,7 +1,7 @@ group: t0_eval task: # Coreference Resolution - - dataset_path: super_glue + - dataset_path: aps/super_glue dataset_name: wsc.fixed use_prompt: promptsource:* training_split: train @@ -27,7 +27,7 @@ task: ignore_case: true ignore_punctuation: true # Natural Language Inference - - dataset_path: super_glue + - dataset_path: aps/super_glue dataset_name: cb use_prompt: promptsource:* training_split: train @@ -39,7 +39,7 @@ task: higher_is_better: true ignore_case: true ignore_punctuation: true - - dataset_path: super_glue + - dataset_path: aps/super_glue dataset_name: rte use_prompt: promptsource:* training_split: train @@ -88,7 +88,7 @@ task: ignore_case: true ignore_punctuation: true # Sentence Completion - - dataset_path: super_glue + - dataset_path: aps/super_glue dataset_name: copa use_prompt: promptsource:* training_split: train @@ -113,7 +113,7 @@ task: ignore_case: true ignore_punctuation: true # Word Sense Disambiguation - - dataset_path: super_glue + - dataset_path: aps/super_glue dataset_name: wic use_prompt: promptsource:* training_split: train diff --git a/lm_eval/tasks/super_glue/boolq/default.yaml b/lm_eval/tasks/super_glue/boolq/default.yaml index 1123761df..0eaec518d 100644 --- a/lm_eval/tasks/super_glue/boolq/default.yaml +++ b/lm_eval/tasks/super_glue/boolq/default.yaml @@ -1,7 +1,7 @@ tag: - super-glue-lm-eval-v1 task: boolq -dataset_path: super_glue +dataset_path: aps/super_glue dataset_name: boolq output_type: multiple_choice training_split: train diff --git a/lm_eval/tasks/super_glue/boolq/seq2seq.yaml b/lm_eval/tasks/super_glue/boolq/seq2seq.yaml index 08211ec11..9b20c022f 100644 --- a/lm_eval/tasks/super_glue/boolq/seq2seq.yaml +++ b/lm_eval/tasks/super_glue/boolq/seq2seq.yaml @@ -1,7 +1,7 @@ tag: - super-glue-lm-eval-v1-seq2seq task: "boolq-seq2seq" -dataset_path: super_glue +dataset_path: aps/super_glue dataset_name: boolq output_type: generate_until training_split: train diff --git a/lm_eval/tasks/super_glue/boolq/t5-prompt.yaml b/lm_eval/tasks/super_glue/boolq/t5-prompt.yaml index b3ceb78d9..c705dbe6e 100644 --- a/lm_eval/tasks/super_glue/boolq/t5-prompt.yaml +++ b/lm_eval/tasks/super_glue/boolq/t5-prompt.yaml @@ -1,7 +1,7 @@ tag: - super-glue-t5-prompt task: super_glue-boolq-t5-prompt -dataset_path: super_glue +dataset_path: aps/super_glue dataset_name: boolq training_split: train validation_split: validation diff --git a/lm_eval/tasks/super_glue/cb/default.yaml b/lm_eval/tasks/super_glue/cb/default.yaml index 550635ed7..61177bc24 100644 --- a/lm_eval/tasks/super_glue/cb/default.yaml +++ b/lm_eval/tasks/super_glue/cb/default.yaml @@ -1,7 +1,7 @@ tag: - super-glue-lm-eval-v1 task: cb -dataset_path: super_glue +dataset_path: aps/super_glue dataset_name: cb output_type: multiple_choice training_split: train diff --git a/lm_eval/tasks/super_glue/cb/t5-prompt.yaml b/lm_eval/tasks/super_glue/cb/t5-prompt.yaml index 8b6f512ca..8ae2eb06b 100644 --- a/lm_eval/tasks/super_glue/cb/t5-prompt.yaml +++ b/lm_eval/tasks/super_glue/cb/t5-prompt.yaml @@ -1,7 +1,7 @@ tag: - super-glue-t5-prompt task: super_glue-cb-t5-prompt -dataset_path: super_glue +dataset_path: aps/super_glue dataset_name: cb training_split: train validation_split: validation diff --git a/lm_eval/tasks/super_glue/copa/default.yaml b/lm_eval/tasks/super_glue/copa/default.yaml index 9e2d1a2f0..fff4312ac 100644 --- a/lm_eval/tasks/super_glue/copa/default.yaml +++ b/lm_eval/tasks/super_glue/copa/default.yaml @@ -1,7 +1,7 @@ tag: - super-glue-lm-eval-v1 task: copa -dataset_path: super_glue +dataset_path: aps/super_glue dataset_name: copa output_type: multiple_choice training_split: train diff --git a/lm_eval/tasks/super_glue/copa/t5-prompt.yaml b/lm_eval/tasks/super_glue/copa/t5-prompt.yaml index 7358b9086..4c64c367c 100644 --- a/lm_eval/tasks/super_glue/copa/t5-prompt.yaml +++ b/lm_eval/tasks/super_glue/copa/t5-prompt.yaml @@ -1,7 +1,7 @@ tag: - super-glue-t5-prompt task: super_glue-copa-t5-prompt -dataset_path: super_glue +dataset_path: aps/super_glue dataset_name: copa training_split: train validation_split: validation diff --git a/lm_eval/tasks/super_glue/multirc/default.yaml b/lm_eval/tasks/super_glue/multirc/default.yaml index c14f7040d..42eba2e1f 100644 --- a/lm_eval/tasks/super_glue/multirc/default.yaml +++ b/lm_eval/tasks/super_glue/multirc/default.yaml @@ -1,7 +1,7 @@ tag: - super-glue-lm-eval-v1 task: multirc -dataset_path: super_glue +dataset_path: aps/super_glue dataset_name: multirc output_type: multiple_choice training_split: train diff --git a/lm_eval/tasks/super_glue/multirc/t5-prompt.yaml b/lm_eval/tasks/super_glue/multirc/t5-prompt.yaml index 566a65ccf..ac948e482 100644 --- a/lm_eval/tasks/super_glue/multirc/t5-prompt.yaml +++ b/lm_eval/tasks/super_glue/multirc/t5-prompt.yaml @@ -1,7 +1,7 @@ tag: - super-glue-t5-prompt task: super_glue-multirc-t5-prompt -dataset_path: super_glue +dataset_path: aps/super_glue dataset_name: multirc training_split: train validation_split: validation diff --git a/lm_eval/tasks/super_glue/record/default.yaml b/lm_eval/tasks/super_glue/record/default.yaml index 9dc924fc7..1d8dd1786 100644 --- a/lm_eval/tasks/super_glue/record/default.yaml +++ b/lm_eval/tasks/super_glue/record/default.yaml @@ -1,7 +1,7 @@ tag: - super-glue-lm-eval-v1 task: record -dataset_path: super_glue +dataset_path: aps/super_glue dataset_name: record output_type: multiple_choice training_split: train diff --git a/lm_eval/tasks/super_glue/record/t5-prompt.yaml b/lm_eval/tasks/super_glue/record/t5-prompt.yaml index f565171fc..d24df1fe4 100644 --- a/lm_eval/tasks/super_glue/record/t5-prompt.yaml +++ b/lm_eval/tasks/super_glue/record/t5-prompt.yaml @@ -1,7 +1,7 @@ tag: - super-glue-t5-prompt task: super_glue-record-t5-prompt -dataset_path: super_glue +dataset_path: aps/super_glue dataset_name: record validation_split: validation output_type: generate_until diff --git a/lm_eval/tasks/super_glue/rte/default.yaml b/lm_eval/tasks/super_glue/rte/default.yaml index b660f36dd..5ad0d3b9d 100644 --- a/lm_eval/tasks/super_glue/rte/default.yaml +++ b/lm_eval/tasks/super_glue/rte/default.yaml @@ -1,7 +1,7 @@ tag: - super-glue-lm-eval-v1 task: sglue_rte -dataset_path: super_glue +dataset_path: aps/super_glue dataset_name: rte output_type: multiple_choice training_split: train diff --git a/lm_eval/tasks/super_glue/rte/t5-prompt.yaml b/lm_eval/tasks/super_glue/rte/t5-prompt.yaml index 27caab0dd..4920563ee 100644 --- a/lm_eval/tasks/super_glue/rte/t5-prompt.yaml +++ b/lm_eval/tasks/super_glue/rte/t5-prompt.yaml @@ -1,7 +1,7 @@ tag: - super-glue-t5-prompt task: super_glue-rte-t5-prompt -dataset_path: super_glue +dataset_path: aps/super_glue dataset_name: rte training_split: train validation_split: validation diff --git a/lm_eval/tasks/super_glue/wic/default.yaml b/lm_eval/tasks/super_glue/wic/default.yaml index 4bb8ed59a..2aa3a6105 100644 --- a/lm_eval/tasks/super_glue/wic/default.yaml +++ b/lm_eval/tasks/super_glue/wic/default.yaml @@ -1,7 +1,7 @@ tag: - super-glue-lm-eval-v1 task: "wic" -dataset_path: super_glue +dataset_path: aps/super_glue dataset_name: wic output_type: multiple_choice training_split: train diff --git a/lm_eval/tasks/super_glue/wic/t5-prompt.yaml b/lm_eval/tasks/super_glue/wic/t5-prompt.yaml index b642a70be..8ea1fd735 100644 --- a/lm_eval/tasks/super_glue/wic/t5-prompt.yaml +++ b/lm_eval/tasks/super_glue/wic/t5-prompt.yaml @@ -1,7 +1,7 @@ tag: - super-glue-t5-prompt task: super_glue-wic-t5-prompt -dataset_path: super_glue +dataset_path: aps/super_glue dataset_name: wic training_split: train validation_split: validation diff --git a/lm_eval/tasks/super_glue/wsc/default.yaml b/lm_eval/tasks/super_glue/wsc/default.yaml index 2cd12679c..39abb7f30 100644 --- a/lm_eval/tasks/super_glue/wsc/default.yaml +++ b/lm_eval/tasks/super_glue/wsc/default.yaml @@ -1,7 +1,7 @@ tag: - super-glue-lm-eval-v1 task: wsc -dataset_path: super_glue +dataset_path: aps/super_glue dataset_name: wsc.fixed output_type: multiple_choice training_split: train diff --git a/lm_eval/tasks/super_glue/wsc/t5-prompt.yaml b/lm_eval/tasks/super_glue/wsc/t5-prompt.yaml index 77bfe7d0d..3ecc67d2a 100644 --- a/lm_eval/tasks/super_glue/wsc/t5-prompt.yaml +++ b/lm_eval/tasks/super_glue/wsc/t5-prompt.yaml @@ -1,7 +1,7 @@ tag: - super-glue-t5-prompt task: super_glue-wsc-t5-prompt -dataset_path: super_glue +dataset_path: aps/super_glue dataset_name: wsc.fixed training_split: train validation_split: validation