fix(kormedmcqa): extract answer choice per paper Appendix B (#3842)

KorMedMCQA computes exact_match on the raw generation, so any model that
answers verbosely (e.g. '정답: B') scores 0 even when correct. This adds the
answer-extraction regex cascade from Appendix B of the KorMedMCQA paper
(arXiv:2403.01469) as an extract_answer filter, wires it into the subtask
template and the group aggregate, and bumps the task version to 3.0. Unit
tests cover the extraction cascade and the filter. Fixes #3103.

lm_eval/tasks/kormedmcqa/README.md

@@ -34,6 +34,9 @@ Homepage: https://huggingface.co/datasets/sean0042/KorMedMCQA
 * `kormedmcqa_pharm`: `Official Korean Pharmacist Examination`
 * `kormedmcqa_dentist`: `Official Korean Dentist Examination`
 
+### Changelog
+- v3.0: added the answer-extraction filter from Appendix B of the paper, so verbose generations (e.g. `정답: B`) are scored on the extracted option letter instead of the raw text. Expect higher scores for models that do not answer with a bare letter.
+
 ### Checklist
 
 For adding novel benchmarks/datasets to the library:

lm_eval/tasks/kormedmcqa/_kormedmcqa.yaml

@@ -8,5 +8,6 @@ aggregate_metric_list:
   - metric: exact_match
     aggregation: mean
     weight_by_size: true
+    filter_list: extract_answer
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/kormedmcqa/_template_yaml

@@ -8,6 +8,11 @@ fewshot_config:
 output_type: generate_until
 doc_to_text: "{{question.strip()}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nE. {{E}}\n정답："
 doc_to_target: "{{['A', 'B', 'C', 'D', 'E'][answer-1]}}"
+filter_list:
+  - name: "extract_answer"
+    filter:
+      - function: !function utils.ExtractChoiceFilter
+      - function: "take_first"
 metric_list:
   - metric: exact_match
     aggregation: mean
@@ -28,4 +33,4 @@ generation_kwargs:
   temperature: 0.0
   max_gen_toks: 1024
 metadata:
-  version: 2.0
+  version: 3.0

lm_eval/tasks/kormedmcqa/utils.py

@@ -0,0 +1,40 @@
+"""Answer extraction for KorMedMCQA.
+
+Implements the answer-extraction cascade from Appendix B of the KorMedMCQA
+paper (https://arxiv.org/abs/2403.01469). Without it, ``exact_match`` is
+computed on the raw generation, so any model that answers verbosely
+(e.g. ``정답: B``) is scored 0 even when correct.
+"""
+
+import re
+
+from lm_eval.api.filter import Filter
+
+
+CHOICE_PATTERNS = [
+    re.compile(r"정답[:\s]*([ABCDE])", re.IGNORECASE),
+    re.compile(r"정답은\s*([ABCDE])\s*입니다", re.IGNORECASE),
+    re.compile(r"\b([ABCDE])\.", re.IGNORECASE),
+    re.compile(r"\b([ABCDE])\b", re.IGNORECASE),
+]
+
+
+def extract_choice(output: str) -> str:
+    """Extract the chosen option letter (A-E) from a model generation.
+
+    Tries each pattern of the paper's cascade in order and returns the last
+    match of the first pattern that matches, or an empty string if no pattern
+    matches.
+    """
+    for pattern in CHOICE_PATTERNS:
+        matches = pattern.findall(output)
+        if matches:
+            return matches[-1].upper()
+    return ""
+
+
+class ExtractChoiceFilter(Filter):
+    """Apply ``extract_choice`` to every model response."""
+
+    def apply(self, resps, docs):
+        return [[extract_choice(resp) for resp in r] for r in resps]

tests/test_kormedmcqa.py

@@ -0,0 +1,39 @@
+import pytest
+
+from lm_eval.tasks.kormedmcqa.utils import ExtractChoiceFilter, extract_choice
+
+
+@pytest.mark.parametrize(
+    "output, expected",
+    [
+        (" B", "B"),
+        ("B", "B"),
+        ("b", "B"),
+        ("정답: B", "B"),
+        ("정답:B", "B"),
+        ("정답 C", "C"),
+        ("**정답: B**", "B"),
+        ("정답은 D 입니다", "D"),
+        ("정답은 D입니다", "D"),
+        ("B.", "B"),
+        ("The answer is (C).", "C"),
+        # last match wins within a pattern, per the paper's reference code
+        ("정답: B\n정답: C", "C"),
+        # earlier patterns in the cascade take precedence over later ones
+        ("A. foo\nB. bar\n정답: E", "E"),
+        # no extractable answer
+        ("잘 모르겠습니다", ""),
+        ("", ""),
+    ],
+)
+def test_kormedmcqa_extract_choice(output, expected):
+    assert extract_choice(output) == expected
+
+
+def test_kormedmcqa_extract_choice_filter():
+    filt = ExtractChoiceFilter()
+
+    resps = [["정답: C"], ["정답은 A 입니다"], ["모르겠어요"]]
+    docs = [{}, {}, {}]
+
+    assert filt.apply(resps, docs) == [["C"], ["A"], [""]]