feat: auto-strip thinking blocks from generations before scoring

lmms_eval/api/reasoning.py

@@ -12,6 +12,11 @@ def strip_reasoning_tags(text: str, tag_pairs: List[List[str]]) -> str:
 
     Returns:
         Cleaned text with reasoning blocks removed.
+
+    Note:
+        Tag matching is case-sensitive (``<THINK>`` is not treated as ``<think>``),
+        and an unclosed opening tag with no matching closing tag is left untouched
+        and passes through unstripped.
     """
     result = text
     for start_tag, end_tag in tag_pairs:

lmms_eval/api/task.py

@@ -154,6 +154,11 @@ class TaskConfig(dict):
     model_specific_generation_kwargs: dict = None
     model_specific_target_kwargs: dict = None
     reasoning_tags: Union[str, list] = None
+    # Opt-in. When True, a StripThinkingFilter is prepended to every filter
+    # pipeline so <think>/<thinking> reasoning blocks are removed before answer
+    # extraction. Defaults False so scoring inputs for existing tasks are
+    # unchanged; ignored when `reasoning_tags` is configured (that path already strips).
+    auto_strip_thinking: bool = False
 
     def __post_init__(self) -> None:
         if self.dataset_path and os.path.exists(os.path.dirname(self.dataset_path)):

lmms_eval/evaluator.py

@@ -1053,6 +1053,31 @@ def _infer_task_request_type(task_obj: Task) -> Optional[str]:
 
     ### Postprocess outputs ###
     # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately)
+
+    # When a task opts in via `auto_strip_thinking`, prepend a StripThinkingFilter to
+    # the FRONT of each existing filter ensemble's chain so answer-extraction filters
+    # (take_first / regex / multi_choice_regex) receive text with the <think>/<thinking>
+    # reasoning blocks already removed. We deliberately do NOT add a sibling ensemble:
+    # FilterEnsembles do not chain (each reads raw inst.resps and writes its own
+    # filtered_resps key), so a sibling would leave extraction filters seeing un-stripped
+    # text and would create a new scored key holding an unselected list.
+    from lmms_eval.filters.transformation import StripThinkingFilter
+
+    for task_output in eval_tasks:
+        task = task_output.task
+        if not hasattr(task, "_filters"):
+            continue
+        if not getattr(getattr(task, "config", None), "auto_strip_thinking", False):
+            continue
+        # Skip when reasoning_tags is already configured (task or CLI): the scoring loop
+        # below strips those blocks, so auto-stripping here would double-strip.
+        cli_reasoning_tags = getattr(cli_args, "reasoning_tags", None) if cli_args else None
+        task_reasoning_tags = getattr(task.config, "reasoning_tags", None)
+        if parse_reasoning_tags_config(cli_value=cli_reasoning_tags, task_value=task_reasoning_tags) is not None:
+            continue
+        for ensemble in task._filters:
+            ensemble.filters.insert(0, StripThinkingFilter())
+
     for task_output in eval_tasks:
         task = task_output.task
         task.apply_filters()

lmms_eval/filters/__init__.py

@@ -12,6 +12,7 @@
     "uppercase": transformation.UppercaseFilter,
     "map": transformation.MapFilter,
     "multi_choice_regex": extraction.MultiChoiceRegexFilter,
+    "strip_thinking": transformation.StripThinkingFilter,
     # TODO: implement this filter. either it should take in an arbitrary "scoring"/reward function
     # that takes an input and returns a scalar and then should select the max reward,
     # or should implement different filters for different ways of handling a reward model's inference.

lmms_eval/filters/transformation.py

@@ -1,4 +1,39 @@
 from lmms_eval.api.filter import Filter
+from lmms_eval.api.reasoning import strip_reasoning_tags
+
+# Default tag pairs for reasoning models.
+_DEFAULT_TAG_PAIRS = [
+    ["<think>", "</think>"],
+    ["<thinking>", "</thinking>"],
+]
+
+
+class StripThinkingFilter(Filter):
+    """Strip reasoning/thinking blocks from model responses.
+
+    Delegates to :func:`lmms_eval.api.reasoning.strip_reasoning_tags` which
+    handles both full ``<think>...</think>`` blocks and the common case where
+    the chat template injects the opening tag as a prompt prefix (so only
+    the closing tag appears in the generated text).
+
+    This filter should run **before** answer-extraction filters (regex, etc.)
+    so they see the clean answer text, not the 50 K reasoning chain.
+
+    Args:
+        tag_pairs: list of ``[open_tag, close_tag]`` pairs.
+            Defaults to ``[["<think>", "</think>"], ["<thinking>", "</thinking>"]]``.
+    """
+
+    def __init__(self, tag_pairs: list = None) -> None:
+        self.tag_pairs = tag_pairs or _DEFAULT_TAG_PAIRS
+
+    def apply(self, resps, docs):
+        def strip_one(text):
+            if not isinstance(text, str):
+                return text
+            return strip_reasoning_tags(text, self.tag_pairs)
+
+        return [[strip_one(r) for r in inst] for inst in resps]
 
 
 class LowercaseFilter(Filter):

test/eval/test_strip_thinking_filter.py

@@ -0,0 +1,89 @@
+"""End-to-end tests for chaining StripThinkingFilter into task filter pipelines.
+
+When a task sets ``auto_strip_thinking``, the evaluator prepends a
+``StripThinkingFilter`` to the FRONT of each existing ``FilterEnsemble``'s chain
+(it does NOT add a sibling ensemble). These tests exercise that wiring at the
+ensemble level and assert the two properties the integration must guarantee:
+
+  (a) the scored/default filter key holds the STRIPPED string that the
+      extraction filters selected -- not a list, and not the un-stripped text; and
+  (b) no extra ``"strip_thinking"`` filter key is created (a sibling ensemble
+      would have produced one, holding an unselected list that later crashes
+      string-based ``process_results``).
+
+See ``lmms_eval/evaluator.py`` (auto_strip_thinking wiring) and
+``lmms_eval/filters/transformation.py`` (StripThinkingFilter).
+"""
+
+from lmms_eval.api.instance import Instance
+from lmms_eval.filters import build_filter_ensemble
+from lmms_eval.filters.transformation import StripThinkingFilter
+
+
+def _make_instance(resps, idx=0, doc_id=0):
+    inst = Instance(
+        request_type="generate_until",
+        arguments=("prompt", {}, None, doc_id, "test_task", "test"),
+        idx=idx,
+        metadata={"task": "test_task", "doc_id": doc_id, "repeats": 1},
+    )
+    inst.resps = resps
+    return inst
+
+
+def _prepend_strip(ensemble):
+    """Mirror the evaluator's auto_strip_thinking wiring: strip at the front."""
+    ensemble.filters.insert(0, StripThinkingFilter())
+    return ensemble
+
+
+def test_strip_thinking_chained_before_take_first_yields_stripped_string():
+    # Minimal task pipeline: a single take_first selection step named "default".
+    ensemble = _prepend_strip(build_filter_ensemble("default", [["take_first", None]]))
+
+    inst = _make_instance(["<think>long chain of reasoning</think>\n\nParis"])
+    ensemble.apply([inst], docs=[None])
+
+    # (a) The scored/default key holds the STRIPPED string -- not a list, and not
+    #     the un-stripped "<think>...</think>Paris".
+    assert inst.filtered_resps["default"] == "Paris"
+    assert isinstance(inst.filtered_resps["default"], str)
+
+    # (b) No sibling "strip_thinking" key is created; "default" is the only key.
+    assert "strip_thinking" not in inst.filtered_resps
+    assert list(inst.filtered_resps.keys()) == ["default"]
+
+
+def test_strip_thinking_runs_before_regex_extraction():
+    # Realistic flexible-extract pipeline: regex extraction, then take_first.
+    ensemble = _prepend_strip(
+        build_filter_ensemble(
+            "flexible-extract",
+            [
+                ["regex", {"regex_pattern": r"answer is \(?([A-D])\)?"}],
+                ["take_first", None],
+            ],
+        )
+    )
+
+    # The reasoning block holds a DECOY "answer is (A)" that must be stripped
+    # before the regex runs; the real answer after </think> is (C). If the strip
+    # did not run first (the old sibling-ensemble bug), regex would see both and
+    # extract the decoy "A".
+    inst = _make_instance(["<think>maybe the answer is (A)?</think> The answer is (C)"])
+    ensemble.apply([inst], docs=[None])
+
+    assert inst.filtered_resps["flexible-extract"] == "C"
+    assert "strip_thinking" not in inst.filtered_resps
+
+
+def test_strip_thinking_is_noop_on_plain_text():
+    # Non-reasoning output passes through unchanged (aside from take_first selection),
+    # so the filter is safe to prepend for non-reasoning models.
+    ensemble = _prepend_strip(build_filter_ensemble("default", [["take_first", None]]))
+
+    inst = _make_instance(["Just a plain answer."])
+    ensemble.apply([inst], docs=[None])
+
+    assert inst.filtered_resps["default"] == "Just a plain answer."
+    assert "strip_thinking" not in inst.filtered_resps