diff --git a/lmms_eval/api/reasoning.py b/lmms_eval/api/reasoning.py index 1268ffdc0..4fc638d6d 100644 --- a/lmms_eval/api/reasoning.py +++ b/lmms_eval/api/reasoning.py @@ -12,6 +12,11 @@ def strip_reasoning_tags(text: str, tag_pairs: List[List[str]]) -> str: Returns: Cleaned text with reasoning blocks removed. + + Note: + Tag matching is case-sensitive (```` is not treated as ````), + and an unclosed opening tag with no matching closing tag is left untouched + and passes through unstripped. """ result = text for start_tag, end_tag in tag_pairs: diff --git a/lmms_eval/api/task.py b/lmms_eval/api/task.py index ec1a94084..a74e81f3d 100755 --- a/lmms_eval/api/task.py +++ b/lmms_eval/api/task.py @@ -154,6 +154,11 @@ class TaskConfig(dict): model_specific_generation_kwargs: dict = None model_specific_target_kwargs: dict = None reasoning_tags: Union[str, list] = None + # Opt-in. When True, a StripThinkingFilter is prepended to every filter + # pipeline so / reasoning blocks are removed before answer + # extraction. Defaults False so scoring inputs for existing tasks are + # unchanged; ignored when `reasoning_tags` is configured (that path already strips). + auto_strip_thinking: bool = False def __post_init__(self) -> None: if self.dataset_path and os.path.exists(os.path.dirname(self.dataset_path)): diff --git a/lmms_eval/evaluator.py b/lmms_eval/evaluator.py index 2ec3e68c6..94dff2014 100755 --- a/lmms_eval/evaluator.py +++ b/lmms_eval/evaluator.py @@ -1053,6 +1053,31 @@ def _infer_task_request_type(task_obj: Task) -> Optional[str]: ### Postprocess outputs ### # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately) + + # When a task opts in via `auto_strip_thinking`, prepend a StripThinkingFilter to + # the FRONT of each existing filter ensemble's chain so answer-extraction filters + # (take_first / regex / multi_choice_regex) receive text with the / + # reasoning blocks already removed. We deliberately do NOT add a sibling ensemble: + # FilterEnsembles do not chain (each reads raw inst.resps and writes its own + # filtered_resps key), so a sibling would leave extraction filters seeing un-stripped + # text and would create a new scored key holding an unselected list. + from lmms_eval.filters.transformation import StripThinkingFilter + + for task_output in eval_tasks: + task = task_output.task + if not hasattr(task, "_filters"): + continue + if not getattr(getattr(task, "config", None), "auto_strip_thinking", False): + continue + # Skip when reasoning_tags is already configured (task or CLI): the scoring loop + # below strips those blocks, so auto-stripping here would double-strip. + cli_reasoning_tags = getattr(cli_args, "reasoning_tags", None) if cli_args else None + task_reasoning_tags = getattr(task.config, "reasoning_tags", None) + if parse_reasoning_tags_config(cli_value=cli_reasoning_tags, task_value=task_reasoning_tags) is not None: + continue + for ensemble in task._filters: + ensemble.filters.insert(0, StripThinkingFilter()) + for task_output in eval_tasks: task = task_output.task task.apply_filters() diff --git a/lmms_eval/filters/__init__.py b/lmms_eval/filters/__init__.py index f6c353f52..5d25bf8f7 100755 --- a/lmms_eval/filters/__init__.py +++ b/lmms_eval/filters/__init__.py @@ -12,6 +12,7 @@ "uppercase": transformation.UppercaseFilter, "map": transformation.MapFilter, "multi_choice_regex": extraction.MultiChoiceRegexFilter, + "strip_thinking": transformation.StripThinkingFilter, # TODO: implement this filter. either it should take in an arbitrary "scoring"/reward function # that takes an input and returns a scalar and then should select the max reward, # or should implement different filters for different ways of handling a reward model's inference. diff --git a/lmms_eval/filters/transformation.py b/lmms_eval/filters/transformation.py index 9842115f9..e2709b3aa 100755 --- a/lmms_eval/filters/transformation.py +++ b/lmms_eval/filters/transformation.py @@ -1,4 +1,39 @@ from lmms_eval.api.filter import Filter +from lmms_eval.api.reasoning import strip_reasoning_tags + +# Default tag pairs for reasoning models. +_DEFAULT_TAG_PAIRS = [ + ["", ""], + ["", ""], +] + + +class StripThinkingFilter(Filter): + """Strip reasoning/thinking blocks from model responses. + + Delegates to :func:`lmms_eval.api.reasoning.strip_reasoning_tags` which + handles both full ``...`` blocks and the common case where + the chat template injects the opening tag as a prompt prefix (so only + the closing tag appears in the generated text). + + This filter should run **before** answer-extraction filters (regex, etc.) + so they see the clean answer text, not the 50 K reasoning chain. + + Args: + tag_pairs: list of ``[open_tag, close_tag]`` pairs. + Defaults to ``[["", ""], ["", ""]]``. + """ + + def __init__(self, tag_pairs: list = None) -> None: + self.tag_pairs = tag_pairs or _DEFAULT_TAG_PAIRS + + def apply(self, resps, docs): + def strip_one(text): + if not isinstance(text, str): + return text + return strip_reasoning_tags(text, self.tag_pairs) + + return [[strip_one(r) for r in inst] for inst in resps] class LowercaseFilter(Filter): diff --git a/test/eval/test_strip_thinking_filter.py b/test/eval/test_strip_thinking_filter.py new file mode 100644 index 000000000..edd02a0b9 --- /dev/null +++ b/test/eval/test_strip_thinking_filter.py @@ -0,0 +1,89 @@ +"""End-to-end tests for chaining StripThinkingFilter into task filter pipelines. + +When a task sets ``auto_strip_thinking``, the evaluator prepends a +``StripThinkingFilter`` to the FRONT of each existing ``FilterEnsemble``'s chain +(it does NOT add a sibling ensemble). These tests exercise that wiring at the +ensemble level and assert the two properties the integration must guarantee: + + (a) the scored/default filter key holds the STRIPPED string that the + extraction filters selected -- not a list, and not the un-stripped text; and + (b) no extra ``"strip_thinking"`` filter key is created (a sibling ensemble + would have produced one, holding an unselected list that later crashes + string-based ``process_results``). + +See ``lmms_eval/evaluator.py`` (auto_strip_thinking wiring) and +``lmms_eval/filters/transformation.py`` (StripThinkingFilter). +""" + +from lmms_eval.api.instance import Instance +from lmms_eval.filters import build_filter_ensemble +from lmms_eval.filters.transformation import StripThinkingFilter + + +def _make_instance(resps, idx=0, doc_id=0): + inst = Instance( + request_type="generate_until", + arguments=("prompt", {}, None, doc_id, "test_task", "test"), + idx=idx, + metadata={"task": "test_task", "doc_id": doc_id, "repeats": 1}, + ) + inst.resps = resps + return inst + + +def _prepend_strip(ensemble): + """Mirror the evaluator's auto_strip_thinking wiring: strip at the front.""" + ensemble.filters.insert(0, StripThinkingFilter()) + return ensemble + + +def test_strip_thinking_chained_before_take_first_yields_stripped_string(): + # Minimal task pipeline: a single take_first selection step named "default". + ensemble = _prepend_strip(build_filter_ensemble("default", [["take_first", None]])) + + inst = _make_instance(["long chain of reasoning\n\nParis"]) + ensemble.apply([inst], docs=[None]) + + # (a) The scored/default key holds the STRIPPED string -- not a list, and not + # the un-stripped "...Paris". + assert inst.filtered_resps["default"] == "Paris" + assert isinstance(inst.filtered_resps["default"], str) + + # (b) No sibling "strip_thinking" key is created; "default" is the only key. + assert "strip_thinking" not in inst.filtered_resps + assert list(inst.filtered_resps.keys()) == ["default"] + + +def test_strip_thinking_runs_before_regex_extraction(): + # Realistic flexible-extract pipeline: regex extraction, then take_first. + ensemble = _prepend_strip( + build_filter_ensemble( + "flexible-extract", + [ + ["regex", {"regex_pattern": r"answer is \(?([A-D])\)?"}], + ["take_first", None], + ], + ) + ) + + # The reasoning block holds a DECOY "answer is (A)" that must be stripped + # before the regex runs; the real answer after is (C). If the strip + # did not run first (the old sibling-ensemble bug), regex would see both and + # extract the decoy "A". + inst = _make_instance(["maybe the answer is (A)? The answer is (C)"]) + ensemble.apply([inst], docs=[None]) + + assert inst.filtered_resps["flexible-extract"] == "C" + assert "strip_thinking" not in inst.filtered_resps + + +def test_strip_thinking_is_noop_on_plain_text(): + # Non-reasoning output passes through unchanged (aside from take_first selection), + # so the filter is safe to prepend for non-reasoning models. + ensemble = _prepend_strip(build_filter_ensemble("default", [["take_first", None]])) + + inst = _make_instance(["Just a plain answer."]) + ensemble.apply([inst], docs=[None]) + + assert inst.filtered_resps["default"] == "Just a plain answer." + assert "strip_thinking" not in inst.filtered_resps