Fix IFEval correctness bugs in if_functions and IFEvalVerifier (allenai#1683)

* Bundle small correctness fixes from allenai#1615/allenai#1618/allenai#1619/allenai#1623/allenai#1625/allenai#1646/allenai#1651/allenai#1655 with regression tests Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

* Address PR review: inline write_header, pricing per 1M tokens, parameterize tests, dedupe if_functions Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

* Trim branch to IFEval-only changes; other fixes moved to allenai#1684/allenai#1685/allenai#1686 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

* Address review: use word-boundary regex in validate_choice to avoid substring false positives Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

* Revert deletion of scripts/eval_constraints/if_functions.py (handled in separate PR) Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

* Revert validate_choice regex change; add TODO comment with suggested word-boundary fix Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

* Update if_functions.py

Author: finbarrtimbers

CHANGELOG.md

@@ -38,6 +38,7 @@ All notable changes to this project will be documented in this file.
 - Add deprecation warning to `finetune.py` pointing users to the OLMo-core SFT implementation (https://github.com/allenai/open-instruct/pull/1574).
 
 ### Fixed
+- Bundle IFEval correctness fixes: `validate_choice` operand direction and `validate_frequency_capital_words` "around" tolerance in `open_instruct/if_functions.py`; `IFEvalVerifier` `ZeroDivisionError` when the instruction list is empty; deduplicate by deleting the unused `scripts/eval_constraints/if_functions.py` copy (supersedes #1615, #1646, #1655).
 - Fix NameError on `streaming_config/vllm_config` when importing from `grpo_fast.py` due to implicit global dependency. (https://github.com/allenai/open-instruct/pull/1675)
 - Fix `grpo.py` token-count inflation by emitting bool `response_masks` from `DataPreparationActor` (instead of int64 doc-id-valued masks) and dropping per-consumer `.bool()` coercions in `grpo_fast.py`, `grpo_utils.py`, and `olmo_core_train_modules.py`. Previously the OLMo-core path summed the doc-id-valued mask in `calculate_token_counts`, inflating `loss_denominator` by ~60× (https://github.com/allenai/open-instruct/pull/1668).
 - Fix hardcoded project version (https://github.com/allenai/open-instruct/pull/1636) by using setuptools scm's automatic versioning.

open_instruct/ground_truth_utils.py

@@ -351,7 +351,7 @@ def __call__(
                 rewards.append(1.0)
             else:
                 rewards.append(0.0)
-        return VerificationResult(score=sum(rewards) / len(rewards))
+        return VerificationResult(score=sum(rewards) / max(len(rewards), 1))
 
 
 class IFEvalVerifierOld(VerifierFunction):

open_instruct/if_functions.py

@@ -360,7 +360,9 @@ def validate_title(text: str) -> bool:
 
 # Choose: From Answer with one of the following options: {options}
 def validate_choice(text: str, options: list) -> bool:
-    return any(text in option for option in options)
+    # substring matching yields false positives (e.g. option "A" matches "Apple"); if this
+    # is a problem, switch to a regex, e.g. re.search(rf"(?<!\w){re.escape(str(option))}(?!\w)", text).
+    return any(option in text for option in options)
 
 
 # Minimum Number Highlighted Section: Highlight at least {N} sections in your answer with markdown, i.e. *highlighted
@@ -430,7 +432,7 @@ def validate_frequency_capital_words(text: str, N: int, quantifier: str) -> bool
     if quantifier == "at least":
         return len(words) >= N
     elif quantifier == "around":
-        return len(words) == N
+        return abs(len(words) - N) <= max(round(N * 0.1), 1)
     elif quantifier == "at most":
         return len(words) <= N
     else:

open_instruct/test_ground_truth_utils.py

@@ -10,6 +10,7 @@
 
 from parameterized import parameterized
 
+from open_instruct import ground_truth_utils
 from open_instruct.ground_truth_utils import (
     F1Verifier,
     GSM8KVerifier,
@@ -229,5 +230,16 @@ def test_cleanup_helpers_are_safe_noops(self):
         self.assertIsNone(asyncio.run(cleanup_all_llm_judge_clients()))
 
 
+class TestIFEvalVerifierEmptyInstructions(unittest.TestCase):
+    """Regression test for PR #1655: IFEvalVerifier crashed with
+    ZeroDivisionError when the constraint's instruction_id list was empty."""
+
+    def test_empty_instruction_list_returns_zero_score(self):
+        verifier = ground_truth_utils.IFEvalVerifier()
+        label = str([{"instruction_id": [], "kwargs": []}])
+        result = verifier(tokenized_prediction=[1, 2, 3], prediction="some non-empty response", label=label)
+        self.assertEqual(result.score, 0.0)
+
+
 if __name__ == "__main__":
     unittest.main()

open_instruct/test_if_functions.py

@@ -0,0 +1,43 @@
+"""Tests for IFEval constraint verifiers in open_instruct.if_functions.
+
+Covers regression fixes from PRs #1615 / #1646 (validate_choice operand
+direction) and #1646 (validate_frequency_capital_words "around" tolerance).
+"""
+
+import unittest
+
+from parameterized import parameterized
+
+from open_instruct import if_functions
+
+
+class TestValidateChoice(unittest.TestCase):
+    @parameterized.expand(
+        [
+            ("option_in_text", "I believe the answer is B", ["A", "B", "C", "D"], True),
+            ("option_appears", "I choose red", ["red", "blue"], True),
+            ("no_option_present", "I choose green", ["red", "blue"], False),
+            ("exact_match", "red", ["red", "blue"], True),
+        ]
+    )
+    def test_validate_choice(self, _name, text, options, expected):
+        self.assertEqual(if_functions.validate_choice(text, options), expected)
+
+
+class TestValidateFrequencyCapitalWords(unittest.TestCase):
+    @parameterized.expand(
+        [
+            ("around_exact", "AAA BBB CCC DDD EEE", 5, "around", True),
+            ("around_off_by_one_low", "AAA BBB CCC DDD", 5, "around", True),
+            ("around_off_by_one_high", "AAA BBB CCC DDD EEE FFF", 5, "around", True),
+            ("around_far_off", "AAA BBB", 5, "around", False),
+            ("at_least_satisfied", "AAA BBB CCC", 2, "at least", True),
+            ("at_most_violated", "AAA BBB CCC DDD", 2, "at most", False),
+        ]
+    )
+    def test_validate_frequency_capital_words(self, _name, text, n, quantifier, expected):
+        self.assertEqual(if_functions.validate_frequency_capital_words(text, n, quantifier), expected)
+
+
+if __name__ == "__main__":
+    unittest.main()