Trim branch to IFEval-only changes; other fixes moved to #1684/#1685/#1686 Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: finbarrtimbers

CHANGELOG.md

@@ -38,7 +38,7 @@ All notable changes to this project will be documented in this file.
 - Add deprecation warning to `finetune.py` pointing users to the OLMo-core SFT implementation (https://github.com/allenai/open-instruct/pull/1574).
 
 ### Fixed
-- Bundle small correctness fixes: `validate_choice` operand direction and `validate_frequency_capital_words` "around" tolerance in `open_instruct/if_functions.py` and `scripts/eval_constraints/if_functions.py`; gpt-4o / gpt-4o-standard output price in `judge_utils.py` (10× undercount); CSV header handling in `save_benchmark_results_to_csv` and `save_completion_lengths`; `_get_batch_logps` division-by-zero on fully-masked sequences in `dpo_utils.py`; `IFEvalVerifier` `ZeroDivisionError` when the instruction list is empty (supersedes #1615, #1618, #1619, #1623, #1625, #1646, #1651, #1655).
+- Bundle IFEval correctness fixes: `validate_choice` operand direction and `validate_frequency_capital_words` "around" tolerance in `open_instruct/if_functions.py`; `IFEvalVerifier` `ZeroDivisionError` when the instruction list is empty; deduplicate by deleting the unused `scripts/eval_constraints/if_functions.py` copy (supersedes #1615, #1646, #1655).
 - Fix NameError on `streaming_config/vllm_config` when importing from `grpo_fast.py` due to implicit global dependency. (https://github.com/allenai/open-instruct/pull/1675)
 - Fix `grpo.py` token-count inflation by emitting bool `response_masks` from `DataPreparationActor` (instead of int64 doc-id-valued masks) and dropping per-consumer `.bool()` coercions in `grpo_fast.py`, `grpo_utils.py`, and `olmo_core_train_modules.py`. Previously the OLMo-core path summed the doc-id-valued mask in `calculate_token_counts`, inflating `loss_denominator` by ~60× (https://github.com/allenai/open-instruct/pull/1668).
 - Fix hardcoded project version (https://github.com/allenai/open-instruct/pull/1636) by using setuptools scm's automatic versioning.

open_instruct/benchmark_generators.py

@@ -57,8 +57,7 @@ def save_completion_lengths(batch_results: list[dict], timestamp: int, batch_idx
     with open(csv_path, "a", newline="") as csvfile:
         fieldnames = ["batch_num", "prompt_num", "completion_length"]
         writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
-        if csvfile.tell() == 0:
-            writer.writeheader()
+        writer.writeheader()
 
         for batch_result in batch_results:
             response_lengths = batch_result["response_lengths"]
@@ -143,7 +142,7 @@ def save_benchmark_results_to_csv(
 
     with csv_path.open("a", newline="") as csvfile:
         writer = csv.DictWriter(csvfile, fieldnames=list(row_data.keys()))
-        if csvfile.tell() == 0:
+        if not csv_path.exists():
             writer.writeheader()
         writer.writerow(row_data)
     logger.info(f"Saved benchmark results to {csv_path}")

open_instruct/dpo_utils.py

@@ -707,7 +707,7 @@ def _get_batch_logps(
     loss_mask = labels[:, 1:] != -100
 
     if average_log_prob:
-        return (per_token_logps * loss_mask).sum(-1) / loss_mask.sum(-1).clamp(min=1)
+        return (per_token_logps * loss_mask).sum(-1) / loss_mask.sum(-1)
     else:
         return (per_token_logps * loss_mask).sum(-1)
 

open_instruct/ground_truth_utils.py

@@ -26,7 +26,7 @@
 from open_instruct import context_window_checker, logger_utils
 from open_instruct.if_functions import IF_FUNCTIONS_MAP
 from open_instruct.IFEvalG import instructions_registry
-from open_instruct.judge_utils import EXTRACTOR_MAP, JUDGE_PROMPT_MAP, PRICE_PER_MILLION_TOKENS, build_messages
+from open_instruct.judge_utils import EXTRACTOR_MAP, JUDGE_PROMPT_MAP, PRICE_PER_TOKEN, build_messages
 from open_instruct.math_utils import (
     get_unnormalized_answer,
     hendrycks_is_equiv,
@@ -743,9 +743,9 @@ def get_cost(self, response, model: str):
         model_name = model.split("/")[-1]  # for litellm, discard the namespace
         model_name = model_name.replace("-standard", "")  # azure OAI models have -standard in the name
         return (
-            PRICE_PER_MILLION_TOKENS.get(model_name, {}).get("input", 0) * response.usage.prompt_tokens
-            + PRICE_PER_MILLION_TOKENS.get(model_name, {}).get("output", 0) * response.usage.completion_tokens
-        ) / 1_000_000
+            PRICE_PER_TOKEN.get(model_name, {}).get("input", 0) * response.usage.prompt_tokens
+            + PRICE_PER_TOKEN.get(model_name, {}).get("output", 0) * response.usage.completion_tokens
+        )
 
     async def async_call(
         self,

open_instruct/judge_utils.py

@@ -5,24 +5,24 @@
 
 logger = logger_utils.setup_logger(__name__)
 
-PRICE_PER_MILLION_TOKENS = {
-    "gpt-4": {"input": 30.0, "output": 60.0},
-    "gpt-3.5-turbo": {"input": 1.5, "output": 2.0},
-    "gpt-4-1106-preview": {"input": 10.0, "output": 30.0},
-    "gpt-4o": {"input": 2.5, "output": 10.0},
-    "gpt-4o-mini": {"input": 0.15, "output": 0.6},
-    "gpt-4o-standard": {"input": 2.5, "output": 10.0},
-    "gpt-4.1": {"input": 2.0, "output": 8.0},
-    "gpt-4.1-standard": {"input": 2.0, "output": 8.0},
-    "gpt-4.1-mini-standard": {"input": 0.4, "output": 1.6},
-    "o3": {"input": 10.0, "output": 40.0},
-    "o3-standard": {"input": 10.0, "output": 40.0},
-    "claude-sonnet": {"input": 3.0, "output": 15.0},
-    "deepseek-chat": {"input": 0.07, "output": 1.0},
-    "deepseek-reasoner": {"input": 0.14, "output": 2.0},
-    "claude-3-7-sonnet-20250219": {"input": 3.0, "output": 15.0},
-    "minimax-m2.7": {"input": 0.3, "output": 1.2},
-    "minimax-m2.7-highspeed": {"input": 0.6, "output": 2.4},
+PRICE_PER_TOKEN = {
+    "gpt-4": {"input": 0.00003, "output": 0.00006},
+    "gpt-3.5-turbo": {"input": 0.0000015, "output": 0.000002},
+    "gpt-4-1106-preview": {"input": 0.00001, "output": 0.00003},
+    "gpt-4o": {"input": 0.0000025, "output": 0.000001},
+    "gpt-4o-mini": {"input": 0.00000015, "output": 0.0000006},
+    "gpt-4o-standard": {"input": 0.0000025, "output": 0.000001},
+    "gpt-4.1": {"input": 0.000002, "output": 0.000008},
+    "gpt-4.1-standard": {"input": 0.000002, "output": 0.000008},
+    "gpt-4.1-mini-standard": {"input": 0.0000004, "output": 0.0000016},
+    "o3": {"input": 0.00001, "output": 0.00004},
+    "o3-standard": {"input": 0.00001, "output": 0.00004},
+    "claude-sonnet": {"input": 0.000003, "output": 0.000015},
+    "deepseek-chat": {"input": 0.00000007, "output": 0.000001},
+    "deepseek-reasoner": {"input": 0.00000014, "output": 0.000002},
+    "claude-3-7-sonnet-20250219": {"input": 0.000003, "output": 0.000015},
+    "minimax-m2.7": {"input": 0.0000003, "output": 0.0000012},
+    "minimax-m2.7-highspeed": {"input": 0.0000006, "output": 0.0000024},
 }
 
 # Define the templates for different judge types

open_instruct/test_benchmark_generators.py

@@ -1,12 +1,8 @@
-import csv
-import pathlib
-import tempfile
 import unittest
-from unittest import mock
 
 import parameterized
 
-from open_instruct import benchmark_generators, utils
+from open_instruct import utils
 
 
 class TestBenchmark(unittest.TestCase):
@@ -18,78 +14,5 @@ def test_get_device_name(self, device_name, expected):
         self.assertEqual(result, expected)
 
 
-class TestSaveCompletionLengths(unittest.TestCase):
-    """Regression test for PR #1623: duplicate headers on every batch."""
-
-    def test_header_written_once_across_batches(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            tmp_path = pathlib.Path(tmp)
-            timestamp = 1234567890
-            batches = [
-                [{"response_lengths": [10, 20]}],
-                [{"response_lengths": [30]}],
-                [{"response_lengths": [40, 50, 60]}],
-            ]
-            with mock.patch.object(benchmark_generators, "DATA_DIR", tmp_path):
-                for batch_idx, batch_results in enumerate(batches):
-                    benchmark_generators.save_completion_lengths(batch_results, timestamp, batch_idx)
-
-            csv_path = tmp_path / f"completion_lengths_{timestamp}.csv"
-            with csv_path.open() as f:
-                rows = list(csv.reader(f))
-
-            self.assertEqual(rows[0], ["batch_num", "prompt_num", "completion_length"])
-            for row in rows[1:]:
-                self.assertNotEqual(row, ["batch_num", "prompt_num", "completion_length"])
-            self.assertEqual(len(rows), 1 + 2 + 1 + 3)
-
-
-class TestSaveBenchmarkResultsToCsv(unittest.TestCase):
-    """Regression test for PR #1619: header never written because the
-    Path('a') open call created the file before the existence check."""
-
-    def _streaming_config(self):
-        return mock.MagicMock(num_unique_prompts_rollout=4, num_samples_per_prompt_rollout=2, response_length=128)
-
-    def _model_config(self):
-        return mock.MagicMock(model_name_or_path="fake/model")
-
-    def _agg(self):
-        return {
-            "total_generation_time": 1.0,
-            "total_weight_sync_time": 0.5,
-            "total_num_new_tokens": 100,
-            "avg_tokens_per_second": 50.0,
-            "avg_mfu": 0.1,
-            "avg_mbu": 0.2,
-            "avg_generation_time": 0.5,
-            "avg_weight_sync_time": 0.25,
-        }
-
-    def test_header_written_exactly_once_across_calls(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            tmp_path = pathlib.Path(tmp)
-            with (
-                mock.patch.object(benchmark_generators, "DATA_DIR", tmp_path),
-                mock.patch.object(benchmark_generators, "aggregate_results", return_value=self._agg()),
-                mock.patch.object(benchmark_generators.utils, "get_git_commit", return_value="abc"),
-            ):
-                for _ in range(3):
-                    benchmark_generators.save_benchmark_results_to_csv(
-                        results=[{}],
-                        total_time=10.0,
-                        streaming_config=self._streaming_config(),
-                        model_config=self._model_config(),
-                    )
-
-            csv_path = tmp_path / "generator_benchmark_results.csv"
-            with csv_path.open() as f:
-                rows = list(csv.reader(f))
-            self.assertEqual(rows[0][0], "git_commit")
-            self.assertEqual(len(rows), 4)
-            for row in rows[1:]:
-                self.assertEqual(row[0], "abc")
-
-
 if __name__ == "__main__":
     unittest.main()

open_instruct/test_dpo_utils.py

@@ -4,7 +4,6 @@
 import unittest
 
 import torch
-from parameterized import parameterized
 
 from open_instruct import dpo_utils
 from open_instruct.dataset_transformation import TokenizerConfig
@@ -88,28 +87,6 @@ def test_rewards_are_detached(self):
         self.assertFalse(rejected_rewards.requires_grad)
 
 
-class TestGetBatchLogps(unittest.TestCase):
-    """Regression tests for PR #1625: division by zero when a sequence has
-    every label masked (`-100`) and `average_log_prob=True`."""
-
-    @parameterized.expand(
-        [
-            (
-                "fully_masked_row_returns_zero_not_nan",
-                [[-1.0, -2.0, -3.0, -4.0], [-1.0, -2.0, -3.0, -4.0]],
-                [[0, 1, 2, 3], [-100, -100, -100, -100]],
-                [-2.0, 0.0],
-            ),
-            ("partially_masked_row_matches_unclamped_average", [[-1.0, -2.0, -3.0, -4.0]], [[0, 1, 2, -100]], [-1.5]),
-        ]
-    )
-    def test_average_log_prob(self, _name, per_token_logps, labels, expected):
-        result = dpo_utils._get_batch_logps(torch.tensor(per_token_logps), torch.tensor(labels), average_log_prob=True)
-        self.assertFalse(torch.isnan(result).any())
-        self.assertFalse(torch.isinf(result).any())
-        torch.testing.assert_close(result, torch.tensor(expected))
-
-
 class TestSimPOLoss(unittest.TestCase):
     """Tests for simpo_loss function."""