From 216794bdd5d52a5773ef980814aee64f7c408996 Mon Sep 17 00:00:00 2001 From: crazywriter1 Date: Tue, 14 Jul 2026 20:42:42 +0300 Subject: [PATCH 1/2] fix(rubrics): make RubricGroup.score_group match rollout metrics and advantages Child rubrics left per-rubric advantages on state/trajectory and score_group merged metrics into preexisting keys, unlike score_rollout and Rubric.score_group. --- verifiers/rubrics/rubric_group.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/verifiers/rubrics/rubric_group.py b/verifiers/rubrics/rubric_group.py index 85807f9023..bf093fe610 100644 --- a/verifiers/rubrics/rubric_group.py +++ b/verifiers/rubrics/rubric_group.py @@ -114,14 +114,24 @@ async def teardown(self): async def score_group(self, states: list[State]): """ Evaluate all reward functions in-place for a group of rollouts. + + Mirrors ``score_rollout`` for metrics (full replace) and ``Rubric.score_group`` + for advantages: child rubrics may write per-rubric advantages/trajectory fields, + so the group recomputes them from the aggregated rewards and overwrites leftovers. """ - aggregated_rewards = [0.0] * len(states) + num_states = len(states) + if num_states == 0: + self.logger.warning("No states to score") + return + + aggregated_rewards = [0.0] * num_states aggregated_metrics: dict[str, list[float]] = {} original_rewards = [state.get("reward", 0.0) for state in states] original_metrics = [ state.get("metrics", {}).copy() if state.get("metrics") else {} for state in states ] + original_advantages = [state.get("advantage") for state in states] for rubric in self.rubrics: await rubric.score_group(states) for i, state in enumerate(states): @@ -132,14 +142,21 @@ async def score_group(self, states: list[State]): aggregated_rewards[i] += rubric_reward for key, value in rubric_metrics.items(): if key not in aggregated_metrics: - aggregated_metrics[key] = [0.0] * len(states) + aggregated_metrics[key] = [0.0] * num_states aggregated_metrics[key][i] += value + # Restore so the next rubric sees the same pre-group inputs. state["reward"] = original_rewards[i] state["metrics"] = original_metrics[i].copy() + state["advantage"] = original_advantages[i] + + avg_reward = sum(aggregated_rewards) / num_states for i, state in enumerate(states): state["reward"] = aggregated_rewards[i] - if aggregated_metrics: - if "metrics" not in state: - state["metrics"] = {} - for key, values in aggregated_metrics.items(): - state["metrics"][key] = values[i] + state["advantage"] = aggregated_rewards[i] - avg_reward + # Children fill traj via if-None; overwrite with the aggregate. + for step in state.get("trajectory") or []: + step["advantage"] = state["advantage"] + step["reward"] = state["reward"] + state["metrics"] = { + key: values[i] for key, values in aggregated_metrics.items() + } From 2dde8b444bdeaf7808f29d9516afd96432b2e099 Mon Sep 17 00:00:00 2001 From: crazywriter1 Date: Tue, 14 Jul 2026 21:06:07 +0300 Subject: [PATCH 2/2] fix(rubrics): preserve intentional trajectory step rewards in RubricGroup.score_group Only fill unset traj reward/advantage after aggregation, matching Rubric.score_group, and restore traj fields between child rubrics so child if-None fills do not stick. --- verifiers/rubrics/rubric_group.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/verifiers/rubrics/rubric_group.py b/verifiers/rubrics/rubric_group.py index bf093fe610..b361b1dd63 100644 --- a/verifiers/rubrics/rubric_group.py +++ b/verifiers/rubrics/rubric_group.py @@ -117,7 +117,8 @@ async def score_group(self, states: list[State]): Mirrors ``score_rollout`` for metrics (full replace) and ``Rubric.score_group`` for advantages: child rubrics may write per-rubric advantages/trajectory fields, - so the group recomputes them from the aggregated rewards and overwrites leftovers. + so the group recomputes state advantages from aggregated rewards and only fills + trajectory fields that are still ``None`` (preserving intentional step values). """ num_states = len(states) if num_states == 0: @@ -132,6 +133,14 @@ async def score_group(self, states: list[State]): for state in states ] original_advantages = [state.get("advantage") for state in states] + # Snapshot traj fields so child if-None fills can be rolled back between rubrics. + original_traj_fields = [ + [ + (step.get("reward"), step.get("advantage")) + for step in state.get("trajectory") or [] + ] + for state in states + ] for rubric in self.rubrics: await rubric.score_group(states) for i, state in enumerate(states): @@ -148,15 +157,22 @@ async def score_group(self, states: list[State]): state["reward"] = original_rewards[i] state["metrics"] = original_metrics[i].copy() state["advantage"] = original_advantages[i] + for step, (reward, advantage) in zip( + state.get("trajectory") or [], original_traj_fields[i] + ): + step["reward"] = reward + step["advantage"] = advantage avg_reward = sum(aggregated_rewards) / num_states for i, state in enumerate(states): state["reward"] = aggregated_rewards[i] state["advantage"] = aggregated_rewards[i] - avg_reward - # Children fill traj via if-None; overwrite with the aggregate. + # Same as Rubric.score_group: fill only unset traj fields. for step in state.get("trajectory") or []: - step["advantage"] = state["advantage"] - step["reward"] = state["reward"] + if step.get("advantage") is None: + step["advantage"] = state["advantage"] + if step.get("reward") is None: + step["reward"] = state["reward"] state["metrics"] = { key: values[i] for key, values in aggregated_metrics.items() }