Keep task scores inside the open interval

Author: PathikSingh

README.md

@@ -183,8 +183,8 @@ The report includes:
 - optional Markdown export
 - optional scripted-vs-random comparison
 
-Normalized score is computed against each task's `max_reward` and capped to the `0.0-1.0` range for clean benchmark reporting.
-The surfaced environment `reward` and benchmark `score` stay in the required `0.0-1.0` range, while raw shaped reward remains available through observation metadata and benchmark report fields for debugging.
+Normalized score is computed against each task's `max_reward` and then clamped strictly inside `(0, 1)` for validator-safe reporting.
+The surfaced environment `reward` and benchmark `score` therefore stay strictly between `0.0` and `1.0`, while raw shaped reward remains available through observation metadata and benchmark report fields for debugging.
 
 ## Submission Compliance Notes
 

benchmark.csv

@@ -1,13 +1,13 @@
-task_id,title,difficulty,terminal_outcome,steps,total_reward,max_reward,reward_ratio,normalized_score
-rollback_alpha,Rollback a broken canary release,easy,success,4,24.4,20.0,1.22,1.0
-hotfix_ci_beta,Fix-forward a blocked release,easy,success,8,25.6,25.0,1.024,1.0
-public_payments_incident,Recover a customer-facing payments incident,medium,success,7,34.1,28.0,1.218,1.0
-auth_fix_forward,Fix-forward a security-sensitive auth release,medium,success,15,48.5,35.0,1.386,1.0
-schema_migration_fix,Repair a release blocked by schema migration failure,medium,success,9,28.3,25.0,1.132,1.0
-customer_timeline_hard,Run a full incident lifecycle with postmortem hygiene,hard,success,10,42.2,38.0,1.111,1.0
-checkout_fix_forward_major,Contain and fix-forward a checkout incident,hard,success,16,51.2,40.0,1.28,1.0
-orders_pause_and_rollback,Pause rollout and rollback an orders release,medium,success,6,29.8,28.0,1.064,1.0
-database_rollback_simple,Rollback a database-impacting release,easy,success,4,24.4,20.0,1.22,1.0
-api_gateway_fix_forward,Fix-forward a broken API gateway release,medium,success,9,28.3,28.0,1.011,1.0
-notification_service_sev2,Handle a sev2 notification service incident,medium,success,7,34.1,28.0,1.218,1.0
-ml_pipeline_hard,Recover a broken ML inference pipeline with full incident lifecycle,hard,success,10,42.2,40.0,1.055,1.0
+task_id,title,difficulty,terminal_outcome,steps,score,total_reward,max_reward,reward_ratio,normalized_score
+rollback_alpha,Rollback a broken canary release,easy,success,4,0.999,24.4,20.0,1.22,0.999
+hotfix_ci_beta,Fix-forward a blocked release,easy,success,8,0.999,25.6,25.0,1.024,0.999
+public_payments_incident,Recover a customer-facing payments incident,medium,success,7,0.999,34.1,28.0,1.218,0.999
+auth_fix_forward,Fix-forward a security-sensitive auth release,medium,success,15,0.999,48.5,35.0,1.386,0.999
+schema_migration_fix,Repair a release blocked by schema migration failure,medium,success,9,0.999,28.3,25.0,1.132,0.999
+customer_timeline_hard,Run a full incident lifecycle with postmortem hygiene,hard,success,10,0.999,42.2,38.0,1.111,0.999
+checkout_fix_forward_major,Contain and fix-forward a checkout incident,hard,success,16,0.999,51.2,40.0,1.28,0.999
+orders_pause_and_rollback,Pause rollout and rollback an orders release,medium,success,6,0.999,29.8,28.0,1.064,0.999
+database_rollback_simple,Rollback a database-impacting release,easy,success,4,0.999,24.4,20.0,1.22,0.999
+api_gateway_fix_forward,Fix-forward a broken API gateway release,medium,success,9,0.999,28.3,28.0,1.011,0.999
+notification_service_sev2,Handle a sev2 notification service incident,medium,success,7,0.999,34.1,28.0,1.218,0.999
+ml_pipeline_hard,Recover a broken ML inference pipeline with full incident lifecycle,hard,success,10,0.999,42.2,40.0,1.055,0.999

benchmark.md

@@ -7,29 +7,29 @@
 - Success: `12/12`
 - Success rate: `100.0%`
 - Average reward: `34.43`
-- Average normalized score: `1.000`
+- Average normalized score: `0.999`
 
 ## By Difficulty
 
 | Difficulty | Tasks | Success | Success Rate | Avg Reward | Avg Normalized |
 |---|---:|---:|---:|---:|---:|
-| easy | 3 | 3 | 100.0% | 24.80 | 1.000 |
-| hard | 3 | 3 | 100.0% | 45.20 | 1.000 |
-| medium | 6 | 6 | 100.0% | 33.85 | 1.000 |
+| easy | 3 | 3 | 100.0% | 24.80 | 0.999 |
+| hard | 3 | 3 | 100.0% | 45.20 | 0.999 |
+| medium | 6 | 6 | 100.0% | 33.85 | 0.999 |
 
 ## Per Task
 
 | Task | Difficulty | Outcome | Steps | Reward | Max Reward | Normalized |
 |---|---|---|---:|---:|---:|---:|
-| rollback_alpha | easy | success | 4 | 24.4 | 20.0 | 1.000 |
-| hotfix_ci_beta | easy | success | 8 | 25.6 | 25.0 | 1.000 |
-| public_payments_incident | medium | success | 7 | 34.1 | 28.0 | 1.000 |
-| auth_fix_forward | medium | success | 15 | 48.5 | 35.0 | 1.000 |
-| schema_migration_fix | medium | success | 9 | 28.3 | 25.0 | 1.000 |
-| customer_timeline_hard | hard | success | 10 | 42.2 | 38.0 | 1.000 |
-| checkout_fix_forward_major | hard | success | 16 | 51.2 | 40.0 | 1.000 |
-| orders_pause_and_rollback | medium | success | 6 | 29.8 | 28.0 | 1.000 |
-| database_rollback_simple | easy | success | 4 | 24.4 | 20.0 | 1.000 |
-| api_gateway_fix_forward | medium | success | 9 | 28.3 | 28.0 | 1.000 |
-| notification_service_sev2 | medium | success | 7 | 34.1 | 28.0 | 1.000 |
-| ml_pipeline_hard | hard | success | 10 | 42.2 | 40.0 | 1.000 |
+| rollback_alpha | easy | success | 4 | 24.4 | 20.0 | 0.999 |
+| hotfix_ci_beta | easy | success | 8 | 25.6 | 25.0 | 0.999 |
+| public_payments_incident | medium | success | 7 | 34.1 | 28.0 | 0.999 |
+| auth_fix_forward | medium | success | 15 | 48.5 | 35.0 | 0.999 |
+| schema_migration_fix | medium | success | 9 | 28.3 | 25.0 | 0.999 |
+| customer_timeline_hard | hard | success | 10 | 42.2 | 38.0 | 0.999 |
+| checkout_fix_forward_major | hard | success | 16 | 51.2 | 40.0 | 0.999 |
+| orders_pause_and_rollback | medium | success | 6 | 29.8 | 28.0 | 0.999 |
+| database_rollback_simple | easy | success | 4 | 24.4 | 20.0 | 0.999 |
+| api_gateway_fix_forward | medium | success | 9 | 28.3 | 28.0 | 0.999 |
+| notification_service_sev2 | medium | success | 7 | 34.1 | 28.0 | 0.999 |
+| ml_pipeline_hard | hard | success | 10 | 42.2 | 40.0 | 0.999 |

benchmark_comparison.md

@@ -5,11 +5,11 @@
 
 | Policy | Success Rate | Avg Reward | Avg Normalized |
 |---|---:|---:|---:|
-| scripted | 100.0% | 34.43 | 1.000 |
+| scripted | 100.0% | 34.43 | 0.999 |
 | random | 25.0% | 15.96 | 0.569 |
 
 ## Delta
 
 - Success rate delta: `75.0%`
 - Average reward delta: `18.47`
-- Average normalized score delta: `0.431`
+- Average normalized score delta: `0.430`

benchmark_report.py

@@ -13,11 +13,11 @@
 
 try:
     from opsgauntlet.models import OpsGauntletAction, ToolCallRequest
-    from opsgauntlet.server.environment import OpsGauntletEnvironment
+    from opsgauntlet.server.environment import OpsGauntletEnvironment, clamp_strict_score
     from opsgauntlet.server.task_bank import TASK_BANK, get_task_by_id
 except ImportError:  # pragma: no cover
     from models import OpsGauntletAction, ToolCallRequest  # type: ignore
-    from server.environment import OpsGauntletEnvironment  # type: ignore
+    from server.environment import OpsGauntletEnvironment, clamp_strict_score  # type: ignore
     from server.task_bank import TASK_BANK, get_task_by_id  # type: ignore
 
 
@@ -43,7 +43,7 @@ def collect_benchmark_results(
                 "score": round(score, 3),
                 "raw_total_reward": round(raw_total_reward, 2),
                 "reward_ratio": round(raw_ratio, 3),
-                "normalized_score": round(max(0.0, min(score, 1.0)), 3),
+                "normalized_score": clamp_strict_score(score),
             }
         )
 

server/environment.py

@@ -30,6 +30,14 @@
 from .task_bank import Task, get_random_task
 from .tool_registry import ToolRegistry
 
+STRICT_SCORE_EPSILON = 0.001
+
+
+def clamp_strict_score(value: float) -> float:
+    """Keep surfaced task scores strictly inside the open unit interval."""
+
+    return round(max(STRICT_SCORE_EPSILON, min(value, 1.0 - STRICT_SCORE_EPSILON)), 3)
+
 
 class OpsGauntletEnvironment(
     Environment[OpsGauntletAction, OpsGauntletObservation, State]
@@ -228,6 +236,6 @@ def _append_timeline(self, result: ToolResult) -> None:
     def _normalized_episode_score(self) -> float:
         assert self._task is not None
         if self._task.max_reward <= 0:
-            return 0.0
+            return clamp_strict_score(0.5)
         normalized = self._raw_total_reward / self._task.max_reward
-        return round(max(0.0, min(normalized, 1.0)), 3)
+        return clamp_strict_score(normalized)

tests/test_environment.py

@@ -59,8 +59,8 @@ def test_rollback_scenario_can_complete_successfully():
     )
 
     assert result.done is True
-    assert result.reward is not None and 0.0 <= result.reward <= 1.0
-    assert result.metadata["episode_score"] == 1.0
+    assert result.reward is not None and 0.0 < result.reward < 1.0
+    assert 0.0 < result.metadata["episode_score"] < 1.0
     assert result.metadata["raw_total_reward"] > 5
     assert "rolled_back_safely" in result.completed_objectives
     assert "internal_comms_sent" in result.completed_objectives
@@ -146,7 +146,7 @@ def test_cannot_close_customer_incident_before_recovery():
 def test_scripted_baseline_solves_all_tasks():
     outcomes = [run_episode(task.task_id, verbose=False) for task in TASK_BANK]
     assert all(item["terminal_outcome"] == "success" for item in outcomes)
-    assert all(0.0 <= item["score"] <= 1.0 for item in outcomes)
+    assert all(0.0 < item["score"] < 1.0 for item in outcomes)
 
 
 def test_auto_policy_defaults_to_scripted_without_api_key(monkeypatch):
@@ -202,9 +202,26 @@ def test_benchmark_report_covers_all_tasks_with_normalized_scores():
     report = collect_benchmark_results(scope="all", seed=7)
     assert report["task_count"] == len(TASK_BANK)
     assert report["success_count"] == len(TASK_BANK)
-    assert 0.0 <= report["average_normalized_score"] <= 1.0
+    assert 0.0 < report["average_normalized_score"] < 1.0
     for item in report["results"]:
-        assert 0.0 <= item["normalized_score"] <= 1.0
+        assert 0.0 < item["normalized_score"] < 1.0
+
+
+def test_failed_terminal_scores_stay_inside_open_interval():
+    env = OpsGauntletEnvironment()
+    observation = env.reset(task_id="rollback_alpha")
+
+    while not observation.done:
+        observation = step(
+            env,
+            "notify_slack",
+            {"channel": "#eng", "message": "premature update"},
+            reasoning="Keep doing the wrong thing until the episode times out.",
+        )
+
+    assert observation.metadata["terminal_outcome"] == "failure"
+    assert 0.0 < observation.reward < 1.0
+    assert 0.0 < observation.metadata["episode_score"] < 1.0
 
 
 def test_root_page_and_robots_are_served():