Clamp task scores inside open interval

Author: codex

inference.py

@@ -38,6 +38,7 @@
     "https://rohan556-openenv-code-review-arena.hf.space",
 ]
 STDOUT_BROKEN = False
+ERROR_SCORE = 0.0001
 
 BASELINE_FINDINGS: dict[str, list[dict[str, Any]]] = {
     "authz_admin_export": [
@@ -321,7 +322,7 @@ def emit_failed_task(task_id: str, step_number: int, expected: int) -> None:
     emit_block(
         "END",
         task=task_id,
-        score=0.0,
+        score=ERROR_SCORE,
         steps=safe_steps,
         grade="error",
         matched=0,

server/grader.py

@@ -27,6 +27,8 @@
     "race_condition": {"race_condition", "concurrency", "double_spend", "atomicity"},
     "xss": {"xss", "cross_site_scripting", "unsafe_html"},
 }
+MIN_OPEN_SCORE = 0.0001
+MAX_OPEN_SCORE = 0.9999
 
 
 @dataclass(frozen=True)
@@ -38,6 +40,10 @@ class MatchBreakdown:
     semantic_score: float
 
 
+def clamp_open_score(value: float) -> float:
+    return min(MAX_OPEN_SCORE, max(MIN_OPEN_SCORE, value))
+
+
 def normalize_text(value: str) -> str:
     return "".join(ch.lower() if ch.isalnum() else " " for ch in value).strip()
 
@@ -218,6 +224,7 @@ def grade_submission(
     if not references:
         false_positive_penalty = min(1.0, 0.32 * len(findings))
         overall = max(0.0, 1.0 - false_positive_penalty - duplicate_penalty)
+        overall = clamp_open_score(overall)
         summary = (
             "Correctly identified that the refactor is clean."
             if not findings
@@ -299,6 +306,7 @@ def grade_submission(
         - 0.14 * missed_penalty
     )
     overall = max(0.0, min(1.0, overall))
+    overall = clamp_open_score(overall)
 
     summary = (
         f"Matched {len(assignments)} of {len(references)} reference findings. "
@@ -322,4 +330,3 @@ def grade_submission(
         assessments=ordered_assessments,
         missed_reference_ids=[reference.id for reference in unmatched_refs],
     )
-

tests/test_environment.py

@@ -146,7 +146,7 @@ def test_emit_failed_task_prints_parseable_error_episode(capsys):
     lines = capsys.readouterr().out.strip().splitlines()
     assert lines == [
         "[STEP] step=1 action=error reward=0.0 done=True phase=error",
-        "[END] task=sql_injection_report_filters score=0.0 steps=1 grade=error matched=0 expected=1",
+        "[END] task=sql_injection_report_filters score=0.0001 steps=1 grade=error matched=0 expected=1",
     ]
 
 

tests/test_grader.py

@@ -52,3 +52,16 @@ def test_clean_task_penalizes_false_positives():
     assert scorecard.overall_score < 0.75
     assert scorecard.false_positive_penalty > 0
 
+
+def test_clean_task_perfect_score_stays_below_one():
+    task = TaskStore().get("safe_logging_refactor")
+    scorecard = grade_submission(task, [], steps_used=1)
+
+    assert 0.0 < scorecard.overall_score < 1.0
+
+
+def test_bad_submission_score_stays_above_zero():
+    task = TaskStore().get("sql_injection_report_filters")
+    scorecard = grade_submission(task, [], steps_used=task.max_steps)
+
+    assert 0.0 < scorecard.overall_score < 1.0