fix(inspect): return None on skip instead of Score.unscored()

Score.unscored() records a NaN value. Custom report renderers (e.g. the
edu-panda-skill-harness eval report) that normalize scores via isinstance(v,
float) treat that NaN as a real 0–1 score and average it into the mean,
poisoning the whole scorer column to NaN. Returning None omits the sample from
this scorer's results entirely — handled cleanly by every Inspect metric and by
naive renderers — and matches the skip convention used by the harness's other
scorers (and rubric_judge). None is a fully-supported Scorer return per the
Scorer protocol (-> Score | None).

Applies to all three skip/error paths: missing/invalid target_grade, no text,
and transient API/parse errors. Tests updated to assert None.

Author: adnanrhussain

integrations/inspect-python/src/learning_commons_inspect_scorers/gla.py

@@ -78,10 +78,10 @@ def gla_scorer(
                         the target also counts as a pass. Set to ``False`` for an
                         exact match.
 
-    Returns ``Score.unscored()`` when ``target_grade_key`` is absent or not a valid
-    grade band, when no text is available to evaluate, or when a transient API/parse
-    error occurs. Re-raises ``ConfigurationError`` and ``InputValidationError`` as
-    task-level failures.
+    Returns ``None`` (skip — the sample is omitted from this scorer's results) when
+    ``target_grade_key`` is absent or not a valid grade band, when no text is
+    available to evaluate, or when a transient API/parse error occurs. Re-raises
+    ``ConfigurationError`` and ``InputValidationError`` as task-level failures.
 
     ``Score.value`` is ``CORRECT`` (pass) or ``INCORRECT`` (fail).
     ``Score.metadata`` contains ``gla_grade``, ``target_grade``, ``alternative_grade``,
@@ -98,35 +98,24 @@ def gla_scorer(
     get_text = _artifact_text if text_source == "artifacts" else _completion_text
 
     async def score(state: TaskState, target: Target) -> Score | None:
+        # Skip paths return None (not Score.unscored()): None omits the sample from
+        # this scorer's results entirely, which every Inspect metric — and custom
+        # report renderers — handle cleanly. Score.unscored() records a NaN value
+        # that naive renderers can mistake for a real score and average into the mean.
         target_grade: str = (state.metadata.get(target_grade_key) or "").strip()
         if not target_grade or target_grade not in GRADE_BANDS:
-            return Score.unscored(
-                explanation=(
-                    f"{target_grade_key!r} is missing or not a valid grade band. "
-                    f"Valid bands: {', '.join(GRADE_BANDS)}"
-                ),
-                metadata={
-                    "target_grade_key": target_grade_key,
-                    "target_grade": target_grade or None,
-                },
-            )
+            return None
 
         text = get_text(state)
         if not text.strip():
-            return Score.unscored(
-                explanation="No text to evaluate (empty completion or no readable artifacts).",
-                metadata={"target_grade": target_grade, "gla_grade": None},
-            )
+            return None
 
         try:
             result = await evaluator.evaluate(GradeLevelAppropriatenessEvaluationInput(text=text))
         except (ConfigurationError, InputValidationError):
             raise  # setup/programming errors — let Inspect surface them as task failures
-        except APIError as exc:  # OutputValidationError is a subclass of APIError
-            return Score.unscored(
-                explanation=f"GLA evaluation failed: {exc}",
-                metadata={"target_grade": target_grade, "gla_grade": None},
-            )
+        except APIError:  # OutputValidationError is a subclass; transient grading failure → skip
+            return None
 
         gla_grade: str = result.answer.score
         passed = gla_grade in _acceptable_bands(target_grade, allow_adjacent)

integrations/inspect-python/tests/test_gla_scorer.py

@@ -2,7 +2,6 @@
 
 from __future__ import annotations
 
-import math
 from pathlib import Path
 from unittest.mock import AsyncMock, MagicMock, patch
 
@@ -140,31 +139,26 @@ async def test_boundary_11ccr_adjacent_passes(self):
         state = _make_state(completion="Sample.", metadata={"target_grade": "11-CCR"})
         assert (await scorer_fn(state, _make_target())).value == CORRECT
 
-    async def test_missing_target_grade_returns_unscored(self):
+    async def test_missing_target_grade_returns_none(self):
         scorer_fn, _ = _make_scorer()
         state = _make_state(completion="Sample.", metadata={})
-        score = await scorer_fn(state, _make_target())
-        assert math.isnan(float(score.value))
+        assert await scorer_fn(state, _make_target()) is None
 
-    async def test_invalid_target_grade_returns_unscored(self):
+    async def test_invalid_target_grade_returns_none(self):
         scorer_fn, _ = _make_scorer()
         state = _make_state(completion="Sample.", metadata={"target_grade": "Grade 5"})
-        score = await scorer_fn(state, _make_target())
-        assert math.isnan(float(score.value))
+        assert await scorer_fn(state, _make_target()) is None
 
-    async def test_empty_completion_returns_unscored(self):
+    async def test_empty_completion_returns_none(self):
         scorer_fn, mock_evaluator = _make_scorer()
         state = _make_state(completion="", metadata={"target_grade": "6-8"})
-        score = await scorer_fn(state, _make_target())
-        assert math.isnan(float(score.value))
+        assert await scorer_fn(state, _make_target()) is None
         mock_evaluator.evaluate.assert_not_called()
 
-    async def test_api_error_returns_unscored(self):
+    async def test_api_error_returns_none(self):
         scorer_fn, _ = _make_scorer(side_effect=APIError("rate limit"))
         state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"})
-        score = await scorer_fn(state, _make_target())
-        assert math.isnan(float(score.value))
-        assert "rate limit" in score.explanation
+        assert await scorer_fn(state, _make_target()) is None
 
     async def test_configuration_error_propagates(self):
         scorer_fn, _ = _make_scorer(side_effect=ConfigurationError("no key"))
@@ -186,11 +180,10 @@ async def test_custom_target_grade_key(self):
         state = _make_state(completion="Sample.", metadata={"expected_grade": "6-8"})
         assert (await scorer_fn(state, _make_target())).value == CORRECT
 
-    async def test_custom_target_grade_key_absent_returns_unscored(self):
+    async def test_custom_target_grade_key_absent_returns_none(self):
         scorer_fn, _ = _make_scorer(target_grade_key="expected_grade")
         state = _make_state(completion="Sample.", metadata={"target_grade": "6-8"})
-        score = await scorer_fn(state, _make_target())
-        assert math.isnan(float(score.value))
+        assert await scorer_fn(state, _make_target()) is None
 
     async def test_custom_grader_model_passed_to_adapter(self):
         with (
@@ -221,16 +214,15 @@ async def test_reads_artifact_files(self, tmp_path: Path):
         called_text = mock_evaluator.evaluate.call_args[0][0].text.value
         assert "mitochondria" in called_text
 
-    async def test_unreadable_artifact_returns_unscored(self):
+    async def test_unreadable_artifact_returns_none(self):
         scorer_fn, mock_evaluator = _make_scorer(text_source="artifacts")
         state = _make_state(
             metadata={
                 "target_grade": "6-8",
                 "artifacts": [{"path": "/nonexistent/path.md", "filename": "missing.md"}],
             }
         )
-        score = await scorer_fn(state, _make_target())
-        assert math.isnan(float(score.value))
+        assert await scorer_fn(state, _make_target()) is None
         mock_evaluator.evaluate.assert_not_called()
 
     async def test_multiple_artifacts_joined_with_separator(self, tmp_path: Path):
@@ -307,7 +299,7 @@ def test_scorer_wires_through_eval(self):
         assert sample.score is not None
         assert sample.score.value == CORRECT
 
-    def test_unscored_sample_does_not_count_as_failure(self):
+    def test_skipped_sample_is_omitted_from_scores(self):
         from unittest.mock import AsyncMock, patch
 
         from inspect_ai import Task, eval
@@ -323,14 +315,15 @@ def test_unscored_sample_does_not_count_as_failure(self):
         ):
             scorer = gla_scorer()
 
-        # Sample has no target_grade — should be unscored, not INCORRECT
+        # Sample has no target_grade — gla_scorer returns None, so it must NOT
+        # contribute a score (no CORRECT/INCORRECT, no NaN to poison the mean).
         task = Task(
             dataset=[Sample(input="Write something.", metadata={})],
             solver=[generate()],
             scorer=scorer,
         )
 
         log = eval(task, model="mockllm/model")[0]
+        assert log.status == "success"
         sample = log.samples[0]
-        assert sample.score is not None
-        assert math.isnan(float(sample.score.value))
+        assert (sample.scores or {}).get("gla_scorer") is None