harbor: score-baseline producer for pinnable baseline numbers (feedback #1)

Add an admin-only way to generate the number to pin: CanonicalVerifier
.measure_baseline(replicates) admin-scores the fixed seed N times on the
selection partition and each target and returns per-key mean/stddev. Exposed
as POST /score/baseline and 'vero harbor score-baseline --replicates N'.
Reuses the existing admin scoring machinery.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: varunursekar

vero/src/vero/harbor/app.py

@@ -43,6 +43,12 @@ class SubmitRequest(BaseModel):
     version: str | None = None
 
 
+class ScoreBaselineRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    replicates: int = 1
+
+
 def _error(status_code: int, message: str):
     async def handler(_request, error):
         return JSONResponse(
@@ -147,6 +153,14 @@ async def finalize(authorization: Annotated[str | None, Header()] = None):
         require_admin(authorization)
         return await verifier.finalize()
 
+    @app.post("/score/baseline")
+    async def score_baseline(
+        body: ScoreBaselineRequest,
+        authorization: Annotated[str | None, Header()] = None,
+    ):
+        require_admin(authorization)
+        return await verifier.measure_baseline(replicates=body.replicates)
+
     @app.get("/evaluations")
     async def evaluations(
         authorization: Annotated[str | None, Header()] = None,

vero/src/vero/harbor/cli.py

@@ -625,6 +625,25 @@ def status_command():
     click.echo(json.dumps(_request("GET", "/status"), indent=2))
 
 
+@harbor.command("score-baseline")
+@click.option(
+    "--token-file",
+    required=True,
+    type=click.Path(path_type=Path, exists=True, dir_okay=False),
+)
+@click.option("--replicates", default=1, show_default=True, type=click.IntRange(min=1))
+def score_baseline_command(token_file, replicates):
+    """Admin-score the fixed seed N times to produce a pinnable baseline number."""
+    token = read_admin_token(token_file)
+    result = _request(
+        "POST",
+        "/score/baseline",
+        payload={"replicates": replicates},
+        headers={"Authorization": f"Bearer {token}"},
+    )
+    click.echo(json.dumps(result, indent=2))
+
+
 @harbor.command("finalize")
 @click.option(
     "--token-file",

vero/src/vero/harbor/verifier.py

@@ -427,6 +427,56 @@ async def _score_target(
         )
         return reward, record.id, None
 
+    async def measure_baseline(self, *, replicates: int = 1) -> dict[str, JsonValue]:
+        """Admin-score the fixed seed to produce pinnable baseline numbers.
+
+        Runs `replicates` trusted evaluations of the baseline on the selection
+        partition and each target and returns per-key mean/stddev, so a stable
+        value can be pinned (baseline_selection_score / target baseline_reward)
+        instead of re-scoring the seed every run."""
+        if replicates < 1:
+            raise ValueError("replicates must be >= 1")
+        baseline = self.selection.baseline_candidate
+        if baseline is None:
+            raise NoCandidateError("no baseline candidate to score")
+
+        def _aggregate(values: list[float | None]) -> dict[str, JsonValue]:
+            clean = [value for value in values if value is not None]
+            return {
+                "values": values,
+                "n": len(clean),
+                "mean": statistics.fmean(clean) if clean else None,
+                "stddev": statistics.pstdev(clean) if len(clean) > 1 else 0.0,
+            }
+
+        result: dict[str, JsonValue] = {
+            "candidate_version": baseline.version,
+            "replicates": replicates,
+        }
+        if (
+            self.selection.backend_id is not None
+            and self.selection.evaluation_set is not None
+            and self.selection.objective is not None
+        ):
+            selection_values: list[float | None] = []
+            for _ in range(replicates):
+                record, _ = await self._rescore_candidate(baseline)
+                selection_values.append(
+                    record.objective.value
+                    if record is not None and record.objective is not None
+                    else None
+                )
+            result["selection"] = _aggregate(selection_values)
+        targets: dict[str, JsonValue] = {}
+        for target in self.targets:
+            target_values: list[float | None] = []
+            for _ in range(replicates):
+                reward, _, error = await self._score_target(baseline, target)
+                target_values.append(None if error is not None else reward)
+            targets[target.reward_key] = _aggregate(target_values)
+        result["targets"] = targets
+        return result
+
     async def _finalize(self) -> VerificationResult:
         try:
             candidate = await self._select_candidate()

vero/tests/test_v05_harbor_verifier.py

@@ -561,3 +561,32 @@ async def test_verifier_floor_fails_safe_when_seed_unmeasurable(tmp_path):
 
     assert result.shipped is False
     assert "baseline floor" in result.errors.get("selection", "")
+
+
+@pytest.mark.asyncio
+async def test_verifier_score_baseline_produces_replicated_means(tmp_path):
+    baseline = _candidate("baseline")
+    engine = FakeEngine(
+        {
+            ("baseline", "selection"): [0.5, 0.6],
+            ("baseline", "test"): [0.4, 0.5],
+        }
+    )
+
+    out = await _verifier(tmp_path, engine, baseline=baseline).measure_baseline(
+        replicates=2
+    )
+
+    assert out["candidate_version"] == "baseline"
+    assert out["replicates"] == 2
+    assert out["selection"]["n"] == 2
+    assert out["selection"]["mean"] == 0.55
+    assert out["targets"]["reward"]["n"] == 2
+    assert out["targets"]["reward"]["mean"] == 0.45
+    # 2 selection re-scores + 2 target scores, all admin.
+    assert engine.calls == [
+        ("baseline", "selection"),
+        ("baseline", "selection"),
+        ("baseline", "test"),
+        ("baseline", "test"),
+    ]