Skip to content

Commit dcd7ffa

Browse files
varunursekarclaude
andcommitted
tests: fix harbor build benchmark tests for the split + candidates flatten
The benchmark-config tests read harness-engineering-bench/ (a separate branch in the stacked split), so skip them when it isn't checked out. Update the paths for the candidates/ -> top-level flatten (all benchmarks are now top-level siblings of gaia/). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 4bbe47b commit dcd7ffa

1 file changed

Lines changed: 11 additions & 3 deletions

File tree

vero/tests/test_v05_harbor_build.py

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,13 @@
2525

2626
BENCHMARK_ROOT = Path(__file__).resolve().parents[2] / "harness-engineering-bench"
2727

28+
# harness-engineering-bench is a separate branch/PR in the stacked split, so
29+
# skip the benchmark-config tests when it isn't checked out here.
30+
_requires_benchmarks = pytest.mark.skipif(
31+
not BENCHMARK_ROOT.exists(),
32+
reason="harness-engineering-bench is not present on this branch",
33+
)
34+
2835

2936
def _git(path: Path, *arguments: str) -> str:
3037
result = subprocess.run(
@@ -37,6 +44,7 @@ def _git(path: Path, *arguments: str) -> str:
3744
return result.stdout.strip()
3845

3946

47+
@_requires_benchmarks
4048
@pytest.mark.parametrize(
4149
("benchmark", "producer_models"),
4250
[
@@ -50,10 +58,9 @@ def _git(path: Path, *arguments: str) -> str:
5058
def test_canonical_benchmarks_isolate_upstream_inference_credentials(
5159
benchmark, producer_models
5260
):
53-
# gaia is promoted to the top level; other task sets live under candidates/.
54-
base = BENCHMARK_ROOT if benchmark == "gaia" else BENCHMARK_ROOT / "candidates"
61+
# All benchmarks live at the top level of harness-engineering-bench.
5562
config = load_harbor_build_config(
56-
base / benchmark / "baseline" / "build.yaml"
63+
BENCHMARK_ROOT / benchmark / "baseline" / "build.yaml"
5764
)
5865

5966
assert config.inference_gateway is not None
@@ -74,6 +81,7 @@ def test_canonical_benchmarks_isolate_upstream_inference_credentials(
7481
assert config.inference_gateway.evaluation.max_tokens == 100000000
7582

7683

84+
@_requires_benchmarks
7785
def test_build_params_override_run_time_knobs_without_rebuild():
7886
path = BENCHMARK_ROOT / "gaia" / "baseline" / "build.yaml"
7987

0 commit comments

Comments
 (0)