Move agent versions to harness config

Author: xeophon

packages/harnesses/README.md

@@ -66,10 +66,13 @@ Command agents use `name@version` specs where their installer supports a
 versioned package or release. Use `@latest` for a moving latest install:
 
 ```toml
-[eval.harness.program]
-# OpenCode
-install_spec = "PrimeIntellect-ai/opencode@latest"
+[eval.harness]
+id = "harnesses.opencode"
+version = "PrimeIntellect-ai/opencode@latest"
+```
 
-# MiniSWEAgent or Pi
-install_spec = "mini-swe-agent@2.2.8"
+```toml
+[eval.harness]
+id = "harnesses.mini_swe_agent"
+version = "mini-swe-agent@2.2.8"
 ```

packages/harnesses/harnesses/mini_swe_agent.py

@@ -1,6 +1,7 @@
 import shlex
 from pathlib import PurePosixPath
 
+from pydantic import PrivateAttr, model_validator
 import verifiers as vf
 from verifiers.v1.utils.sandbox_python_utils import python_runtime_setup_command
 
@@ -16,23 +17,23 @@
 MINI_SWE_AGENT_DEFAULT_SYSTEM_PROMPT_PATH = "/mini-swe-agent/system.txt"
 MINI_SWE_AGENT_DEFAULT_LOG_PATH = "/logs/agent/mini-swe-agent.log"
 MINI_SWE_AGENT_DEFAULT_TRAJECTORY_PATH = "/logs/agent/mini-swe-agent.traj.json"
-MINI_SWE_AGENT_DEFAULT_INSTALL_SPEC = "mini-swe-agent@2.2.8"
+MINI_SWE_AGENT_DEFAULT_VERSION = "mini-swe-agent@2.2.8"
 MINI_SWE_AGENT_DEFAULT_CONFIG_SPEC = "mini"
 MINI_SWE_AGENT_DEFAULT_MODEL_CLASS = "litellm"
 MINI_SWE_AGENT_DEFAULT_ENVIRONMENT_TIMEOUT = 120
 
 
 def build_mini_swe_agent_install_script(
-    install_spec: str = MINI_SWE_AGENT_DEFAULT_INSTALL_SPEC,
+    version: str = MINI_SWE_AGENT_DEFAULT_VERSION,
     prefix_dir: str = DEFAULT_PREFIX_DIR,
 ) -> str:
     root = shlex.quote(str(PurePosixPath(prefix_dir).parent))
     prefix = shlex.quote(prefix_dir)
     site_packages = shlex.quote(f"{prefix_dir.rstrip('/')}/site-packages")
-    name, version = split_versioned_agent_spec(install_spec)
+    name, pin = split_versioned_agent_spec(version)
     requirement = name
-    if version and version != "latest":
-        requirement = f"{name}=={version}"
+    if pin and pin != "latest":
+        requirement = f"{name}=={pin}"
     return f"""\
 set -e
 {python_runtime_setup_command()}
@@ -51,12 +52,13 @@ def build_mini_swe_agent_install_script(
 
 
 class MiniSWEAgentProgramConfig(vf.ProgramConfig):
+    _version: str = PrivateAttr(MINI_SWE_AGENT_DEFAULT_VERSION)
+
     agent_workdir: str = MINI_SWE_AGENT_DEFAULT_AGENT_WORKDIR
     instruction_path: str = MINI_SWE_AGENT_DEFAULT_INSTRUCTION_PATH
     system_prompt_path: str = MINI_SWE_AGENT_DEFAULT_SYSTEM_PROMPT_PATH
     log_path: str = MINI_SWE_AGENT_DEFAULT_LOG_PATH
     trajectory_path: str = MINI_SWE_AGENT_DEFAULT_TRAJECTORY_PATH
-    install_spec: str = MINI_SWE_AGENT_DEFAULT_INSTALL_SPEC
     config_spec: str = MINI_SWE_AGENT_DEFAULT_CONFIG_SPEC
     model_class: str = MINI_SWE_AGENT_DEFAULT_MODEL_CLASS
     environment_timeout: int = MINI_SWE_AGENT_DEFAULT_ENVIRONMENT_TIMEOUT
@@ -112,7 +114,7 @@ def resolve(self) -> vf.ProgramConfig:
             config_args.extend(["-c", shlex.quote(spec)])
 
         setup = build_mini_swe_agent_install_script(
-            install_spec=self.install_spec,
+            version=self._version,
         )
         log_dir = str(PurePosixPath(self.log_path).parent)
         trajectory_dir = str(PurePosixPath(self.trajectory_path).parent)
@@ -155,9 +157,15 @@ def resolve(self) -> vf.ProgramConfig:
 
 
 class MiniSWEAgentConfig(vf.HarnessConfig):
+    version: str = MINI_SWE_AGENT_DEFAULT_VERSION
     program: MiniSWEAgentProgramConfig = MiniSWEAgentProgramConfig()
     max_turns: int = 4
 
+    @model_validator(mode="after")
+    def apply_version(self):
+        self.program._version = self.version
+        return self
+
 
 class MiniSWEAgent(vf.Harness[MiniSWEAgentConfig]):
     config: MiniSWEAgentConfig

packages/harnesses/harnesses/opencode.py

@@ -2,12 +2,13 @@
 import shlex
 from pathlib import PurePosixPath
 
+from pydantic import PrivateAttr, model_validator
 import verifiers as vf
 from verifiers.v1.utils.mcp_proxy_utils import proxy_command
 
 from .utils import split_versioned_agent_spec
 
-OPENCODE_DEFAULT_INSTALL_SPEC = "PrimeIntellect-ai/opencode@1.1.63-rl2"
+OPENCODE_DEFAULT_VERSION = "PrimeIntellect-ai/opencode@1.1.63-rl2"
 OPENCODE_DEFAULT_AGENT_WORKDIR = "/app"
 OPENCODE_DEFAULT_INSTRUCTION_PATH = "/opencode/instruction.txt"
 OPENCODE_DEFAULT_SYSTEM_PROMPT_PATH = "/opencode/system.txt"
@@ -43,14 +44,15 @@
 
 
 class OpenCodeProgramConfig(vf.ProgramConfig):
+    _version: str = PrivateAttr(OPENCODE_DEFAULT_VERSION)
+
     agent_workdir: str = OPENCODE_DEFAULT_AGENT_WORKDIR
     instruction_path: str = OPENCODE_DEFAULT_INSTRUCTION_PATH
     system_prompt_path: str = OPENCODE_DEFAULT_SYSTEM_PROMPT_PATH
     log_path: str = OPENCODE_DEFAULT_LOG_PATH
     disabled_tools: list[str] = OPENCODE_DEFAULT_DISABLED_TOOLS
     allow_git: bool = False
     disable_compaction: bool = True
-    install_spec: str = OPENCODE_DEFAULT_INSTALL_SPEC
     install_ripgrep: bool = True
     provider_timeout_ms: int = 3_600_000
 
@@ -75,7 +77,7 @@ def resolve(self) -> vf.ProgramConfig:
             if self.install_ripgrep
             else ""
         )
-        repo, version = split_versioned_agent_spec(self.install_spec)
+        repo, version = split_versioned_agent_spec(self._version)
         path = "releases/latest/download"
         if version and version != "latest":
             tag = version if version.startswith("v") else f"v{version}"
@@ -197,9 +199,15 @@ class OpenCodeConfig(vf.HarnessConfig):
     system_prompt: vf.PromptInput | vf.SystemPromptConfig | None = (
         OPENCODE_DEFAULT_SYSTEM_PROMPT
     )
+    version: str = OPENCODE_DEFAULT_VERSION
     program: OpenCodeProgramConfig = OpenCodeProgramConfig()
     max_turns: int = 4
 
+    @model_validator(mode="after")
+    def apply_version(self):
+        self.program._version = self.version
+        return self
+
 
 class OpenCode(vf.Harness[OpenCodeConfig]):
     config: OpenCodeConfig

packages/harnesses/harnesses/pi.py

@@ -2,10 +2,11 @@
 import shlex
 from pathlib import PurePosixPath
 
+from pydantic import PrivateAttr, model_validator
 import verifiers as vf
 from verifiers.v1.utils.mcp_proxy_utils import proxy_command
 
-PI_DEFAULT_INSTALL_SPEC = "@earendil-works/pi-coding-agent@latest"
+PI_DEFAULT_VERSION = "@earendil-works/pi-coding-agent@latest"
 PI_DEFAULT_WORKDIR = "/app"
 PI_DEFAULT_INSTRUCTION_PATH = "/pi/instruction.txt"
 PI_DEFAULT_SYSTEM_PROMPT_PATH = "/pi/system.txt"
@@ -14,11 +15,12 @@
 
 
 class PiProgramConfig(vf.ProgramConfig):
+    _version: str = PrivateAttr(PI_DEFAULT_VERSION)
+
     agent_workdir: str = PI_DEFAULT_WORKDIR
     instruction_path: str = PI_DEFAULT_INSTRUCTION_PATH
     system_prompt_path: str = PI_DEFAULT_SYSTEM_PROMPT_PATH
     log_path: str = PI_DEFAULT_LOG_PATH
-    install_spec: str = PI_DEFAULT_INSTALL_SPEC
     install_mcp_adapter: bool = True
     sandbox: vf.SandboxConfig | None = vf.SandboxConfig()
 
@@ -82,7 +84,7 @@ def resolve(self) -> vf.ProgramConfig:
 npm install -g --ignore-scripts n
 n 22.19.0
 hash -r
-npm install -g --ignore-scripts {shlex.quote(self.install_spec)}
+npm install -g --ignore-scripts {shlex.quote(self._version)}
 """
         artifacts = vf.ArtifactsConfig.model_validate(
             {
@@ -127,9 +129,15 @@ class PiConfig(vf.HarnessConfig):
     system_prompt: vf.PromptInput | vf.SystemPromptConfig | None = (
         PI_DEFAULT_SYSTEM_PROMPT
     )
+    version: str = PI_DEFAULT_VERSION
     program: PiProgramConfig = PiProgramConfig()
     max_turns: int = 4
 
+    @model_validator(mode="after")
+    def apply_version(self):
+        self.program._version = self.version
+        return self
+
 
 class Pi(vf.Harness[PiConfig]):
     config: PiConfig

packages/harnesses/harnesses/terminus_2.py

@@ -1,25 +1,27 @@
 import shlex
 from pathlib import PurePosixPath
 
+from pydantic import PrivateAttr, model_validator
 import verifiers as vf
 from verifiers.v1.utils.sandbox_python_utils import SANDBOX_BIN_DIR, uv_setup_command
 
 TERMINUS_2_DEFAULT_AGENT_WORKDIR = "/app"
 TERMINUS_2_DEFAULT_INSTRUCTION_PATH = "/terminus_2/instruction.md"
 TERMINUS_2_DEFAULT_SYSTEM_PROMPT_PATH = "/terminus_2/system_prompt.txt"
 TERMINUS_2_DEFAULT_LOG_PATH = "/logs/agent/terminus_2.log"
-TERMINUS_2_DEFAULT_HARBOR_INSTALL_SPEC = "harbor==0.6.6"
+TERMINUS_2_DEFAULT_VERSION = "harbor==0.6.6"
 TERMINUS_2_DEFAULT_PYTHON_VERSION = "3.12"
 TERMINUS_2_DEFAULT_MODEL_NAME = "openai/gpt-4.1-mini"
 TERMINUS_2_DEFAULT_API_BASE_URL = "https://api.pinference.ai/api/v1"
 
 
 class Terminus2ProgramConfig(vf.ProgramConfig):
+    _version: str = PrivateAttr(TERMINUS_2_DEFAULT_VERSION)
+
     agent_workdir: str = TERMINUS_2_DEFAULT_AGENT_WORKDIR
     instruction_path: str = TERMINUS_2_DEFAULT_INSTRUCTION_PATH
     system_prompt_path: str = TERMINUS_2_DEFAULT_SYSTEM_PROMPT_PATH
     log_path: str = TERMINUS_2_DEFAULT_LOG_PATH
-    harbor_install_spec: str = TERMINUS_2_DEFAULT_HARBOR_INSTALL_SPEC
     python_version: str = TERMINUS_2_DEFAULT_PYTHON_VERSION
     model_name: str = TERMINUS_2_DEFAULT_MODEL_NAME
     api_base_url: str = TERMINUS_2_DEFAULT_API_BASE_URL
@@ -171,7 +173,7 @@ async def main() -> None:
 cd "$TERMINUS_2_WORKDIR"
 uv --no-config run --no-project --quiet \
   --python {shlex.quote(self.python_version)} \
-  --with {shlex.quote(self.harbor_install_spec)} \
+  --with {shlex.quote(self._version)} \
   python - <<'PY' 2>&1 | tee -a {shlex.quote(self.log_path)}
 {agent_script}
 PY
@@ -186,8 +188,14 @@ async def main() -> None:
 
 
 class Terminus2Config(vf.HarnessConfig):
+    version: str = TERMINUS_2_DEFAULT_VERSION
     program: Terminus2ProgramConfig = Terminus2ProgramConfig()
 
+    @model_validator(mode="after")
+    def apply_version(self):
+        self.program._version = self.version
+        return self
+
 
 class Terminus2(vf.Harness[Terminus2Config]):
     config: Terminus2Config

tests/test_composable_env.py

@@ -193,8 +193,8 @@ def test_taskset_repr():
     assert "3" in repr(ts)
 
 
-def test_composable_mini_swe_agent_unversioned_install_spec_uses_unpinned_requirement():
-    setup = build_mini_swe_agent_install_script(install_spec="  mini-swe-agent  ")
+def test_composable_mini_swe_agent_unversioned_version_uses_unpinned_requirement():
+    setup = build_mini_swe_agent_install_script(version="  mini-swe-agent  ")
 
     assert (
         "vf_python_install --target /opt/mini-swe-agent/prefix/site-packages mini-swe-agent"
@@ -203,9 +203,9 @@ def test_composable_mini_swe_agent_unversioned_install_spec_uses_unpinned_requir
     assert "mini-swe-agent==mini-swe-agent" not in setup
 
 
-def test_composable_opencode_unversioned_install_spec_uses_latest_download_url():
+def test_composable_opencode_unversioned_version_uses_latest_download_url():
     setup = build_opencode_install_script(
-        install_spec="  PrimeIntellect-ai/opencode  ",
+        version="  PrimeIntellect-ai/opencode  ",
         install_ripgrep=False,
     )
 

tests/test_v1_harbor_cli.py

@@ -25,10 +25,10 @@
     Terminus2Config,
     Terminus2ProgramConfig,
 )
-from harnesses.pi import PI_DEFAULT_INSTALL_SPEC
+from harnesses.pi import PI_DEFAULT_VERSION
 from harnesses.terminus_2 import (
     TERMINUS_2_DEFAULT_API_BASE_URL,
-    TERMINUS_2_DEFAULT_HARBOR_INSTALL_SPEC,
+    TERMINUS_2_DEFAULT_VERSION,
     TERMINUS_2_DEFAULT_MODEL_NAME,
     Terminus2,
 )
@@ -287,18 +287,18 @@ def test_opencode_config_owns_opencode_harness_fields() -> None:
 
 
 @pytest.mark.parametrize(
-    "install_spec",
+    "version",
     ["PrimeIntellect-ai/opencode@latest", "  PrimeIntellect-ai/opencode  "],
 )
-def test_opencode_latest_install_spec_uses_latest_download_url(
-    install_spec: str,
+def test_opencode_latest_version_uses_latest_download_url(
+    version: str,
 ) -> None:
     harness = OpenCode(
         config=OpenCodeConfig(
+            version=version,
             program=OpenCodeProgramConfig(
-                install_spec=install_spec,
                 install_ripgrep=False,
-            )
+            ),
         )
     )
     program = cast(dict[str, object], harness.config.program.data())
@@ -308,12 +308,10 @@ def test_opencode_latest_install_spec_uses_latest_download_url(
     assert "OPENCODE_RELEASE_PATH=releases/latest/download" in setup
 
 
-def test_opencode_custom_install_spec_uses_versioned_release() -> None:
+def test_opencode_custom_version_uses_versioned_release() -> None:
     harness = OpenCode(
         config=OpenCodeConfig(
-            program=OpenCodeProgramConfig(
-                install_spec="Example/open-code@v2.0.0",
-            )
+            version="Example/open-code@v2.0.0",
         )
     )
     program = cast(dict[str, object], harness.config.program.data())
@@ -395,9 +393,9 @@ def test_pi_harness_writes_intercepted_model_and_mcp_config() -> None:
 
     assert "apt-get -o Acquire::Retries=3 update" in setup
     assert "apt-get -o Acquire::Retries=3 install" in setup
-    assert harness.config.program.install_spec == PI_DEFAULT_INSTALL_SPEC
-    assert PI_DEFAULT_INSTALL_SPEC == "@earendil-works/pi-coding-agent@latest"
-    assert f"npm install -g --ignore-scripts {PI_DEFAULT_INSTALL_SPEC}" in setup
+    assert harness.config.version == PI_DEFAULT_VERSION
+    assert PI_DEFAULT_VERSION == "@earendil-works/pi-coding-agent@latest"
+    assert f"npm install -g --ignore-scripts {PI_DEFAULT_VERSION}" in setup
     assert "mariozechner" not in setup
     assert '"baseUrl": "${OPENAI_BASE_URL}"' in mcp_setup
     assert '"api": "openai-completions"' in mcp_setup
@@ -407,12 +405,8 @@ def test_pi_harness_writes_intercepted_model_and_mcp_config() -> None:
     assert f'"command": "{SANDBOX_PYTHON}"' in mcp_setup
 
 
-def test_pi_harness_preserves_scoped_npm_install_specs() -> None:
-    harness = Pi(
-        config=PiConfig(
-            program=PiProgramConfig(install_spec="@anthropic-ai/claude-code@1.2.3")
-        )
-    )
+def test_pi_harness_preserves_scoped_npm_versions() -> None:
+    harness = Pi(config=PiConfig(version="@anthropic-ai/claude-code@1.2.3"))
     program = cast(dict[str, object], harness.config.program.data())
     setup = cast(str, program["setup"])
 
@@ -448,7 +442,7 @@ def test_terminus_2_harness_builds_sandbox_program() -> None:
 
     run_script = cast(str, command[2])
     assert "TERMINUS_2_WORKDIR=/workspace" in run_script
-    assert f"--with {TERMINUS_2_DEFAULT_HARBOR_INSTALL_SPEC}" in run_script
+    assert f"--with {TERMINUS_2_DEFAULT_VERSION}" in run_script
     assert "git+https://github.com" not in run_script
     assert "max_turns=7" in run_script