Add versioned agent install specs (#1506)

Author: xeophon

packages/harnesses/README.md

@@ -59,3 +59,17 @@ own a reusable execution mechanism.
 Harness implementations resolve to one `ProgramConfig` shape. Command harness
 configs may expose task-relevant execution knobs, but the harness owns command
 construction, channel wiring, sandbox placement, and artifacts.
+
+## Agent Versions
+
+Command agents use `name@version` specs where their installer supports a
+versioned package or release. Use `@latest` for a moving latest install:
+
+```toml
+[eval.harness.program]
+# OpenCode
+release = "PrimeIntellect-ai/opencode@latest"
+
+# MiniSWEAgent or Pi
+package = "mini-swe-agent@2.2.8"
+```

packages/harnesses/harnesses/mini_swe_agent.py

@@ -4,6 +4,8 @@
 import verifiers as vf
 from verifiers.v1.utils.sandbox_python_utils import python_runtime_setup_command
 
+from .utils import split_versioned_agent_spec
+
 DEFAULT_INSTALL_DIR = "/opt/mini-swe-agent"
 DEFAULT_PREFIX_DIR = f"{DEFAULT_INSTALL_DIR}/prefix"
 DEFAULT_SITE_PACKAGES_DIR = f"{DEFAULT_PREFIX_DIR}/site-packages"
@@ -14,41 +16,30 @@
 MINI_SWE_AGENT_DEFAULT_SYSTEM_PROMPT_PATH = "/mini-swe-agent/system.txt"
 MINI_SWE_AGENT_DEFAULT_LOG_PATH = "/logs/agent/mini-swe-agent.log"
 MINI_SWE_AGENT_DEFAULT_TRAJECTORY_PATH = "/logs/agent/mini-swe-agent.traj.json"
-MINI_SWE_AGENT_DEFAULT_PACKAGE_VERSION = "2.2.8"
-MINI_SWE_AGENT_DEFAULT_PACKAGE_SHA256 = (
-    "694df4de1337e665e3cd82e99f93374f573bf52b8e7c362ac5d8045ad9f7c37c"
-)
+MINI_SWE_AGENT_DEFAULT_PACKAGE = "mini-swe-agent@2.2.8"
 MINI_SWE_AGENT_DEFAULT_CONFIG_SPEC = "mini"
 MINI_SWE_AGENT_DEFAULT_MODEL_CLASS = "litellm"
 MINI_SWE_AGENT_DEFAULT_ENVIRONMENT_TIMEOUT = 120
 
 
 def build_mini_swe_agent_install_script(
-    package_version: str = MINI_SWE_AGENT_DEFAULT_PACKAGE_VERSION,
-    package_sha256: str = MINI_SWE_AGENT_DEFAULT_PACKAGE_SHA256,
+    package: str = MINI_SWE_AGENT_DEFAULT_PACKAGE,
     prefix_dir: str = DEFAULT_PREFIX_DIR,
 ) -> str:
     install_dir = str(PurePosixPath(prefix_dir).parent)
     site_packages_dir = f"{prefix_dir.rstrip('/')}/site-packages"
-    wheel_filename = f"mini_swe_agent-{package_version}-py3-none-any.whl"
-    wheel_url = (
-        f"https://files.pythonhosted.org/packages/py3/m/mini-swe-agent/{wheel_filename}"
-    )
     setup_prefix_dir = shlex.quote(prefix_dir)
     setup_site_packages_dir = shlex.quote(site_packages_dir)
+    package_name, package_version = split_versioned_agent_spec(package)
+    package_requirement = package_name
+    if package_version and package_version != "latest":
+        package_requirement = f"{package_name}=={package_version}"
     return f"""\
 set -e
 {python_runtime_setup_command()}
 rm -rf {setup_prefix_dir}
 mkdir -p {shlex.quote(install_dir)} {setup_prefix_dir}/bin {setup_site_packages_dir} {shlex.quote(DEFAULT_LOG_DIR)} /mini-swe-agent
-MINI_SWE_AGENT_WHEEL_DIR="$(mktemp -d)"
-trap 'rm -rf "$MINI_SWE_AGENT_WHEEL_DIR"' EXIT
-MINI_SWE_AGENT_WHEEL="$MINI_SWE_AGENT_WHEEL_DIR/{wheel_filename}"
-MINI_SWE_AGENT_WHEEL_URL={shlex.quote(wheel_url)}
-export MINI_SWE_AGENT_WHEEL MINI_SWE_AGENT_WHEEL_URL
-"$VF_PYTHON" -c 'import os, urllib.request; urllib.request.urlretrieve(os.environ["MINI_SWE_AGENT_WHEEL_URL"], os.environ["MINI_SWE_AGENT_WHEEL"])'
-echo "{package_sha256}  $MINI_SWE_AGENT_WHEEL" | sha256sum -c -
-vf_python_install --target {setup_site_packages_dir} "$MINI_SWE_AGENT_WHEEL"
+vf_python_install --target {setup_site_packages_dir} {shlex.quote(package_requirement)}
 echo "$VF_PYTHON" > {setup_prefix_dir}/python
 cat > {setup_prefix_dir}/bin/mini <<'EOF'
 #!/usr/bin/env sh
@@ -66,8 +57,7 @@ class MiniSWEAgentProgramConfig(vf.ProgramConfig):
     system_prompt_path: str = MINI_SWE_AGENT_DEFAULT_SYSTEM_PROMPT_PATH
     log_path: str = MINI_SWE_AGENT_DEFAULT_LOG_PATH
     trajectory_path: str = MINI_SWE_AGENT_DEFAULT_TRAJECTORY_PATH
-    package_version: str = MINI_SWE_AGENT_DEFAULT_PACKAGE_VERSION
-    package_sha256: str = MINI_SWE_AGENT_DEFAULT_PACKAGE_SHA256
+    package: str = MINI_SWE_AGENT_DEFAULT_PACKAGE
     config_spec: str = MINI_SWE_AGENT_DEFAULT_CONFIG_SPEC
     model_class: str = MINI_SWE_AGENT_DEFAULT_MODEL_CLASS
     environment_timeout: int = MINI_SWE_AGENT_DEFAULT_ENVIRONMENT_TIMEOUT
@@ -125,8 +115,7 @@ def resolve(self) -> vf.ProgramConfig:
             config_args.extend(["-c", shlex.quote(spec)])
 
         setup = build_mini_swe_agent_install_script(
-            package_version=self.package_version,
-            package_sha256=self.package_sha256,
+            package=self.package,
         )
         log_dir = str(PurePosixPath(self.log_path).parent)
         trajectory_dir = str(PurePosixPath(self.trajectory_path).parent)

packages/harnesses/harnesses/opencode.py

@@ -5,11 +5,9 @@
 import verifiers as vf
 from verifiers.v1.utils.mcp_proxy_utils import proxy_command
 
-OPENCODE_DEFAULT_RELEASE_REPO = "PrimeIntellect-ai/opencode"
-OPENCODE_DEFAULT_RELEASE_VERSION = "1.1.63-rl2"
-OPENCODE_DEFAULT_RELEASE_SHA256 = (
-    "47f4102796da50769e27d2c9ea6a9cf7941f76898390cb497278cab39c4b6ed4"
-)
+from .utils import split_versioned_agent_spec
+
+OPENCODE_DEFAULT_RELEASE = "PrimeIntellect-ai/opencode@1.1.63-rl2"
 OPENCODE_DEFAULT_AGENT_WORKDIR = "/app"
 OPENCODE_DEFAULT_INSTRUCTION_PATH = "/opencode/instruction.txt"
 OPENCODE_DEFAULT_SYSTEM_PROMPT_PATH = "/opencode/system.txt"
@@ -52,9 +50,7 @@ class OpenCodeProgramConfig(vf.ProgramConfig):
     disabled_tools: list[str] = OPENCODE_DEFAULT_DISABLED_TOOLS
     allow_git: bool = False
     disable_compaction: bool = True
-    release_repo: str = OPENCODE_DEFAULT_RELEASE_REPO
-    release_version: str = OPENCODE_DEFAULT_RELEASE_VERSION
-    release_sha256: str = OPENCODE_DEFAULT_RELEASE_SHA256
+    release: str = OPENCODE_DEFAULT_RELEASE
     install_ripgrep: bool = True
     provider_timeout_ms: int = 3_600_000
 
@@ -79,18 +75,24 @@ def resolve(self) -> vf.ProgramConfig:
             if self.install_ripgrep
             else ""
         )
-        sha256_check = (
-            f'echo "{self.release_sha256}  /tmp/opencode.tar.gz" | sha256sum -c -'
-        )
+        release_repo, release_version = split_versioned_agent_spec(self.release)
+        release_path = "releases/latest/download"
+        if release_version and release_version != "latest":
+            release_tag = (
+                release_version
+                if release_version.startswith("v")
+                else f"v{release_version}"
+            )
+            release_path = f"releases/download/{release_tag}"
         # Acquire::Retries=3 mitigates transient archive.ubuntu.com CDN sync
         # mismatches that fail fresh-sandbox apt-get calls mid-rollout.
         setup = f"""\
 set -e
 apt-get -o Acquire::Retries=3 update -qq && apt-get -o Acquire::Retries=3 install -y -qq curl tar ca-certificates > /dev/null 2>&1
 {rg_install}
 
-OPENCODE_RELEASE_REPO={shlex.quote(self.release_repo)}
-OPENCODE_RELEASE_VERSION={shlex.quote(self.release_version)}
+OPENCODE_RELEASE_REPO={shlex.quote(release_repo)}
+OPENCODE_RELEASE_PATH={shlex.quote(release_path)}
 
 case "$(uname -m)" in
   x86_64) OPENCODE_ARCH=x64 ;;
@@ -99,15 +101,13 @@ def resolve(self) -> vf.ProgramConfig:
 esac
 
 OPENCODE_ASSET="opencode-linux-$OPENCODE_ARCH.tar.gz"
-OPENCODE_RELEASE_TAG="${{OPENCODE_RELEASE_VERSION#v}}"
-OPENCODE_RELEASE_URL="https://github.com/$OPENCODE_RELEASE_REPO/releases/download/v$OPENCODE_RELEASE_TAG/$OPENCODE_ASSET"
+OPENCODE_RELEASE_URL="https://github.com/$OPENCODE_RELEASE_REPO/$OPENCODE_RELEASE_PATH/$OPENCODE_ASSET"
 
 mkdir -p "$HOME/.opencode/bin"
 if [ -x "$HOME/.opencode/bin/opencode" ]; then
   echo "OpenCode already installed, skipping download"
 else
   curl -fsSL "$OPENCODE_RELEASE_URL" -o /tmp/opencode.tar.gz
-  {sha256_check}
   tar -xzf /tmp/opencode.tar.gz -C /tmp
   install -m 755 /tmp/opencode "$HOME/.opencode/bin/opencode"
   rm -f /tmp/opencode.tar.gz /tmp/opencode

packages/harnesses/harnesses/pi.py

@@ -5,7 +5,7 @@
 import verifiers as vf
 from verifiers.v1.utils.mcp_proxy_utils import proxy_command
 
-PI_DEFAULT_PACKAGE = "@earendil-works/pi-coding-agent"
+PI_DEFAULT_PACKAGE = "@earendil-works/pi-coding-agent@latest"
 PI_DEFAULT_WORKDIR = "/app"
 PI_DEFAULT_INSTRUCTION_PATH = "/pi/instruction.txt"
 PI_DEFAULT_SYSTEM_PROMPT_PATH = "/pi/system.txt"

packages/harnesses/harnesses/utils/__init__.py

@@ -1 +1,10 @@
 """Internal harness utilities."""
+
+
+def split_versioned_agent_spec(spec: str) -> tuple[str, str | None]:
+    """Split an agent install spec written as name[@version]."""
+    spec = spec.strip()
+    name, _, version = spec.rpartition("@")
+    if not name:
+        return spec, None
+    return name, version or None

tests/test_composable_env.py

@@ -17,6 +17,12 @@
     TaskSet,
     discover_sibling_dir,
 )
+from verifiers.envs.experimental.composable.harnesses.mini_swe_agent import (
+    build_mini_swe_agent_install_script,
+)
+from verifiers.envs.experimental.composable.harnesses.opencode import (
+    build_install_script as build_opencode_install_script,
+)
 
 
 # ── Mock Rubrics ──────────────────────────────────────────────────────
@@ -187,6 +193,27 @@ def test_taskset_repr():
     assert "3" in repr(ts)
 
 
+def test_composable_mini_swe_agent_unversioned_package_uses_unpinned_requirement():
+    setup = build_mini_swe_agent_install_script(package="  mini-swe-agent  ")
+
+    assert (
+        "vf_python_install --target /opt/mini-swe-agent/prefix/site-packages mini-swe-agent"
+        in setup
+    )
+    assert "mini-swe-agent==mini-swe-agent" not in setup
+
+
+def test_composable_opencode_unversioned_release_uses_latest_download_url():
+    setup = build_opencode_install_script(
+        release="  PrimeIntellect-ai/opencode  ",
+        install_ripgrep=False,
+    )
+
+    assert "OPENCODE_RELEASE_REPO=PrimeIntellect-ai/opencode" in setup
+    assert "OPENCODE_RELEASE_PATH=releases/latest/download" in setup
+    assert "releases/download/vPrimeIntellect-ai/opencode" not in setup
+
+
 @pytest.mark.asyncio
 async def test_composable_env_exports_task_workdir():
     taskset = MockSandboxTaskSet(dataset=_make_dataset(), name="test")

tests/test_multiturn_env.py

@@ -189,15 +189,15 @@ async def env_response(self, messages, state, **kwargs):  # type: ignore[overrid
 
             async def add_model_response(self, state, prompt_messages, response):  # type: ignore[override]
                 await super().add_model_response(state, prompt_messages, response)
-                await asyncio.sleep(0.05)
+                await asyncio.sleep(1)
 
         env = SlowMultiTurnEnv(
             client=mock_client,
             model="test-model",
             dataset=sample_chat_dataset,
             parser=Parser(),
             rubric=Rubric(),
-            timeout_seconds=0.01,
+            timeout_seconds=0.2,
         )
         mock_client.set_default_response("Still going")
 

tests/test_v1_harbor_cli.py

@@ -279,11 +279,47 @@ def test_opencode_config_owns_opencode_harness_fields() -> None:
     assert harness.config.max_turns == 2
     assert "apt-get -o Acquire::Retries=3 update" in setup
     assert "apt-get -o Acquire::Retries=3 install" in setup
+    assert "OPENCODE_RELEASE_REPO=PrimeIntellect-ai/opencode" in setup
+    assert "OPENCODE_RELEASE_PATH=releases/download/v1.1.63-rl2" in setup
     assert "/workspace" in cast(str, command[2])
     assert '"webfetch": false' in cast(str, mcp_setup)
     assert "/opencode/system.txt" in cast(dict[str, object], program["files"])
 
 
+@pytest.mark.parametrize(
+    "release", ["PrimeIntellect-ai/opencode@latest", "  PrimeIntellect-ai/opencode  "]
+)
+def test_opencode_latest_release_uses_latest_download_url(release: str) -> None:
+    harness = OpenCode(
+        config=OpenCodeConfig(
+            program=OpenCodeProgramConfig(
+                release=release,
+                install_ripgrep=False,
+            )
+        )
+    )
+    program = cast(dict[str, object], harness.config.program.data())
+    setup = cast(str, program["setup"])
+
+    assert "OPENCODE_RELEASE_REPO=PrimeIntellect-ai/opencode" in setup
+    assert "OPENCODE_RELEASE_PATH=releases/latest/download" in setup
+
+
+def test_opencode_custom_release_uses_versioned_spec() -> None:
+    harness = OpenCode(
+        config=OpenCodeConfig(
+            program=OpenCodeProgramConfig(
+                release="Example/open-code@v2.0.0",
+            )
+        )
+    )
+    program = cast(dict[str, object], harness.config.program.data())
+    setup = cast(str, program["setup"])
+
+    assert "OPENCODE_RELEASE_REPO=Example/open-code" in setup
+    assert "OPENCODE_RELEASE_PATH=releases/download/v2.0.0" in setup
+
+
 @pytest.mark.parametrize(
     ("harness_cls", "config_cls", "program_cls"),
     [
@@ -357,7 +393,7 @@ def test_pi_harness_writes_intercepted_model_and_mcp_config() -> None:
     assert "apt-get -o Acquire::Retries=3 update" in setup
     assert "apt-get -o Acquire::Retries=3 install" in setup
     assert harness.config.program.package == PI_DEFAULT_PACKAGE
-    assert PI_DEFAULT_PACKAGE == "@earendil-works/pi-coding-agent"
+    assert PI_DEFAULT_PACKAGE == "@earendil-works/pi-coding-agent@latest"
     assert f"npm install -g --ignore-scripts {PI_DEFAULT_PACKAGE}" in setup
     assert "mariozechner" not in setup
     assert '"baseUrl": "${OPENAI_BASE_URL}"' in mcp_setup
@@ -368,6 +404,18 @@ def test_pi_harness_writes_intercepted_model_and_mcp_config() -> None:
     assert f'"command": "{SANDBOX_PYTHON}"' in mcp_setup
 
 
+def test_pi_harness_preserves_scoped_npm_package_versions() -> None:
+    harness = Pi(
+        config=PiConfig(
+            program=PiProgramConfig(package="@anthropic-ai/claude-code@1.2.3")
+        )
+    )
+    program = cast(dict[str, object], harness.config.program.data())
+    setup = cast(str, program["setup"])
+
+    assert "npm install -g --ignore-scripts @anthropic-ai/claude-code@1.2.3" in setup
+
+
 def test_terminus_2_harness_builds_sandbox_program() -> None:
     harness = Terminus2(
         config=Terminus2Config(

tests/test_v1_mini_swe_agent.py

@@ -80,11 +80,38 @@ def test_mini_swe_agent_builds_sandbox_program():
     assert "model.model_kwargs.parallel_tool_calls=true" in script
     assert "apt-get -o Acquire::Retries=3 update" in cast(str, program["setup"])
     assert "apt-get -o Acquire::Retries=3 install" in cast(str, program["setup"])
+    assert "mini-swe-agent==2.2.8" in cast(str, program["setup"])
     assert "/mini-swe-agent/prompt.txt" in cast(dict[str, object], program["files"])
     assert "/mini-swe-agent/system.txt" in cast(dict[str, object], program["files"])
     assert "mini_swe_agent_log" in cast(dict[str, object], program["artifacts"])
 
 
+@pytest.mark.parametrize("package", ["mini-swe-agent@latest", "  mini-swe-agent  "])
+def test_mini_swe_agent_latest_package_uses_unpinned_pip_requirement(package: str):
+    harness = MiniSWEAgent(
+        config=MiniSWEAgentConfig(program=MiniSWEAgentProgramConfig(package=package))
+    )
+    program = cast(dict[str, Any], harness.config.program.data())
+    setup = cast(str, program["setup"])
+
+    assert (
+        "vf_python_install --target /opt/mini-swe-agent/prefix/site-packages mini-swe-agent"
+        in setup
+    )
+
+
+def test_mini_swe_agent_pinned_package_uses_pip_requirement():
+    harness = MiniSWEAgent(
+        config=MiniSWEAgentConfig(
+            program=MiniSWEAgentProgramConfig(package="mini-swe-agent@2.2.7")
+        )
+    )
+    program = cast(dict[str, Any], harness.config.program.data())
+    setup = cast(str, program["setup"])
+
+    assert "mini-swe-agent==2.2.7" in setup
+
+
 def test_mini_swe_agent_composes_with_harbor_taskset(
     tmp_path: Path, monkeypatch: pytest.MonkeyPatch
 ):

verifiers/envs/experimental/composable/harnesses/__init__.py

@@ -9,7 +9,7 @@
 )
 from verifiers.envs.experimental.composable.harnesses.opencode import (
     DEFAULT_DISABLED_TOOLS,
-    DEFAULT_RELEASE_SHA256,
+    DEFAULT_RELEASE,
     DEFAULT_SYSTEM_PROMPT,
     OPENCODE_INSTALL_SCRIPT,
     build_install_script as build_opencode_install_script,
@@ -39,7 +39,7 @@
     "build_opencode_run_command",
     "OPENCODE_INSTALL_SCRIPT",
     "DEFAULT_DISABLED_TOOLS",
-    "DEFAULT_RELEASE_SHA256",
+    "DEFAULT_RELEASE",
     "DEFAULT_SYSTEM_PROMPT",
     "mini_swe_agent_harness",
     "build_mini_swe_agent_install_script",