From 07e5e515132480b31e825bb8aec6000b8f7e7a46 Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Tue, 23 Jun 2026 16:04:09 +0200 Subject: [PATCH 1/7] Add interception-only Docker networking --- tests/v1/test_runtimes.py | 210 +++++++++++++++++++++++++++ verifiers/v1/ARCHITECTURE.md | 7 +- verifiers/v1/GUIDE.md | 18 +++ verifiers/v1/env.py | 3 +- verifiers/v1/interception/pool.py | 34 +++-- verifiers/v1/rollout.py | 20 ++- verifiers/v1/runtimes/__init__.py | 12 +- verifiers/v1/runtimes/base.py | 49 +++++-- verifiers/v1/runtimes/docker.py | 234 +++++++++++++++++++++++++++--- 9 files changed, 532 insertions(+), 55 deletions(-) create mode 100644 tests/v1/test_runtimes.py diff --git a/tests/v1/test_runtimes.py b/tests/v1/test_runtimes.py new file mode 100644 index 0000000000..4b5d288bf6 --- /dev/null +++ b/tests/v1/test_runtimes.py @@ -0,0 +1,210 @@ +"""Runtime topology tests, including an optional local-Docker isolation proof.""" + +import asyncio +import subprocess +import uuid +from typing import Literal + +import pytest + +import verifiers.v1.runtimes.docker as docker_module +from verifiers.v1.interception.pool import InterceptionPool +from verifiers.v1.errors import SandboxError +from verifiers.v1.runtimes import ( + DockerConfig, + runtime_is_local, + runtime_reaches_host_locally, +) +from verifiers.v1.runtimes.base import ProgramResult +from verifiers.v1.runtimes.docker import DockerRuntime + + +def result(stdout: str = "", stderr: str = "", exit_code: int = 0) -> ProgramResult: + return ProgramResult(exit_code=exit_code, stdout=stdout, stderr=stderr) + + +@pytest.mark.parametrize( + ("network_access", "network"), + [("full", "host"), ("interception", "bridge")], +) +async def test_docker_start_uses_setup_network( + monkeypatch, + network_access: Literal["full", "interception"], + network: str, +) -> None: + calls: list[tuple[str, ...]] = [] + + async def fake_docker(*args: str) -> ProgramResult: + calls.append(args) + return result("29.0.0" if args[0] == "version" else "abcdef1234567890") + + monkeypatch.setattr(docker_module, "docker", fake_docker) + runtime = DockerRuntime(DockerConfig(network_access=network_access), name="vf-test") + await runtime.start() + + run = next(args for args in calls if args[0] == "run") + assert run[run.index("--network") + 1] == network + if network == "bridge": + assert ("--cap-drop", "NET_ADMIN") == run[4:6] + + +def test_interception_docker_is_local_but_needs_a_host_tunnel() -> None: + config = DockerConfig(network_access="interception") + + assert runtime_is_local(config) + assert not runtime_reaches_host_locally(config) + assert not InterceptionPool(config, multiplex=1).host_is_local + + +async def test_docker_seals_agent_behind_fixed_interception_relay(monkeypatch) -> None: + calls: list[tuple[str, ...]] = [] + + async def fake_docker(*args: str) -> ProgramResult: + calls.append(args) + return result("ok") + + monkeypatch.setattr(docker_module, "docker", fake_docker) + runtime = DockerRuntime(DockerConfig(network_access="interception"), name="vf-test") + runtime._container = runtime.name + + endpoint = await runtime.seal_agent_network( + "https://rollout.tunnel.example/v1?dialect=responses" + ) + + assert endpoint == "http://vf-interception:8080/v1?dialect=responses" + assert calls[0][:4] == ("network", "create", "--internal", "--opt") + relay = next(args for args in calls if args[0] == "create") + config = relay[-1] + assert "nginx:1.28.3-alpine" in relay + assert "proxy_pass https://rollout.tunnel.example;" in config + assert "proxy_set_header Host rollout.tunnel.example;" in config + assert "proxy_ssl_verify on;" in config + assert ( + "network", + "connect", + "vf-test-network", + "vf-test", + ) in calls + assert ("network", "disconnect", "bridge", "vf-test") in calls + + +async def test_docker_rejects_non_http_interception_endpoint(monkeypatch) -> None: + async def unexpected_docker(*args: str) -> ProgramResult: + raise AssertionError(f"unexpected docker call: {args}") + + monkeypatch.setattr(docker_module, "docker", unexpected_docker) + runtime = DockerRuntime(DockerConfig(network_access="interception"), name="vf-test") + runtime._container = runtime.name + + with pytest.raises(SandboxError, match="invalid interception endpoint"): + await runtime.seal_agent_network("file:///tmp/interception.sock") + + +def test_docker_cleanup_removes_relay_container_and_network(monkeypatch) -> None: + calls: list[list[str]] = [] + + def fake_run(argv: list[str], **kwargs) -> subprocess.CompletedProcess: + calls.append(argv) + return subprocess.CompletedProcess(argv, 0) + + monkeypatch.setattr(docker_module.subprocess, "run", fake_run) + runtime = DockerRuntime(DockerConfig(network_access="interception"), name="vf-test") + runtime._container = "vf-test" + runtime._relay = "vf-test-relay" + runtime._network = "vf-test-network" + + runtime.cleanup() + runtime.cleanup() + + assert calls == [ + ["docker", "rm", "--force", "vf-test-relay"], + ["docker", "rm", "--force", "vf-test"], + ["docker", "network", "rm", "vf-test-network"], + ] + + +@pytest.mark.integration +@pytest.mark.docker +async def test_interception_network_allows_relay_and_blocks_direct_egress() -> None: + version = await docker_module.docker("version", "--format", "{{.Server.Version}}") + if version.exit_code != 0: + pytest.skip("Docker daemon is not available") + + suffix = uuid.uuid4().hex[:12] + upstream = f"vf-upstream-{suffix}" + runtime = DockerRuntime( + DockerConfig(network_access="interception"), name=f"vf-agent-{suffix}" + ) + server = await docker_module.docker( + "run", + "--detach", + "--network", + "bridge", + "--name", + upstream, + "python:3.11-slim", + "python", + "-m", + "http.server", + "8000", + ) + if server.exit_code != 0: + pytest.fail(f"test upstream failed to start: {server.stderr}") + try: + for _ in range(20): + ready = await docker_module.docker( + "exec", + upstream, + "python", + "-c", + "import socket; socket.create_connection(('127.0.0.1', 8000), 1)", + ) + if ready.exit_code == 0: + break + await asyncio.sleep(0.05) + else: + pytest.fail("test upstream did not become ready") + await runtime.start() + inspected = await docker_module.docker( + "inspect", + "--format", + "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", + upstream, + ) + upstream_ip = inspected.stdout.strip() + assert upstream_ip + + endpoint = await runtime.seal_agent_network(f"http://{upstream_ip}:8000/") + allowed = await runtime.run( + [ + "python", + "-c", + "import sys, urllib.request; urllib.request.urlopen(sys.argv[1], timeout=3).read()", + endpoint, + ], + {}, + ) + direct = await runtime.run( + [ + "python", + "-c", + "import socket, sys; socket.create_connection((sys.argv[1], 8000), 1)", + upstream_ip, + ], + {}, + ) + internet = await runtime.run( + [ + "python", + "-c", + "import socket; socket.create_connection(('1.1.1.1', 80), 1)", + ], + {}, + ) + + assert allowed.exit_code == 0, allowed.stderr + assert direct.exit_code != 0 + assert internet.exit_code != 0 + finally: + await runtime.stop() + await docker_module.docker("rm", "--force", upstream) diff --git a/verifiers/v1/ARCHITECTURE.md b/verifiers/v1/ARCHITECTURE.md index 047575ad71..297f5ef58f 100644 --- a/verifiers/v1/ARCHITECTURE.md +++ b/verifiers/v1/ARCHITECTURE.md @@ -144,8 +144,11 @@ thousands of servers or tunnels. A `Runtime` (`runtimes/base.py`) is the single contract for *where* code runs: `start`/`stop`/`cleanup`, `run(argv, env)` and `run_background(...)`, `run_uv_script(...)`, -`read`/`write`, and `expose(port)` (the URL by which the host reaches a port inside the -runtime — localhost for subprocess, a tunnel for prime/modal). The same contract backs the +`read`/`write`, `seal_agent_network(endpoint)`, and `expose(port)` (the URL by which the host +reaches a port inside the runtime — localhost for subprocess, a tunnel for prime/modal). +`seal_agent_network` is the setup-to-execution boundary: normally a no-op, while an +interception-only Docker runtime replaces setup networking with an internal network and a fixed +reverse relay to the interception tunnel. The same contract backs the harness, a task's tool servers, and the user simulator, so any of them runs in any backend: `subprocess` (local, `/tmp/` workspace, own process group), `docker` (local container), `prime` (remote sandbox), `modal` (remote function). diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index ab2f7d3b71..70f8357407 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -627,6 +627,24 @@ uv run eval gsm8k-v1 -n 1 --harness.runtime.type modal # remote modal sand A taskset that sets `NEEDS_CONTAINER` (or a task with an `image`) refuses the subprocess runtime — pass `docker` / `prime` / `modal`. +## Interception-only Docker agents + +Set `network_access = "interception"` to let Docker setup download harness dependencies, then +remove internet access before the agent starts: + +```toml +harness = { id = "codex", runtime = { type = "docker", network_access = "interception" } } +``` + +The runtime moves the agent onto an internal per-rollout network and rewrites its model endpoint +through a fixed reverse relay to the interception tunnel. The relay cannot proxy another host, and +the agent has no route to the internet, host, or other containers. The restriction remains active +through finalization and scoring so a child process left by the agent cannot regain network access. +Docker pulls the pinned official `nginx:1.28.3-alpine` relay image on first use. + +This mode currently supports tasksets without MCP tools. User simulators must run in their own +runtime rather than `colocated`; both cases fail early with a clear rollout error. + --- # CLI reference diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index 7113b2af81..b1c1e8e6ae 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -30,6 +30,7 @@ RuntimeConfig, SubprocessConfig, runtime_is_local, + runtime_reaches_host_locally, ) from verifiers.v1.task import Task from verifiers.v1.taskset import TasksetConfig @@ -400,6 +401,6 @@ async def shared_tools(self, tasks: list[Task]): if not any(server.config.shared for server in servers): yield {} return - harness_is_local = runtime_is_local(self.harness.config.runtime) + harness_is_local = runtime_reaches_host_locally(self.harness.config.runtime) async with serve_shared(servers, harness_is_local=harness_is_local) as urls: yield urls diff --git a/verifiers/v1/interception/pool.py b/verifiers/v1/interception/pool.py index 3b25d6e3f3..01184af4a4 100644 --- a/verifiers/v1/interception/pool.py +++ b/verifiers/v1/interception/pool.py @@ -1,14 +1,12 @@ """A pool of shared interception servers, grown on demand, so N concurrent rollouts need ~N/multiplex servers + tunnels rather than one each. -Behind a remote runtime each rollout's interception endpoint needs a tunnel, and tunnel -creation is rate-capped per API token — so one-tunnel-per-rollout caps how wide a remote -eval (or env server) can fan out. Each shared `InterceptionServer` serves up to `multiplex` -rollouts behind one tunnel (created via a host-side exposer runtime of the harness's runtime -type). The pool is elastic: `acquire` reuses a server with a free slot, else brings up a new -one — so it fits both the bounded eval runner and the env server's unbounded request load. -The harness is unchanged: it authenticates with a per-rollout secret, which is what the -server routes by. +A harness runtime that cannot reach the host locally needs a tunnel for each interception +endpoint, and tunnel creation is rate-capped per API token — so one tunnel per rollout caps +fan-out. Each shared `InterceptionServer` serves up to `multiplex` rollouts behind one tunnel. +The pool is elastic: `acquire` reuses a server with a free slot, else brings up a new one — so it +fits both the bounded eval runner and the env server's unbounded request load. The harness is +unchanged: it authenticates with a per-rollout secret, which is what the server routes by. """ import asyncio @@ -17,7 +15,12 @@ from dataclasses import dataclass from verifiers.v1.interception.server import InterceptionServer, RolloutSession -from verifiers.v1.runtimes import HOST, RuntimeConfig, reachable_url, runtime_is_local +from verifiers.v1.runtimes import ( + HOST, + RuntimeConfig, + reachable_url, + runtime_reaches_host_locally, +) logger = logging.getLogger(__name__) @@ -35,11 +38,10 @@ class InterceptionPool: bringing up a new server when all are at capacity.""" def __init__(self, runtime_config: RuntimeConfig, multiplex: int) -> None: - # The harness runtime's topology decides reachability: a remote one needs a host tunnel - # to the interception port, a local one is reached at localhost. Read off the runtime - # class (no provisioning) — the pool never runs a sandbox. + # The harness config decides reachability: provider sandboxes and interception-only Docker + # need a host tunnel; ordinary local runtimes use localhost. The pool provisions neither. self.runtime_type = runtime_config.type - self.is_local = runtime_is_local(runtime_config) + self.host_is_local = runtime_reaches_host_locally(runtime_config) self.multiplex = max(1, multiplex) self._servers: list[PooledServer] = [] self._lock = asyncio.Lock() @@ -61,10 +63,10 @@ async def _entry(self) -> PooledServer: return entry server = InterceptionServer() await self._stack.enter_async_context(server) - # The interception server is a HOST service the harness reaches: localhost for a local - # harness runtime, a tunnel for a remote one. Owned by the pool's stack, torn down with it. + # The interception server is a HOST service the harness reaches through localhost or a + # tunnel. The pool's stack owns both and tears them down together. url = await self._stack.enter_async_context( - reachable_url(HOST, server.port, consumer_is_local=self.is_local) + reachable_url(HOST, server.port, consumer_is_local=self.host_is_local) ) entry = PooledServer(server, url) self._servers.append(entry) diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index f7b56364d8..9557f1b796 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -26,6 +26,7 @@ RolloutError, TasksetError, ToolsetError, + UserError, boundary, ) from verifiers.v1.interception import ( @@ -181,6 +182,22 @@ async def run(self) -> Trace: ): async with boundary(ToolsetError, "building tool servers"): tool_servers = self.taskset.tools(self.task) + if runtime.interception_only and tool_servers: + raise ValueError( + "Docker network_access='interception' does not yet support MCP " + "tool servers; use a taskset without tools" + ) + async with boundary(UserError, "building user simulator"): + user = self.taskset.user(self.task) + if ( + runtime.interception_only + and user is not None + and user.config.colocated + ): + raise ValueError( + "Docker network_access='interception' does not support a colocated " + "user simulator; run the user in its own runtime" + ) async with ( serve_tools( tool_servers, @@ -192,7 +209,7 @@ async def run(self) -> Trace: state_base=state_base, ) as urls, serve_user( - self.taskset.user(self.task), + user, self.task, harness_runtime=runtime, state_port=state_port, @@ -206,6 +223,7 @@ async def run(self) -> Trace: "conversation; set task.prompt or have Taskset.user return " "a simulator" ) + endpoint = await runtime.seal_agent_network(endpoint) # setup done — the harness is now driving now = time.time() trace.timing.setup.end = now diff --git a/verifiers/v1/runtimes/__init__.py b/verifiers/v1/runtimes/__init__.py index 7b0eb5e755..d98786c0a3 100644 --- a/verifiers/v1/runtimes/__init__.py +++ b/verifiers/v1/runtimes/__init__.py @@ -47,19 +47,23 @@ def make_runtime(config: RuntimeConfig, name: str | None = None) -> Runtime: def runtime_is_local(config: RuntimeConfig) -> bool: - """Whether a runtime of this config shares the host network (so a program inside it reaches a - host service at localhost, no tunnel) — read off the runtime class, without provisioning one. - The interception pool / rollout use it to decide whether to tunnel their host port via - `host_endpoint`.""" + """Whether this is a local runtime rather than a provider sandbox. Config-specific network + restrictions do not change this topology; use `runtime_reaches_host_locally` for host URLs.""" return _runtime_cls(config).is_local +def runtime_reaches_host_locally(config: RuntimeConfig) -> bool: + """Whether this runtime config reaches a host service at localhost rather than by tunnel.""" + return _runtime_cls(config).config_reaches_host_locally(config) + + __all__ = [ "ProgramResult", "Runtime", "RuntimeConfig", "make_runtime", "runtime_is_local", + "runtime_reaches_host_locally", "host_endpoint", "reachable_url", "HOST", diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index 5670f89bb2..6fce471dde 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -102,10 +102,9 @@ def cleanup_at_exit() -> None: class Runtime(ABC): is_local: ClassVar[bool] = True - """Whether this runtime shares the host network — a program inside it reaches a host service - at localhost (no tunnel) and a service inside it is reachable at localhost. True for - subprocess / docker(--network host); remote runtimes (modal/prime) override to False (they - need a tunnel each way: `host_endpoint` inward, `expose` outward).""" + """Whether this runtime is ordinarily on the host network. True for subprocess / Docker; + remote runtimes (Modal/Prime) override to False. Config-specific restrictions may still make + a local runtime require a tunnel to reach the host; use `host_is_local` for that direction.""" def __init__(self, name: str | None = None) -> None: self.name = name or f"vf-{uuid.uuid4().hex[:12]}" @@ -129,6 +128,21 @@ def descriptor(self) -> str | None: runtime: subprocess workdir, docker image, prime sandbox id.""" return None + @classmethod + def config_reaches_host_locally(cls, config: object) -> bool: + """Whether this config reaches a host service at localhost rather than through a tunnel.""" + return cls.is_local + + @property + def host_is_local(self) -> bool: + """Whether this instance reaches a host service at localhost.""" + return type(self).config_reaches_host_locally(self.config) + + @property + def interception_only(self) -> bool: + """Whether agent execution is limited to the interception endpoint.""" + return False + # --- lifecycle --- @abstractmethod @@ -160,6 +174,14 @@ async def run_program(self, argv: list[str], env: dict[str, str]) -> ProgramResu still retry individual safe transport operations underneath `run`.""" return await self.run(argv, env) + async def seal_agent_network(self, endpoint: str) -> str: + """Apply the runtime's agent-execution network policy and return its reachable endpoint. + + Setup has already completed when this is called. Most runtimes keep their existing + network and endpoint; Docker's interception-only mode replaces both for agent execution. + """ + return endpoint + async def run_background( self, argv: list[str], env: dict[str, str], log: str ) -> None: @@ -299,11 +321,11 @@ async def open_tunnel( @contextlib.asynccontextmanager async def host_endpoint(port: int, is_local: bool, labels: list[str] | None = None): """Yield a URL a program *inside a runtime* uses to reach a HOST service on `port`. A local - runtime shares the host network → localhost; a remote one needs a host-side reverse tunnel - (`prime_tunnel`), torn down on exit. This is the host-side, provider-agnostic counterpart to - `Runtime.expose` (which publishes a port running *inside* a runtime) — so the runtime only - reports `is_local` and callers (interception pool, rollout, tool serving) bridge to the host - here, rather than every runtime reimplementing the tunnel.""" + connection uses localhost; any config that cannot reach the host locally needs a host-side + reverse tunnel (`prime_tunnel`), torn down on exit. This is the host-side, provider-agnostic + counterpart to `Runtime.expose` (which publishes a port running *inside* a runtime) — so it only + reports whether the active config reaches the host locally and callers (interception pool, + rollout, tool serving) bridge here, rather than every runtime reimplementing the tunnel.""" if is_local: yield f"http://127.0.0.1:{port}" return @@ -341,6 +363,7 @@ class _Host: and publishes nothing itself (it's reached *into* via `host_endpoint`, not via `expose`).""" is_local = True + host_is_local = True HOST = _Host() @@ -357,16 +380,16 @@ async def reachable_url( (publish *out* of a runtime) and `host_endpoint` (reach *into* the host from a runtime). `service` is the `Runtime` the service runs in, or `HOST` (a host-network service). `consumer` is - the consuming `Runtime` (used for the colocated check and its locality); leave it `None` for a - host consumer or an eval-level consumer with no single instance (a shared tool reused by every - rollout's harness) and pass its locality as `consumer_is_local`: + the consuming `Runtime` (used for the colocated check and its host reachability); leave it + `None` for a host consumer or an eval-level consumer with no single instance (a shared tool + reused by every rollout's harness) and pass its locality as `consumer_is_local`: - same location (a colocated tool in the consumer's own runtime, or host -> host): localhost; - the service runs in a sandbox (a remote runtime): its own published URL (`expose`), reachable from anywhere; - the service is on the host network: localhost to a host-network consumer, else a host tunnel (`host_endpoint`).""" - is_local = consumer.is_local if consumer is not None else consumer_is_local + is_local = consumer.host_is_local if consumer is not None else consumer_is_local if service is consumer: # colocated in the consumer's runtime (or host -> host) yield f"http://127.0.0.1:{port}" elif ( diff --git a/verifiers/v1/runtimes/docker.py b/verifiers/v1/runtimes/docker.py index 75fc3fe7e7..5bab1aec38 100644 --- a/verifiers/v1/runtimes/docker.py +++ b/verifiers/v1/runtimes/docker.py @@ -1,16 +1,19 @@ -"""Local Docker runtime: run the program in a container sharing the host network. +"""Local Docker runtime with full or interception-only agent networking. -On Linux the container shares the host network (`--network host`), so it reaches -the interception server on localhost directly — no tunnel needed. +Full mode shares the host network. Interception mode provisions over an ordinary bridge, then +moves the container onto an internal per-rollout network whose only peer is a fixed reverse proxy +to the interception tunnel before the agent starts. """ import asyncio import contextlib import logging +import re import shlex import subprocess from pathlib import PurePosixPath from typing import Literal +from urllib.parse import urlsplit, urlunsplit from pydantic_config import BaseConfig @@ -19,11 +22,19 @@ logger = logging.getLogger(__name__) +# The trusted, fixed-upstream reverse relay. It never receives the Docker socket. +_RELAY_IMAGE = "nginx:1.28.3-alpine" +_RELAY_HOST = "vf-interception" +_RELAY_PORT = 8080 + class DockerConfig(BaseConfig): type: Literal["docker"] = "docker" image: str = "python:3.11-slim" workdir: str = "/app" + network_access: Literal["full", "interception"] = "full" + """Agent-execution networking. `full` preserves host networking; `interception` allows + internet-backed setup, then limits the agent to the rollout's interception endpoint.""" # TaskResources in Modal's units (also settable per-task via Task.resources). cpu: float | None = None """Pin the container to this many CPU cores (docker `--cpus`). None = unlimited.""" @@ -54,19 +65,29 @@ async def docker(*args: str) -> ProgramResult: class DockerRuntime(Runtime): - """Runs the program in a local Docker container reachable over the host network.""" + """Runs programs in a local container, optionally sealing agent egress to interception.""" def __init__(self, config: DockerConfig, name: str | None = None) -> None: super().__init__(name) self.config = config self._container: str | None = None # our `--name` (used for exec/rm) self._container_id: str | None = None # docker's short id (for display) + self._relay: str | None = None + self._network: str | None = None self._stopped = False @property def descriptor(self) -> str | None: return self._container_id + @classmethod + def config_reaches_host_locally(cls, config: object) -> bool: + return isinstance(config, DockerConfig) and config.network_access == "full" + + @property + def interception_only(self) -> bool: + return self.config.network_access == "interception" + async def start(self) -> None: try: version = await docker("version", "--format", "{{.Server.Version}}") @@ -95,11 +116,14 @@ async def start(self) -> None: _, gpu_count = parse_gpu(self.config.gpu) if gpu_count: limits += ["--gpus", str(gpu_count)] + network = "bridge" if self.interception_only else "host" + capabilities = ["--cap-drop", "NET_ADMIN"] if self.interception_only else [] run = await docker( "run", "--detach", "--network", - "host", + network, + *capabilities, *limits, "--workdir", self.config.workdir, @@ -115,9 +139,173 @@ async def start(self) -> None: :12 ] # `docker run -d` prints the container id logger.info( - "docker: started container %s (image=%s)", + "docker: started container %s (image=%s, network=%s)", self._container, self.config.image, + self.config.network_access, + ) + + async def seal_agent_network(self, endpoint: str) -> str: + if not self.interception_only: + return endpoint + if self._container is None: + raise SandboxError("docker container is not running") + + parsed = urlsplit(endpoint) + host = parsed.hostname + try: + port = parsed.port + except ValueError as e: + raise SandboxError( + f"invalid interception endpoint {endpoint!r}: {e}" + ) from e + if ( + parsed.scheme not in {"http", "https"} + or host is None + or parsed.username is not None + or parsed.password is not None + or re.fullmatch(r"[A-Za-z0-9.-]+", host) is None + ): + raise SandboxError(f"invalid interception endpoint {endpoint!r}") + + authority = f"{host}:{port}" if port is not None else host + upstream = f"{parsed.scheme}://{authority}" + self._network = f"{self.name}-network" + self._relay = f"{self.name}-relay" + # Static proxy_pass + Host/SNI prevent the agent from selecting another upstream. + config = f""" +worker_processes 1; +pid /tmp/nginx.pid; +error_log stderr warn; +events {{ worker_connections 128; }} +http {{ + access_log off; + client_body_temp_path /tmp/client_body; + proxy_temp_path /tmp/proxy; + fastcgi_temp_path /tmp/fastcgi; + uwsgi_temp_path /tmp/uwsgi; + scgi_temp_path /tmp/scgi; + server {{ + listen {_RELAY_PORT}; + client_max_body_size 0; + location / {{ + proxy_pass {upstream}; + proxy_set_header Host {authority}; + proxy_ssl_server_name on; + proxy_ssl_name {host}; + proxy_ssl_verify on; + proxy_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt; + proxy_ssl_verify_depth 3; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_request_buffering off; + proxy_buffering off; + proxy_next_upstream off; + proxy_read_timeout 86400s; + proxy_send_timeout 86400s; + }} + }} +}} +""" + + created = await docker( + "network", + "create", + "--internal", + "--opt", + "com.docker.network.bridge.gateway_mode_ipv4=isolated", + "--label", + f"verifiers.runtime={self.name}", + self._network, + ) + if created.exit_code != 0: + raise SandboxError( + f"docker internal network creation failed: {created.stderr.strip()}" + ) + relay = await docker( + "create", + "--name", + self._relay, + "--network", + "bridge", + "--label", + f"verifiers.runtime={self.name}", + "--user", + "101:101", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--read-only", + "--tmpfs", + "/tmp:rw,noexec,nosuid,size=16m,uid=101,gid=101", + "--entrypoint", + "sh", + _RELAY_IMAGE, + "-c", + 'printf "%s" "$1" > /tmp/nginx.conf && ' + 'exec nginx -c /tmp/nginx.conf -g "daemon off;"', + "vf-relay", + config, + ) + if relay.exit_code != 0: + raise SandboxError( + f"docker interception relay creation failed: {relay.stderr.strip()}" + ) + connected = await docker( + "network", + "connect", + "--alias", + _RELAY_HOST, + self._network, + self._relay, + ) + if connected.exit_code != 0: + raise SandboxError( + f"docker interception relay network failed: {connected.stderr.strip()}" + ) + started = await docker("start", self._relay) + if started.exit_code != 0: + raise SandboxError( + f"docker interception relay start failed: {started.stderr.strip()}" + ) + for _ in range(20): + ready = await docker( + "exec", self._relay, "nginx", "-t", "-c", "/tmp/nginx.conf" + ) + if ready.exit_code == 0: + break + await asyncio.sleep(0.05) + else: + logs = await docker("logs", self._relay) + raise SandboxError( + "docker interception relay did not become ready: " + f"{(logs.stderr or logs.stdout).strip()[-2000:]}" + ) + + connected = await docker("network", "connect", self._network, self._container) + if connected.exit_code != 0: + raise SandboxError( + f"docker agent network connection failed: {connected.stderr.strip()}" + ) + disconnected = await docker("network", "disconnect", "bridge", self._container) + if disconnected.exit_code != 0: + raise SandboxError( + f"docker setup network removal failed: {disconnected.stderr.strip()}" + ) + logger.info( + "docker: sealed container %s to interception relay %s", + self._container, + self._relay, + ) + return urlunsplit( + ( + "http", + f"{_RELAY_HOST}:{_RELAY_PORT}", + parsed.path, + parsed.query, + "", + ) ) async def run(self, argv: list[str], env: dict[str, str]) -> ProgramResult: @@ -187,16 +375,26 @@ async def write(self, path: str, data: bytes) -> None: ) def cleanup(self) -> None: - if self._container is None or self._stopped: + if self._stopped or not any((self._container, self._relay, self._network)): return - self._stopped = ( - True # idempotency guard; keep `_container` so the name still shows - ) - logger.debug("docker: removing container %s", self._container) - with contextlib.suppress(Exception): - subprocess.run( - ["docker", "rm", "--force", self._container], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=30, - ) + self._stopped = True # keep the names so descriptors and logs survive teardown + for container in (self._relay, self._container): + if container is None: + continue + logger.debug("docker: removing container %s", container) + with contextlib.suppress(Exception): + subprocess.run( + ["docker", "rm", "--force", container], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=30, + ) + if self._network is not None: + logger.debug("docker: removing network %s", self._network) + with contextlib.suppress(Exception): + subprocess.run( + ["docker", "network", "rm", self._network], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=30, + ) From 74160228978fb86a6a5f75c49e6e1a7b94b89ef7 Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Tue, 23 Jun 2026 16:07:37 +0200 Subject: [PATCH 2/7] Remove runtime tests from PR --- tests/v1/test_runtimes.py | 210 -------------------------------------- 1 file changed, 210 deletions(-) delete mode 100644 tests/v1/test_runtimes.py diff --git a/tests/v1/test_runtimes.py b/tests/v1/test_runtimes.py deleted file mode 100644 index 4b5d288bf6..0000000000 --- a/tests/v1/test_runtimes.py +++ /dev/null @@ -1,210 +0,0 @@ -"""Runtime topology tests, including an optional local-Docker isolation proof.""" - -import asyncio -import subprocess -import uuid -from typing import Literal - -import pytest - -import verifiers.v1.runtimes.docker as docker_module -from verifiers.v1.interception.pool import InterceptionPool -from verifiers.v1.errors import SandboxError -from verifiers.v1.runtimes import ( - DockerConfig, - runtime_is_local, - runtime_reaches_host_locally, -) -from verifiers.v1.runtimes.base import ProgramResult -from verifiers.v1.runtimes.docker import DockerRuntime - - -def result(stdout: str = "", stderr: str = "", exit_code: int = 0) -> ProgramResult: - return ProgramResult(exit_code=exit_code, stdout=stdout, stderr=stderr) - - -@pytest.mark.parametrize( - ("network_access", "network"), - [("full", "host"), ("interception", "bridge")], -) -async def test_docker_start_uses_setup_network( - monkeypatch, - network_access: Literal["full", "interception"], - network: str, -) -> None: - calls: list[tuple[str, ...]] = [] - - async def fake_docker(*args: str) -> ProgramResult: - calls.append(args) - return result("29.0.0" if args[0] == "version" else "abcdef1234567890") - - monkeypatch.setattr(docker_module, "docker", fake_docker) - runtime = DockerRuntime(DockerConfig(network_access=network_access), name="vf-test") - await runtime.start() - - run = next(args for args in calls if args[0] == "run") - assert run[run.index("--network") + 1] == network - if network == "bridge": - assert ("--cap-drop", "NET_ADMIN") == run[4:6] - - -def test_interception_docker_is_local_but_needs_a_host_tunnel() -> None: - config = DockerConfig(network_access="interception") - - assert runtime_is_local(config) - assert not runtime_reaches_host_locally(config) - assert not InterceptionPool(config, multiplex=1).host_is_local - - -async def test_docker_seals_agent_behind_fixed_interception_relay(monkeypatch) -> None: - calls: list[tuple[str, ...]] = [] - - async def fake_docker(*args: str) -> ProgramResult: - calls.append(args) - return result("ok") - - monkeypatch.setattr(docker_module, "docker", fake_docker) - runtime = DockerRuntime(DockerConfig(network_access="interception"), name="vf-test") - runtime._container = runtime.name - - endpoint = await runtime.seal_agent_network( - "https://rollout.tunnel.example/v1?dialect=responses" - ) - - assert endpoint == "http://vf-interception:8080/v1?dialect=responses" - assert calls[0][:4] == ("network", "create", "--internal", "--opt") - relay = next(args for args in calls if args[0] == "create") - config = relay[-1] - assert "nginx:1.28.3-alpine" in relay - assert "proxy_pass https://rollout.tunnel.example;" in config - assert "proxy_set_header Host rollout.tunnel.example;" in config - assert "proxy_ssl_verify on;" in config - assert ( - "network", - "connect", - "vf-test-network", - "vf-test", - ) in calls - assert ("network", "disconnect", "bridge", "vf-test") in calls - - -async def test_docker_rejects_non_http_interception_endpoint(monkeypatch) -> None: - async def unexpected_docker(*args: str) -> ProgramResult: - raise AssertionError(f"unexpected docker call: {args}") - - monkeypatch.setattr(docker_module, "docker", unexpected_docker) - runtime = DockerRuntime(DockerConfig(network_access="interception"), name="vf-test") - runtime._container = runtime.name - - with pytest.raises(SandboxError, match="invalid interception endpoint"): - await runtime.seal_agent_network("file:///tmp/interception.sock") - - -def test_docker_cleanup_removes_relay_container_and_network(monkeypatch) -> None: - calls: list[list[str]] = [] - - def fake_run(argv: list[str], **kwargs) -> subprocess.CompletedProcess: - calls.append(argv) - return subprocess.CompletedProcess(argv, 0) - - monkeypatch.setattr(docker_module.subprocess, "run", fake_run) - runtime = DockerRuntime(DockerConfig(network_access="interception"), name="vf-test") - runtime._container = "vf-test" - runtime._relay = "vf-test-relay" - runtime._network = "vf-test-network" - - runtime.cleanup() - runtime.cleanup() - - assert calls == [ - ["docker", "rm", "--force", "vf-test-relay"], - ["docker", "rm", "--force", "vf-test"], - ["docker", "network", "rm", "vf-test-network"], - ] - - -@pytest.mark.integration -@pytest.mark.docker -async def test_interception_network_allows_relay_and_blocks_direct_egress() -> None: - version = await docker_module.docker("version", "--format", "{{.Server.Version}}") - if version.exit_code != 0: - pytest.skip("Docker daemon is not available") - - suffix = uuid.uuid4().hex[:12] - upstream = f"vf-upstream-{suffix}" - runtime = DockerRuntime( - DockerConfig(network_access="interception"), name=f"vf-agent-{suffix}" - ) - server = await docker_module.docker( - "run", - "--detach", - "--network", - "bridge", - "--name", - upstream, - "python:3.11-slim", - "python", - "-m", - "http.server", - "8000", - ) - if server.exit_code != 0: - pytest.fail(f"test upstream failed to start: {server.stderr}") - try: - for _ in range(20): - ready = await docker_module.docker( - "exec", - upstream, - "python", - "-c", - "import socket; socket.create_connection(('127.0.0.1', 8000), 1)", - ) - if ready.exit_code == 0: - break - await asyncio.sleep(0.05) - else: - pytest.fail("test upstream did not become ready") - await runtime.start() - inspected = await docker_module.docker( - "inspect", - "--format", - "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", - upstream, - ) - upstream_ip = inspected.stdout.strip() - assert upstream_ip - - endpoint = await runtime.seal_agent_network(f"http://{upstream_ip}:8000/") - allowed = await runtime.run( - [ - "python", - "-c", - "import sys, urllib.request; urllib.request.urlopen(sys.argv[1], timeout=3).read()", - endpoint, - ], - {}, - ) - direct = await runtime.run( - [ - "python", - "-c", - "import socket, sys; socket.create_connection((sys.argv[1], 8000), 1)", - upstream_ip, - ], - {}, - ) - internet = await runtime.run( - [ - "python", - "-c", - "import socket; socket.create_connection(('1.1.1.1', 80), 1)", - ], - {}, - ) - - assert allowed.exit_code == 0, allowed.stderr - assert direct.exit_code != 0 - assert internet.exit_code != 0 - finally: - await runtime.stop() - await docker_module.docker("rm", "--force", upstream) From b63ed87722d0a11894202597220d1257c23fb2fd Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Thu, 25 Jun 2026 12:51:49 +0200 Subject: [PATCH 3/7] Use offline flag for Docker interception --- verifiers/v1/GUIDE.md | 8 ++++---- verifiers/v1/rollout.py | 4 ++-- verifiers/v1/runtimes/docker.py | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index 70f8357407..da8b42038a 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -627,13 +627,13 @@ uv run eval gsm8k-v1 -n 1 --harness.runtime.type modal # remote modal sand A taskset that sets `NEEDS_CONTAINER` (or a task with an `image`) refuses the subprocess runtime — pass `docker` / `prime` / `modal`. -## Interception-only Docker agents +## Offline Docker agents -Set `network_access = "interception"` to let Docker setup download harness dependencies, then -remove internet access before the agent starts: +Set `network_access = false` to let Docker setup download harness dependencies, then remove direct +web access before the agent starts while preserving its connection to the interception server: ```toml -harness = { id = "codex", runtime = { type = "docker", network_access = "interception" } } +harness = { id = "codex", runtime = { type = "docker", network_access = false } } ``` The runtime moves the agent onto an internal per-rollout network and rewrites its model endpoint diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 9557f1b796..1ee448f10e 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -184,7 +184,7 @@ async def run(self) -> Trace: tool_servers = self.taskset.tools(self.task) if runtime.interception_only and tool_servers: raise ValueError( - "Docker network_access='interception' does not yet support MCP " + "Docker network_access=false does not yet support MCP " "tool servers; use a taskset without tools" ) async with boundary(UserError, "building user simulator"): @@ -195,7 +195,7 @@ async def run(self) -> Trace: and user.config.colocated ): raise ValueError( - "Docker network_access='interception' does not support a colocated " + "Docker network_access=false does not support a colocated " "user simulator; run the user in its own runtime" ) async with ( diff --git a/verifiers/v1/runtimes/docker.py b/verifiers/v1/runtimes/docker.py index 5bab1aec38..dd786612fe 100644 --- a/verifiers/v1/runtimes/docker.py +++ b/verifiers/v1/runtimes/docker.py @@ -32,9 +32,9 @@ class DockerConfig(BaseConfig): type: Literal["docker"] = "docker" image: str = "python:3.11-slim" workdir: str = "/app" - network_access: Literal["full", "interception"] = "full" - """Agent-execution networking. `full` preserves host networking; `interception` allows - internet-backed setup, then limits the agent to the rollout's interception endpoint.""" + network_access: bool = True + """Agent-execution web access. When false, setup can still use the internet, then the agent + is limited to the rollout's interception endpoint.""" # TaskResources in Modal's units (also settable per-task via Task.resources). cpu: float | None = None """Pin the container to this many CPU cores (docker `--cpus`). None = unlimited.""" @@ -82,11 +82,11 @@ def descriptor(self) -> str | None: @classmethod def config_reaches_host_locally(cls, config: object) -> bool: - return isinstance(config, DockerConfig) and config.network_access == "full" + return isinstance(config, DockerConfig) and config.network_access @property def interception_only(self) -> bool: - return self.config.network_access == "interception" + return not self.config.network_access async def start(self) -> None: try: From 83c11a62c375a4d50b29a56a45240bb8f377330c Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Thu, 25 Jun 2026 15:20:17 +0200 Subject: [PATCH 4/7] Use host interception port for offline Docker --- verifiers/v1/ARCHITECTURE.md | 10 +- verifiers/v1/GUIDE.md | 16 +- verifiers/v1/env.py | 3 +- verifiers/v1/interception/pool.py | 34 ++-- verifiers/v1/interception/server.py | 21 +- verifiers/v1/rollout.py | 2 +- verifiers/v1/runtimes/__init__.py | 9 +- verifiers/v1/runtimes/base.py | 41 ++-- verifiers/v1/runtimes/docker.py | 300 ++++++++++++---------------- 9 files changed, 197 insertions(+), 239 deletions(-) diff --git a/verifiers/v1/ARCHITECTURE.md b/verifiers/v1/ARCHITECTURE.md index 297f5ef58f..97218cccb9 100644 --- a/verifiers/v1/ARCHITECTURE.md +++ b/verifiers/v1/ARCHITECTURE.md @@ -136,9 +136,9 @@ Behind the endpoint sits one of two clients: trainable. Interception servers are pooled and multiplexed (`interception/pool.py`): one `PooledServer` -serves up to `multiplex` concurrent rollouts (each with its own secret) behind a single tunnel, -and the pool brings up more elastically — so thousands of concurrent rollouts don't mean -thousands of servers or tunnels. +serves up to `multiplex` concurrent rollouts (each with its own secret) behind one host endpoint, +and the pool brings up more elastically. Remote runtimes share a tunnel; offline Docker agents +share a listener on the Docker bridge gateway. ## Runtimes @@ -147,8 +147,8 @@ A `Runtime` (`runtimes/base.py`) is the single contract for *where* code runs: `read`/`write`, `seal_agent_network(endpoint)`, and `expose(port)` (the URL by which the host reaches a port inside the runtime — localhost for subprocess, a tunnel for prime/modal). `seal_agent_network` is the setup-to-execution boundary: normally a no-op, while an -interception-only Docker runtime replaces setup networking with an internal network and a fixed -reverse relay to the interception tunnel. The same contract backs the +interception-only Docker runtime installs an immutable network-namespace `OUTPUT` allowlist that +leaves only its host interception port reachable. The same contract backs the harness, a task's tool servers, and the user simulator, so any of them runs in any backend: `subprocess` (local, `/tmp/` workspace, own process group), `docker` (local container), `prime` (remote sandbox), `modal` (remote function). diff --git a/verifiers/v1/GUIDE.md b/verifiers/v1/GUIDE.md index da8b42038a..a6f5b92246 100644 --- a/verifiers/v1/GUIDE.md +++ b/verifiers/v1/GUIDE.md @@ -636,11 +636,17 @@ web access before the agent starts while preserving its connection to the interc harness = { id = "codex", runtime = { type = "docker", network_access = false } } ``` -The runtime moves the agent onto an internal per-rollout network and rewrites its model endpoint -through a fixed reverse relay to the interception tunnel. The relay cannot proxy another host, and -the agent has no route to the internet, host, or other containers. The restriction remains active -through finalization and scoring so a child process left by the agent cannot regain network access. -Docker pulls the pinned official `nginx:1.28.3-alpine` relay image on first use. +This mode targets rootful Docker Engine on Linux and requires the host process to run as root with +`iptables` and `nsenter`. Setup runs on Docker's bridge with ordinary egress. Before the harness +starts, the interception server also listens on Docker's host gateway and the runtime enters the +container's network namespace to install an `OUTPUT` allowlist: loopback remains available, Docker +DNS is blocked, the interception gateway and port are accepted, and everything else is rejected. +IPv6 and the container's `NET_ADMIN`/`NET_RAW` capabilities are disabled. The endpoint becomes +`http://host.docker.internal:/v1`; no public tunnel or relay container is involved. The +namespace rules disappear with the container, so they do not mutate the host's global firewall. + +The restriction remains active through finalization and scoring so a child process left by the +agent cannot regain network access. This mode currently supports tasksets without MCP tools. User simulators must run in their own runtime rather than `colocated`; both cases fail early with a clear rollout error. diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index b1c1e8e6ae..7113b2af81 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -30,7 +30,6 @@ RuntimeConfig, SubprocessConfig, runtime_is_local, - runtime_reaches_host_locally, ) from verifiers.v1.task import Task from verifiers.v1.taskset import TasksetConfig @@ -401,6 +400,6 @@ async def shared_tools(self, tasks: list[Task]): if not any(server.config.shared for server in servers): yield {} return - harness_is_local = runtime_reaches_host_locally(self.harness.config.runtime) + harness_is_local = runtime_is_local(self.harness.config.runtime) async with serve_shared(servers, harness_is_local=harness_is_local) as urls: yield urls diff --git a/verifiers/v1/interception/pool.py b/verifiers/v1/interception/pool.py index 01184af4a4..ff3926ebe1 100644 --- a/verifiers/v1/interception/pool.py +++ b/verifiers/v1/interception/pool.py @@ -1,12 +1,7 @@ -"""A pool of shared interception servers, grown on demand, so N concurrent rollouts need -~N/multiplex servers + tunnels rather than one each. +"""A pool of shared interception servers, grown on demand. -A harness runtime that cannot reach the host locally needs a tunnel for each interception -endpoint, and tunnel creation is rate-capped per API token — so one tunnel per rollout caps -fan-out. Each shared `InterceptionServer` serves up to `multiplex` rollouts behind one tunnel. -The pool is elastic: `acquire` reuses a server with a free slot, else brings up a new one — so it -fits both the bounded eval runner and the env server's unbounded request load. The harness is -unchanged: it authenticates with a per-rollout secret, which is what the server routes by. +Each `InterceptionServer` serves up to `multiplex` rollouts. Remote runtimes share its host tunnel; +offline Docker agents share its Docker-gateway listener. The pool grows when every server is full. """ import asyncio @@ -17,10 +12,12 @@ from verifiers.v1.interception.server import InterceptionServer, RolloutSession from verifiers.v1.runtimes import ( HOST, + DockerConfig, RuntimeConfig, reachable_url, - runtime_reaches_host_locally, + runtime_is_local, ) +from verifiers.v1.runtimes.docker import docker_bridge_gateway logger = logging.getLogger(__name__) @@ -38,10 +35,13 @@ class InterceptionPool: bringing up a new server when all are at capacity.""" def __init__(self, runtime_config: RuntimeConfig, multiplex: int) -> None: - # The harness config decides reachability: provider sandboxes and interception-only Docker - # need a host tunnel; ordinary local runtimes use localhost. The pool provisions neither. self.runtime_type = runtime_config.type - self.host_is_local = runtime_reaches_host_locally(runtime_config) + self.is_local = runtime_is_local(runtime_config) + self.offline_docker = ( + isinstance(runtime_config, DockerConfig) + and not runtime_config.network_access + ) + self.interception_host: str | None = None self.multiplex = max(1, multiplex) self._servers: list[PooledServer] = [] self._lock = asyncio.Lock() @@ -61,12 +61,14 @@ async def _entry(self) -> PooledServer: for entry in self._servers: if entry.load < self.multiplex: return entry - server = InterceptionServer() + if self.offline_docker and self.interception_host is None: + self.interception_host = await docker_bridge_gateway() + server = InterceptionServer(host=self.interception_host) await self._stack.enter_async_context(server) - # The interception server is a HOST service the harness reaches through localhost or a - # tunnel. The pool's stack owns both and tears them down together. + # Local harnesses start with loopback (offline Docker rewrites it at seal); provider + # sandboxes share a host tunnel. The pool's stack owns both endpoints. url = await self._stack.enter_async_context( - reachable_url(HOST, server.port, consumer_is_local=self.host_is_local) + reachable_url(HOST, server.port, consumer_is_local=self.is_local) ) entry = PooledServer(server, url) self._servers.append(entry) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index c8fa2a0119..e084b72af8 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -1,7 +1,7 @@ """The interception server: harness chat-completions, caught and proxied. Every rollout runs an harness program whose OpenAI-style calls are caught here: a small -localhost server routes each `POST /v1/chat/completions` to our `Client`, records the turn +host server routes each `POST /v1/chat/completions` to our `Client`, records the turn into the trace's message graph, and returns the result in OpenAI shape. We inject `OPENAI_BASE_URL`/`OPENAI_API_KEY` so the program's SDK talks to us. Both non-streaming and SSE requests are supported. @@ -57,7 +57,7 @@ _MAX_REQUEST_BODY = 1024**3 # 1 GiB (aiohttp's default is 1 MiB) _KEEPALIVE_INTERVAL_SECONDS = 3 _STREAM_QUEUE_MAXSIZE = 16 -# The server binds loopback; callers reach it via localhost or a host tunnel (see `reachable_url`). +# The server binds loopback; offline Docker also binds the Docker bridge gateway. _HOST = "127.0.0.1" @@ -167,15 +167,16 @@ async def refused(self) -> str | None: class InterceptionServer: - """A localhost server that proxies model calls for one or more rollouts. It serves every + """A host server that proxies model calls for one or more rollouts. It serves every registered dialect's routes (see `dialects.DIALECTS`), so a request's wire format is resolved from the endpoint it arrived on — not declared by the harness. Each rollout `register`s a `RolloutSession` and gets back a secret; each request is routed to the session whose secret matches its bearer token. A single server can multiplex many rollouts (the basis for `interception.pool`); used 1:1 it's just a server with one session.""" - def __init__(self) -> None: + def __init__(self, host: str | None = None) -> None: self.sessions: dict[str, RolloutSession] = {} + self.hosts = (_HOST,) if host in {None, _HOST} else (_HOST, host) self.port = 0 self.runner: web.AppRunner | None = None @@ -220,14 +221,20 @@ async def __aenter__(self) -> "InterceptionServer": app.router.add_get("/task", self.handle_task_get) self.runner = web.AppRunner(app) await self.runner.setup() - site = web.TCPSite(self.runner, _HOST, 0) + site = web.TCPSite(self.runner, self.hosts[0], 0) await site.start() self.port = site._server.sockets[0].getsockname()[1] # actual ephemeral port - logger.info("interception up: url=http://%s:%d", _HOST, self.port) + try: + for host in self.hosts[1:]: + await web.TCPSite(self.runner, host, self.port).start() + except BaseException: + await self.runner.cleanup() + raise + logger.info("interception up: hosts=%s port=%d", self.hosts, self.port) return self async def __aexit__(self, *exc) -> None: - logger.info("interception down: url=http://%s:%d", _HOST, self.port) + logger.info("interception down: hosts=%s port=%d", self.hosts, self.port) if self.runner is not None: await self.runner.cleanup() diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 1ee448f10e..ea76662c54 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -126,7 +126,7 @@ async def _serve_interception( ): yield endpoint, secret, state_port, state_base else: - async with InterceptionServer() as server: + async with InterceptionServer(host=runtime.interception_host) as server: secret = server.register(session) # a HOST service the harness (in `runtime`) reaches: localhost or a tunnel async with reachable_url(HOST, server.port, consumer=runtime) as url: diff --git a/verifiers/v1/runtimes/__init__.py b/verifiers/v1/runtimes/__init__.py index d98786c0a3..9add12e00f 100644 --- a/verifiers/v1/runtimes/__init__.py +++ b/verifiers/v1/runtimes/__init__.py @@ -47,23 +47,16 @@ def make_runtime(config: RuntimeConfig, name: str | None = None) -> Runtime: def runtime_is_local(config: RuntimeConfig) -> bool: - """Whether this is a local runtime rather than a provider sandbox. Config-specific network - restrictions do not change this topology; use `runtime_reaches_host_locally` for host URLs.""" + """Whether a runtime of this config runs on the host rather than in a provider sandbox.""" return _runtime_cls(config).is_local -def runtime_reaches_host_locally(config: RuntimeConfig) -> bool: - """Whether this runtime config reaches a host service at localhost rather than by tunnel.""" - return _runtime_cls(config).config_reaches_host_locally(config) - - __all__ = [ "ProgramResult", "Runtime", "RuntimeConfig", "make_runtime", "runtime_is_local", - "runtime_reaches_host_locally", "host_endpoint", "reachable_url", "HOST", diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index 6fce471dde..ae643945d0 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -102,9 +102,9 @@ def cleanup_at_exit() -> None: class Runtime(ABC): is_local: ClassVar[bool] = True - """Whether this runtime is ordinarily on the host network. True for subprocess / Docker; - remote runtimes (Modal/Prime) override to False. Config-specific restrictions may still make - a local runtime require a tunnel to reach the host; use `host_is_local` for that direction.""" + """Whether this runtime runs locally rather than in a provider sandbox. Local services use + loopback by default; interception-only Docker rewrites its endpoint when networking is sealed. + Remote runtimes (Modal/Prime) override to False.""" def __init__(self, name: str | None = None) -> None: self.name = name or f"vf-{uuid.uuid4().hex[:12]}" @@ -128,21 +128,16 @@ def descriptor(self) -> str | None: runtime: subprocess workdir, docker image, prime sandbox id.""" return None - @classmethod - def config_reaches_host_locally(cls, config: object) -> bool: - """Whether this config reaches a host service at localhost rather than through a tunnel.""" - return cls.is_local - - @property - def host_is_local(self) -> bool: - """Whether this instance reaches a host service at localhost.""" - return type(self).config_reaches_host_locally(self.config) - @property def interception_only(self) -> bool: """Whether agent execution is limited to the interception endpoint.""" return False + @property + def interception_host(self) -> str | None: + """Additional host address on which the interception server must listen.""" + return None + # --- lifecycle --- @abstractmethod @@ -178,7 +173,7 @@ async def seal_agent_network(self, endpoint: str) -> str: """Apply the runtime's agent-execution network policy and return its reachable endpoint. Setup has already completed when this is called. Most runtimes keep their existing - network and endpoint; Docker's interception-only mode replaces both for agent execution. + network and endpoint; Docker's interception-only mode installs its namespace policy. """ return endpoint @@ -321,11 +316,10 @@ async def open_tunnel( @contextlib.asynccontextmanager async def host_endpoint(port: int, is_local: bool, labels: list[str] | None = None): """Yield a URL a program *inside a runtime* uses to reach a HOST service on `port`. A local - connection uses localhost; any config that cannot reach the host locally needs a host-side - reverse tunnel (`prime_tunnel`), torn down on exit. This is the host-side, provider-agnostic - counterpart to `Runtime.expose` (which publishes a port running *inside* a runtime) — so it only - reports whether the active config reaches the host locally and callers (interception pool, - rollout, tool serving) bridge here, rather than every runtime reimplementing the tunnel.""" + runtime shares the host network → localhost; a remote one needs a host-side reverse tunnel + (`prime_tunnel`), torn down on exit. This is the host-side, provider-agnostic counterpart to + `Runtime.expose` (which publishes a port running *inside* a runtime) — so the runtime only + reports `is_local` and callers bridge to the host here.""" if is_local: yield f"http://127.0.0.1:{port}" return @@ -363,7 +357,6 @@ class _Host: and publishes nothing itself (it's reached *into* via `host_endpoint`, not via `expose`).""" is_local = True - host_is_local = True HOST = _Host() @@ -380,16 +373,16 @@ async def reachable_url( (publish *out* of a runtime) and `host_endpoint` (reach *into* the host from a runtime). `service` is the `Runtime` the service runs in, or `HOST` (a host-network service). `consumer` is - the consuming `Runtime` (used for the colocated check and its host reachability); leave it - `None` for a host consumer or an eval-level consumer with no single instance (a shared tool - reused by every rollout's harness) and pass its locality as `consumer_is_local`: + the consuming `Runtime` (used for the colocated check and its locality); leave it `None` for a + host consumer or an eval-level consumer with no single instance (a shared tool reused by every + rollout's harness) and pass its locality as `consumer_is_local`: - same location (a colocated tool in the consumer's own runtime, or host -> host): localhost; - the service runs in a sandbox (a remote runtime): its own published URL (`expose`), reachable from anywhere; - the service is on the host network: localhost to a host-network consumer, else a host tunnel (`host_endpoint`).""" - is_local = consumer.host_is_local if consumer is not None else consumer_is_local + is_local = consumer.is_local if consumer is not None else consumer_is_local if service is consumer: # colocated in the consumer's runtime (or host -> host) yield f"http://127.0.0.1:{port}" elif ( diff --git a/verifiers/v1/runtimes/docker.py b/verifiers/v1/runtimes/docker.py index dd786612fe..1823e818d0 100644 --- a/verifiers/v1/runtimes/docker.py +++ b/verifiers/v1/runtimes/docker.py @@ -1,15 +1,16 @@ """Local Docker runtime with full or interception-only agent networking. -Full mode shares the host network. Interception mode provisions over an ordinary bridge, then -moves the container onto an internal per-rollout network whose only peer is a fixed reverse proxy -to the interception tunnel before the agent starts. +Full mode shares the host network. Interception mode provisions on Docker's bridge, then installs +a network-namespace firewall that limits the agent to the host interception port. """ import asyncio import contextlib +import json import logging -import re +import os import shlex +import shutil import subprocess from pathlib import PurePosixPath from typing import Literal @@ -22,11 +23,6 @@ logger = logging.getLogger(__name__) -# The trusted, fixed-upstream reverse relay. It never receives the Docker socket. -_RELAY_IMAGE = "nginx:1.28.3-alpine" -_RELAY_HOST = "vf-interception" -_RELAY_PORT = 8080 - class DockerConfig(BaseConfig): type: Literal["docker"] = "docker" @@ -64,6 +60,44 @@ async def docker(*args: str) -> ProgramResult: ) +async def network_iptables(pid: int, *args: str) -> ProgramResult: + """Run host iptables inside a container's network namespace.""" + proc = await asyncio.create_subprocess_exec( + "nsenter", + "--target", + str(pid), + "--net", + "iptables", + *args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + return ProgramResult( + exit_code=proc.returncode or 0, + stdout=stdout.decode(errors="replace"), + stderr=stderr.decode(errors="replace"), + ) + + +async def docker_bridge_gateway() -> str: + """Return the IPv4 address through which containers reach host services.""" + result = await docker( + "network", + "inspect", + "--format", + "{{(index .IPAM.Config 0).Gateway}}", + "bridge", + ) + gateway = result.stdout.strip() + if result.exit_code != 0 or not gateway: + raise SandboxError( + "could not resolve the Docker host gateway: " + f"{(result.stderr or result.stdout).strip()}" + ) + return gateway + + class DockerRuntime(Runtime): """Runs programs in a local container, optionally sealing agent egress to interception.""" @@ -72,22 +106,21 @@ def __init__(self, config: DockerConfig, name: str | None = None) -> None: self.config = config self._container: str | None = None # our `--name` (used for exec/rm) self._container_id: str | None = None # docker's short id (for display) - self._relay: str | None = None - self._network: str | None = None + self._host_gateway: str | None = None self._stopped = False @property def descriptor(self) -> str | None: return self._container_id - @classmethod - def config_reaches_host_locally(cls, config: object) -> bool: - return isinstance(config, DockerConfig) and config.network_access - @property def interception_only(self) -> bool: return not self.config.network_access + @property + def interception_host(self) -> str | None: + return self._host_gateway if self.interception_only else None + async def start(self) -> None: try: version = await docker("version", "--format", "{{.Server.Version}}") @@ -107,6 +140,32 @@ async def start(self) -> None: raise RuntimeError( f"docker runtime selected but the Docker daemon is not reachable: {detail}{hint}" ) + + network = "host" + options: list[str] = [] + if self.interception_only: + missing = [ + cmd for cmd in ("iptables", "nsenter") if shutil.which(cmd) is None + ] + if os.geteuid() != 0 or missing: + detail = f"; missing {', '.join(missing)}" if missing else "" + raise RuntimeError( + "Docker network_access=false requires root on a Linux Docker host " + f"with iptables and nsenter{detail}" + ) + self._host_gateway = await docker_bridge_gateway() + network = "bridge" + options = [ + "--add-host", + f"host.docker.internal:{self._host_gateway}", + "--cap-drop", + "NET_ADMIN", + "--cap-drop", + "NET_RAW", + "--sysctl", + "net.ipv6.conf.all.disable_ipv6=1", + ] + self._container = self.name limits: list[str] = [] if self.config.cpu is not None: @@ -116,14 +175,12 @@ async def start(self) -> None: _, gpu_count = parse_gpu(self.config.gpu) if gpu_count: limits += ["--gpus", str(gpu_count)] - network = "bridge" if self.interception_only else "host" - capabilities = ["--cap-drop", "NET_ADMIN"] if self.interception_only else [] run = await docker( "run", "--detach", "--network", network, - *capabilities, + *options, *limits, "--workdir", self.config.workdir, @@ -142,7 +199,7 @@ async def start(self) -> None: "docker: started container %s (image=%s, network=%s)", self._container, self.config.image, - self.config.network_access, + network, ) async def seal_agent_network(self, endpoint: str) -> str: @@ -151,8 +208,10 @@ async def seal_agent_network(self, endpoint: str) -> str: if self._container is None: raise SandboxError("docker container is not running") + if self._host_gateway is None: + raise SandboxError("docker host gateway is not available") + parsed = urlsplit(endpoint) - host = parsed.hostname try: port = parsed.port except ValueError as e: @@ -160,148 +219,59 @@ async def seal_agent_network(self, endpoint: str) -> str: f"invalid interception endpoint {endpoint!r}: {e}" ) from e if ( - parsed.scheme not in {"http", "https"} - or host is None + parsed.scheme != "http" + or parsed.hostname not in {"127.0.0.1", "localhost"} + or port is None or parsed.username is not None or parsed.password is not None - or re.fullmatch(r"[A-Za-z0-9.-]+", host) is None ): raise SandboxError(f"invalid interception endpoint {endpoint!r}") - authority = f"{host}:{port}" if port is not None else host - upstream = f"{parsed.scheme}://{authority}" - self._network = f"{self.name}-network" - self._relay = f"{self.name}-relay" - # Static proxy_pass + Host/SNI prevent the agent from selecting another upstream. - config = f""" -worker_processes 1; -pid /tmp/nginx.pid; -error_log stderr warn; -events {{ worker_connections 128; }} -http {{ - access_log off; - client_body_temp_path /tmp/client_body; - proxy_temp_path /tmp/proxy; - fastcgi_temp_path /tmp/fastcgi; - uwsgi_temp_path /tmp/uwsgi; - scgi_temp_path /tmp/scgi; - server {{ - listen {_RELAY_PORT}; - client_max_body_size 0; - location / {{ - proxy_pass {upstream}; - proxy_set_header Host {authority}; - proxy_ssl_server_name on; - proxy_ssl_name {host}; - proxy_ssl_verify on; - proxy_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt; - proxy_ssl_verify_depth 3; - proxy_http_version 1.1; - proxy_set_header Connection ""; - proxy_request_buffering off; - proxy_buffering off; - proxy_next_upstream off; - proxy_read_timeout 86400s; - proxy_send_timeout 86400s; - }} - }} -}} -""" - - created = await docker( - "network", - "create", - "--internal", - "--opt", - "com.docker.network.bridge.gateway_mode_ipv4=isolated", - "--label", - f"verifiers.runtime={self.name}", - self._network, - ) - if created.exit_code != 0: - raise SandboxError( - f"docker internal network creation failed: {created.stderr.strip()}" - ) - relay = await docker( - "create", - "--name", - self._relay, - "--network", - "bridge", - "--label", - f"verifiers.runtime={self.name}", - "--user", - "101:101", - "--cap-drop", - "ALL", - "--security-opt", - "no-new-privileges", - "--read-only", - "--tmpfs", - "/tmp:rw,noexec,nosuid,size=16m,uid=101,gid=101", - "--entrypoint", - "sh", - _RELAY_IMAGE, - "-c", - 'printf "%s" "$1" > /tmp/nginx.conf && ' - 'exec nginx -c /tmp/nginx.conf -g "daemon off;"', - "vf-relay", - config, - ) - if relay.exit_code != 0: - raise SandboxError( - f"docker interception relay creation failed: {relay.stderr.strip()}" - ) - connected = await docker( - "network", - "connect", - "--alias", - _RELAY_HOST, - self._network, - self._relay, - ) - if connected.exit_code != 0: - raise SandboxError( - f"docker interception relay network failed: {connected.stderr.strip()}" - ) - started = await docker("start", self._relay) - if started.exit_code != 0: - raise SandboxError( - f"docker interception relay start failed: {started.stderr.strip()}" - ) - for _ in range(20): - ready = await docker( - "exec", self._relay, "nginx", "-t", "-c", "/tmp/nginx.conf" - ) - if ready.exit_code == 0: - break - await asyncio.sleep(0.05) - else: - logs = await docker("logs", self._relay) - raise SandboxError( - "docker interception relay did not become ready: " - f"{(logs.stderr or logs.stdout).strip()[-2000:]}" - ) + inspected = await docker("inspect", self._container) + if inspected.exit_code != 0: + raise SandboxError(f"docker inspect failed: {inspected.stderr.strip()}") + details = json.loads(inspected.stdout)[0] + pid = details["State"]["Pid"] + chain = "VF-AGENT" + # Build the chain before activating it. Docker DNS can proxy through the host namespace, + # so reject its loopback listener before allowing the container's own loopback traffic. + commands = [ + ("-N", chain), + ( + "-A", + chain, + "-d", + self._host_gateway, + "-p", + "tcp", + "--dport", + str(port), + "-j", + "ACCEPT", + ), + ("-A", chain, "-d", "127.0.0.11", "-j", "REJECT"), + ("-A", chain, "-o", "lo", "-j", "ACCEPT"), + ("-A", chain, "-j", "REJECT"), + ("-I", "OUTPUT", "1", "-j", chain), + ] + for command in commands: + installed = await network_iptables(pid, "--wait", *command) + if installed.exit_code != 0: + raise SandboxError( + "docker agent firewall failed: " + f"{(installed.stderr or installed.stdout).strip()}" + ) - connected = await docker("network", "connect", self._network, self._container) - if connected.exit_code != 0: - raise SandboxError( - f"docker agent network connection failed: {connected.stderr.strip()}" - ) - disconnected = await docker("network", "disconnect", "bridge", self._container) - if disconnected.exit_code != 0: - raise SandboxError( - f"docker setup network removal failed: {disconnected.stderr.strip()}" - ) logger.info( - "docker: sealed container %s to interception relay %s", + "docker: sealed container %s to interception port %s:%d", self._container, - self._relay, + self._host_gateway, + port, ) return urlunsplit( ( "http", - f"{_RELAY_HOST}:{_RELAY_PORT}", + f"host.docker.internal:{port}", parsed.path, parsed.query, "", @@ -375,26 +345,14 @@ async def write(self, path: str, data: bytes) -> None: ) def cleanup(self) -> None: - if self._stopped or not any((self._container, self._relay, self._network)): + if self._stopped or self._container is None: return self._stopped = True # keep the names so descriptors and logs survive teardown - for container in (self._relay, self._container): - if container is None: - continue - logger.debug("docker: removing container %s", container) - with contextlib.suppress(Exception): - subprocess.run( - ["docker", "rm", "--force", container], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=30, - ) - if self._network is not None: - logger.debug("docker: removing network %s", self._network) - with contextlib.suppress(Exception): - subprocess.run( - ["docker", "network", "rm", self._network], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=30, - ) + logger.debug("docker: removing container %s", self._container) + with contextlib.suppress(Exception): + subprocess.run( + ["docker", "rm", "--force", self._container], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=30, + ) From 8d2399aa38e6cdaf58031929006dafdf40596d3f Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:01:26 +0200 Subject: [PATCH 5/7] refactor(v1): generalize runtime network policies --- verifiers/v1/env.py | 6 +- verifiers/v1/interception/__init__.py | 8 +- verifiers/v1/mcp/launch.py | 53 ++--- verifiers/v1/rollout.py | 24 ++- verifiers/v1/runtimes/__init__.py | 2 +- verifiers/v1/runtimes/base.py | 55 +++-- verifiers/v1/runtimes/docker.py | 279 +++++++++++++++++--------- verifiers/v1/runtimes/network.py | 23 +++ 8 files changed, 299 insertions(+), 151 deletions(-) create mode 100644 verifiers/v1/runtimes/network.py diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index 3918373602..312f5b66d1 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -414,6 +414,8 @@ async def shared_tools(self): if not servers: yield {} return - harness_is_local = runtime_reaches_host_locally(self.harness.config.runtime) - async with serve_shared(servers, harness_is_local=harness_is_local) as shared: + harness_reaches_host = runtime_reaches_host_locally(self.harness.config.runtime) + async with serve_shared( + servers, harness_reaches_host=harness_reaches_host + ) as shared: yield shared diff --git a/verifiers/v1/interception/__init__.py b/verifiers/v1/interception/__init__.py index cc03831615..5840c05034 100644 --- a/verifiers/v1/interception/__init__.py +++ b/verifiers/v1/interception/__init__.py @@ -40,7 +40,7 @@ def requires_tunnel( - harness_is_local: bool, + harness_reaches_host: bool, server_configs: Iterable[BaseConfig] = (), shared: "Iterable[SharedToolServer]" = (), ) -> bool: @@ -48,12 +48,12 @@ def requires_tunnel( host network: the harness itself, a live `shared` server in a remote runtime, or a tool/user server config placing one there (each reaches the `/state` channel from its own runtime). Skipped as non-consumers: a `colocated` server (shares the - harness's runtime, covered by `harness_is_local`), a config-`url` server (external — + harness's runtime, covered by `harness_reaches_host`), a config-`url` server (external — it connects out), and an `external` shared server (outside the state machinery entirely). False means every consumer reaches the server at localhost.""" - if not harness_is_local: + if not harness_reaches_host: return True - if any(not s.external and not s.local for s in shared): + if any(not s.external and not s.reaches_host for s in shared): return True for config in server_configs: if getattr(config, "url", None) or config.colocated: diff --git a/verifiers/v1/mcp/launch.py b/verifiers/v1/mcp/launch.py index 159640ea39..5da947ea9e 100644 --- a/verifiers/v1/mcp/launch.py +++ b/verifiers/v1/mcp/launch.py @@ -17,7 +17,7 @@ from typing import TYPE_CHECKING from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit -from verifiers.v1.errors import RolloutError, ToolsetError, UserError +from verifiers.v1.errors import RolloutError, SandboxError, ToolsetError, UserError from verifiers.v1.interception.tunnel import PrimeTunnel from verifiers.v1.mcp.server import STATE_SECRET_PARAM, STATE_URL_PARAM, ServerBase from verifiers.v1.runtimes import ( @@ -217,21 +217,24 @@ async def serve_in_runtime( @contextlib.asynccontextmanager async def reachable_url( - service: Runtime, port: int, *, colocated: bool, consumer_is_local: bool + service: Runtime, port: int, *, colocated: bool, consumer_reaches_host: bool ) -> AsyncIterator[str]: """Yield the URL a consumer uses to reach the server at (`service`, `port`), over two primitives: `Runtime.expose` (publish a port out of a sandbox) and a host `Tunnel` (reach into the host from a remote runtime). `colocated` = the server shares the consumer's - runtime; `consumer_is_local` = the consumer is on the host network. + runtime; `consumer_reaches_host` = the consumer reaches the host network directly. - `colocated` -> localhost (same runtime, in-sandbox or host loopback); - - the server runs in a remote sandbox -> its own published URL (`expose`), reachable anywhere; + - the server is not host-reachable -> its published URL (`expose`), reachable anywhere; - else it's on the host network -> localhost to a local consumer, a host tunnel to a remote one.""" if colocated: yield f"http://127.0.0.1:{port}" - elif not service.is_local: # in a remote sandbox → it publishes its own port - yield await service.expose(port) - elif consumer_is_local: # host network, local consumer → localhost, no tunnel + elif not service.reachable_from_host_locally: + url = await service.expose(port) + if url is None: + raise SandboxError(f"{service.type} runtime cannot expose port {port}") + yield url + elif consumer_reaches_host: # host network, direct consumer → localhost, no tunnel yield f"http://127.0.0.1:{port}" else: # host network, remote consumer → a host tunnel publishes the port outward async with PrimeTunnel().expose(port) as url: @@ -243,7 +246,7 @@ async def serve( server: ServerBase, harness_runtime: Runtime | None = None, for_host: bool = False, - harness_is_local: bool = True, + harness_reaches_host: bool = True, *, state_secret: str = "", state_base: str | None = None, @@ -254,11 +257,9 @@ async def serve( runtime = harness_runtime else: runtime = make_runtime(cfg.runtime) - if runtime.interception_only: + if runtime.network_policy.restricted: error = UserError if for_host else ToolsetError - raise error( - "Docker network_access=false is only supported for harness runtimes" - ) + raise error("network policies are only applied to harness runtimes") await runtime.start() stack.push_async_callback(runtime.stop) # Only consumers outside the server runtime need its fixed published port. Colocated tools @@ -279,19 +280,22 @@ async def serve( # Who consumes the server decides reachability: a user sim is reached by the HOST # (`for_host`, always local, never colocated with it); a tool by the harness — colocated # when it shares the harness's runtime, reached with the harness's locality (read off the - # harness runtime when there is one, else `harness_is_local` for an eval-level shared tool). + # harness runtime when there is one, else `harness_reaches_host` for a shared tool). if for_host: - colocated, consumer_is_local = False, True + colocated, consumer_reaches_host = False, True else: colocated = runtime is harness_runtime - consumer_is_local = ( + consumer_reaches_host = ( harness_runtime.reaches_host_locally if harness_runtime is not None - else harness_is_local + else harness_reaches_host ) base = await stack.enter_async_context( reachable_url( - runtime, port, colocated=colocated, consumer_is_local=consumer_is_local + runtime, + port, + colocated=colocated, + consumer_reaches_host=consumer_reaches_host, ) ) yield f"{base.rstrip('/')}/mcp" @@ -300,7 +304,7 @@ async def serve( @dataclass(frozen=True) class SharedToolServer: """One live taskset-scoped (shared) server, as the rollouts see it: its eval-level - `url` plus whether its runtime is `local` (host-reachable) — a remote one is an + `url` plus whether its runtime reaches the host directly — otherwise it is an interception consumer, so the interception must be exposed for it to reach the `/state` channel (see `Environment._requires_tunnel`). An `external` server (a config-`url` endpoint) was not launched by the framework and sits outside its state @@ -308,17 +312,17 @@ class SharedToolServer: secret sent to a third party).""" url: str - local: bool + reaches_host: bool external: bool = False @contextlib.asynccontextmanager -async def serve_shared(toolsets: list[Toolset], harness_is_local: bool = True): +async def serve_shared(toolsets: list[Toolset], harness_reaches_host: bool = True): """Start the taskset-scoped (shared) tool servers ONCE for a whole eval, each in its OWN `runtime`, and yield `{name: SharedToolServer}` reachable by every rollout's harness. Reachability mirrors a per-rollout tool, but there's no single harness runtime to read locality off — the caller (`Environment.shared_tools`) passes the harness runtime's - `harness_is_local`, so a host tool gets one host bridge (tunnel) when the harness runs + `harness_reaches_host`, so a host tool gets one host bridge when the harness runs remotely, and a remote tool runtime publishes its own URL. Torn down when the eval ends. A shared server is task-agnostic — the taskset carries no per-row data — so its `setup` gets no task (its `setup_task` is never called; the per-rollout servers fetch @@ -343,14 +347,15 @@ async def serve_shared(toolsets: list[Toolset], harness_is_local: bool = True): ) if cfg.url: # already running remotely; nothing launched, nothing to bridge servers[name] = SharedToolServer( - url=cfg.url, local=False, external=True + url=cfg.url, reaches_host=False, external=True ) else: url = await stack.enter_async_context( - serve(toolset, harness_is_local=harness_is_local) + serve(toolset, harness_reaches_host=harness_reaches_host) ) servers[name] = SharedToolServer( - url=url, local=runtime_reaches_host_locally(cfg.runtime) + url=url, + reaches_host=runtime_reaches_host_locally(cfg.runtime), ) logger.info("shared tool server '%s': %s", name, servers[name].url) yield servers diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index a544ecbcf9..28a4e8a17a 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -151,14 +151,21 @@ async def run(self) -> Trace: await self.harness.setup(runtime) async with boundary(ToolsetError, "building tool servers"): tool_servers = self.task.tool_servers() - if runtime.interception_only and (tool_servers or self.shared_tools): + if not runtime.supports_colocated_tools and any( + server.config.colocated and not server.config.url + for server in tool_servers + ): raise ToolsetError( - "Docker network_access=false does not support MCP tool servers" + "this harness network policy does not support colocated MCP servers" ) user = self.task.user_server() - if runtime.interception_only and user is not None and user.config.colocated: + if ( + not runtime.supports_colocated_user + and user is not None + and user.config.colocated + ): raise UserError( - "Docker network_access=false does not support a colocated user server" + "this harness network policy does not support a colocated user server" ) # `base_url` is the interception server's reachable URL for this rollout. The # harness reaches the model at `{base_url}/v1`; tool/user servers reach this @@ -189,7 +196,14 @@ async def run(self) -> Trace: "conversation; set task.prompt or declare a simulator " "class on Task.user" ) - endpoint = await runtime.seal_agent_network(endpoint) + routes = await runtime.apply_network_policy( + { + "model": endpoint, + **{f"mcp:{name}": url for name, url in urls.items()}, + } + ) + endpoint = routes["model"] + urls = {name: routes[f"mcp:{name}"] for name in urls} now = time.time() trace.timing.setup.end = now trace.timing.generation.start = now diff --git a/verifiers/v1/runtimes/__init__.py b/verifiers/v1/runtimes/__init__.py index 2216ab88d5..fe6a7ac13d 100644 --- a/verifiers/v1/runtimes/__init__.py +++ b/verifiers/v1/runtimes/__init__.py @@ -53,7 +53,7 @@ def runtime_is_local(config: RuntimeConfig) -> bool: def runtime_reaches_host_locally(config: RuntimeConfig) -> bool: """Whether this config reaches host services at localhost rather than through a tunnel.""" - return runtime_is_local(config) and getattr(config, "network_access", True) + return _runtime_cls(config).config_reaches_host_locally(config) __all__ = [ diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index 6566b2f6d5..9d3f83dcae 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -15,6 +15,8 @@ from pydantic_config import BaseConfig +from verifiers.v1.errors import SandboxError +from verifiers.v1.runtimes.network import NetworkPolicy from verifiers.v1.utils.aio import run_shielded logger = logging.getLogger(__name__) @@ -107,13 +109,15 @@ class BaseRuntimeInfo(BaseConfig): class Runtime(ABC): is_local: ClassVar[bool] = True - """Whether this runtime shares the host network — a program inside it reaches a host service - at localhost (no tunnel) and a service inside it is reachable at localhost. True for - subprocess / docker(--network host); remote runtimes (modal/prime) override to False (they - need a tunnel each way: a host `Tunnel` (interception.tunnel) inward, `expose` outward).""" + """Whether this runtime is provisioned on the local machine rather than by a provider.""" info: BaseRuntimeInfo + @classmethod + def config_reaches_host_locally(cls, _config: BaseConfig) -> bool: + """Whether this runtime config reaches host services without a tunnel.""" + return cls.is_local + def __init__(self, name: str | None = None) -> None: self.name = name or f"vf-{uuid.uuid4().hex[:12]}" self._uv_interpreters: dict[str, str] = {} @@ -129,13 +133,28 @@ def descriptor(self) -> str | None: @property def reaches_host_locally(self) -> bool: - """Whether this runtime reaches host services at localhost.""" + """Whether programs here reach host services at localhost, without a tunnel.""" + return type(self).config_reaches_host_locally(self.config) + + @property + def reachable_from_host_locally(self) -> bool: + """Whether the host reaches services here at localhost, without exposure.""" return self.is_local @property - def interception_only(self) -> bool: - """Whether agent execution is limited to the interception endpoint.""" - return self.is_local and not self.reaches_host_locally + def network_policy(self) -> NetworkPolicy: + """Egress policy applied at the trusted setup-to-execution boundary.""" + return NetworkPolicy() + + @property + def supports_colocated_tools(self) -> bool: + """Whether colocated MCP servers work under this runtime policy.""" + return True + + @property + def supports_colocated_user(self) -> bool: + """Whether a colocated user server is host-reachable under this policy.""" + return True @abstractmethod async def start(self) -> None: @@ -174,9 +193,14 @@ async def run_program(self, argv: list[str], env: dict[str, str]) -> ProgramResu still retry individual safe transport operations underneath `run`.""" return await self.run(argv, env) - async def seal_agent_network(self, endpoint: str) -> str: - """Apply the runtime's execution network policy after trusted setup.""" - return endpoint + async def apply_network_policy(self, routes: dict[str, str]) -> dict[str, str]: + """Apply the policy once after setup and map required framework route URLs. + + The policy remains active until `stop()` tears down the runtime. + """ + if self.network_policy.restricted: + raise SandboxError(f"{self.type} runtime does not support network policies") + return routes async def run_background( self, argv: list[str], env: dict[str, str], log: str @@ -263,14 +287,13 @@ def published_port(self) -> int | None: """A fixed port this runtime exposes to the outside at startup, declared up front to the provider (Modal forwards only ports named at `Sandbox.create`). When set, a server placed here binds it instead of a host-chosen free port, and `expose` returns its public URL. - `None` for host-networked runtimes (subprocess/docker), which pick a free port and are - reached over the shared host network.""" + `None` for runtimes reached directly from the host, which pick a free port.""" return None async def expose(self, port: int) -> str | None: """Publish a port running *inside this runtime* to a URL reachable from the host/outside, - or None when local (it's on the host network — reach it at localhost). A remote runtime - overrides this with the provider's native port exposure (modal `tunnels()`, prime - `client.expose`), torn down with the sandbox in `stop()`. The reverse of a host `Tunnel` + or None when `reachable_from_host_locally` is true. A remote runtime overrides this + with the provider's native port exposure (modal `tunnels()`, prime `client.expose`), + torn down with the sandbox in `stop()`. The reverse of a host `Tunnel` (interception.tunnel, which reaches a host port from inside a runtime).""" return None diff --git a/verifiers/v1/runtimes/docker.py b/verifiers/v1/runtimes/docker.py index d1d6df0a2e..d26bc34971 100644 --- a/verifiers/v1/runtimes/docker.py +++ b/verifiers/v1/runtimes/docker.py @@ -1,13 +1,13 @@ -"""Local Docker runtime with optional interception-only agent networking.""" +"""Local Docker runtime with policy-controlled agent networking.""" import asyncio -import contextlib +import json import logging import re import shlex import subprocess from pathlib import PurePosixPath -from typing import Literal +from typing import Literal, cast from urllib.parse import urlsplit, urlunsplit from pydantic_config import BaseConfig @@ -19,12 +19,13 @@ Runtime, parse_gpu, ) +from verifiers.v1.runtimes.network import NetworkPolicy logger = logging.getLogger(__name__) -_RELAY_IMAGE = "nginx:1.28.3-alpine" -_RELAY_HOST = "interception" -_RELAY_PORT = 8080 +_EGRESS_IMAGE = "nginx:1.28.3-alpine" +_EGRESS_HOST = "egress" +_EGRESS_PORT = 8080 class DockerConfig(BaseConfig): @@ -32,7 +33,7 @@ class DockerConfig(BaseConfig): image: str = "python:3.11-slim" workdir: str = "/app" network_access: bool = True - """Allow agent web access. Setup always has access; false seals execution to interception.""" + """Allow arbitrary agent egress. Setup has access; false then allows framework routes only.""" # TaskData.resources uses these units; non-default runtime config values take precedence. cpu: float | None = None """Pin the container to this many CPU cores (docker `--cpus`). None = unlimited.""" @@ -79,12 +80,29 @@ def __init__(self, config: DockerConfig, name: str | None = None) -> None: self.config = config self.info = DockerRuntimeInfo(**config.model_dump()) self._container: str | None = None # our `--name` (used for exec/rm) - self._relay: str | None = None - self._network: str | None = None + self._egress: str | None = None + self._setup_network: str | None = None + self._execution_network: str | None = None self._stopped = False + @classmethod + def config_reaches_host_locally(cls, config: BaseConfig) -> bool: + return cast(DockerConfig, config).network_access + + @property + def reachable_from_host_locally(self) -> bool: + return self.config.network_access + + @property + def network_policy(self) -> NetworkPolicy: + return NetworkPolicy(allow=None if self.config.network_access else []) + @property - def reaches_host_locally(self) -> bool: + def supports_colocated_tools(self) -> bool: + return self.config.network_access + + @property + def supports_colocated_user(self) -> bool: return self.config.network_access async def start(self) -> None: @@ -115,7 +133,17 @@ async def start(self) -> None: _, gpu_count = parse_gpu(self.config.gpu) if gpu_count: limits += ["--gpus", str(gpu_count)] - network = "host" if self.config.network_access else "bridge" + if self.network_policy.restricted: + self._setup_network = f"{self.name}-setup" + await docker_checked( + "network", + "create", + "--label", + f"verifiers.runtime={self.name}", + self._setup_network, + error="docker setup network creation failed", + ) + network = self._setup_network or "host" restrictions = ( [] if self.config.network_access @@ -154,31 +182,75 @@ async def start(self) -> None: network, ) - async def seal_agent_network(self, endpoint: str) -> str: - if not self.interception_only: - return endpoint - if self._container is None: + async def apply_network_policy(self, routes: dict[str, str]) -> dict[str, str]: + if not self.network_policy.restricted: + return routes + if self._container is None or self._setup_network is None: raise SandboxError("docker container is not running") - parsed = urlsplit(endpoint) - host = parsed.hostname - try: - port = parsed.port - except ValueError as e: - raise SandboxError( - f"invalid interception endpoint {endpoint!r}: {e}" - ) from e - if ( - parsed.scheme not in {"http", "https"} - or host is None - or parsed.username is not None - or parsed.password is not None - or re.fullmatch(r"[A-Za-z0-9.-]+", host) is None - ): - raise SandboxError(f"invalid interception endpoint {endpoint!r}") - - authority = f"{host}:{port}" if port is not None else host - upstream = f"{parsed.scheme}://{authority}" + parsed_routes = {} + servers = [] + for offset, (name, url) in enumerate(routes.items()): + parsed = urlsplit(url) + host = parsed.hostname + try: + port = parsed.port + except ValueError as e: + raise SandboxError( + f"invalid required network route {url!r}: {e}" + ) from e + if ( + parsed.scheme not in {"http", "https"} + or host is None + or parsed.username is not None + or parsed.password is not None + or re.fullmatch(r"[A-Za-z0-9.-]+", host) is None + ): + raise SandboxError(f"invalid required network route {url!r}") + path = parsed.path or "/" + if ( + not path.isascii() + or (path != "/" and not path.strip("/")) + or any( + not char.isprintable() or char.isspace() or char == "$" + for char in path + ) + ): + raise SandboxError(f"invalid required network route {url!r}") + authority = f"{host}:{port}" if port is not None else host + relay_port = _EGRESS_PORT + offset + parsed_routes[name] = parsed, relay_port + locations = ( + f"location / {{ proxy_pass {parsed.scheme}://{authority}; }}" + if path == "/" + else f""" + location = {json.dumps(path)} {{ proxy_pass {parsed.scheme}://{authority}; }} + location ^~ {json.dumps(f"{path.rstrip('/')}/")} {{ + proxy_pass {parsed.scheme}://{authority}; + }} + location / {{ return 403; }}""" + ) + servers.append( + f""" + server {{ + listen {relay_port}; + client_max_body_size 0; + proxy_set_header Host {authority}; + proxy_ssl_server_name on; + proxy_ssl_name {host}; + proxy_ssl_verify on; + proxy_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_request_buffering off; + proxy_buffering off; + proxy_next_upstream off; + proxy_read_timeout 86400s; + proxy_send_timeout 86400s; + {locations} + }} +""" + ) config = f""" worker_processes 1; pid /tmp/nginx.pid; @@ -191,51 +263,36 @@ async def seal_agent_network(self, endpoint: str) -> str: fastcgi_temp_path /tmp/fastcgi; uwsgi_temp_path /tmp/uwsgi; scgi_temp_path /tmp/scgi; - server {{ - listen {_RELAY_PORT}; - client_max_body_size 0; - location / {{ - proxy_pass {upstream}; - proxy_set_header Host {authority}; - proxy_ssl_server_name on; - proxy_ssl_name {host}; - proxy_ssl_verify on; - proxy_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt; - proxy_http_version 1.1; - proxy_set_header Connection ""; - proxy_request_buffering off; - proxy_buffering off; - proxy_next_upstream off; - proxy_read_timeout 86400s; - proxy_send_timeout 86400s; - }} - }} -}} +{"".join(servers)}}} """ - self._relay = f"{self.name}-relay" - self._network = f"{self.name}-network" + self._egress = f"{self.name}-egress" + self._execution_network = f"{self.name}-execution" await docker_checked( "network", "create", "--internal", + "--opt", + "com.docker.network.bridge.gateway_mode_ipv4=isolated", "--label", f"verifiers.runtime={self.name}", - self._network, - error="docker internal network creation failed", + self._execution_network, + error="docker execution network creation failed", ) await docker_checked( "create", "--name", - self._relay, + self._egress, "--network", - "bridge", + self._setup_network, "--label", f"verifiers.runtime={self.name}", "--user", "101:101", "--cap-drop", "ALL", + "--sysctl", + "net.ipv4.ip_forward=0", "--security-opt", "no-new-privileges", "--read-only", @@ -243,67 +300,68 @@ async def seal_agent_network(self, endpoint: str) -> str: "/tmp:rw,noexec,nosuid,size=16m,uid=101,gid=101", "--entrypoint", "sh", - _RELAY_IMAGE, + _EGRESS_IMAGE, "-c", 'printf "%s" "$1" > /tmp/nginx.conf && ' 'exec nginx -c /tmp/nginx.conf -g "daemon off;"', - "vf-relay", + "vf-egress", config, - error="docker interception relay creation failed", + error="docker egress creation failed", ) await docker_checked( "network", "connect", "--alias", - _RELAY_HOST, - self._network, - self._relay, - error="docker interception relay connection failed", - ) - await docker_checked( - "start", self._relay, error="docker interception relay start failed" + _EGRESS_HOST, + self._execution_network, + self._egress, + error="docker egress network connection failed", ) + await docker_checked("start", self._egress, error="docker egress start failed") for _ in range(20): ready = await docker( - "exec", self._relay, "nginx", "-t", "-c", "/tmp/nginx.conf" + "exec", self._egress, "nginx", "-t", "-c", "/tmp/nginx.conf" ) if ready.exit_code == 0: break await asyncio.sleep(0.05) else: - logs = await docker("logs", self._relay) + logs = await docker("logs", self._egress) raise SandboxError( - "docker interception relay did not become ready: " + "docker egress did not become ready: " f"{(logs.stderr or logs.stdout).strip()[-2000:]}" ) await docker_checked( "network", "connect", - self._network, + self._execution_network, self._container, - error="docker agent network connection failed", + error="docker execution network connection failed", ) await docker_checked( "network", "disconnect", - "bridge", + self._setup_network, self._container, - error="docker agent network isolation failed", + error="docker setup network disconnection failed", ) logger.info( - "docker: sealed container %s to interception relay %s", + "docker: applied network policy to %s via %s", self._container, - self._relay, + self._egress, ) - return urlunsplit( - ( - "http", - f"{_RELAY_HOST}:{_RELAY_PORT}", - parsed.path, - parsed.query, - "", + return { + name: urlunsplit( + ( + "http", + f"{_EGRESS_HOST}:{port}", + parsed.path, + parsed.query, + "", + ) ) - ) + for name, (parsed, port) in parsed_routes.items() + } async def run(self, argv: list[str], env: dict[str, str]) -> ProgramResult: env_args = [arg for k, v in env.items() for arg in ("--env", f"{k}={v}")] @@ -371,28 +429,51 @@ async def write(self, path: str, data: bytes) -> None: ) def cleanup(self) -> None: - if self._stopped or not any((self._container, self._relay, self._network)): - return - self._stopped = ( - True # keep names and ids so descriptors and logs survive teardown + resources = ( + self._container, + self._egress, + self._setup_network, + self._execution_network, ) - for container in (self._relay, self._container): + if self._stopped or not any(resources): + return + for attr in ("_egress", "_container"): + container = getattr(self, attr) if container is None: continue logger.debug("docker: removing container %s", container) - with contextlib.suppress(Exception): - subprocess.run( + try: + result = subprocess.run( ["docker", "rm", "--force", container], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30, ) - if self._network is not None: - logger.debug("docker: removing network %s", self._network) - with contextlib.suppress(Exception): - subprocess.run( - ["docker", "network", "rm", self._network], + except Exception: + continue + if result.returncode == 0: + setattr(self, attr, None) + for attr in ("_execution_network", "_setup_network"): + network = getattr(self, attr) + if network is None: + continue + logger.debug("docker: removing network %s", network) + try: + result = subprocess.run( + ["docker", "network", "rm", network], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30, ) + except Exception: + continue + if result.returncode == 0: + setattr(self, attr, None) + self._stopped = not any( + ( + self._container, + self._egress, + self._setup_network, + self._execution_network, + ) + ) diff --git a/verifiers/v1/runtimes/network.py b/verifiers/v1/runtimes/network.py new file mode 100644 index 0000000000..58d214f1d1 --- /dev/null +++ b/verifiers/v1/runtimes/network.py @@ -0,0 +1,23 @@ +"""Internal execution-network policy shared by runtime backends.""" + +from pydantic_config import BaseConfig + + +class NetworkPolicy(BaseConfig): + """Agent egress rules applied at the trusted setup-to-execution boundary. + + Framework routes such as model and MCP endpoints are passed separately to the + runtime. They remain reachable regardless of these rules and may be rewritten + through backend-specific relays. + + This stays internal until network target syntax and backend support are defined. + """ + + allow: list[str] | None = None + """Allowed network targets. None leaves arbitrary egress enabled; [] denies it.""" + block: list[str] = [] + """Blocked arbitrary-egress targets. These take precedence over allow rules.""" + + @property + def restricted(self) -> bool: + return self.allow is not None or bool(self.block) From 407b9792bae462dc30a768514d207854cc8ad0f6 Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Tue, 14 Jul 2026 19:34:27 +0200 Subject: [PATCH 6/7] fix(v1): restore direct Docker interception --- verifiers/v1/env.py | 17 ++++++++++-- verifiers/v1/interception/__init__.py | 32 ++++++++++------------- verifiers/v1/interception/pool.py | 16 +++++++++--- verifiers/v1/interception/server.py | 27 +++++++++---------- verifiers/v1/rollout.py | 6 +++-- verifiers/v1/runtimes/base.py | 5 ++++ verifiers/v1/runtimes/docker.py | 37 ++++++++++++++++++++++++--- 7 files changed, 97 insertions(+), 43 deletions(-) diff --git a/verifiers/v1/env.py b/verifiers/v1/env.py index 312f5b66d1..824b102e48 100644 --- a/verifiers/v1/env.py +++ b/verifiers/v1/env.py @@ -23,11 +23,13 @@ from verifiers.v1.retries import RetryConfig from verifiers.v1.rollout import Rollout from verifiers.v1.runtimes import ( + DockerConfig, RuntimeConfig, SubprocessConfig, runtime_is_local, runtime_reaches_host_locally, ) +from verifiers.v1.runtimes.docker import docker_interception_host from verifiers.v1.task import Task, resolve_server_config from verifiers.v1.taskset import Taskset, TasksetConfig from verifiers.v1.utils.generic import generic_type @@ -372,8 +374,19 @@ async def serving(self): what keeps both eval runners (in-process and env-server) on one serving path. Build episodes inside this context; the resources are torn down on exit.""" async with self.shared_tools() as shared: + tunneled = self._requires_tunnel(shared) + runtime = self.harness.config.runtime + extra_host = ( + await docker_interception_host() + if isinstance(runtime, DockerConfig) + and not runtime.network_access + and not tunneled + else None + ) interception = make_interception( - self.config.interception, requires_tunnel=self._requires_tunnel(shared) + self.config.interception, + requires_tunnel=tunneled, + extra_host=extra_host, ) async with interception: self._shared_tools = shared @@ -403,7 +416,7 @@ def _requires_tunnel(self, shared: dict[str, SharedToolServer]) -> bool: for server_cls in server_classes ] return requires_tunnel( - runtime_reaches_host_locally(self.harness.config.runtime), + runtime_is_local(self.harness.config.runtime), configs, shared.values(), ) diff --git a/verifiers/v1/interception/__init__.py b/verifiers/v1/interception/__init__.py index 5840c05034..e56f6e6ac3 100644 --- a/verifiers/v1/interception/__init__.py +++ b/verifiers/v1/interception/__init__.py @@ -40,18 +40,15 @@ def requires_tunnel( - harness_reaches_host: bool, + harness_is_local: bool, server_configs: Iterable[BaseConfig] = (), shared: "Iterable[SharedToolServer]" = (), ) -> bool: - """Whether the interception must be exposed via a tunnel — some consumer is off the - host network: the harness itself, a live `shared` server in a remote runtime, or a - tool/user server config placing one there (each reaches the `/state` channel from - its own runtime). Skipped as non-consumers: a `colocated` server (shares the - harness's runtime, covered by `harness_reaches_host`), a config-`url` server (external — - it connects out), and an `external` shared server (outside the state machinery - entirely). False means every consumer reaches the server at localhost.""" - if not harness_reaches_host: + """Whether interception needs a public tunnel because any consumer is remote. + + Local runtimes map framework routes through their network policy. Colocated servers + share that route; configured URLs and external shared servers do not consume state.""" + if not harness_is_local: return True if any(not s.external and not s.reaches_host for s in shared): return True @@ -64,18 +61,17 @@ def requires_tunnel( def make_interception( - config: InterceptionConfig, *, requires_tunnel: bool + config: InterceptionConfig, + *, + requires_tunnel: bool, + extra_host: str | None = None, ) -> Interception: - """The interception for a config, picked by type (the host-side counterpart to - `make_runtime`). With `requires_tunnel` (some consumer is off the host network — see - the `requires_tunnel` util) each server is exposed via its configured tunnel; without - it they get none and are reached at localhost. The caller computes it (see - `Environment._requires_tunnel`).""" + """Build the configured interception shape for the resolved network topology.""" if isinstance(config, InterceptionServerConfig): - return InterceptionServer(config, requires_tunnel) + return InterceptionServer(config, requires_tunnel, extra_host=extra_host) if isinstance(config, StaticInterceptionPoolConfig): - return StaticInterceptionPool(config, requires_tunnel) - return ElasticInterceptionPool(config, requires_tunnel) + return StaticInterceptionPool(config, requires_tunnel, extra_host=extra_host) + return ElasticInterceptionPool(config, requires_tunnel, extra_host=extra_host) __all__ = [ diff --git a/verifiers/v1/interception/pool.py b/verifiers/v1/interception/pool.py index 324260635d..1088aa408e 100644 --- a/verifiers/v1/interception/pool.py +++ b/verifiers/v1/interception/pool.py @@ -47,12 +47,17 @@ class StaticInterceptionPool(Interception): operator's call (it's the shape for pre-provisioned/bring-your-own endpoints).""" def __init__( - self, config: StaticInterceptionPoolConfig, requires_tunnel: bool = False + self, + config: StaticInterceptionPoolConfig, + requires_tunnel: bool = False, + *, + extra_host: str | None = None, ) -> None: super().__init__() self.config = config self.servers = [ - InterceptionServer(server, requires_tunnel) for server in config.servers + InterceptionServer(server, requires_tunnel, extra_host=extra_host) + for server in config.servers ] async def start(self) -> None: @@ -91,10 +96,13 @@ def __init__( self, config: ElasticInterceptionPoolConfig | None = None, requires_tunnel: bool = False, + *, + extra_host: str | None = None, ) -> None: super().__init__() self.config = config or ElasticInterceptionPoolConfig() self.requires_tunnel = requires_tunnel + self.extra_host = extra_host self.servers: list[InterceptionServer] = [] self._lock = asyncio.Lock() @@ -110,7 +118,9 @@ async def _server(self) -> InterceptionServer: return server # Pin prime explicitly — the only tunnel kind that can be minted on demand. server = InterceptionServer( - InterceptionServerConfig(tunnel=PrimeTunnelConfig()), self.requires_tunnel + InterceptionServerConfig(tunnel=PrimeTunnelConfig()), + self.requires_tunnel, + extra_host=self.extra_host, ) await self.stack.enter_async_context(server) self.servers.append(server) diff --git a/verifiers/v1/interception/server.py b/verifiers/v1/interception/server.py index a232266a07..4ad438ad24 100644 --- a/verifiers/v1/interception/server.py +++ b/verifiers/v1/interception/server.py @@ -98,17 +98,14 @@ class InterceptionServerConfig(BaseInterceptionConfig): class InterceptionServer(Interception): - """A server that proxies model calls for one or more rollouts — and is itself the - single-server `Interception` (the pools compose several of these). With - `requires_tunnel` (some consumer is off the host network) it mints its configured - tunnel; on `start` it then binds where the tunnel says (`bind_host`/`bind_port`) and - sets `base_url` — the one URL every consumer reaches it at — to the tunnel's public - URL. Without, every consumer is on the host network: it binds loopback, tunnel-free.""" + """A model proxy for one or more rollouts, optionally exposed through a tunnel.""" def __init__( self, config: InterceptionServerConfig | None = None, requires_tunnel: bool = False, + *, + extra_host: str | None = None, ) -> None: super().__init__() self.sessions: dict[str, RolloutSession] = {} @@ -116,7 +113,7 @@ def __init__( self.tunnel: Tunnel | None = ( make_tunnel(self.config.tunnel) if requires_tunnel else None ) - self.host = "127.0.0.1" + self.hosts = ("127.0.0.1",) + ((extra_host,) if extra_host else ()) self.port = 0 self.base_url = "" # set by `start` self.runner: web.AppRunner | None = None @@ -176,19 +173,19 @@ async def start(self) -> None: self.runner = web.AppRunner(app) await self.runner.setup() self.stack.push_async_callback(self.runner.cleanup) - # No tunnel → every consumer shares the host network: bind loopback on any ephemeral - # port. Otherwise the tunnel says where to bind for it to reach the port, and - # `expose` publishes it. + # Local Docker also listens on its bridge gateway while localhost stays canonical. if self.tunnel is None: - self.host, bind_port = "127.0.0.1", 0 + hosts, bind_port = self.hosts, 0 else: - self.host, bind_port = self.tunnel.bind_host, self.tunnel.bind_port - site = web.TCPSite(self.runner, self.host, bind_port) + hosts, bind_port = (self.tunnel.bind_host,), self.tunnel.bind_port + site = web.TCPSite(self.runner, hosts[0], bind_port) await site.start() self.port = site._server.sockets[0].getsockname()[1] # actual bound port - logger.info("interception up: url=http://%s:%d", self.host, self.port) + for host in hosts[1:]: + await web.TCPSite(self.runner, host, self.port).start() + logger.info("interception up: hosts=%s port=%d", hosts, self.port) self.stack.callback( - logger.info, "interception down: url=http://%s:%d", self.host, self.port + logger.info, "interception down: hosts=%s port=%d", hosts, self.port ) if self.tunnel is None: self.base_url = f"http://127.0.0.1:{self.port}" diff --git a/verifiers/v1/rollout.py b/verifiers/v1/rollout.py index 28a4e8a17a..2b4deaab35 100644 --- a/verifiers/v1/rollout.py +++ b/verifiers/v1/rollout.py @@ -91,11 +91,13 @@ async def _serve_interception( yield slot return tunneled = requires_tunnel( - runtime.reaches_host_locally, + runtime.is_local, [server.config for server in servers], self.shared_tools.values(), ) - server = InterceptionServer(requires_tunnel=tunneled) + server = InterceptionServer( + requires_tunnel=tunneled, extra_host=runtime.interception_host + ) async with server: async with server.acquire(session) as slot: yield slot diff --git a/verifiers/v1/runtimes/base.py b/verifiers/v1/runtimes/base.py index 9d3f83dcae..3e2b35145a 100644 --- a/verifiers/v1/runtimes/base.py +++ b/verifiers/v1/runtimes/base.py @@ -141,6 +141,11 @@ def reachable_from_host_locally(self) -> bool: """Whether the host reaches services here at localhost, without exposure.""" return self.is_local + @property + def interception_host(self) -> str | None: + """Additional host address where local interception must listen.""" + return None + @property def network_policy(self) -> NetworkPolicy: """Egress policy applied at the trusted setup-to-execution boundary.""" diff --git a/verifiers/v1/runtimes/docker.py b/verifiers/v1/runtimes/docker.py index d26bc34971..e99701f933 100644 --- a/verifiers/v1/runtimes/docker.py +++ b/verifiers/v1/runtimes/docker.py @@ -6,6 +6,7 @@ import re import shlex import subprocess +import sys from pathlib import PurePosixPath from typing import Literal, cast from urllib.parse import urlsplit, urlunsplit @@ -74,6 +75,22 @@ async def docker_checked(*args: str, error: str) -> ProgramResult: raise SandboxError(f"{error}: {detail}") +async def docker_interception_host() -> str | None: + """Return the host address used by native Linux containers.""" + if sys.platform != "linux": + return None + result = await docker( + "network", "inspect", "--format", "{{(index .IPAM.Config 0).Gateway}}", "bridge" + ) + gateway = result.stdout.strip() + if result.exit_code != 0 or not gateway: + raise SandboxError( + "could not resolve the Docker host gateway: " + f"{(result.stderr or result.stdout).strip()}" + ) + return gateway + + class DockerRuntime(Runtime): def __init__(self, config: DockerConfig, name: str | None = None) -> None: super().__init__(name) @@ -83,6 +100,7 @@ def __init__(self, config: DockerConfig, name: str | None = None) -> None: self._egress: str | None = None self._setup_network: str | None = None self._execution_network: str | None = None + self._interception_host: str | None = None self._stopped = False @classmethod @@ -93,6 +111,10 @@ def config_reaches_host_locally(cls, config: BaseConfig) -> bool: def reachable_from_host_locally(self) -> bool: return self.config.network_access + @property + def interception_host(self) -> str | None: + return self._interception_host + @property def network_policy(self) -> NetworkPolicy: return NetworkPolicy(allow=None if self.config.network_access else []) @@ -134,6 +156,7 @@ async def start(self) -> None: if gpu_count: limits += ["--gpus", str(gpu_count)] if self.network_policy.restricted: + self._interception_host = await docker_interception_host() self._setup_network = f"{self.name}-setup" await docker_checked( "network", @@ -218,15 +241,21 @@ async def apply_network_policy(self, routes: dict[str, str]) -> dict[str, str]: ): raise SandboxError(f"invalid required network route {url!r}") authority = f"{host}:{port}" if port is not None else host + upstream_host = ( + "host.docker.internal" if host in {"127.0.0.1", "localhost"} else host + ) + upstream_authority = ( + f"{upstream_host}:{port}" if port is not None else upstream_host + ) relay_port = _EGRESS_PORT + offset parsed_routes[name] = parsed, relay_port locations = ( - f"location / {{ proxy_pass {parsed.scheme}://{authority}; }}" + f"location / {{ proxy_pass {parsed.scheme}://{upstream_authority}; }}" if path == "/" else f""" - location = {json.dumps(path)} {{ proxy_pass {parsed.scheme}://{authority}; }} + location = {json.dumps(path)} {{ proxy_pass {parsed.scheme}://{upstream_authority}; }} location ^~ {json.dumps(f"{path.rstrip('/')}/")} {{ - proxy_pass {parsed.scheme}://{authority}; + proxy_pass {parsed.scheme}://{upstream_authority}; }} location / {{ return 403; }}""" ) @@ -285,6 +314,8 @@ async def apply_network_policy(self, routes: dict[str, str]) -> dict[str, str]: self._egress, "--network", self._setup_network, + "--add-host", + f"host.docker.internal:{self._interception_host or 'host-gateway'}", "--label", f"verifiers.runtime={self.name}", "--user", From 4d23b5acf45a9e3a9ec6bdbfd25c7b4195012322 Mon Sep 17 00:00:00 2001 From: Xeophon <46377542+xeophon@users.noreply.github.com> Date: Tue, 14 Jul 2026 20:04:29 +0200 Subject: [PATCH 7/7] fix(v1): preserve shared tool reachability --- verifiers/v1/mcp/launch.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/verifiers/v1/mcp/launch.py b/verifiers/v1/mcp/launch.py index 5a0c747508..1643a461b9 100644 --- a/verifiers/v1/mcp/launch.py +++ b/verifiers/v1/mcp/launch.py @@ -403,7 +403,11 @@ async def serve_tools( urls[name] = server.url logger.info("tool server '%s' (shared, external): %s", name, server.url) continue - url = harness_runtime.host_url(server.url) if server.local else server.url + url = ( + harness_runtime.host_url(server.url) + if server.reaches_host + else server.url + ) urls[name] = _shared_url_for_rollout(url, state_base, state_secret) # The tagged URL contains the bearer secret; log only the untagged base URL. logger.info("tool server '%s' (shared): %s", name, server.url)