Preserve interception for offline Docker agents

verifiers/v1/env.py

@@ -23,10 +23,13 @@
 from verifiers.v1.retries import RetryConfig
 from verifiers.v1.rollout import Rollout
 from verifiers.v1.runtimes import (
+    DockerConfig,
     RuntimeConfig,
     SubprocessConfig,
     runtime_is_local,
+    runtime_reaches_host_locally,
 )
+from verifiers.v1.runtimes.docker import docker_interception_host
 from verifiers.v1.task import Task, resolve_server_config
 from verifiers.v1.taskset import Taskset, TasksetConfig
 from verifiers.v1.utils.generic import generic_type
@@ -371,8 +374,19 @@ async def serving(self):
         what keeps both eval runners (in-process and env-server) on one serving path. Build
         episodes inside this context; the resources are torn down on exit."""
         async with self.shared_tools() as shared:
+            tunneled = self._requires_tunnel(shared)
+            runtime = self.harness.config.runtime
+            extra_host = (
+                await docker_interception_host()
+                if isinstance(runtime, DockerConfig)
+                and not runtime.network_access
+                and not tunneled
+                else None
+            )
             interception = make_interception(
-                self.config.interception, requires_tunnel=self._requires_tunnel(shared)
+                self.config.interception,
+                requires_tunnel=tunneled,
+                extra_host=extra_host,
             )
             async with interception:
                 self._shared_tools = shared
@@ -402,7 +416,9 @@ def _requires_tunnel(self, shared: dict[str, SharedToolServer]) -> bool:
             for server_cls in server_classes
         ]
         return requires_tunnel(
-            runtime_is_local(self.harness.config.runtime), configs, shared.values()
+            runtime_is_local(self.harness.config.runtime),
+            configs,
+            shared.values(),
         )
 
     @contextlib.asynccontextmanager
@@ -411,6 +427,8 @@ async def shared_tools(self):
         if not servers:
             yield {}
             return
-        harness_is_local = runtime_is_local(self.harness.config.runtime)
-        async with serve_shared(servers, harness_is_local=harness_is_local) as shared:
+        harness_reaches_host = runtime_reaches_host_locally(self.harness.config.runtime)
+        async with serve_shared(
+            servers, harness_reaches_host=harness_reaches_host
+        ) as shared:
             yield shared

verifiers/v1/interception/__init__.py

@@ -25,7 +25,7 @@
     InterceptionServer,
     InterceptionServerConfig,
 )
-from verifiers.v1.runtimes import runtime_is_local
+from verifiers.v1.runtimes import runtime_reaches_host_locally
 
 if TYPE_CHECKING:
     from verifiers.v1.mcp import SharedToolServer
@@ -44,38 +44,34 @@ def requires_tunnel(
     server_configs: Iterable[BaseConfig] = (),
     shared: "Iterable[SharedToolServer]" = (),
 ) -> bool:
-    """Whether the interception must be exposed via a tunnel — some consumer is off the
-    host network: the harness itself, a live `shared` server in a remote runtime, or a
-    tool/user server config placing one there (each reaches the `/state` channel from
-    its own runtime). Skipped as non-consumers: a `colocated` server (shares the
-    harness's runtime, covered by `harness_is_local`), a config-`url` server (external —
-    it connects out), and an `external` shared server (outside the state machinery
-    entirely). False means every consumer reaches the server at localhost."""
+    """Whether interception needs a public tunnel because any consumer is remote.
+
+    Local runtimes map framework routes through their network policy. Colocated servers
+    share that route; configured URLs and external shared servers do not consume state."""
     if not harness_is_local:
         return True
-    if any(not s.external and not s.local for s in shared):
+    if any(not s.external and not s.reaches_host for s in shared):
         return True
     for config in server_configs:
         if getattr(config, "url", None) or config.colocated:
             continue
-        if not runtime_is_local(config.runtime):
+        if not runtime_reaches_host_locally(config.runtime):
             return True
     return False
 
 
 def make_interception(
-    config: InterceptionConfig, *, requires_tunnel: bool
+    config: InterceptionConfig,
+    *,
+    requires_tunnel: bool,
+    extra_host: str | None = None,
 ) -> Interception:
-    """The interception for a config, picked by type (the host-side counterpart to
-    `make_runtime`). With `requires_tunnel` (some consumer is off the host network — see
-    the `requires_tunnel` util) each server is exposed via its configured tunnel; without
-    it they get none and are reached at localhost. The caller computes it (see
-    `Environment._requires_tunnel`)."""
+    """Build the configured interception shape for the resolved network topology."""
     if isinstance(config, InterceptionServerConfig):
-        return InterceptionServer(config, requires_tunnel)
+        return InterceptionServer(config, requires_tunnel, extra_host=extra_host)
     if isinstance(config, StaticInterceptionPoolConfig):
-        return StaticInterceptionPool(config, requires_tunnel)
-    return ElasticInterceptionPool(config, requires_tunnel)
+        return StaticInterceptionPool(config, requires_tunnel, extra_host=extra_host)
+    return ElasticInterceptionPool(config, requires_tunnel, extra_host=extra_host)
 
 
 __all__ = [

verifiers/v1/interception/pool.py

@@ -47,12 +47,17 @@ class StaticInterceptionPool(Interception):
     operator's call (it's the shape for pre-provisioned/bring-your-own endpoints)."""
 
     def __init__(
-        self, config: StaticInterceptionPoolConfig, requires_tunnel: bool = False
+        self,
+        config: StaticInterceptionPoolConfig,
+        requires_tunnel: bool = False,
+        *,
+        extra_host: str | None = None,
     ) -> None:
         super().__init__()
         self.config = config
         self.servers = [
-            InterceptionServer(server, requires_tunnel) for server in config.servers
+            InterceptionServer(server, requires_tunnel, extra_host=extra_host)
+            for server in config.servers
         ]
 
     async def start(self) -> None:
@@ -91,10 +96,13 @@ def __init__(
         self,
         config: ElasticInterceptionPoolConfig | None = None,
         requires_tunnel: bool = False,
+        *,
+        extra_host: str | None = None,
     ) -> None:
         super().__init__()
         self.config = config or ElasticInterceptionPoolConfig()
         self.requires_tunnel = requires_tunnel
+        self.extra_host = extra_host
         self.servers: list[InterceptionServer] = []
         self._lock = asyncio.Lock()
 
@@ -110,7 +118,9 @@ async def _server(self) -> InterceptionServer:
                 return server
         # Pin prime explicitly — the only tunnel kind that can be minted on demand.
         server = InterceptionServer(
-            InterceptionServerConfig(tunnel=PrimeTunnelConfig()), self.requires_tunnel
+            InterceptionServerConfig(tunnel=PrimeTunnelConfig()),
+            self.requires_tunnel,
+            extra_host=self.extra_host,
         )
         await self.stack.enter_async_context(server)
         self.servers.append(server)

verifiers/v1/interception/server.py

@@ -98,25 +98,22 @@ class InterceptionServerConfig(BaseInterceptionConfig):
 
 
 class InterceptionServer(Interception):
-    """A server that proxies model calls for one or more rollouts — and is itself the
-    single-server `Interception` (the pools compose several of these). With
-    `requires_tunnel` (some consumer is off the host network) it mints its configured
-    tunnel; on `start` it then binds where the tunnel says (`bind_host`/`bind_port`) and
-    sets `base_url` — the one URL every consumer reaches it at — to the tunnel's public
-    URL. Without, every consumer is on the host network: it binds loopback, tunnel-free."""
+    """A model proxy for one or more rollouts, optionally exposed through a tunnel."""
 
     def __init__(
         self,
         config: InterceptionServerConfig | None = None,
         requires_tunnel: bool = False,
+        *,
+        extra_host: str | None = None,
     ) -> None:
         super().__init__()
         self.sessions: dict[str, RolloutSession] = {}
         self.config = config or InterceptionServerConfig()
         self.tunnel: Tunnel | None = (
             make_tunnel(self.config.tunnel) if requires_tunnel else None
         )
-        self.host = "127.0.0.1"
+        self.hosts = ("127.0.0.1",) + ((extra_host,) if extra_host else ())
         self.port = 0
         self.base_url = ""  # set by `start`
         self.runner: web.AppRunner | None = None
@@ -176,19 +173,19 @@ async def start(self) -> None:
         self.runner = web.AppRunner(app)
         await self.runner.setup()
         self.stack.push_async_callback(self.runner.cleanup)
-        # No tunnel → every consumer shares the host network: bind loopback on any ephemeral
-        # port. Otherwise the tunnel says where to bind for it to reach the port, and
-        # `expose` publishes it.
+        # Local Docker also listens on its bridge gateway while localhost stays canonical.
         if self.tunnel is None:
-            self.host, bind_port = "127.0.0.1", 0
+            hosts, bind_port = self.hosts, 0
         else:
-            self.host, bind_port = self.tunnel.bind_host, self.tunnel.bind_port
-        site = web.TCPSite(self.runner, self.host, bind_port)
+            hosts, bind_port = (self.tunnel.bind_host,), self.tunnel.bind_port
+        site = web.TCPSite(self.runner, hosts[0], bind_port)
         await site.start()
         self.port = site._server.sockets[0].getsockname()[1]  # actual bound port
-        logger.info("interception up: url=http://%s:%d", self.host, self.port)
+        for host in hosts[1:]:
+            await web.TCPSite(self.runner, host, self.port).start()
+        logger.info("interception up: hosts=%s port=%d", hosts, self.port)
         self.stack.callback(
-            logger.info, "interception down: url=http://%s:%d", self.host, self.port
+            logger.info, "interception down: hosts=%s port=%d", hosts, self.port
         )
         if self.tunnel is None:
             self.base_url = f"http://127.0.0.1:{self.port}"

verifiers/v1/mcp/launch.py

@@ -17,13 +17,13 @@
 from typing import TYPE_CHECKING
 from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
 
-from verifiers.v1.errors import RolloutError, ToolsetError, UserError
+from verifiers.v1.errors import RolloutError, SandboxError, ToolsetError, UserError
 from verifiers.v1.interception.tunnel import PrimeTunnel
 from verifiers.v1.mcp.server import STATE_SECRET_PARAM, STATE_URL_PARAM, ServerBase
 from verifiers.v1.runtimes import (
     Runtime,
     make_runtime,
-    runtime_is_local,
+    runtime_reaches_host_locally,
 )
 from verifiers.v1.runtimes.base import _ENSURE_UV
 from verifiers.v1.types import Messages
@@ -217,21 +217,24 @@ async def serve_in_runtime(
 
 @contextlib.asynccontextmanager
 async def reachable_url(
-    service: Runtime, port: int, *, colocated: bool, consumer_is_local: bool
+    service: Runtime, port: int, *, colocated: bool, consumer_reaches_host: bool
 ) -> AsyncIterator[str]:
     """Yield the URL a consumer uses to reach the server at (`service`, `port`), over two
     primitives: `Runtime.expose` (publish a port out of a sandbox) and a host `Tunnel` (reach
     into the host from a remote runtime). `colocated` = the server shares the consumer's
-    runtime; `consumer_is_local` = the consumer is on the host network.
+    runtime; `consumer_reaches_host` = the consumer reaches the host network directly.
 
     - `colocated` -> localhost (same runtime, in-sandbox or host loopback);
-    - the server runs in a remote sandbox -> its own published URL (`expose`), reachable anywhere;
+    - the server is not host-reachable -> its published URL (`expose`), reachable anywhere;
     - else it's on the host network -> localhost to a local consumer, a host tunnel to a remote one."""
     if colocated:
         yield f"http://127.0.0.1:{port}"
-    elif not service.is_local:  # in a remote sandbox → it publishes its own port
-        yield await service.expose(port)
-    elif consumer_is_local:  # host network, local consumer → localhost, no tunnel
+    elif not service.reachable_from_host_locally:
+        url = await service.expose(port)
+        if url is None:
+            raise SandboxError(f"{service.type} runtime cannot expose port {port}")
+        yield url
+    elif consumer_reaches_host:  # host network, direct consumer → localhost, no tunnel
         yield f"http://127.0.0.1:{port}"
     else:  # host network, remote consumer → a host tunnel publishes the port outward
         async with PrimeTunnel().expose(port) as url:
@@ -243,7 +246,7 @@ async def serve(
     server: ServerBase,
     harness_runtime: Runtime | None = None,
     for_host: bool = False,
-    harness_is_local: bool = True,
+    harness_reaches_host: bool = True,
     *,
     state_secret: str = "",
     state_base: str | None = None,
@@ -254,6 +257,9 @@ async def serve(
             runtime = harness_runtime
         else:
             runtime = make_runtime(cfg.runtime)
+            if runtime.network_policy.restricted:
+                error = UserError if for_host else ToolsetError
+                raise error("network policies are only applied to harness runtimes")
             await runtime.start()
             stack.push_async_callback(runtime.stop)
         # Only consumers outside the server runtime need its fixed published port. Colocated tools
@@ -276,19 +282,22 @@ async def serve(
         # Who consumes the server decides reachability: a user sim is reached by the HOST
         # (`for_host`, always local, never colocated with it); a tool by the harness — colocated
         # when it shares the harness's runtime, reached with the harness's locality (read off the
-        # harness runtime when there is one, else `harness_is_local` for an eval-level shared tool).
+        # harness runtime when there is one, else `harness_reaches_host` for a shared tool).
         if for_host:
-            colocated, consumer_is_local = False, True
+            colocated, consumer_reaches_host = False, True
         else:
             colocated = runtime is harness_runtime
-            consumer_is_local = (
-                harness_runtime.is_local
+            consumer_reaches_host = (
+                harness_runtime.reaches_host_locally
                 if harness_runtime is not None
-                else harness_is_local
+                else harness_reaches_host
             )
         base = await stack.enter_async_context(
             reachable_url(
-                runtime, port, colocated=colocated, consumer_is_local=consumer_is_local
+                runtime,
+                port,
+                colocated=colocated,
+                consumer_reaches_host=consumer_reaches_host,
             )
         )
         if not for_host and not colocated and harness_runtime is not None:
@@ -299,25 +308,25 @@ async def serve(
 @dataclass(frozen=True)
 class SharedToolServer:
     """One live taskset-scoped (shared) server, as the rollouts see it: its eval-level
-    `url` plus whether its runtime is `local` (host-reachable) — a remote one is an
+    `url` plus whether its runtime reaches the host directly — otherwise it is an
     interception consumer, so the interception must be exposed for it to reach the
     `/state` channel (see `Environment._requires_tunnel`). An `external` server (a
     config-`url` endpoint) was not launched by the framework and sits outside its state
     machinery entirely: rollouts get its URL bare — no state tag (and no per-rollout
     secret sent to a third party)."""
 
     url: str
-    local: bool
+    reaches_host: bool
     external: bool = False
 
 
 @contextlib.asynccontextmanager
-async def serve_shared(toolsets: list[Toolset], harness_is_local: bool = True):
+async def serve_shared(toolsets: list[Toolset], harness_reaches_host: bool = True):
     """Start the taskset-scoped (shared) tool servers ONCE for a whole eval, each in its OWN
     `runtime`, and yield `{name: SharedToolServer}` reachable by every rollout's harness.
     Reachability mirrors a per-rollout tool, but there's no single harness runtime to read
     locality off — the caller (`Environment.shared_tools`) passes the harness runtime's
-    `harness_is_local`, so a host tool gets one host bridge (tunnel) when the harness runs
+    `harness_reaches_host`, so a host tool gets one host bridge when the harness runs
     remotely, and a remote tool runtime publishes its own URL. Torn down when the eval ends.
     A shared server is task-agnostic — the taskset carries no per-row data — so its `setup`
     gets no task (its `setup_task` is never called; the per-rollout servers fetch
@@ -342,14 +351,15 @@ async def serve_shared(toolsets: list[Toolset], harness_is_local: bool = True):
                 )
             if cfg.url:  # already running remotely; nothing launched, nothing to bridge
                 servers[name] = SharedToolServer(
-                    url=cfg.url, local=False, external=True
+                    url=cfg.url, reaches_host=False, external=True
                 )
             else:
                 url = await stack.enter_async_context(
-                    serve(toolset, harness_is_local=harness_is_local)
+                    serve(toolset, harness_reaches_host=harness_reaches_host)
                 )
                 servers[name] = SharedToolServer(
-                    url=url, local=runtime_is_local(cfg.runtime)
+                    url=url,
+                    reaches_host=runtime_reaches_host_locally(cfg.runtime),
                 )
             logger.info("shared tool server '%s': %s", name, servers[name].url)
         yield servers
@@ -393,7 +403,11 @@ async def serve_tools(
                 urls[name] = server.url
                 logger.info("tool server '%s' (shared, external): %s", name, server.url)
                 continue
-            url = harness_runtime.host_url(server.url) if server.local else server.url
+            url = (
+                harness_runtime.host_url(server.url)
+                if server.reaches_host
+                else server.url
+            )
             urls[name] = _shared_url_for_rollout(url, state_base, state_secret)
             # The tagged URL contains the bearer secret; log only the untagged base URL.
             logger.info("tool server '%s' (shared): %s", name, server.url)