From d21612be16fc9c8674031d17e9deb6121193381c Mon Sep 17 00:00:00 2001 From: Sandro Date: Tue, 23 Jun 2026 16:54:54 +0100 Subject: [PATCH 01/10] feat(sandbox): add egress proxy wire schemas --- daiv/core/sandbox/schemas.dump.json | 247 ++++++++++++++++++ daiv/core/sandbox/schemas.py | 51 +++- .../core/sandbox/test_schema_consistency.py | 5 + tests/unit_tests/core/sandbox/test_schemas.py | 25 ++ 4 files changed, 327 insertions(+), 1 deletion(-) diff --git a/daiv/core/sandbox/schemas.dump.json b/daiv/core/sandbox/schemas.dump.json index c759c916..3f73bbbe 100644 --- a/daiv/core/sandbox/schemas.dump.json +++ b/daiv/core/sandbox/schemas.dump.json @@ -99,6 +99,253 @@ "title": "ApplyMutationsResponse", "type": "object" }, + "EgressConfigRequest": { + "$defs": { + "EgressPolicy": { + "properties": { + "default": { + "default": "deny", + "description": "Reachability for unlisted hosts.", + "enum": [ + "deny", + "allow" + ], + "title": "Default", + "type": "string" + }, + "intercept": { + "default": "all", + "description": "'all' MITMs every reachable host; 'credentialed' MITMs only inject hosts.", + "enum": [ + "all", + "credentialed" + ], + "title": "Intercept", + "type": "string" + }, + "rules": { + "items": { + "$ref": "#/$defs/EgressRule" + }, + "title": "Rules", + "type": "array" + } + }, + "title": "EgressPolicy", + "type": "object" + }, + "EgressRule": { + "properties": { + "host": { + "description": "Destination host glob (e.g. 'github.com', '*.githubusercontent.com').", + "title": "Host", + "type": "string" + }, + "inject": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Name of the secret whose header is injected for this host.", + "title": "Inject" + }, + "methods": { + "description": "Allowed HTTP methods, or ['*'] for any.", + "items": { + "type": "string" + }, + "title": "Methods", + "type": "array" + } + }, + "required": [ + "host" + ], + "title": "EgressRule", + "type": "object" + }, + "EgressSecret": { + "properties": { + "header": { + "description": "Header name to set (e.g. 'Authorization', 'PRIVATE-TOKEN').", + "title": "Header", + "type": "string" + }, + "value": { + "description": "Header value; redacted in logs/repr.", + "format": "password", + "title": "Value", + "type": "string", + "writeOnly": true + } + }, + "required": [ + "header", + "value" + ], + "title": "EgressSecret", + "type": "object" + } + }, + "properties": { + "policy": { + "$ref": "#/$defs/EgressPolicy" + }, + "secrets": { + "additionalProperties": { + "$ref": "#/$defs/EgressSecret" + }, + "title": "Secrets", + "type": "object" + } + }, + "title": "EgressConfigRequest", + "type": "object" + }, + "EgressConfigResponse": { + "properties": { + "ok": { + "default": true, + "description": "True when the policy was provisioned to the sidecar.", + "title": "Ok", + "type": "boolean" + } + }, + "title": "EgressConfigResponse", + "type": "object" + }, + "EgressPolicy": { + "$defs": { + "EgressRule": { + "properties": { + "host": { + "description": "Destination host glob (e.g. 'github.com', '*.githubusercontent.com').", + "title": "Host", + "type": "string" + }, + "inject": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Name of the secret whose header is injected for this host.", + "title": "Inject" + }, + "methods": { + "description": "Allowed HTTP methods, or ['*'] for any.", + "items": { + "type": "string" + }, + "title": "Methods", + "type": "array" + } + }, + "required": [ + "host" + ], + "title": "EgressRule", + "type": "object" + } + }, + "properties": { + "default": { + "default": "deny", + "description": "Reachability for unlisted hosts.", + "enum": [ + "deny", + "allow" + ], + "title": "Default", + "type": "string" + }, + "intercept": { + "default": "all", + "description": "'all' MITMs every reachable host; 'credentialed' MITMs only inject hosts.", + "enum": [ + "all", + "credentialed" + ], + "title": "Intercept", + "type": "string" + }, + "rules": { + "items": { + "$ref": "#/$defs/EgressRule" + }, + "title": "Rules", + "type": "array" + } + }, + "title": "EgressPolicy", + "type": "object" + }, + "EgressRule": { + "properties": { + "host": { + "description": "Destination host glob (e.g. 'github.com', '*.githubusercontent.com').", + "title": "Host", + "type": "string" + }, + "inject": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Name of the secret whose header is injected for this host.", + "title": "Inject" + }, + "methods": { + "description": "Allowed HTTP methods, or ['*'] for any.", + "items": { + "type": "string" + }, + "title": "Methods", + "type": "array" + } + }, + "required": [ + "host" + ], + "title": "EgressRule", + "type": "object" + }, + "EgressSecret": { + "properties": { + "header": { + "description": "Header name to set (e.g. 'Authorization', 'PRIVATE-TOKEN').", + "title": "Header", + "type": "string" + }, + "value": { + "description": "Header value; redacted in logs/repr.", + "format": "password", + "title": "Value", + "type": "string", + "writeOnly": true + } + }, + "required": [ + "header", + "value" + ], + "title": "EgressSecret", + "type": "object" + }, "FsDeleteRequest": { "properties": { "path": { diff --git a/daiv/core/sandbox/schemas.py b/daiv/core/sandbox/schemas.py index 505f1565..0503839c 100644 --- a/daiv/core/sandbox/schemas.py +++ b/daiv/core/sandbox/schemas.py @@ -3,7 +3,7 @@ from enum import StrEnum from typing import Literal -from pydantic import Base64Bytes, BaseModel, Field, computed_field, field_validator, model_validator +from pydantic import Base64Bytes, BaseModel, Field, SecretStr, computed_field, field_validator, model_validator class StartSessionRequest(BaseModel): @@ -226,3 +226,52 @@ class FsDeleteResponse(BaseModel): @property def ok(self) -> bool: return self.error is None + + +# --- Egress proxy wire schemas ----------------------------------------------- +# +# Mirror of the daiv-sandbox ``Egress*`` schemas; kept structurally identical so the +# schema-drift test (test_schema_consistency.py) passes. ``value`` is ``SecretStr`` for +# log redaction — the client sends the plaintext via ``get_secret_value()``. + + +class EgressRule(BaseModel): + host: str = Field(description="Destination host glob (e.g. 'github.com', '*.githubusercontent.com').") + methods: list[str] = Field(default_factory=lambda: ["*"], description="Allowed HTTP methods, or ['*'] for any.") + inject: str | None = Field(default=None, description="Name of the secret whose header is injected for this host.") + + @field_validator("methods", mode="after") + @classmethod + def _upper(cls, value: list[str]) -> list[str]: + if not value: + raise ValueError("methods must not be empty; use ['*'] to allow all methods") + return [m.upper() for m in value] + + +class EgressSecret(BaseModel): + header: str = Field(description="Header name to set (e.g. 'Authorization', 'PRIVATE-TOKEN').") + value: SecretStr = Field(description="Header value; redacted in logs/repr.") + + +class EgressPolicy(BaseModel): + default: Literal["deny", "allow"] = Field(default="deny", description="Reachability for unlisted hosts.") + intercept: Literal["all", "credentialed"] = Field( + default="all", description="'all' MITMs every reachable host; 'credentialed' MITMs only inject hosts." + ) + rules: list[EgressRule] = Field(default_factory=list) + + +class EgressConfigRequest(BaseModel): + policy: EgressPolicy = Field(default_factory=EgressPolicy) + secrets: dict[str, EgressSecret] = Field(default_factory=dict) + + @model_validator(mode="after") + def _injects_resolve(self) -> EgressConfigRequest: + for rule in self.policy.rules: + if rule.inject is not None and rule.inject not in self.secrets: + raise ValueError(f"rule for host {rule.host!r} references unknown secret {rule.inject!r}") + return self + + +class EgressConfigResponse(BaseModel): + ok: bool = Field(default=True, description="True when the policy was provisioned to the sidecar.") diff --git a/tests/unit_tests/core/sandbox/test_schema_consistency.py b/tests/unit_tests/core/sandbox/test_schema_consistency.py index 649739d0..72e77cd9 100644 --- a/tests/unit_tests/core/sandbox/test_schema_consistency.py +++ b/tests/unit_tests/core/sandbox/test_schema_consistency.py @@ -47,6 +47,11 @@ def _normalize(schema): "FsEditResponse", "FsDeleteRequest", "FsDeleteResponse", + "EgressRule", + "EgressSecret", + "EgressPolicy", + "EgressConfigRequest", + "EgressConfigResponse", ] # Types that exist on both sides but are deliberately allowed to diverge. diff --git a/tests/unit_tests/core/sandbox/test_schemas.py b/tests/unit_tests/core/sandbox/test_schemas.py index 64942ffe..5e8843c1 100644 --- a/tests/unit_tests/core/sandbox/test_schemas.py +++ b/tests/unit_tests/core/sandbox/test_schemas.py @@ -82,3 +82,28 @@ def test_fs_delete_response_removed_and_ok(): failed = FsDeleteResponse(error=FsError(code=FsErrorCode.IS_A_DIRECTORY, message="is a directory")) assert failed.ok is False and failed.error.code is FsErrorCode.IS_A_DIRECTORY + + +def test_egress_rule_uppercases_methods_and_rejects_empty(): + from core.sandbox.schemas import EgressRule + + assert EgressRule(host="x", methods=["get", "post"]).methods == ["GET", "POST"] + with pytest.raises(ValueError): + EgressRule(host="x", methods=[]) + + +def test_egress_config_request_rejects_dangling_inject(): + from core.sandbox.schemas import EgressConfigRequest, EgressPolicy, EgressRule + + with pytest.raises(ValueError, match="unknown secret"): + EgressConfigRequest(policy=EgressPolicy(rules=[EgressRule(host="x", inject="missing")]), secrets={}) + + +def test_egress_secret_value_is_redacted_in_repr(): + from pydantic import SecretStr + + from core.sandbox.schemas import EgressSecret + + s = EgressSecret(header="Authorization", value=SecretStr("Bearer t")) + assert "Bearer t" not in repr(s) + assert s.value.get_secret_value() == "Bearer t" From 747cef6cfd3d3df02f79e370ece028586f7b5111 Mon Sep 17 00:00:00 2001 From: Sandro Date: Tue, 23 Jun 2026 17:00:05 +0100 Subject: [PATCH 02/10] feat(sandbox): client.configure_egress provisions the sidecar policy --- daiv/core/sandbox/client.py | 21 ++++++++++++++++ tests/unit_tests/core/sandbox/test_client.py | 25 ++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/daiv/core/sandbox/client.py b/daiv/core/sandbox/client.py index 93605d12..21a31076 100644 --- a/daiv/core/sandbox/client.py +++ b/daiv/core/sandbox/client.py @@ -12,6 +12,8 @@ from .schemas import ( ApplyMutationsRequest, ApplyMutationsResponse, + EgressConfigRequest, + EgressConfigResponse, FsDeleteRequest, FsDeleteResponse, FsEditRequest, @@ -226,6 +228,25 @@ async def run_commands(self, session_id: str, request: RunCommandsRequest) -> Ru response.raise_for_status() return RunCommandsResponse.model_validate(response.json()) + async def configure_egress(self, session_id: str, request: EgressConfigRequest) -> EgressConfigResponse: + """Provision (replace) the egress policy + secrets on the session's sidecar proxy. + + Idempotent on the sandbox side. Secret values are sent in plaintext (the sidecar needs them); + ``SecretStr`` keeps them redacted in logs/repr, so the payload is built explicitly here rather + than via ``request.model_dump(mode="json")``, which would mask the values. Raises + ``httpx.HTTPStatusError`` on a non-2xx (404 = egress not enabled on the sandbox, 409 = the + session has no proxy); the caller decides how to react. + """ + payload = { + "policy": request.policy.model_dump(mode="json"), + "secrets": { + name: {"header": s.header, "value": s.value.get_secret_value()} for name, s in request.secrets.items() + }, + } + response = await self._client.post(f"session/{session_id}/egress/", json=payload) + response.raise_for_status() + return EgressConfigResponse.model_validate(response.json()) + async def session_exists(self, session_id: str) -> bool: """Return True if the session container exists on the sandbox. diff --git a/tests/unit_tests/core/sandbox/test_client.py b/tests/unit_tests/core/sandbox/test_client.py index 5a6ac03d..aa184993 100644 --- a/tests/unit_tests/core/sandbox/test_client.py +++ b/tests/unit_tests/core/sandbox/test_client.py @@ -358,3 +358,28 @@ def test_is_transient_sandbox_error_true_for_request_error_with_no_response(): from core.sandbox.client import is_transient_sandbox_error assert is_transient_sandbox_error(httpx.ConnectError("refused")) is True + + +async def test_configure_egress_posts_plaintext_secret_and_uppercased_methods(fake_settings, mock_post): + from core.sandbox.client import DAIVSandboxClient + from core.sandbox.schemas import EgressConfigRequest, EgressPolicy, EgressRule, EgressSecret + + mock_post["json_body"] = {"ok": True} + req = EgressConfigRequest( + policy=EgressPolicy( + default="deny", + intercept="credentialed", + rules=[EgressRule(host="*.github.com", methods=["get"], inject="gh")], + ), + secrets={"gh": EgressSecret(header="Authorization", value=SecretStr("Bearer t"))}, + ) + + async with DAIVSandboxClient() as client: + resp = await client.configure_egress("sid", req) + + assert mock_post["url"] == "session/sid/egress/" + body = mock_post["kwargs"]["json"] + assert body["policy"]["default"] == "deny" + assert body["policy"]["rules"][0]["methods"] == ["GET"] # validator uppercased + assert body["secrets"]["gh"] == {"header": "Authorization", "value": "Bearer t"} # plaintext on wire + assert resp.ok is True From ae97345d3d2a1cbfdda6528c2de8771f5aa0627c Mon Sep 17 00:00:00 2001 From: Sandro Date: Tue, 23 Jun 2026 17:07:51 +0100 Subject: [PATCH 03/10] feat(sandbox-envs): store per-env egress policy + encrypted secrets --- ...ment__egress_secrets_encrypted_and_more.py | 20 +++++++ daiv/sandbox_envs/models.py | 57 ++++++++++++++++++- tests/unit_tests/sandbox_envs/test_models.py | 48 ++++++++++++++++ 3 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 daiv/sandbox_envs/migrations/0005_sandboxenvironment__egress_secrets_encrypted_and_more.py diff --git a/daiv/sandbox_envs/migrations/0005_sandboxenvironment__egress_secrets_encrypted_and_more.py b/daiv/sandbox_envs/migrations/0005_sandboxenvironment__egress_secrets_encrypted_and_more.py new file mode 100644 index 00000000..c194201b --- /dev/null +++ b/daiv/sandbox_envs/migrations/0005_sandboxenvironment__egress_secrets_encrypted_and_more.py @@ -0,0 +1,20 @@ +# Generated by Django 6.0.6 on 2026-06-23 16:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [("sandbox_envs", "0004_repo_ids")] + + operations = [ + migrations.AddField( + model_name="sandboxenvironment", + name="_egress_secrets_encrypted", + field=models.TextField(blank=True, editable=False, null=True), + ), + migrations.AddField( + model_name="sandboxenvironment", + name="egress_policy", + field=models.JSONField(blank=True, default=None, null=True, verbose_name="egress policy"), + ), + ] diff --git a/daiv/sandbox_envs/models.py b/daiv/sandbox_envs/models.py index a9d282e7..74800d2e 100644 --- a/daiv/sandbox_envs/models.py +++ b/daiv/sandbox_envs/models.py @@ -20,6 +20,9 @@ _REPO_ID_RE = re.compile(r"^[^\s/]+(/[^\s/]+)+$") ENV_VARS_MAX_ENTRIES = 100 ENV_VARS_MAX_ENCRYPTED_SIZE = 32 * 1024 # 32 KiB +EGRESS_MAX_RULES = 100 +EGRESS_MAX_SECRETS = 100 +EGRESS_SECRETS_MAX_ENCRYPTED_SIZE = 32 * 1024 # 32 KiB _UNSET = object() @@ -101,6 +104,10 @@ class SandboxEnvironment(TimeStampedModel): is_default = models.BooleanField(_("is default"), default=False) + egress_policy = models.JSONField(_("egress policy"), null=True, blank=True, default=None) + _egress_secrets_encrypted = models.TextField(blank=True, null=True, editable=False) # noqa: DJ001 — NULL = no secrets + egress_secrets = EncryptedJSONFieldDescriptor("egress_secrets") + objects = SandboxEnvironmentQuerySet.as_manager() class Meta: @@ -128,11 +135,14 @@ class Meta: ] def __init__(self, *args, **kwargs) -> None: - # Route ``env_vars`` through the descriptor so ``Manager.create(env_vars=...)`` works. + # Route ``env_vars``/``egress_secrets`` through their descriptors so ``Manager.create(...)`` works. env_vars_value = kwargs.pop("env_vars", _UNSET) + egress_secrets_value = kwargs.pop("egress_secrets", _UNSET) super().__init__(*args, **kwargs) if env_vars_value is not _UNSET: self.env_vars = env_vars_value + if egress_secrets_value is not _UNSET: + self.egress_secrets = egress_secrets_value def __str__(self) -> str: return f"{self.get_scope_display()}: {self.name}" @@ -178,6 +188,11 @@ def is_global_default(self) -> bool: """ return self.scope == Scope.GLOBAL and bool(self.is_default) + @property + def has_egress(self) -> bool: + """True iff this env carries an egress policy (``egress_policy is not None``).""" + return self.egress_policy is not None + def can_delete(self) -> tuple[bool, str | None]: """Row-level invariant gate for delete. ``promote_as_default`` must run first if this is the global default.""" @@ -196,6 +211,7 @@ def clean(self) -> None: self.base_image = self.base_image.strip() self._validate_env_vars() self._validate_repo_ids() + self._validate_egress() def _validate_env_vars(self) -> None: from core.encryption import DecryptionError @@ -271,6 +287,45 @@ def _validate_repo_ids(self) -> None: % {"repos": ", ".join(overlap), "name": other.name} }) + def _validate_egress(self) -> None: + from pydantic import ValidationError as PydanticValidationError + + from core.encryption import DecryptionError + from core.sandbox.schemas import EgressConfigRequest, EgressPolicy, EgressSecret + + if self.egress_policy is None: + return # no egress configured + if not isinstance(self.egress_policy, dict): + raise ValidationError({"egress_policy": _("Egress policy must be an object.")}) + + raw = self._egress_secrets_encrypted or "" + if len(raw) > EGRESS_SECRETS_MAX_ENCRYPTED_SIZE: + raise ValidationError({"egress_secrets": _("Egress secrets exceed 32 KiB encrypted.")}) + try: + secrets_raw = self.egress_secrets or {} + except DecryptionError as err: + raise ValidationError({ + NON_FIELD_ERRORS: _( + "Existing egress secrets could not be decrypted. Re-enter all secret values, " + "or restore DAIV_ENCRYPTION_KEY." + ) + }) from err + if not isinstance(secrets_raw, dict): + raise ValidationError({"egress_secrets": _("Egress secrets must be an object.")}) + if len(self.egress_policy.get("rules") or []) > EGRESS_MAX_RULES: + raise ValidationError({"egress_policy": _("Too many egress rules (max %d).") % EGRESS_MAX_RULES}) + if len(secrets_raw) > EGRESS_MAX_SECRETS: + raise ValidationError({"egress_secrets": _("Too many egress secrets (max %d).") % EGRESS_MAX_SECRETS}) + + # Single source of truth for shape + the inject-resolves invariant: build the wire request. + try: + EgressConfigRequest( + policy=EgressPolicy.model_validate(self.egress_policy), + secrets={name: EgressSecret(**s) for name, s in secrets_raw.items()}, + ) + except (PydanticValidationError, TypeError, ValueError) as err: + raise ValidationError({"egress_policy": _("Invalid egress configuration: %s") % err}) from err + def promote_as_default(self) -> None: """Atomically demote any other GLOBAL default and mark this env as default. diff --git a/tests/unit_tests/sandbox_envs/test_models.py b/tests/unit_tests/sandbox_envs/test_models.py index f4390842..72ccc548 100644 --- a/tests/unit_tests/sandbox_envs/test_models.py +++ b/tests/unit_tests/sandbox_envs/test_models.py @@ -402,3 +402,51 @@ def test_can_delete_allows_user_env(user): def test_can_delete_allows_non_default_global(_clear_global_envs): env = SandboxEnvironment.objects.create(scope=Scope.GLOBAL, name="g", base_image="g", is_default=False) assert env.can_delete() == (True, None) + + +def _egress_env(**over): + """Unsaved GLOBAL env carrying an egress config, for clean()/validator unit tests.""" + base = { + "scope": Scope.GLOBAL, + "name": "egress", + "base_image": "python:3.12", + "egress_policy": { + "default": "deny", + "intercept": "credentialed", + "rules": [{"host": "*.github.com", "methods": ["GET"], "inject": "gh"}], + }, + "egress_secrets": {"gh": {"header": "Authorization", "value": "Bearer t"}}, + } + base.update(over) + return SandboxEnvironment(**base) + + +def test_has_egress_predicate(): + assert _egress_env().has_egress is True + assert _egress_env(egress_policy=None).has_egress is False + + +def test_validate_egress_accepts_valid_config(): + _egress_env()._validate_egress() # must not raise + + +def test_validate_egress_rejects_dangling_inject(): + from django.core.exceptions import ValidationError + + env = _egress_env(egress_secrets={}) # rule injects "gh" but no such secret + with pytest.raises(ValidationError): + env._validate_egress() + + +def test_validate_egress_rejects_bad_default(): + from django.core.exceptions import ValidationError + + env = _egress_env(egress_policy={"default": "sometimes", "intercept": "all", "rules": []}) + with pytest.raises(ValidationError): + env._validate_egress() + + +def test_egress_secrets_encrypted_at_rest(): + env = _egress_env() + assert env._egress_secrets_encrypted # ciphertext populated by the descriptor + assert "Bearer t" not in env._egress_secrets_encrypted From 27e506caf22981959faa57baefcb539d1aa55c55 Mon Sep 17 00:00:00 2001 From: Sandro Date: Tue, 23 Jun 2026 17:12:56 +0100 Subject: [PATCH 04/10] feat(sandbox-envs): carry typed egress config through the runtime merge --- daiv/codebase/context.py | 2 + daiv/sandbox_envs/services.py | 27 ++++++ .../unit_tests/sandbox_envs/test_services.py | 91 +++++++++++++++++++ 3 files changed, 120 insertions(+) diff --git a/daiv/codebase/context.py b/daiv/codebase/context.py index 96ee169d..eb224bc4 100644 --- a/daiv/codebase/context.py +++ b/daiv/codebase/context.py @@ -12,6 +12,7 @@ from codebase.repo_config import RepositoryConfig # noqa: TC001 from core.sandbox.client import DAIVSandboxClient, reset_run_sandbox_client, set_run_sandbox_client from core.sandbox.command_policy import SandboxCommandPolicy # noqa: TC001 +from core.sandbox.schemas import EgressConfigRequest # noqa: TC001 if TYPE_CHECKING: from collections.abc import AsyncIterator @@ -38,6 +39,7 @@ class SandboxRuntime: cpus: float | None env_vars: dict[str, str] command_policy: SandboxCommandPolicy + egress: EgressConfigRequest | None = None @property def enabled(self) -> bool: diff --git a/daiv/sandbox_envs/services.py b/daiv/sandbox_envs/services.py index 832fde21..11e22c97 100644 --- a/daiv/sandbox_envs/services.py +++ b/daiv/sandbox_envs/services.py @@ -14,6 +14,7 @@ from activity.services import RepoTarget from codebase.context import SandboxRuntime + from core.sandbox.schemas import EgressConfigRequest logger = logging.getLogger("daiv.sandbox_envs") @@ -29,10 +30,14 @@ class SandboxEnvOverride: memory_bytes: int | None cpus: float | None env_vars: dict[str, str] + egress: EgressConfigRequest | None = None def row_to_override(env: SandboxEnvironment) -> SandboxEnvOverride: + from pydantic import ValidationError as PydanticValidationError + from core.encryption import DecryptionError + from core.sandbox.schemas import EgressConfigRequest, EgressPolicy, EgressSecret try: env_vars_rows = env.env_vars or [] @@ -41,6 +46,20 @@ def row_to_override(env: SandboxEnvironment) -> SandboxEnvOverride: # keep going. The descriptor already logs at exception level. logger.error("env_vars decryption failed for SandboxEnvironment id=%s; dropping env_vars", env.id) env_vars_rows = [] + + egress = None + if env.egress_policy is not None: + try: + secrets_raw = env.egress_secrets or {} + egress = EgressConfigRequest( + policy=EgressPolicy.model_validate(env.egress_policy), + secrets={name: EgressSecret(**s) for name, s in secrets_raw.items()}, + ) + except DecryptionError, PydanticValidationError, TypeError, ValueError: + # A half/invalid egress config must never reach the sidecar; drop it and log loudly. + logger.error("egress config unusable for SandboxEnvironment id=%s; dropping egress", env.id) + egress = None + return SandboxEnvOverride( base_image=env.base_image or None, network_enabled=env.network_enabled, @@ -51,6 +70,7 @@ def row_to_override(env: SandboxEnvironment) -> SandboxEnvOverride: for entry in env_vars_rows if entry.get("name") and entry.get("value") is not None }, + egress=egress, ) @@ -228,6 +248,12 @@ def pick(field: str, runtime_default): return v return runtime_default + egress = None + if per_run is not None and per_run.egress is not None: + egress = per_run.egress + elif global_default is not None and global_default.egress is not None: + egress = global_default.egress + return SandboxRuntime( base_image=pick("base_image", None), network_enabled=pick("network_enabled", False), @@ -235,6 +261,7 @@ def pick(field: str, runtime_default): cpus=pick("cpus", None), env_vars={**(global_default.env_vars if global_default else {}), **(per_run.env_vars if per_run else {})}, command_policy=SandboxCommandPolicy(), + egress=egress, ) diff --git a/tests/unit_tests/sandbox_envs/test_services.py b/tests/unit_tests/sandbox_envs/test_services.py index 7e91e6e5..75988f5e 100644 --- a/tests/unit_tests/sandbox_envs/test_services.py +++ b/tests/unit_tests/sandbox_envs/test_services.py @@ -489,3 +489,94 @@ def test_build_env_trigger_updated_uses_action_key(global_env): def test_build_env_trigger_deleted_uses_action_key(global_env): assert "env-deleted" in build_env_trigger(global_env, "deleted") + + +def _egress_request(host: str): + from pydantic import SecretStr + + from core.sandbox.schemas import EgressConfigRequest, EgressPolicy, EgressRule, EgressSecret + + return EgressConfigRequest( + policy=EgressPolicy(rules=[EgressRule(host=host, inject="t")]), + secrets={"t": EgressSecret(header="Authorization", value=SecretStr("Bearer x"))}, + ) + + +@pytest.mark.asyncio +@pytest.mark.django_db(transaction=True) +async def test_row_to_override_builds_egress(): + from sandbox_envs.services import row_to_override + + env = await SandboxEnvironment.objects.acreate( + scope=Scope.GLOBAL, + name="eg", + base_image="python:3.12", + egress_policy={ + "default": "deny", + "intercept": "credentialed", + "rules": [{"host": "*.github.com", "methods": ["GET"], "inject": "gh"}], + }, + egress_secrets={"gh": {"header": "Authorization", "value": "Bearer t"}}, + ) + override = row_to_override(env) + assert override.egress is not None + assert override.egress.policy.rules[0].host == "*.github.com" + assert override.egress.secrets["gh"].value.get_secret_value() == "Bearer t" + + +@pytest.mark.asyncio +@pytest.mark.django_db(transaction=True) +async def test_row_to_override_drops_egress_when_malformed(): + from sandbox_envs.services import row_to_override + + # Dangling inject (no matching secret) — stored directly, bypassing clean(). + env = await SandboxEnvironment.objects.acreate( + scope=Scope.GLOBAL, + name="bad", + base_image="python:3.12", + egress_policy={"default": "deny", "rules": [{"host": "x", "inject": "missing"}]}, + egress_secrets={}, + ) + override = row_to_override(env) + assert override.egress is None + + +def test_merge_prefers_per_run_egress(): + from sandbox_envs.services import SandboxEnvOverride, merge_sandbox_runtime + + per_run = SandboxEnvOverride( + base_image=None, + network_enabled=None, + memory_bytes=None, + cpus=None, + env_vars={}, + egress=_egress_request("per-run.example"), + ) + global_default = SandboxEnvOverride( + base_image=None, + network_enabled=None, + memory_bytes=None, + cpus=None, + env_vars={}, + egress=_egress_request("global.example"), + ) + rt = merge_sandbox_runtime(per_run=per_run, global_default=global_default) + assert rt.egress.policy.rules[0].host == "per-run.example" + + +def test_merge_falls_back_to_global_egress(): + from sandbox_envs.services import SandboxEnvOverride, merge_sandbox_runtime + + per_run = SandboxEnvOverride( + base_image=None, network_enabled=None, memory_bytes=None, cpus=None, env_vars={}, egress=None + ) + global_default = SandboxEnvOverride( + base_image=None, + network_enabled=None, + memory_bytes=None, + cpus=None, + env_vars={}, + egress=_egress_request("global.example"), + ) + rt = merge_sandbox_runtime(per_run=per_run, global_default=global_default) + assert rt.egress.policy.rules[0].host == "global.example" From 4b2bae5882d0fd9c2426155ebbe424a782ed2730 Mon Sep 17 00:00:00 2001 From: Sandro Date: Tue, 23 Jun 2026 17:21:11 +0100 Subject: [PATCH 05/10] feat(sandbox): provision egress policy at session start, fail-closed --- daiv/automation/agent/middlewares/sandbox.py | 30 +++++++ .../agent/middlewares/test_sandbox.py | 83 +++++++++++++++++++ 2 files changed, 113 insertions(+) diff --git a/daiv/automation/agent/middlewares/sandbox.py b/daiv/automation/agent/middlewares/sandbox.py index 2f0671ef..c9a8c30b 100644 --- a/daiv/automation/agent/middlewares/sandbox.py +++ b/daiv/automation/agent/middlewares/sandbox.py @@ -291,6 +291,34 @@ def _make_global_skills_archive() -> bytes | None: return buf.getvalue() +class SandboxEgressUnavailableError(RuntimeError): + """Raised when a session's resolved egress policy cannot be provisioned to the sandbox sidecar + (the sandbox returned 404 'egress not enabled' or 409 'session has no proxy'). Fail-closed: the + run must not proceed with whatever raw network the sandbox would otherwise grant.""" + + +async def _provision_egress(client, session_id: str, sb) -> None: + """Provision the sidecar egress policy for a network-enabled session that requires it. + + No-op when the resolved env defines no egress policy or network is off. A 404/409 is converted to + ``SandboxEgressUnavailableError`` (fail-closed); other HTTP errors propagate unchanged for the + existing transient-retry handling. + """ + if sb.egress is None or not sb.network_enabled: + return + try: + await client.configure_egress(session_id, sb.egress) + except httpx.HTTPStatusError as exc: + status = exc.response.status_code + if status in (404, 409): + raise SandboxEgressUnavailableError( + f"The resolved sandbox environment requires the egress proxy, but the sandbox returned " + f"{status} for egress provisioning. Enable DAIV_SANDBOX_EGRESS_PROXY_ENABLED on the " + f"sandbox (and ensure the session is network-enabled)." + ) from exc + raise + + class BashFailure(Enum): """Why a bash invocation produced no result, mapped to the guidance the agent gets. @@ -518,6 +546,7 @@ async def abefore_agent(self, state: StateT, runtime: Runtime[RuntimeCtx]) -> di prior_session_id = state.get("session_id") if prior_session_id and await self._session_exists(client, prior_session_id): self._bind_session(prior_session_id) + await _provision_egress(client, prior_session_id, runtime.context.sandbox) logger.info("Reusing warm sandbox session %s", prior_session_id) return {"session_id": prior_session_id} @@ -537,6 +566,7 @@ async def abefore_agent(self, state: StateT, runtime: Runtime[RuntimeCtx]) -> di asyncio.to_thread(_make_repo_archive, str(working_dir)), asyncio.to_thread(_make_global_skills_archive) ) await client.seed_session(session_id, repo_archive=repo_archive, skills_archive=skills_archive) + await _provision_egress(client, session_id, sb) except Exception: logger.exception("Failed to build or seed sandbox session %s", session_id) try: diff --git a/tests/unit_tests/automation/agent/middlewares/test_sandbox.py b/tests/unit_tests/automation/agent/middlewares/test_sandbox.py index 4fd96011..ed3c0829 100644 --- a/tests/unit_tests/automation/agent/middlewares/test_sandbox.py +++ b/tests/unit_tests/automation/agent/middlewares/test_sandbox.py @@ -93,6 +93,7 @@ def _make_runtime() -> MagicMock: sb.memory_bytes = 1 sb.cpus = 1 sb.env_vars = None + sb.egress = None runtime.context.gitrepo.working_dir = "/tmp/repo" # noqa: S108 return runtime @@ -761,3 +762,85 @@ async def test_soft_fails_to_false_on_transport_error(self): client = Mock(session_exists=AsyncMock(side_effect=httpx.HTTPError("boom"))) assert await SandboxMiddleware._session_exists(client, "warm-1") is False + + +class TestSandboxEgress: + def _runtime_with_egress(self): + from core.sandbox.schemas import EgressConfigRequest, EgressPolicy + + runtime = _make_runtime() + runtime.context.sandbox.network_enabled = True + runtime.context.sandbox.egress = EgressConfigRequest(policy=EgressPolicy(default="allow")) + return runtime, runtime.context.sandbox.egress + + async def test_provisions_egress_on_fresh_create(self): + client = MagicMock() + client.start_session = AsyncMock(return_value="sess-e") + client.seed_session = AsyncMock() + client.configure_egress = AsyncMock() + mw = SandboxMiddleware( + agent_root="/workspace/repo", client=client, sandbox_backend=SandboxFileBackend(client=client) + ) + runtime, egress = self._runtime_with_egress() + with ( + patch("automation.agent.middlewares.sandbox._make_repo_archive", return_value=b""), + patch("automation.agent.middlewares.sandbox._make_global_skills_archive", return_value=None), + ): + await mw.abefore_agent({}, runtime) + client.configure_egress.assert_awaited_once_with("sess-e", egress) + + async def test_provisions_egress_on_warm_reuse(self): + client = MagicMock() + client.session_exists = AsyncMock(return_value=True) + client.start_session = AsyncMock() + client.configure_egress = AsyncMock() + mw = SandboxMiddleware( + agent_root="/workspace/repo", client=client, sandbox_backend=SandboxFileBackend(client=client) + ) + runtime, egress = self._runtime_with_egress() + result = await mw.abefore_agent({"session_id": "sess-warm"}, runtime) + assert result == {"session_id": "sess-warm"} + client.start_session.assert_not_awaited() + client.configure_egress.assert_awaited_once_with("sess-warm", egress) + + async def test_skips_egress_when_none(self): + client = MagicMock() + client.start_session = AsyncMock(return_value="sess-x") + client.seed_session = AsyncMock() + client.configure_egress = AsyncMock() + mw = SandboxMiddleware( + agent_root="/workspace/repo", client=client, sandbox_backend=SandboxFileBackend(client=client) + ) + runtime = _make_runtime() # network_enabled False, egress None + with ( + patch("automation.agent.middlewares.sandbox._make_repo_archive", return_value=b""), + patch("automation.agent.middlewares.sandbox._make_global_skills_archive", return_value=None), + ): + await mw.abefore_agent({}, runtime) + client.configure_egress.assert_not_awaited() + + @pytest.mark.parametrize("status", [404, 409]) + async def test_egress_failure_is_fail_closed_and_closes_session(self, status: int): + import httpx + + from automation.agent.middlewares.sandbox import SandboxEgressUnavailableError + + err = httpx.HTTPStatusError("nope", request=httpx.Request("POST", "x"), response=httpx.Response(status)) + client = MagicMock() + client.start_session = AsyncMock(return_value="sess-fc") + client.seed_session = AsyncMock() + client.configure_egress = AsyncMock(side_effect=err) + client.close_session = AsyncMock() + client.close = AsyncMock() + mw = SandboxMiddleware( + agent_root="/workspace/repo", client=client, sandbox_backend=SandboxFileBackend(client=client) + ) + runtime, _ = self._runtime_with_egress() + with ( + patch("automation.agent.middlewares.sandbox._make_repo_archive", return_value=b""), + patch("automation.agent.middlewares.sandbox._make_global_skills_archive", return_value=None), + pytest.raises(SandboxEgressUnavailableError), + ): + await mw.abefore_agent({}, runtime) + client.close_session.assert_awaited_once_with("sess-fc", force=True) + client.close.assert_not_awaited() # the run owns the transport From 4a55517ef1e3ef94daa41b497fe0b4196c00e9c7 Mon Sep 17 00:00:00 2001 From: Sandro Date: Tue, 23 Jun 2026 17:25:08 +0100 Subject: [PATCH 06/10] docs(sandbox-envs): document per-env egress policy and deny-all caveat --- CHANGELOG.md | 1 + daiv/sandbox_envs/models.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 769b597b..808a7834 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added per-environment network egress policy for sandbox environments. A `SandboxEnvironment` can now carry an `egress_policy` (default/intercept + per-host rules) and encrypted `egress_secrets` (named header credentials); when set, the agent provisions them onto the daiv-sandbox MITM egress proxy at session start so credentials are injected by the sidecar and never enter the sandbox container. Provisioning is fail-closed: if the configured sandbox does not have the egress proxy enabled (`DAIV_SANDBOX_EGRESS_PROXY_ENABLED`), a run using an egress-configured environment aborts rather than falling back to unrestricted network. Note: once the sandbox egress proxy is enabled, network-enabled environments **without** an egress policy receive the sidecar's default deny-all (no connectivity) — configure an egress policy on those environments. Configurable via Django admin / API for now (form UI to follow). - Added `release_orphan_queued_threads` management command to recover `QUEUED` activities left behind on threads with no active sibling (rare TOCTOU loss). - Added per-domain auth headers for the `web_fetch` tool to the configuration UI under **Web Fetch → Per-domain auth headers**. Each row pairs a domain (exact match) with an HTTP header name and a header value (encrypted at rest). Replaces the previous env-only `AUTOMATION_WEB_FETCH_AUTH_HEADERS` setting; the new env override is `DAIV_WEB_FETCH_AUTH_HEADERS` (same JSON shape). Operators using the old name must rename it. - Added a "Start a run" page at `/dashboard/runs/new/` for launching new agent runs from the UI, and a "Retry" button on terminal non-webhook activities that pre-fills the form with the original prompt, repository, ref, and max-mode flag. diff --git a/daiv/sandbox_envs/models.py b/daiv/sandbox_envs/models.py index 74800d2e..324a438b 100644 --- a/daiv/sandbox_envs/models.py +++ b/daiv/sandbox_envs/models.py @@ -104,6 +104,10 @@ class SandboxEnvironment(TimeStampedModel): is_default = models.BooleanField(_("is default"), default=False) + # Per-env egress policy provisioned to the daiv-sandbox sidecar proxy when network is enabled. + # ``None`` = no egress configured (legacy behavior: raw network when the sandbox proxy is off). + # NOTE: when the sandbox has EGRESS_PROXY_ENABLED, a network-enabled env with egress_policy=None + # gets the sidecar's deny-all default — configure a policy to grant connectivity. egress_policy = models.JSONField(_("egress policy"), null=True, blank=True, default=None) _egress_secrets_encrypted = models.TextField(blank=True, null=True, editable=False) # noqa: DJ001 — NULL = no secrets egress_secrets = EncryptedJSONFieldDescriptor("egress_secrets") From aee24162d9911dc1cf8c482071eec4a554c0854d Mon Sep 17 00:00:00 2001 From: Sandro Date: Tue, 23 Jun 2026 22:03:19 +0100 Subject: [PATCH 07/10] fix(sandbox): harden egress fail-closed semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only HTTP 404 (egress proxy disabled — permanent) converts to SandboxEgressUnavailableError. 409 (ambiguous "session busy" vs "no proxy", already classified as transient) and 5xx now propagate unchanged instead of being mislabeled as a permanent "enable the proxy" diagnosis; they still fail closed via the existing cleanup. Re-provision egress on warm session reuse so the guarantee holds on resumed turns too, without force-closing the resumable container on a possibly transient blip. When stored egress config is unusable (rotated DAIV_ENCRYPTION_KEY, hand-edited row), fail closed to an empty deny-all policy rather than dropping to None — None would let a network-enabled session fall back to raw network or the sidecar default. Extract EgressConfigRequest.from_stored() as the single source of truth for mapping persisted policy/secrets onto the wire schema, and simplify egress precedence in merge_sandbox_runtime via the generic pick() helper. --- CHANGELOG.md | 2 +- daiv/automation/agent/middlewares/sandbox.py | 65 ++++++------ daiv/core/sandbox/client.py | 6 +- daiv/core/sandbox/schemas.py | 12 +++ daiv/sandbox_envs/models.py | 9 +- daiv/sandbox_envs/services.py | 30 +++--- .../agent/middlewares/test_sandbox.py | 99 ++++++++++++------- tests/unit_tests/sandbox_envs/test_models.py | 22 +++++ .../unit_tests/sandbox_envs/test_services.py | 92 ++++++++++------- 9 files changed, 218 insertions(+), 119 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 808a7834..ef739b9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added per-environment network egress policy for sandbox environments. A `SandboxEnvironment` can now carry an `egress_policy` (default/intercept + per-host rules) and encrypted `egress_secrets` (named header credentials); when set, the agent provisions them onto the daiv-sandbox MITM egress proxy at session start so credentials are injected by the sidecar and never enter the sandbox container. Provisioning is fail-closed: if the configured sandbox does not have the egress proxy enabled (`DAIV_SANDBOX_EGRESS_PROXY_ENABLED`), a run using an egress-configured environment aborts rather than falling back to unrestricted network. Note: once the sandbox egress proxy is enabled, network-enabled environments **without** an egress policy receive the sidecar's default deny-all (no connectivity) — configure an egress policy on those environments. Configurable via Django admin / API for now (form UI to follow). +- Added per-environment network egress policy for sandbox environments. A `SandboxEnvironment` can now carry an `egress_policy` (default/intercept + per-host rules) and encrypted `egress_secrets` (named header credentials); when set, the agent provisions them onto the daiv-sandbox MITM egress proxy at session start so credentials are injected by the sidecar and never enter the sandbox container. Provisioning is fail-closed: if the configured sandbox does not have the egress proxy enabled (`DAIV_SANDBOX_EGRESS_PROXY_ENABLED`), a run using an egress-configured environment aborts rather than falling back to unrestricted network. Note: once the sandbox egress proxy is enabled, network-enabled environments **without** an egress policy receive the sidecar's default deny-all (no connectivity) — configure an egress policy on those environments. Set programmatically via the ORM for now (admin/API/form UI to follow). - Added `release_orphan_queued_threads` management command to recover `QUEUED` activities left behind on threads with no active sibling (rare TOCTOU loss). - Added per-domain auth headers for the `web_fetch` tool to the configuration UI under **Web Fetch → Per-domain auth headers**. Each row pairs a domain (exact match) with an HTTP header name and a header value (encrypted at rest). Replaces the previous env-only `AUTOMATION_WEB_FETCH_AUTH_HEADERS` setting; the new env override is `DAIV_WEB_FETCH_AUTH_HEADERS` (same JSON shape). Operators using the old name must rename it. - Added a "Start a run" page at `/dashboard/runs/new/` for launching new agent runs from the UI, and a "Retry" button on terminal non-webhook activities that pre-fills the form with the original prompt, repository, ref, and max-mode flag. diff --git a/daiv/automation/agent/middlewares/sandbox.py b/daiv/automation/agent/middlewares/sandbox.py index c9a8c30b..7b32ffc8 100644 --- a/daiv/automation/agent/middlewares/sandbox.py +++ b/daiv/automation/agent/middlewares/sandbox.py @@ -20,7 +20,7 @@ from automation.agent.conf import settings as agent_settings from automation.agent.constants import BUILTIN_SKILLS_PATH from automation.agent.middlewares.file_system import SandboxFileBackend # noqa: TC001 -from codebase.context import RuntimeCtx # noqa: TC001 +from codebase.context import RuntimeCtx, SandboxRuntime # noqa: TC001 from core.conf import settings from core.sandbox.client import DAIVSandboxClient, is_transient_sandbox_error from core.sandbox.command_parser import CommandParseError, parse_command @@ -292,31 +292,9 @@ def _make_global_skills_archive() -> bytes | None: class SandboxEgressUnavailableError(RuntimeError): - """Raised when a session's resolved egress policy cannot be provisioned to the sandbox sidecar - (the sandbox returned 404 'egress not enabled' or 409 'session has no proxy'). Fail-closed: the - run must not proceed with whatever raw network the sandbox would otherwise grant.""" - - -async def _provision_egress(client, session_id: str, sb) -> None: - """Provision the sidecar egress policy for a network-enabled session that requires it. - - No-op when the resolved env defines no egress policy or network is off. A 404/409 is converted to - ``SandboxEgressUnavailableError`` (fail-closed); other HTTP errors propagate unchanged for the - existing transient-retry handling. - """ - if sb.egress is None or not sb.network_enabled: - return - try: - await client.configure_egress(session_id, sb.egress) - except httpx.HTTPStatusError as exc: - status = exc.response.status_code - if status in (404, 409): - raise SandboxEgressUnavailableError( - f"The resolved sandbox environment requires the egress proxy, but the sandbox returned " - f"{status} for egress provisioning. Enable DAIV_SANDBOX_EGRESS_PROXY_ENABLED on the " - f"sandbox (and ensure the session is network-enabled)." - ) from exc - raise + """Raised when a session's resolved egress policy cannot be provisioned because the sandbox has + the egress proxy disabled (HTTP 404 'egress proxy not enabled'). Fail-closed: the run must not + proceed with whatever raw network the sandbox would otherwise grant.""" class BashFailure(Enum): @@ -523,6 +501,31 @@ async def _session_exists(client: DAIVSandboxClient, session_id: str) -> bool: ) return False + @staticmethod + async def _provision_egress(client: DAIVSandboxClient, session_id: str, sb: SandboxRuntime | None) -> None: + """Provision the sidecar egress policy for a network-enabled session that requires it. + + No-op when there is no resolved sandbox runtime, the env defines no egress policy, or network + is off. Only a **404** (egress + proxy disabled on the sandbox — unambiguous and permanent) is converted to the actionable + ``SandboxEgressUnavailableError``. Every other status propagates unchanged, including a 409 — on + the egress endpoint 409 is ambiguous ("Session is busy" transient lock contention, or "Session + has no egress proxy") and the project already classifies 409 as transient + (``TRANSIENT_SANDBOX_STATUS``), so it must not be mislabeled as a permanent "enable the proxy" + diagnosis. A propagated error still fails closed: the caller's setup path aborts the run. + """ + if sb is None or sb.egress is None or not sb.network_enabled: + return + try: + await client.configure_egress(session_id, sb.egress) + except httpx.HTTPStatusError as exc: + if exc.response.status_code == 404: + raise SandboxEgressUnavailableError( + "The resolved sandbox environment requires the egress proxy, but the sandbox returned " + "404 for egress provisioning. Enable DAIV_SANDBOX_EGRESS_PROXY_ENABLED on the sandbox." + ) from exc + raise + async def abefore_agent(self, state: StateT, runtime: Runtime[RuntimeCtx]) -> dict[str, str] | None: """ Bind a sandbox session — reusing the prior turn's warm session when possible. @@ -546,7 +549,13 @@ async def abefore_agent(self, state: StateT, runtime: Runtime[RuntimeCtx]) -> di prior_session_id = state.get("session_id") if prior_session_id and await self._session_exists(client, prior_session_id): self._bind_session(prior_session_id) - await _provision_egress(client, prior_session_id, runtime.context.sandbox) + # Re-provision egress on every warm reuse so the fail-closed guarantee holds on resumed + # turns too. A failure here propagates and aborts the run (fail-closed). Unlike the + # fresh-create path below we deliberately do NOT force-close the container: it's a healthy + # resumable session, the failure may be transient (e.g. a 409 "Session is busy"), and the + # next turn re-runs this check (or the reaper reclaims it). Force-closing here would throw + # away a good warm container on a transient blip. + await self._provision_egress(client, prior_session_id, runtime.context.sandbox) logger.info("Reusing warm sandbox session %s", prior_session_id) return {"session_id": prior_session_id} @@ -566,7 +575,7 @@ async def abefore_agent(self, state: StateT, runtime: Runtime[RuntimeCtx]) -> di asyncio.to_thread(_make_repo_archive, str(working_dir)), asyncio.to_thread(_make_global_skills_archive) ) await client.seed_session(session_id, repo_archive=repo_archive, skills_archive=skills_archive) - await _provision_egress(client, session_id, sb) + await self._provision_egress(client, session_id, sb) except Exception: logger.exception("Failed to build or seed sandbox session %s", session_id) try: diff --git a/daiv/core/sandbox/client.py b/daiv/core/sandbox/client.py index 21a31076..b8ee764d 100644 --- a/daiv/core/sandbox/client.py +++ b/daiv/core/sandbox/client.py @@ -234,8 +234,10 @@ async def configure_egress(self, session_id: str, request: EgressConfigRequest) Idempotent on the sandbox side. Secret values are sent in plaintext (the sidecar needs them); ``SecretStr`` keeps them redacted in logs/repr, so the payload is built explicitly here rather than via ``request.model_dump(mode="json")``, which would mask the values. Raises - ``httpx.HTTPStatusError`` on a non-2xx (404 = egress not enabled on the sandbox, 409 = the - session has no proxy); the caller decides how to react. + ``httpx.HTTPStatusError`` on a non-2xx; the caller decides how to react. Status meanings: + 404 = egress proxy disabled on the sandbox (permanent); 409 = either "Session is busy" + (transient lock contention — see ``TRANSIENT_SANDBOX_STATUS``) or "Session has no egress + proxy"; other 5xx are transient. """ payload = { "policy": request.policy.model_dump(mode="json"), diff --git a/daiv/core/sandbox/schemas.py b/daiv/core/sandbox/schemas.py index 0503839c..0340df05 100644 --- a/daiv/core/sandbox/schemas.py +++ b/daiv/core/sandbox/schemas.py @@ -272,6 +272,18 @@ def _injects_resolve(self) -> EgressConfigRequest: raise ValueError(f"rule for host {rule.host!r} references unknown secret {rule.inject!r}") return self + @classmethod + def from_stored(cls, policy: dict, secrets: dict) -> EgressConfigRequest: + """Build a wire request from a stored ``egress_policy`` dict + decrypted ``egress_secrets`` dict. + + Single source of truth for how persisted egress config maps onto the wire schemas (including + the inject-resolves invariant). Raises ``pydantic.ValidationError`` / ``TypeError`` / ``ValueError`` + on a malformed stored shape; callers decide how to react (reject on save vs. fail-closed at runtime). + """ + return cls( + policy=EgressPolicy.model_validate(policy), secrets={name: EgressSecret(**s) for name, s in secrets.items()} + ) + class EgressConfigResponse(BaseModel): ok: bool = Field(default=True, description="True when the policy was provisioned to the sidecar.") diff --git a/daiv/sandbox_envs/models.py b/daiv/sandbox_envs/models.py index 324a438b..664834dc 100644 --- a/daiv/sandbox_envs/models.py +++ b/daiv/sandbox_envs/models.py @@ -295,7 +295,7 @@ def _validate_egress(self) -> None: from pydantic import ValidationError as PydanticValidationError from core.encryption import DecryptionError - from core.sandbox.schemas import EgressConfigRequest, EgressPolicy, EgressSecret + from core.sandbox.schemas import EgressConfigRequest if self.egress_policy is None: return # no egress configured @@ -321,12 +321,9 @@ def _validate_egress(self) -> None: if len(secrets_raw) > EGRESS_MAX_SECRETS: raise ValidationError({"egress_secrets": _("Too many egress secrets (max %d).") % EGRESS_MAX_SECRETS}) - # Single source of truth for shape + the inject-resolves invariant: build the wire request. + # Authoritative shape + inject-resolves check (count/size caps handled above): build the wire request. try: - EgressConfigRequest( - policy=EgressPolicy.model_validate(self.egress_policy), - secrets={name: EgressSecret(**s) for name, s in secrets_raw.items()}, - ) + EgressConfigRequest.from_stored(self.egress_policy, secrets_raw) except (PydanticValidationError, TypeError, ValueError) as err: raise ValidationError({"egress_policy": _("Invalid egress configuration: %s") % err}) from err diff --git a/daiv/sandbox_envs/services.py b/daiv/sandbox_envs/services.py index 11e22c97..26bbbd2c 100644 --- a/daiv/sandbox_envs/services.py +++ b/daiv/sandbox_envs/services.py @@ -37,7 +37,7 @@ def row_to_override(env: SandboxEnvironment) -> SandboxEnvOverride: from pydantic import ValidationError as PydanticValidationError from core.encryption import DecryptionError - from core.sandbox.schemas import EgressConfigRequest, EgressPolicy, EgressSecret + from core.sandbox.schemas import EgressConfigRequest try: env_vars_rows = env.env_vars or [] @@ -50,15 +50,17 @@ def row_to_override(env: SandboxEnvironment) -> SandboxEnvOverride: egress = None if env.egress_policy is not None: try: - secrets_raw = env.egress_secrets or {} - egress = EgressConfigRequest( - policy=EgressPolicy.model_validate(env.egress_policy), - secrets={name: EgressSecret(**s) for name, s in secrets_raw.items()}, - ) + egress = EgressConfigRequest.from_stored(env.egress_policy, env.egress_secrets or {}) except DecryptionError, PydanticValidationError, TypeError, ValueError: - # A half/invalid egress config must never reach the sidecar; drop it and log loudly. - logger.error("egress config unusable for SandboxEnvironment id=%s; dropping egress", env.id) - egress = None + # The env *intended* restricted egress but its config is unusable (e.g. a rotated + # DAIV_ENCRYPTION_KEY left the secrets undecryptable, or the row was hand-edited). + # Fail closed: substitute an empty deny-all policy rather than dropping to None — None + # would let a network-enabled session fall back to raw network (proxy off) or rely on + # the sidecar's default (proxy on). Deny-all denies connectivity (proxy on) or trips the + # fail-closed abort in _provision_egress (proxy off). Never reach the sidecar with a + # half/invalid config. The descriptor already logs the decryption failure at exception level. + logger.error("egress config unusable for SandboxEnvironment id=%s; failing closed to deny-all", env.id) + egress = EgressConfigRequest() return SandboxEnvOverride( base_image=env.base_image or None, @@ -248,20 +250,16 @@ def pick(field: str, runtime_default): return v return runtime_default - egress = None - if per_run is not None and per_run.egress is not None: - egress = per_run.egress - elif global_default is not None and global_default.egress is not None: - egress = global_default.egress - return SandboxRuntime( base_image=pick("base_image", None), network_enabled=pick("network_enabled", False), memory_bytes=pick("memory_bytes", None), cpus=pick("cpus", None), + # Egress is opt-in; its "absent" value is None, so the generic precedence in pick() fits exactly + # (per-run wins, else global, else None). env_vars below is a union, which is why it can't. + egress=pick("egress", None), env_vars={**(global_default.env_vars if global_default else {}), **(per_run.env_vars if per_run else {})}, command_policy=SandboxCommandPolicy(), - egress=egress, ) diff --git a/tests/unit_tests/automation/agent/middlewares/test_sandbox.py b/tests/unit_tests/automation/agent/middlewares/test_sandbox.py index ed3c0829..e7b4377b 100644 --- a/tests/unit_tests/automation/agent/middlewares/test_sandbox.py +++ b/tests/unit_tests/automation/agent/middlewares/test_sandbox.py @@ -1,6 +1,7 @@ import io import json import tarfile +from contextlib import contextmanager from typing import TYPE_CHECKING from unittest.mock import AsyncMock, MagicMock, Mock, patch @@ -11,6 +12,7 @@ from automation.agent.middlewares.sandbox import ( SANDBOX_SYSTEM_PROMPT, BashFailure, + SandboxEgressUnavailableError, SandboxMiddleware, _run_bash_commands, ) @@ -765,6 +767,21 @@ async def test_soft_fails_to_false_on_transport_error(self): class TestSandboxEgress: + def _mw(self, client) -> SandboxMiddleware: + return SandboxMiddleware( + agent_root="/workspace/repo", client=client, sandbox_backend=SandboxFileBackend(client=client) + ) + + @staticmethod + @contextmanager + def _patch_archives(): + """Stub the archive builders so the fresh-create path doesn't touch the filesystem.""" + with ( + patch("automation.agent.middlewares.sandbox._make_repo_archive", return_value=b""), + patch("automation.agent.middlewares.sandbox._make_global_skills_archive", return_value=None), + ): + yield + def _runtime_with_egress(self): from core.sandbox.schemas import EgressConfigRequest, EgressPolicy @@ -778,15 +795,9 @@ async def test_provisions_egress_on_fresh_create(self): client.start_session = AsyncMock(return_value="sess-e") client.seed_session = AsyncMock() client.configure_egress = AsyncMock() - mw = SandboxMiddleware( - agent_root="/workspace/repo", client=client, sandbox_backend=SandboxFileBackend(client=client) - ) runtime, egress = self._runtime_with_egress() - with ( - patch("automation.agent.middlewares.sandbox._make_repo_archive", return_value=b""), - patch("automation.agent.middlewares.sandbox._make_global_skills_archive", return_value=None), - ): - await mw.abefore_agent({}, runtime) + with self._patch_archives(): + await self._mw(client).abefore_agent({}, runtime) client.configure_egress.assert_awaited_once_with("sess-e", egress) async def test_provisions_egress_on_warm_reuse(self): @@ -794,37 +805,66 @@ async def test_provisions_egress_on_warm_reuse(self): client.session_exists = AsyncMock(return_value=True) client.start_session = AsyncMock() client.configure_egress = AsyncMock() - mw = SandboxMiddleware( - agent_root="/workspace/repo", client=client, sandbox_backend=SandboxFileBackend(client=client) - ) runtime, egress = self._runtime_with_egress() - result = await mw.abefore_agent({"session_id": "sess-warm"}, runtime) + result = await self._mw(client).abefore_agent({"session_id": "sess-warm"}, runtime) assert result == {"session_id": "sess-warm"} client.start_session.assert_not_awaited() client.configure_egress.assert_awaited_once_with("sess-warm", egress) + async def test_egress_404_on_warm_reuse_is_fail_closed_and_preserves_session(self): + import httpx + + # The fail-closed guarantee must hold on every resumed turn, not just fresh create. On warm + # reuse the container is deliberately NOT force-closed on failure (it's resumable — preserved + # for the next turn's retry or the reaper), unlike the fresh-create path. + err = httpx.HTTPStatusError("nope", request=httpx.Request("POST", "x"), response=httpx.Response(404)) + client = MagicMock() + client.session_exists = AsyncMock(return_value=True) + client.start_session = AsyncMock() + client.configure_egress = AsyncMock(side_effect=err) + client.close_session = AsyncMock() + runtime, _ = self._runtime_with_egress() + with pytest.raises(SandboxEgressUnavailableError): + await self._mw(client).abefore_agent({"session_id": "sess-warm"}, runtime) + client.start_session.assert_not_awaited() + client.close_session.assert_not_awaited() # warm container preserved for retry/reaper + async def test_skips_egress_when_none(self): client = MagicMock() client.start_session = AsyncMock(return_value="sess-x") client.seed_session = AsyncMock() client.configure_egress = AsyncMock() - mw = SandboxMiddleware( - agent_root="/workspace/repo", client=client, sandbox_backend=SandboxFileBackend(client=client) - ) runtime = _make_runtime() # network_enabled False, egress None - with ( - patch("automation.agent.middlewares.sandbox._make_repo_archive", return_value=b""), - patch("automation.agent.middlewares.sandbox._make_global_skills_archive", return_value=None), - ): - await mw.abefore_agent({}, runtime) + with self._patch_archives(): + await self._mw(client).abefore_agent({}, runtime) client.configure_egress.assert_not_awaited() - @pytest.mark.parametrize("status", [404, 409]) - async def test_egress_failure_is_fail_closed_and_closes_session(self, status: int): + async def test_egress_404_is_fail_closed_and_closes_session(self): import httpx - from automation.agent.middlewares.sandbox import SandboxEgressUnavailableError + # 404 = the egress proxy is disabled on the sandbox (DAIV_SANDBOX_EGRESS_PROXY_ENABLED off): + # unambiguous and permanent, so it converts to the actionable fail-closed error. + err = httpx.HTTPStatusError("nope", request=httpx.Request("POST", "x"), response=httpx.Response(404)) + client = MagicMock() + client.start_session = AsyncMock(return_value="sess-fc") + client.seed_session = AsyncMock() + client.configure_egress = AsyncMock(side_effect=err) + client.close_session = AsyncMock() + client.close = AsyncMock() + runtime, _ = self._runtime_with_egress() + with self._patch_archives(), pytest.raises(SandboxEgressUnavailableError): + await self._mw(client).abefore_agent({}, runtime) + client.close_session.assert_awaited_once_with("sess-fc", force=True) + client.close.assert_not_awaited() # the run owns the transport + @pytest.mark.parametrize("status", [409, 500]) + async def test_egress_non_404_propagates_raw_and_closes_session(self, status: int): + import httpx + + # 409 is ambiguous on the egress endpoint ("Session is busy" = transient lock contention, or + # "Session has no egress proxy") and 5xx is transient; neither maps to the permanent + # "proxy disabled" diagnosis, so they must propagate unchanged (still fail-closed via the + # fresh-create cleanup) rather than be mislabeled as SandboxEgressUnavailableError. err = httpx.HTTPStatusError("nope", request=httpx.Request("POST", "x"), response=httpx.Response(status)) client = MagicMock() client.start_session = AsyncMock(return_value="sess-fc") @@ -832,15 +872,8 @@ async def test_egress_failure_is_fail_closed_and_closes_session(self, status: in client.configure_egress = AsyncMock(side_effect=err) client.close_session = AsyncMock() client.close = AsyncMock() - mw = SandboxMiddleware( - agent_root="/workspace/repo", client=client, sandbox_backend=SandboxFileBackend(client=client) - ) runtime, _ = self._runtime_with_egress() - with ( - patch("automation.agent.middlewares.sandbox._make_repo_archive", return_value=b""), - patch("automation.agent.middlewares.sandbox._make_global_skills_archive", return_value=None), - pytest.raises(SandboxEgressUnavailableError), - ): - await mw.abefore_agent({}, runtime) + with self._patch_archives(), pytest.raises(httpx.HTTPStatusError) as raised: + await self._mw(client).abefore_agent({}, runtime) + assert not isinstance(raised.value, SandboxEgressUnavailableError) # raw error, not converted client.close_session.assert_awaited_once_with("sess-fc", force=True) - client.close.assert_not_awaited() # the run owns the transport diff --git a/tests/unit_tests/sandbox_envs/test_models.py b/tests/unit_tests/sandbox_envs/test_models.py index 72ccc548..1054344e 100644 --- a/tests/unit_tests/sandbox_envs/test_models.py +++ b/tests/unit_tests/sandbox_envs/test_models.py @@ -450,3 +450,25 @@ def test_egress_secrets_encrypted_at_rest(): env = _egress_env() assert env._egress_secrets_encrypted # ciphertext populated by the descriptor assert "Bearer t" not in env._egress_secrets_encrypted + + +def test_validate_egress_rejects_too_many_rules(): + from django.core.exceptions import ValidationError + + from sandbox_envs.models import EGRESS_MAX_RULES + + rules = [{"host": f"h{i}.example"} for i in range(EGRESS_MAX_RULES + 1)] + env = _egress_env(egress_policy={"default": "deny", "intercept": "all", "rules": rules}, egress_secrets={}) + with pytest.raises(ValidationError, match="Too many egress rules"): + env._validate_egress() + + +def test_validate_egress_rejects_too_many_secrets(): + from django.core.exceptions import ValidationError + + from sandbox_envs.models import EGRESS_MAX_SECRETS + + secrets = {f"s{i}": {"header": "Authorization", "value": "v"} for i in range(EGRESS_MAX_SECRETS + 1)} + env = _egress_env(egress_policy={"default": "deny", "intercept": "all", "rules": []}, egress_secrets=secrets) + with pytest.raises(ValidationError, match="Too many egress secrets"): + env._validate_egress() diff --git a/tests/unit_tests/sandbox_envs/test_services.py b/tests/unit_tests/sandbox_envs/test_services.py index 75988f5e..0705921a 100644 --- a/tests/unit_tests/sandbox_envs/test_services.py +++ b/tests/unit_tests/sandbox_envs/test_services.py @@ -502,6 +502,12 @@ def _egress_request(host: str): ) +def _override(**over): + """A ``SandboxEnvOverride`` with all-None scaffolding; pass only the field(s) under test.""" + base = {"base_image": None, "network_enabled": None, "memory_bytes": None, "cpus": None, "env_vars": {}} + return SandboxEnvOverride(**(base | over)) + + @pytest.mark.asyncio @pytest.mark.django_db(transaction=True) async def test_row_to_override_builds_egress(): @@ -526,10 +532,11 @@ async def test_row_to_override_builds_egress(): @pytest.mark.asyncio @pytest.mark.django_db(transaction=True) -async def test_row_to_override_drops_egress_when_malformed(): +async def test_row_to_override_fails_closed_to_deny_all_when_malformed(): from sandbox_envs.services import row_to_override - # Dangling inject (no matching secret) — stored directly, bypassing clean(). + # Dangling inject (no matching secret) — stored directly, bypassing clean(). The env *intended* + # restricted egress, so dropping to None (raw network) would be fail-open; we substitute deny-all. env = await SandboxEnvironment.objects.acreate( scope=Scope.GLOBAL, name="bad", @@ -538,45 +545,64 @@ async def test_row_to_override_drops_egress_when_malformed(): egress_secrets={}, ) override = row_to_override(env) - assert override.egress is None + assert override.egress is not None + assert override.egress.policy.default == "deny" + assert override.egress.policy.rules == [] + assert override.egress.secrets == {} -def test_merge_prefers_per_run_egress(): - from sandbox_envs.services import SandboxEnvOverride, merge_sandbox_runtime - - per_run = SandboxEnvOverride( - base_image=None, - network_enabled=None, - memory_bytes=None, - cpus=None, - env_vars={}, - egress=_egress_request("per-run.example"), +@pytest.mark.asyncio +@pytest.mark.django_db(transaction=True) +async def test_row_to_override_fails_closed_to_deny_all_on_decryption_error(mocker): + from sandbox_envs.services import row_to_override + + from core.encryption import DecryptionError + + env = await SandboxEnvironment.objects.acreate( + scope=Scope.GLOBAL, + name="rotated", + base_image="python:3.12", + egress_policy={"default": "deny", "rules": [{"host": "*.github.com", "inject": "gh"}]}, + egress_secrets={"gh": {"header": "Authorization", "value": "Bearer t"}}, ) - global_default = SandboxEnvOverride( - base_image=None, - network_enabled=None, - memory_bytes=None, - cpus=None, - env_vars={}, - egress=_egress_request("global.example"), + + # Simulate a rotated/lost DAIV_ENCRYPTION_KEY: the still-present egress_policy can no longer be + # paired with its decryptable secrets. Must fail closed (deny-all), never drop to raw network. + def _raise(instance): + raise DecryptionError("bad key") + + mocker.patch.object(type(env), "egress_secrets", new_callable=lambda: property(fget=_raise)) + override = row_to_override(env) + assert override.egress is not None + assert override.egress.policy.default == "deny" + assert override.egress.policy.rules == [] + + +def test_merge_prefers_per_run_egress(): + from sandbox_envs.services import merge_sandbox_runtime + + rt = merge_sandbox_runtime( + per_run=_override(egress=_egress_request("per-run.example")), + global_default=_override(egress=_egress_request("global.example")), ) - rt = merge_sandbox_runtime(per_run=per_run, global_default=global_default) assert rt.egress.policy.rules[0].host == "per-run.example" def test_merge_falls_back_to_global_egress(): - from sandbox_envs.services import SandboxEnvOverride, merge_sandbox_runtime + from sandbox_envs.services import merge_sandbox_runtime - per_run = SandboxEnvOverride( - base_image=None, network_enabled=None, memory_bytes=None, cpus=None, env_vars={}, egress=None - ) - global_default = SandboxEnvOverride( - base_image=None, - network_enabled=None, - memory_bytes=None, - cpus=None, - env_vars={}, - egress=_egress_request("global.example"), + rt = merge_sandbox_runtime( + per_run=_override(egress=None), global_default=_override(egress=_egress_request("global.example")) ) - rt = merge_sandbox_runtime(per_run=per_run, global_default=global_default) assert rt.egress.policy.rules[0].host == "global.example" + + +def test_merge_egress_is_none_when_neither_side_has_it(): + from sandbox_envs.services import merge_sandbox_runtime + + # Egress is opt-in: no policy on either side must never materialize one (it would otherwise + # silently apply an unintended network posture). + rt = merge_sandbox_runtime( + per_run=_override(network_enabled=True, egress=None), global_default=_override(egress=None) + ) + assert rt.egress is None From 27ff0737760102934b6f69b8be1efecef8d88c35 Mon Sep 17 00:00:00 2001 From: Sandro Date: Thu, 25 Jun 2026 15:17:58 +0100 Subject: [PATCH 08/10] fix(sandbox): align egress contract with CA-gated proxy model The earlier egress commits described the sandbox proxy as gated by a DAIV_SANDBOX_EGRESS_PROXY_ENABLED boolean flag, but daiv-sandbox enables egress by configuring a shared MITM CA (EGRESS_CA_CERT_FILE + EGRESS_CA_KEY_FILE) instead. A network-enabled session is built as a triad and reaches the internet only through the per-session proxy; if no CA is configured the run is rejected up front rather than dropping to raw network. Correct the contract everywhere it was described: the actionable SandboxEgressUnavailableError message, the provisioning docstrings, the SandboxEnvironment/services comments, and the test rationale comments. Clarify that the 404 fail-closed path is effectively unreachable on a fresh create (rejected at start_session) but still matters for warm reuse after CA rotation. Wire the local docker setup to the real mechanism: mount ./data/certs read-only and set DAIV_SANDBOX_EGRESS_CA_{CERT,KEY}_FILE, replacing the raw-network DAIV_SANDBOX_EXTRA_HOSTS sibling-resolution approach with egress-only routing through the proxy. --- CHANGELOG.md | 2 +- daiv/automation/agent/middlewares/sandbox.py | 26 ++++++++++++------- daiv/sandbox_envs/models.py | 8 +++--- daiv/sandbox_envs/services.py | 10 +++---- docker-compose.yml | 3 +++ docker/local/sandbox/config.env | 15 ++++++----- .../agent/middlewares/test_sandbox.py | 4 +-- .../unit_tests/sandbox_envs/test_services.py | 4 +-- 8 files changed, 42 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef739b9e..9ce5223d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added per-environment network egress policy for sandbox environments. A `SandboxEnvironment` can now carry an `egress_policy` (default/intercept + per-host rules) and encrypted `egress_secrets` (named header credentials); when set, the agent provisions them onto the daiv-sandbox MITM egress proxy at session start so credentials are injected by the sidecar and never enter the sandbox container. Provisioning is fail-closed: if the configured sandbox does not have the egress proxy enabled (`DAIV_SANDBOX_EGRESS_PROXY_ENABLED`), a run using an egress-configured environment aborts rather than falling back to unrestricted network. Note: once the sandbox egress proxy is enabled, network-enabled environments **without** an egress policy receive the sidecar's default deny-all (no connectivity) — configure an egress policy on those environments. Set programmatically via the ORM for now (admin/API/form UI to follow). +- Added per-environment network egress policy for sandbox environments. A `SandboxEnvironment` can now carry an `egress_policy` (default/intercept + per-host rules) and encrypted `egress_secrets` (named header credentials); when set, the agent provisions them onto the daiv-sandbox MITM egress proxy at session start so credentials are injected by the sidecar and never enter the sandbox container. The egress proxy is mandatory for network-enabled sessions: if the sandbox deployment has no egress proxy configured, a network-enabled run is rejected rather than falling back to unrestricted network. Note: a network-enabled environment **without** an egress policy receives the sidecar's default deny-all (no connectivity) — configure an egress policy on those environments. Set programmatically via the ORM for now (admin/API/form UI to follow). - Added `release_orphan_queued_threads` management command to recover `QUEUED` activities left behind on threads with no active sibling (rare TOCTOU loss). - Added per-domain auth headers for the `web_fetch` tool to the configuration UI under **Web Fetch → Per-domain auth headers**. Each row pairs a domain (exact match) with an HTTP header name and a header value (encrypted at rest). Replaces the previous env-only `AUTOMATION_WEB_FETCH_AUTH_HEADERS` setting; the new env override is `DAIV_WEB_FETCH_AUTH_HEADERS` (same JSON shape). Operators using the old name must rename it. - Added a "Start a run" page at `/dashboard/runs/new/` for launching new agent runs from the UI, and a "Retry" button on terminal non-webhook activities that pre-fills the form with the original prompt, repository, ref, and max-mode flag. diff --git a/daiv/automation/agent/middlewares/sandbox.py b/daiv/automation/agent/middlewares/sandbox.py index 7b32ffc8..548aa88c 100644 --- a/daiv/automation/agent/middlewares/sandbox.py +++ b/daiv/automation/agent/middlewares/sandbox.py @@ -293,8 +293,9 @@ def _make_global_skills_archive() -> bytes | None: class SandboxEgressUnavailableError(RuntimeError): """Raised when a session's resolved egress policy cannot be provisioned because the sandbox has - the egress proxy disabled (HTTP 404 'egress proxy not enabled'). Fail-closed: the run must not - proceed with whatever raw network the sandbox would otherwise grant.""" + no egress proxy configured (HTTP 404 'Egress proxy not configured' — no shared egress CA). + Fail-closed: an environment that requires a restricted egress policy must not run without that + policy in force.""" class BashFailure(Enum): @@ -506,13 +507,17 @@ async def _provision_egress(client: DAIVSandboxClient, session_id: str, sb: Sand """Provision the sidecar egress policy for a network-enabled session that requires it. No-op when there is no resolved sandbox runtime, the env defines no egress policy, or network - is off. Only a **404** (egress - proxy disabled on the sandbox — unambiguous and permanent) is converted to the actionable - ``SandboxEgressUnavailableError``. Every other status propagates unchanged, including a 409 — on - the egress endpoint 409 is ambiguous ("Session is busy" transient lock contention, or "Session - has no egress proxy") and the project already classifies 409 as transient - (``TRANSIENT_SANDBOX_STATUS``), so it must not be mislabeled as a permanent "enable the proxy" - diagnosis. A propagated error still fails closed: the caller's setup path aborts the run. + is off. Only a **404** (egress proxy not configured on the sandbox — no shared egress CA, + unambiguous and permanent) is converted to the actionable ``SandboxEgressUnavailableError``. + On a fresh create this 404 is effectively unreachable: a network-enabled ``start_session`` is + rejected up front (400) when the sandbox has no egress proxy, so the run aborts before reaching + here. It survives for warm reuse — if the shared CA is rotated out between the session's start + and a later turn's re-provision, ``configure_egress`` 404s and we still want the actionable + diagnosis. Every other status propagates unchanged, including a 409 — on the egress endpoint 409 + is ambiguous ("Session is busy" transient lock contention, or "Session has no egress proxy") and + the project already classifies 409 as transient (``TRANSIENT_SANDBOX_STATUS``), so it must not be + mislabeled as a permanent "configure the proxy" diagnosis. A propagated error still fails closed: + the caller's setup path aborts the run. """ if sb is None or sb.egress is None or not sb.network_enabled: return @@ -522,7 +527,8 @@ async def _provision_egress(client: DAIVSandboxClient, session_id: str, sb: Sand if exc.response.status_code == 404: raise SandboxEgressUnavailableError( "The resolved sandbox environment requires the egress proxy, but the sandbox returned " - "404 for egress provisioning. Enable DAIV_SANDBOX_EGRESS_PROXY_ENABLED on the sandbox." + "404 for egress provisioning. Configure the shared egress CA (EGRESS_CA_CERT_FILE + " + "EGRESS_CA_KEY_FILE) on the sandbox deployment to enable the egress proxy." ) from exc raise diff --git a/daiv/sandbox_envs/models.py b/daiv/sandbox_envs/models.py index 664834dc..cd7e217d 100644 --- a/daiv/sandbox_envs/models.py +++ b/daiv/sandbox_envs/models.py @@ -105,9 +105,11 @@ class SandboxEnvironment(TimeStampedModel): is_default = models.BooleanField(_("is default"), default=False) # Per-env egress policy provisioned to the daiv-sandbox sidecar proxy when network is enabled. - # ``None`` = no egress configured (legacy behavior: raw network when the sandbox proxy is off). - # NOTE: when the sandbox has EGRESS_PROXY_ENABLED, a network-enabled env with egress_policy=None - # gets the sidecar's deny-all default — configure a policy to grant connectivity. + # ``None`` = no egress policy configured for this env. The daiv-sandbox egress proxy is mandatory + # for network-enabled sessions: a network-enabled env on a sandbox with no egress proxy configured + # (no shared egress CA) is rejected at session start — there is no raw-network fallback. + # NOTE: when the proxy IS configured, a network-enabled env with egress_policy=None gets the + # sidecar's default deny-all (no connectivity) — configure a policy to grant connectivity. egress_policy = models.JSONField(_("egress policy"), null=True, blank=True, default=None) _egress_secrets_encrypted = models.TextField(blank=True, null=True, editable=False) # noqa: DJ001 — NULL = no secrets egress_secrets = EncryptedJSONFieldDescriptor("egress_secrets") diff --git a/daiv/sandbox_envs/services.py b/daiv/sandbox_envs/services.py index 26bbbd2c..a29313d5 100644 --- a/daiv/sandbox_envs/services.py +++ b/daiv/sandbox_envs/services.py @@ -54,11 +54,11 @@ def row_to_override(env: SandboxEnvironment) -> SandboxEnvOverride: except DecryptionError, PydanticValidationError, TypeError, ValueError: # The env *intended* restricted egress but its config is unusable (e.g. a rotated # DAIV_ENCRYPTION_KEY left the secrets undecryptable, or the row was hand-edited). - # Fail closed: substitute an empty deny-all policy rather than dropping to None — None - # would let a network-enabled session fall back to raw network (proxy off) or rely on - # the sidecar's default (proxy on). Deny-all denies connectivity (proxy on) or trips the - # fail-closed abort in _provision_egress (proxy off). Never reach the sidecar with a - # half/invalid config. The descriptor already logs the decryption failure at exception level. + # Fail closed: substitute an empty deny-all policy rather than dropping to None. None + # would leave a network-enabled session to rely on the sidecar's implicit default; an + # explicit deny-all honors the env's restricted-egress intent without depending on that + # default. Never reach the sidecar with a half/invalid config. The descriptor already + # logs the decryption failure at exception level. logger.error("egress config unusable for SandboxEnvironment id=%s; failing closed to deny-all", env.id) egress = EgressConfigRequest() diff --git a/docker-compose.yml b/docker-compose.yml index a8d3b76c..4e6c0129 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -173,6 +173,9 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock - ./../daiv-sandbox/daiv_sandbox:/home/app/daiv_sandbox + # Shared egress CA (cert + key) read by the service to provision per-session MITM sidecars. + # See DAIV_SANDBOX_EGRESS_CA_*_FILE in docker/local/sandbox/config.env. + - ./data/certs:/home/app/certs:ro depends_on: redis: condition: service_healthy diff --git a/docker/local/sandbox/config.env b/docker/local/sandbox/config.env index 1ead9ece..1d03ca8b 100644 --- a/docker/local/sandbox/config.env +++ b/docker/local/sandbox/config.env @@ -3,11 +3,12 @@ DAIV_SANDBOX_RUNTIME=runsc DAIV_SANDBOX_LOG_LEVEL=DEBUG DAIV_SANDBOX_ENVIRONMENT=local DAIV_SANDBOX_REDIS_URL=redis://redis:6379/1 -# Attach network-enabled cmd-executor containers to the compose network so they can -# reach sibling services (e.g. gitlab:8929). +# Upstream network for the egress sidecar's second NIC (its route out). Pinned to the compose +# network so the proxy can reach sibling services (e.g. gitlab:8929). Network-enabled sandboxes +# never attach here directly — they egress only through the per-session proxy. DAIV_SANDBOX_NETWORK=daiv_default -# Under gVisor (runsc) the cmd-executor can't use Docker's embedded resolver (127.0.0.11) that the -# user-defined network injects, so resolv.conf is repointed at real upstreams (DAIV_SANDBOX_DNS, -# default 1.1.1.1,8.8.8.8) for external names, and these sibling services are pinned in /etc/hosts. -DAIV_SANDBOX_EXTRA_HOSTS=gitlab - +# Shared egress CA. Setting both enables network egress: a network-enabled session is built as a +# triad (internal network + mitmdump sidecar + sandbox) and reaches the internet only through the +# proxy. Paths point at ./data/certs, mounted read-only into the container (see docker-compose.yml). +DAIV_SANDBOX_EGRESS_CA_CERT_FILE=/home/app/certs/egress-ca.crt +DAIV_SANDBOX_EGRESS_CA_KEY_FILE=/home/app/certs/egress-ca.key diff --git a/tests/unit_tests/automation/agent/middlewares/test_sandbox.py b/tests/unit_tests/automation/agent/middlewares/test_sandbox.py index e7b4377b..3678b7da 100644 --- a/tests/unit_tests/automation/agent/middlewares/test_sandbox.py +++ b/tests/unit_tests/automation/agent/middlewares/test_sandbox.py @@ -842,7 +842,7 @@ async def test_skips_egress_when_none(self): async def test_egress_404_is_fail_closed_and_closes_session(self): import httpx - # 404 = the egress proxy is disabled on the sandbox (DAIV_SANDBOX_EGRESS_PROXY_ENABLED off): + # 404 = the egress proxy is not configured on the sandbox (no shared egress CA): # unambiguous and permanent, so it converts to the actionable fail-closed error. err = httpx.HTTPStatusError("nope", request=httpx.Request("POST", "x"), response=httpx.Response(404)) client = MagicMock() @@ -863,7 +863,7 @@ async def test_egress_non_404_propagates_raw_and_closes_session(self, status: in # 409 is ambiguous on the egress endpoint ("Session is busy" = transient lock contention, or # "Session has no egress proxy") and 5xx is transient; neither maps to the permanent - # "proxy disabled" diagnosis, so they must propagate unchanged (still fail-closed via the + # "proxy not configured" diagnosis, so they must propagate unchanged (still fail-closed via the # fresh-create cleanup) rather than be mislabeled as SandboxEgressUnavailableError. err = httpx.HTTPStatusError("nope", request=httpx.Request("POST", "x"), response=httpx.Response(status)) client = MagicMock() diff --git a/tests/unit_tests/sandbox_envs/test_services.py b/tests/unit_tests/sandbox_envs/test_services.py index 0705921a..f08d5398 100644 --- a/tests/unit_tests/sandbox_envs/test_services.py +++ b/tests/unit_tests/sandbox_envs/test_services.py @@ -536,7 +536,7 @@ async def test_row_to_override_fails_closed_to_deny_all_when_malformed(): from sandbox_envs.services import row_to_override # Dangling inject (no matching secret) — stored directly, bypassing clean(). The env *intended* - # restricted egress, so dropping to None (raw network) would be fail-open; we substitute deny-all. + # restricted egress, so dropping to None (discarding that intent) is unsafe; we substitute deny-all. env = await SandboxEnvironment.objects.acreate( scope=Scope.GLOBAL, name="bad", @@ -567,7 +567,7 @@ async def test_row_to_override_fails_closed_to_deny_all_on_decryption_error(mock ) # Simulate a rotated/lost DAIV_ENCRYPTION_KEY: the still-present egress_policy can no longer be - # paired with its decryptable secrets. Must fail closed (deny-all), never drop to raw network. + # paired with its decryptable secrets. Must fail closed (deny-all), never drop to None. def _raise(instance): raise DecryptionError("bad key") From 080ca6635d488dd13a08d73d2782f063d661c38f Mon Sep 17 00:00:00 2001 From: Sandro Rodrigues Date: Tue, 30 Jun 2026 10:55:45 +0100 Subject: [PATCH 09/10] feat(sandbox-envs): per-environment egress policy UI (#1336) --- CHANGELOG.md | 4 +- .../accounts/templates/accounts/_sidebar.html | 2 +- daiv/accounts/templates/base_app.html | 2 +- daiv/automation/agent/git_manager.py | 34 +- daiv/automation/agent/middlewares/sandbox.py | 84 ++- daiv/codebase/clients/base.py | 47 ++ daiv/codebase/clients/github/client.py | 14 + daiv/codebase/clients/gitlab/client.py | 64 ++- daiv/codebase/clients/gitlab/clone_tokens.py | 12 + daiv/codebase/context.py | 7 +- daiv/core/sandbox/client.py | 39 +- daiv/core/sandbox/schemas.py | 23 +- daiv/core/utils.py | 22 + daiv/mcp_server/server.py | 2 +- daiv/sandbox_envs/forms.py | 186 +++++- .../migrations/0006_drop_network_enabled.py | 25 + daiv/sandbox_envs/models.py | 60 +- daiv/sandbox_envs/services.py | 101 +++- .../static/sandbox_envs/js/egress-editor.js | 111 ++++ .../sandbox_envs/js/resource-control.js | 19 +- .../templates/sandbox_envs/_form_body.html | 142 +++-- .../templates/sandbox_envs/_scripts.html | 1 + docs/features/sandbox-environments.md | 45 +- .../agent/middlewares/test_sandbox.py | 104 ++-- .../codebase/clients/github/test_client.py | 24 + .../codebase/clients/gitlab/test_client.py | 140 +++++ .../clients/gitlab/test_clone_tokens.py | 15 + tests/unit_tests/codebase/test_context.py | 66 +++ tests/unit_tests/core/sandbox/test_client.py | 57 +- tests/unit_tests/core/sandbox/test_schemas.py | 29 + .../unit_tests/mcp_server/test_server_envs.py | 27 + tests/unit_tests/sandbox_envs/test_forms.py | 535 +++++++++++++++++- .../sandbox_envs/test_migrations.py | 50 ++ tests/unit_tests/sandbox_envs/test_models.py | 53 +- .../unit_tests/sandbox_envs/test_services.py | 355 +++++++++--- tests/unit_tests/sandbox_envs/test_views.py | 40 +- 36 files changed, 2153 insertions(+), 388 deletions(-) create mode 100644 daiv/sandbox_envs/migrations/0006_drop_network_enabled.py create mode 100644 daiv/sandbox_envs/static/sandbox_envs/js/egress-editor.js create mode 100644 tests/unit_tests/sandbox_envs/test_migrations.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ce5223d..7621bbf5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added per-environment network egress policy for sandbox environments. A `SandboxEnvironment` can now carry an `egress_policy` (default/intercept + per-host rules) and encrypted `egress_secrets` (named header credentials); when set, the agent provisions them onto the daiv-sandbox MITM egress proxy at session start so credentials are injected by the sidecar and never enter the sandbox container. The egress proxy is mandatory for network-enabled sessions: if the sandbox deployment has no egress proxy configured, a network-enabled run is rejected rather than falling back to unrestricted network. Note: a network-enabled environment **without** an egress policy receives the sidecar's default deny-all (no connectivity) — configure an egress policy on those environments. Set programmatically via the ORM for now (admin/API/form UI to follow). +- Sandbox environments now automatically allow and authenticate their repository's git platform (GitLab/GitHub) for git operations over HTTPS in the sandbox. DAIV injects an `Authorization: Basic` credential at the egress proxy, built from a short-lived token — the same project-scoped ephemeral clone token for GitLab, and a `contents:write` installation token for GitHub — so `git fetch`/clone/push of the repository works without configuring a per-environment host rule. Because DAIV pushes from inside the sandbox, this applies even to **Network Off** environments when a push token exists: the environment is opened solely for its git platform (everything else stays blocked) so publishing always works, while token-less eval/benchmark runs stay fully network-isolated. The rule is applied at runtime (never stored on the environment) and takes precedence over a user-defined rule for the same host. +- Added a per-environment **network egress policy** for sandbox environments, configurable from the environment form when network is enabled. Restrict outbound traffic to an allow-list of hosts (glob patterns), attach per-host HTTP credentials (header name + value, encrypted at rest and never rendered back), scope each host to **all methods** or **read-only** (`GET, HEAD, OPTIONS`), and set the `default` (deny/allow) reachability for unlisted hosts. The proxy always TLS-inspects every reachable host, so per-host credentials and method rules are always enforced. Credentials are injected by the daiv-sandbox egress proxy and never enter the sandbox container. The egress proxy is mandatory for network-enabled sessions: if the sandbox deployment has no egress proxy configured, a network-enabled run is rejected rather than falling back to unrestricted network. - Added `release_orphan_queued_threads` management command to recover `QUEUED` activities left behind on threads with no active sibling (rare TOCTOU loss). - Added per-domain auth headers for the `web_fetch` tool to the configuration UI under **Web Fetch → Per-domain auth headers**. Each row pairs a domain (exact match) with an HTTP header name and a header value (encrypted at rest). Replaces the previous env-only `AUTOMATION_WEB_FETCH_AUTH_HEADERS` setting; the new env override is `DAIV_WEB_FETCH_AUTH_HEADERS` (same JSON shape). Operators using the old name must rename it. - Added a "Start a run" page at `/dashboard/runs/new/` for launching new agent runs from the UI, and a "Retry" button on terminal non-webhook activities that pre-fills the form with the original prompt, repository, ref, and max-mode flag. @@ -41,6 +42,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Hardened the agent's workspace file tools (`read_file`/`write_file`/`edit_file`/`ls`/`glob`/`grep`) against sandbox transport faults: a transient failure — most notably the `409 "Session is busy"` the sandbox returns when batched tool calls contend on the one-at-a-time session — now degrades to a retryable tool-result error instead of aborting the whole run, matching the `bash` tool. The `bash` tool's own classifier now also treats `409` as transient (retry once) rather than permanent. - Hardened MCP tool loading so a single broken or slow MCP server can no longer freeze chats and runs. Each server's tools are now loaded independently with a per-server timeout (`MCP_TOOL_LOAD_TIMEOUT`, default 30s); a server that times out or errors is skipped instead of blocking the whole agent. +- GitLab clones now self-heal from a stale cached clone token. The short-lived project-scoped clone token is cached for a day, but the underlying project access token can die sooner (revoked, or the project/instance reset); previously the dead token was served to — and failed — every clone of that project until the cache window closed. A clone rejected for authentication while using the ephemeral token now drops it and retries once with a freshly minted one (the PAT fallback is not retried, since re-minting cannot change that credential). - Enforced "at most one active (`READY`/`RUNNING`) API/MCP Activity per `thread_id`" at the DB layer via a partial unique constraint. Concurrent submissions on the same thread cleanly fall back to `QUEUED` instead of both running. - Made the FIFO dispatcher race-safe: an atomic compare-and-swap (`UPDATE filter(status=QUEUED) → READY`) prevents two terminal events on the same thread from double-promoting the same queued sibling. Dispatch failures now set `finished_at` and iterate via a loop instead of recursive signal re-entry. The loop bails after `MAX_CONSECUTIVE_DISPATCH_FAILURES` (3) consecutive failures so a transient broker outage does not mass-fail an entire QUEUED backlog — remaining rows stay QUEUED for `release_orphan_queued_threads`. - Guarded the post-enqueue `task_result_id` save in both `asubmit_batch_runs` and `dispatch_next_in_thread`: a DB blip after a successful broker enqueue now marks the row FAILED (and releases queued siblings) instead of stranding it in READY with no task linkage. diff --git a/daiv/accounts/templates/accounts/_sidebar.html b/daiv/accounts/templates/accounts/_sidebar.html index ab3cc78d..4f8b2328 100644 --- a/daiv/accounts/templates/accounts/_sidebar.html +++ b/daiv/accounts/templates/accounts/_sidebar.html @@ -1,7 +1,7 @@ {% load i18n icon_tags nav_tags %}