feat(tasks): inject agent OTLP telemetry into cloud sandboxes (#71100)

Author: tatoalo

docs/internal/sandboxes-setup-guide.md

@@ -177,6 +177,21 @@ SANDBOX_MCP_URL=https://<mcp-8787-subdomain>.ngrok-free.app/mcp
 
 `SANDBOX_MCP_URL` overrides the `host.docker.internal` default (which only resolves from local Docker sandboxes, not Modal). Without it, sandbox agents can't reach the MCP server and lose access to the PostHog `execute-sql`, query, and tool-calling stack.
 
+### Agent run telemetry (optional)
+
+To ship agent-server run metadata to PostHog Logs, set both of the first two; the third additionally produces one APM trace per run (root `task_run` span, a `turn` span per prompt, a `tool_call:<kind>` span per tool call) with trace/span ids stamped on the log records:
+
+```bash
+SANDBOX_AGENT_OTEL_LOGS_URL=http://localhost:8000/i/v1/logs  # or https://us.i.posthog.com/i/v1/logs
+SANDBOX_AGENT_OTEL_LOGS_TOKEN=<project API key of the telemetry project>
+SANDBOX_AGENT_OTEL_TRACES_URL=http://localhost:8000/i/v1/traces  # optional, enables APM spans
+```
+
+In cloud, emission is additionally gated per run by the `tasks-agent-run-otel-telemetry` feature flag (org-targeted, stamped into run state at dispatch; it also gates the scout run-log mirror). `DEBUG` bypasses the flag, so locally these settings are the only switch. They're injected into the sandbox as `POSTHOG_AGENT_OTEL_LOGS_URL`/`_TOKEN`/`POSTHOG_AGENT_OTEL_TRACES_URL` (deliberately not standard `OTEL_*` names, so OTel SDKs in user code don't auto-export into the telemetry project).
+The agent-server exports run/turn/tool lifecycle metadata (never message content or tool arguments), tagged with `run_id`/`task_id`/`team_id`/`user_id`/`distinct_id` resource attributes and `service.name=posthog-code-agent`.
+Telemetry stays off when either of the first two vars is unset.
+For local Docker sandboxes the localhost URLs are rewritten to `host.docker.internal` automatically; local ingestion requires the `capture-logs` service to be running.
+
 ### MCP server `.env`
 
 `MODAL_DOCKER` (and the local Docker provider) both depend on the MCP server running at `localhost:8787`. The server reads its config from `services/mcp/.env` — without it, things like `POSTHOG_API_BASE_URL`, the UI-apps token, and analytics keys are missing and the server will either refuse to start or return broken responses to the sandbox.
@@ -237,6 +252,39 @@ repositories.
 
 > **Note:** This only works with `SANDBOX_PROVIDER=docker`.
 
+### Task-run log mirroring to PostHog Logs (dogfooding)
+
+Task-run log entries (the JSONL appended to object storage via `TaskRun.append_log`) are also mirrored into the PostHog Logs product,
+so runs can be browsed and sampled in the Logs UI instead of fetching S3 blobs.
+
+In production there is no transport of its own: entries are emitted as structured stdout log lines (`event=task_run_log`),
+and the per-cluster OTel collector that already ships all container stdout into the region's internal PostHog project picks them up.
+The collector parses each JSON key into a queryable attribute and turns the emitted `request_id` (the run uuid) into a trace id,
+so one run groups as a trace and can be pulled up with an attribute filter on `task_run_id`.
+
+```bash
+# Which task origins to mirror (comma-separated). Defaults to signals scouts only.
+# Set empty to disable.
+TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS=signals_scout
+```
+
+The mirror also has a **direct OTLP leg** that ships each batch straight to a logs ingest endpoint:
+
+```bash
+TASK_RUN_LOGS_MIRROR_OTLP_URL=http://localhost:8000/i/v1/logs  # prod: https://us.i.posthog.com/i/v1/logs
+TASK_RUN_LOGS_MIRROR_OTLP_TOKEN=<project API key of the internal logs project>
+```
+
+The token pins the destination: scout runs execute for customer teams,
+but their mirrored transcripts must only ever land in — and bill — PostHog's own internal logs project, never the customer's.
+Records arrive under `service.name=task-run-log-mirror` with the run uuid as the trace id.
+
+Locally the direct leg is the only delivery path: `append_log` runs in the host Django process,
+and the dev collector (`otel-collector-config.dev.yaml`) only tails docker-compose container stdout,
+so without these settings the mirrored lines only show up in the Django phrocs pane.
+
+Mirroring failures are logged and never break the run's log write.
+
 ### How `MODAL_DOCKER` works
 
 When both `SANDBOX_PROVIDER=MODAL_DOCKER` and `LOCAL_POSTHOG_CODE_MONOREPO_ROOT` are set:

posthog/settings/temporal.py

@@ -51,6 +51,14 @@
 SANDBOX_LLM_GATEWAY_URL: str | None = get_from_env("SANDBOX_LLM_GATEWAY_URL", None, optional=True)
 SANDBOX_MCP_URL: str | None = get_from_env("SANDBOX_MCP_URL", None, optional=True)
 
+# OTLP destinations for agent-server run telemetry (PostHog Logs/APM).
+# Full ingest URLs (e.g. https://us.i.posthog.com/i/v1/logs and .../i/v1/traces)
+# plus the project API key of the telemetry project. Telemetry stays off unless
+# URL + token are set; the traces URL additionally enables APM spans.
+SANDBOX_AGENT_OTEL_LOGS_URL: str | None = get_from_env("SANDBOX_AGENT_OTEL_LOGS_URL", None, optional=True)
+SANDBOX_AGENT_OTEL_LOGS_TOKEN: str | None = get_from_env("SANDBOX_AGENT_OTEL_LOGS_TOKEN", None, optional=True)
+SANDBOX_AGENT_OTEL_TRACES_URL: str | None = get_from_env("SANDBOX_AGENT_OTEL_TRACES_URL", None, optional=True)
+
 # client_id of the OAuthApplication used to mint the access token the PostHog setup wizard
 # uses when it runs inside a task sandbox (the "run the wizard in the cloud" onboarding path).
 # It must be the wizard's own app so the LLM gateway authorizes the token like a normal wizard
@@ -94,6 +102,24 @@
     "TASKS_CREDENTIAL_REFRESH_INITIAL_DELAY_SECONDS", 0, type_cast=int
 )
 
+# Mirror persisted task-run logs into the PostHog Logs product (dogfooding).
+# Entries appended to a run's S3 JSONL log are also emitted as structured stdout log lines;
+# the per-cluster OTel collector already ships container stdout into the region's internal
+# PostHog project's Logs, so no transport or credentials are needed here. Only runs whose
+# task origin_product is in this list are mirrored — scoped to signals scouts for now;
+# widen the list to cover more task origins, or set it empty to disable.
+TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS: list[str] = get_list(
+    os.getenv("TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS", "signals_scout")
+)
+
+# Direct OTLP delivery for the mirror above. The token pins the destination: scout runs
+# execute for customer teams, but their mirrored logs must only ever land in (and bill)
+# PostHog's own internal logs project — so this is the internal project's API key, never
+# derived from the run's team. Point locally at the dev logs ingest to see mirrored runs
+# in /logs. Unset disables the direct leg (stdout emission for the collector remains).
+TASK_RUN_LOGS_MIRROR_OTLP_URL: str | None = get_from_env("TASK_RUN_LOGS_MIRROR_OTLP_URL", None, optional=True)
+TASK_RUN_LOGS_MIRROR_OTLP_TOKEN: str | None = get_from_env("TASK_RUN_LOGS_MIRROR_OTLP_TOKEN", None, optional=True)
+
 TEMPORAL_LOG_LEVEL_PRODUCE: str = os.getenv("TEMPORAL_LOG_LEVEL_PRODUCE", "DEBUG")
 TEMPORAL_EXTERNAL_LOGS_QUEUE_SIZE: int = get_from_env("TEMPORAL_EXTERNAL_LOGS_QUEUE_SIZE", 0, type_cast=int)
 

products/tasks/backend/constants.py

@@ -7,6 +7,10 @@
 AGENT_PROXY_KEEP_STREAM_OPEN_FEATURE_FLAG = "tasks-agent-proxy-keep-stream-open"
 MODAL_VM_SANDBOX_FEATURE_FLAG = "tasks-modal-vm-sandbox"
 MODAL_NETWORK_ALLOWLIST_FEATURE_FLAG = "tasks-modal-network-allowlist"
+AGENT_RUN_OTEL_TELEMETRY_FEATURE_FLAG = "tasks-agent-run-otel-telemetry"
+# Run-state key the telemetry flag decision is stamped under at dispatch (temporal/client.py).
+# Consumers read the stamp, so the decision stays stable for the run's whole lifetime.
+AGENT_OTEL_TELEMETRY_STATE_KEY = "agent_otel_telemetry_enabled"
 
 
 def vm_sandbox_allowed_origin_products(payload: object) -> set[str]:
@@ -365,6 +369,9 @@ def vm_sandbox_allowed_origins(*, distinct_id: str, organization_id: str) -> set
         "GH_TOKEN",
         "LLM_GATEWAY_URL",
         "POSTHOG_RESUME_RUN_ID",
+        "POSTHOG_AGENT_OTEL_LOGS_URL",
+        "POSTHOG_AGENT_OTEL_LOGS_TOKEN",
+        "POSTHOG_AGENT_OTEL_TRACES_URL",
         "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC",
         "DISABLE_TELEMETRY",
         "DISABLE_ERROR_REPORTING",

products/tasks/backend/facade/api.py

@@ -37,6 +37,7 @@
 from posthog.models.integration import Integration
 
 from products.tasks.backend.constants import (
+    AGENT_OTEL_TELEMETRY_STATE_KEY,
     MAX_CUSTOM_IMAGES_PER_TEAM,
     MAX_CUSTOM_IMAGES_PER_USER,
     RESERVED_SANDBOX_ENVIRONMENT_VARIABLE_KEYS,
@@ -1716,6 +1717,12 @@ def _sync_automation_schedule(automation: TaskAutomation) -> None:
         "wizard_head_branch",
         "use_modal_directory_resume_snapshots",
         "use_modal_vm_sandbox",
+        # Rollout stamps written once at dispatch by _capture_run_feature_flags; a PATCHable
+        # value would let a task controller bypass the org feature flags (for telemetry, that
+        # means injecting the internal OTLP capture token into their sandbox and re-enabling
+        # the run-log mirror with the rollout off).
+        AGENT_OTEL_TELEMETRY_STATE_KEY,
+        "sandbox_event_ingest_enabled",
         "snapshot_external_id",
         "snapshot_kind",
         "snapshot_mount_path",

products/tasks/backend/feature_flags.py

@@ -4,6 +4,8 @@
 
 import posthoganalytics
 
+from products.tasks.backend.constants import AGENT_OTEL_TELEMETRY_STATE_KEY, AGENT_RUN_OTEL_TELEMETRY_FEATURE_FLAG
+
 logger = logging.getLogger(__name__)
 
 NATIVE_STEERING_SIGNALS_FEATURE_FLAG = "tasks-native-steering-signals"
@@ -26,3 +28,32 @@ def is_native_steering_signals_enabled() -> bool:
     except Exception:
         logger.exception("native_steering_signals_feature_flag_check_failed")
         return False
+
+
+def is_agent_otel_telemetry_enabled(*, distinct_id: str, organization_id: str) -> bool:
+    """Org-gated rollout of agent-run OTel telemetry; fail-closed when evaluation fails."""
+    try:
+        return bool(
+            posthoganalytics.feature_enabled(
+                AGENT_RUN_OTEL_TELEMETRY_FEATURE_FLAG,
+                distinct_id=distinct_id,
+                groups={"organization": organization_id},
+                group_properties={"organization": {"id": organization_id}},
+                only_evaluate_locally=False,
+                send_feature_flag_events=False,
+            )
+        )
+    except Exception:
+        logger.exception("agent_otel_telemetry_flag_check_failed")
+        return False
+
+
+def agent_otel_telemetry_enabled_for_state(state: dict | None) -> bool:
+    """Per-run telemetry decision, read from the flag value stamped into run state at dispatch.
+
+    DEBUG bypasses the flag: the analytics SDK is disabled in local dev, where the
+    telemetry env settings / mirror settings are themselves the opt-in.
+    """
+    if settings.DEBUG:
+        return True
+    return (state or {}).get(AGENT_OTEL_TELEMETRY_STATE_KEY) is True

products/tasks/backend/logic/services/agentsh.py

@@ -64,6 +64,8 @@ def _port_from_url(url: str | None) -> int | None:
     "SANDBOX_API_URL",
     "SANDBOX_LLM_GATEWAY_URL",
     "SANDBOX_MCP_URL",
+    "SANDBOX_AGENT_OTEL_LOGS_URL",
+    "SANDBOX_AGENT_OTEL_TRACES_URL",
 )
 
 

products/tasks/backend/logic/services/docker_sandbox.py

@@ -84,6 +84,8 @@
     {
         "POSTHOG_API_URL",
         "POSTHOG_SITE_URL",
+        "POSTHOG_AGENT_OTEL_LOGS_URL",
+        "POSTHOG_AGENT_OTEL_TRACES_URL",
         "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT",
         "OTEL_EXPORTER_OTLP_ENDPOINT",
     }