Merge branch 'main' into posthog-code/group-configure-settings

Author: puemos

.github/workflows/agent-release.yml

@@ -57,6 +57,9 @@ jobs:
       - name: Build enricher (agent dependency)
         run: pnpm --filter @posthog/enricher run build
 
+      - name: Build harness (agent dependency)
+        run: pnpm --filter @posthog/harness run build
+
       - name: Build the package
         run: pnpm --filter agent run build
 

packages/agent/README.md

@@ -74,6 +74,8 @@ Four modes defined in `src/execution-mode.ts`:
 
 In cloud background mode, permissions are always auto-approved. In interactive mode, the permission system is active and configurable per session. Tool categorization lives in `src/adapters/claude/tools.ts` — each tool belongs to a group (read, write, bash, search, web, agent) and modes whitelist groups.
 
+Cloud provisioning can pass `--posthogExecPermissionRegex <regex>` to require one-time client approval for matching PostHog MCP `exec` sub-tools in every interactive cloud Claude and Codex permission mode. Non-matching sub-tools never prompt. Locally, hands-off modes stay hands-off: Claude `auto` and `bypassPermissions`, and Codex `auto` and `full-access`, auto-approve matching sub-tools; other local modes prompt. Matching is case-insensitive against the delegated name in `call [--json] <sub-tool> ...`. These prompts offer Claude users an always-allow choice remembered in local repository settings; Codex approvals remain one-time. An invalid or empty regex is logged and falls back to the default. Background runs keep their existing auto-approval behavior. The default is `(^|-)(partial-update|update|patch|delete|destroy)(-|$)`.
+
 ## ACP connection layer
 
 `createAcpConnection()` in `src/adapters/acp-connection.ts` is the heart of the package. It's a factory that returns a `{ clientStreams, cleanup }` object — a pair of ndJson `ReadableStream`/`WritableStream` that the caller uses to speak ACP.
@@ -142,9 +144,9 @@ JWT validation (`src/server/jwt.ts`) uses RS256 with a configurable public key.
 
 When `POST /command` receives a `user_message`, it doesn't handle it directly — it calls `clientConnection.prompt()` on the ACP `ClientSideConnection`, which sends a `session/prompt` message through the ACP streams to the agent. Similarly, `cancel` sends `session/cancel`. This means all commands follow the same path as in-process calls from PostHog Code, with the HTTP layer just being a thin translation.
 
-### Auto-approval in cloud mode
+### Permission routing in cloud mode
 
-The `AgentServer` provides a `requestPermission` callback to the `ClientSideConnection` that always selects the "allow" option. In background mode this is necessary (no human to ask). In interactive mode it currently does the same, with a TODO for future per-tool approval via SSE round-trips.
+The `AgentServer` provides the `requestPermission` callback to the `ClientSideConnection`. Background mode selects an allow option automatically. Interactive mode relays approvals that need a person over SSE and parks them until a client responds; other requests follow the selected permission mode.
 
 ### Checkpoint capture
 
@@ -157,6 +159,7 @@ npx agent-server \
   --port 3001 \
   --mode interactive \
   --repositoryPath /path/to/repo \
+  --posthogExecPermissionRegex '(^|-)(partial-update|update|patch|delete|destroy)(-|$)' \
   --taskId task_123 \
   --runId run_456
 ```

packages/agent/package.json

@@ -145,6 +145,7 @@
     "@posthog/shared": "workspace:*",
     "@posthog/git": "workspace:*",
     "@posthog/enricher": "workspace:*",
+    "@posthog/harness": "workspace:*",
     "@types/bun": "latest",
     "@types/tar": "^6.1.13",
     "msw": "^2.12.7",
@@ -167,7 +168,6 @@
     "@opentelemetry/resources": "^2.0.0",
     "@opentelemetry/sdk-logs": "^0.208.0",
     "@opentelemetry/semantic-conventions": "^1.28.0",
-    "@posthog/harness": "workspace:*",
     "@types/jsonwebtoken": "^9.0.10",
     "commander": "^14.0.2",
     "fflate": "^0.8.2",

packages/agent/src/adapters/claude/claude-agent.resume-model.test.ts

@@ -1,7 +1,8 @@
-import { mkdtempSync, rmSync } from "node:fs";
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
 import * as os from "node:os";
 import * as path from "node:path";
 import type { AgentSideConnection } from "@agentclientprotocol/sdk";
+import type { HookInput, Options } from "@anthropic-ai/claude-agent-sdk";
 import { afterAll, beforeEach, describe, expect, it, vi } from "vitest";
 import { DEFAULT_GATEWAY_MODEL } from "../../gateway-models";
 
@@ -38,11 +39,13 @@ function makeQueryHandle(): SdkQueryHandle {
 }
 
 const createdQueries: SdkQueryHandle[] = [];
+const createdQueryOptions: Options[] = [];
 
 vi.mock("@anthropic-ai/claude-agent-sdk", () => ({
-  query: vi.fn(() => {
+  query: vi.fn(({ options }: { options: Options }) => {
     const handle = makeQueryHandle();
     createdQueries.push(handle);
+    createdQueryOptions.push(options);
     return handle;
   }),
   getSessionMessages: vi.fn().mockResolvedValue([]),
@@ -87,6 +90,14 @@ const cwd = mkdtempSync(path.join(os.tmpdir(), "claude-agent-test-cwd-"));
 const configDir = mkdtempSync(
   path.join(os.tmpdir(), "claude-agent-test-config-"),
 );
+const permissionCwd = mkdtempSync(
+  path.join(os.tmpdir(), "claude-agent-permission-test-cwd-"),
+);
+mkdirSync(path.join(permissionCwd, ".claude"), { recursive: true });
+writeFileSync(
+  path.join(permissionCwd, ".claude", "settings.json"),
+  JSON.stringify({ permissions: { allow: ["mcp__posthog__exec"] } }),
+);
 const savedEnv = {
   ANTHROPIC_BASE_URL: process.env.ANTHROPIC_BASE_URL,
   CLAUDE_CONFIG_DIR: process.env.CLAUDE_CONFIG_DIR,
@@ -95,6 +106,7 @@ const savedEnv = {
 afterAll(() => {
   rmSync(cwd, { recursive: true, force: true });
   rmSync(configDir, { recursive: true, force: true });
+  rmSync(permissionCwd, { recursive: true, force: true });
   process.env.ANTHROPIC_BASE_URL = savedEnv.ANTHROPIC_BASE_URL;
   process.env.CLAUDE_CONFIG_DIR = savedEnv.CLAUDE_CONFIG_DIR;
   if (savedEnv.ANTHROPIC_BASE_URL === undefined) {
@@ -105,9 +117,10 @@ afterAll(() => {
   }
 });
 
-describe("ClaudeAcpAgent session model on resume", () => {
+describe("ClaudeAcpAgent session creation", () => {
   beforeEach(() => {
     createdQueries.length = 0;
+    createdQueryOptions.length = 0;
     nextInitPromise = Promise.resolve({
       result: "success",
       commands: [],
@@ -119,6 +132,154 @@ describe("ClaudeAcpAgent session model on resume", () => {
     process.env.CLAUDE_CONFIG_DIR = configDir;
   });
 
+  async function runPostHogExecPreToolUse(
+    options: Options,
+    subTool: string,
+  ): Promise<string | undefined> {
+    const input = {
+      session_id: "permission-session",
+      transcript_path: "/tmp/transcript",
+      cwd: permissionCwd,
+      hook_event_name: "PreToolUse",
+      tool_name: "mcp__posthog__exec",
+      tool_use_id: "toolu_permission",
+      tool_input: { command: `call ${subTool} {}` },
+    } as HookInput;
+
+    for (const hook of (options.hooks?.PreToolUse ?? []).flatMap(
+      (entry) => entry.hooks ?? [],
+    )) {
+      const result = await hook(input, undefined, {
+        signal: new AbortController().signal,
+      });
+      const decision = (
+        result as {
+          hookSpecificOutput?: { permissionDecision?: string };
+        }
+      ).hookSpecificOutput?.permissionDecision;
+      if (decision) return decision;
+    }
+
+    return undefined;
+  }
+
+  it.each(["new", "resume", "load"] as const)(
+    "uses the default PostHog exec permission regex for local %s sessions when metadata omits it",
+    async (sessionKind) => {
+      const agent = makeAgent();
+      const sessionIds = {
+        new: "0197a000-0000-7000-8000-000000000101",
+        resume: "0197a000-0000-7000-8000-000000000102",
+        load: "0197a000-0000-7000-8000-000000000103",
+      };
+      const params = {
+        sessionId: sessionIds[sessionKind],
+        cwd: permissionCwd,
+        mcpServers: [],
+        _meta: { taskRunId: `run-permission-${sessionKind}` },
+      };
+
+      if (sessionKind === "new") {
+        await agent.newSession(params);
+      } else if (sessionKind === "resume") {
+        await agent.resumeSession(params);
+      } else {
+        await agent.loadSession(params);
+      }
+
+      expect(createdQueryOptions).toHaveLength(1);
+      await expect(
+        runPostHogExecPreToolUse(
+          createdQueryOptions[0] as Options,
+          "dashboard-update",
+        ),
+      ).resolves.toBe("ask");
+    },
+  );
+
+  it("uses an explicit PostHog exec permission regex instead of the default", async () => {
+    const agent = makeAgent();
+
+    await agent.newSession({
+      cwd: permissionCwd,
+      mcpServers: [],
+      _meta: {
+        taskRunId: "run-permission-custom",
+        posthogExecPermissionRegex: "(^|-)archive(-|$)",
+      },
+    });
+
+    expect(createdQueryOptions).toHaveLength(1);
+    await expect(
+      runPostHogExecPreToolUse(
+        createdQueryOptions[0] as Options,
+        "dashboard-update",
+      ),
+    ).resolves.toBe("allow");
+    await expect(
+      runPostHogExecPreToolUse(
+        createdQueryOptions[0] as Options,
+        "dashboard-archive",
+      ),
+    ).resolves.toBe("ask");
+  });
+
+  it.each(["[", ""])(
+    "warns and falls back to the default regex when metadata carries the invalid regex %j",
+    async (posthogExecPermissionRegex) => {
+      const agent = makeAgent();
+      const warnSpy = vi.spyOn(agent.logger, "warn");
+
+      await agent.newSession({
+        cwd: permissionCwd,
+        mcpServers: [],
+        _meta: {
+          taskRunId: "run-permission-invalid",
+          posthogExecPermissionRegex,
+        },
+      });
+
+      expect(warnSpy).toHaveBeenCalledWith(
+        expect.stringContaining("Invalid posthogExecPermissionRegex"),
+        expect.anything(),
+      );
+      expect(createdQueryOptions).toHaveLength(1);
+      await expect(
+        runPostHogExecPreToolUse(
+          createdQueryOptions[0] as Options,
+          "dashboard-update",
+        ),
+      ).resolves.toBe("ask");
+      await expect(
+        runPostHogExecPreToolUse(
+          createdQueryOptions[0] as Options,
+          "dashboard-get",
+        ),
+      ).resolves.toBe("allow");
+    },
+  );
+
+  it.each([
+    { environment: "local", expectedCloudMode: false },
+    { environment: "cloud", expectedCloudMode: true },
+  ] as const)(
+    "records $environment sessions as cloudMode=$expectedCloudMode",
+    async ({ environment, expectedCloudMode }) => {
+      const agent = makeAgent();
+
+      await agent.newSession({
+        cwd,
+        mcpServers: [],
+        _meta: { environment, taskRunId: `run-${environment}` },
+      });
+
+      expect(
+        (agent as unknown as { session: { cloudMode: boolean } }).session
+          .cloudMode,
+      ).toBe(expectedCloudMode);
+    },
+  );
+
   // The SDK does not carry the model across resume — without an explicit
   // setModel the resumed session silently runs the SDK default (opus).
   it.each([

packages/agent/src/adapters/claude/claude-agent.streamed-text.test.ts

@@ -149,7 +149,17 @@ function textDelta(sessionId: string, text: string) {
   };
 }
 
-function assistantMessage(sessionId: string, apiId: string, text: string) {
+function assistantMessage(
+  sessionId: string,
+  apiId: string,
+  text: string,
+  usage?: {
+    input_tokens: number;
+    output_tokens: number;
+    cache_read_input_tokens: number;
+    cache_creation_input_tokens: number;
+  },
+) {
   return {
     type: "assistant",
     parent_tool_use_id: null,
@@ -159,6 +169,20 @@ function assistantMessage(sessionId: string, apiId: string, text: string) {
       id: apiId,
       role: "assistant",
       content: [{ type: "text", text }],
+      ...(usage ? { usage } : {}),
+    },
+  };
+}
+
+function compactBoundary(sessionId: string) {
+  return {
+    type: "system",
+    subtype: "compact_boundary",
+    session_id: sessionId,
+    uuid: "compact-1",
+    compact_metadata: {
+      trigger: "auto",
+      pre_tokens: 434_000,
     },
   };
 }
@@ -192,6 +216,23 @@ function messageChunkTexts(
     .map((update) => update?.content?.text ?? "");
 }
 
+function usageUpdates(
+  calls: ClientMocks["sessionUpdate"]["mock"]["calls"],
+): Array<{ used: number; size: number }> {
+  return calls.flatMap(([call]) => {
+    const update = (
+      call as {
+        update?: { sessionUpdate?: string; used?: number; size?: number };
+      }
+    ).update;
+    return update?.sessionUpdate === "usage_update" &&
+      typeof update.used === "number" &&
+      typeof update.size === "number"
+      ? [{ used: update.used, size: update.size }]
+      : [];
+  });
+}
+
 describe("ClaudeAcpAgent.prompt — streamed assistant text wiring", () => {
   beforeEach(() => {
     vi.clearAllMocks();
@@ -243,6 +284,48 @@ describe("ClaudeAcpAgent.prompt — streamed assistant text wiring", () => {
     ]);
   });
 
+  it("does not replace known context usage with incomplete gateway snapshots", async () => {
+    const { agent, client } = makeAgent();
+    const sessionId = "s-context-usage";
+    const { query, input } = installFakeSession(agent, sessionId);
+    const session = (
+      agent as unknown as {
+        session: { contextUsed?: number; lastContextWindowSize?: number };
+      }
+    ).session;
+    session.contextUsed = 434_000;
+    session.lastContextWindowSize = 1_000_000;
+    vi.mocked(query.getContextUsage).mockRejectedValue(
+      new Error("context usage unavailable"),
+    );
+
+    const promptPromise = agent.prompt({
+      sessionId,
+      prompt: [{ type: "text", text: "continue" }],
+    });
+    await tick();
+
+    await echoUserMessage(query, input);
+    await send(query, messageStart(sessionId, "msg-context"));
+    await send(query, compactBoundary(sessionId));
+    await send(
+      query,
+      assistantMessage(sessionId, "msg-context", "done", {
+        input_tokens: 440_000,
+        output_tokens: 100,
+        cache_read_input_tokens: 0,
+        cache_creation_input_tokens: 0,
+      }),
+    );
+    await send(query, resultSuccess(sessionId));
+    await promptPromise;
+
+    const updates = usageUpdates(client.sessionUpdate.mock.calls);
+    expect(updates.length).toBeGreaterThan(0);
+    expect(updates.every(({ used }) => used >= 434_000)).toBe(true);
+    expect(updates.every(({ size }) => size === 1_000_000)).toBe(true);
+  });
+
   it("keeps the original turn open until a pending steer is consumed", async () => {
     const { agent, client } = makeAgent();
     const sessionId = "s-steer-ordering";