Skip to content

Commit 9f09dbc

Browse files
authored
feat(agent): report per-run token usage to TaskRun.state
The agent-server now accumulates each settled turn's ACP usage (input/output/cache read/cache write/thought tokens) into run-level totals and PATCHes the snapshot into TaskRun.state.token_usage after every turn. Best-effort and adapter-agnostic: both the claude and codex adapters resolve prompt() with usage. This gives cloud runs a persisted, queryable token expenditure record even when a run never completes cleanly. Generated-By: PostHog Code Task-Id: 2d6eed7f-9209-4d1f-be54-ae3e22bf2990
1 parent 12cfd51 commit 9f09dbc

4 files changed

Lines changed: 204 additions & 0 deletions

File tree

packages/agent/src/server/agent-server.test.ts

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -685,6 +685,78 @@ describe("AgentServer HTTP Mode", () => {
685685
expect(testServer.posthogAPI.updateTaskRun).not.toHaveBeenCalled();
686686
});
687687

688+
function createUsageTestServer() {
689+
const testServer = new AgentServer({
690+
port,
691+
jwtPublicKey: TEST_PUBLIC_KEY,
692+
repositoryPath: repo.path,
693+
apiUrl: "http://localhost:8000",
694+
apiKey: "test-api-key",
695+
projectId: 1,
696+
mode: "interactive",
697+
taskId: "test-task-id",
698+
runId: "test-run-id",
699+
}) as unknown as {
700+
session: { payload: JwtPayload } | null;
701+
posthogAPI: { updateTaskRun: ReturnType<typeof vi.fn> };
702+
recordTurnUsage(usage: unknown): void;
703+
};
704+
testServer.posthogAPI = { updateTaskRun: vi.fn(async () => ({})) };
705+
testServer.session = {
706+
payload: {
707+
run_id: "run-1",
708+
task_id: "task-1",
709+
team_id: 1,
710+
user_id: 1,
711+
distinct_id: "distinct-id",
712+
mode: "interactive",
713+
},
714+
};
715+
return testServer;
716+
}
717+
718+
it("reports cumulative run token usage into TaskRun.state after each settled turn", () => {
719+
const testServer = createUsageTestServer();
720+
const turnUsage = {
721+
inputTokens: 100,
722+
outputTokens: 50,
723+
cachedReadTokens: 10,
724+
cachedWriteTokens: 5,
725+
totalTokens: 165,
726+
};
727+
728+
testServer.recordTurnUsage(turnUsage);
729+
testServer.recordTurnUsage(turnUsage);
730+
731+
expect(testServer.posthogAPI.updateTaskRun).toHaveBeenCalledTimes(2);
732+
// The second report carries run-cumulative totals, not per-turn figures.
733+
expect(testServer.posthogAPI.updateTaskRun).toHaveBeenLastCalledWith(
734+
"task-1",
735+
"run-1",
736+
{
737+
state: {
738+
token_usage: {
739+
input_tokens: 200,
740+
output_tokens: 100,
741+
cache_read_tokens: 20,
742+
cache_write_tokens: 10,
743+
thought_tokens: 0,
744+
total_tokens: 330,
745+
turns: 2,
746+
},
747+
},
748+
},
749+
);
750+
});
751+
752+
it("does not report anything when a turn settles without usage", () => {
753+
const testServer = createUsageTestServer();
754+
755+
testServer.recordTurnUsage(undefined);
756+
757+
expect(testServer.posthogAPI.updateTaskRun).not.toHaveBeenCalled();
758+
});
759+
688760
function createFailureTestServer() {
689761
const appendRawLine = vi.fn();
690762
const testServer = new AgentServer({

packages/agent/src/server/agent-server.ts

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ import {
7777
} from "./cloud-prompt";
7878
import { TaskRunEventStreamSender } from "./event-stream-sender";
7979
import { type JwtPayload, JwtValidationError, validateJwt } from "./jwt";
80+
import { RunUsageAccumulator } from "./run-usage";
8081
import {
8182
handoffLocalGitStateSchema,
8283
jsonRpcRequestSchema,
@@ -308,6 +309,7 @@ export class AgentServer {
308309
private eventStreamSender: TaskRunEventStreamSender | null = null;
309310
private questionRelayedToSlack = false;
310311
private adapterEmittedTurnComplete = false;
312+
private readonly runUsage = new RunUsageAccumulator();
311313
private detectedPrUrl: string | null = null;
312314
// Reset per session. `evaluatedPrUrls` dedupes per URL; `prAttributionChain` serializes
313315
// attributions so the most recently created PR in a run wins.
@@ -947,6 +949,7 @@ export class AgentServer {
947949
void this.syncCloudBranchMetadata(this.session.payload);
948950
}
949951

952+
this.recordTurnUsage(result.usage);
950953
this.broadcastTurnComplete(result.stopReason);
951954

952955
if (result.stopReason === "end_turn") {
@@ -1659,6 +1662,7 @@ export class AgentServer {
16591662
void this.syncCloudBranchMetadata(payload);
16601663
}
16611664

1665+
this.recordTurnUsage(result.usage);
16621666
this.broadcastTurnComplete(result.stopReason);
16631667

16641668
if (result.stopReason === "end_turn") {
@@ -1810,6 +1814,7 @@ export class AgentServer {
18101814
void this.syncCloudBranchMetadata(payload);
18111815
}
18121816

1817+
this.recordTurnUsage(result.usage);
18131818
this.broadcastTurnComplete(result.stopReason);
18141819

18151820
if (result.stopReason === "end_turn") {
@@ -3611,6 +3616,24 @@ ${signedCommitInstructions}${prLinkInstructions}${shellEfficiencyInstructions}
36113616
return result.success ? result.data : null;
36123617
}
36133618

3619+
/**
3620+
* Accumulates a settled turn's token usage into the run total and reports it
3621+
* to the backend, merged into `TaskRun.state.token_usage`. Best-effort: a
3622+
* reporting failure must never affect the turn outcome.
3623+
*/
3624+
private recordTurnUsage(usage: PromptResponse["usage"]): void {
3625+
if (!this.runUsage.add(usage)) return;
3626+
const payload = this.session?.payload;
3627+
if (!payload) return;
3628+
void this.posthogAPI
3629+
.updateTaskRun(payload.task_id, payload.run_id, {
3630+
state: { token_usage: this.runUsage.snapshot() },
3631+
})
3632+
.catch((error) => {
3633+
this.logger.warn("Failed to report run token usage", error);
3634+
});
3635+
}
3636+
36143637
private broadcastTurnComplete(stopReason: string): void {
36153638
if (!this.session) return;
36163639
if (this.adapterEmittedTurnComplete) {
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
import { describe, expect, test } from "vitest";
2+
import { RunUsageAccumulator } from "./run-usage";
3+
4+
describe("RunUsageAccumulator", () => {
5+
test("accumulates across turns and defaults nullable ACP fields to 0", () => {
6+
const acc = new RunUsageAccumulator();
7+
8+
// Claude-shaped turn: cache components present, no thought tokens.
9+
expect(
10+
acc.add({
11+
inputTokens: 100,
12+
outputTokens: 50,
13+
cachedReadTokens: 10,
14+
cachedWriteTokens: 5,
15+
totalTokens: 165,
16+
}),
17+
).toBe(true);
18+
// Codex-shaped turn: null cache writes, reasoning as thought tokens.
19+
expect(
20+
acc.add({
21+
inputTokens: 200,
22+
outputTokens: 80,
23+
cachedReadTokens: null,
24+
cachedWriteTokens: null,
25+
thoughtTokens: 40,
26+
totalTokens: 320,
27+
}),
28+
).toBe(true);
29+
30+
expect(acc.snapshot()).toEqual({
31+
input_tokens: 300,
32+
output_tokens: 130,
33+
cache_read_tokens: 10,
34+
cache_write_tokens: 5,
35+
thought_tokens: 40,
36+
total_tokens: 485,
37+
turns: 2,
38+
});
39+
});
40+
41+
test.each([[null], [undefined]])(
42+
"ignores a turn that settles with %s usage",
43+
(usage) => {
44+
const acc = new RunUsageAccumulator();
45+
expect(acc.add(usage)).toBe(false);
46+
expect(acc.snapshot().turns).toBe(0);
47+
},
48+
);
49+
50+
test("snapshot is a copy — later turns don't mutate earlier snapshots", () => {
51+
const acc = new RunUsageAccumulator();
52+
acc.add({ inputTokens: 1, outputTokens: 1, totalTokens: 2 });
53+
const first = acc.snapshot();
54+
acc.add({ inputTokens: 1, outputTokens: 1, totalTokens: 2 });
55+
expect(first.turns).toBe(1);
56+
expect(acc.snapshot().turns).toBe(2);
57+
});
58+
});
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
import type { Usage } from "@agentclientprotocol/sdk";
2+
3+
/**
4+
* Cumulative token usage for a task run, shaped for `TaskRun.state.token_usage`
5+
* (snake_case, matching the backend's state conventions). `turns` counts the
6+
* settled turns that contributed usage, giving consumers a per-turn denominator.
7+
*/
8+
export type RunTokenUsage = {
9+
input_tokens: number;
10+
output_tokens: number;
11+
cache_read_tokens: number;
12+
cache_write_tokens: number;
13+
thought_tokens: number;
14+
total_tokens: number;
15+
turns: number;
16+
};
17+
18+
/**
19+
* Accumulates per-turn ACP `Usage` into run-level totals. The ACP usage fields
20+
* are optional and nullable, so every component defaults to 0 to keep the sums
21+
* numeric across adapters (codex reports no cache writes, claude no thought
22+
* tokens on some models).
23+
*/
24+
export class RunUsageAccumulator {
25+
private totals: RunTokenUsage = {
26+
input_tokens: 0,
27+
output_tokens: 0,
28+
cache_read_tokens: 0,
29+
cache_write_tokens: 0,
30+
thought_tokens: 0,
31+
total_tokens: 0,
32+
turns: 0,
33+
};
34+
35+
/** Adds a settled turn's usage. Returns false when there was nothing to add. */
36+
add(usage: Usage | null | undefined): boolean {
37+
if (!usage) return false;
38+
this.totals.input_tokens += usage.inputTokens ?? 0;
39+
this.totals.output_tokens += usage.outputTokens ?? 0;
40+
this.totals.cache_read_tokens += usage.cachedReadTokens ?? 0;
41+
this.totals.cache_write_tokens += usage.cachedWriteTokens ?? 0;
42+
this.totals.thought_tokens += usage.thoughtTokens ?? 0;
43+
this.totals.total_tokens += usage.totalTokens ?? 0;
44+
this.totals.turns += 1;
45+
return true;
46+
}
47+
48+
snapshot(): RunTokenUsage {
49+
return { ...this.totals };
50+
}
51+
}

0 commit comments

Comments
 (0)