Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions packages/api-client/src/spend-analysis.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,12 @@ export interface SpendAnalysisModelRow {
output_tokens: number;
}

export interface SpendAnalysisDayRow {
day: string;
event_count: number;
cost_usd: number;
}

export interface SpendAnalysisBreakdown<TRow> {
items: TRow[];
truncated: boolean;
Expand All @@ -40,6 +46,8 @@ export interface SpendAnalysisResponse {
by_product: SpendAnalysisBreakdown<SpendAnalysisProductRow>;
by_tool: SpendAnalysisBreakdown<SpendAnalysisToolRow>;
by_model: SpendAnalysisBreakdown<SpendAnalysisModelRow>;
// Optional until the backend by_day rollout reaches every deployment.
by_day?: SpendAnalysisBreakdown<SpendAnalysisDayRow>;
// `top_traces` is still in the backend response shape (always empty) per
// posthog/posthog#59796. Renderer code does not consume it; left out of the
// TS type so future readers see only what we actually use.
Expand Down
72 changes: 71 additions & 1 deletion packages/core/src/billing/spendAnalysisFormat.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
import { describe, expect, it } from "vitest";
import { formatTokens } from "./spendAnalysisFormat";
import {
fillSpendDays,
formatTokens,
type SpendAnalysisWindow,
windowToDateFrom,
windowToDays,
} from "./spendAnalysisFormat";

describe("formatTokens", () => {
it.each([
Expand All @@ -15,3 +21,67 @@ describe("formatTokens", () => {
expect(formatTokens(input)).toBe(expected);
});
});

describe("windowToDateFrom", () => {
it.each<[SpendAnalysisWindow, string]>([
["7d", "-6dStart"],
["30d", "-29dStart"],
["90d", "-89dStart"],
])("maps %s to the day-aligned %s", (window, expected) => {
expect(windowToDateFrom(window)).toBe(expected);
});
});

describe("windowToDays", () => {
it.each<[SpendAnalysisWindow, number]>([
["7d", 7],
["30d", 30],
["90d", 90],
])("maps %s to %d", (window, expected) => {
expect(windowToDays(window)).toBe(expected);
});
});

describe("fillSpendDays", () => {
it("zero-fills days without rows across the window", () => {
const filled = fillSpendDays(
[
{ day: "2026-07-01", event_count: 3, cost_usd: 1.5 },
{ day: "2026-07-03", event_count: 1, cost_usd: 0.25 },
],
"2026-07-01T00:00:00Z",
"2026-07-04T12:00:00Z",
);
expect(filled).toEqual([
{ day: "2026-07-01", event_count: 3, cost_usd: 1.5 },
{ day: "2026-07-02", event_count: 0, cost_usd: 0 },
{ day: "2026-07-03", event_count: 1, cost_usd: 0.25 },
{ day: "2026-07-04", event_count: 0, cost_usd: 0 },
]);
});

it("returns one zeroed row per day when there are no rows", () => {
const filled = fillSpendDays(
[],
"2026-07-01T06:00:00Z",
"2026-07-03T00:00:00Z",
);
expect(filled.map((d) => d.day)).toEqual([
"2026-07-01",
"2026-07-02",
"2026-07-03",
]);
expect(filled.every((d) => d.cost_usd === 0 && d.event_count === 0)).toBe(
true,
);
});

it("caps runaway windows instead of looping unbounded", () => {
const filled = fillSpendDays(
[],
"2020-01-01T00:00:00Z",
"2026-01-01T00:00:00Z",
);
expect(filled.length).toBe(100);
});
});
56 changes: 55 additions & 1 deletion packages/core/src/billing/spendAnalysisFormat.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import type { SpendAnalysisDayRow } from "./spendAnalysisTypes";

export function formatUsd(amount: number): string {
if (amount === 0) return "$0";
if (amount < 0.01) return "<$0.01";
Expand All @@ -12,12 +14,64 @@ export function formatTokens(n: number): string {
return n.toString();
}

export type SpendAnalysisWindow = "7d" | "30d" | "90d";

// Day-aligned (`dStart`) so an N-day window resolves to exactly N UTC calendar
// days including today, giving the daily chart exactly N bars.
export function windowToDateFrom(window: SpendAnalysisWindow): string {
return `-${windowToDays(window) - 1}dStart`;
}

export function windowToDays(window: SpendAnalysisWindow): number {
return Number.parseInt(window, 10);
}

export function windowDays(fromIso: string, toIso: string): number {
const fromMs = new Date(fromIso).getTime();
const toMs = new Date(toIso).getTime();
return Math.max(1, Math.round((toMs - fromMs) / (1000 * 60 * 60 * 24)));
// Ceil: a day-aligned window ending mid-day still covers N calendar days.
return Math.max(1, Math.ceil((toMs - fromMs) / DAY_MS));
}

export function formatWindow(fromIso: string, toIso: string): string {
return `${windowDays(fromIso, toIso)} days`;
}

const DAY_MS = 86_400_000;
const MAX_FILLED_DAYS = 100;

export interface SpendAnalysisFilledDay {
day: string;
event_count: number;
cost_usd: number;
}

export function fillSpendDays(
items: SpendAnalysisDayRow[],
fromIso: string,
toIso: string,
): SpendAnalysisFilledDay[] {
const byDay = new Map(items.map((row) => [row.day, row]));
const from = new Date(fromIso);
const start = Date.UTC(
from.getUTCFullYear(),
from.getUTCMonth(),
from.getUTCDate(),
);
const end = new Date(toIso).getTime();
const filled: SpendAnalysisFilledDay[] = [];
for (
let t = start;
t <= end && filled.length < MAX_FILLED_DAYS;
t += DAY_MS
) {
const day = new Date(t).toISOString().slice(0, 10);
const row = byDay.get(day);
filled.push({
day,
event_count: row?.event_count ?? 0,
cost_usd: row?.cost_usd ?? 0,
});
}
return filled;
}
2 changes: 1 addition & 1 deletion packages/core/src/billing/spendAnalysisPrompt.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ describe("buildAnalysisPrompt", () => {
it("instructs the agent not to query external data", () => {
const prompt = buildAnalysisPrompt(makeResponse());
expect(prompt).toContain(
"do **not** try to query PostHog AI observability or any external data source",
"Do **not** try to query PostHog AI observability or any external data source",
);
});
});
8 changes: 4 additions & 4 deletions packages/core/src/billing/spendAnalysisPrompt.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,13 @@ const PLAYBOOK = `## What to look at

Use this playbook to interpret the numbers above. Apply the levers in order of impact; not every lever applies to every user.

1. **Input tokens are the bill, not the tool calls themselves.** "Avg input" per tool is the context size dragged along on every call. A tool being expensive almost never means the tool itself is expensive — it means there were many calls each carrying a fat context. The biggest lever is conversation length, not which tool gets called: compact aggressively at logical checkpoints, start fresh sessions for unrelated tasks, avoid backtracking ("actually try X instead") because that re-runs all the prior context plus the alternative.
1. **Input tokens are the bill, not the tool calls themselves.** "Avg input" per tool is the context size dragged along on every call. A tool being expensive almost never means the tool itself is expensive. It means there were many calls each carrying a fat context. The biggest lever is conversation length, not which tool gets called: compact aggressively at logical checkpoints, start fresh sessions for unrelated tasks, avoid backtracking ("actually try X instead") because that re-runs all the prior context plus the alternative.

2. **Model choice.** Look at the "By model" table. If most generations are on the most expensive available model, switching the default to a mid-tier model and only escalating for genuinely hard reasoning is often the single biggest dollar saver. The cheapest tier is essentially free per call for routine work (run a test, check git status, grep for a string).

3. **Subagent hygiene.** The Agent / subagent tool typically has a high avg input because subagents inherit a brief plus the tool registry. They're worth their cost when they protect the main conversation from a long exploration; they're not worth it for "read one file" or "grep one pattern" — use the direct tool.
3. **Subagent hygiene.** The Agent / subagent tool typically has a high avg input because subagents inherit a brief plus the tool registry. They're worth their cost when they protect the main conversation from a long exploration; they're not worth it for "read one file" or "grep one pattern". Use the direct tool for those.

4. **No-tool replies.** If the "By tool" table has a "(no tool)" row, that's the model replying with pure text no action. Some of that is unavoidable (answering a question), some is the model thinking out loud or asking clarifying questions when it could just act. If this share is greater than ~10% of spend, more directive prompts ("Just do X" instead of "What do you think about X?") cut a round-trip per task.
4. **No-tool replies.** If the "By tool" table has a "(no tool)" row, that's the model replying with pure text, no action. Some of that is unavoidable (answering a question), some is the model thinking out loud or asking clarifying questions when it could just act. If this share is greater than ~10% of spend, more directive prompts ("Just do X" instead of "What do you think about X?") cut a round-trip per task.

5. **MCP / tool-registry overhead.** Tool calls that route through MCP (or any plugin layer that ships a tool registry on every turn) often show inflated avg input. If the user has many MCP servers enabled, pruning the ones they don't use shrinks the per-call overhead.

Expand Down Expand Up @@ -93,7 +93,7 @@ export function buildAnalysisPrompt(data: SpendAnalysisResponse): string {

return `Here is my PostHog Code LLM spend for the last ${windowLabel}. Help me understand what's driving the cost and what concrete changes I should make to reduce it.

Work only from the tables below — do **not** try to query PostHog AI observability or any external data source. The numbers here are everything you have. Rank advice by impact, lead with the biggest lever, and keep each suggestion concrete and actionable.
Work only from the tables below. Do **not** try to query PostHog AI observability or any external data source. The numbers here are everything you have. Rank advice by impact, lead with the biggest lever, and keep each suggestion concrete and actionable.

## My spend

Expand Down
8 changes: 8 additions & 0 deletions packages/core/src/billing/spendAnalysisTypes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,12 @@ export interface SpendAnalysisModelRow {
output_tokens: number;
}

export interface SpendAnalysisDayRow {
day: string;
event_count: number;
cost_usd: number;
}

export interface SpendAnalysisBreakdown<TRow> {
items: TRow[];
truncated: boolean;
Expand All @@ -40,4 +46,6 @@ export interface SpendAnalysisResponse {
by_product: SpendAnalysisBreakdown<SpendAnalysisProductRow>;
by_tool: SpendAnalysisBreakdown<SpendAnalysisToolRow>;
by_model: SpendAnalysisBreakdown<SpendAnalysisModelRow>;
// Optional until the backend by_day rollout reaches every deployment.
by_day?: SpendAnalysisBreakdown<SpendAnalysisDayRow>;
}
6 changes: 3 additions & 3 deletions packages/core/src/billing/spendSuggestions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,20 +23,20 @@ export function deriveSpendSuggestions(data: SpendAnalysisResponse): string[] {
const top = toolItems[0];
if (top.share_of_scoped > 0.35 && top.tool) {
suggestions.push(
`${top.tool} drives ${Math.round(top.share_of_scoped * 100)}% of your PostHog Code spend averaging ${formatTokens(top.avg_input_tokens)} input tokens per call.`,
`${top.tool} drives ${Math.round(top.share_of_scoped * 100)}% of your PostHog Code spend, averaging ${formatTokens(top.avg_input_tokens)} input tokens per call.`,
);
}
const noToolRow = toolItems.find((r) => r.tool === null);
if (noToolRow && noToolRow.share_of_scoped > 0.1) {
suggestions.push(
`${Math.round(noToolRow.share_of_scoped * 100)}% is spent on generations that take no tool action pure text replies. Consider tighter prompts or stopping the agent earlier.`,
`${Math.round(noToolRow.share_of_scoped * 100)}% is spent on generations that take no tool action: pure text replies. Consider tighter prompts or stopping the agent earlier.`,
);
}
}

if (suggestions.length === 0) {
suggestions.push(
"Your spend is fairly evenly distributed across tools — no single hotspot stands out.",
"Your spend is fairly evenly distributed across tools. No single hotspot stands out.",
);
}

Expand Down
14 changes: 12 additions & 2 deletions packages/shared/src/analytics-events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -626,6 +626,14 @@ export interface InboxReportScrolledProperties {
time_since_open_ms: number;
}

export interface UsageViewedProperties {
is_pro: boolean;
/** Monthly bucket percent (0-100), null when usage is unavailable. */
sustained_used_percent: number | null;
/** Daily bucket percent (0-100), null when usage is unavailable. */
burst_used_percent: number | null;
}

export interface SpendAnalysisTaskOpenedProperties {
/** Total LLM spend in USD across all products for the analysed window. */
total_cost_usd: number;
Expand Down Expand Up @@ -1129,7 +1137,8 @@ export const ANALYTICS_EVENTS = {
SCOUT_CHAT_STARTED: "Scout chat started",
SCOUT_ACTION: "Scout action",

// Spend analysis events
// Usage and spend analysis events
USAGE_VIEWED: "Usage viewed",
SPEND_ANALYSIS_TASK_OPENED: "Spend analysis task opened",

// Prompt history events
Expand Down Expand Up @@ -1281,7 +1290,8 @@ export type EventPropertyMap = {
[ANALYTICS_EVENTS.SCOUT_CHAT_STARTED]: ScoutChatStartedProperties;
[ANALYTICS_EVENTS.SCOUT_ACTION]: ScoutActionProperties;

// Spend analysis events
// Usage and spend analysis events
[ANALYTICS_EVENTS.USAGE_VIEWED]: UsageViewedProperties;
[ANALYTICS_EVENTS.SPEND_ANALYSIS_TASK_OPENED]: SpendAnalysisTaskOpenedProperties;

// Prompt history events
Expand Down
1 change: 1 addition & 0 deletions packages/shared/src/flags.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
export const BILLING_FLAG = "posthog-code-billing";
export const SPEND_ANALYSIS_FLAG = "posthog-code-spend-analysis";
export const EXPERIMENT_SUGGESTIONS_FLAG =
"posthog-code-experiment-suggestions";
export const SYNC_CLOUD_TASKS_FLAG = "posthog-code-sync-cloud-tasks";
Expand Down
Loading
Loading