Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions packages/core/src/setup/identifiers.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import type { SpendAnalysisResponse } from "@posthog/core/billing/spendAnalysisTypes";
import type { ActivityEntry } from "@posthog/core/setup/setupState";
import type { StaleFlagPayload } from "@posthog/core/setup/suggestions";
import type { DiscoveredTask } from "@posthog/core/setup/types";
Expand Down Expand Up @@ -61,6 +62,8 @@ export interface ISetupRunService {
repoPath: string,
): Promise<"initialized" | "not_installed" | "installed_no_init">;
findStaleFlagSuggestions(repoPath: string): Promise<StaleFlagPayload[]>;
/** The user's recent personal LLM spend (default window). Null when unauthenticated or the endpoint is unavailable. */
getSpendAnalysis(): Promise<SpendAnalysisResponse | null>;

/** Whether experiment-tier suggestions are enabled (feature flag or dev build). */
includeExperiments(): boolean;
Expand Down
44 changes: 44 additions & 0 deletions packages/core/src/setup/setupRunService.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ function makePort(overrides: Partial<ISetupRunService> = {}): ISetupRunService {
subscribeSessionEvents: vi.fn(() => ({ unsubscribe: () => {} })),
detectPosthogInstallState: vi.fn(async () => "not_installed" as const),
findStaleFlagSuggestions: vi.fn(async () => []),
getSpendAnalysis: vi.fn(async () => null),
includeExperiments: vi.fn(() => false),
trackDiscoveryStarted: vi.fn(),
trackDiscoveryCompleted: vi.fn(),
Expand Down Expand Up @@ -144,6 +145,49 @@ describe("SetupRunService enricher", () => {
expect(store.getEnricherStatus(REPO)).toBe("done");
});

it("adds the ai-usage suggestion when spend is above threshold, regardless of install state", async () => {
const port = makePort({
getSpendAnalysis: vi.fn(async () => ({
summary: {
date_from: "2026-06-07T00:00:00Z",
date_to: "2026-07-07T00:00:00Z",
product: null,
total_cost_usd: 50,
event_count: 100,
scoped_cost_usd: 40,
scoped_event_count: 80,
},
by_product: { items: [], truncated: false },
by_tool: { items: [], truncated: false },
by_model: { items: [], truncated: false },
})),
});
const service = new SetupRunService(port, store, noopLogger);

service.startEnricherForRepo(REPO);
await flush();

const ids = store.discoveredTasks.map((t) => t.id);
expect(ids).toContain("ai-usage-optimization");
expect(store.getEnricherStatus(REPO)).toBe("done");
});

it("still completes enrichment when spend analysis fails", async () => {
const port = makePort({
getSpendAnalysis: vi.fn(async () => {
throw new Error("spend endpoint unavailable");
}),
});
const service = new SetupRunService(port, store, noopLogger);

service.startEnricherForRepo(REPO);
await flush();

const ids = store.discoveredTasks.map((t) => t.id);
expect(ids).not.toContain("ai-usage-optimization");
expect(store.getEnricherStatus(REPO)).toBe("done");
});

it("marks enrichment failed when install-state detection throws", async () => {
const port = makePort({
detectPosthogInstallState: vi.fn(async () => {
Expand Down
19 changes: 19 additions & 0 deletions packages/core/src/setup/setupRunService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import {
nextActivityId,
} from "@posthog/core/setup/sessionUpdate";
import {
buildAiUsageSuggestion,
buildPosthogSetupSuggestion,
buildSdkHealthSuggestion,
buildStaleFlagSuggestion,
Expand Down Expand Up @@ -117,6 +118,7 @@ export class SetupRunService {
repoPath: directory,
});
}
await this.injectAiUsageSuggestion(directory);
this.store.completeEnrichment(directory);
} catch (err) {
this.logger.warn("Enricher run failed", { error: err });
Expand All @@ -140,6 +142,23 @@ export class SetupRunService {
}
}

// Independent of the repo's PostHog install state — the user's own spend is
// what makes the suggestion relevant, so it runs on every enricher pass.
private async injectAiUsageSuggestion(directory: string): Promise<void> {
try {
const spend = await this.port.getSpendAnalysis();
if (!spend) return;
const suggestion = buildAiUsageSuggestion(spend);
if (!suggestion) return;
this.store.addEnricherSuggestionIfMissing({
...suggestion,
repoPath: directory,
});
} catch (err) {
this.logger.warn("Failed to build AI usage suggestion", { error: err });
}
}

private async runDiscovery(directory: string): Promise<void> {
const abort = new AbortController();
const discoveryStartedAt = Date.now();
Expand Down
70 changes: 70 additions & 0 deletions packages/core/src/setup/suggestions.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import type { SpendAnalysisResponse } from "@posthog/core/billing/spendAnalysisTypes";
import {
AI_USAGE_MIN_SPEND_USD,
buildAiUsageSuggestion,
buildPosthogSetupSuggestion,
buildSdkHealthSuggestion,
buildStaleFlagSuggestion,
Expand Down Expand Up @@ -63,6 +66,73 @@ describe("buildSdkHealthSuggestion", () => {
});
});

describe("buildAiUsageSuggestion", () => {
function makeSpend(
overrides: Partial<SpendAnalysisResponse["summary"]> = {},
topTool?: Partial<SpendAnalysisResponse["by_tool"]["items"][number]>,
): SpendAnalysisResponse {
return {
summary: {
date_from: "2026-06-07T00:00:00Z",
date_to: "2026-07-07T00:00:00Z",
product: null,
total_cost_usd: 120,
event_count: 5000,
scoped_cost_usd: 90,
scoped_event_count: 4000,
...overrides,
},
by_product: { items: [], truncated: false },
by_tool: {
items: topTool
? [
{
tool: "Bash",
generation_count: 900,
cost_usd: 45,
share_of_scoped: 0.5,
avg_input_tokens: 42_000,
...topTool,
},
]
: [],
truncated: false,
},
by_model: { items: [], truncated: false },
};
}

it("returns null below the minimum spend threshold", () => {
const data = makeSpend({ scoped_cost_usd: AI_USAGE_MIN_SPEND_USD - 1 });
expect(buildAiUsageSuggestion(data)).toBeNull();
});

it("has a stable id so dismissal sticks across re-runs", () => {
expect(buildAiUsageSuggestion(makeSpend())?.id).toBe(
"ai-usage-optimization",
);
});

it("summarizes spend and window in the description", () => {
const task = buildAiUsageSuggestion(makeSpend());
expect(task?.description).toContain("$90.00");
expect(task?.description).toContain("30 days");
});

it("calls out the top tool when it dominates spend", () => {
const task = buildAiUsageSuggestion(makeSpend({}, {}));
expect(task?.description).toContain("Bash alone drives 50%");
expect(task?.description).toContain("42k input tokens per call");
});

it("omits the hotspot line when no tool dominates", () => {
const task = buildAiUsageSuggestion(
makeSpend({}, { share_of_scoped: 0.2 }),
);
expect(task?.description).not.toContain("alone drives");
});
});

describe("buildPosthogSetupSuggestion", () => {
it("returns the install suggestion when not installed", () => {
const task = buildPosthogSetupSuggestion("not_installed");
Expand Down
44 changes: 44 additions & 0 deletions packages/core/src/setup/suggestions.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
import {
formatTokens,
formatUsd,
formatWindow,
} from "@posthog/core/billing/spendAnalysisFormat";
import type { SpendAnalysisResponse } from "@posthog/core/billing/spendAnalysisTypes";
import type { DiscoveredTask } from "@posthog/core/setup/types";

export interface StaleFlagPayload {
Expand Down Expand Up @@ -32,6 +38,44 @@ export function buildStaleFlagSuggestion(
};
}

// Below this much PostHog Code spend in the analysis window there's nothing
// meaningful to optimize, so the suggestion stays hidden rather than nagging
// light users.
export const AI_USAGE_MIN_SPEND_USD = 5;

// Share of scoped spend a single tool must reach before it's called out as a
// hotspot in the suggestion description.
const AI_USAGE_HOTSPOT_SHARE = 0.35;

export function buildAiUsageSuggestion(
data: SpendAnalysisResponse,
): DiscoveredTask | null {
const { summary } = data;
if (summary.scoped_cost_usd < AI_USAGE_MIN_SPEND_USD) return null;

const window = formatWindow(summary.date_from, summary.date_to);
const topTool = data.by_tool.items[0];
const hotspot =
topTool?.tool && topTool.share_of_scoped >= AI_USAGE_HOTSPOT_SHARE
? ` ${topTool.tool} alone drives ${Math.round(topTool.share_of_scoped * 100)}% of that, averaging ${formatTokens(topTool.avg_input_tokens)} input tokens per call.`
: "";

return {
// Stable id so dismissal sticks across re-runs.
id: "ai-usage-optimization",
source: "enricher",
category: "ai_usage",
title: "Optimize your AI usage",
description: `You've spent ${formatUsd(summary.scoped_cost_usd)} on PostHog Code in the last ${window}.${hotspot} Repeated workflows, re-explained context, and recurring one-off analyses usually hide easy savings.`,
impact:
"Token spend compounds quietly: the same context re-sent every session, the same analysis re-run every week. Capturing repeated work as skills and scheduled reports cuts cost — and gets you answers without having to ask.",
recommendation:
'Click "Implement as new task" — the agent audits your recent sessions and memory files, then proposes concrete changes ranked by impact: new skills for repeated workflows, memory-file edits, and scheduled reports for recurring questions.',
prompt:
"Audit how I've been using PostHog Code and find ways to get better results for fewer tokens. Review my recent sessions for: repeated workflows worth capturing as a reusable skill, recurring questions better served by a scheduled scout or AI report, memory files (CLAUDE.md, AGENTS.md) that are bloated, stale, or missing context I keep re-explaining, and prompts that ran long or retried because they were under-specified. Deliver a short report of findings ranked by impact — each with the evidence behind it and the estimated saving — then offer to implement the top quick wins.",
};
}

export function buildSdkHealthSuggestion(): DiscoveredTask {
return {
id: "posthog-sdk-health",
Expand Down
3 changes: 2 additions & 1 deletion packages/core/src/setup/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ export interface DiscoveredTask {
| "event_tracking"
| "funnel"
| "posthog_setup"
| "experiment";
| "experiment"
| "ai_usage";
source: DiscoveredTaskSource;
file?: string;
lineHint?: number;
Expand Down
3 changes: 2 additions & 1 deletion packages/shared/src/analytics-events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -482,7 +482,8 @@ type SetupDiscoveredTaskCategory =
| "event_tracking"
| "funnel"
| "posthog_setup"
| "experiment";
| "experiment"
| "ai_usage";

export interface SetupDiscoveryStartedProperties {
discovery_task_id: string;
Expand Down
2 changes: 2 additions & 0 deletions packages/ui/src/features/setup/categoryConfig.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
Flag,
Flask,
Funnel,
Gauge,
Lightning,
Lock,
Sparkle,
Expand Down Expand Up @@ -39,6 +40,7 @@ export const CATEGORY_CONFIG: Record<
funnel: { icon: Funnel, color: "violet", label: "Funnel" },
posthog_setup: { icon: Sparkle, color: "violet", label: "PostHog setup" },
experiment: { icon: Flask, color: "purple", label: "Experiment" },
ai_usage: { icon: Gauge, color: "cyan", label: "AI usage" },
};

// Fallback when a `DiscoveredTask.category` somehow doesn't match the map
Expand Down
10 changes: 10 additions & 0 deletions packages/ui/src/features/setup/setupRunServiceImpl.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import type { PostHogAPIClient } from "@posthog/api-client/posthog-client";
import type { SpendAnalysisResponse } from "@posthog/core/billing/spendAnalysisTypes";
import type {
DiscoveryFailureReason,
DiscoverySignalSource,
Expand Down Expand Up @@ -155,6 +156,15 @@ export class SetupRunServiceImpl implements ISetupRunService {
});
}

// Uses a fresh authenticated client rather than `this.client`: the enricher
// path runs before (and independently of) getDiscoveryContext().
async getSpendAnalysis(): Promise<SpendAnalysisResponse | null> {
const authState = await fetchAuthState();
const client = createAuthenticatedClient(authState);
if (!client) return null;
return client.getPersonalSpendAnalysis();
}

includeExperiments(): boolean {
return (
resolveService<FeatureFlags>(FEATURE_FLAGS).isEnabled(
Expand Down
Loading