From 47b176c1393ea22a5cf5aa090497a4783b2df8f0 Mon Sep 17 00:00:00 2001 From: Vasco de Krijger <66254686+vdekrijger@users.noreply.github.com> Date: Tue, 7 Jul 2026 10:35:25 +0200 Subject: [PATCH 1/3] feat(canvas): add "Optimize your AI usage" new-task suggestion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a starter-prompt card to CHANNEL_TASK_SUGGESTIONS that kicks off a task to review recent agent/task usage and find efficiency improvements — reusable skills, memory-file tuning, and token-cost reductions. Generated-By: PostHog Code Task-Id: e5484dae-c3c0-473b-a1ea-12f4cc3d9ab5 --- .../ui/src/features/canvas/channelTaskSuggestions.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/ui/src/features/canvas/channelTaskSuggestions.ts b/packages/ui/src/features/canvas/channelTaskSuggestions.ts index 5529b1447e..4d38dd2bdf 100644 --- a/packages/ui/src/features/canvas/channelTaskSuggestions.ts +++ b/packages/ui/src/features/canvas/channelTaskSuggestions.ts @@ -6,6 +6,7 @@ import { Cube, CurrencyDollar, Flask, + Gauge, Wrench, } from "@phosphor-icons/react"; import type { SuggestedPrompt } from "@posthog/ui/features/task-detail/components/SuggestedPromptCard"; @@ -72,6 +73,15 @@ export const CHANNEL_TASK_SUGGESTIONS: SuggestedPrompt[] = [ prompt: "Interpret the results of an experiment — explain what the metrics show, whether it's significant, and what to do next.\n\n\nUser input:\n- Experiment name or key:\n- What decision are you trying to make (optional):", }, + { + label: "Optimize your AI usage", + description: "Cut token spend with skills & memory tuning", + icon: Gauge, + color: "cyan", + mode: "auto", + prompt: + "Review my recent agent and task usage to find ways to work more efficiently. Look for repeated work a reusable skill would capture, memory files that could be tightened or split, and prompt patterns that burn tokens. Recommend concrete improvements — new skills, memory edits, or best practices — to reduce token cost.\n\n\nUser input:\n- Time period to review (optional):\n- What to prioritize (cost, speed, accuracy — optional):", + }, { label: "Fix a bug", description: "Track down and fix a problem in the code", From 87f3e9046f34d900d8056a584eab54bd673eee43 Mon Sep 17 00:00:00 2001 From: Vasco de Krijger <66254686+vdekrijger@users.noreply.github.com> Date: Tue, 7 Jul 2026 10:41:45 +0200 Subject: [PATCH 2/3] refactor(canvas): sharpen the "Optimize your AI usage" starter prompt Name the concrete evidence to review (repeated workflows, recurring questions, memory bloat, under-specified prompts), point each finding at the right PostHog Code feature (skills, scouts, scheduled AI reports, memory edits), and shape the deliverable: an impact-ranked report with evidence and estimated savings, plus an offer to implement quick wins. Generated-By: PostHog Code Task-Id: e5484dae-c3c0-473b-a1ea-12f4cc3d9ab5 --- packages/ui/src/features/canvas/channelTaskSuggestions.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ui/src/features/canvas/channelTaskSuggestions.ts b/packages/ui/src/features/canvas/channelTaskSuggestions.ts index 4d38dd2bdf..70af77ad9f 100644 --- a/packages/ui/src/features/canvas/channelTaskSuggestions.ts +++ b/packages/ui/src/features/canvas/channelTaskSuggestions.ts @@ -75,12 +75,12 @@ export const CHANNEL_TASK_SUGGESTIONS: SuggestedPrompt[] = [ }, { label: "Optimize your AI usage", - description: "Cut token spend with skills & memory tuning", + description: "Turn repeated work into skills, scouts & savings", icon: Gauge, color: "cyan", mode: "auto", prompt: - "Review my recent agent and task usage to find ways to work more efficiently. Look for repeated work a reusable skill would capture, memory files that could be tightened or split, and prompt patterns that burn tokens. Recommend concrete improvements — new skills, memory edits, or best practices — to reduce token cost.\n\n\nUser input:\n- Time period to review (optional):\n- What to prioritize (cost, speed, accuracy — optional):", + "Audit how I've been using PostHog Code and find ways to get better results for fewer tokens. Review my recent tasks and sessions for: repeated workflows worth capturing as a reusable skill, recurring questions better served by a scheduled scout or AI report, memory files that are bloated, stale, or missing context I keep re-explaining, and prompts that ran long or retried because they were under-specified. Deliver a short report of findings ranked by impact — each with the evidence behind it and the estimated saving — then offer to implement the top quick wins.\n\n\nUser input:\n- Time period to review (optional):\n- What to prioritize (token cost, speed, quality — optional):", }, { label: "Fix a bug", From 1bdd45cc9dc3b3c4d67805dbfbade63f3017c80a Mon Sep 17 00:00:00 2001 From: Vasco de Krijger <66254686+vdekrijger@users.noreply.github.com> Date: Tue, 7 Jul 2026 11:00:30 +0200 Subject: [PATCH 3/3] feat(setup): spend-driven "Optimize your AI usage" discovered task MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the static channel starter card with a real enricher-produced discovered task. On each enricher pass the setup orchestration fetches the user's personal LLM spend analysis and, when recent PostHog Code spend crosses a threshold, surfaces an "Optimize your AI usage" suggestion in the new-task panel — including the dominant tool hotspot as evidence. The one-click prompt audits recent sessions for skill-worthy repeated workflows, memory-file bloat, and recurring questions better served by scheduled reports. Adds an ai_usage category (enricher-only, like posthog_setup), a getSpendAnalysis port method implemented via the authenticated PostHog API client, and unit tests for the builder and orchestration paths. Generated-By: PostHog Code Task-Id: e5484dae-c3c0-473b-a1ea-12f4cc3d9ab5 --- packages/core/src/setup/identifiers.ts | 3 + .../core/src/setup/setupRunService.test.ts | 44 ++++++++++++ packages/core/src/setup/setupRunService.ts | 19 +++++ packages/core/src/setup/suggestions.test.ts | 70 +++++++++++++++++++ packages/core/src/setup/suggestions.ts | 44 ++++++++++++ packages/core/src/setup/types.ts | 3 +- packages/shared/src/analytics-events.ts | 3 +- .../features/canvas/channelTaskSuggestions.ts | 10 --- .../ui/src/features/setup/categoryConfig.ts | 2 + .../src/features/setup/setupRunServiceImpl.ts | 10 +++ 10 files changed, 196 insertions(+), 12 deletions(-) diff --git a/packages/core/src/setup/identifiers.ts b/packages/core/src/setup/identifiers.ts index 924ce97203..378d395ce9 100644 --- a/packages/core/src/setup/identifiers.ts +++ b/packages/core/src/setup/identifiers.ts @@ -1,3 +1,4 @@ +import type { SpendAnalysisResponse } from "@posthog/core/billing/spendAnalysisTypes"; import type { ActivityEntry } from "@posthog/core/setup/setupState"; import type { StaleFlagPayload } from "@posthog/core/setup/suggestions"; import type { DiscoveredTask } from "@posthog/core/setup/types"; @@ -61,6 +62,8 @@ export interface ISetupRunService { repoPath: string, ): Promise<"initialized" | "not_installed" | "installed_no_init">; findStaleFlagSuggestions(repoPath: string): Promise; + /** The user's recent personal LLM spend (default window). Null when unauthenticated or the endpoint is unavailable. */ + getSpendAnalysis(): Promise; /** Whether experiment-tier suggestions are enabled (feature flag or dev build). */ includeExperiments(): boolean; diff --git a/packages/core/src/setup/setupRunService.test.ts b/packages/core/src/setup/setupRunService.test.ts index 89c205af1c..05ea66595d 100644 --- a/packages/core/src/setup/setupRunService.test.ts +++ b/packages/core/src/setup/setupRunService.test.ts @@ -91,6 +91,7 @@ function makePort(overrides: Partial = {}): ISetupRunService { subscribeSessionEvents: vi.fn(() => ({ unsubscribe: () => {} })), detectPosthogInstallState: vi.fn(async () => "not_installed" as const), findStaleFlagSuggestions: vi.fn(async () => []), + getSpendAnalysis: vi.fn(async () => null), includeExperiments: vi.fn(() => false), trackDiscoveryStarted: vi.fn(), trackDiscoveryCompleted: vi.fn(), @@ -144,6 +145,49 @@ describe("SetupRunService enricher", () => { expect(store.getEnricherStatus(REPO)).toBe("done"); }); + it("adds the ai-usage suggestion when spend is above threshold, regardless of install state", async () => { + const port = makePort({ + getSpendAnalysis: vi.fn(async () => ({ + summary: { + date_from: "2026-06-07T00:00:00Z", + date_to: "2026-07-07T00:00:00Z", + product: null, + total_cost_usd: 50, + event_count: 100, + scoped_cost_usd: 40, + scoped_event_count: 80, + }, + by_product: { items: [], truncated: false }, + by_tool: { items: [], truncated: false }, + by_model: { items: [], truncated: false }, + })), + }); + const service = new SetupRunService(port, store, noopLogger); + + service.startEnricherForRepo(REPO); + await flush(); + + const ids = store.discoveredTasks.map((t) => t.id); + expect(ids).toContain("ai-usage-optimization"); + expect(store.getEnricherStatus(REPO)).toBe("done"); + }); + + it("still completes enrichment when spend analysis fails", async () => { + const port = makePort({ + getSpendAnalysis: vi.fn(async () => { + throw new Error("spend endpoint unavailable"); + }), + }); + const service = new SetupRunService(port, store, noopLogger); + + service.startEnricherForRepo(REPO); + await flush(); + + const ids = store.discoveredTasks.map((t) => t.id); + expect(ids).not.toContain("ai-usage-optimization"); + expect(store.getEnricherStatus(REPO)).toBe("done"); + }); + it("marks enrichment failed when install-state detection throws", async () => { const port = makePort({ detectPosthogInstallState: vi.fn(async () => { diff --git a/packages/core/src/setup/setupRunService.ts b/packages/core/src/setup/setupRunService.ts index 229a3977ab..84d4bfd424 100644 --- a/packages/core/src/setup/setupRunService.ts +++ b/packages/core/src/setup/setupRunService.ts @@ -10,6 +10,7 @@ import { nextActivityId, } from "@posthog/core/setup/sessionUpdate"; import { + buildAiUsageSuggestion, buildPosthogSetupSuggestion, buildSdkHealthSuggestion, buildStaleFlagSuggestion, @@ -117,6 +118,7 @@ export class SetupRunService { repoPath: directory, }); } + await this.injectAiUsageSuggestion(directory); this.store.completeEnrichment(directory); } catch (err) { this.logger.warn("Enricher run failed", { error: err }); @@ -140,6 +142,23 @@ export class SetupRunService { } } + // Independent of the repo's PostHog install state — the user's own spend is + // what makes the suggestion relevant, so it runs on every enricher pass. + private async injectAiUsageSuggestion(directory: string): Promise { + try { + const spend = await this.port.getSpendAnalysis(); + if (!spend) return; + const suggestion = buildAiUsageSuggestion(spend); + if (!suggestion) return; + this.store.addEnricherSuggestionIfMissing({ + ...suggestion, + repoPath: directory, + }); + } catch (err) { + this.logger.warn("Failed to build AI usage suggestion", { error: err }); + } + } + private async runDiscovery(directory: string): Promise { const abort = new AbortController(); const discoveryStartedAt = Date.now(); diff --git a/packages/core/src/setup/suggestions.test.ts b/packages/core/src/setup/suggestions.test.ts index 610f19ff0e..b9fd9fd166 100644 --- a/packages/core/src/setup/suggestions.test.ts +++ b/packages/core/src/setup/suggestions.test.ts @@ -1,4 +1,7 @@ +import type { SpendAnalysisResponse } from "@posthog/core/billing/spendAnalysisTypes"; import { + AI_USAGE_MIN_SPEND_USD, + buildAiUsageSuggestion, buildPosthogSetupSuggestion, buildSdkHealthSuggestion, buildStaleFlagSuggestion, @@ -63,6 +66,73 @@ describe("buildSdkHealthSuggestion", () => { }); }); +describe("buildAiUsageSuggestion", () => { + function makeSpend( + overrides: Partial = {}, + topTool?: Partial, + ): SpendAnalysisResponse { + return { + summary: { + date_from: "2026-06-07T00:00:00Z", + date_to: "2026-07-07T00:00:00Z", + product: null, + total_cost_usd: 120, + event_count: 5000, + scoped_cost_usd: 90, + scoped_event_count: 4000, + ...overrides, + }, + by_product: { items: [], truncated: false }, + by_tool: { + items: topTool + ? [ + { + tool: "Bash", + generation_count: 900, + cost_usd: 45, + share_of_scoped: 0.5, + avg_input_tokens: 42_000, + ...topTool, + }, + ] + : [], + truncated: false, + }, + by_model: { items: [], truncated: false }, + }; + } + + it("returns null below the minimum spend threshold", () => { + const data = makeSpend({ scoped_cost_usd: AI_USAGE_MIN_SPEND_USD - 1 }); + expect(buildAiUsageSuggestion(data)).toBeNull(); + }); + + it("has a stable id so dismissal sticks across re-runs", () => { + expect(buildAiUsageSuggestion(makeSpend())?.id).toBe( + "ai-usage-optimization", + ); + }); + + it("summarizes spend and window in the description", () => { + const task = buildAiUsageSuggestion(makeSpend()); + expect(task?.description).toContain("$90.00"); + expect(task?.description).toContain("30 days"); + }); + + it("calls out the top tool when it dominates spend", () => { + const task = buildAiUsageSuggestion(makeSpend({}, {})); + expect(task?.description).toContain("Bash alone drives 50%"); + expect(task?.description).toContain("42k input tokens per call"); + }); + + it("omits the hotspot line when no tool dominates", () => { + const task = buildAiUsageSuggestion( + makeSpend({}, { share_of_scoped: 0.2 }), + ); + expect(task?.description).not.toContain("alone drives"); + }); +}); + describe("buildPosthogSetupSuggestion", () => { it("returns the install suggestion when not installed", () => { const task = buildPosthogSetupSuggestion("not_installed"); diff --git a/packages/core/src/setup/suggestions.ts b/packages/core/src/setup/suggestions.ts index fa406195c0..a3e3374ed1 100644 --- a/packages/core/src/setup/suggestions.ts +++ b/packages/core/src/setup/suggestions.ts @@ -1,3 +1,9 @@ +import { + formatTokens, + formatUsd, + formatWindow, +} from "@posthog/core/billing/spendAnalysisFormat"; +import type { SpendAnalysisResponse } from "@posthog/core/billing/spendAnalysisTypes"; import type { DiscoveredTask } from "@posthog/core/setup/types"; export interface StaleFlagPayload { @@ -32,6 +38,44 @@ export function buildStaleFlagSuggestion( }; } +// Below this much PostHog Code spend in the analysis window there's nothing +// meaningful to optimize, so the suggestion stays hidden rather than nagging +// light users. +export const AI_USAGE_MIN_SPEND_USD = 5; + +// Share of scoped spend a single tool must reach before it's called out as a +// hotspot in the suggestion description. +const AI_USAGE_HOTSPOT_SHARE = 0.35; + +export function buildAiUsageSuggestion( + data: SpendAnalysisResponse, +): DiscoveredTask | null { + const { summary } = data; + if (summary.scoped_cost_usd < AI_USAGE_MIN_SPEND_USD) return null; + + const window = formatWindow(summary.date_from, summary.date_to); + const topTool = data.by_tool.items[0]; + const hotspot = + topTool?.tool && topTool.share_of_scoped >= AI_USAGE_HOTSPOT_SHARE + ? ` ${topTool.tool} alone drives ${Math.round(topTool.share_of_scoped * 100)}% of that, averaging ${formatTokens(topTool.avg_input_tokens)} input tokens per call.` + : ""; + + return { + // Stable id so dismissal sticks across re-runs. + id: "ai-usage-optimization", + source: "enricher", + category: "ai_usage", + title: "Optimize your AI usage", + description: `You've spent ${formatUsd(summary.scoped_cost_usd)} on PostHog Code in the last ${window}.${hotspot} Repeated workflows, re-explained context, and recurring one-off analyses usually hide easy savings.`, + impact: + "Token spend compounds quietly: the same context re-sent every session, the same analysis re-run every week. Capturing repeated work as skills and scheduled reports cuts cost — and gets you answers without having to ask.", + recommendation: + 'Click "Implement as new task" — the agent audits your recent sessions and memory files, then proposes concrete changes ranked by impact: new skills for repeated workflows, memory-file edits, and scheduled reports for recurring questions.', + prompt: + "Audit how I've been using PostHog Code and find ways to get better results for fewer tokens. Review my recent sessions for: repeated workflows worth capturing as a reusable skill, recurring questions better served by a scheduled scout or AI report, memory files (CLAUDE.md, AGENTS.md) that are bloated, stale, or missing context I keep re-explaining, and prompts that ran long or retried because they were under-specified. Deliver a short report of findings ranked by impact — each with the evidence behind it and the estimated saving — then offer to implement the top quick wins.", + }; +} + export function buildSdkHealthSuggestion(): DiscoveredTask { return { id: "posthog-sdk-health", diff --git a/packages/core/src/setup/types.ts b/packages/core/src/setup/types.ts index 643f6cc498..2caf35d813 100644 --- a/packages/core/src/setup/types.ts +++ b/packages/core/src/setup/types.ts @@ -16,7 +16,8 @@ export interface DiscoveredTask { | "event_tracking" | "funnel" | "posthog_setup" - | "experiment"; + | "experiment" + | "ai_usage"; source: DiscoveredTaskSource; file?: string; lineHint?: number; diff --git a/packages/shared/src/analytics-events.ts b/packages/shared/src/analytics-events.ts index e692c2f1ae..aa6fe91d59 100644 --- a/packages/shared/src/analytics-events.ts +++ b/packages/shared/src/analytics-events.ts @@ -482,7 +482,8 @@ type SetupDiscoveredTaskCategory = | "event_tracking" | "funnel" | "posthog_setup" - | "experiment"; + | "experiment" + | "ai_usage"; export interface SetupDiscoveryStartedProperties { discovery_task_id: string; diff --git a/packages/ui/src/features/canvas/channelTaskSuggestions.ts b/packages/ui/src/features/canvas/channelTaskSuggestions.ts index 70af77ad9f..5529b1447e 100644 --- a/packages/ui/src/features/canvas/channelTaskSuggestions.ts +++ b/packages/ui/src/features/canvas/channelTaskSuggestions.ts @@ -6,7 +6,6 @@ import { Cube, CurrencyDollar, Flask, - Gauge, Wrench, } from "@phosphor-icons/react"; import type { SuggestedPrompt } from "@posthog/ui/features/task-detail/components/SuggestedPromptCard"; @@ -73,15 +72,6 @@ export const CHANNEL_TASK_SUGGESTIONS: SuggestedPrompt[] = [ prompt: "Interpret the results of an experiment — explain what the metrics show, whether it's significant, and what to do next.\n\n\nUser input:\n- Experiment name or key:\n- What decision are you trying to make (optional):", }, - { - label: "Optimize your AI usage", - description: "Turn repeated work into skills, scouts & savings", - icon: Gauge, - color: "cyan", - mode: "auto", - prompt: - "Audit how I've been using PostHog Code and find ways to get better results for fewer tokens. Review my recent tasks and sessions for: repeated workflows worth capturing as a reusable skill, recurring questions better served by a scheduled scout or AI report, memory files that are bloated, stale, or missing context I keep re-explaining, and prompts that ran long or retried because they were under-specified. Deliver a short report of findings ranked by impact — each with the evidence behind it and the estimated saving — then offer to implement the top quick wins.\n\n\nUser input:\n- Time period to review (optional):\n- What to prioritize (token cost, speed, quality — optional):", - }, { label: "Fix a bug", description: "Track down and fix a problem in the code", diff --git a/packages/ui/src/features/setup/categoryConfig.ts b/packages/ui/src/features/setup/categoryConfig.ts index 3cce2e5f68..34e52f11eb 100644 --- a/packages/ui/src/features/setup/categoryConfig.ts +++ b/packages/ui/src/features/setup/categoryConfig.ts @@ -6,6 +6,7 @@ import { Flag, Flask, Funnel, + Gauge, Lightning, Lock, Sparkle, @@ -39,6 +40,7 @@ export const CATEGORY_CONFIG: Record< funnel: { icon: Funnel, color: "violet", label: "Funnel" }, posthog_setup: { icon: Sparkle, color: "violet", label: "PostHog setup" }, experiment: { icon: Flask, color: "purple", label: "Experiment" }, + ai_usage: { icon: Gauge, color: "cyan", label: "AI usage" }, }; // Fallback when a `DiscoveredTask.category` somehow doesn't match the map diff --git a/packages/ui/src/features/setup/setupRunServiceImpl.ts b/packages/ui/src/features/setup/setupRunServiceImpl.ts index b45537c5fc..f8ce6b40d0 100644 --- a/packages/ui/src/features/setup/setupRunServiceImpl.ts +++ b/packages/ui/src/features/setup/setupRunServiceImpl.ts @@ -1,4 +1,5 @@ import type { PostHogAPIClient } from "@posthog/api-client/posthog-client"; +import type { SpendAnalysisResponse } from "@posthog/core/billing/spendAnalysisTypes"; import type { DiscoveryFailureReason, DiscoverySignalSource, @@ -155,6 +156,15 @@ export class SetupRunServiceImpl implements ISetupRunService { }); } + // Uses a fresh authenticated client rather than `this.client`: the enricher + // path runs before (and independently of) getDiscoveryContext(). + async getSpendAnalysis(): Promise { + const authState = await fetchAuthState(); + const client = createAuthenticatedClient(authState); + if (!client) return null; + return client.getPersonalSpendAnalysis(); + } + includeExperiments(): boolean { return ( resolveService(FEATURE_FLAGS).isEnabled(