Skip to content

Commit d292fab

Browse files
ofershapofershap
committed
fix: reduce alert noise and make spend alerts actionable
Spend alerts: remove technical fields (type/metric/threshold/incident grid), replace σ notation with human-readable "Nx above team avg", add cost driver diagnosis (thinking %, max-mode %) to z-score alerts matching what trends already had. Same cleanup applied to email alerts. Plan exhaustion: replace daily spam with delta-based alerting — only fires when N+ new users exhaust since last alert (default 5, configurable). Shows who the new users are. Add enable/disable toggle and min-delta setting to Settings page. Auto-resets at cycle boundary. Made-with: Cursor
1 parent 88e96cb commit d292fab

7 files changed

Lines changed: 118 additions & 96 deletions

File tree

‎src/app/api/cron/route.ts‎

Lines changed: 33 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,13 @@ import { runDetection } from "@/lib/anomaly/detector";
44
import { processNewAnomalies } from "@/lib/incidents";
55
import { sendAlerts } from "@/lib/alerts";
66
import { sendPlanExhaustionAlert, sendCycleSummary } from "@/lib/alerts/slack";
7-
import { getMetadata, setMetadata, getPlanExhaustionStats, getCycleSummaryData } from "@/lib/data";
7+
import {
8+
getMetadata,
9+
setMetadata,
10+
getPlanExhaustionStats,
11+
getCycleSummaryData,
12+
getConfig,
13+
} from "@/lib/data";
814

915
export const dynamic = "force-dynamic";
1016
export const maxDuration = 300;
@@ -59,30 +65,46 @@ export async function POST(request: Request) {
5965
}
6066

6167
try {
62-
const today = new Date().toISOString().split("T")[0] ?? "";
63-
const lastExhaustionAlert = getMetadata("last_plan_exhaustion_alert");
64-
65-
if (lastExhaustionAlert !== today) {
68+
const config = getConfig();
69+
if (!config.planExhaustion.enabled) {
70+
results.planExhaustionAlert = "disabled";
71+
} else {
6672
const planStats = getPlanExhaustionStats();
67-
if (planStats.summary.users_exhausted > 0) {
73+
const currentCount = planStats.summary.users_exhausted;
74+
const lastCountStr = getMetadata("last_plan_exhaustion_count");
75+
const lastCountRaw = lastCountStr ? parseInt(lastCountStr, 10) : 0;
76+
const lastCount = lastCountRaw > currentCount ? 0 : lastCountRaw;
77+
const delta = currentCount - lastCount;
78+
const minDelta = config.planExhaustion.minDeltaSinceLastAlert;
79+
80+
if (currentCount === 0) {
81+
results.planExhaustionAlert = "none_exhausted";
82+
} else if (lastCount === 0 || delta >= minDelta) {
83+
const newUsers = lastCount === 0 ? planStats.users : planStats.users.slice(-delta);
84+
const nameList = newUsers
85+
.slice(0, 5)
86+
.map((u) => u.name || u.email.split("@")[0])
87+
.join(", ");
88+
const moreCount = newUsers.length > 5 ? newUsers.length - 5 : 0;
89+
6890
const sent = await sendPlanExhaustionAlert(
6991
{
70-
totalPlanExhausted: planStats.summary.users_exhausted,
92+
totalPlanExhausted: currentCount,
7193
totalActive: planStats.summary.total_active,
94+
newSinceLastAlert: delta > 0 ? delta : currentCount,
95+
newUserNames: nameList + (moreCount > 0 ? ` +${moreCount} more` : ""),
7296
},
7397
{ dashboardUrl: process.env.DASHBOARD_URL },
7498
);
7599
if (sent) {
76-
setMetadata("last_plan_exhaustion_alert", today);
100+
setMetadata("last_plan_exhaustion_count", String(currentCount));
77101
results.planExhaustionAlert = "sent";
78102
} else {
79103
results.planExhaustionAlert = "failed";
80104
}
81105
} else {
82-
results.planExhaustionAlert = "none_exhausted";
106+
results.planExhaustionAlert = `skipped (delta ${delta} < min ${minDelta})`;
83107
}
84-
} else {
85-
results.planExhaustionAlert = "already_sent_today";
86108
}
87109
} catch (error) {
88110
results.planExhaustionAlert = {

‎src/app/settings/settings-client.tsx‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,37 @@ export function SettingsClient({ config: initial }: SettingsClientProps) {
248248
</div>
249249
</Section>
250250

251+
<Section
252+
title="Plan Exhaustion Alerts"
253+
description="Notify when users exceed their included plan allowance"
254+
>
255+
<Toggle
256+
label="Enable plan exhaustion alerts"
257+
description="Send Slack alerts when users exceed their plan"
258+
checked={config.planExhaustion?.enabled ?? true}
259+
onChange={(v) =>
260+
setConfig({
261+
...config,
262+
planExhaustion: { ...config.planExhaustion, enabled: v },
263+
})
264+
}
265+
/>
266+
{(config.planExhaustion?.enabled ?? true) && (
267+
<Field
268+
label="Min new users to alert"
269+
value={config.planExhaustion?.minDeltaSinceLastAlert ?? 5}
270+
onChange={(v) =>
271+
setConfig({
272+
...config,
273+
planExhaustion: { ...config.planExhaustion, minDeltaSinceLastAlert: v },
274+
})
275+
}
276+
hint="skip alert unless N+ new users exhausted since last alert"
277+
unit="users"
278+
/>
279+
)}
280+
</Section>
281+
251282
<Section title="Alerting Behavior" description="Control which anomaly types are generated">
252283
<Toggle
253284
label="Info-level anomalies"

‎src/lib/alerts/email.ts‎

Lines changed: 3 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -7,29 +7,10 @@ function getClient(): Resend | null {
77
return new Resend(key);
88
}
99

10-
function formatValue(metric: string, value: number): string {
11-
switch (metric) {
12-
case "spend":
13-
case "cycle_spend":
14-
return `$${(value / 100).toFixed(2)}`;
15-
case "cost_per_req":
16-
return `$${(value / 100).toFixed(2)}/req`;
17-
case "tokens":
18-
return `${(value / 1_000_000).toFixed(2)}M tokens`;
19-
case "requests":
20-
return `${value.toFixed(0)} requests`;
21-
default:
22-
return `${value}`;
23-
}
24-
}
25-
2610
function buildHtml(anomaly: Anomaly, incident: Incident, dashboardUrl?: string): string {
2711
const severityColor = anomaly.severity === "critical" ? "#dc2626" : "#f59e0b";
28-
const diagnosisHtml = anomaly.diagnosisModel
29-
? `<tr><td style="padding:8px;color:#6b7280">Diagnosis</td><td style="padding:8px">Primary model: <code>${anomaly.diagnosisModel}</code></td></tr>`
30-
: "";
3112
const linkHtml = dashboardUrl
32-
? `<p style="margin-top:16px"><a href="${dashboardUrl}/users/${encodeURIComponent(anomaly.userEmail)}">View user dashboard</a> · <a href="${dashboardUrl}/anomalies">View all anomalies</a></p>`
13+
? `<p style="margin-top:16px"><a href="${dashboardUrl}/users/${encodeURIComponent(anomaly.userEmail)}">View user details</a> · <a href="${dashboardUrl}/anomalies">All anomalies</a></p>`
3314
: "";
3415

3516
return `
@@ -39,17 +20,8 @@ function buildHtml(anomaly: Anomaly, incident: Incident, dashboardUrl?: string):
3920
</div>
4021
<div style="border:1px solid #e5e7eb;border-top:none;padding:16px;border-radius:0 0 8px 8px">
4122
<p style="font-size:16px;font-weight:600">${anomaly.message}</p>
42-
<table style="width:100%;border-collapse:collapse">
43-
<tr><td style="padding:8px;color:#6b7280">User</td><td style="padding:8px">${anomaly.userEmail}</td></tr>
44-
<tr><td style="padding:8px;color:#6b7280">Type</td><td style="padding:8px">${anomaly.type}</td></tr>
45-
<tr><td style="padding:8px;color:#6b7280">Metric</td><td style="padding:8px">${anomaly.metric}</td></tr>
46-
<tr><td style="padding:8px;color:#6b7280">Value</td><td style="padding:8px">${formatValue(anomaly.metric, anomaly.value)}</td></tr>
47-
<tr><td style="padding:8px;color:#6b7280">Threshold</td><td style="padding:8px">${formatValue(anomaly.metric, anomaly.threshold)}</td></tr>
48-
<tr><td style="padding:8px;color:#6b7280">Incident</td><td style="padding:8px">#${incident.id}</td></tr>
49-
${diagnosisHtml}
50-
</table>
5123
${linkHtml}
52-
<p style="margin-top:16px;color:#9ca3af;font-size:12px">Detected at ${anomaly.detectedAt} · cursor-usage-tracker</p>
24+
<p style="margin-top:16px;color:#9ca3af;font-size:12px">Incident #${incident.id} · ${anomaly.detectedAt} · cursor-usage-tracker</p>
5325
</div>
5426
</div>
5527
`;
@@ -79,7 +51,7 @@ export async function sendEmailAlert(
7951
const { error } = await resend.emails.send({
8052
from,
8153
to,
82-
subject: `${severityPrefix} Cursor Usage Alert: ${anomaly.userEmail} — ${anomaly.metric}`,
54+
subject: `${severityPrefix} ${anomaly.message}`,
8355
html: buildHtml(anomaly, incident, options.dashboardUrl),
8456
});
8557
if (error) {

‎src/lib/alerts/slack.ts‎

Lines changed: 20 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -15,22 +15,6 @@ function severityEmoji(severity: string): string {
1515
return severity === "critical" ? ":rotating_light:" : ":warning:";
1616
}
1717

18-
function formatValue(metric: string, value: number): string {
19-
switch (metric) {
20-
case "spend":
21-
case "cycle_spend":
22-
return `$${(value / 100).toFixed(2)}`;
23-
case "cost_per_req":
24-
return `$${(value / 100).toFixed(2)}/req`;
25-
case "tokens":
26-
return `${(value / 1_000_000).toFixed(2)}M`;
27-
case "requests":
28-
return `${value.toFixed(0)}`;
29-
default:
30-
return `${value}`;
31-
}
32-
}
33-
3418
function buildAlertBlocks(
3519
anomaly: Anomaly,
3620
incident: Incident,
@@ -52,44 +36,14 @@ function buildAlertBlocks(
5236
text: `*${anomaly.message}*`,
5337
},
5438
},
55-
{
56-
type: "section",
57-
fields: [
58-
{ type: "mrkdwn", text: `*User:*\n${anomaly.userEmail}` },
59-
{ type: "mrkdwn", text: `*Type:*\n${anomaly.type}` },
60-
{ type: "mrkdwn", text: `*Metric:*\n${anomaly.metric}` },
61-
{
62-
type: "mrkdwn",
63-
text: `*Value:*\n${formatValue(anomaly.metric, anomaly.value)}`,
64-
},
65-
{
66-
type: "mrkdwn",
67-
text: `*Threshold:*\n${formatValue(anomaly.metric, anomaly.threshold)}`,
68-
},
69-
{
70-
type: "mrkdwn",
71-
text: `*Incident:*\n#${incident.id}`,
72-
},
73-
],
74-
},
7539
];
7640

77-
if (anomaly.diagnosisModel) {
78-
blocks.push({
79-
type: "section",
80-
text: {
81-
type: "mrkdwn",
82-
text: `*Primary model:* \`${anomaly.diagnosisModel}\``,
83-
},
84-
});
85-
}
86-
8741
if (dashboardUrl) {
8842
blocks.push({
8943
type: "section",
9044
text: {
9145
type: "mrkdwn",
92-
text: `<${dashboardUrl}/users/${encodeURIComponent(anomaly.userEmail)}|View user dashboard> · <${dashboardUrl}/anomalies|View all anomalies>`,
46+
text: `<${dashboardUrl}/users/${encodeURIComponent(anomaly.userEmail)}|View user details> · <${dashboardUrl}/anomalies|All anomalies>`,
9347
},
9448
});
9549
}
@@ -99,7 +53,7 @@ function buildAlertBlocks(
9953
elements: [
10054
{
10155
type: "mrkdwn",
102-
text: `Detected at ${anomaly.detectedAt} · cursor-usage-tracker`,
56+
text: `Incident #${incident.id} · ${anomaly.detectedAt} · cursor-usage-tracker`,
10357
},
10458
],
10559
});
@@ -225,13 +179,18 @@ export async function sendSlackAlert(
225179
}
226180

227181
const blocks = buildAlertBlocks(anomaly, incident, options.dashboardUrl);
228-
const text = `${severityEmoji(anomaly.severity)} ${anomaly.message} — ${anomaly.userEmail}`;
182+
const text = `${severityEmoji(anomaly.severity)} ${anomaly.message}`;
229183

230184
return postToSlack(token, channel, text, blocks);
231185
}
232186

233187
export async function sendPlanExhaustionAlert(
234-
summary: { totalPlanExhausted: number; totalActive: number },
188+
summary: {
189+
totalPlanExhausted: number;
190+
totalActive: number;
191+
newSinceLastAlert: number;
192+
newUserNames: string;
193+
},
235194
options: { dashboardUrl?: string } = {},
236195
): Promise<boolean> {
237196
const token = process.env.SLACK_BOT_TOKEN;
@@ -243,6 +202,8 @@ export async function sendPlanExhaustionAlert(
243202
return false;
244203
}
245204

205+
const pct = Math.round((summary.totalPlanExhausted / summary.totalActive) * 100);
206+
246207
const blocks: SlackBlock[] = [
247208
{
248209
type: "header",
@@ -252,7 +213,14 @@ export async function sendPlanExhaustionAlert(
252213
type: "section",
253214
text: {
254215
type: "mrkdwn",
255-
text: `*${summary.totalPlanExhausted}/${summary.totalActive}* active users have exceeded their included plan this cycle`,
216+
text: `*${summary.newSinceLastAlert} new users* exceeded their plan (${summary.totalPlanExhausted}/${summary.totalActive} total, ${pct}%)`,
217+
},
218+
},
219+
{
220+
type: "section",
221+
text: {
222+
type: "mrkdwn",
223+
text: `*New:* ${summary.newUserNames}`,
256224
},
257225
},
258226
];
@@ -274,7 +242,7 @@ export async function sendPlanExhaustionAlert(
274242
return postToSlack(
275243
token,
276244
channel,
277-
`Cursor — ${summary.totalPlanExhausted} users exceeded plan`,
245+
`Cursor — ${summary.newSinceLastAlert} new users exceeded plan (${summary.totalPlanExhausted} total)`,
278246
blocks,
279247
);
280248
}

‎src/lib/anomaly/zscore.ts‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import type { Anomaly, DetectionConfig } from "../types";
2-
import { getDailySpendWithNames, getLatestDailySpendDate } from "../data";
2+
import { getDailySpendWithNames, getLatestDailySpendDate, getUserCostDrivers } from "../data";
33

44
const MIN_DAILY_SPEND_CENTS = 5000;
55

@@ -8,6 +8,24 @@ function computeZScore(value: number, mean: number, stddev: number): number {
88
return (value - mean) / stddev;
99
}
1010

11+
function buildCostDriverNote(email: string, date?: string): string {
12+
const drivers = getUserCostDrivers(email, date);
13+
if (!drivers) return "";
14+
15+
const parts: string[] = [];
16+
if (drivers.thinking_pct > 0) {
17+
const thinkingDollars = (drivers.thinking_spend_cents / 100).toFixed(0);
18+
parts.push(`${drivers.thinking_pct}% thinking ($${thinkingDollars})`);
19+
}
20+
if (drivers.max_pct > 0) {
21+
const maxDollars = (drivers.max_spend_cents / 100).toFixed(0);
22+
parts.push(`${drivers.max_pct}% max-mode ($${maxDollars})`);
23+
}
24+
25+
if (parts.length === 0) return "";
26+
return ` · cost drivers: ${parts.join(", ")}`;
27+
}
28+
1129
export function detectZScoreAnomalies(config: DetectionConfig): Anomaly[] {
1230
const anomalies: Anomaly[] = [];
1331
const now = new Date().toISOString();
@@ -35,14 +53,16 @@ export function detectZScoreAnomalies(config: DetectionConfig): Anomaly[] {
3553
if (spendZ > multiplier) {
3654
const userDollars = (user.spend_cents / 100).toFixed(2);
3755
const meanDollars = (teamSpendMean / 100).toFixed(2);
56+
const ratio = (user.spend_cents / teamSpendMean).toFixed(1);
57+
const driverNote = buildCostDriverNote(user.email, targetDate);
3858
anomalies.push({
3959
userEmail: user.email,
4060
type: "zscore",
4161
severity: spendZ > multiplier * 3 ? "critical" : "warning",
4262
metric: "spend",
4363
value: user.spend_cents,
4464
threshold: teamSpendMean + multiplier * teamSpendStddev,
45-
message: `${user.name}: daily spend $${userDollars} is ${spendZ.toFixed(1)}σ above team mean ($${meanDollars}) — model: ${user.most_used_model}`,
65+
message: `${user.name}: daily spend $${userDollars} is ${ratio}x above team avg ($${meanDollars}), model: ${user.most_used_model || "unknown"}${driverNote}`,
4666
detectedAt: now,
4767
resolvedAt: null,
4868
alertedAt: null,

‎src/lib/data/sqlite.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -829,6 +829,7 @@ export function getConfig(): DetectionConfig {
829829
thresholds: { ...DEFAULT_CONFIG.thresholds, ...stored.thresholds },
830830
zscore: { ...DEFAULT_CONFIG.zscore, ...stored.zscore },
831831
trends: { ...DEFAULT_CONFIG.trends, ...stored.trends },
832+
planExhaustion: { ...DEFAULT_CONFIG.planExhaustion, ...stored.planExhaustion },
832833
cronIntervalMinutes: stored.cronIntervalMinutes ?? DEFAULT_CONFIG.cronIntervalMinutes,
833834
enableInfoAnomalies: stored.enableInfoAnomalies ?? DEFAULT_CONFIG.enableInfoAnomalies,
834835
};

‎src/lib/types.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -397,6 +397,10 @@ export interface DetectionConfig {
397397
costPerReqMinSpendCents: number;
398398
costPerReqMinAbsoluteCents: number;
399399
};
400+
planExhaustion: {
401+
enabled: boolean;
402+
minDeltaSinceLastAlert: number;
403+
};
400404
cronIntervalMinutes: number;
401405
enableInfoAnomalies: boolean;
402406
}
@@ -418,6 +422,10 @@ export const DEFAULT_CONFIG: DetectionConfig = {
418422
costPerReqMinSpendCents: 2000,
419423
costPerReqMinAbsoluteCents: 500,
420424
},
425+
planExhaustion: {
426+
enabled: true,
427+
minDeltaSinceLastAlert: 5,
428+
},
421429
cronIntervalMinutes: 60,
422430
enableInfoAnomalies: false,
423431
};

0 commit comments

Comments
 (0)