Skip to content

Commit 41d186c

Browse files
izadoesdevizadoesdev
committed
feat(insights): add reflection pass to filter low-signal cards
Adds a ruthless-editor LLM step that scores generated insight cards on actionability x novelty x impact and drops anything below the keep threshold before persistence. Wires reflectAndRank into generation, adjusts persistence to handle filtered output, and trims prompts.
1 parent df87cb5 commit 41d186c

7 files changed

Lines changed: 347 additions & 151 deletions

File tree

‎apps/insights/package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@
1919
"bullmq": "^5.66.5",
2020
"dayjs": "^1.11.19",
2121
"elysia": "catalog:",
22-
"evlog": "catalog:"
22+
"evlog": "catalog:",
23+
"zod": "catalog:"
2324
},
2425
"devDependencies": {
2526
"@databuddy/test": "workspace:*"

‎apps/insights/src/generation.ts‎

Lines changed: 83 additions & 117 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,13 @@ import dayjs from "dayjs";
2727
import { resolveInsightsBilling } from "./billing";
2828
import { type DetectedSignal, detectSignals, wowWindow } from "./detection";
2929
import { detectFunnelGoalSignals } from "./funnel-detection";
30-
import { enrichSignals, type EnrichedSignal } from "./enrichment";
30+
import { enrichSignals } from "./enrichment";
3131
import {
3232
type GeneratedWebsiteInsight,
3333
maxInsights,
3434
persistWebsiteInsights,
3535
} from "./persistence";
36+
import { reflectAndRank } from "./reflection";
3637
import { resolveInsightsForWebsite } from "./resolution";
3738
import {
3839
buildInvestigationPrompt,
@@ -131,31 +132,8 @@ function normalizeAllowedTools(
131132
return TOOL_NAMES.filter((t) => allowed.has(t));
132133
}
133134

134-
function validateCollectedInsights(
135-
insights: ParsedInsight[],
136-
context: {
137-
config: InsightGenerationConfigSnapshot;
138-
organizationId: string;
139-
websiteId: string;
140-
}
141-
): ParsedInsight[] {
142-
const validated = validateInsights(insights);
143-
if (validated.warnings.length > 0) {
144-
emitInsightsEvent("warn", "generation.validation_warnings", {
145-
organization_id: context.organizationId,
146-
website_id: context.websiteId,
147-
input_count: insights.length,
148-
output_count: validated.insights.length,
149-
warning_count: validated.warnings.length,
150-
warnings: validated.warnings,
151-
});
152-
}
153-
return validated.insights.slice(0, maxInsights(context.config));
154-
}
155-
156135
interface AnalyzeWebsiteResult {
157136
detectedSignals: DetectedSignal[];
158-
detectionSucceeded: boolean;
159137
hadData: boolean;
160138
insights: ParsedInsight[];
161139
}
@@ -196,51 +174,41 @@ async function analyzeWebsite(params: {
196174
website_id: params.websiteId,
197175
duration_ms: Math.round(performance.now() - startedAt),
198176
});
199-
return {
200-
insights: [],
201-
detectedSignals: [],
202-
hadData: false,
203-
detectionSucceeded: false,
204-
};
177+
return { insights: [], detectedSignals: [], hadData: false };
205178
}
206179

207-
let detectedSignals: DetectedSignal[] = [];
208-
let detectionSucceeded = false;
209-
let enrichedSignals: EnrichedSignal[] = [];
210-
try {
211-
const detectParams = {
212-
websiteId: params.websiteId,
213-
lookbackDays: params.config.lookbackDays,
214-
timezone: params.config.timezone,
215-
};
216-
const [metricSignals, funnelGoalSignals] = await Promise.all([
217-
detectSignals(detectParams),
218-
detectFunnelGoalSignals(detectParams),
219-
]);
220-
detectedSignals = [...metricSignals, ...funnelGoalSignals].sort(
221-
(a, b) => Math.abs(b.deltaPercent) - Math.abs(a.deltaPercent)
222-
);
223-
detectionSucceeded = true;
224-
if (detectedSignals.length > 0) {
225-
const githubToken = params.githubRepo
226-
? await getOAuthToken("github", params.organizationId, params.userId)
227-
: null;
180+
const detectParams = {
181+
websiteId: params.websiteId,
182+
lookbackDays: params.config.lookbackDays,
183+
timezone: params.config.timezone,
184+
};
185+
const [metricSignals, funnelGoalSignals] = await Promise.all([
186+
detectSignals(detectParams),
187+
detectFunnelGoalSignals(detectParams),
188+
]);
189+
const detectedSignals = [...metricSignals, ...funnelGoalSignals].sort(
190+
(a, b) => Math.abs(b.deltaPercent) - Math.abs(a.deltaPercent)
191+
);
228192

229-
enrichedSignals = await enrichSignals(detectedSignals, {
230-
websiteId: params.websiteId,
231-
timezone: params.config.timezone,
232-
lookbackDays: params.config.lookbackDays,
233-
githubRepo: params.githubRepo,
234-
githubToken,
235-
});
236-
}
237-
} catch (err) {
238-
emitInsightsEvent("warn", "generation.detection_failed", {
239-
error: String(err),
193+
if (detectedSignals.length === 0) {
194+
emitInsightsEvent("info", "generation.agent.skipped_no_signals", {
195+
organization_id: params.organizationId,
196+
website_id: params.websiteId,
197+
duration_ms: Math.round(performance.now() - startedAt),
240198
});
199+
return { insights: [], detectedSignals, hadData: true };
241200
}
242201

243-
const investigationMode = enrichedSignals.length > 0;
202+
const githubToken = params.githubRepo
203+
? await getOAuthToken("github", params.organizationId, params.userId)
204+
: null;
205+
const enrichedSignals = await enrichSignals(detectedSignals, {
206+
websiteId: params.websiteId,
207+
timezone: params.config.timezone,
208+
lookbackDays: params.config.lookbackDays,
209+
githubRepo: params.githubRepo,
210+
githubToken,
211+
});
244212

245213
const [
246214
annotationContext,
@@ -271,22 +239,19 @@ async function analyzeWebsite(params: {
271239
const siteBlock = siteContext
272240
? `\n\nProduct context (cached from homepage):\n${siteContext}`
273241
: '\nScrape "/" first to understand the product.';
274-
const userPrompt = investigationMode
275-
? buildInvestigationPrompt(enrichedSignals, {
276-
domain: params.domain,
277-
githubRepo: params.githubRepo,
278-
period: params.period,
279-
timezone: params.config.timezone,
280-
historyBlock,
281-
annotationContext,
282-
downvotedBlock,
283-
suppressedBlock,
284-
capabilitiesBlock,
285-
orgContext,
286-
siteContext: siteBlock,
287-
})
288-
: `Analyze ${params.domain} (${currentRange.from} to ${currentRange.to} vs ${previousRange.from} to ${previousRange.to}, ${params.config.timezone}). Use web_metrics with period="both" to compare periods efficiently.${siteBlock}${capabilitiesBlock}
289-
${orgContext}${annotationContext}${historyBlock}${suppressedBlock}${downvotedBlock}`;
242+
const userPrompt = buildInvestigationPrompt(enrichedSignals, {
243+
domain: params.domain,
244+
githubRepo: params.githubRepo,
245+
period: params.period,
246+
timezone: params.config.timezone,
247+
historyBlock,
248+
annotationContext,
249+
downvotedBlock,
250+
suppressedBlock,
251+
capabilitiesBlock,
252+
orgContext,
253+
siteContext: siteBlock,
254+
});
290255

291256
const { tools: analyticsTools } = createInsightsAgentTools({
292257
websiteId: params.websiteId,
@@ -315,15 +280,14 @@ ${orgContext}${annotationContext}${historyBlock}${suppressedBlock}${downvotedBlo
315280
config: params.config,
316281
domain: params.domain,
317282
hasCriticalSignals: enrichedSignals.some((s) => s.severity === "critical"),
318-
investigationMode,
319283
organizationId: params.organizationId,
320284
startedAt,
321285
userId: params.userId,
322286
userPrompt,
323287
websiteId: params.websiteId,
324288
});
325289

326-
return { insights, detectedSignals, hadData: true, detectionSucceeded };
290+
return { insights, detectedSignals, hadData: true };
327291
}
328292

329293
async function runInsightsAgent(params: {
@@ -332,7 +296,6 @@ async function runInsightsAgent(params: {
332296
config: InsightGenerationConfigSnapshot;
333297
domain: string;
334298
hasCriticalSignals: boolean;
335-
investigationMode: boolean;
336299
organizationId: string;
337300
startedAt: number;
338301
userId?: string;
@@ -379,26 +342,11 @@ async function runInsightsAgent(params: {
379342
model: ai.wrap(INSIGHTS_MODELS[modelKey]),
380343
instructions: {
381344
role: "system",
382-
content: buildSystemPrompt(params.config, {
383-
investigationMode: params.investigationMode,
384-
}),
345+
content: buildSystemPrompt(params.config),
385346
providerOptions: ANTHROPIC_CACHE_1H,
386347
},
387348
tools: allToolsWithEmit,
388-
stopWhen: (event) => {
389-
if (stepCountIs(params.config.maxSteps)(event)) {
390-
return true;
391-
}
392-
if (
393-
collected.length >= maxInsights(params.config) &&
394-
event.steps
395-
.at(-1)
396-
?.toolCalls.some((tc) => tc?.toolName === "emit_insight")
397-
) {
398-
return true;
399-
}
400-
return false;
401-
},
349+
stopWhen: stepCountIs(params.config.maxSteps),
402350
onStepFinish: ({ usage, finishReason, toolCalls }) => {
403351
toolCallCount += toolCalls.length;
404352
emitInsightsEvent("info", "generation.agent.step_finished", {
@@ -446,35 +394,53 @@ async function runInsightsAgent(params: {
446394
websiteId: params.websiteId,
447395
});
448396

449-
if (collected.length > 0) {
450-
const validated = validateCollectedInsights(collected, {
451-
config: params.config,
452-
organizationId: params.organizationId,
453-
websiteId: params.websiteId,
454-
});
455-
emitInsightsEvent("info", "generation.agent.completed", {
397+
if (collected.length === 0) {
398+
emitInsightsEvent("warn", "generation.agent.missing_output", {
456399
organization_id: params.organizationId,
457400
website_id: params.websiteId,
458401
duration_ms: Math.round(performance.now() - params.startedAt),
459-
raw_output_count: collected.length,
460-
output_count: validated.length,
461402
tool_call_count: toolCallCount,
462403
});
463-
setInsightsLog({
464-
generation_mode: "agent",
465-
tool_call_count: toolCallCount,
466-
generated_candidate_count: validated.length,
467-
});
468-
return validated;
404+
return [];
469405
}
470406

471-
emitInsightsEvent("warn", "generation.agent.missing_output", {
407+
const validation = validateInsights(collected);
408+
if (validation.warnings.length > 0) {
409+
emitInsightsEvent("warn", "generation.validation_warnings", {
410+
organization_id: params.organizationId,
411+
website_id: params.websiteId,
412+
input_count: collected.length,
413+
output_count: validation.insights.length,
414+
warning_count: validation.warnings.length,
415+
warnings: validation.warnings,
416+
});
417+
}
418+
const selected = await reflectAndRank(
419+
validation.insights,
420+
maxInsights(params.config),
421+
{
422+
billingCustomerId: params.billingCustomerId,
423+
chatId: appContext.chatId,
424+
organizationId: params.organizationId,
425+
userId: params.userId,
426+
websiteId: params.websiteId,
427+
}
428+
);
429+
emitInsightsEvent("info", "generation.agent.completed", {
472430
organization_id: params.organizationId,
473431
website_id: params.websiteId,
474432
duration_ms: Math.round(performance.now() - params.startedAt),
433+
raw_output_count: collected.length,
434+
validated_count: validation.insights.length,
435+
output_count: selected.length,
475436
tool_call_count: toolCallCount,
476437
});
477-
return [];
438+
setInsightsLog({
439+
generation_mode: "agent",
440+
tool_call_count: toolCallCount,
441+
generated_candidate_count: selected.length,
442+
});
443+
return selected;
478444
} catch (error) {
479445
captureInsightsError(error, "generation.agent.failed", {
480446
organization_id: params.organizationId,
@@ -631,7 +597,7 @@ export async function generateWebsiteInsights(
631597
websiteId: site.id,
632598
runId: input.runId,
633599
detectedSignals: analysis.detectedSignals,
634-
canRecover: analysis.hadData && analysis.detectionSucceeded,
600+
canRecover: analysis.hadData,
635601
});
636602
} catch (error) {
637603
captureInsightsError(error, "generation.resolution.failed", {

‎apps/insights/src/persistence.ts‎

Lines changed: 27 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,30 @@ function dedupeKeyFor(insight: GeneratedWebsiteInsight): string {
8787
});
8888
}
8989

90+
interface DedupeKeyRow {
91+
changePercent: number | null;
92+
dedupeKey: string | null;
93+
sentiment: string;
94+
subjectKey: string;
95+
title: string;
96+
type: string;
97+
websiteId: string;
98+
}
99+
100+
function resolveDedupeKey(row: DedupeKeyRow): string {
101+
return (
102+
row.dedupeKey ??
103+
insightDedupeKey({
104+
websiteId: row.websiteId,
105+
type: row.type as ParsedInsight["type"],
106+
sentiment: row.sentiment as ParsedInsight["sentiment"],
107+
changePercent: row.changePercent,
108+
subjectKey: row.subjectKey,
109+
title: row.title,
110+
})
111+
);
112+
}
113+
90114
async function fetchInsightDedupeKeyToIdMap(
91115
organizationId: string,
92116
cooldownHours: number
@@ -114,16 +138,7 @@ async function fetchInsightDedupeKeyToIdMap(
114138

115139
const map = new Map<string, string>();
116140
for (const row of rows) {
117-
const key =
118-
row.dedupeKey ??
119-
insightDedupeKey({
120-
websiteId: row.websiteId,
121-
type: row.type as ParsedInsight["type"],
122-
sentiment: row.sentiment as ParsedInsight["sentiment"],
123-
changePercent: row.changePercent,
124-
subjectKey: row.subjectKey,
125-
title: row.title,
126-
});
141+
const key = resolveDedupeKey(row);
127142
if (!map.has(key)) {
128143
map.set(key, row.id);
129144
}
@@ -169,16 +184,7 @@ async function fetchDismissedBaselines(
169184

170185
const map = new Map<string, DismissedBaseline>();
171186
for (const row of rows) {
172-
const key =
173-
row.dedupeKey ??
174-
insightDedupeKey({
175-
websiteId: row.websiteId,
176-
type: row.type as ParsedInsight["type"],
177-
sentiment: row.sentiment as ParsedInsight["sentiment"],
178-
changePercent: row.changePercent,
179-
subjectKey: row.subjectKey,
180-
title: row.title,
181-
});
187+
const key = resolveDedupeKey(row);
182188
if (!map.has(key)) {
183189
map.set(key, {
184190
changePercent: row.changePercent,
@@ -290,10 +296,7 @@ export async function persistWebsiteInsights(params: {
290296
evidence: insight.evidence ?? null,
291297
investigationDepth: insight.investigationDepth ?? null,
292298
actions: insight.actions ?? null,
293-
metrics:
294-
insight.metrics.length > 0
295-
? (insight.metrics as InsightMetricRow[])
296-
: null,
299+
metrics: insight.metrics as InsightMetricRow[],
297300
timezone: params.config.timezone,
298301
currentPeriodFrom: params.period.current.from,
299302
currentPeriodTo: params.period.current.to,

0 commit comments

Comments
 (0)