fix(analytics): correct identity funnels links and uptime

Author: izadoesdev

apps/api/src/integration/profile-handlers.test.ts

@@ -331,13 +331,69 @@ describe("getTraitDistribution", () => {
 		});
 
 		const distribution = await getTraitDistribution(website.id);
-		expect(distribution.identifiedProfiles).toBe(3);
+		expect(distribution).toMatchObject({
+			hasMoreKeys: false,
+			hasMoreValues: false,
+			identifiedProfiles: 3,
+			returnedTraitKeys: 2,
+			totalTraitKeys: 2,
+			valuesPerKey: 20,
+		});
 		expect(distribution.traits).toEqual([
 			{ key: "beta", value: "true", profiles: 1 },
 			{ key: "plan", value: "pro", profiles: 2 },
 			{ key: "plan", value: "free", profiles: 1 },
 		]);
 	});
+
+	iit("gives every returned key a value before adding more values per key", async () => {
+		const org = await insertOrganization();
+		const website = await insertWebsite({ organizationId: org.id });
+		const keys = Array.from({ length: 11 }, (_, index) => `trait_${index}`);
+
+		await Promise.all(
+			Array.from({ length: 20 }, (_, valueIndex) =>
+				seedProfile(website.id, `user_${valueIndex}`, {
+					traits: Object.fromEntries(
+						keys.map((key) => [key, `value_${valueIndex}`])
+					),
+				})
+			)
+		);
+
+		const distribution = await getTraitDistribution(website.id);
+		expect(distribution).toMatchObject({
+			hasMoreKeys: false,
+			hasMoreValues: true,
+			returnedTraitKeys: 11,
+			totalTraitKeys: 11,
+			valuesPerKey: 18,
+		});
+		expect(distribution.traits).toHaveLength(198);
+		expect(new Set(distribution.traits.map((trait) => trait.key))).toEqual(
+			new Set(keys)
+		);
+	});
+
+	iit("reports when lower-coverage keys are omitted by the row bound", async () => {
+		const org = await insertOrganization();
+		const website = await insertWebsite({ organizationId: org.id });
+		await seedProfile(website.id, "user_1", {
+			traits: Object.fromEntries(
+				Array.from({ length: 201 }, (_, index) => [`trait_${index}`, true])
+			),
+		});
+
+		const distribution = await getTraitDistribution(website.id);
+		expect(distribution).toMatchObject({
+			hasMoreKeys: true,
+			hasMoreValues: false,
+			returnedTraitKeys: 200,
+			totalTraitKeys: 201,
+			valuesPerKey: 1,
+		});
+		expect(distribution.traits).toHaveLength(200);
+	});
 });
 
 describe("resolveTraitSegment", () => {

apps/dashboard/app/(main)/websites/[id]/errors/_components/error-summary-stats.tsx

@@ -100,7 +100,7 @@ export const ErrorSummaryStats = ({
 			/>
 			<ErrorStatCard
 				icon={TrendUpIcon}
-				title="Sessions with errors"
+				title="Session error rate"
 				value={`${(errorSummary.errorRate || 0).toFixed(2)}%`}
 				variant="warning"
 			/>

apps/dashboard/app/(main)/websites/[id]/funnels/_components/funnel-analytics.tsx

@@ -110,9 +110,10 @@ export function FunnelAnalytics({
 				total_users_completed: selectedReferrerData.completed_users,
 				overall_conversion_rate: selectedReferrerData.conversion_rate,
 				avg_completion_time: 0,
-				avg_completion_time_formatted: "0s",
+				avg_completion_time_formatted: "—",
 				biggest_dropoff_step: 1,
 				biggest_dropoff_rate: 100 - selectedReferrerData.conversion_rate,
+				duration_available: false,
 				steps_analytics: [],
 			}
 		: data;
@@ -127,6 +128,7 @@ export function FunnelAnalytics({
 	const errorInsights = data?.error_insights;
 	const hasErrorCorrelation =
 		errorInsights &&
+		errorInsights.available &&
 		errorInsights.dropoffs_with_errors > 0 &&
 		errorInsights.error_correlation_rate > 0;
 
@@ -216,7 +218,7 @@ export function FunnelAnalytics({
 						v < 60 ? `${Math.round(v)}s` : `${Math.round(v / 60)}m`
 					}
 					icon={ClockIcon}
-					showChart={hasChartData}
+					showChart={hasChartData && displayData.duration_available}
 					title="Avg Time"
 					value={displayData.avg_completion_time_formatted || "—"}
 				/>

apps/dashboard/app/(main)/websites/[id]/funnels/_components/funnel-flow.tsx

@@ -157,7 +157,7 @@ export function FunnelFlow({ steps }: FunnelFlowProps) {
 										<span className="truncate font-medium text-foreground">
 											{step.step_name}
 										</span>
-										{step.error_count > 0 && (
+										{step.error_context_available && step.error_count > 0 && (
 											<Tooltip
 												content={
 													<div className="max-w-xs space-y-1.5">

apps/dashboard/hooks/use-goals.ts

@@ -17,7 +17,9 @@ export interface GoalAnalyticsData {
 	avg_completion_time_formatted: string;
 	biggest_dropoff_rate: number;
 	biggest_dropoff_step: number;
+	duration_available: boolean;
 	error_insights: {
+		available: boolean;
 		total_errors: number;
 		sessions_with_errors: number;
 		dropoffs_with_errors: number;
@@ -29,6 +31,7 @@ export interface GoalAnalyticsData {
 		conversion_rate: number;
 		dropoff_rate: number;
 		dropoffs: number;
+		error_context_available: boolean;
 		error_count: number;
 		error_rate: number;
 		step_name: string;

apps/dashboard/types/funnels.ts

@@ -43,6 +43,7 @@ export interface FunnelStepAnalytics {
 	conversion_rate: number;
 	dropoff_rate: number;
 	dropoffs: number;
+	error_context_available: boolean;
 	error_count: number;
 	error_rate: number;
 	step_name: string;
@@ -53,6 +54,7 @@ export interface FunnelStepAnalytics {
 }
 
 export interface FunnelErrorInsights {
+	available: boolean;
 	dropoffs_with_errors: number;
 	error_correlation_rate: number;
 	sessions_with_errors: number;
@@ -74,6 +76,7 @@ export interface FunnelAnalyticsData {
 	avg_completion_time_formatted: string;
 	biggest_dropoff_rate: number;
 	biggest_dropoff_step: number;
+	duration_available: boolean;
 	error_insights?: FunnelErrorInsights;
 	overall_conversion_rate: number;
 	steps_analytics: FunnelStepAnalytics[];

packages/ai/src/ai/mcp/tools.ts

@@ -17,10 +17,12 @@ import {
 	LinkFolderSelectorSchema,
 	LinkFolderWithUsageSchema,
 	LinkRowOutputSchema,
+	getLinkSummary,
 	listLinkFolders,
 	listLinks,
 	parseLinkRow,
 	resolveLinkFolder,
+	searchLinks,
 	summarizeLink,
 	summarizeLinkFolder,
 	summarizeLinkFoldersWithUsage,
@@ -943,15 +945,15 @@ const listLinkFoldersTool = defineMcpTool(
 		}
 
 		const rpcContext = buildRpcContext(ctx);
-		const [folders, links] = await Promise.all([
+		const [folders, summary] = await Promise.all([
 			listLinkFolders(rpcContext, orgId),
-			listLinks(rpcContext, orgId),
+			getLinkSummary(rpcContext, orgId),
 		]);
 
 		return {
-			folders: summarizeLinkFoldersWithUsage(folders, links),
+			folders: summarizeLinkFoldersWithUsage(folders),
 			count: folders.length,
-			unfiledCount: links.filter((link) => !link.folderId).length,
+			unfiledCount: summary.unfiledTotal,
 			hint:
 				folders.length === 0
 					? "No link folders exist yet. Leave links unfiled unless the user creates a folder in Databuddy."
@@ -964,7 +966,7 @@ const listLinksTool = defineMcpTool(
 	{
 		name: "list_links",
 		description:
-			"List short links and existing folders for the website's organization. Use to enumerate all links before referencing one or choosing where a new link should go.",
+			"List the newest short links and existing folders for the website's organization. The count covers the full catalog; use search_links to find a specific older link.",
 		inputSchema: z.object({
 			...WebsiteSelectorSchema,
 		}),
@@ -984,24 +986,29 @@ const listLinksTool = defineMcpTool(
 			throw new McpToolError("not_found", orgId.message);
 		}
 		const rpcContext = buildRpcContext(ctx);
-		const [links, folders] = await Promise.all([
+		const [page, folders, summary] = await Promise.all([
 			listLinks(rpcContext, orgId),
 			listLinkFolders(rpcContext, orgId),
+			getLinkSummary(rpcContext, orgId),
 		]);
-		if (links.length === 0) {
+		if (page.items.length === 0) {
 			return {
-				links,
+				links: page.items,
 				count: 0,
-				folders: summarizeLinkFoldersWithUsage(folders, links),
-				unfiledCount: 0,
+				folders: summarizeLinkFoldersWithUsage(folders),
+				unfiledCount: summary.unfiledTotal,
 				hint: "No links yet for this organization.",
 			};
 		}
 		return {
-			links: links.map((link) => summarizeLink(link, folders)),
-			count: links.length,
-			folders: summarizeLinkFoldersWithUsage(folders, links),
-			unfiledCount: links.filter((link) => !link.folderId).length,
+			links: page.items.map((link) => summarizeLink(link, folders)),
+			count: summary.total,
+			folders: summarizeLinkFoldersWithUsage(folders, page.items),
+			unfiledCount: summary.unfiledTotal,
+			hint:
+				page.hasMore || summary.total > page.items.length
+					? `Showing the ${page.items.length} newest of ${summary.total} links. Use search_links for older links.`
+					: undefined,
 		};
 	}
 );
@@ -1010,12 +1017,14 @@ const searchLinksTool = defineMcpTool(
 	{
 		name: "search_links",
 		description:
-			"Find short links matching a substring on name, slug, target URL, or external ID. Use when you know part of a link identifier.",
+			"Find short links across the full catalog matching a substring on name, slug, target URL, or external ID. Returns at most the newest 50 matches.",
 		inputSchema: z.object({
 			...WebsiteSelectorSchema,
 			query: z
 				.string()
+				.trim()
 				.min(1)
+				.max(255)
 				.describe("Search query (matches name, slug, URL, or external ID)"),
 		}),
 		outputSchema: z.object({
@@ -1031,30 +1040,23 @@ const searchLinksTool = defineMcpTool(
 				})
 			),
 			count: z.number(),
+			hasMore: z.boolean(),
 		}),
 		resolveWebsite: true,
-		ratelimit: { limit: 60, windowSec: 60 },
+		ratelimit: { limit: 20, windowSec: 60 },
 	},
 	async (input, ctx) => {
 		const orgId = await getOrganizationId(getResolvedWebsiteId(ctx));
 		if (orgId instanceof Error) {
 			throw new McpToolError("not_found", orgId.message);
 		}
 		const rpcContext = buildRpcContext(ctx);
-		const [allLinks, folders] = await Promise.all([
-			listLinks(rpcContext, orgId),
+		const [page, folders] = await Promise.all([
+			searchLinks(rpcContext, orgId, input.query),
 			listLinkFolders(rpcContext, orgId),
 		]);
-		const queryLower = input.query.toLowerCase();
-		const matches = allLinks.filter(
-			(link) =>
-				link.name.toLowerCase().includes(queryLower) ||
-				link.slug.toLowerCase().includes(queryLower) ||
-				link.targetUrl.toLowerCase().includes(queryLower) ||
-				link.externalId?.toLowerCase().includes(queryLower)
-		);
 		return {
-			links: matches.map((link) => ({
+			links: page.items.map((link) => ({
 				id: link.id,
 				name: link.name,
 				slug: link.slug,
@@ -1063,7 +1065,8 @@ const searchLinksTool = defineMcpTool(
 				folder: summarizeLink(link, folders).folder,
 				externalId: link.externalId,
 			})),
-			count: matches.length,
+			count: page.items.length,
+			hasMore: page.hasMore,
 		};
 	}
 );

packages/ai/src/ai/prompts/clickhouse-schema.ts

@@ -1,6 +1,7 @@
 import {
 	AGENT_TABLE_COLUMNS,
 	AGENT_TENANT_COLUMN_BY_TABLE,
+	CUSTOM_EVENTS_VISITOR_KEY,
 	EVENTS_VISITOR_KEY,
 	validateAgentSQL,
 } from "@databuddy/db/clickhouse";
@@ -191,7 +192,7 @@ const GUIDELINES = `## Query Guidelines
 - Geographic data (country, region, city) exists only on analytics.events, NOT on web_vitals_spans or error_spans. Join via session_id if needed.
 - All timestamps are in UTC.
 - analytics.custom_events.properties contains JSON strings — use JSONExtractString(properties, 'key') to parse.
-- Identity: \`anonymous_id\` is a per-device id; \`profile_id\` is the customer-assigned user id ('' when anonymous, on analytics.events, analytics.custom_events, and analytics.revenue). To count people, dedupe identified users across devices with \`uniq(${EVENTS_VISITOR_KEY})\` (on custom_events/revenue wrap the fallback: \`if(profile_id != '', profile_id, ifNull(anonymous_id, ''))\` since anonymous_id is Nullable there). To count only identified users: \`uniqIf(profile_id, profile_id != '')\`. error_spans/web_vitals_spans/outgoing_links have no profile_id — resolve an identified user's rows there via their anonymous_ids from analytics.events.
+- Identity: \`anonymous_id\` is a per-device id; \`profile_id\` is the customer-assigned user id ('' when anonymous, on analytics.events, analytics.custom_events, and analytics.revenue). To count people, dedupe identified users across devices with \`uniq(${EVENTS_VISITOR_KEY})\`; on custom_events/revenue use \`uniq(${CUSTOM_EVENTS_VISITOR_KEY})\` because anonymous_id is Nullable and rows missing both identifiers must not count as a person. To count only identified users: \`uniqIf(profile_id, profile_id != '')\`. error_spans/web_vitals_spans/outgoing_links have no profile_id — resolve an identified user's rows there via their anonymous_ids from analytics.events.
 
 ## Aggregate function preferences
 - Percentiles: use \`quantileTDigest(p)(col)\` for p50/p75/p95/p99. Plain \`quantile(p)\` uses reservoir sampling and is noisy at the tails (~10% error at p99). \`quantileTDigest\` is within 0.1% of exact at the same memory cost.

packages/ai/src/ai/tools/link-catalog.test.ts

@@ -1,6 +1,8 @@
 import { describe, expect, it } from "bun:test";
 import {
 	type LinkFolder,
+	fetchLinkCatalogPage,
+	fetchLinkSummary,
 	resolveLinkFolderFromList,
 } from "./link-catalog";
 
@@ -80,3 +82,45 @@ describe("resolveLinkFolderFromList", () => {
 		}
 	});
 });
+
+describe("paginated link catalog", () => {
+	it("uses one bounded server-side search beyond the legacy 1,000-row cap", async () => {
+		const source = Array.from({ length: 1001 }, (_, index) => ({
+			id: `link-${index}`,
+			name: `Example ${index}`,
+			slug: `example-${index}`,
+			targetUrl: `https://example.com/${index}`,
+		}));
+		const calls: Array<{ limit: number; offset: number; search?: string }> = [];
+
+		const page = await fetchLinkCatalogPage(async (input) => {
+			calls.push(input);
+			const matches = source.filter((link) =>
+				link.name.toLowerCase().includes(input.search?.toLowerCase() ?? "")
+			);
+			const items = matches.slice(input.offset, input.offset + input.limit);
+			return {
+				hasMore: input.offset + items.length < matches.length,
+				items,
+			};
+		}, { search: "Example 1000" });
+
+		expect(page.items).toHaveLength(1);
+		expect(page.items[0]?.id).toBe("link-1000");
+		expect(page.total).toBeUndefined();
+		expect(calls).toEqual([
+			{ limit: 50, offset: 0, search: "Example 1000" },
+		]);
+	});
+
+	it("loads exact catalog and unfiled totals with one summary call", async () => {
+		const calls: Array<{ search?: string }> = [];
+		const summary = await fetchLinkSummary(async (input) => {
+			calls.push(input);
+			return { total: 7, unfiledTotal: 3 };
+		}, "campaign");
+
+		expect(summary).toEqual({ total: 7, unfiledTotal: 3 });
+		expect(calls).toEqual([{ search: "campaign" }]);
+	});
+});