Merge pull request #540 from databuddy-analytics/staging

Release: identity segmentation, SDK reliability, ingest gates, OG rebrand

Author: izadoesdev

apps/api/src/billing/autumn.ts

@@ -1,4 +1,5 @@
 import { auth } from "@databuddy/auth";
+import { getRedisCache } from "@databuddy/redis";
 import { getBillingCustomerId, getMemberRole } from "@databuddy/rpc";
 import { autumnHandler } from "autumn-js/fetch";
 import { useLogger } from "evlog/elysia";
@@ -66,9 +67,102 @@ async function stripPrivilegedBody(request: Request): Promise<Request> {
 
 const autumn = autumnHandler({ identify: identifyAutumnCustomer });
 
+const AUTUMN_CACHE_TTL_SEC: Record<string, number> = {
+	getOrCreateCustomer: 30,
+	listPlans: 300,
+};
+
+function autumnPathSegment(request: Request): string {
+	const { pathname } = new URL(request.url);
+	return pathname.split("/").at(-1) ?? "";
+}
+
+function autumnCacheKey(segment: string, customerId: string): string {
+	return `autumn:proxy:${segment}:${customerId}`;
+}
+
+async function invalidateAutumnCustomerCache(request: Request) {
+	const identity = await identifyAutumnCustomer(request).catch(() => null);
+	if (!identity?.customerId) {
+		return;
+	}
+	await getRedisCache()
+		.del(autumnCacheKey("getOrCreateCustomer", identity.customerId))
+		.catch(() => {});
+}
+
+async function readAutumnCache(key: string): Promise<Response | null> {
+	const hit = await getRedisCache()
+		.get(key)
+		.catch(() => null);
+	if (!hit) {
+		return null;
+	}
+	try {
+		const { body, contentType } = JSON.parse(hit) as {
+			body: string;
+			contentType: string;
+		};
+		return new Response(body, {
+			status: 200,
+			headers: { "content-type": contentType },
+		});
+	} catch {
+		return null;
+	}
+}
+
+async function writeAutumnCache(
+	key: string,
+	ttlSec: number,
+	response: Response
+) {
+	const body = await response.clone().text();
+	await getRedisCache()
+		.setex(
+			key,
+			ttlSec,
+			JSON.stringify({
+				body,
+				contentType: response.headers.get("content-type") ?? "application/json",
+			})
+		)
+		.catch(() => {});
+}
+
 export async function handleAutumnRequest(request: Request) {
 	const sanitized = await stripPrivilegedBody(request);
-	return autumn(withAutumnApiPath(sanitized));
+	const segment = autumnPathSegment(sanitized);
+	const ttlSec = AUTUMN_CACHE_TTL_SEC[segment];
+
+	if (ttlSec === undefined) {
+		const response = await autumn(withAutumnApiPath(sanitized));
+		if (
+			response.ok &&
+			sanitized.method !== "GET" &&
+			sanitized.method !== "HEAD"
+		) {
+			await invalidateAutumnCustomerCache(sanitized);
+		}
+		return response;
+	}
+
+	const identity = await identifyAutumnCustomer(sanitized).catch(() => null);
+	if (!identity?.customerId) {
+		return autumn(withAutumnApiPath(sanitized));
+	}
+
+	const key = autumnCacheKey(segment, identity.customerId);
+	const cached = await readAutumnCache(key);
+	if (cached) {
+		return cached;
+	}
+
+	const response = await autumn(withAutumnApiPath(sanitized));
+	if (response.status === 200) {
+		await writeAutumnCache(key, ttlSec, response);
+	}
+	return response;
 }
 
 async function loadSession(request: Request) {

apps/api/src/integration/profile-handlers.test.ts

@@ -7,7 +7,11 @@ import {
 } from "@databuddy/db/schema";
 import { eq } from "@databuddy/db";
 import { appRouter, type Context } from "@databuddy/rpc";
-import { splitTraits, upsertProfile } from "@databuddy/services/identity";
+import {
+	resolveTraitSegment,
+	splitTraits,
+	upsertProfile,
+} from "@databuddy/services/identity";
 import {
 	addToOrganization,
 	cleanup,
@@ -246,3 +250,89 @@ describe("upsertProfile trait history", () => {
 		).toHaveLength(0);
 	});
 });
+
+describe("profiles.traitKeys / traitValues", () => {
+	iit("lists distinct keys and values for the website only", async () => {
+		const user = await signUp();
+		const org = await insertOrganization();
+		await addToOrganization(user.id, org.id, "member");
+		const website = await insertWebsite({ organizationId: org.id });
+		const otherWebsite = await insertWebsite({ organizationId: org.id });
+		await seedProfile(website.id, "user_1", {
+			traits: { plan: "pro", beta: true },
+		});
+		await seedProfile(website.id, "user_2", { traits: { plan: "free" } });
+		await seedProfile(otherWebsite.id, "user_3", {
+			traits: { region: "eu" },
+		});
+		const ctx = userContext(user, org.id);
+
+		const keys = await call(appRouter.profiles.traitKeys, ctx)({
+			websiteId: website.id,
+		});
+		expect(keys).toEqual(["beta", "plan"]);
+
+		const values = await call(appRouter.profiles.traitValues, ctx)({
+			websiteId: website.id,
+			key: "plan",
+		});
+		expect(values).toEqual(["free", "pro"]);
+	});
+
+	iit("rejects non-members", async () => {
+		const user = await signUp();
+		const org = await insertOrganization();
+		const website = await insertWebsite({ organizationId: org.id });
+		const ctx = userContext(user, org.id);
+
+		await expectCode(
+			call(appRouter.profiles.traitKeys, ctx)({ websiteId: website.id }),
+			"FORBIDDEN"
+		);
+	});
+});
+
+describe("resolveTraitSegment", () => {
+	iit("resolves profile ids by trait predicate scoped to the website", async () => {
+		const org = await insertOrganization();
+		const website = await insertWebsite({ organizationId: org.id });
+		const otherWebsite = await insertWebsite({ organizationId: org.id });
+		await seedProfile(website.id, "user_1", { traits: { plan: "pro" } });
+		await seedProfile(website.id, "user_2", { traits: { plan: "free" } });
+		await seedProfile(website.id, "user_3", {
+			traits: { plan: "pro", beta: true },
+		});
+		await seedProfile(otherWebsite.id, "user_4", { traits: { plan: "pro" } });
+
+		const pro = await resolveTraitSegment(website.id, [
+			{ field: "trait:plan", op: "eq", value: "pro" },
+		]);
+		expect(pro.sort()).toEqual(["user_1", "user_3"]);
+
+		const proBeta = await resolveTraitSegment(website.id, [
+			{ field: "trait:plan", op: "eq", value: "pro" },
+			{ field: "trait:beta", op: "eq", value: "true" },
+		]);
+		expect(proBeta).toEqual(["user_3"]);
+
+		const notPro = await resolveTraitSegment(website.id, [
+			{ field: "trait:plan", op: "ne", value: "pro" },
+		]);
+		expect(notPro).toEqual(["user_2"]);
+
+		const inList = await resolveTraitSegment(website.id, [
+			{ field: "trait:plan", op: "in", value: ["free", "trial"] },
+		]);
+		expect(inList).toEqual(["user_2"]);
+
+		const emptyIn = await resolveTraitSegment(website.id, [
+			{ field: "trait:plan", op: "in", value: [] },
+		]);
+		expect(emptyIn).toEqual([]);
+
+		const emptyNotIn = await resolveTraitSegment(website.id, [
+			{ field: "trait:plan", op: "not_in", value: [] },
+		]);
+		expect(emptyNotIn.sort()).toEqual(["user_1", "user_2", "user_3"]);
+	});
+});

apps/api/src/routes/query.ts

@@ -10,7 +10,13 @@ import {
 import { and, db, eq, inArray } from "@databuddy/db";
 import { profiles } from "@databuddy/db/schema";
 import { config } from "@databuddy/env/app";
-import { revealPii } from "@databuddy/services/identity";
+import {
+	isTraitFilterField,
+	resolveTraitSegment,
+	revealPii,
+	type TraitFilter,
+	TraitFilterError,
+} from "@databuddy/services/identity";
 import { validateTimezone } from "@databuddy/validation";
 import { readBooleanEnv } from "@databuddy/env/boolean";
 import { ratelimit } from "@databuddy/redis/rate-limit";
@@ -882,8 +888,6 @@ interface QueryResult {
 	success: boolean;
 }
 
-const PROFILE_ENRICHED_TYPES = new Set(["profile_list"]);
-
 async function attachProfileIdentities(
 	websiteId: string,
 	results: QueryResult[]
@@ -959,6 +963,39 @@ async function executeDynamicQuery(
 			? (scope?.organizationWebsiteIds ?? [])
 			: undefined;
 
+	const requestFilters = (request.filters || []) as Filter[];
+	const traitFilters = requestFilters.filter((f) =>
+		isTraitFilterField(f.field)
+	);
+	let effectiveFilters = requestFilters;
+	let traitError: string | null = null;
+	if (traitFilters.length > 0) {
+		if (projectType === "website") {
+			try {
+				const segment = await resolveTraitSegment(
+					projectId,
+					traitFilters as TraitFilter[]
+				);
+				effectiveFilters = [
+					...requestFilters.filter((f) => !isTraitFilterField(f.field)),
+					{ field: "profile_id", op: "in", value: segment },
+				];
+			} catch (error) {
+				if (error instanceof TraitFilterError) {
+					traitError = error.message;
+				} else {
+					captureError(error, {
+						route: "v1/query",
+						step: "resolve_trait_segment",
+					});
+					traitError = "Trait filter failed";
+				}
+			}
+		} else {
+			traitError = "Trait filters are only supported for website queries";
+		}
+	}
+
 	// Org-level custom_events queries: builder scans by owner_id (= organizationId
 	// set at ingestion) via primary key instead of matching website_id.
 	const hasCustomEventsQueries = request.parameters.some((param) => {
@@ -983,6 +1020,16 @@ async function executeDynamicQuery(
 			return { id, error: `Unknown query type: ${name}` };
 		}
 
+		if (traitError) {
+			return { id, error: traitError };
+		}
+		if (
+			traitFilters.length > 0 &&
+			!isFilterFieldAllowed(config, "profile_id")
+		) {
+			return { id, error: `Trait filters are not supported for ${name}` };
+		}
+
 		if (
 			(paramFrom && !isNormalizedQueryDate(paramFrom)) ||
 			(paramTo && !isNormalizedQueryDate(paramTo))
@@ -1015,7 +1062,7 @@ async function executeDynamicQuery(
 					paramFrom,
 					paramTo
 				),
-				filters: ((request.filters || []) as Filter[]).filter((f) =>
+				filters: effectiveFilters.filter((f) =>
 					isFilterFieldAllowed(config, f.field)
 				),
 				limit: request.limit || 100,
@@ -1078,7 +1125,7 @@ async function executeDynamicQuery(
 
 		if (projectType === "website") {
 			const profileResults = validParameters
-				.filter((p) => PROFILE_ENRICHED_TYPES.has(p.request.type))
+				.filter((p) => p.request.type === "profile_list")
 				.flatMap((p) => resultMap.get(p.id) ?? []);
 			if (profileResults.length > 0) {
 				await attachProfileIdentities(projectId, profileResults).catch(

apps/basket/src/routes/identify.test.ts

@@ -184,7 +184,7 @@ describe("pii protection", () => {
 
 describe("denyApiKeyIdentify", () => {
 	const orgKey = { id: "key_1", organizationId: "org_1" } as ApiKeyRow;
-	const website = { organizationId: "org_1" };
+	const website = { organizationId: "org_1", status: "ACTIVE" };
 	const scopeMock = vi.mocked(hasWebsiteScope);
 
 	test("requires a websiteId", () => {
@@ -211,10 +211,23 @@ describe("denyApiKeyIdentify", () => {
 	test("rejects websites from another organization", () => {
 		scopeMock.mockReturnValueOnce(true);
 		expect(
-			denyApiKeyIdentify(orgKey, "site_1", { organizationId: "org_2" })
+			denyApiKeyIdentify(orgKey, "site_1", {
+				organizationId: "org_2",
+				status: "ACTIVE",
+			})
 		).toBe("website_scope_mismatch");
 	});
 
+	test("rejects websites that are not active", () => {
+		scopeMock.mockReturnValueOnce(true);
+		expect(
+			denyApiKeyIdentify(orgKey, "site_1", {
+				organizationId: "org_1",
+				status: "INACTIVE",
+			})
+		).toBe("website_not_active");
+	});
+
 	test("rejects keys without an organization", () => {
 		scopeMock.mockReturnValueOnce(true);
 		const userKey = { id: "key_2", organizationId: null } as ApiKeyRow;

apps/basket/src/routes/identify.ts

@@ -25,12 +25,13 @@ export type ApiKeyIdentifyDenial =
 	| "missing_website_id"
 	| "missing_scope"
 	| "website_not_found"
-	| "website_scope_mismatch";
+	| "website_scope_mismatch"
+	| "website_not_active";
 
 export function denyApiKeyIdentify(
 	apiKey: ApiKeyRow,
 	websiteId: string | undefined,
-	website: { organizationId: string | null } | null
+	website: { organizationId: string | null; status: string } | null
 ): ApiKeyIdentifyDenial | null {
 	if (!websiteId) {
 		return "missing_website_id";
@@ -47,6 +48,9 @@ export function denyApiKeyIdentify(
 	) {
 		return "website_scope_mismatch";
 	}
+	if (website.status !== "ACTIVE") {
+		return "website_not_active";
+	}
 	return null;
 }
 
@@ -55,6 +59,7 @@ const DENIAL_ERRORS: Record<ApiKeyIdentifyDenial, () => Error> = {
 	missing_scope: basketErrors.trackMissingScope,
 	website_not_found: basketErrors.trackWebsiteNotFound,
 	website_scope_mismatch: basketErrors.trackWebsiteScopeMismatch,
+	website_not_active: basketErrors.trackWebsiteNotFound,
 };
 
 type IdentifyTarget =