Skip to content

Commit 9533578

Browse files
authored
Merge pull request #536 from databuddy-analytics/staging
Release: user identity, encryption at rest, env schema enforcement
2 parents 117bd62 + 7ba7a5e commit 9533578

138 files changed

Lines changed: 5021 additions & 264 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/dashboard-e2e.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ jobs:
4343
REDIS_URL: redis://localhost:6379
4444
BULLMQ_REDIS_URL: redis://localhost:6379
4545
BETTER_AUTH_SECRET: databuddy-e2e-secret-at-least-32-bytes
46+
BETTER_AUTH_URL: http://localhost:3000
47+
AI_API_KEY: e2e-ai-api-key
48+
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
4649
RESEND_API_KEY: re_e2e_dummy
4750
AUTUMN_SECRET_KEY: e2e-autumn-secret
4851

.github/workflows/health-check.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,8 @@ jobs:
112112
-e AUTUMN_SECRET_KEY=test-autumn-secret-key \
113113
-e EMAIL_API_KEY=test-email-api-key \
114114
-e IP_HASH_SALT=test-ip-hash-salt \
115+
-e AI_API_KEY=test-ai-api-key \
116+
-e CLICKHOUSE_URL=http://default:@localhost:8123/analytics \
115117
api:test
116118
117119
echo "Waiting for API to start..."

CLAUDE.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,7 @@ cd packages/db && DATABASE_URL="postgres://databuddy:databuddy_dev_password@loca
185185
- **Never hand-write dependency versions.** Use `bun add <pkg>` to add dependencies. Hand-written version ranges drift from lockfile reality and cause phantom resolution bugs.
186186
- **Shared test helpers over local copies.** `expectCode`, `userContext`, `apiKeyContext`, env setup — these live in `@databuddy/test`. If you're about to define a helper that already exists there, import it instead.
187187
- **Scope maps must match.** If `RESOURCE_SCOPE_OVERRIDES` changes in `packages/api-keys/src/scopes.ts`, the integration tests in `link-handlers.test.ts` and `with-workspace.test.ts` must be updated to match. The link resource is mapped there (`read:links` for read, `write:links` for create/update/delete), as is the flag resource (`manage:flags` for create/update/delete). Both are enforced solely by `withWorkspace`; there are no separate pre-check layers. New resources also need role grants in `packages/auth/src/permissions.ts` (statement plus each role).
188+
- **Identity columns are governed by `packages/db/src/clickhouse/identity.ts`.** `PROFILE_ID_TABLES` lists every ClickHouse table carrying `profile_id`; `identity.test.ts` enforces that each entry has a CREATE column, an idempotent migration, and `profile_id` + `anonymous_id` in the agent SQL allowlist. Adding `profile_id` to a table means adding it there and following the failing test. Query-side identity stitching must use `EVENTS_VISITOR_KEY` / `CUSTOM_EVENTS_VISITOR_KEY` / `visitorMatch()` from that module — never inline the expression. Profile write semantics (trait splitting, upserts) live in `@databuddy/services/identity`; `profile_id` values are customer-supplied and are never salted, unlike `anonymous_id`.
188189

189190
## AI Policy Note
190191

apps/api/src/http/errors.ts

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { config } from "@databuddy/env/app";
12
import { EvlogError, parseError } from "evlog";
23

34
interface AppErrorContext {
@@ -23,6 +24,8 @@ const HTTP_STATUS_BY_ERROR_CODE: Record<string, number> = {
2324
VALIDATION: 422,
2425
};
2526

27+
const PROTECTED_RESOURCE_METADATA_URL = `${config.urls.api}/.well-known/oauth-protected-resource`;
28+
2629
export function handleAppError({ error, code }: AppErrorContext) {
2730
const parsed = parseError(error);
2831
const statusCode = getStatusCode({
@@ -45,6 +48,13 @@ export function handleAppError({ error, code }: AppErrorContext) {
4548
isClientError,
4649
statusCode,
4750
});
51+
const headers: Record<string, string> = {
52+
"Content-Type": "application/json",
53+
};
54+
if (statusCode === 401) {
55+
headers["WWW-Authenticate"] =
56+
`Bearer resource_metadata="${PROTECTED_RESOURCE_METADATA_URL}"`;
57+
}
4858

4959
return new Response(
5060
JSON.stringify({
@@ -57,7 +67,7 @@ export function handleAppError({ error, code }: AppErrorContext) {
5767
? { link: parsed.link }
5868
: {}),
5969
}),
60-
{ status: statusCode, headers: { "Content-Type": "application/json" } }
70+
{ status: statusCode, headers }
6171
);
6272
}
6373

apps/api/src/index.ts

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import "./polyfills/compression";
2+
import { env as apiEnv } from "@databuddy/env/api";
23
import cors from "@elysiajs/cors";
34
import { Elysia } from "elysia";
45
import { evlog } from "evlog/elysia";
@@ -20,6 +21,7 @@ import {
2021
} from "@/rpc/handlers";
2122
import { openApiHandler } from "@/rpc/openapi";
2223
import { agent } from "./routes/agent";
24+
import { discovery } from "./routes/discovery";
2325
import { health } from "./routes/health";
2426
import { integrations } from "./routes/integrations";
2527
import { mcp } from "./routes/mcp";
@@ -54,6 +56,18 @@ function handleOpenApiEndpoint({ request }: RequestContext) {
5456
return handleAuthenticatedOrpcRequest(request, handleOpenApiRequest);
5557
}
5658

59+
function handleOpenApiJson({ request }: RequestContext) {
60+
const url = new URL(request.url);
61+
url.pathname = "/spec.json";
62+
const specRequest = new Request(url, {
63+
headers: request.headers,
64+
method: request.method,
65+
signal: request.signal,
66+
});
67+
68+
return handleOpenApiReference({ request: specRequest });
69+
}
70+
5771
function handleOpenApiRequest(orpcRequest: Request, context: OrpcContext) {
5872
return openApiHandler.handle(orpcRequest, {
5973
prefix: "/",
@@ -76,14 +90,7 @@ const app = new Elysia({ precompile: true })
7690
)
7791
.use(publicApi)
7892
.use(health)
79-
.get(
80-
"/.well-known/oauth-authorization-server",
81-
() =>
82-
new Response(null, {
83-
status: 404,
84-
headers: { "Cache-Control": "no-store" },
85-
})
86-
)
93+
.use(discovery)
8794
.use(webhooks)
8895
.mount(AUTUMN_API_PREFIX, handleAutumnRequest)
8996
.use(query)
@@ -92,6 +99,7 @@ const app = new Elysia({ precompile: true })
9299
.use(mcp)
93100
.all("/rpc/*", handleRpcEndpoint, { parse: "none" })
94101
.all("/", handleOpenApiReference, { parse: "none" })
102+
.all("/openapi.json", handleOpenApiJson, { parse: "none" })
95103
.all("/spec.json", handleOpenApiReference, { parse: "none" })
96104
.all("/*", handleOpenApiEndpoint, { parse: "none" })
97105
.onError(handleAppError);
@@ -101,6 +109,6 @@ registerShutdownHooks();
101109

102110
export default {
103111
fetch: app.fetch,
104-
port: Number.parseInt(process.env.PORT ?? "3001", 10),
112+
port: Number.parseInt(apiEnv.PORT, 10),
105113
idleTimeout: BUN_IDLE_TIMEOUT_SECONDS,
106114
};
Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
import "@databuddy/test/env";
2+
3+
import { profileAliases, profiles } from "@databuddy/db/schema";
4+
import { appRouter, type Context } from "@databuddy/rpc";
5+
import {
6+
addToOrganization,
7+
cleanup,
8+
db,
9+
expectCode,
10+
hasTestDb,
11+
insertOrganization,
12+
insertWebsite,
13+
reset,
14+
signUp,
15+
userContext,
16+
} from "@databuddy/test";
17+
import { createProcedureClient } from "@orpc/server";
18+
import { afterAll, beforeEach, describe, expect, it } from "vitest";
19+
20+
const iit = hasTestDb ? it : it.skip;
21+
22+
function call<T>(procedure: T, ctx: Context) {
23+
return createProcedureClient(procedure as any, { context: ctx });
24+
}
25+
26+
beforeEach(() => reset());
27+
afterAll(() => cleanup());
28+
29+
async function seedProfile(
30+
websiteId: string,
31+
profileId: string,
32+
overrides: Partial<typeof profiles.$inferInsert> = {}
33+
) {
34+
await db()
35+
.insert(profiles)
36+
.values({
37+
websiteId,
38+
profileId,
39+
email: `${profileId}@example.com`,
40+
displayName: `User ${profileId}`,
41+
traits: { plan: "pro" },
42+
...overrides,
43+
});
44+
}
45+
46+
describe("profiles.getByIds", () => {
47+
iit("returns profiles for an organization member", async () => {
48+
const user = await signUp();
49+
const org = await insertOrganization();
50+
await addToOrganization(user.id, org.id, "member");
51+
const website = await insertWebsite({ organizationId: org.id });
52+
await seedProfile(website.id, "user_1");
53+
await seedProfile(website.id, "user_2", { email: null });
54+
55+
const result = await call(appRouter.profiles.getByIds, {
56+
...userContext(user, org.id),
57+
})({
58+
websiteId: website.id,
59+
profileIds: ["user_1", "user_2", "user_missing"],
60+
});
61+
62+
expect(result).toHaveLength(2);
63+
const first = result.find((p) => p.profileId === "user_1");
64+
expect(first?.email).toBe("user_1@example.com");
65+
expect(first?.displayName).toBe("User user_1");
66+
expect(first?.traits).toEqual({ plan: "pro" });
67+
});
68+
69+
iit("does not leak profiles from another organization's website", async () => {
70+
const outsider = await signUp();
71+
const outsiderOrg = await insertOrganization();
72+
await addToOrganization(outsider.id, outsiderOrg.id, "member");
73+
74+
const org = await insertOrganization();
75+
const website = await insertWebsite({ organizationId: org.id });
76+
await seedProfile(website.id, "user_1");
77+
78+
await expectCode(
79+
call(appRouter.profiles.getByIds, {
80+
...userContext(outsider, outsiderOrg.id),
81+
})({
82+
websiteId: website.id,
83+
profileIds: ["user_1"],
84+
}),
85+
"FORBIDDEN"
86+
);
87+
});
88+
});
89+
90+
describe("profiles.get", () => {
91+
iit("returns a profile with its device aliases", async () => {
92+
const user = await signUp();
93+
const org = await insertOrganization();
94+
await addToOrganization(user.id, org.id, "member");
95+
const website = await insertWebsite({ organizationId: org.id });
96+
await seedProfile(website.id, "user_1");
97+
await db().insert(profileAliases).values([
98+
{ websiteId: website.id, anonymousId: "anon_a", profileId: "user_1" },
99+
{ websiteId: website.id, anonymousId: "anon_b", profileId: "user_1" },
100+
]);
101+
102+
const result = await call(appRouter.profiles.get, {
103+
...userContext(user, org.id),
104+
})({ websiteId: website.id, profileId: "user_1" });
105+
106+
expect(result?.profileId).toBe("user_1");
107+
expect(result?.anonymousIds?.sort()).toEqual(["anon_a", "anon_b"]);
108+
});
109+
110+
iit("returns null for an unknown profile", async () => {
111+
const user = await signUp();
112+
const org = await insertOrganization();
113+
await addToOrganization(user.id, org.id, "member");
114+
const website = await insertWebsite({ organizationId: org.id });
115+
116+
const result = await call(appRouter.profiles.get, {
117+
...userContext(user, org.id),
118+
})({ websiteId: website.id, profileId: "user_ghost" });
119+
120+
expect(result).toBeNull();
121+
});
122+
});

apps/api/src/routes/agent.ts

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import {
3232
import { auth } from "@databuddy/auth";
3333
import { db, eq } from "@databuddy/db";
3434
import { agentChats } from "@databuddy/db/schema";
35+
import { config } from "@databuddy/env/app";
3536
import {
3637
appendStreamChunk,
3738
clearActiveStream,
@@ -70,12 +71,22 @@ import { captureError, mergeWideEvent } from "@databuddy/ai/lib/tracing";
7071
import { getAccessibleWebsites } from "@databuddy/ai/lib/accessible-websites";
7172
import { warnAgentStreamRedisSideEffect } from "./agent-stream-errors";
7273

74+
const PROTECTED_RESOURCE_METADATA_URL = `${config.urls.api}/.well-known/oauth-protected-resource`;
75+
7376
function jsonError(status: number, code: string, message: string): Response {
77+
const headers: Record<string, string> = {
78+
"Content-Type": "application/json",
79+
};
80+
if (status === 401) {
81+
headers["WWW-Authenticate"] =
82+
`Bearer resource_metadata="${PROTECTED_RESOURCE_METADATA_URL}"`;
83+
}
84+
7485
return new Response(
7586
JSON.stringify({ success: false, error: message, code }),
7687
{
7788
status,
78-
headers: { "Content-Type": "application/json" },
89+
headers,
7990
}
8091
);
8192
}
@@ -505,6 +516,8 @@ export const agent = new Elysia({ prefix: "/v1/agent" })
505516
.onBeforeHandle(({ isAuthenticated, set }) => {
506517
if (!isAuthenticated) {
507518
set.status = 401;
519+
set.headers["WWW-Authenticate"] =
520+
`Bearer resource_metadata="${PROTECTED_RESOURCE_METADATA_URL}"`;
508521
return {
509522
success: false,
510523
error: "Authentication required",

0 commit comments

Comments
 (0)