Skip to content

Commit d09a180

Browse files
BilalG1aadesh18N2D4
authored
clickhouse user sync (#1159)
<!-- Make sure you've read the CONTRIBUTING.md guidelines: https://github.com/stack-auth/stack-auth/blob/dev/CONTRIBUTING.md --> <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Real-time AI search with project-scoped analytics and dynamic query execution; streaming AI responses replace the placeholder flow. * External DB sync adds ClickHouse support: users sync, sync metadata tracking, tenancy-aware status, and per-mapping throttling. * AI assistant UI shows expandable tool-invocation results and streams via the real AI pipeline. * **Chores** * Dashboard dependencies and workspace exclusions updated; development OpenAI env var added; editor config flag toggled. * **Tests** * E2E coverage extended to validate ClickHouse user sync and analytics queries. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Co-authored-by: aadesh18 <110230993+aadesh18@users.noreply.github.com> Co-authored-by: Konsti Wohlwend <n2d4xc@gmail.com>
1 parent 6673e63 commit d09a180

25 files changed

Lines changed: 1750 additions & 972 deletions

File tree

‎.github/workflows/e2e-source-of-truth-api-tests.yaml‎

Lines changed: 0 additions & 178 deletions
This file was deleted.

‎.vscode/settings.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,5 +174,6 @@
174174
"((?:<!-- *)?(?:#|// @|//|./\\*+|<!--|--|\\* @|{!|{{!--|{{!) *(?:IDEA)(?:\\s*\\([^)]+\\))?:?)((?!\\w)(?: *-->| *\\*/| *!}| *--}}| *}}|(?= *(?:[^:]//|/\\*+|<!--|@|--|{!|{{!--|{{!))|(?: +[^\\n@]*?)(?= *(?:[^:]//|/\\*+|<!--|@|--(?!>)|{!|{{!--|{{!))|(?: +[^@\\n]+)?))": [],
175175
},
176176
"editor.formatOnSaveMode": "file",
177-
"git.ignoreLimitWarning": true
177+
"git.ignoreLimitWarning": true,
178+
"chatgpt.commentCodeLensEnabled": false
178179
}

‎apps/backend/scripts/clickhouse-migrations.ts‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,24 @@ export async function runClickhouseMigrations() {
1111
});
1212
// todo: create migration files
1313
await client.exec({ query: EXTERNAL_ANALYTICS_DB_SQL });
14+
await client.exec({ query: SYNC_METADATA_TABLE_SQL });
1415
await client.exec({ query: EVENTS_TABLE_BASE_SQL });
1516
await client.exec({ query: EVENTS_VIEW_SQL });
17+
await client.exec({ query: USERS_TABLE_BASE_SQL });
18+
await client.exec({ query: USERS_VIEW_SQL });
19+
await client.exec({ query: TOKEN_REFRESH_EVENT_ROW_FORMAT_MUTATION_SQL });
1620
const queries = [
1721
"REVOKE ALL PRIVILEGES ON *.* FROM limited_user;",
1822
"REVOKE ALL FROM limited_user;",
1923
"GRANT SELECT ON default.events TO limited_user;",
24+
"GRANT SELECT ON default.users TO limited_user;",
2025
];
2126
await client.exec({
2227
query: "CREATE ROW POLICY IF NOT EXISTS events_project_isolation ON default.events FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user",
2328
});
29+
await client.exec({
30+
query: "CREATE ROW POLICY IF NOT EXISTS users_project_isolation ON default.users FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user",
31+
});
2432
for (const query of queries) {
2533
await client.exec({ query });
2634
}
@@ -52,6 +60,101 @@ SELECT *
5260
FROM analytics_internal.events;
5361
`;
5462

63+
// Normalizes legacy $token-refresh rows (camelCase JSON) to the new format:
64+
// - Row identity stays in columns (project_id/branch_id/user_id)
65+
// - data JSON becomes { refresh_token_id, is_anonymous, ip_info } (snake_case)
66+
// Assumption: all legacy rows have the camelCase format.
67+
const TOKEN_REFRESH_EVENT_ROW_FORMAT_MUTATION_SQL = `
68+
ALTER TABLE analytics_internal.events
69+
UPDATE
70+
data = CAST(concat(
71+
'{',
72+
'\"refresh_token_id\":', toJSONString(JSONExtractString(toJSONString(data), 'refreshTokenId')), ',',
73+
'\"is_anonymous\":', toJSONString(JSONExtract(toJSONString(data), 'isAnonymous', 'Bool')), ',',
74+
'\"ip_info\":', if(
75+
JSONExtractString(toJSONString(data), 'ipInfo.ip') = '',
76+
'null',
77+
concat(
78+
'{',
79+
'\"ip\":', toJSONString(JSONExtractString(toJSONString(data), 'ipInfo.ip')), ',',
80+
'\"is_trusted\":', toJSONString(JSONExtract(toJSONString(data), 'ipInfo.isTrusted', 'Bool')), ',',
81+
'\"country_code\":', toJSONString(JSONExtract(toJSONString(data), 'ipInfo.countryCode', 'Nullable(String)')), ',',
82+
'\"region_code\":', toJSONString(JSONExtract(toJSONString(data), 'ipInfo.regionCode', 'Nullable(String)')), ',',
83+
'\"city_name\":', toJSONString(JSONExtract(toJSONString(data), 'ipInfo.cityName', 'Nullable(String)')), ',',
84+
'\"latitude\":', toJSONString(JSONExtract(toJSONString(data), 'ipInfo.latitude', 'Nullable(Float64)')), ',',
85+
'\"longitude\":', toJSONString(JSONExtract(toJSONString(data), 'ipInfo.longitude', 'Nullable(Float64)')), ',',
86+
'\"tz_identifier\":', toJSONString(JSONExtract(toJSONString(data), 'ipInfo.tzIdentifier', 'Nullable(String)')),
87+
'}'
88+
)
89+
),
90+
'}'
91+
) AS JSON)
92+
WHERE event_type = '$token-refresh'
93+
AND JSONHas(toJSONString(data), 'refreshTokenId');
94+
`;
95+
96+
const USERS_TABLE_BASE_SQL = `
97+
CREATE TABLE IF NOT EXISTS analytics_internal.users (
98+
project_id String,
99+
branch_id String,
100+
id UUID,
101+
display_name Nullable(String),
102+
profile_image_url Nullable(String),
103+
primary_email Nullable(String),
104+
primary_email_verified UInt8,
105+
signed_up_at DateTime64(3, 'UTC'),
106+
client_metadata JSON,
107+
client_read_only_metadata JSON,
108+
server_metadata JSON,
109+
is_anonymous UInt8,
110+
restricted_by_admin UInt8,
111+
restricted_by_admin_reason Nullable(String),
112+
restricted_by_admin_private_details Nullable(String),
113+
sync_sequence_id Int64,
114+
sync_is_deleted UInt8,
115+
sync_created_at DateTime64(3, 'UTC') DEFAULT now64(3)
116+
)
117+
ENGINE ReplacingMergeTree(sync_sequence_id)
118+
PARTITION BY toYYYYMM(signed_up_at)
119+
ORDER BY (project_id, branch_id, id);
120+
`;
121+
122+
const USERS_VIEW_SQL = `
123+
CREATE OR REPLACE VIEW default.users
124+
SQL SECURITY DEFINER
125+
AS
126+
SELECT
127+
project_id,
128+
branch_id,
129+
id,
130+
display_name,
131+
profile_image_url,
132+
primary_email,
133+
primary_email_verified,
134+
signed_up_at,
135+
client_metadata,
136+
client_read_only_metadata,
137+
server_metadata,
138+
is_anonymous,
139+
restricted_by_admin,
140+
restricted_by_admin_reason,
141+
restricted_by_admin_private_details
142+
FROM analytics_internal.users
143+
FINAL
144+
WHERE sync_is_deleted = 0;
145+
`;
146+
147+
const SYNC_METADATA_TABLE_SQL = `
148+
CREATE TABLE IF NOT EXISTS analytics_internal._stack_sync_metadata (
149+
tenancy_id UUID,
150+
mapping_name String,
151+
last_synced_sequence_id Int64,
152+
updated_at DateTime64(3, 'UTC') DEFAULT now64(3)
153+
)
154+
ENGINE ReplacingMergeTree(updated_at)
155+
ORDER BY (tenancy_id, mapping_name);
156+
`;
157+
55158
const EXTERNAL_ANALYTICS_DB_SQL = `
56159
CREATE DATABASE IF NOT EXISTS analytics_internal;
57160
`;

0 commit comments

Comments
 (0)