Skip to content

Commit 907a983

Browse files
BilalG1BilalG1
authored
Clickhouse sync fixing (#1198)
<!-- Make sure you've read the CONTRIBUTING.md guidelines: https://github.com/stack-auth/stack-auth/blob/dev/CONTRIBUTING.md -->
1 parent 6dd8fac commit 907a983

10 files changed

Lines changed: 36 additions & 23 deletions

File tree

‎.github/workflows/db-migration-backwards-compatibility.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,9 @@ jobs:
150150
- name: Wait on Svix
151151
run: pnpx wait-on tcp:localhost:8113
152152

153+
- name: Wait on ClickHouse
154+
run: pnpx wait-on http://localhost:8136/ping
155+
153156
- name: Initialize database
154157
run: pnpm run db:init
155158

‎.github/workflows/e2e-api-tests.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,9 @@ jobs:
106106
- name: Wait on QStash
107107
run: pnpx wait-on tcp:localhost:8125
108108

109+
- name: Wait on ClickHouse
110+
run: pnpx wait-on http://localhost:8136/ping
111+
109112
- name: Initialize database
110113
run: pnpm run db:init
111114

‎.github/workflows/e2e-custom-base-port-api-tests.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,9 @@ jobs:
100100
- name: Wait on QStash
101101
run: pnpx wait-on tcp:localhost:6725
102102

103+
- name: Wait on ClickHouse
104+
run: pnpx wait-on http://localhost:6736/ping
105+
103106
- name: Initialize database
104107
run: pnpm run db:init
105108

‎apps/backend/scripts/clickhouse-migrations.ts‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -70,28 +70,28 @@ ALTER TABLE analytics_internal.events
7070
UPDATE
7171
data = CAST(concat(
7272
'{',
73-
'\"refresh_token_id\":', toJSONString(JSONExtractString(toJSONString(data), 'refreshTokenId')), ',',
74-
'\"is_anonymous\":', toJSONString(JSONExtract(toJSONString(data), 'isAnonymous', 'Bool')), ',',
75-
'\"ip_info\":', if(
76-
JSONExtractString(toJSONString(data), 'ipInfo.ip') = '',
73+
'"refresh_token_id":', toJSONString(data.refreshTokenId::String), ',',
74+
'"is_anonymous":', if(ifNull(data.isAnonymous::Nullable(Bool), false), 'true', 'false'), ',',
75+
'"ip_info":', if(
76+
isNull(data.ipInfo.ip::Nullable(String)),
7777
'null',
7878
concat(
7979
'{',
80-
'\"ip\":', toJSONString(JSONExtractString(toJSONString(data), 'ipInfo.ip')), ',',
81-
'\"is_trusted\":', toJSONString(JSONExtract(toJSONString(data), 'ipInfo.isTrusted', 'Bool')), ',',
82-
'\"country_code\":', toJSONString(JSONExtract(toJSONString(data), 'ipInfo.countryCode', 'Nullable(String)')), ',',
83-
'\"region_code\":', toJSONString(JSONExtract(toJSONString(data), 'ipInfo.regionCode', 'Nullable(String)')), ',',
84-
'\"city_name\":', toJSONString(JSONExtract(toJSONString(data), 'ipInfo.cityName', 'Nullable(String)')), ',',
85-
'\"latitude\":', toJSONString(JSONExtract(toJSONString(data), 'ipInfo.latitude', 'Nullable(Float64)')), ',',
86-
'\"longitude\":', toJSONString(JSONExtract(toJSONString(data), 'ipInfo.longitude', 'Nullable(Float64)')), ',',
87-
'\"tz_identifier\":', toJSONString(JSONExtract(toJSONString(data), 'ipInfo.tzIdentifier', 'Nullable(String)')),
80+
'"ip":', toJSONString(data.ipInfo.ip::String), ',',
81+
'"is_trusted":', if(ifNull(data.ipInfo.isTrusted::Nullable(Bool), false), 'true', 'false'), ',',
82+
'"country_code":', if(isNull(data.ipInfo.countryCode::Nullable(String)), 'null', toJSONString(data.ipInfo.countryCode::String)), ',',
83+
'"region_code":', if(isNull(data.ipInfo.regionCode::Nullable(String)), 'null', toJSONString(data.ipInfo.regionCode::String)), ',',
84+
'"city_name":', if(isNull(data.ipInfo.cityName::Nullable(String)), 'null', toJSONString(data.ipInfo.cityName::String)), ',',
85+
'"latitude":', if(isNull(data.ipInfo.latitude::Nullable(Float64)), 'null', toString(data.ipInfo.latitude::Float64)), ',',
86+
'"longitude":', if(isNull(data.ipInfo.longitude::Nullable(Float64)), 'null', toString(data.ipInfo.longitude::Float64)), ',',
87+
'"tz_identifier":', if(isNull(data.ipInfo.tzIdentifier::Nullable(String)), 'null', toJSONString(data.ipInfo.tzIdentifier::String)),
8888
'}'
8989
)
9090
),
9191
'}'
9292
) AS JSON)
9393
WHERE event_type = '$token-refresh'
94-
AND JSONHas(toJSONString(data), 'refreshTokenId');
94+
AND data.refreshTokenId::Nullable(String) IS NOT NULL;
9595
`;
9696

9797
// Normalizes legacy $sign-up-rule-trigger rows (camelCase JSON) to the new format:

‎apps/backend/src/app/api/latest/internal/external-db-sync/poller/route.ts‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,13 +172,11 @@ export const GET = createSmartRouteHandler({
172172
}
173173

174174
const flowControl = options.flowControl as UpstashRequest["flowControl"];
175-
const deduplicationId = options.deduplicationId as UpstashRequest["deduplicationId"];
176175

177176
return {
178177
url: fullUrl,
179178
body: options.body,
180179
...(flowControl ? { flowControl } : {}),
181-
...(deduplicationId ? { deduplicationId } : {})
182180
};
183181
}
184182

‎apps/backend/src/lib/external-db-sync-queue.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,7 @@ export async function enqueueExternalDbSyncBatch(tenancyIds: string[]): Promise<
3333
json_build_object(
3434
'url', '/api/latest/internal/external-db-sync/sync-engine',
3535
'body', json_build_object('tenancyId', t.tenancy_id),
36-
'flowControl', json_build_object('key', 'sentinel-sync-key', 'parallelism', 20),
37-
'deduplicationId', t.tenancy_id
36+
'flowControl', json_build_object('key', 'sentinel-sync-key', 'parallelism', 20)
3837
),
3938
NULL,
4039
'sentinel-sync-key-' || t.tenancy_id

‎apps/backend/src/lib/external-db-sync.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -566,6 +566,9 @@ async function syncPostgresMapping(
566566
if (rows.length === 0) {
567567
break;
568568
}
569+
if (rows.length > 1) {
570+
console.log("db-sync-postgres: more than 1 row returned from source db fetch", { tenancyId, numRows: rows.length });
571+
}
569572

570573
await pushRowsToExternalDb(
571574
externalClient,
@@ -644,6 +647,9 @@ async function syncClickhouseMapping(
644647
if (rows.length === 0) {
645648
break;
646649
}
650+
if (rows.length > 1) {
651+
console.log("db-sync-clickhouse: more than 1 row returned from source db fetch", { tenancyId, numRows: rows.length });
652+
}
647653

648654
await pushRowsToClickhouse(
649655
client,

‎apps/e2e/tests/backend/endpoints/api/v1/auth/sign-up-rules.test.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -266,8 +266,8 @@ describe("sign-up rules", () => {
266266
"status": 403,
267267
"body": {
268268
"code": "SIGN_UP_REJECTED",
269-
"details": { "message": "Your sign up was rejected. Please contact us for more information." },
270-
"error": "Your sign up was rejected. Please contact us for more information.",
269+
"details": { "message": "Your sign up was rejected by an administrator's sign-up rule." },
270+
"error": "Your sign up was rejected by an administrator's sign-up rule.",
271271
},
272272
"headers": Headers {
273273
"x-stack-known-error": "SIGN_UP_REJECTED",

‎package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@
2626
"stop-deps": "POSTGRES_DELAY_MS=0 pnpm run deps-compose kill && POSTGRES_DELAY_MS=0 pnpm run deps-compose down -v",
2727
"wait-until-postgres-is-ready:pg_isready": "until pg_isready -h localhost -p ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}28 && pg_isready -h localhost -p ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}34; do sleep 1; done",
2828
"wait-until-postgres-is-ready": "command -v pg_isready >/dev/null 2>&1 && pnpm run wait-until-postgres-is-ready:pg_isready || sleep 10 # not everyone has pg_isready installed, so we fallback to sleeping",
29-
"start-deps:no-delay": "pnpm pre && pnpm run deps-compose up --detach --build && pnpm run wait-until-postgres-is-ready && pnpm run db:init && echo \"\\nDependencies started in the background as Docker containers. 'pnpm run stop-deps' to stop them\"n",
29+
"wait-until-clickhouse-is-ready": "pnpx wait-on http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}36/ping",
30+
"start-deps:no-delay": "pnpm pre && pnpm run deps-compose up --detach --build && pnpm run wait-until-postgres-is-ready && pnpm run wait-until-clickhouse-is-ready && pnpm run db:init && echo \"\\nDependencies started in the background as Docker containers. 'pnpm run stop-deps' to stop them\"n",
3031
"start-deps": "POSTGRES_DELAY_MS=${POSTGRES_DELAY_MS:-0} pnpm run start-deps:no-delay",
3132
"restart-deps": "pnpm pre && pnpm run stop-deps && pnpm run start-deps",
3233
"restart-deps:no-delay": "pnpm pre && pnpm run stop-deps && pnpm run start-deps:no-delay",

‎packages/stack-shared/src/config/db-sync-mappings.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,9 @@ export const DEFAULT_DB_SYNC_MAPPINGS = {
3535
primary_email Nullable(String),
3636
primary_email_verified UInt8,
3737
signed_up_at DateTime64(3, 'UTC'),
38-
client_metadata JSON,
39-
client_read_only_metadata JSON,
40-
server_metadata JSON,
38+
client_metadata String,
39+
client_read_only_metadata String,
40+
server_metadata String,
4141
is_anonymous UInt8,
4242
restricted_by_admin UInt8,
4343
restricted_by_admin_reason Nullable(String),

0 commit comments

Comments
 (0)