Skip to content

Commit d1e4bd2

Browse files
authored
Merge branch 'dev' into dashboard/changelog
2 parents da92216 + 2634095 commit d1e4bd2

180 files changed

Lines changed: 22523 additions & 1208 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.github/workflows/docker-server-build-run.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,13 @@ jobs:
2222
docker run -d --name db -e POSTGRES_USER=postgres -e POSTGRES_PASSWORD=password -e POSTGRES_DB=stackframe -p 8128:5432 postgres:latest
2323
sleep 5
2424
docker logs db
25+
26+
- name: Setup clickhouse
27+
run: |
28+
docker run -d --name clickhouse -e CLICKHOUSE_DB=analytics -e CLICKHOUSE_USER=stackframe -e CLICKHOUSE_PASSWORD=password -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 -p 8133:8123 clickhouse/clickhouse-server:25.10
29+
sleep 5
30+
docker logs clickhouse
31+
2532
2633
- name: Build Docker image
2734
run: docker build -f docker/server/Dockerfile -t server .
Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
name: Publish Swift SDK to prerelease repo
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
paths:
8+
- 'sdks/implementations/swift/**'
9+
10+
concurrency:
11+
group: ${{ github.workflow }}-${{ github.ref }}
12+
cancel-in-progress: false # Don't cancel publishing in progress
13+
14+
jobs:
15+
publish:
16+
runs-on: ubuntu-latest
17+
18+
steps:
19+
- name: Checkout source repo
20+
uses: actions/checkout@v4
21+
with:
22+
path: source
23+
24+
- name: Read version from package.json
25+
id: version
26+
run: |
27+
VERSION=$(jq -r '.version' source/sdks/implementations/swift/package.json)
28+
echo "version=$VERSION" >> $GITHUB_OUTPUT
29+
echo "Swift SDK version: $VERSION"
30+
31+
- name: Check if tag already exists in target repo
32+
id: check-tag
33+
run: |
34+
TAG="v${{ steps.version.outputs.version }}"
35+
echo "Checking if tag $TAG exists in stack-auth/swift-sdk-prerelease..."
36+
37+
# Use the GitHub API to check if the tag exists
38+
HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" \
39+
-H "Authorization: Bearer ${{ secrets.SWIFT_SDK_PUBLISH_TOKEN }}" \
40+
-H "Accept: application/vnd.github+json" \
41+
"https://api.github.com/repos/stack-auth/swift-sdk-prerelease/git/refs/tags/$TAG")
42+
43+
if [ "$HTTP_STATUS" = "200" ]; then
44+
echo "Tag $TAG already exists, skipping publish"
45+
echo "exists=true" >> $GITHUB_OUTPUT
46+
else
47+
echo "Tag $TAG does not exist, will publish"
48+
echo "exists=false" >> $GITHUB_OUTPUT
49+
fi
50+
51+
- name: Clone target repo
52+
if: steps.check-tag.outputs.exists == 'false'
53+
run: |
54+
git clone https://x-access-token:${{ secrets.SWIFT_SDK_PUBLISH_TOKEN }}@github.com/stack-auth/swift-sdk-prerelease.git target
55+
56+
- name: Copy Swift SDK to target repo
57+
if: steps.check-tag.outputs.exists == 'false'
58+
run: |
59+
# Remove all files except .git from target
60+
cd target
61+
find . -maxdepth 1 -not -name '.git' -not -name '.' -exec rm -rf {} +
62+
cd ..
63+
64+
# Copy everything from Swift SDK
65+
cp -r source/sdks/implementations/swift/* target/
66+
cp source/sdks/implementations/swift/.gitignore target/ 2>/dev/null || true
67+
68+
# Remove package.json (it's only for turborepo integration, not part of the Swift package)
69+
rm -f target/package.json
70+
71+
- name: Commit and push to target repo
72+
if: steps.check-tag.outputs.exists == 'false'
73+
run: |
74+
cd target
75+
git config user.email "github-actions[bot]@users.noreply.github.com"
76+
git config user.name "github-actions[bot]"
77+
78+
git add -A
79+
80+
# Check if there are changes to commit
81+
if git diff --staged --quiet; then
82+
echo "No changes to commit"
83+
else
84+
git commit -m "Release v${{ steps.version.outputs.version }}"
85+
fi
86+
87+
# Create and push tag
88+
TAG="v${{ steps.version.outputs.version }}"
89+
git tag "$TAG"
90+
git push origin main --tags
91+
92+
echo "Successfully published Swift SDK v${{ steps.version.outputs.version }}"
93+
94+
- name: Summary
95+
run: |
96+
if [ "${{ steps.check-tag.outputs.exists }}" = "true" ]; then
97+
echo "::notice::Skipped publishing - tag v${{ steps.version.outputs.version }} already exists"
98+
else
99+
echo "::notice::Published Swift SDK v${{ steps.version.outputs.version }} to stack-auth/swift-sdk-prerelease"
100+
fi

apps/backend/.env

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,13 @@ STACK_QSTASH_TOKEN=
8181
STACK_QSTASH_CURRENT_SIGNING_KEY=
8282
STACK_QSTASH_NEXT_SIGNING_KEY=
8383

84+
# Clickhouse
85+
STACK_CLICKHOUSE_URL=# URL of the Clickhouse instance
86+
STACK_CLICKHOUSE_ADMIN_USER=# username of the admin account
87+
STACK_CLICKHOUSE_ADMIN_PASSWORD=# password of the admin account
88+
STACK_CLICKHOUSE_EXTERNAL_PASSWORD=# a randomly generated secure string. The user account will be created automatically
89+
90+
8491
# Misc
8592
STACK_ACCESS_TOKEN_EXPIRATION_TIME=# enter the expiration time for the access token here. Optional, don't specify it for default value
8693
STACK_SETUP_ADMIN_GITHUB_ID=# enter the account ID of the admin user here, and after running the seed script they will be able to access the internal project in the Stack dashboard. Optional, don't specify it for default value

apps/backend/.env.development

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,3 +75,9 @@ STACK_QSTASH_URL=http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}25
7575
STACK_QSTASH_TOKEN=eyJVc2VySUQiOiJkZWZhdWx0VXNlciIsIlBhc3N3b3JkIjoiZGVmYXVsdFBhc3N3b3JkIn0=
7676
STACK_QSTASH_CURRENT_SIGNING_KEY=sig_7kYjw48mhY7kAjqNGcy6cr29RJ6r
7777
STACK_QSTASH_NEXT_SIGNING_KEY=sig_5ZB6DVzB1wjE8S6rZ7eenA8Pdnhs
78+
79+
# Clickhouse
80+
STACK_CLICKHOUSE_URL=http://localhost:${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}36
81+
STACK_CLICKHOUSE_ADMIN_USER=stackframe
82+
STACK_CLICKHOUSE_ADMIN_PASSWORD=PASSWORD-PLACEHOLDER--9gKyMxJeMx
83+
STACK_CLICKHOUSE_EXTERNAL_PASSWORD=PASSWORD-PLACEHOLDER--EZeHscBMzE

apps/backend/package.json

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@stackframe/stack-backend",
3-
"version": "2.8.60",
3+
"version": "2.8.62",
44
"repository": "https://github.com/stack-auth/stack-auth",
55
"private": true,
66
"type": "module",
@@ -25,6 +25,7 @@
2525
"codegen": "pnpm run with-env pnpm run generate-migration-imports && pnpm run with-env bash -c 'if [ \"$STACK_ACCELERATE_ENABLED\" = \"true\" ]; then pnpm run prisma generate --no-engine; else pnpm run codegen-prisma; fi' && pnpm run codegen-docs && pnpm run codegen-route-info",
2626
"codegen:watch": "concurrently -n \"prisma,docs,route-info,migration-imports\" -k \"pnpm run codegen-prisma:watch\" \"pnpm run codegen-docs:watch\" \"pnpm run codegen-route-info:watch\" \"pnpm run generate-migration-imports:watch\"",
2727
"psql-inner": "psql $(echo $STACK_DATABASE_CONNECTION_STRING | sed 's/\\?.*$//')",
28+
"clickhouse": "pnpm run with-env clickhouse-client --host localhost --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}37 --user stackframe --password PASSWORD-PLACEHOLDER--9gKyMxJeMx",
2829
"psql": "pnpm run with-env:dev pnpm run psql-inner",
2930
"prisma-studio": "pnpm run with-env:dev prisma studio --port ${NEXT_PUBLIC_STACK_PORT_PREFIX:-81}06 --browser none",
3031
"prisma:dev": "pnpm run with-env:dev prisma",
@@ -41,7 +42,7 @@
4142
"codegen-docs:watch": "pnpm run with-env tsx watch --exclude '**/node_modules/**' --clear-screen=false scripts/generate-openapi-fumadocs.ts",
4243
"generate-keys": "pnpm run with-env tsx scripts/generate-keys.ts",
4344
"db-seed-script": "pnpm run db:seed",
44-
"verify-data-integrity": "pnpm run with-env:dev tsx scripts/verify-data-integrity.ts",
45+
"verify-data-integrity": "pnpm run with-env:dev tsx scripts/verify-data-integrity/index.ts",
4546
"run-email-queue": "pnpm run with-env:dev tsx scripts/run-email-queue.ts"
4647
},
4748
"prisma": {
@@ -50,6 +51,7 @@
5051
"dependencies": {
5152
"@ai-sdk/openai": "^1.3.23",
5253
"@aws-sdk/client-s3": "^3.855.0",
54+
"@clickhouse/client": "^1.14.0",
5355
"@node-oauth/oauth2-server": "^5.1.0",
5456
"@opentelemetry/api": "^1.9.0",
5557
"@opentelemetry/api-logs": "^0.53.0",
@@ -86,7 +88,7 @@
8688
"freestyle-sandboxes": "^0.1.6",
8789
"jose": "^6.1.3",
8890
"json-diff": "^1.0.6",
89-
"next": "16.1.1",
91+
"next": "16.1.5",
9092
"nodemailer": "^6.9.10",
9193
"oidc-provider": "^8.5.1",
9294
"openid-client": "5.6.4",
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
-- SPLIT_STATEMENT_SENTINEL
2+
-- SINGLE_STATEMENT_SENTINEL
3+
-- RUN_OUTSIDE_TRANSACTION_SENTINEL
4+
-- Add index on createdAt for efficient range queries (used by ClickHouse migration and similar count queries).
5+
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_event_created_at ON "Event" ("createdAt");
6+
7+
-- SPLIT_STATEMENT_SENTINEL
8+
-- SINGLE_STATEMENT_SENTINEL
9+
-- RUN_OUTSIDE_TRANSACTION_SENTINEL
10+
-- Add composite index (createdAt, id) for cursor-based pagination queries.
11+
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_event_created_at_id ON "Event" ("createdAt", "id");

apps/backend/prisma/schema.prisma

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -659,6 +659,8 @@ model Event {
659659
// =============================== END END USER PROPERTIES ===============================
660660
661661
@@index([data(ops: JsonbPathOps)], type: Gin)
662+
@@index([createdAt])
663+
@@index([createdAt, id])
662664
}
663665

664666
// An IP address that was seen in an event. Use the location fields instead of refetching the location from the ip, as the real-world geoip data may have changed since the event was logged.
Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
import { getClickhouseAdminClient } from "@/lib/clickhouse";
2+
import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
3+
4+
export async function runClickhouseMigrations() {
5+
console.log("[Clickhouse] Running Clickhouse migrations...");
6+
const client = getClickhouseAdminClient();
7+
const clickhouseExternalPassword = getEnvVariable("STACK_CLICKHOUSE_EXTERNAL_PASSWORD");
8+
await client.exec({
9+
query: "CREATE USER IF NOT EXISTS limited_user IDENTIFIED WITH sha256_password BY {clickhouseExternalPassword:String}",
10+
query_params: { clickhouseExternalPassword },
11+
});
12+
// todo: create migration files
13+
await client.exec({ query: EXTERNAL_ANALYTICS_DB_SQL });
14+
await client.exec({ query: EVENTS_TABLE_BASE_SQL });
15+
await client.exec({ query: EVENTS_VIEW_SQL });
16+
const queries = [
17+
"REVOKE ALL PRIVILEGES ON *.* FROM limited_user;",
18+
"REVOKE ALL FROM limited_user;",
19+
"GRANT SELECT ON default.events TO limited_user;",
20+
];
21+
await client.exec({
22+
query: "CREATE ROW POLICY IF NOT EXISTS events_project_isolation ON default.events FOR SELECT USING project_id = getSetting('SQL_project_id') AND branch_id = getSetting('SQL_branch_id') TO limited_user",
23+
});
24+
for (const query of queries) {
25+
await client.exec({ query });
26+
}
27+
console.log("[Clickhouse] Clickhouse migrations complete");
28+
await client.close();
29+
}
30+
31+
const EVENTS_TABLE_BASE_SQL = `
32+
CREATE TABLE IF NOT EXISTS analytics_internal.events (
33+
event_type LowCardinality(String),
34+
event_at DateTime64(3, 'UTC'),
35+
data JSON,
36+
project_id String,
37+
branch_id String,
38+
user_id String,
39+
team_id String,
40+
refresh_token_id String,
41+
is_anonymous Boolean,
42+
session_id String,
43+
ip_address String,
44+
created_at DateTime64(3, 'UTC') DEFAULT now64(3)
45+
)
46+
ENGINE MergeTree
47+
PARTITION BY toYYYYMM(event_at)
48+
ORDER BY (project_id, branch_id, event_at);
49+
`;
50+
51+
const EVENTS_VIEW_SQL = `
52+
CREATE OR REPLACE VIEW default.events
53+
SQL SECURITY DEFINER
54+
AS
55+
SELECT *
56+
FROM analytics_internal.events;
57+
`;
58+
59+
const EXTERNAL_ANALYTICS_DB_SQL = `
60+
CREATE DATABASE IF NOT EXISTS analytics_internal;
61+
`;

apps/backend/scripts/db-migrations.ts

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,25 @@ import { applyMigrations } from "@/auto-migrations";
22
import { MIGRATION_FILES_DIR, getMigrationFiles } from "@/auto-migrations/utils";
33
import { Prisma } from "@/generated/prisma/client";
44
import { globalPrismaClient, globalPrismaSchema, sqlQuoteIdent } from "@/prisma-client";
5-
import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
65
import { spawnSync } from "child_process";
76
import fs from "fs";
87
import path from "path";
98
import * as readline from "readline";
109
import { seed } from "../prisma/seed";
10+
import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
11+
import { runClickhouseMigrations } from "./clickhouse-migrations";
12+
import { getClickhouseAdminClient } from "@/lib/clickhouse";
13+
14+
const getClickhouseClient = () => getClickhouseAdminClient();
1115

1216
const dropSchema = async () => {
1317
await globalPrismaClient.$executeRaw(Prisma.sql`DROP SCHEMA ${sqlQuoteIdent(globalPrismaSchema)} CASCADE`);
1418
await globalPrismaClient.$executeRaw(Prisma.sql`CREATE SCHEMA ${sqlQuoteIdent(globalPrismaSchema)}`);
1519
await globalPrismaClient.$executeRaw(Prisma.sql`GRANT ALL ON SCHEMA ${sqlQuoteIdent(globalPrismaSchema)} TO postgres`);
1620
await globalPrismaClient.$executeRaw(Prisma.sql`GRANT ALL ON SCHEMA ${sqlQuoteIdent(globalPrismaSchema)} TO public`);
21+
const clickhouseClient = getClickhouseClient();
22+
await clickhouseClient.command({ query: "DROP DATABASE IF EXISTS analytics_internal" });
23+
await clickhouseClient.command({ query: "CREATE DATABASE IF NOT EXISTS analytics_internal" });
1724
};
1825

1926

@@ -163,6 +170,8 @@ const migrate = async (selectedMigrationFiles?: { migrationName: string, sql: st
163170

164171
console.log('='.repeat(60) + '\n');
165172

173+
await runClickhouseMigrations();
174+
166175
return result;
167176
};
168177

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
import { getEnvVariable } from "@stackframe/stack-shared/dist/utils/env";
2+
import { StackAssertionError } from "@stackframe/stack-shared/dist/utils/errors";
3+
import { deepPlainEquals, filterUndefined } from "@stackframe/stack-shared/dist/utils/objects";
4+
import { deindent } from "@stackframe/stack-shared/dist/utils/strings";
5+
6+
export type EndpointOutput = {
7+
status: number,
8+
responseJson: any,
9+
};
10+
11+
export type OutputData = Record<string, EndpointOutput[]>;
12+
13+
export type ExpectStatusCode = <T = any>(
14+
expectedStatusCode: number,
15+
endpoint: string,
16+
request: RequestInit,
17+
) => Promise<T>;
18+
19+
export function createApiHelpers(options: {
20+
currentOutputData: OutputData,
21+
targetOutputData?: OutputData,
22+
}) {
23+
const { currentOutputData, targetOutputData } = options;
24+
25+
function appendOutputData(endpoint: string, output: EndpointOutput) {
26+
if (!(endpoint in currentOutputData)) {
27+
currentOutputData[endpoint] = [];
28+
}
29+
const newLength = currentOutputData[endpoint].push(output);
30+
if (targetOutputData) {
31+
if (!(endpoint in targetOutputData)) {
32+
throw new StackAssertionError(deindent`
33+
Output data mismatch for endpoint ${endpoint}:
34+
Expected ${endpoint} to be in targetOutputData, but it is not.
35+
`, { endpoint });
36+
}
37+
if (targetOutputData[endpoint].length < newLength) {
38+
throw new StackAssertionError(deindent`
39+
Output data mismatch for endpoint ${endpoint}:
40+
Expected ${targetOutputData[endpoint].length} outputs but got at least ${newLength}.
41+
`, { endpoint });
42+
}
43+
if (!(deepPlainEquals(targetOutputData[endpoint][newLength - 1], output))) {
44+
throw new StackAssertionError(deindent`
45+
Output data mismatch for endpoint ${endpoint}:
46+
Expected output[${JSON.stringify(endpoint)}][${newLength - 1}] to be:
47+
${JSON.stringify(targetOutputData[endpoint][newLength - 1], null, 2)}
48+
but got:
49+
${JSON.stringify(output, null, 2)}.
50+
`, { endpoint });
51+
}
52+
}
53+
}
54+
55+
const expectStatusCode: ExpectStatusCode = async (expectedStatusCode, endpoint, request) => {
56+
const apiUrl = new URL(getEnvVariable("NEXT_PUBLIC_STACK_API_URL"));
57+
const response = await fetch(new URL(endpoint, apiUrl), {
58+
...request,
59+
headers: {
60+
"x-stack-disable-artificial-development-delay": "yes",
61+
"x-stack-development-disable-extended-logging": "yes",
62+
...filterUndefined(request.headers ?? {}),
63+
},
64+
});
65+
66+
const responseText = await response.text();
67+
68+
if (response.status !== expectedStatusCode) {
69+
throw new StackAssertionError(deindent`
70+
Expected status code ${expectedStatusCode} but got ${response.status} for ${endpoint}:
71+
72+
${responseText}
73+
`, { request, response });
74+
}
75+
76+
const responseJson = JSON.parse(responseText);
77+
const currentOutput: EndpointOutput = {
78+
status: response.status,
79+
responseJson,
80+
};
81+
82+
appendOutputData(endpoint, currentOutput);
83+
84+
return responseJson;
85+
};
86+
87+
return {
88+
appendOutputData,
89+
expectStatusCode,
90+
};
91+
}
92+

0 commit comments

Comments
 (0)