Merge branch 'dev' into promptless/document-apple-bundle-ids

Author: madster456

.github/workflows/db-migration-backwards-compatibility.yaml

@@ -215,7 +215,7 @@ jobs:
         run: pnpm test
 
       - name: Verify data integrity
-        run: pnpm run verify-data-integrity
+        run: pnpm run verify-data-integrity --no-bail
 
       - name: Print Docker Compose logs
         if: always()

.github/workflows/e2e-api-tests.yaml

@@ -156,7 +156,7 @@ jobs:
         run: pnpm test ${{ matrix.freestyle-mode == 'prod' && '--min-workers=1 --max-workers=1' || '' }}
 
       - name: Verify data integrity
-        run: pnpm run verify-data-integrity
+        run: pnpm run verify-data-integrity --no-bail
 
       - name: Print Docker Compose logs
         if: always()

.github/workflows/e2e-custom-base-port-api-tests.yaml

@@ -150,7 +150,7 @@ jobs:
         run: pnpm test
 
       - name: Verify data integrity
-        run: pnpm run verify-data-integrity
+        run: pnpm run verify-data-integrity --no-bail
 
       - name: Print Docker Compose logs
         if: always()

.github/workflows/e2e-source-of-truth-api-tests.yaml

@@ -156,7 +156,7 @@ jobs:
         run: pnpm test
 
       - name: Verify data integrity
-        run: pnpm run verify-data-integrity
+        run: pnpm run verify-data-integrity --no-bail
 
       - name: Print Docker Compose logs
         if: always()

apps/backend/prisma/migrations/20260201240000_event_created_at_index/migration.sql

@@ -2,10 +2,10 @@
 -- SINGLE_STATEMENT_SENTINEL
 -- RUN_OUTSIDE_TRANSACTION_SENTINEL
 -- Add index on createdAt for efficient range queries (used by ClickHouse migration and similar count queries).
-CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_event_created_at ON "Event" ("createdAt");
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "Event_createdAt_idx" ON /* SCHEMA_NAME_SENTINEL */."Event" ("createdAt");
 
 -- SPLIT_STATEMENT_SENTINEL
 -- SINGLE_STATEMENT_SENTINEL
 -- RUN_OUTSIDE_TRANSACTION_SENTINEL
 -- Add composite index (createdAt, id) for cursor-based pagination queries.
-CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_event_created_at_id ON "Event" ("createdAt", "id");
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "Event_createdAt_id_idx" ON /* SCHEMA_NAME_SENTINEL */."Event" ("createdAt", "id");

apps/backend/scripts/clickhouse-migrations.ts

@@ -35,12 +35,8 @@ CREATE TABLE IF NOT EXISTS analytics_internal.events (
     data             JSON,
     project_id       String,
     branch_id        String,
-    user_id          String,
-    team_id          String,
-    refresh_token_id String,
-    is_anonymous     Boolean,
-    session_id       String,
-    ip_address       String,
+    user_id          Nullable(String),
+    team_id          Nullable(String),
     created_at DateTime64(3, 'UTC') DEFAULT now64(3)
 )
 ENGINE MergeTree

apps/backend/scripts/verify-data-integrity/index.ts

@@ -21,8 +21,6 @@ const USE_MOCK_STRIPE_API = STRIPE_SECRET_KEY === "sk_test_mockstripekey";
 let targetOutputData: OutputData | undefined = undefined;
 const currentOutputData: OutputData = {};
 
-const recurse = createRecurse();
-
 async function main() {
   console.log();
   console.log();
@@ -80,6 +78,13 @@ async function main() {
   const shouldVerifyOutput = flags.includes("--verify-output");
   const shouldSkipNeon = flags.includes("--skip-neon");
   const recentFirst = flags.includes("--recent-first");
+  const noBail = flags.includes("--no-bail");
+
+  const { recurse, collectedErrors } = createRecurse({ noBail });
+
+  if (noBail) {
+    console.log(`Running in no-bail mode: will continue on errors and report all at the end.`);
+  }
 
   if (shouldSaveOutput) {
     console.log(`Will save output to ${OUTPUT_FILE_PATH}`);
@@ -318,6 +323,27 @@ async function main() {
     console.log(`Output saved to ${OUTPUT_FILE_PATH}`);
   }
 
+  // Report collected errors if in no-bail mode
+  if (collectedErrors.length > 0) {
+    console.log();
+    console.log();
+    console.log();
+    console.log();
+    console.log("===================================================");
+    console.log(`\x1b[41mFAILED\x1b[0m! Found ${collectedErrors.length} error(s):`);
+    console.log();
+    for (let i = 0; i < collectedErrors.length; i++) {
+      const { context, error } = collectedErrors[i];
+      console.log(`--- Error ${i + 1}/${collectedErrors.length} ---`);
+      console.log(`Context: ${context}`);
+      console.error(error);
+      console.log();
+    }
+    console.log("===================================================");
+    console.log();
+    process.exit(1);
+  }
+
   console.log();
   console.log();
   console.log();

apps/backend/scripts/verify-data-integrity/recurse.ts

@@ -1,15 +1,23 @@
 export type RecurseFunction = (progressPrefix: string, inner: (recurse: RecurseFunction) => Promise<void>) => Promise<void>;
 
-export function createRecurse(): RecurseFunction {
+export type CollectedError = {
+  context: string,
+  error: unknown,
+};
+
+export function createRecurse(options: { noBail: boolean }): { recurse: RecurseFunction, collectedErrors: CollectedError[] } {
   let lastProgress = performance.now() - 9999999999;
+  const collectedErrors: CollectedError[] = [];
 
   const _recurse = async (
     progressPrefix: string | ((...args: any[]) => void),
     inner: Parameters<RecurseFunction>[1],
+    contextPath: string = "",
   ): Promise<void> => {
     const progressFunc = typeof progressPrefix === "function" ? progressPrefix : (...args: any[]) => {
       console.log(`${progressPrefix}`, ...args);
     };
+    const currentContext = typeof progressPrefix === "string" ? progressPrefix : contextPath;
     if (performance.now() - lastProgress > 1000) {
       progressFunc();
       lastProgress = performance.now();
@@ -19,14 +27,24 @@ export function createRecurse(): RecurseFunction {
         (progressPrefix, inner) => _recurse(
           (...args) => progressFunc(progressPrefix, ...args),
           inner,
+          `${currentContext} > ${typeof progressPrefix === "string" ? progressPrefix : ""}`,
         ),
       );
     } catch (error) {
       progressFunc(`\x1b[41mERROR\x1b[0m!`);
-      throw error;
+      if (options.noBail) {
+        collectedErrors.push({
+          context: currentContext,
+          error,
+        });
+      } else {
+        throw error;
+      }
     }
   };
 
-  return _recurse;
+  const recurse: RecurseFunction = (progressPrefix, inner) => _recurse(progressPrefix, inner, progressPrefix);
+
+  return { recurse, collectedErrors };
 }
 

apps/backend/src/app/api/latest/internal/clickhouse/migrate-events/route.tsx

@@ -1,10 +1,11 @@
 import type { Prisma } from "@/generated/prisma/client";
 import { getClickhouseAdminClient, isClickhouseConfigured } from "@/lib/clickhouse";
+import { endUserIpInfoSchema, type EndUserIpInfo } from "@/lib/events";
 import { DEFAULT_BRANCH_ID } from "@/lib/tenancies";
 import { globalPrismaClient } from "@/prisma-client";
 import { createSmartRouteHandler } from "@/route-handlers/smart-route-handler";
 import { yupNumber, yupObject, yupString } from "@stackframe/stack-shared/dist/schema-fields";
-import { StatusError } from "@stackframe/stack-shared/dist/utils/errors";
+import { StackAssertionError, StatusError, throwErr } from "@stackframe/stack-shared/dist/utils/errors";
 
 type Cursor = {
   created_at_millis: number,
@@ -22,41 +23,86 @@ const parseMillisOrThrow = (value: number | undefined, field: string) => {
   return parsed;
 };
 
-const createClickhouseRows = (event: {
+type EventWithIpInfo = {
   id: string,
   systemEventTypeIds: string[],
-  data: any,
+  data: unknown,
   eventEndedAt: Date,
   eventStartedAt: Date,
   isWide: boolean,
-}) => {
+  isEndUserIpInfoGuessTrusted: boolean,
+  endUserIpInfoGuess: {
+    ip: string,
+    countryCode: string | null,
+    regionCode: string | null,
+    cityName: string | null,
+    latitude: number | null,
+    longitude: number | null,
+    tzIdentifier: string | null,
+  } | null,
+};
+
+/**
+ * Transforms a $session-activity event from Postgres into a $token-refresh event for ClickHouse.
+ *
+ * The $session-activity event has:
+ * - sessionId (from SessionActivityEventType)
+ * - userId, branchId, isAnonymous, teamId (from UserActivityEventType)
+ * - projectId (from ProjectEventType via inheritance)
+ *
+ * The $token-refresh event needs:
+ * - projectId, branchId, organizationId (null), userId, refreshTokenId, isAnonymous, ipInfo
+ */
+const createClickhouseRow = (event: EventWithIpInfo) => {
   const dataRecord = typeof event.data === "object" && event.data !== null ? event.data as Record<string, unknown> : {};
-  const clickhouseEventData = {
-    ...dataRecord,
-    is_wide: event.isWide,
-    event_started_at: event.eventStartedAt,
-    event_ended_at: event.eventEndedAt,
-  };
-  const projectId = typeof dataRecord.projectId === "string" ? dataRecord.projectId : "";
-  const branchId = DEFAULT_BRANCH_ID;
-  const userId = typeof dataRecord.userId === "string" ? dataRecord.userId : "";
-  const teamId = typeof dataRecord.teamId === "string" ? dataRecord.teamId : "";
-  const sessionId = typeof dataRecord.sessionId === "string" ? dataRecord.sessionId : "";
+
+  // Extract fields from the old $session-activity format
+  const projectId = typeof dataRecord.projectId === "string" ? dataRecord.projectId : throwErr(new StackAssertionError("projectId is required"));
+  const branchId = typeof dataRecord.branchId === "string" ? dataRecord.branchId : DEFAULT_BRANCH_ID;
+  const userId = typeof dataRecord.userId === "string" && dataRecord.userId ? dataRecord.userId : throwErr(new StackAssertionError("userId is required"));
+  // sessionId becomes refreshTokenId in the new schema
+  const refreshTokenId = typeof dataRecord.sessionId === "string" ? dataRecord.sessionId : throwErr(new StackAssertionError("sessionId is required"));
+  // isAnonymous may not exist on old events, default to false
   const isAnonymous = typeof dataRecord.isAnonymous === "boolean" ? dataRecord.isAnonymous : false;
 
-  const eventTypes = [...new Set(event.systemEventTypeIds)];
+  // Build ipInfo from the event's endUserIpInfoGuess relation
+  let ipInfo: EndUserIpInfo | null = null;
+  if (event.endUserIpInfoGuess) {
+    const ip = event.endUserIpInfoGuess;
+    ipInfo = {
+      ip: ip.ip,
+      isTrusted: event.isEndUserIpInfoGuessTrusted,
+      countryCode: ip.countryCode ?? undefined,
+      regionCode: ip.regionCode ?? undefined,
+      cityName: ip.cityName ?? undefined,
+      latitude: ip.latitude ?? undefined,
+      longitude: ip.longitude ?? undefined,
+      tzIdentifier: ip.tzIdentifier ?? undefined,
+    };
+    // Validate against schema
+    ipInfo = endUserIpInfoSchema.nullable().defined().validateSync(ipInfo, { stripUnknown: true });
+  }
+
+  // Build the data object matching TokenRefreshEventType schema
+  const tokenRefreshData = {
+    projectId,
+    branchId,
+    organizationId: null,
+    userId: userId,
+    refreshTokenId,
+    isAnonymous,
+    ipInfo,
+  };
 
-  return eventTypes.map(eventType => ({
-    event_type: eventType,
+  return {
+    event_type: '$token-refresh',
     event_at: event.eventEndedAt,
-    data: clickhouseEventData,
+    data: tokenRefreshData,
     project_id: projectId,
     branch_id: branchId,
     user_id: userId,
-    team_id: teamId,
-    session_id: sessionId,
-    is_anonymous: isAnonymous,
-  }));
+    team_id: null,
+  };
 };
 
 export const POST = createSmartRouteHandler({
@@ -106,11 +152,17 @@ export const POST = createSmartRouteHandler({
     const cursorId = body.cursor?.id;
     const limit = body.limit;
 
+    const laterOfMinCreatedAtOrCursorCreatedAt = !cursorCreatedAt || minCreatedAt > cursorCreatedAt ? minCreatedAt : cursorCreatedAt;
+
     const baseWhere: Prisma.EventWhereInput = {
       createdAt: {
-        gte: minCreatedAt,
+        gte: laterOfMinCreatedAtOrCursorCreatedAt,
         lt: maxCreatedAt,
       },
+      // Only migrate $session-activity events (translated to $token-refresh in ClickHouse)
+      systemEventTypeIds: {
+        has: '$session-activity',
+      },
     };
 
     const cursorFilter: Prisma.EventWhereInput | undefined = (cursorCreatedAt && cursorId) ? {
@@ -131,6 +183,9 @@ export const POST = createSmartRouteHandler({
         { id: "asc" },
       ],
       take: limit,
+      include: {
+        endUserIpInfoGuess: true,
+      },
     });
 
     let insertedRows = 0;
@@ -141,22 +196,19 @@ export const POST = createSmartRouteHandler({
         throw new StatusError(StatusError.ServiceUnavailable, "ClickHouse is not configured");
       }
       const clickhouseClient = getClickhouseAdminClient();
-      const rowsByEvent = events.map(createClickhouseRows);
-      const rowsToInsert = rowsByEvent.flat();
-      migratedEvents = rowsByEvent.reduce((acc, rows) => acc + (rows.length ? 1 : 0), 0);
-
-      if (rowsToInsert.length) {
-        await clickhouseClient.insert({
-          table: "analytics_internal.events",
-          values: rowsToInsert,
-          format: "JSONEachRow",
-          clickhouse_settings: {
-            date_time_input_format: "best_effort",
-            async_insert: 1,
-          },
-        });
-        insertedRows = rowsToInsert.length;
-      }
+      const rowsToInsert = events.map(createClickhouseRow);
+      migratedEvents = events.length;
+
+      await clickhouseClient.insert({
+        table: "analytics_internal.events",
+        values: rowsToInsert,
+        format: "JSONEachRow",
+        clickhouse_settings: {
+          date_time_input_format: "best_effort",
+          async_insert: 1,
+        },
+      });
+      insertedRows = rowsToInsert.length;
     }
 
     const lastEvent = events.at(-1);