Merge branch 'dev' into dashboard/changelog

Author: madster456

apps/dashboard/src/app/(main)/(protected)/projects/[projectId]/clickhouse-migration/page-client.tsx

@@ -1,36 +1,26 @@
 "use client";
 
+import { Alert, Button, Card, CardContent, CardHeader, CardTitle, Input, Typography } from "@/components/ui";
 import { ClickhouseMigrationRequest, ClickhouseMigrationResponse } from "@stackframe/stack-shared/dist/interface/admin-interface";
-import { Button, Card, CardContent, CardHeader, CardTitle, Input, Typography, Alert } from "@/components/ui";
+import { notFound } from "next/navigation";
 import React from "react";
 import { PageLayout } from "../page-layout";
 import { useAdminApp } from "../use-admin-app";
-import { notFound } from "next/navigation";
 
 type MigrationCursor = {
   createdAtMillis: number,
   id: string,
 };
 
 type MigrationSnapshot = {
-  totalEvents: number,
-  processedEvents: number,
-  remainingEvents: number,
   migratedEvents: number,
-  skippedExistingEvents: number,
   insertedRows: number,
-  progress: number,
   nextCursor: MigrationCursor | null,
 };
 
 const normalizeResponse = (response: ClickhouseMigrationResponse): MigrationSnapshot => ({
-  totalEvents: response.total_events,
-  processedEvents: response.processed_events,
-  remainingEvents: response.remaining_events,
   migratedEvents: response.migrated_events,
-  skippedExistingEvents: response.skipped_existing_events,
   insertedRows: response.inserted_rows,
-  progress: response.progress,
   nextCursor: response.next_cursor ? {
     createdAtMillis: response.next_cursor.created_at_millis,
     id: response.next_cursor.id,
@@ -46,13 +36,16 @@ export default function PageClient() {
   const [minCreatedAt, setMinCreatedAt] = React.useState("");
   const [maxCreatedAt, setMaxCreatedAt] = React.useState("");
   const [limit, setLimit] = React.useState(1000);
-  const [stats, setStats] = React.useState<MigrationSnapshot | null>(null);
   const [cursor, setCursor] = React.useState<MigrationCursor | null>(null);
   const [running, setRunning] = React.useState(false);
   const runningRef = React.useRef(false);
   const cursorRef = React.useRef<MigrationCursor | null>(null);
   const timeWindowRef = React.useRef<{ minCreatedAtMillis: number, maxCreatedAtMillis: number } | null>(null);
   const [error, setError] = React.useState<string | null>(null);
+  const [totalMigratedEvents, setTotalMigratedEvents] = React.useState(0);
+  const [totalInsertedRows, setTotalInsertedRows] = React.useState(0);
+  const [batchCount, setBatchCount] = React.useState(0);
+  const [done, setDone] = React.useState(false);
 
   const parseCreatedAtMillis = React.useCallback((value: string | undefined) => {
     if (!value) return null;
@@ -87,7 +80,9 @@ export default function PageClient() {
   const runBatch = React.useCallback(async () => {
     const response = await adminInterface.migrateEventsToClickhouse(buildRequestBody());
     const snapshot = normalizeResponse(response);
-    setStats(snapshot);
+    setTotalMigratedEvents(prev => prev + snapshot.migratedEvents);
+    setTotalInsertedRows(prev => prev + snapshot.insertedRows);
+    setBatchCount(prev => prev + 1);
     cursorRef.current = snapshot.nextCursor;
     setCursor(snapshot.nextCursor);
     return snapshot;
@@ -103,8 +98,11 @@ export default function PageClient() {
     cursorRef.current = null;
     timeWindowRef.current = null;
     setCursor(null);
-    setStats(null);
     setError(null);
+    setTotalMigratedEvents(0);
+    setTotalInsertedRows(0);
+    setBatchCount(0);
+    setDone(false);
   }, [stopMigration]);
 
   const startMigration = React.useCallback(async () => {
@@ -128,6 +126,7 @@ export default function PageClient() {
       while (runningRef.current) {
         const snapshot = await runBatch();
         if (!snapshot.nextCursor) {
+          setDone(true);
           stopMigration();
           break;
         }
@@ -138,8 +137,6 @@ export default function PageClient() {
     }
   }, [maxCreatedAt, minCreatedAt, parseCreatedAtMillis, runBatch, stopMigration]);
 
-  const progressPercent = Math.min(100, Math.max(0, Math.round((stats?.progress ?? 0) * 100)));
-
   if (stackAdminApp.projectId !== "internal") {
     return notFound();
   }
@@ -186,7 +183,7 @@ export default function PageClient() {
                 <Input
                   type="number"
                   min={1}
-                  max={1000}
+                  max={100_000}
                   value={limit}
                   onChange={(e) => {
                     setLimit(Number(e.target.value) || 0);
@@ -225,40 +222,22 @@ export default function PageClient() {
             <CardTitle>Status</CardTitle>
           </CardHeader>
           <CardContent className="space-y-3">
-            <div className="h-3 w-full rounded-full bg-muted">
-              <div
-                className="h-3 rounded-full bg-gradient-to-r from-blue-500 to-emerald-500"
-                style={{ width: `${progressPercent}%` }}
-              />
-            </div>
-            <div className="flex items-center justify-between">
-              <Typography variant="secondary">Progress</Typography>
-              <Typography type="label">{progressPercent}%</Typography>
-            </div>
             <div className="grid grid-cols-2 gap-3 text-sm">
               <div>
-                <Typography variant="secondary">Processed</Typography>
-                <Typography type="label">{stats?.processedEvents ?? 0}</Typography>
-              </div>
-              <div>
-                <Typography variant="secondary">Remaining</Typography>
-                <Typography type="label">{stats?.remainingEvents ?? 0}</Typography>
+                <Typography variant="secondary">Events migrated</Typography>
+                <Typography type="label">{totalMigratedEvents.toLocaleString()}</Typography>
               </div>
               <div>
-                <Typography variant="secondary">Migrated this run</Typography>
-                <Typography type="label">{stats?.migratedEvents ?? 0}</Typography>
+                <Typography variant="secondary">Rows inserted</Typography>
+                <Typography type="label">{totalInsertedRows.toLocaleString()}</Typography>
               </div>
               <div>
-                <Typography variant="secondary">Inserted rows</Typography>
-                <Typography type="label">{stats?.insertedRows ?? 0}</Typography>
+                <Typography variant="secondary">Batches completed</Typography>
+                <Typography type="label">{batchCount.toLocaleString()}</Typography>
               </div>
               <div>
-                <Typography variant="secondary">Total in scope</Typography>
-                <Typography type="label">{stats?.totalEvents ?? 0}</Typography>
-              </div>
-              <div className="">
                 <Typography variant="secondary">State</Typography>
-                <Typography type="label">{running ? "Running" : "Idle"}</Typography>
+                <Typography type="label">{done ? "Done" : running ? "Running" : "Idle"}</Typography>
               </div>
             </div>
           </CardContent>

packages/stack-shared/src/interface/admin-interface.ts

@@ -3,13 +3,13 @@ import { KnownErrors } from "../known-errors";
 import { branchConfigSourceSchema } from "../schema-fields";
 import { AccessToken, InternalSession, RefreshToken } from "../sessions";
 import { Result } from "../utils/results";
+import type { AnalyticsQueryOptions, AnalyticsQueryResponse } from "./crud/analytics";
 import { EmailOutboxCrud } from "./crud/email-outbox";
 import { InternalEmailsCrud } from "./crud/emails";
 import { InternalApiKeysCrud } from "./crud/internal-api-keys";
 import { ProjectPermissionDefinitionsCrud } from "./crud/project-permissions";
 import { ProjectsCrud } from "./crud/projects";
 import { SvixTokenCrud } from "./crud/svix-token";
-import type { AnalyticsQueryOptions, AnalyticsQueryResponse } from "./crud/analytics";
 import { TeamPermissionDefinitionsCrud } from "./crud/team-permissions";
 import type { Transaction, TransactionType } from "./crud/transactions";
 import { ServerAuthApplicationOptions, StackServerInterface } from "./server-interface";
@@ -56,13 +56,8 @@ export type ClickhouseMigrationRequest = {
 };
 
 export type ClickhouseMigrationResponse = {
-  total_events: number,
-  processed_events: number,
-  remaining_events: number,
   migrated_events: number,
-  skipped_existing_events: number,
   inserted_rows: number,
-  progress: number,
   next_cursor: {
     created_at_millis: number,
     id: string,