[pull] main from tldraw:main

apps/dotcom/client/src/pages/admin.tsx

@@ -5,6 +5,7 @@ import { fetch } from 'tldraw'
 import { TlaButton } from '../tla/components/TlaButton/TlaButton'
 import { useTldrawUser } from '../tla/hooks/useUser'
 import styles from './admin.module.css'
+import { saveMigrationLog } from './migrationLogsDB'
 
 // Helper component for structured data display
 function StructuredDataDisplay({ data }: { data: ZStoreData }) {
@@ -502,6 +503,7 @@ function BatchMigrateUsersToGroups() {
 	const [eventSource, setEventSource] = useState<EventSource | null>(null)
 	const [sleepMs, setSleepMs] = useState(100)
 	const logContainerRef = useRef<HTMLDivElement>(null)
+	const shouldContinueRef = useRef(true)
 
 	// Cleanup EventSource on unmount
 	useEffect(() => {
@@ -538,6 +540,7 @@ function BatchMigrateUsersToGroups() {
 	}, [])
 
 	const stopMigration = useCallback(() => {
+		shouldContinueRef.current = false
 		if (eventSource) {
 			eventSource.close()
 			setEventSource(null)
@@ -553,58 +556,87 @@ function BatchMigrateUsersToGroups() {
 		}
 
 		setIsMigrating(true)
+		shouldContinueRef.current = true
 		setError(null)
 		setProgressLog([])
 		setIsComplete(false)
 		setStats({ successCount: 0, failureCount: 0, totalUsers: 0, usersToMigrate: 0, progress: 0 })
 
-		try {
-			const params = new URLSearchParams({
-				sleepMs: sleepMs.toString(),
-			})
-			const es = new EventSource(`/api/app/admin/migrate_users_batch?${params}`)
-			setEventSource(es)
-
-			es.onmessage = (event) => {
-				const data = JSON.parse(event.data)
-
-				const timestamp = new Date(data.timestamp).toLocaleTimeString()
-				const logEntry = `[${timestamp}] ${data.message}`
-
-				// Keep only the last 500 log entries to prevent memory issues
-				setProgressLog((prev) => {
-					const updated = [...prev, logEntry]
-					return updated.length > 500 ? updated.slice(-500) : updated
-				})
-
-				// Update stats from details
-				if (data.details) {
-					setStats(data.details)
-				}
+		const startBatch = () => {
+			return new Promise<void>((resolve, reject) => {
+				try {
+					const params = new URLSearchParams({
+						sleepMs: sleepMs.toString(),
+					})
+					const es = new EventSource(`/api/app/admin/migrate_users_batch?${params}`)
+					setEventSource(es)
+
+					es.onmessage = async (event) => {
+						const data = JSON.parse(event.data)
+
+						const timestamp = new Date(data.timestamp).toLocaleTimeString()
+						const logEntry = `[${timestamp}] ${data.message}`
+
+						// Keep only the last 500 log entries to prevent memory issues
+						setProgressLog((prev) => {
+							const updated = [...prev, logEntry]
+							return updated.length > 500 ? updated.slice(-500) : updated
+						})
+
+						// Save failure events to IndexedDB
+						if (data.step === 'failure') {
+							try {
+								await saveMigrationLog(data)
+							} catch (err) {
+								console.error('Failed to save migration log to IndexedDB:', err)
+							}
+						}
+
+						// Update stats from details
+						if (data.details) {
+							setStats(data.details)
+						}
+
+						if (data.type === 'complete') {
+							es.close()
+							setEventSource(null)
+							if (data.hasMore && shouldContinueRef.current) {
+								// Start next batch
+								setTimeout(() => startBatch().then(resolve).catch(reject), 100)
+							} else {
+								setIsComplete(true)
+								setIsMigrating(false)
+								resolve()
+							}
+						} else if (data.type === 'error') {
+							setError(data.message)
+							setIsMigrating(false)
+							es.close()
+							setEventSource(null)
+							reject(new Error(data.message))
+						}
+					}
 
-				if (data.type === 'complete') {
-					setIsComplete(true)
-					setIsMigrating(false)
-					es.close()
-					setEventSource(null)
-				} else if (data.type === 'error') {
-					setError(data.message)
+					es.onerror = () => {
+						setError('Connection failed')
+						setIsMigrating(false)
+						es.close()
+						setEventSource(null)
+						reject(new Error('Connection failed'))
+					}
+				} catch (err) {
+					setError(err instanceof Error ? err.message : 'Unknown error occurred')
 					setIsMigrating(false)
-					es.close()
 					setEventSource(null)
+					reject(err)
 				}
-			}
+			})
+		}
 
-			es.onerror = () => {
-				setError('Connection failed')
-				setIsMigrating(false)
-				es.close()
-				setEventSource(null)
-			}
-		} catch (err) {
-			setError(err instanceof Error ? err.message : 'Unknown error occurred')
-			setIsMigrating(false)
-			setEventSource(null)
+		try {
+			await startBatch()
+		} catch (_err) {
+			// Error already handled in startBatch
 		}
 	}, [sleepMs])
 

apps/dotcom/client/src/pages/migrationLogsDB.ts

@@ -0,0 +1,26 @@
+import { openDB, type IDBPDatabase } from 'idb'
+
+const DB_NAME = 'migration-errors'
+const DB_VERSION = 1
+const STORE_NAME = 'errors'
+
+let dbPromise: Promise<IDBPDatabase> | null = null
+
+function getDB(): Promise<IDBPDatabase> {
+	if (!dbPromise) {
+		dbPromise = openDB(DB_NAME, DB_VERSION, {
+			upgrade(db, _oldVersion, _newVersion, _transaction) {
+				db.createObjectStore(STORE_NAME, {
+					keyPath: 'id',
+					autoIncrement: true,
+				})
+			},
+		})
+	}
+	return dbPromise
+}
+
+export async function saveMigrationLog(entry: object): Promise<void> {
+	const db = await getDB()
+	await db.add(STORE_NAME, entry)
+}

apps/dotcom/sync-worker/src/adminRoutes.ts

@@ -1,5 +1,5 @@
 import { TlaFile } from '@tldraw/dotcom-shared'
-import { assert, sleep, uniqueId } from '@tldraw/utils'
+import { assert, retry, sleep, uniqueId } from '@tldraw/utils'
 import { createRouter } from '@tldraw/worker-shared'
 import { StatusError, json } from 'itty-router'
 import { sql } from 'kysely'
@@ -93,7 +93,7 @@ export const adminRoutes = createRouter<Environment>()
 
 						sendProgress('starting', 'Beginning batch user migration process...')
 
-						await startUserMigration(env, sendProgress, shouldStop, sleepMs)
+						const hasMore = await startUserMigration(env, sendProgress, shouldStop, sleepMs)
 
 						// Send completion event
 						const completionEvent = {
@@ -103,6 +103,7 @@ export const adminRoutes = createRouter<Environment>()
 								? 'Batch migration stopped by user'
 								: 'Batch migration completed successfully',
 							timestamp: Date.now(),
+							hasMore,
 						}
 						controller.enqueue(
 							new TextEncoder().encode(`data: ${JSON.stringify(completionEvent)}\n\n`)
@@ -415,7 +416,8 @@ async function startUserMigration(
 	sendProgress: (step: string, message: string, details?: any) => void,
 	shouldStop: () => boolean,
 	sleepTime: number = 100
-) {
+): Promise<boolean> {
+	const batchSize = 50
 	const pg = createPostgresConnectionPool(env, '/app/admin/migrate_users_batch')
 
 	sendProgress('query', 'Fetching users without groups_backend flag...')
@@ -439,13 +441,14 @@ async function startUserMigration(
 
 	if (usersToMigrate === 0) {
 		sendProgress('complete', 'No users to migrate')
-		return
+		return false
 	}
 
 	const failures: Array<{ userId: string; email: string; error: string }> = []
+	let processedCount = 0
 
 	// Process users in batches
-	while (true) {
+	while (processedCount < batchSize) {
 		const userRow = await getNextUnmigratedUser(pg)
 		if (!userRow) {
 			break
@@ -464,21 +467,23 @@ async function startUserMigration(
 		})
 
 		try {
-			const user = getUserDurableObject(env, userRow.id)
-
-			const result = await sql<{
-				files_migrated: number
-				pinned_files_migrated: number
-				flag_added: boolean
-			}>`SELECT * FROM migrate_user_to_groups(${userRow.id}, ${uniqueId()})`.execute(pg)
-			await user.admin_forceHardReboot(userRow.id)
-
-			successCount++
-			sendProgress('success', `Successfully migrated user ${userRow.email}`, {
-				userId: userRow.id,
-				email: userRow.email,
-				result: result.rows[0],
-				...getStats(),
+			await retry(async () => {
+				const user = getUserDurableObject(env, userRow.id)
+
+				const result = await sql<{
+					files_migrated: number
+					pinned_files_migrated: number
+					flag_added: boolean
+				}>`SELECT * FROM migrate_user_to_groups(${userRow.id}, ${uniqueId()})`.execute(pg)
+				await user.admin_forceHardReboot(userRow.id)
+
+				successCount++
+				sendProgress('success', `Successfully migrated user ${userRow.email}`, {
+					userId: userRow.id,
+					email: userRow.email,
+					result: result.rows[0],
+					...getStats(),
+				})
 			})
 		} catch (error) {
 			failureCount++
@@ -489,19 +494,32 @@ async function startUserMigration(
 				error: errorMessage,
 			})
 
+			// Send failure event to client so it can be stored in the log
 			sendProgress('failure', `Failed to migrate ${userRow.email}`, {
 				userId: userRow.id,
 				email: userRow.email,
 				error: errorMessage,
 				...getStats(),
 			})
+
+			// Stop processing immediately after reporting the failure
+			sendProgress('summary', 'Migration stopped due to failure', {
+				failures: failures.length > 0 ? failures : undefined,
+			})
+			return false
 		}
 
+		processedCount++
+
 		// Brief pause between migrations to avoid overwhelming the system
 		await sleep(sleepTime)
 	}
 
 	sendProgress('summary', 'Migration batch complete', {
 		failures: failures.length > 0 ? failures : undefined,
 	})
+
+	// Check if there are more users to migrate
+	const remainingUsers = await getNumUnmigratedUsers(pg)
+	return remainingUsers > 0
 }

packages/tldraw/setupVitest.js

@@ -1,3 +1,31 @@
+import { vi } from 'vitest'
+
+// Mock the breakpoints context to provide a default breakpoint value in tests
+// This prevents errors when components use useBreakpoint() without a BreakpointProvider
+vi.mock('./src/lib/ui/context/breakpoints', async () => {
+	const actual = await vi.importActual('./src/lib/ui/context/breakpoints')
+	return {
+		...actual,
+		useBreakpoint: () => 7, // PORTRAIT_BREAKPOINT.DESKTOP
+	}
+})
+
+// Mock the translation hooks to provide default translation functions in tests
+// This prevents errors when components use useTranslation() without a TldrawUiTranslationProvider
+vi.mock('./src/lib/ui/hooks/useTranslation/useTranslation', async () => {
+	const actual = await vi.importActual('./src/lib/ui/hooks/useTranslation/useTranslation')
+	return {
+		...actual,
+		useCurrentTranslation: () => ({
+			locale: 'en',
+			label: 'English',
+			dir: 'ltr',
+			messages: {},
+		}),
+		useTranslation: () => (key) => key || '', // Return the key itself as the translation
+	}
+})
+
 // Vitest setup file for tldraw package
 // Converted from setupTests.js to provide the same polyfills and global setup