Add retries for persistence (tldraw#7050)

Add a lengthy retry system with sentry error reporting to see how many
retries we max out at.

### Change type

- [x] `other`

### API changes
internal changes only


<!-- CURSOR_SUMMARY -->
---

> [!NOTE]
> Introduce persistence retry with thresholds, analytics logging, and
client notifications; add custom server events and update retry util
signature.
> 
> - **Sync Worker**:
> - **Persistence Retries**: Wrap `persistToDatabase` in `retry(...)`
with up to `PERSIST_RETRIES_MAX` attempts; log `persist_success` with
attempt count.
> - **Client Notifications**: Broadcast `TLCustomServerEvent`
(`persistence_bad` at threshold, `persistence_good` on recovery) via
`broadcastPersistenceEvent`.
> - **Analytics**: Extend `logEvent` to handle `persist_success`;
improved error logging on DB update.
> - **Constants**: Add `PERSIST_RETRIES_NOTIFY_THRESHOLD` and
`PERSIST_RETRIES_MAX`.
> - **Client**:
> - Track custom server events by adding `onCustomMessageReceived` in
`useSync` and calling `trackEvent(message.type)`.
> - **Shared/Utils**:
>   - Add `TLCustomServerEvent` type in `@tldraw/dotcom-shared`.
> - Change `retry` signature to pass `{ attempt, remaining, total }`;
update API report accordingly.
> 
> <sup>Written by [Cursor
Bugbot](https://cursor.com/dashboard?tab=bugbot) for commit
6980329. This will update automatically
on new commits. Configure
[here](https://cursor.com/dashboard?tab=bugbot).</sup>
<!-- /CURSOR_SUMMARY -->

Author: ds300

apps/dotcom/client/src/tla/components/TlaEditor/TlaEditor.tsx

@@ -1,4 +1,4 @@
-import { getLicenseKey } from '@tldraw/dotcom-shared'
+import { TLCustomServerEvent, getLicenseKey } from '@tldraw/dotcom-shared'
 import { FairyEntity } from '@tldraw/fairy-shared'
 import { useSync } from '@tldraw/sync'
 import { Suspense, lazy, useCallback, useEffect, useMemo, useRef, useState } from 'react'
@@ -30,7 +30,7 @@ import {
 import { ThemeUpdater } from '../../../components/ThemeUpdater/ThemeUpdater'
 import { useOpenUrlAndTrack } from '../../../hooks/useOpenUrlAndTrack'
 import { useRoomLoadTracking } from '../../../hooks/useRoomLoadTracking'
-import { useHandleUiEvents } from '../../../utils/analytics'
+import { trackEvent, useHandleUiEvents } from '../../../utils/analytics'
 import { assetUrls } from '../../../utils/assetUrls'
 import { MULTIPLAYER_SERVER } from '../../../utils/config'
 import { createAssetFromUrl } from '../../../utils/createAssetFromUrl'
@@ -209,6 +209,9 @@ function TlaEditorInner({ fileSlug, deepLinks }: TlaEditorProps) {
 		}, [fileSlug, hasUser, getUserToken]),
 		assets,
 		userInfo: app?.tlUser.userPreferences,
+		onCustomMessageReceived: useCallback((message: TLCustomServerEvent) => {
+			trackEvent(message.type)
+		}, []),
 		getUserPresence: useCallback(
 			(store: TLStore, userInfo: TLPresenceUserInfo): TLPresenceStateInfo | null => {
 				const defaultPresence = getDefaultUserPresence(store, userInfo)

apps/dotcom/sync-worker/src/TLDrawDurableObject.ts

@@ -1,6 +1,6 @@
 /// <reference no-default-lib="true"/>
-/// <reference types="@cloudflare/workers-types" />
 
+import { R2Bucket } from '@cloudflare/workers-types'
 import { SupabaseClient } from '@supabase/supabase-js'
 import {
 	APP_ASSET_UPLOAD_ENDPOINT,
@@ -14,6 +14,7 @@ import {
 	ROOM_PREFIX,
 	ROOM_SIZE_LIMIT_MB,
 	SNAPSHOT_PREFIX,
+	TLCustomServerEvent,
 	TlaFile,
 	type RoomOpenMode,
 } from '@tldraw/dotcom-shared'
@@ -528,6 +529,10 @@ export class TLDrawDurableObject extends DurableObject {
 
 	logEvent(event: TLServerEvent) {
 		switch (event.type) {
+			case 'persist_success': {
+				this.writeEvent(event.type, { doubles: [event.attempts] })
+				break
+			}
 			case 'room': {
 				// we would add user/connection ids here if we could
 				this.writeEvent(event.name, { blobs: [event.roomId] })
@@ -801,43 +806,71 @@ export class TLDrawDurableObject extends DurableObject {
 		})
 	}
 
+	broadcastPersistenceEvent(event: TLCustomServerEvent) {
+		this._room?.then((r) => {
+			for (const session of r.getSessions()) {
+				r.sendCustomMessage(session.sessionId, event)
+			}
+		})
+	}
+
 	// Save the room to r2
 	async persistToDatabase() {
-		try {
-			await this.executionQueue.push(async () => {
-				// check whether the worker was woken up to persist after having gone to sleep
-				if (!this._room) return
-				const slug = this.documentInfo.slug
-				const room = await this.getRoom()
-				const clock = room.getCurrentDocumentClock()
-				if (this._lastPersistedClock === clock) return
-				if (this._isRestoring) return
-
-				const snapshot = room.getCurrentSnapshot()
-				this.maybeAssociateFileAssets()
-
-				const key = getR2KeyForRoom({ slug: slug, isApp: this.documentInfo.isApp })
-				await this._uploadSnapshotToR2(room, snapshot, key)
-
-				this._lastPersistedClock = clock
-
-				// Update the updatedAt timestamp in the database
-				if (this.documentInfo.isApp) {
-					// don't await on this because otherwise
-					// if this logic is invoked during another db transaction
-					// (e.g. when publishing a file)
-					// that transaction will deadlock
-					this.db
-						.updateTable('file')
-						.set({ updatedAt: new Date().getTime() })
-						.where('id', '=', this.documentInfo.slug)
-						.execute()
-						.catch((e) => this.reportError(e))
-				}
+		await this.executionQueue
+			.push(async () => {
+				await retry(
+					async ({ attempt }) => {
+						if (attempt === PERSIST_RETRIES_NOTIFY_THRESHOLD) {
+							this.broadcastPersistenceEvent({ type: 'persistence_bad' })
+						}
+						// check whether the worker was woken up to persist after having gone to sleep
+						if (!this._room) return
+						const slug = this.documentInfo.slug
+						const room = await this.getRoom()
+						const clock = room.getCurrentDocumentClock()
+						if (this._lastPersistedClock === clock) return
+						if (this._isRestoring) return
+
+						const snapshot = room.getCurrentSnapshot()
+						this.maybeAssociateFileAssets()
+
+						const key = getR2KeyForRoom({ slug: slug, isApp: this.documentInfo.isApp })
+						await this._uploadSnapshotToR2(room, snapshot, key)
+
+						this.logEvent({ type: 'persist_success', attempts: attempt })
+						this._lastPersistedClock = clock
+						if (attempt >= PERSIST_RETRIES_NOTIFY_THRESHOLD) {
+							this.broadcastPersistenceEvent({ type: 'persistence_good' })
+						}
+
+						// Update the updatedAt timestamp in the database
+						if (this.documentInfo.isApp) {
+							// don't await on this because otherwise
+							// if this logic is invoked during another db transaction
+							// (e.g. when publishing a file)
+							// that transaction will deadlock
+							this.db
+								.updateTable('file')
+								.set({ updatedAt: new Date().getTime() })
+								.where('id', '=', this.documentInfo.slug)
+								.execute()
+								.catch((e) => {
+									this.logEvent({
+										type: 'room',
+										roomId: this.documentInfo.slug,
+										name: 'failed_persist_to_db',
+									})
+									this.reportError(e)
+								})
+						}
+					},
+					{ attempts: PERSIST_RETRIES_MAX, waitDuration: 2000 }
+				)
+			})
+			.catch((e) => {
+				this.logEvent({ type: 'room', roomId: this.documentInfo.slug, name: 'fail_persist' })
+				this.reportError(e)
 			})
-		} catch (e) {
-			this.reportError(e)
-		}
 	}
 
 	private async _uploadSnapshotToR2(
@@ -1143,3 +1176,6 @@ async function listAllObjectKeys(bucket: R2Bucket, prefix: string): Promise<stri
 
 	return keys
 }
+
+const PERSIST_RETRIES_NOTIFY_THRESHOLD = 10
+const PERSIST_RETRIES_MAX = 100

apps/dotcom/sync-worker/src/types.ts

@@ -124,6 +124,10 @@ export type TLServerEvent =
 			messageType: string
 			messageLength: number
 	  }
+	| {
+			type: 'persist_success'
+			attempts: number
+	  }
 
 export type TLPostgresReplicatorRebootSource =
 	| 'constructor'

packages/dotcom-shared/src/types.ts

@@ -224,3 +224,5 @@ export interface SubmitFeedbackRequestBody {
 }
 
 export const MAX_PROBLEM_DESCRIPTION_LENGTH = 2000
+
+export type TLCustomServerEvent = { type: 'persistence_good' } | { type: 'persistence_bad' }

packages/utils/api-report.api.md

@@ -421,7 +421,11 @@ export const Result: {
 };
 
 // @internal
-export function retry<T>(fn: () => Promise<T>, { attempts, waitDuration, abortSignal, matchError, }?: {
+export function retry<T>(fn: (args: {
+    attempt: number;
+    remaining: number;
+    total: number;
+}) => Promise<T>, { attempts, waitDuration, abortSignal, matchError, }?: {
     abortSignal?: AbortSignal;
     attempts?: number;
     matchError?(error: unknown): boolean;

packages/utils/src/lib/retry.ts

@@ -48,7 +48,7 @@ import { sleep } from './control'
  * @internal
  */
 export async function retry<T>(
-	fn: () => Promise<T>,
+	fn: (args: { attempt: number; remaining: number; total: number }) => Promise<T>,
 	{
 		attempts = 3,
 		waitDuration = 1000,
@@ -65,7 +65,7 @@ export async function retry<T>(
 	for (let i = 0; i < attempts; i++) {
 		if (abortSignal?.aborted) throw new Error('aborted')
 		try {
-			return await fn()
+			return await fn({ attempt: i, remaining: attempts - i, total: attempts })
 		} catch (e) {
 			if (matchError && !matchError(e)) throw e
 			error = e