fix(infra): add retry logic for transient R2 and service binding errors (tldraw#8280)

Sentry shows multiple transient Cloudflare R2 errors (`internal error
(10001)`, `connectivity issue`, `network connection lost`, `unspecified
error (0)`) and service binding errors (`proxy request failed`) in the
tldrawusercontent worker. Found these when investigating user reports of
image uploads not working for them.

This PR wraps R2 `head`, `put`, and `get` calls in `retry()` (3
attempts, 500ms wait) with an error matcher for known transient
patterns. The upload body is buffered to `ArrayBuffer` before retries
since `ReadableStream` is single-use. Service binding calls
(`validateUpload`, `confirmUpload`) get the same treatment.

### Change type

- [x] `bugfix`

### Test plan

- Cannot be manually tested (transient infra errors)

### Release notes

- Fix transient R2 upload/download failures by adding retry logic

### Code changes

| Section    | LOC change |
| ---------- | ---------- |
| Core code  | +30 / -7   |
| Apps       | +14 / -4   |

Author: MitjaBezensek

apps/dotcom/sync-worker/src/worker.ts

@@ -318,7 +318,8 @@ export default class Worker extends WorkerEntrypoint<Environment> {
 				await db
 					.insertInto('asset')
 					.values({ objectName, fileId, userId })
-					.executeTakeFirstOrThrow()
+					.onConflict((oc) => oc.column('objectName').doNothing())
+					.execute()
 				message.ack()
 			} catch (_e) {
 				message.retry({

apps/dotcom/tldrawusercontent-worker/src/worker.ts

@@ -9,6 +9,8 @@ import {
 	handleUserAssetUpload,
 	isAllowedOrigin,
 	notFound,
+	retry,
+	TRANSIENT_RETRY_OPTIONS,
 } from '@tldraw/worker-shared'
 import { WorkerEntrypoint } from 'cloudflare:workers'
 import { cors } from 'itty-router'
@@ -43,7 +45,10 @@ export default class Worker extends WorkerEntrypoint<Environment> {
 			let userId: string | null = null
 			if (fileId) {
 				const authHeader = request.headers.get('Authorization')
-				const validation = await this.env.SYNC_WORKER.validateUpload(fileId, authHeader)
+				const validation = await retry(
+					() => this.env.SYNC_WORKER.validateUpload(fileId, authHeader),
+					TRANSIENT_RETRY_OPTIONS
+				)
 				if (!validation.ok) {
 					const status = validation.error === 'File not found' ? 404 : 403
 					return Response.json({ error: validation.error }, { status })
@@ -64,7 +69,10 @@ export default class Worker extends WorkerEntrypoint<Environment> {
 			// will pick up the association later if this fails.
 			if (res.status === 200 && fileId) {
 				try {
-					await this.env.SYNC_WORKER.confirmUpload(objectName, fileId, userId)
+					await retry(
+						() => this.env.SYNC_WORKER.confirmUpload(objectName, fileId, userId),
+						TRANSIENT_RETRY_OPTIONS
+					)
 				} catch (e) {
 					console.error('Failed to queue asset association', e)
 				}

packages/worker-shared/src/index.ts

@@ -1,6 +1,7 @@
 /// <reference no-default-lib="true"/>
 /// <reference types="@cloudflare/workers-types" />
 
+export { retry } from '@tldraw/utils'
 export { handleExtractBookmarkMetadataRequest } from './bookmarks'
 export { forbidden, notFound } from './errors'
 export {
@@ -12,4 +13,9 @@ export {
 } from './handleRequest'
 export { blockUnknownOrigins, isAllowedOrigin } from './origins'
 export { createSentry } from './sentry'
-export { handleUserAssetGet, handleUserAssetUpload, type R2BucketLike } from './userAssetUploads'
+export {
+	TRANSIENT_RETRY_OPTIONS,
+	handleUserAssetGet,
+	handleUserAssetUpload,
+	type R2BucketLike,
+} from './userAssetUploads'

packages/worker-shared/src/userAssetUploads.ts

@@ -1,6 +1,20 @@
+import { retry } from '@tldraw/utils'
 import { IRequest } from 'itty-router'
 import { notFound } from './errors'
 
+function isTransientWorkerError(error: unknown): boolean {
+	const msg = String(error)
+	return /internal error|connectivity|network connection lost|service temporarily unavailable|proxy request failed|unspecified error|connection (refused|reset|timed?\s?out)/i.test(
+		msg
+	)
+}
+
+export const TRANSIENT_RETRY_OPTIONS = {
+	attempts: 3,
+	waitDuration: 500,
+	matchError: isTransientWorkerError,
+} as const
+
 // Minimal interface for R2Bucket operations used in this file
 // This avoids type conflicts between ambient and imported Cloudflare types
 // Using 'any' for return types to allow compatibility with different R2Bucket type definitions
@@ -57,13 +71,18 @@ export async function handleUserAssetUpload({
 	body: ReadableStream | null
 	headers: Headers
 }): Promise<Response> {
-	if (await bucket.head(objectName)) {
+	const existing = await retry(() => bucket.head(objectName), TRANSIENT_RETRY_OPTIONS)
+	if (existing) {
 		return Response.json({ error: 'Asset already exists' }, { status: 409 })
 	}
 
-	const object = await bucket.put(objectName, body, {
-		httpMetadata: headers,
-	})
+	// Buffer body so retries can re-send (ReadableStream is single-use)
+	const buffer = body ? await new Response(body).arrayBuffer() : null
+
+	const object = await retry(
+		() => bucket.put(objectName, buffer, { httpMetadata: headers }),
+		TRANSIENT_RETRY_OPTIONS
+	)
 
 	return Response.json({ object: objectName }, { headers: { etag: object.httpEtag } })
 }
@@ -115,10 +134,10 @@ export async function handleUserAssetGet({
 		return cachedResponse
 	}
 
-	const object = await bucket.get(objectName, {
-		range: request.headers,
-		onlyIf: request.headers,
-	})
+	const object = await retry(
+		() => bucket.get(objectName, { range: request.headers, onlyIf: request.headers }),
+		TRANSIENT_RETRY_OPTIONS
+	)
 
 	if (!object) {
 		return notFound()