Retrying requestRaw and chunked sandbox uploads

Author: richiemcilroy

packages/sdk-ts/src/http.test.ts

@@ -561,4 +561,101 @@ describe('HttpClient', () => {
       }
     })
   })
+
+  describe('requestRaw', () => {
+    test('returns Response directly on success', async () => {
+      globalThis.fetch = mock(async () =>
+        new Response('file contents', { status: 200 }),
+      ) as unknown as typeof fetch
+
+      const client = createClient()
+      const response = await client.requestRaw({ method: 'GET', path: '/v1/files' })
+      expect(response.status).toBe(200)
+      expect(await response.text()).toBe('file contents')
+    })
+
+    test('retries on 500 errors', async () => {
+      let calls = 0
+      globalThis.fetch = mock(async () => {
+        calls++
+        if (calls === 1) {
+          return new Response('internal error', { status: 500 })
+        }
+        return new Response('ok', { status: 200 })
+      }) as unknown as typeof fetch
+
+      const client = createClient({ retries: 1 })
+      const response = await client.requestRaw({ method: 'PUT', path: '/v1/files' })
+      expect(response.status).toBe(200)
+      expect(calls).toBe(2)
+    })
+
+    test('retries on network errors', async () => {
+      let calls = 0
+      globalThis.fetch = mock(async () => {
+        calls++
+        if (calls === 1) {
+          throw new TypeError('fetch failed')
+        }
+        return new Response('ok', { status: 200 })
+      }) as unknown as typeof fetch
+
+      const client = createClient({ retries: 1 })
+      const response = await client.requestRaw({ method: 'PUT', path: '/v1/files' })
+      expect(response.status).toBe(200)
+      expect(calls).toBe(2)
+    })
+
+    test('throws after exhausting retries on 500', async () => {
+      globalThis.fetch = mock(async () =>
+        new Response(JSON.stringify(errorBody('internal_error', 'Server error')), {
+          status: 500,
+          headers: { 'Content-Type': 'application/json' },
+        }),
+      ) as unknown as typeof fetch
+
+      const client = createClient({ retries: 1 })
+      try {
+        await client.requestRaw({ method: 'PUT', path: '/v1/files' })
+        expect.unreachable('should have thrown')
+      } catch (err) {
+        expect(err).toBeInstanceOf(SandchestError)
+        expect((err as SandchestError).status).toBe(500)
+      }
+    })
+
+    test('does not retry on 4xx client errors', async () => {
+      let calls = 0
+      globalThis.fetch = mock(async () => {
+        calls++
+        return new Response(
+          JSON.stringify(errorBody('not_found', 'Not found', 'req_1')),
+          { status: 404, headers: { 'Content-Type': 'application/json' } },
+        )
+      }) as unknown as typeof fetch
+
+      const client = createClient({ retries: 2 })
+      try {
+        await client.requestRaw({ method: 'GET', path: '/v1/files' })
+        expect.unreachable('should have thrown')
+      } catch (err) {
+        expect(err).toBeInstanceOf(NotFoundError)
+        expect(calls).toBe(1)
+      }
+    })
+
+    test('wraps network errors into ConnectionError after exhausting retries', async () => {
+      globalThis.fetch = mock(async () => {
+        throw new TypeError('fetch failed')
+      }) as unknown as typeof fetch
+
+      const client = createClient({ retries: 1 })
+      try {
+        await client.requestRaw({ method: 'PUT', path: '/v1/files' })
+        expect.unreachable('should have thrown')
+      } catch (err) {
+        expect(err).toBeInstanceOf(ConnectionError)
+      }
+    })
+  })
 })

packages/sdk-ts/src/http.ts

@@ -187,32 +187,60 @@ export class HttpClient {
     }
 
     const timeoutMs = options.timeout ?? this.timeout
-    const controller = new AbortController()
-    const timer = setTimeout(() => controller.abort(), timeoutMs)
-
-    try {
-      const response = await fetch(url, {
-        method: options.method,
-        headers,
-        body: options.body,
-        signal: controller.signal,
-      })
+    let lastError: Error | undefined
+
+    for (let attempt = 0; attempt <= this.retries; attempt++) {
+      if (attempt > 0) {
+        await sleep(backoffDelay(attempt - 1))
+      }
+
+      const controller = new AbortController()
+      const timer = setTimeout(() => controller.abort(), timeoutMs)
+
+      try {
+        const response = await fetch(url, {
+          method: options.method,
+          headers,
+          body: options.body,
+          signal: controller.signal,
+        })
+
+        clearTimeout(timer)
+
+        if (response.ok) {
+          return response
+        }
 
-      clearTimeout(timer)
+        // Retry on server errors
+        if (response.status >= 500 && attempt < this.retries) {
+          await response.text().catch(() => {}) // drain body to free connection
+          lastError = new SandchestError({
+            code: 'internal_error',
+            message: `HTTP ${response.status}`,
+            status: response.status,
+            requestId: response.headers.get('x-request-id') ?? '',
+          })
+          continue
+        }
 
-      if (!response.ok) {
         const errorBody = (await response.json().catch(() => null)) as ApiErrorBody | null
         const requestId = errorBody?.request_id ?? response.headers.get('x-request-id') ?? ''
         const message = errorBody?.message ?? `HTTP ${response.status}`
         throw this.parseErrorResponse(response.status, message, requestId, errorBody)
-      }
+      } catch (error) {
+        clearTimeout(timer)
+        if (error instanceof SandchestError) throw error
+
+        if (attempt < this.retries) {
+          lastError = error instanceof Error ? error : new Error(String(error))
+          continue
+        }
 
-      return response
-    } catch (error) {
-      clearTimeout(timer)
-      if (error instanceof SandchestError) throw error
-      throw this.wrapRawError(error, timeoutMs)
+        throw this.wrapRawError(lastError ?? error, timeoutMs)
+      }
     }
+
+    throw this.wrapRawError(lastError, timeoutMs)
   }
 
   /** Wrap non-SDK errors into typed TimeoutError or ConnectionError. */

packages/sdk-ts/src/sandbox.ts

@@ -40,6 +40,8 @@ const WAIT_READY_DEFAULT_TIMEOUT = 120_000
 const WAIT_READY_POLL_INTERVAL = 1_000
 const TEXT_ENCODER = new TextEncoder()
 const TEXT_DECODER = new TextDecoder()
+/** Uploads larger than this threshold are split into chunks to avoid body-size issues. */
+const UPLOAD_CHUNK_SIZE = 2 * 1024 * 1024 // 2 MB
 const VALIDATE_TAR_SCRIPT = `
 import posixpath
 import sys
@@ -201,14 +203,49 @@ export class Sandbox {
         await this.fs.upload(path, TEXT_ENCODER.encode(content))
       },
       uploadDir: async (path: string, tarball: Uint8Array): Promise<void> => {
-        const tmpPath = `/tmp/.sandchest-upload-${crypto.randomUUID()}.tar.gz`
-        await this._http.requestRaw({
-          method: 'PUT',
-          path: `/v1/sandboxes/${this.id}/files`,
-          query: { path: tmpPath, batch: true },
-          body: tarball,
-          headers: { 'Content-Type': 'application/octet-stream' },
-        })
+        const uuid = crypto.randomUUID()
+        const tmpPath = `/tmp/.sandchest-upload-${uuid}.tar.gz`
+        const chunkPaths: string[] = []
+
+        // Upload the tarball — chunked for large files to avoid body-size issues
+        if (tarball.byteLength <= UPLOAD_CHUNK_SIZE) {
+          await this._http.requestRaw({
+            method: 'PUT',
+            path: `/v1/sandboxes/${this.id}/files`,
+            query: { path: tmpPath, batch: true },
+            body: tarball,
+            headers: { 'Content-Type': 'application/octet-stream' },
+          })
+        } else {
+          const totalChunks = Math.ceil(tarball.byteLength / UPLOAD_CHUNK_SIZE)
+          for (let i = 0; i < totalChunks; i++) {
+            const start = i * UPLOAD_CHUNK_SIZE
+            const end = Math.min(start + UPLOAD_CHUNK_SIZE, tarball.byteLength)
+            const chunkPath = `/tmp/.sandchest-chunk-${uuid}-${String(i).padStart(4, '0')}`
+            chunkPaths.push(chunkPath)
+            await this._http.requestRaw({
+              method: 'PUT',
+              path: `/v1/sandboxes/${this.id}/files`,
+              query: { path: chunkPath },
+              body: tarball.slice(start, end),
+              headers: { 'Content-Type': 'application/octet-stream' },
+            })
+          }
+
+          // Concatenate chunks into the final tarball
+          const catResult = await this._execSync(
+            ['sh', '-c', `cat /tmp/.sandchest-chunk-${uuid}-* > ${tmpPath}`],
+            { operation: 'uploadDir:concat' },
+          )
+          if (catResult.exit_code !== 0) {
+            throw new ExecFailedError({
+              operation: 'uploadDir:concat',
+              exitCode: catResult.exit_code,
+              stderr: catResult.stderr,
+            })
+          }
+        }
+
         try {
           const mkdirResult = await this._execSync(['mkdir', '-p', path], {
             operation: 'uploadDir:mkdir',
@@ -255,9 +292,12 @@ export class Sandbox {
             })
           }
         } finally {
-          await this._execSync(['rm', '-f', tmpPath], {
-            operation: 'uploadDir:cleanup',
-          }).catch(() => {})
+          // Clean up tarball and any chunk files
+          const cleanupPaths = [tmpPath, ...chunkPaths]
+          await this._execSync(
+            ['rm', '-f', ...cleanupPaths],
+            { operation: 'uploadDir:cleanup' },
+          ).catch(() => {})
         }
       },
       download: async (path: string): Promise<Uint8Array> => {
@@ -362,6 +402,15 @@ export class Sandbox {
 
         cmd.push('--', validated.url, dest)
 
+        // Ensure the parent directory exists (sandbox root FS is read-only outside writable paths)
+        const parentDir = dest.includes('/') ? dest.slice(0, dest.lastIndexOf('/')) : undefined
+        if (parentDir && parentDir !== '') {
+          await this._execSync(['mkdir', '-p', parentDir], {
+            operation: 'git.clone:mkdir',
+            timeout: 5,
+          }).catch(() => {}) // Best-effort — parent may already exist
+        }
+
         const result = await this._execSync(cmd, {
           operation: 'git.clone',
           timeout: options?.timeout ?? 120,