Fix R2 persistence: three interacting bugs + sandbox API workarounds

Fix three bugs that together prevented R2 backup/restore from working:

Bug 1 - waitForProcess didn't handle 'starting' status:
  The poll loop only waited while status was 'running', exiting
  immediately if a fast command was still 'starting'.

Bug 2 - exitCode null check broke config detection:
  exitCode is often undefined in the sandbox API. Switched to
  stdout-based detection (echo exists) instead of exitCode checks.

Bug 3 - should_restore_from_r2() race condition:
  Called three times (config, workspace, skills) but the config
  restore copied .last-sync locally, making timestamps match and
  skipping workspace/skills. Now evaluates once with DO_RESTORE flag.

Additional fixes discovered during debugging:

- proc.status is a readonly snapshot that never updates. Use
  getStatus() to poll for actual status changes.

- getStatus() itself returns 'running' indefinitely for shell
  commands (known sandbox API behavior). Sync now starts the rsync
  chain and polls for the timestamp file instead of waiting on
  process status.

- isR2Mounted used getLogs().stdout which was empty (process hadn't
  completed). Now uses stdout marker pattern with waitForProcess.

- Exclude .git from rsync (workspace/.git/ has 50+ hook files,
  each taking seconds over s3fs).

- Make workspace/skills rsync non-fatal (directories may not exist
  in fresh containers).

- Replace inline wait loops in debug.ts with shared waitForProcess.

- Cron handler now uses fire-and-forget sync instead of polling.
  The scheduled handler has a strict time limit — polling competes
  with slow s3fs operations and exceeds it, causing an unhandled
  exception that resets the Durable Object and kills the container.

- Add comprehensive e2e tests covering sync, restart persistence,
  and workspace marker file survival.

Fixes #212, #228, #102, #86

Author: andreasjansson

src/gateway/index.ts

@@ -1,5 +1,5 @@
 export { buildEnvVars } from './env';
 export { mountR2Storage } from './r2';
 export { findExistingMoltbotProcess, ensureMoltbotGateway } from './process';
-export { syncToR2 } from './sync';
+export { fireAndForgetSync, syncToR2 } from './sync';
 export { waitForProcess } from './utils';

src/gateway/r2.test.ts

@@ -145,8 +145,8 @@ describe('mountR2Storage', () => {
     it('returns true if mount fails but check shows it is actually mounted', async () => {
       const { sandbox, mountBucketMock, startProcessMock } = createMockSandbox();
       startProcessMock
-        .mockResolvedValueOnce(createMockProcess(''))
-        .mockResolvedValueOnce(createMockProcess('s3fs on /data/moltbot type fuse.s3fs\n'));
+        .mockResolvedValueOnce(createMockProcess('not-mounted\n'))
+        .mockResolvedValueOnce(createMockProcess('mounted\n'));
 
       mountBucketMock.mockRejectedValue(new Error('Transient error'));
 

src/gateway/r2.ts

@@ -1,23 +1,24 @@
 import type { Sandbox } from '@cloudflare/sandbox';
 import type { MoltbotEnv } from '../types';
 import { R2_MOUNT_PATH, getR2BucketName } from '../config';
+import { waitForProcess } from './utils';
 
 /**
- * Check if R2 is already mounted by looking at the mount table
+ * Check if R2 is already mounted by looking at the mount table.
+ * Uses stdout marker pattern because getLogs() often returns empty.
  */
 async function isR2Mounted(sandbox: Sandbox): Promise<boolean> {
   try {
-    const proc = await sandbox.startProcess(`mount | grep "s3fs on ${R2_MOUNT_PATH}"`);
-    // Wait for the command to complete
-    let attempts = 0;
-    while (proc.status === 'running' && attempts < 10) {
-      // eslint-disable-next-line no-await-in-loop -- intentional sequential polling
-      await new Promise((r) => setTimeout(r, 200));
-      attempts++;
-    }
+    const proc = await sandbox.startProcess(
+      `mount | grep -q "s3fs on ${R2_MOUNT_PATH}" && echo mounted || echo not-mounted`,
+    );
+    await waitForProcess(proc, 5000);
     const logs = await proc.getLogs();
-    // If stdout has content, the mount exists
-    const mounted = !!(logs.stdout && logs.stdout.includes('s3fs'));
+    const mounted = !!(
+      logs.stdout &&
+      logs.stdout.includes('mounted') &&
+      !logs.stdout.includes('not-mounted')
+    );
     console.log('isR2Mounted check:', mounted, 'stdout:', logs.stdout?.slice(0, 100));
     return mounted;
   } catch (err) {
@@ -34,15 +35,13 @@ async function isR2Mounted(sandbox: Sandbox): Promise<boolean> {
  * @returns true if mounted successfully, false otherwise
  */
 export async function mountR2Storage(sandbox: Sandbox, env: MoltbotEnv): Promise<boolean> {
-  // Skip if R2 credentials are not configured
   if (!env.R2_ACCESS_KEY_ID || !env.R2_SECRET_ACCESS_KEY || !env.CF_ACCOUNT_ID) {
     console.log(
       'R2 storage not configured (missing R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, or CF_ACCOUNT_ID)',
     );
     return false;
   }
 
-  // Check if already mounted first - this avoids errors and is faster
   if (await isR2Mounted(sandbox)) {
     console.log('R2 bucket already mounted at', R2_MOUNT_PATH);
     return true;
@@ -53,7 +52,6 @@ export async function mountR2Storage(sandbox: Sandbox, env: MoltbotEnv): Promise
     console.log('Mounting R2 bucket', bucketName, 'at', R2_MOUNT_PATH);
     await sandbox.mountBucket(bucketName, R2_MOUNT_PATH, {
       endpoint: `https://${env.CF_ACCOUNT_ID}.r2.cloudflarestorage.com`,
-      // Pass credentials explicitly since we use R2_* naming instead of AWS_*
       credentials: {
         accessKeyId: env.R2_ACCESS_KEY_ID,
         secretAccessKey: env.R2_SECRET_ACCESS_KEY,
@@ -65,13 +63,12 @@ export async function mountR2Storage(sandbox: Sandbox, env: MoltbotEnv): Promise
     const errorMessage = err instanceof Error ? err.message : String(err);
     console.log('R2 mount error:', errorMessage);
 
-    // Check again if it's mounted - the error might be misleading
+    // Check again if it's mounted - the error might be misleading (e.g. "already mounted")
     if (await isR2Mounted(sandbox)) {
       console.log('R2 bucket is mounted despite error');
       return true;
     }
 
-    // Don't fail if mounting fails - moltbot can still run without persistent storage
     console.error('Failed to mount R2 bucket:', err);
     return false;
   }

src/gateway/sync.test.ts

@@ -42,9 +42,9 @@ describe('syncToR2', () => {
     it('returns error when source has no config file', async () => {
       const { sandbox, startProcessMock } = createMockSandbox();
       startProcessMock
-        .mockResolvedValueOnce(createMockProcess('s3fs on /data/moltbot type fuse.s3fs\n'))
-        .mockResolvedValueOnce(createMockProcess('', { exitCode: 1 })) // No openclaw.json
-        .mockResolvedValueOnce(createMockProcess('', { exitCode: 1 })); // No clawdbot.json either
+        .mockResolvedValueOnce(createMockProcess('mounted\n'))
+        .mockResolvedValueOnce(createMockProcess('')) // No openclaw.json
+        .mockResolvedValueOnce(createMockProcess('')); // No clawdbot.json either
 
       const env = createMockEnvWithR2();
 
@@ -56,62 +56,76 @@ describe('syncToR2', () => {
   });
 
   describe('sync execution', () => {
-    it('returns success when sync completes', async () => {
+    it('returns success when timestamp file is written', async () => {
       const { sandbox, startProcessMock } = createMockSandbox();
       const timestamp = '2026-01-27T12:00:00+00:00';
 
-      // Calls: mount check, check openclaw.json, rsync, cat timestamp
+      // Calls: mount check, config detect, rsync (all-in-one), cat timestamp
       startProcessMock
-        .mockResolvedValueOnce(createMockProcess('s3fs on /data/moltbot type fuse.s3fs\n'))
-        .mockResolvedValueOnce(createMockProcess('ok'))
-        .mockResolvedValueOnce(createMockProcess(''))
-        .mockResolvedValueOnce(createMockProcess(timestamp));
+        .mockResolvedValueOnce(createMockProcess('mounted\n'))
+        .mockResolvedValueOnce(createMockProcess('exists'))
+        .mockResolvedValueOnce(createMockProcess('')) // rsync chain
+        .mockResolvedValueOnce(createMockProcess(timestamp)); // cat timestamp
 
       const env = createMockEnvWithR2();
-
       const result = await syncToR2(sandbox, env);
 
       expect(result.success).toBe(true);
       expect(result.lastSync).toBe(timestamp);
     });
 
-    it('returns error when rsync fails (no timestamp created)', async () => {
+    it('falls back to legacy clawdbot config directory', async () => {
       const { sandbox, startProcessMock } = createMockSandbox();
+      const timestamp = '2026-01-27T12:00:00+00:00';
 
-      // Calls: mount check, check openclaw.json, rsync (fails), cat timestamp (empty)
       startProcessMock
-        .mockResolvedValueOnce(createMockProcess('s3fs on /data/moltbot type fuse.s3fs\n'))
-        .mockResolvedValueOnce(createMockProcess('ok'))
-        .mockResolvedValueOnce(createMockProcess('', { exitCode: 1 }))
-        .mockResolvedValueOnce(createMockProcess(''));
+        .mockResolvedValueOnce(createMockProcess('mounted\n'))
+        .mockResolvedValueOnce(createMockProcess('')) // No openclaw.json
+        .mockResolvedValueOnce(createMockProcess('exists')) // clawdbot.json found
+        .mockResolvedValueOnce(createMockProcess('')) // rsync chain
+        .mockResolvedValueOnce(createMockProcess(timestamp));
 
       const env = createMockEnvWithR2();
-
       const result = await syncToR2(sandbox, env);
 
+      expect(result.success).toBe(true);
+
+      // rsync chain should reference .clawdbot
+      const rsyncCall = startProcessMock.mock.calls[3][0];
+      expect(rsyncCall).toContain('/root/.clawdbot/');
+    });
+
+    it('returns error when no timestamp after sync', async () => {
+      const { sandbox, startProcessMock } = createMockSandbox();
+
+      startProcessMock
+        .mockResolvedValueOnce(createMockProcess('mounted\n'))
+        .mockResolvedValueOnce(createMockProcess('exists'))
+        .mockResolvedValueOnce(createMockProcess('')) // rsync chain
+        .mockResolvedValue(createMockProcess('')); // all cat polls return empty
+
+      const env = createMockEnvWithR2();
+      const result = await syncToR2(sandbox, env, 10, 3); // fast: 10ms interval, 3 polls
+
       expect(result.success).toBe(false);
-      expect(result.error).toBe('Sync failed');
+      expect(result.error).toBe('Sync timed out');
     });
 
-    it('verifies rsync command is called with correct flags', async () => {
+    it('verifies rsync command excludes .git', async () => {
       const { sandbox, startProcessMock } = createMockSandbox();
       const timestamp = '2026-01-27T12:00:00+00:00';
 
       startProcessMock
-        .mockResolvedValueOnce(createMockProcess('s3fs on /data/moltbot type fuse.s3fs\n'))
-        .mockResolvedValueOnce(createMockProcess('ok'))
+        .mockResolvedValueOnce(createMockProcess('mounted\n'))
+        .mockResolvedValueOnce(createMockProcess('exists'))
         .mockResolvedValueOnce(createMockProcess(''))
         .mockResolvedValueOnce(createMockProcess(timestamp));
 
       const env = createMockEnvWithR2();
-
       await syncToR2(sandbox, env);
 
-      // Third call should be rsync to openclaw/ R2 prefix
       const rsyncCall = startProcessMock.mock.calls[2][0];
-      expect(rsyncCall).toContain('rsync');
-      expect(rsyncCall).toContain('--no-times');
-      expect(rsyncCall).toContain('--delete');
+      expect(rsyncCall).toContain("--exclude='.git'");
       expect(rsyncCall).toContain('/root/.openclaw/');
       expect(rsyncCall).toContain('/data/moltbot/openclaw/');
     });