Merge pull request #235 from cloudflare/fix-r2-persistence

Fix R2 persistence: three interacting bugs + sandbox API workarounds

Author: andreasjansson

src/gateway/index.ts

@@ -1,5 +1,5 @@
 export { buildEnvVars } from './env';
 export { mountR2Storage } from './r2';
 export { findExistingMoltbotProcess, ensureMoltbotGateway } from './process';
-export { syncToR2 } from './sync';
+export { fireAndForgetSync, syncToR2 } from './sync';
 export { waitForProcess } from './utils';

src/gateway/r2.test.ts

@@ -145,8 +145,8 @@ describe('mountR2Storage', () => {
     it('returns true if mount fails but check shows it is actually mounted', async () => {
       const { sandbox, mountBucketMock, startProcessMock } = createMockSandbox();
       startProcessMock
-        .mockResolvedValueOnce(createMockProcess(''))
-        .mockResolvedValueOnce(createMockProcess('s3fs on /data/moltbot type fuse.s3fs\n'));
+        .mockResolvedValueOnce(createMockProcess('not-mounted\n'))
+        .mockResolvedValueOnce(createMockProcess('mounted\n'));
 
       mountBucketMock.mockRejectedValue(new Error('Transient error'));
 

src/gateway/r2.ts

@@ -1,23 +1,24 @@
 import type { Sandbox } from '@cloudflare/sandbox';
 import type { MoltbotEnv } from '../types';
 import { R2_MOUNT_PATH, getR2BucketName } from '../config';
+import { waitForProcess } from './utils';
 
 /**
- * Check if R2 is already mounted by looking at the mount table
+ * Check if R2 is already mounted by looking at the mount table.
+ * Uses stdout marker pattern because getLogs() often returns empty.
  */
 async function isR2Mounted(sandbox: Sandbox): Promise<boolean> {
   try {
-    const proc = await sandbox.startProcess(`mount | grep "s3fs on ${R2_MOUNT_PATH}"`);
-    // Wait for the command to complete
-    let attempts = 0;
-    while (proc.status === 'running' && attempts < 10) {
-      // eslint-disable-next-line no-await-in-loop -- intentional sequential polling
-      await new Promise((r) => setTimeout(r, 200));
-      attempts++;
-    }
+    const proc = await sandbox.startProcess(
+      `mount | grep -q "s3fs on ${R2_MOUNT_PATH}" && echo mounted || echo not-mounted`,
+    );
+    await waitForProcess(proc, 5000);
     const logs = await proc.getLogs();
-    // If stdout has content, the mount exists
-    const mounted = !!(logs.stdout && logs.stdout.includes('s3fs'));
+    const mounted = !!(
+      logs.stdout &&
+      logs.stdout.includes('mounted') &&
+      !logs.stdout.includes('not-mounted')
+    );
     console.log('isR2Mounted check:', mounted, 'stdout:', logs.stdout?.slice(0, 100));
     return mounted;
   } catch (err) {
@@ -34,15 +35,13 @@ async function isR2Mounted(sandbox: Sandbox): Promise<boolean> {
  * @returns true if mounted successfully, false otherwise
  */
 export async function mountR2Storage(sandbox: Sandbox, env: MoltbotEnv): Promise<boolean> {
-  // Skip if R2 credentials are not configured
   if (!env.R2_ACCESS_KEY_ID || !env.R2_SECRET_ACCESS_KEY || !env.CF_ACCOUNT_ID) {
     console.log(
       'R2 storage not configured (missing R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, or CF_ACCOUNT_ID)',
     );
     return false;
   }
 
-  // Check if already mounted first - this avoids errors and is faster
   if (await isR2Mounted(sandbox)) {
     console.log('R2 bucket already mounted at', R2_MOUNT_PATH);
     return true;
@@ -53,7 +52,6 @@ export async function mountR2Storage(sandbox: Sandbox, env: MoltbotEnv): Promise
     console.log('Mounting R2 bucket', bucketName, 'at', R2_MOUNT_PATH);
     await sandbox.mountBucket(bucketName, R2_MOUNT_PATH, {
       endpoint: `https://${env.CF_ACCOUNT_ID}.r2.cloudflarestorage.com`,
-      // Pass credentials explicitly since we use R2_* naming instead of AWS_*
       credentials: {
         accessKeyId: env.R2_ACCESS_KEY_ID,
         secretAccessKey: env.R2_SECRET_ACCESS_KEY,
@@ -65,13 +63,12 @@ export async function mountR2Storage(sandbox: Sandbox, env: MoltbotEnv): Promise
     const errorMessage = err instanceof Error ? err.message : String(err);
     console.log('R2 mount error:', errorMessage);
 
-    // Check again if it's mounted - the error might be misleading
+    // Check again if it's mounted - the error might be misleading (e.g. "already mounted")
     if (await isR2Mounted(sandbox)) {
       console.log('R2 bucket is mounted despite error');
       return true;
     }
 
-    // Don't fail if mounting fails - moltbot can still run without persistent storage
     console.error('Failed to mount R2 bucket:', err);
     return false;
   }

src/gateway/sync.test.ts

@@ -42,9 +42,9 @@ describe('syncToR2', () => {
     it('returns error when source has no config file', async () => {
       const { sandbox, startProcessMock } = createMockSandbox();
       startProcessMock
-        .mockResolvedValueOnce(createMockProcess('s3fs on /data/moltbot type fuse.s3fs\n'))
-        .mockResolvedValueOnce(createMockProcess('', { exitCode: 1 })) // No openclaw.json
-        .mockResolvedValueOnce(createMockProcess('', { exitCode: 1 })); // No clawdbot.json either
+        .mockResolvedValueOnce(createMockProcess('mounted\n'))
+        .mockResolvedValueOnce(createMockProcess('')) // No openclaw.json
+        .mockResolvedValueOnce(createMockProcess('')); // No clawdbot.json either
 
       const env = createMockEnvWithR2();
 
@@ -56,62 +56,76 @@ describe('syncToR2', () => {
   });
 
   describe('sync execution', () => {
-    it('returns success when sync completes', async () => {
+    it('returns success when timestamp file is written', async () => {
       const { sandbox, startProcessMock } = createMockSandbox();
       const timestamp = '2026-01-27T12:00:00+00:00';
 
-      // Calls: mount check, check openclaw.json, rsync, cat timestamp
+      // Calls: mount check, config detect, rsync (all-in-one), cat timestamp
       startProcessMock
-        .mockResolvedValueOnce(createMockProcess('s3fs on /data/moltbot type fuse.s3fs\n'))
-        .mockResolvedValueOnce(createMockProcess('ok'))
-        .mockResolvedValueOnce(createMockProcess(''))
-        .mockResolvedValueOnce(createMockProcess(timestamp));
+        .mockResolvedValueOnce(createMockProcess('mounted\n'))
+        .mockResolvedValueOnce(createMockProcess('exists'))
+        .mockResolvedValueOnce(createMockProcess('')) // rsync chain
+        .mockResolvedValueOnce(createMockProcess(timestamp)); // cat timestamp
 
       const env = createMockEnvWithR2();
-
       const result = await syncToR2(sandbox, env);
 
       expect(result.success).toBe(true);
       expect(result.lastSync).toBe(timestamp);
     });
 
-    it('returns error when rsync fails (no timestamp created)', async () => {
+    it('falls back to legacy clawdbot config directory', async () => {
       const { sandbox, startProcessMock } = createMockSandbox();
+      const timestamp = '2026-01-27T12:00:00+00:00';
 
-      // Calls: mount check, check openclaw.json, rsync (fails), cat timestamp (empty)
       startProcessMock
-        .mockResolvedValueOnce(createMockProcess('s3fs on /data/moltbot type fuse.s3fs\n'))
-        .mockResolvedValueOnce(createMockProcess('ok'))
-        .mockResolvedValueOnce(createMockProcess('', { exitCode: 1 }))
-        .mockResolvedValueOnce(createMockProcess(''));
+        .mockResolvedValueOnce(createMockProcess('mounted\n'))
+        .mockResolvedValueOnce(createMockProcess('')) // No openclaw.json
+        .mockResolvedValueOnce(createMockProcess('exists')) // clawdbot.json found
+        .mockResolvedValueOnce(createMockProcess('')) // rsync chain
+        .mockResolvedValueOnce(createMockProcess(timestamp));
 
       const env = createMockEnvWithR2();
-
       const result = await syncToR2(sandbox, env);
 
+      expect(result.success).toBe(true);
+
+      // rsync chain should reference .clawdbot
+      const rsyncCall = startProcessMock.mock.calls[3][0];
+      expect(rsyncCall).toContain('/root/.clawdbot/');
+    });
+
+    it('returns error when no timestamp after sync', async () => {
+      const { sandbox, startProcessMock } = createMockSandbox();
+
+      startProcessMock
+        .mockResolvedValueOnce(createMockProcess('mounted\n'))
+        .mockResolvedValueOnce(createMockProcess('exists'))
+        .mockResolvedValueOnce(createMockProcess('')) // rsync chain
+        .mockResolvedValue(createMockProcess('')); // all cat polls return empty
+
+      const env = createMockEnvWithR2();
+      const result = await syncToR2(sandbox, env, 10, 3); // fast: 10ms interval, 3 polls
+
       expect(result.success).toBe(false);
-      expect(result.error).toBe('Sync failed');
+      expect(result.error).toBe('Sync timed out');
     });
 
-    it('verifies rsync command is called with correct flags', async () => {
+    it('verifies rsync command excludes .git', async () => {
       const { sandbox, startProcessMock } = createMockSandbox();
       const timestamp = '2026-01-27T12:00:00+00:00';
 
       startProcessMock
-        .mockResolvedValueOnce(createMockProcess('s3fs on /data/moltbot type fuse.s3fs\n'))
-        .mockResolvedValueOnce(createMockProcess('ok'))
+        .mockResolvedValueOnce(createMockProcess('mounted\n'))
+        .mockResolvedValueOnce(createMockProcess('exists'))
         .mockResolvedValueOnce(createMockProcess(''))
         .mockResolvedValueOnce(createMockProcess(timestamp));
 
       const env = createMockEnvWithR2();
-
       await syncToR2(sandbox, env);
 
-      // Third call should be rsync to openclaw/ R2 prefix
       const rsyncCall = startProcessMock.mock.calls[2][0];
-      expect(rsyncCall).toContain('rsync');
-      expect(rsyncCall).toContain('--no-times');
-      expect(rsyncCall).toContain('--delete');
+      expect(rsyncCall).toContain("--exclude='.git'");
       expect(rsyncCall).toContain('/root/.openclaw/');
       expect(rsyncCall).toContain('/data/moltbot/openclaw/');
     });