fix: add IDB backing store corruption healing mechanism

leshniak · claude · leshniak · commit 6569dc798a61 · 2026-05-26T14:29:09.000+02:00
Adds a Dexie-style heal pattern to createStore for Chromium's Internal error opening backing store error (884K errors/month). - isBackingStoreError() detects the Chromium-specific corruption - Shared healAttemptsRemaining counter (3, reset on success) - On backing store error: clear cached connection, retry once - Clear dbp on rejection so retries get fresh indexedDB.open() - 5 new tests: mid-session heal, init heal, budget exhaustion, budget reset, error classification No deleteDatabase(), no provider swap, no UI changes. Scoped to IDBKeyValProvider only -- SQLite provider untouched. Ref: Expensify/App#90636 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
diff --git a/lib/storage/providers/IDBKeyValProvider/createStore.ts b/lib/storage/providers/IDBKeyValProvider/createStore.ts
@@ -2,12 +2,25 @@ import * as IDB from 'idb-keyval';
 import type {UseStore} from 'idb-keyval';
 import * as Logger from '../../../Logger';
 
+const HEAL_ATTEMPTS_MAX = 3;
+
+/**
+ * Detects the Chromium-specific IDB backing store corruption error.
+ * Fires when LevelDB files backing IndexedDB are corrupted and Chrome's
+ * internal recovery (RepairDB -> delete -> recreate) also fails.
+ * https://github.com/Expensify/App/issues/87862
+ */
+function isBackingStoreError(error: unknown): boolean {
+    return error instanceof DOMException && error.name === 'UnknownError' && error.message.includes('Internal error opening backing store');
+}
+
 // This is a copy of the createStore function from idb-keyval, we need a custom implementation
 // because we need to create the database manually in order to ensure that the store exists before we use it.
 // If the store does not exist, idb-keyval will throw an error
 // source: https://github.com/jakearchibald/idb-keyval/blob/9d19315b4a83897df1e0193dccdc29f78466a0f3/src/index.ts#L12
 function createStore(dbName: string, storeName: string): UseStore {
     let dbp: Promise<IDBDatabase> | undefined;
+    let healAttemptsRemaining = HEAL_ATTEMPTS_MAX;
 
     const attachHandlers = (db: IDBDatabase) => {
         // Browsers may close idle IDB connections at any time, especially Safari.
@@ -36,11 +49,11 @@ function createStore(dbName: string, storeName: string): UseStore {
         request.onupgradeneeded = () => request.result.createObjectStore(storeName);
         dbp = IDB.promisifyRequest(request);
 
-        dbp.then(
-            attachHandlers,
-            // eslint-disable-next-line @typescript-eslint/no-empty-function
-            () => {},
-        );
+        dbp.then(attachHandlers, () => {
+            // Clear the cached rejected promise so the next operation retries
+            // with a fresh indexedDB.open() instead of returning the same rejection.
+            dbp = undefined;
+        });
         return dbp;
     };
 
@@ -67,8 +80,9 @@ function createStore(dbName: string, storeName: string): UseStore {
         };
 
         dbp = IDB.promisifyRequest(request);
-        // eslint-disable-next-line @typescript-eslint/no-empty-function
-        dbp.then(attachHandlers, () => {});
+        dbp.then(attachHandlers, () => {
+            dbp = undefined;
+        });
         return dbp;
     };
 
@@ -78,23 +92,45 @@ function createStore(dbName: string, storeName: string): UseStore {
             .then((db) => callback(db.transaction(storeName, txMode).objectStore(storeName)));
     }
 
-    // If the connection was closed between getDB() resolving and db.transaction() executing,
-    // the transaction throws InvalidStateError. We catch it and retry once with a fresh connection.
+    function resetHealBudget<T>(result: T): T {
+        healAttemptsRemaining = HEAL_ATTEMPTS_MAX;
+        return result;
+    }
+
+    // Handles two recoverable error classes:
+    // 1. InvalidStateError — connection closed between getDB() resolving and db.transaction().
+    //    Retry once with a fresh connection.
+    // 2. Backing store corruption (Chromium UnknownError) — close + reopen the IDB connection.
+    //    Bounded by a shared heal budget (3 attempts, reset on success).
+    //    Mirrors Dexie's PR1398_maxLoop pattern: https://github.com/dexie/Dexie.js/blob/master/src/functions/temp-transaction.ts
     return (txMode, callback) =>
-        executeTransaction(txMode, callback).catch((error) => {
-            if (error instanceof DOMException && error.name === 'InvalidStateError') {
-                Logger.logAlert('IDB InvalidStateError, retrying with fresh connection', {
-                    dbName,
-                    storeName,
-                    txMode,
-                    errorMessage: error.message,
-                });
-                dbp = undefined;
-                // Retry only once — this call is not wrapped, so if it also fails the error propagates normally.
-                return executeTransaction(txMode, callback);
-            }
-            throw error;
-        });
+        executeTransaction(txMode, callback)
+            .then(resetHealBudget)
+            .catch((error) => {
+                if (error instanceof DOMException && error.name === 'InvalidStateError') {
+                    Logger.logAlert('IDB InvalidStateError, retrying with fresh connection', {
+                        dbName,
+                        storeName,
+                        txMode,
+                        errorMessage: error.message,
+                    });
+                    dbp = undefined;
+                    return executeTransaction(txMode, callback).then(resetHealBudget);
+                }
+
+                if (isBackingStoreError(error) && healAttemptsRemaining > 0) {
+                    healAttemptsRemaining--;
+                    Logger.logInfo('IDB heal: backing store error, attempting close + reopen', {
+                        dbName,
+                        storeName,
+                        healAttemptsRemaining,
+                    });
+                    dbp = undefined;
+                    return executeTransaction(txMode, callback).then(resetHealBudget);
+                }
+
+                throw error;
+            });
 }
 
 export default createStore;
diff --git a/tests/unit/storage/providers/createStoreTest.ts b/tests/unit/storage/providers/createStoreTest.ts
@@ -245,4 +245,172 @@ describe('createStore', () => {
             expect(result).toBe('value');
         });
     });
+
+    describe('backing store healing', () => {
+        /**
+         * Helper: creates a DOMException matching Chromium's backing store corruption error.
+         */
+        function backingStoreError() {
+            return new DOMException('Internal error opening backing store for indexedDB.open.', 'UnknownError');
+        }
+
+        it('should heal mid-session by closing and reopening the connection', async () => {
+            const store = createStore(uniqueDBName(), STORE_NAME);
+
+            await store('readwrite', (s) => {
+                s.put('value', 'key1');
+                return IDB.promisifyRequest(s.transaction);
+            });
+
+            const original = IDBDatabase.prototype.transaction;
+            let callCount = 0;
+            jest.spyOn(IDBDatabase.prototype, 'transaction').mockImplementation(function (this: IDBDatabase, ...args) {
+                callCount++;
+                if (callCount === 1) {
+                    throw backingStoreError();
+                }
+                return original.apply(this, args);
+            });
+
+            const result = await store('readonly', (s) => IDB.promisifyRequest(s.get('key1')));
+            expect(result).toBe('value');
+            expect(callCount).toBe(2);
+            expect(logInfoSpy).toHaveBeenCalledWith('IDB heal: backing store error, attempting close + reopen', expect.objectContaining({healAttemptsRemaining: 2}));
+        });
+
+        it('should heal on init when indexedDB.open() rejects with UnknownError', async () => {
+            const dbName = uniqueDBName();
+
+            // Pre-create the DB with data
+            const setupStore = createStore(dbName, STORE_NAME);
+            await setupStore('readwrite', (s) => {
+                s.put('persisted', 'key1');
+                return IDB.promisifyRequest(s.transaction);
+            });
+
+            // Fresh store instance (simulates app restart — dbp starts undefined)
+            const store = createStore(dbName, STORE_NAME);
+
+            // Mock indexedDB.open to fail twice, then succeed.
+            // First failure: initial open in the first store() call.
+            // Second failure: the heal retry's open in that same call — propagates error.
+            // Third call (second store() invocation): succeeds, proving budget still has 1 left.
+            const realOpen = indexedDB.open.bind(indexedDB);
+            let openCallCount = 0;
+            jest.spyOn(indexedDB, 'open').mockImplementation((name: string, version?: number) => {
+                openCallCount++;
+                if (openCallCount <= 2) {
+                    const req = {} as IDBOpenDBRequest;
+                    Promise.resolve().then(() => {
+                        Object.defineProperty(req, 'error', {value: backingStoreError(), configurable: true});
+                        req.onerror?.(new Event('error') as Event & {target: IDBOpenDBRequest});
+                    });
+                    return req;
+                }
+                return realOpen(name, version);
+            });
+
+            // First call: open fails, heal retry also fails — error propagates
+            await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('key1')))).rejects.toThrow('Internal error opening backing store');
+
+            // Second call: open succeeds (mock exhausted), heals
+            const result = await store('readonly', (s) => IDB.promisifyRequest(s.get('key1')));
+            expect(result).toBe('persisted');
+            expect(openCallCount).toBeGreaterThanOrEqual(3);
+        });
+
+        it('should stop healing after budget exhausts across multiple operations', async () => {
+            const store = createStore(uniqueDBName(), STORE_NAME);
+
+            // All opens fail permanently
+            jest.spyOn(indexedDB, 'open').mockImplementation(() => {
+                const req = {} as IDBOpenDBRequest;
+                Promise.resolve().then(() => {
+                    Object.defineProperty(req, 'error', {value: backingStoreError(), configurable: true});
+                    req.onerror?.(new Event('error') as Event & {target: IDBOpenDBRequest});
+                });
+                return req;
+            });
+
+            // Each call consumes 1 heal attempt (initial fails, heal retry also fails, propagates)
+            await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('k')))).rejects.toThrow('Internal error opening backing store');
+            await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('k')))).rejects.toThrow('Internal error opening backing store');
+            await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('k')))).rejects.toThrow('Internal error opening backing store');
+
+            // Budget exhausted — 4th call should NOT attempt healing
+            logInfoSpy.mockClear();
+            await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('k')))).rejects.toThrow('Internal error opening backing store');
+            expect(logInfoSpy).not.toHaveBeenCalledWith(expect.stringContaining('IDB heal'), expect.anything());
+        });
+
+        it('should reset heal budget after a successful operation', async () => {
+            const dbName = uniqueDBName();
+            const store = createStore(dbName, STORE_NAME);
+
+            await store('readwrite', (s) => {
+                s.put('value', 'key1');
+                return IDB.promisifyRequest(s.transaction);
+            });
+
+            const original = IDBDatabase.prototype.transaction;
+
+            // Drain budget to 1 remaining: fail twice, each heals successfully
+            for (let i = 0; i < 2; i++) {
+                let callCount = 0;
+                const spy = jest.spyOn(IDBDatabase.prototype, 'transaction').mockImplementation(function (this: IDBDatabase, ...args) {
+                    callCount++;
+                    if (callCount === 1) {
+                        throw backingStoreError();
+                    }
+                    spy.mockRestore();
+                    return original.apply(this, args);
+                });
+                await store('readonly', (s) => IDB.promisifyRequest(s.get('key1')));            }
+
+            // Clean success — resets budget to 3
+            jest.restoreAllMocks();
+            logInfoSpy = jest.spyOn(Logger, 'logInfo');
+            await store('readonly', (s) => IDB.promisifyRequest(s.get('key1')));
+
+            // Now fail 3 more times — all should still heal (proving budget was reset)
+            for (let i = 0; i < 3; i++) {
+                let callCount = 0;
+                const spy = jest.spyOn(IDBDatabase.prototype, 'transaction').mockImplementation(function (this: IDBDatabase, ...args) {
+                    callCount++;
+                    if (callCount === 1) {
+                        throw backingStoreError();
+                    }
+                    spy.mockRestore();
+                    return original.apply(this, args);
+                });
+                const result = await store('readonly', (s) => IDB.promisifyRequest(s.get('key1')));                expect(result).toBe('value');
+            }
+        });
+
+        it('should not attempt healing for non-backing-store errors', async () => {
+            const store = createStore(uniqueDBName(), STORE_NAME);
+
+            await store('readwrite', (s) => {
+                s.put('value', 'key1');
+                return IDB.promisifyRequest(s.transaction);
+            });
+
+            // UnknownError but different message
+            jest.spyOn(IDBDatabase.prototype, 'transaction').mockImplementation(() => {
+                throw new DOMException('Some other unknown error', 'UnknownError');
+            });
+            await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('key1')))).rejects.toThrow('Some other unknown error');
+
+            jest.restoreAllMocks();
+            logInfoSpy = jest.spyOn(Logger, 'logInfo');
+
+            // QuotaExceededError
+            jest.spyOn(IDBDatabase.prototype, 'transaction').mockImplementation(() => {
+                throw new DOMException('Quota exceeded', 'QuotaExceededError');
+            });
+            await expect(store('readonly', (s) => IDB.promisifyRequest(s.get('key1')))).rejects.toThrow('Quota exceeded');
+
+            expect(logInfoSpy).not.toHaveBeenCalledWith(expect.stringContaining('IDB heal'), expect.anything());
+        });
+    });
 });
