Merge pull request Expensify#780 from callstack-internal/fix/idb-corruption-detect-and-heal

fix: add IDB healing mechanism for backing store corruption and connection lost errors

Author: Julesssss

lib/storage/providers/IDBKeyValProvider/createStore.ts

@@ -2,12 +2,50 @@ import * as IDB from 'idb-keyval';
 import type {UseStore} from 'idb-keyval';
 import * as Logger from '../../../Logger';
 
+const HEAL_ATTEMPTS_MAX = 3;
+
+/**
+ * Detects the Chromium-specific IDB backing store corruption error.
+ * Fires when LevelDB files backing IndexedDB are corrupted and Chrome's
+ * internal recovery (RepairDB -> delete -> recreate) also fails.
+ */
+function isBackingStoreError(error: unknown): boolean {
+    return (error instanceof Error || error instanceof DOMException) && (error as Error).message.includes('Internal error opening backing store');
+}
+
+/**
+ * Detects Safari/WebKit IDB connection termination errors.
+ * Fires when Safari kills the IDB server process for backgrounded tabs.
+ * WebKit bugs: https://bugs.webkit.org/show_bug.cgi?id=197050, https://bugs.webkit.org/show_bug.cgi?id=201483
+ */
+function isConnectionLostError(error: unknown): boolean {
+    if (!(error instanceof Error || error instanceof DOMException)) return false;
+    const msg = (error as Error).message.toLowerCase();
+    return msg.includes('connection to indexed database server lost') || msg.includes('connection is closing');
+}
+
+function isInvalidStateError(error: unknown): boolean {
+    return (error instanceof Error || error instanceof DOMException) && (error as Error).name === 'InvalidStateError';
+}
+
+/** Errors that trigger a budgeted heal-and-retry in store(). */
+function isBudgetedHealError(error: unknown): boolean {
+    return isBackingStoreError(error) || isConnectionLostError(error);
+}
+
+function getBudgetedHealErrorLabel(error: unknown): string {
+    if (isBackingStoreError(error)) return 'backing store';
+    if (isConnectionLostError(error)) return 'connection lost';
+    return 'unknown';
+}
+
 // This is a copy of the createStore function from idb-keyval, we need a custom implementation
 // because we need to create the database manually in order to ensure that the store exists before we use it.
 // If the store does not exist, idb-keyval will throw an error
 // source: https://github.com/jakearchibald/idb-keyval/blob/9d19315b4a83897df1e0193dccdc29f78466a0f3/src/index.ts#L12
 function createStore(dbName: string, storeName: string): UseStore {
     let dbp: Promise<IDBDatabase> | undefined;
+    let healAttemptsRemaining = HEAL_ATTEMPTS_MAX;
 
     const attachHandlers = (db: IDBDatabase) => {
         // Browsers may close idle IDB connections at any time, especially Safari.
@@ -30,18 +68,27 @@ function createStore(dbName: string, storeName: string): UseStore {
         };
     };
 
+    // Cache the open promise and attach handlers + rejection cleanup.
+    // On rejection, clears dbp so the next operation retries with a fresh indexedDB.open()
+    // instead of returning the same rejected promise.
+    // Guard: only clear if dbp hasn't been replaced by a concurrent heal/retry.
+    function cacheOpenPromise(openPromise: Promise<IDBDatabase>) {
+        dbp = openPromise;
+        const currentPromise = openPromise;
+        openPromise.then(attachHandlers, () => {
+            if (dbp !== currentPromise) {
+                return;
+            }
+            dbp = undefined;
+        });
+        return openPromise;
+    }
+
     const getDB = () => {
         if (dbp) return dbp;
         const request = indexedDB.open(dbName);
         request.onupgradeneeded = () => request.result.createObjectStore(storeName);
-        dbp = IDB.promisifyRequest(request);
-
-        dbp.then(
-            attachHandlers,
-            // eslint-disable-next-line @typescript-eslint/no-empty-function
-            () => {},
-        );
-        return dbp;
+        return cacheOpenPromise(IDB.promisifyRequest(request));
     };
 
     // Ensures the store exists in the DB. If missing, bumps the version to trigger
@@ -66,10 +113,7 @@ function createStore(dbName: string, storeName: string): UseStore {
             updatedDatabase.createObjectStore(storeName);
         };
 
-        dbp = IDB.promisifyRequest(request);
-        // eslint-disable-next-line @typescript-eslint/no-empty-function
-        dbp.then(attachHandlers, () => {});
-        return dbp;
+        return cacheOpenPromise(IDB.promisifyRequest(request));
     };
 
     function executeTransaction<T>(txMode: IDBTransactionMode, callback: (store: IDBObjectStore) => T | PromiseLike<T>): Promise<T> {
@@ -78,23 +122,64 @@ function createStore(dbName: string, storeName: string): UseStore {
             .then((db) => callback(db.transaction(storeName, txMode).objectStore(storeName)));
     }
 
-    // If the connection was closed between getDB() resolving and db.transaction() executing,
-    // the transaction throws InvalidStateError. We catch it and retry once with a fresh connection.
+    function resetHealBudget<T>(result: T): T {
+        healAttemptsRemaining = HEAL_ATTEMPTS_MAX;
+        return result;
+    }
+
+    // Handles three recoverable error classes:
+    // 1. InvalidStateError — connection closed between getDB() resolving and db.transaction().
+    //    Retry once with a fresh connection. No budget limit (transient, always worth one reopen).
+    // 2. Backing store corruption (Chromium UnknownError) — drop cached connection and reopen.
+    // 3. Connection lost (Safari UnknownError) — IDB server terminated for backgrounded tabs.
+    //    Both 2 and 3 share a heal budget (3 attempts, reset on success).
+    //    Mirrors Dexie's PR1398_maxLoop pattern: https://github.com/dexie/Dexie.js/blob/master/src/functions/temp-transaction.ts
+    // Note: concurrent store() calls share the budget. Under overlapping failures each caller
+    // decrements independently, so the budget may drain faster than one-per-incident. This is
+    // acceptable — same as Dexie's approach — and the budget resets on any success.
     return (txMode, callback) =>
-        executeTransaction(txMode, callback).catch((error) => {
-            if (error instanceof DOMException && error.name === 'InvalidStateError') {
-                Logger.logAlert('IDB InvalidStateError, retrying with fresh connection', {
-                    dbName,
-                    storeName,
-                    txMode,
-                    errorMessage: error.message,
-                });
-                dbp = undefined;
-                // Retry only once — this call is not wrapped, so if it also fails the error propagates normally.
-                return executeTransaction(txMode, callback);
-            }
-            throw error;
-        });
+        executeTransaction(txMode, callback)
+            .then(resetHealBudget)
+            .catch((error) => {
+                if (isInvalidStateError(error)) {
+                    Logger.logInfo('IDB InvalidStateError — dropping cached connection and retrying', {
+                        dbName,
+                        storeName,
+                        txMode,
+                        errorMessage: error instanceof Error ? error.message : String(error),
+                    });
+                    dbp = undefined;
+                    return executeTransaction(txMode, callback).then(resetHealBudget);
+                }
+
+                if (isBudgetedHealError(error) && healAttemptsRemaining > 0) {
+                    healAttemptsRemaining--;
+                    const label = getBudgetedHealErrorLabel(error);
+                    Logger.logInfo(`IDB heal: ${label} error detected — dropping cached connection and reopening (${healAttemptsRemaining} attempts left)`, {
+                        dbName,
+                        storeName,
+                    });
+                    dbp = undefined;
+                    return executeTransaction(txMode, callback).then((result) => {
+                        Logger.logInfo(`IDB heal: successfully recovered after ${label} error`, {dbName, storeName});
+                        return resetHealBudget(result);
+                    });
+                }
+
+                if (isBudgetedHealError(error)) {
+                    Logger.logAlert(`IDB heal: ${getBudgetedHealErrorLabel(error)} error — heal budget exhausted, giving up`, {
+                        dbName,
+                        storeName,
+                    });
+                } else {
+                    Logger.logAlert('IDB error is not recoverable, giving up', {
+                        dbName,
+                        storeName,
+                        errorMessage: error instanceof Error ? error.message : String(error),
+                    });
+                }
+                throw error;
+            });
 }
 
 export default createStore;