feat: weblock controlled opfs pool (fwd-port #24740)

Author: mverzilli

yarn-project/kv-store/src/sqlite-opfs/errors.ts

@@ -42,3 +42,56 @@ const SQLITE3MC_DECRYPT_ERROR_PATTERNS: readonly RegExp[] = [
 export function isDecryptFailureMessage(message: string): boolean {
   return SQLITE3MC_DECRYPT_ERROR_PATTERNS.some(p => p.test(message));
 }
+<<<<<<< HEAD
+=======
+
+/**
+ * Error thrown by sqlite-opfs when the on-disk database image is corrupt
+ * (`SQLITE_CORRUPT`, result code 11 — "database disk image is malformed").
+ *
+ * Distinct from {@link SqliteEncryptionError}: no key recovers a corrupt image,
+ * so there is nothing to retry. The only escape is to delete the store and start
+ * fresh, so consumers pattern-match on `instanceof SqliteCorruptionError` to wipe
+ * and reopen rather than surfacing a dead-end error.
+ **/
+export class SqliteCorruptionError extends Error {
+  constructor(message: string, opts?: { cause?: unknown }) {
+    super(message, opts?.cause !== undefined ? { cause: opts.cause } : undefined);
+    this.name = 'SqliteCorruptionError';
+  }
+}
+
+/** Error thrown when another browser context already owns a store's OPFS pool. */
+export class SqlitePoolBusyError extends Error {
+  constructor(public readonly poolDirectory: string) {
+    super(`SQLite-OPFS pool "${poolDirectory}" is already in use by another store instance`);
+    this.name = 'SqlitePoolBusyError';
+  }
+}
+
+/** Error thrown when the browser context does not expose the Web Locks API required by the OPFS SAH pool. */
+export class SqliteWebLocksUnavailableError extends Error {
+  constructor() {
+    super('SQLite-OPFS requires the Web Locks API, but it is unavailable in this browser context');
+    this.name = 'SqliteWebLocksUnavailableError';
+  }
+}
+
+/**
+ * Strings/codes raised by SQLite when a database image is corrupt. Kept disjoint
+ * from {@link SQLITE3MC_DECRYPT_ERROR_PATTERNS}: "file is not a database"
+ * (SQLITE_NOTADB) is the decrypt signal, whereas a malformed image is the
+ * genuinely-unrecoverable SQLITE_CORRUPT.
+ **/
+const SQLITE_CORRUPTION_ERROR_PATTERNS: readonly RegExp[] = [
+  /database disk image is malformed/i,
+  /\bSQLITE_CORRUPT\b/i,
+];
+
+/**
+ * Returns `true` if `message` matches one of the known SQLite corruption strings.
+ **/
+export function isCorruptionMessage(message: string): boolean {
+  return SQLITE_CORRUPTION_ERROR_PATTERNS.some(p => p.test(message));
+}
+>>>>>>> 386f120fb2 (feat: weblock controlled opfs pool (#24740))

yarn-project/kv-store/src/sqlite-opfs/index.ts

@@ -5,6 +5,7 @@ import { initStoreForRollupAndSchemaVersion } from '../utils.js';
 import { AztecSQLiteOPFSStore } from './store.js';
 
 export { AztecSQLiteOPFSStore } from './store.js';
+<<<<<<< HEAD
 export { SqliteEncryptionError } from './errors.js';
 export type { SqliteEncryptionErrorCode } from './errors.js';
 
@@ -23,6 +24,16 @@ export async function createStore(
   const store = await AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), name, false);
   return initStoreForRollupAndSchemaVersion(store, schemaVersion, config.rollupAddress, log);
 }
+=======
+export {
+  SqliteCorruptionError,
+  SqliteEncryptionError,
+  SqlitePoolBusyError,
+  SqliteWebLocksUnavailableError,
+} from './errors.js';
+export type { SqliteEncryptionErrorCode } from './errors.js';
+export { OPFS_POOL_DIR_PREFIX, deletePoolDirectory, deleteStore, listStores, storePoolDirectory } from './manage.js';
+>>>>>>> 386f120fb2 (feat: weblock controlled opfs pool (#24740))
 
 export function openTmpStore(ephemeral: boolean = false): Promise<AztecSQLiteOPFSStore> {
   return AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), undefined, ephemeral);

yarn-project/kv-store/src/sqlite-opfs/manage.ts

@@ -0,0 +1,46 @@
+import { normalizePoolDirectory, withPoolLock } from './pool_lock.js';
+
+/** Prefix for the per-store OPFS SAH pool directories owned by this package. */
+export const OPFS_POOL_DIR_PREFIX = '.aztec-kv-';
+
+/**
+ * OPFS directory holding a store's SAH pool. One directory per store: the SAH-pool VFS allows only one
+ * concurrent instance per directory and its capacity does not grow automatically, so sharing a pool across
+ * stores would make concurrently opened stores contend for locks and orphaned stores exhaust pool slots.
+ */
+export function storePoolDirectory(effectiveName: string): string {
+  return `${OPFS_POOL_DIR_PREFIX}${effectiveName}`;
+}
+
+/**
+ * Lists the names of every persistent sqlite-opfs store in this origin, by enumerating the per-store pool
+ * directories. Includes stores other than the current one, so wallets can surface and clean up data for
+ * networks/versions no longer in use.
+ */
+export async function listStores(): Promise<string[]> {
+  const root = await navigator.storage.getDirectory();
+  const names: string[] = [];
+  for await (const [entryName, handle] of root.entries()) {
+    if (handle.kind === 'directory' && entryName.startsWith(OPFS_POOL_DIR_PREFIX)) {
+      names.push(entryName.slice(OPFS_POOL_DIR_PREFIX.length));
+    }
+  }
+  return names;
+}
+
+/**
+ * Permanently deletes a store by effective name (as returned by {@link listStores}). The store must be closed:
+ * an open store holds the directory's Web Lock and the removal will reject with `SqlitePoolBusyError`.
+ */
+export async function deleteStore(effectiveName: string): Promise<void> {
+  await deletePoolDirectory(storePoolDirectory(effectiveName));
+}
+
+/** Permanently deletes one OPFS SAH-pool directory after exclusively locking it. */
+export async function deletePoolDirectory(poolDirectory: string): Promise<void> {
+  poolDirectory = normalizePoolDirectory(poolDirectory);
+  await withPoolLock(poolDirectory, async () => {
+    const root = await navigator.storage.getDirectory();
+    await root.removeEntry(poolDirectory, { recursive: true });
+  });
+}

yarn-project/kv-store/src/sqlite-opfs/pool_lock.ts

@@ -0,0 +1,62 @@
+import { SqlitePoolBusyError, SqliteWebLocksUnavailableError } from './errors.js';
+
+export const DEFAULT_SAH_POOL_DIRECTORY = '.aztec-kv';
+
+const WEB_LOCK_PREFIX = 'aztec.sqlite-opfs.pool:';
+
+export interface PoolLockLease {
+  release(): Promise<void>;
+}
+
+export function normalizePoolDirectory(poolDirectory?: string): string {
+  const path = poolDirectory
+    ?.split('/')
+    .filter(segment => segment.length > 0)
+    .join('/');
+  return path || DEFAULT_SAH_POOL_DIRECTORY;
+}
+
+export async function acquirePoolLock(poolDirectory?: string): Promise<PoolLockLease> {
+  if (!navigator.locks) {
+    throw new SqliteWebLocksUnavailableError();
+  }
+  const normalizedDirectory = normalizePoolDirectory(poolDirectory);
+  const acquired = Promise.withResolvers<Lock | null>();
+  const hold = Promise.withResolvers<void>();
+  const request = navigator.locks.request(
+    `${WEB_LOCK_PREFIX}${normalizedDirectory}`,
+    { mode: 'exclusive', ifAvailable: true },
+    async lock => {
+      acquired.resolve(lock);
+      if (lock) {
+        await hold.promise;
+      }
+    },
+  );
+  request.catch(acquired.reject);
+
+  if (!(await acquired.promise)) {
+    await request;
+    throw new SqlitePoolBusyError(normalizedDirectory);
+  }
+
+  let released = false;
+  return {
+    release: async () => {
+      if (!released) {
+        released = true;
+        hold.resolve();
+      }
+      await request;
+    },
+  };
+}
+
+export async function withPoolLock<T>(poolDirectory: string, callback: () => Promise<T>): Promise<T> {
+  const lease = await acquirePoolLock(poolDirectory);
+  try {
+    return await callback();
+  } finally {
+    await lease.release();
+  }
+}

yarn-project/kv-store/src/sqlite-opfs/store.ts

@@ -14,6 +14,7 @@ import { SqliteEncryptionError } from './errors.js';
 import { SQLiteOPFSAztecMap } from './map.js';
 import type { ResultRow, SqlValue, WorkerRequest, WorkerResponse } from './messages.js';
 import { SQLiteOPFSAztecMultiMap } from './multi_map.js';
+import { type PoolLockLease, acquirePoolLock, normalizePoolDirectory } from './pool_lock.js';
 import { SQLiteOPFSAztecSet } from './set.js';
 import { SQLiteOPFSAztecSingleton } from './singleton.js';
 
@@ -36,12 +37,14 @@ export class AztecSQLiteOPFSStore implements AztecAsyncKVStore {
   #nextId = 0;
   #inTx = false;
   #closed = false;
+  #workerFailed = false;
 
   private constructor(
     worker: Worker,
     name: string,
     log: Logger,
     public readonly isEphemeral: boolean,
+    private readonly poolLock?: PoolLockLease,
   ) {
     this.#worker = worker;
     this.#name = name;
@@ -57,6 +60,7 @@ export class AztecSQLiteOPFSStore implements AztecAsyncKVStore {
       handler.resolve(ev.data);
     };
     this.#worker.onerror = ev => {
+      this.#workerFailed = true;
       this.#log.error(`SQLite worker crashed: ${ev.message}`);
       this.#rejectPending(`SQLite worker crashed: ${ev.message}`);
     };
@@ -70,6 +74,10 @@ export class AztecSQLiteOPFSStore implements AztecAsyncKVStore {
    * required when multiple stores coexist in the same tab, because the SAH Pool holds
    * an exclusive lock on its directory.
    *
+   * Persistent stores hold an origin-wide Web Lock for the pool directory until close
+   * or delete. If another store instance already owns it, open fails immediately with
+   * `SqlitePoolBusyError`.
+   *
    * Pass `encryptionKey` (exactly 32 bytes) to enable at-rest encryption via sqlite3mc's
    * ChaCha20 page cipher. The key buffer is **transferred** to the worker — its
    * ArrayBuffer detaches on the caller side after `postMessage`. This is intentional:
@@ -102,22 +110,26 @@ export class AztecSQLiteOPFSStore implements AztecAsyncKVStore {
     log.debug(
       `Opening SQLite-OPFS ${ephemeral ? 'ephemeral ' : ''}${encryptionKey ? 'encrypted ' : ''}database ${dbName}`,
     );
-    const worker = new Worker(new URL('./worker.js', import.meta.url), { type: 'module' });
-    const store = new AztecSQLiteOPFSStore(worker, dbName, log, ephemeral);
-    // Transfer (not clone) the key buffer to the worker so we don't leave a
-    // second copy on the main thread. Caveat: this detaches the caller's
-    // encryptionKey.buffer — subsequent reads from the same Uint8Array are empty.
-    const transfer = encryptionKey ? [encryptionKey.buffer as ArrayBuffer] : undefined;
+    const effectivePoolDirectory = ephemeral ? undefined : normalizePoolDirectory(poolDirectory);
+    const poolLock = effectivePoolDirectory ? await acquirePoolLock(effectivePoolDirectory) : undefined;
+    let worker: Worker | undefined;
     try {
+      worker = new Worker(new URL('./worker.js', import.meta.url), { type: 'module' });
+      const store = new AztecSQLiteOPFSStore(worker, dbName, log, ephemeral, poolLock);
+      // Transfer (not clone) the key buffer to the worker so we don't leave a
+      // second copy on the main thread. Caveat: this detaches the caller's
+      // encryptionKey.buffer — subsequent reads from the same Uint8Array are empty.
+      const transfer = encryptionKey ? [encryptionKey.buffer as ArrayBuffer] : undefined;
       await store.#sendRequest(
-        { type: 'init', id: store.#allocId(), dbName, ephemeral, poolDirectory, encryptionKey },
+        { type: 'init', id: store.#allocId(), dbName, ephemeral, poolDirectory: effectivePoolDirectory, encryptionKey },
         transfer,
       );
+      return store;
     } catch (err) {
-      worker.terminate();
+      worker?.terminate();
+      await poolLock?.release();
       throw err;
     }
-    return store;
   }
 
   openMap<K extends Key, V extends Value>(name: string): AztecAsyncMap<K, V> {
@@ -179,12 +191,16 @@ export class AztecSQLiteOPFSStore implements AztecAsyncKVStore {
       return;
     }
     this.#closed = true;
-    await this.#txQueue.end();
-    await this.#sendRequest({ type: 'deleteDb', id: this.#allocId(), dbName: this.#name }).catch(err =>
-      this.#log.warn(`SQLite deleteDb failed: ${err instanceof Error ? err.message : err}`),
-    );
-    this.#worker.terminate();
-    this.#rejectPending('SQLite store deleted');
+    try {
+      await this.#txQueue.end();
+      await this.#sendRequest({ type: 'deleteDb', id: this.#allocId(), dbName: this.#name }).catch(err =>
+        this.#log.warn(`SQLite deleteDb failed: ${err instanceof Error ? err.message : err}`),
+      );
+    } finally {
+      this.#worker.terminate();
+      this.#rejectPending('SQLite store deleted');
+      await this.poolLock?.release();
+    }
   }
 
   /**
@@ -204,10 +220,14 @@ export class AztecSQLiteOPFSStore implements AztecAsyncKVStore {
       return;
     }
     this.#closed = true;
-    await this.#txQueue.end();
-    await this.#sendRequest({ type: 'close', id: this.#allocId() }).catch(() => {});
-    this.#worker.terminate();
-    this.#rejectPending('SQLite store closed');
+    try {
+      await this.#txQueue.end();
+      await this.#sendRequest({ type: 'close', id: this.#allocId() }).catch(() => {});
+    } finally {
+      this.#worker.terminate();
+      this.#rejectPending('SQLite store closed');
+      await this.poolLock?.release();
+    }
   }
 
   backupTo(_dstPath: string, _compact?: boolean): Promise<void> {
@@ -268,6 +288,9 @@ export class AztecSQLiteOPFSStore implements AztecAsyncKVStore {
   }
 
   #sendRequest(req: WorkerRequest, transfer?: Transferable[]): Promise<WorkerResponse> {
+    if (this.#workerFailed) {
+      return Promise.reject(new Error('SQLite worker has crashed'));
+    }
     return new Promise<WorkerResponse>((resolve, reject) => {
       this.#pending.set(req.id, {
         resolve: resp => {