chore: forward port sqlite changes (#24947)

Forward port the following PRs from v5-next to next:

- #24631
- #24647
- #24739
- #24740
 - #24743

---------

Co-authored-by: Gregorio Juliana <gregojquiros@gmail.com>

Author: mverzilli

playground/src/components/navbar/components/NetworkSelector.tsx

@@ -7,7 +7,7 @@ import AddIcon from '@mui/icons-material/Add';
 import { AddNetworksDialog } from './AddNetworkDialog';
 import CircularProgress from '@mui/material/CircularProgress';
 import KeyboardArrowDownIcon from '@mui/icons-material/KeyboardArrowDown';
-import { createStore } from '@aztec/kv-store/sqlite-opfs';
+import { AztecSQLiteOPFSStore, storePoolDirectory } from '@aztec/kv-store/sqlite-opfs';
 import { AztecContext } from '../../../aztecContext';
 import { navbarButtonStyle, navbarSelect } from '../../../styles/common';
 import { NETWORKS } from '../../../utils/networks';
@@ -50,10 +50,12 @@ export function NetworkSelector() {
       }
       setIsContextInitialized(true);
       WebLogger.create(setLogs, setTotalLogCount);
-      const store = await createStore('playground_data', {
-        dataDirectory: 'playground',
-        dataStoreMapSizeKb: 1e6,
-      });
+      const store = await AztecSQLiteOPFSStore.open(
+        WebLogger.getInstance().createLogger('playground_data'),
+        'playground_data',
+        false,
+        storePoolDirectory('playground_data'),
+      );
       const playgroundDB = PlaygroundDB.getInstance();
       playgroundDB.init(store, WebLogger.getInstance().createLogger('playground_db').info);
       setPlaygroundDB(PlaygroundDB.getInstance());

yarn-project/kv-store/scripts/run-browser-tests.sh

@@ -15,7 +15,9 @@ set -euo pipefail
 
 cd "$(dirname "$0")/.."
 
-files=$(find src/deprecated/indexeddb src/sqlite-opfs -name '*.test.ts' 2>/dev/null | sort)
+# -type f: vitest failure screenshots land in __screenshots__/<test-file>/, creating
+# directories whose names match the *.test.ts glob.
+files=$(find src/deprecated/indexeddb src/sqlite-opfs -type f -name '*.test.ts' 2>/dev/null | sort)
 
 if [ -z "$files" ]; then
   echo "No test files found in src/deprecated/indexeddb or src/sqlite-opfs"

yarn-project/kv-store/src/deprecated/indexeddb/index.ts

@@ -1,34 +1,9 @@
-import { type Logger, createLogger } from '@aztec/foundation/log';
-import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
+import { createLogger } from '@aztec/foundation/log';
 
-import { initStoreForRollupAndSchemaVersion } from '../../utils.js';
 import { AztecIndexedDBStore } from './store.js';
 
 export { AztecIndexedDBStore } from './store.js';
 
-/**
- * @deprecated The IndexedDB backend is being retired. Use `@aztec/kv-store/sqlite-opfs` instead.
- */
-export async function createStore(
-  name: string,
-  config: DataStoreConfig,
-  schemaVersion: number | undefined = undefined,
-  log: Logger = createLogger('kv-store'),
-) {
-  let { dataDirectory } = config;
-  if (typeof dataDirectory !== 'undefined') {
-    dataDirectory = `${dataDirectory}/${name}`;
-  }
-
-  log.info(
-    dataDirectory
-      ? `Creating ${name} data store at directory ${dataDirectory} with map size ${config.dataStoreMapSizeKb} KB`
-      : `Creating ${name} ephemeral data store with map size ${config.dataStoreMapSizeKb} KB`,
-  );
-  const store = await AztecIndexedDBStore.open(createLogger('kv-store:indexeddb'), dataDirectory ?? '', false);
-  return initStoreForRollupAndSchemaVersion(store, schemaVersion, config.rollupAddress, log);
-}
-
 /**
  * @deprecated The IndexedDB backend is being retired. Use `@aztec/kv-store/sqlite-opfs` instead.
  */

yarn-project/kv-store/src/sqlite-opfs/errors.test.ts

@@ -0,0 +1,30 @@
+import { describe, expect, it } from 'vitest';
+
+import { isCorruptionMessage, isDecryptFailureMessage } from './errors.js';
+
+/**
+ * The two classifiers must stay disjoint: a decrypt failure (wrong/missing key)
+ * is recoverable by re-keying, whereas a corrupt image is not. Mixing them would
+ * make the worker mis-tag failures and consumers take the wrong recovery path.
+ */
+describe('sqlite-opfs error classification', () => {
+  it('classifies SQLITE_CORRUPT / malformed-image messages as corruption', () => {
+    expect(isCorruptionMessage('SQLITE_CORRUPT: sqlite3 result code 11: database disk image is malformed')).toBe(true);
+    expect(isCorruptionMessage('database disk image is malformed')).toBe(true);
+  });
+
+  it('does not classify decrypt-failure messages as corruption', () => {
+    expect(isCorruptionMessage('file is not a database')).toBe(false);
+    expect(isCorruptionMessage('file is encrypted or is not a database')).toBe(false);
+  });
+
+  it('classifies sqlite3mc decrypt-failure messages as decrypt failures', () => {
+    expect(isDecryptFailureMessage('file is not a database')).toBe(true);
+    expect(isDecryptFailureMessage('file is encrypted or is not a database')).toBe(true);
+  });
+
+  it('does not classify corruption messages as decrypt failures', () => {
+    expect(isDecryptFailureMessage('database disk image is malformed')).toBe(false);
+    expect(isDecryptFailureMessage('SQLITE_CORRUPT: sqlite3 result code 11')).toBe(false);
+  });
+});

yarn-project/kv-store/src/sqlite-opfs/errors.ts

@@ -42,3 +42,53 @@ const SQLITE3MC_DECRYPT_ERROR_PATTERNS: readonly RegExp[] = [
 export function isDecryptFailureMessage(message: string): boolean {
   return SQLITE3MC_DECRYPT_ERROR_PATTERNS.some(p => p.test(message));
 }
+
+/**
+ * Error thrown by sqlite-opfs when the on-disk database image is corrupt
+ * (`SQLITE_CORRUPT`, result code 11 — "database disk image is malformed").
+ *
+ * Distinct from {@link SqliteEncryptionError}: no key recovers a corrupt image,
+ * so there is nothing to retry. The only escape is to delete the store and start
+ * fresh, so consumers pattern-match on `instanceof SqliteCorruptionError` to wipe
+ * and reopen rather than surfacing a dead-end error.
+ **/
+export class SqliteCorruptionError extends Error {
+  constructor(message: string, opts?: { cause?: unknown }) {
+    super(message, opts?.cause !== undefined ? { cause: opts.cause } : undefined);
+    this.name = 'SqliteCorruptionError';
+  }
+}
+
+/** Error thrown when another browser context already owns a store's OPFS pool. */
+export class SqlitePoolBusyError extends Error {
+  constructor(public readonly poolDirectory: string) {
+    super(`SQLite-OPFS pool "${poolDirectory}" is already in use by another store instance`);
+    this.name = 'SqlitePoolBusyError';
+  }
+}
+
+/** Error thrown when the browser context does not expose the Web Locks API required by the OPFS SAH pool. */
+export class SqliteWebLocksUnavailableError extends Error {
+  constructor() {
+    super('SQLite-OPFS requires the Web Locks API, but it is unavailable in this browser context');
+    this.name = 'SqliteWebLocksUnavailableError';
+  }
+}
+
+/**
+ * Strings/codes raised by SQLite when a database image is corrupt. Kept disjoint
+ * from {@link SQLITE3MC_DECRYPT_ERROR_PATTERNS}: "file is not a database"
+ * (SQLITE_NOTADB) is the decrypt signal, whereas a malformed image is the
+ * genuinely-unrecoverable SQLITE_CORRUPT.
+ **/
+const SQLITE_CORRUPTION_ERROR_PATTERNS: readonly RegExp[] = [
+  /database disk image is malformed/i,
+  /\bSQLITE_CORRUPT\b/i,
+];
+
+/**
+ * Returns `true` if `message` matches one of the known SQLite corruption strings.
+ **/
+export function isCorruptionMessage(message: string): boolean {
+  return SQLITE_CORRUPTION_ERROR_PATTERNS.some(p => p.test(message));
+}

yarn-project/kv-store/src/sqlite-opfs/index.ts

@@ -1,28 +1,16 @@
-import { type Logger, createLogger } from '@aztec/foundation/log';
-import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
+import { createLogger } from '@aztec/foundation/log';
 
-import { initStoreForRollupAndSchemaVersion } from '../utils.js';
 import { AztecSQLiteOPFSStore } from './store.js';
 
 export { AztecSQLiteOPFSStore } from './store.js';
-export { SqliteEncryptionError } from './errors.js';
+export {
+  SqliteCorruptionError,
+  SqliteEncryptionError,
+  SqlitePoolBusyError,
+  SqliteWebLocksUnavailableError,
+} from './errors.js';
 export type { SqliteEncryptionErrorCode } from './errors.js';
-
-export async function createStore(
-  name: string,
-  config: DataStoreConfig,
-  schemaVersion: number | undefined = undefined,
-  log: Logger = createLogger('kv-store'),
-) {
-  const { dataDirectory } = config;
-  log.info(
-    dataDirectory
-      ? `Creating ${name} SQLite-OPFS data store with map size ${config.dataStoreMapSizeKb} KB`
-      : `Creating ${name} ephemeral SQLite-OPFS data store with map size ${config.dataStoreMapSizeKb} KB`,
-  );
-  const store = await AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), name, false);
-  return initStoreForRollupAndSchemaVersion(store, schemaVersion, config.rollupAddress, log);
-}
+export { OPFS_POOL_DIR_PREFIX, deletePoolDirectory, deleteStore, listStores, storePoolDirectory } from './manage.js';
 
 export function openTmpStore(ephemeral: boolean = false): Promise<AztecSQLiteOPFSStore> {
   return AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), undefined, ephemeral);

yarn-project/kv-store/src/sqlite-opfs/manage.ts

@@ -0,0 +1,46 @@
+import { normalizePoolDirectory, withPoolLock } from './pool_lock.js';
+
+/** Prefix for the per-store OPFS SAH pool directories owned by this package. */
+export const OPFS_POOL_DIR_PREFIX = '.aztec-kv-';
+
+/**
+ * OPFS directory holding a store's SAH pool. One directory per store: the SAH-pool VFS allows only one
+ * concurrent instance per directory and its capacity does not grow automatically, so sharing a pool across
+ * stores would make concurrently opened stores contend for locks and orphaned stores exhaust pool slots.
+ */
+export function storePoolDirectory(effectiveName: string): string {
+  return `${OPFS_POOL_DIR_PREFIX}${effectiveName}`;
+}
+
+/**
+ * Lists the names of every persistent sqlite-opfs store in this origin, by enumerating the per-store pool
+ * directories. Includes stores other than the current one, so wallets can surface and clean up data for
+ * networks/versions no longer in use.
+ */
+export async function listStores(): Promise<string[]> {
+  const root = await navigator.storage.getDirectory();
+  const names: string[] = [];
+  for await (const [entryName, handle] of root.entries()) {
+    if (handle.kind === 'directory' && entryName.startsWith(OPFS_POOL_DIR_PREFIX)) {
+      names.push(entryName.slice(OPFS_POOL_DIR_PREFIX.length));
+    }
+  }
+  return names;
+}
+
+/**
+ * Permanently deletes a store by effective name (as returned by {@link listStores}). The store must be closed:
+ * an open store holds the directory's Web Lock and the removal will reject with `SqlitePoolBusyError`.
+ */
+export async function deleteStore(effectiveName: string): Promise<void> {
+  await deletePoolDirectory(storePoolDirectory(effectiveName));
+}
+
+/** Permanently deletes one OPFS SAH-pool directory after exclusively locking it. */
+export async function deletePoolDirectory(poolDirectory: string): Promise<void> {
+  poolDirectory = normalizePoolDirectory(poolDirectory);
+  await withPoolLock(poolDirectory, async () => {
+    const root = await navigator.storage.getDirectory();
+    await root.removeEntry(poolDirectory, { recursive: true });
+  });
+}