feat: backport kv-store sqlite encryption (#22759) to v4-next (#22927)

## Summary

Backport of #22759 (`feat: kv-store sqlite backend with page level
encryption`) to `backport-to-v4-next-staging`.

The auto-cherry-pick failed on
`yarn-project/kv-store/src/sqlite-opfs/worker.ts`. The conflict was
localized to the `ensurePool` function: the v4-next branch already
contains #22721 (kv-store browser test hangs fix), which added a
`poolDirectory = directory;` assignment, while #22759 introduced a local
`s = sqlite3` alias and the sqlite3mc VFS registration call. Both
changes are compatible and have been combined.

## Commit structure

1. `1944712b` — original cherry-pick committed as-is with the conflict
marker preserved in `worker.ts`, so reviewers can see exactly what
conflicted.
2. `0c397e2c` — conflict resolution: keeps `poolDirectory` tracking from
#22721 + the local `s` alias and `sqlite3mc_vfs_create` call from
#22759.

No build-fixes commit was needed — `kv-store` and `sqlite3mc-wasm` both
typecheck clean against the resolved tree.

## Verification

- `yarn install` succeeded.
- `tsgo --noEmit` is clean for both `kv-store` and `sqlite3mc-wasm`.
- Diff stat matches the original PR: 26 files, +699/-14.

## Original PR

- #22759
- Merge commit: `97c6e48fe9bf269d12fa0640e4e9303f5e7cbae5`

ClaudeBox log: https://claudebox.work/s/6446d1f92380c25a?run=1

ClaudeBox log: https://claudebox.work/s/6446d1f92380c25a?run=1

Author: mverzilli

yarn-project/.prettierignore

@@ -20,3 +20,5 @@ noir-protocol-circuits-types/src/vk_tree.ts
 noir-protocol-circuits-types/src/client_artifacts_helper.ts
 constants/src/constants.gen.ts
 cli/src/config/generated
+sqlite3mc-wasm/vendor/
+

yarn-project/bootstrap.sh

@@ -119,6 +119,9 @@ function compile_all {
     return
   fi
 
+  # Ensure the pinned version sqlite3mc-wasm upstream artifacts are present before any package builds.
+  ./sqlite3mc-wasm/scripts/vendor.sh ensure
+
   compile_project ::: constants foundation stdlib blob-lib builder ethereum l1-artifacts
 
   # Call all projects that have a generation stage.

yarn-project/kv-store/package.json

@@ -31,8 +31,8 @@
     "@aztec/ethereum": "workspace:^",
     "@aztec/foundation": "workspace:^",
     "@aztec/native": "workspace:^",
+    "@aztec/sqlite3mc-wasm": "workspace:^",
     "@aztec/stdlib": "workspace:^",
-    "@sqlite.org/sqlite-wasm": "3.50.4-build1",
     "idb": "^8.0.0",
     "lmdb": "^3.2.0",
     "msgpackr": "^1.11.2",

yarn-project/kv-store/src/bench/sqlite-opfs-encrypted/map_bench.test.ts

@@ -0,0 +1,36 @@
+/**
+ * Map benchmark suite for the SQLite-OPFS backend with page-level encryption
+ * enabled (sqlite3mc ChaCha20). Runs the exact same workload as
+ * `bench/sqlite-opfs/map_bench.test.ts` — the delta between the two runs is
+ * the full cost of page-level encryption on reads and writes.
+ *
+ * Uses persistent OPFS-backed stores (sqlite3mc does not support encryption
+ * on :memory: databases). Each run creates its own unique SAH Pool directory.
+ *
+ * Skipped by default; set VITE_BENCH=1 (and VITE_SQLITE_OPFS=1) to run.
+ */
+import { createLogger } from '@aztec/foundation/log';
+
+import { mockLogger } from '../../interfaces/utils.js';
+import { AztecSQLiteOPFSStore } from '../../sqlite-opfs/store.js';
+import { describeAztecMapBench } from '../shared_map_bench.js';
+
+const shouldRun = (import.meta as ImportMeta & { env?: { VITE_BENCH?: string } }).env?.VITE_BENCH === '1';
+
+if (shouldRun) {
+  describeAztecMapBench(
+    'SQLite-OPFS (chacha20)',
+    () => {
+      const key = globalThis.crypto.getRandomValues(new Uint8Array(32));
+      const name = `bench-enc-${Date.now()}`;
+      const dir = `/bench-enc-pool-${Date.now()}`;
+      return AztecSQLiteOPFSStore.open(mockLogger, name, false, dir, key);
+    },
+    createLogger('kv-store:map:benchmarks:sqlite-opfs-chacha20'),
+    () => {},
+  );
+} else {
+  describe.skip('SQLite-OPFS (chacha20) Map benchmarks (set VITE_BENCH=1 to run)', () => {
+    it('skipped', () => {});
+  });
+}

yarn-project/kv-store/src/sqlite-opfs/encrypted_store.test.ts

@@ -0,0 +1,89 @@
+/**
+ * Tests for sqlite3mc-backed page-level encryption.
+ * Exercises the `encryptionKey` parameter added to `AztecSQLiteOPFSStore.open()`.
+ * Mixes ephemeral and persistent stores so we cover both `:memory:` and OPFS paths.
+ */
+import { describe, expect, it } from 'vitest';
+
+import { mockLogger } from '../interfaces/utils.js';
+import { AztecSQLiteOPFSStore } from './store.js';
+
+function randomKey(): Uint8Array {
+  return globalThis.crypto.getRandomValues(new Uint8Array(32));
+}
+
+/**
+ * Clone a key into a fresh Uint8Array. `AztecSQLiteOPFSStore.open()` transfers
+ * the key's ArrayBuffer to the worker (detaching it on the caller side), so
+ * callers that open multiple stores or re-use a key must pass a copy per call.
+ */
+function cloneKey(k: Uint8Array): Uint8Array {
+  return new Uint8Array(k);
+}
+
+describe('AztecSQLiteOPFSStore with encryption', () => {
+  it('rejects keys that are not 32 bytes', async () => {
+    const short = new Uint8Array(16);
+    await expect(AztecSQLiteOPFSStore.open(mockLogger, undefined, true, undefined, short)).rejects.toThrow(
+      /encryptionKey must be 32 bytes/,
+    );
+  });
+
+  it('rejects encryption on ephemeral (:memory:) stores', async () => {
+    const key = randomKey();
+    await expect(AztecSQLiteOPFSStore.open(mockLogger, undefined, true, undefined, key)).rejects.toThrow(
+      /not supported for ephemeral/,
+    );
+  });
+
+  it('round-trips values with the same key (persistent)', async () => {
+    const key = randomKey();
+    const name = `test-enc-${Date.now()}`;
+    const dir = `/test-enc-pool-${Date.now()}`;
+    const a = await AztecSQLiteOPFSStore.open(mockLogger, name, false, dir, cloneKey(key));
+    const mapA = a.openMap<string, string>('m');
+    await mapA.set('k1', 'v1');
+    await a.close();
+
+    const b = await AztecSQLiteOPFSStore.open(mockLogger, name, false, dir, cloneKey(key));
+    const mapB = b.openMap<string, string>('m');
+    expect(await mapB.getAsync('k1')).toBe('v1');
+    await b.delete();
+  });
+
+  it('fails to open with the wrong key', async () => {
+    const key = randomKey();
+    const name = `test-wrong-${Date.now()}`;
+    const dir = `/test-wrong-pool-${Date.now()}`;
+    const a = await AztecSQLiteOPFSStore.open(mockLogger, name, false, dir, cloneKey(key));
+    await a.openMap<string, string>('m').set('k1', 'v1');
+    await a.close();
+
+    await expect(async () => {
+      const b = await AztecSQLiteOPFSStore.open(mockLogger, name, false, dir, randomKey());
+      await b.openMap<string, string>('m').getAsync('k1');
+      await b.close();
+    }).rejects.toThrow();
+
+    const c = await AztecSQLiteOPFSStore.open(mockLogger, name, false, dir, cloneKey(key));
+    await c.delete();
+  });
+
+  it('fails to open an encrypted DB without a key', async () => {
+    const key = randomKey();
+    const name = `test-nokey-${Date.now()}`;
+    const dir = `/test-nokey-pool-${Date.now()}`;
+    const a = await AztecSQLiteOPFSStore.open(mockLogger, name, false, dir, cloneKey(key));
+    await a.openMap<string, string>('m').set('k1', 'v1');
+    await a.close();
+
+    await expect(async () => {
+      const b = await AztecSQLiteOPFSStore.open(mockLogger, name, false, dir);
+      await b.openMap<string, string>('m').getAsync('k1');
+      await b.close();
+    }).rejects.toThrow();
+
+    const c = await AztecSQLiteOPFSStore.open(mockLogger, name, false, dir, cloneKey(key));
+    await c.delete();
+  });
+});

yarn-project/kv-store/src/sqlite-opfs/encrypted_store_smoke.test.ts

@@ -0,0 +1,78 @@
+/**
+ * Smoke test: runs the same basic read/write dance against every kv-store interface
+ * using an encrypted store, to verify the interfaces are oblivious to the cipher
+ * layer. Each test uses its own persistent OPFS directory because sqlite3mc does
+ * not support encryption on ephemeral (:memory:) stores.
+ */
+import { afterEach, describe, expect, it } from 'vitest';
+
+import { mockLogger } from '../interfaces/utils.js';
+import { AztecSQLiteOPFSStore } from './store.js';
+
+function randomKey(): Uint8Array {
+  return globalThis.crypto.getRandomValues(new Uint8Array(32));
+}
+
+describe('encrypted store smoke: interfaces', () => {
+  let openedStore: AztecSQLiteOPFSStore | undefined;
+
+  afterEach(async () => {
+    if (openedStore) {
+      await openedStore.delete();
+      openedStore = undefined;
+    }
+  });
+
+  async function openStore(label: string): Promise<AztecSQLiteOPFSStore> {
+    const key = randomKey();
+    const name = `smoke-${label}-${Date.now()}`;
+    const dir = `/smoke-${label}-pool-${Date.now()}`;
+    const s = await AztecSQLiteOPFSStore.open(mockLogger, name, false, dir, key);
+    openedStore = s;
+    return s;
+  }
+
+  it('AztecMap', async () => {
+    const s = await openStore('map');
+    const m = s.openMap<string, number>('m');
+    await m.set('a', 1);
+    await m.set('b', 2);
+    expect(await m.getAsync('a')).toBe(1);
+    expect(await m.getAsync('b')).toBe(2);
+  });
+
+  it('AztecSet', async () => {
+    const s = await openStore('set');
+    const set = s.openSet<string>('s');
+    await set.add('a');
+    await set.add('b');
+    expect(await set.hasAsync('a')).toBe(true);
+    expect(await set.hasAsync('z')).toBe(false);
+  });
+
+  it('AztecSingleton', async () => {
+    const s = await openStore('singleton');
+    const sing = s.openSingleton<{ n: number }>('sg');
+    await sing.set({ n: 42 });
+    expect(await sing.getAsync()).toEqual({ n: 42 });
+  });
+
+  it('AztecArray', async () => {
+    const s = await openStore('array');
+    const arr = s.openArray<string>('a');
+    await arr.push('x', 'y');
+    expect(await arr.lengthAsync()).toBe(2);
+  });
+
+  it('AztecMultiMap', async () => {
+    const s = await openStore('multimap');
+    const mm = s.openMultiMap<string, number>('mm');
+    await mm.set('k', 1);
+    await mm.set('k', 2);
+    const vals: number[] = [];
+    for await (const v of mm.getValuesAsync('k')) {
+      vals.push(v);
+    }
+    expect(vals.sort()).toEqual([1, 2]);
+  });
+});

yarn-project/kv-store/src/sqlite-opfs/index.ts

@@ -25,3 +25,13 @@ export async function createStore(
 export function openTmpStore(ephemeral: boolean = false): Promise<AztecSQLiteOPFSStore> {
   return AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), undefined, ephemeral);
 }
+
+/**
+ * Convenience helper for tests and consumers that want an encrypted sqlite-opfs
+ * store without dealing with the full `open()` parameter order. Key must be 32
+ * bytes. Creates a fresh persistent store (sqlite3mc does not support encryption
+ * on ephemeral `:memory:` databases) in an auto-generated OPFS directory.
+ */
+export function openEncryptedStore(encryptionKey: Uint8Array, name?: string, poolDirectory?: string) {
+  return AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), name, false, poolDirectory, encryptionKey);
+}

yarn-project/kv-store/src/sqlite-opfs/messages.ts

@@ -11,7 +11,7 @@ export type SqlValue = string | number | bigint | null | Uint8Array;
 export type ResultRow = SqlValue[];
 
 export type WorkerRequest =
-  | { type: 'init'; id: number; dbName: string; ephemeral: boolean; poolDirectory?: string }
+  | { type: 'init'; id: number; dbName: string; ephemeral: boolean; poolDirectory?: string; encryptionKey?: Uint8Array }
   | { type: 'close'; id: number }
   | { type: 'deleteDb'; id: number; dbName: string }
   | { type: 'run'; id: number; sql: string; bind?: SqlValue[] }

yarn-project/kv-store/src/sqlite-opfs/store.ts

@@ -68,18 +68,48 @@ export class AztecSQLiteOPFSStore implements AztecAsyncKVStore {
    * Pass `poolDirectory` to place the SAH Pool in a non-default OPFS subdirectory —
    * required when multiple stores coexist in the same tab, because the SAH Pool holds
    * an exclusive lock on its directory.
+   *
+   * Pass `encryptionKey` (exactly 32 bytes) to enable at-rest encryption via sqlite3mc's
+   * ChaCha20 page cipher. The key buffer is **transferred** to the worker — its
+   * ArrayBuffer detaches on the caller side after `postMessage`. This is intentional:
+   * the API encodes a one-key-one-owner invariant. A caller that wants to use the same
+   * key for multiple stores must explicitly clone it per call (e.g.
+   * `new Uint8Array(savedKey)`), making the duplication a visible, deliberate decision
+   * rather than a silent structured-clone operation. The default path (one `.open()`,
+   * one consumption of the key) leaves zero key bytes on the main thread after the call.
    */
   static async open(
     log: Logger,
     name?: string,
     ephemeral: boolean = false,
     poolDirectory?: string,
+    encryptionKey?: Uint8Array,
   ): Promise<AztecSQLiteOPFSStore> {
+    if (encryptionKey !== undefined && encryptionKey.length !== 32) {
+      throw new Error(`encryptionKey must be 32 bytes (got ${encryptionKey.length})`);
+    }
+    if (encryptionKey !== undefined && ephemeral) {
+      throw new Error('encryptionKey is not supported for ephemeral (:memory:) stores');
+    }
     const dbName = name && !ephemeral ? name : `tmp-${globalThis.crypto.getRandomValues(new Uint8Array(8)).join('')}`;
-    log.debug(`Opening SQLite-OPFS ${ephemeral ? 'ephemeral ' : ''}database ${dbName}`);
+    log.debug(
+      `Opening SQLite-OPFS ${ephemeral ? 'ephemeral ' : ''}${encryptionKey ? 'encrypted ' : ''}database ${dbName}`,
+    );
     const worker = new Worker(new URL('./worker.js', import.meta.url), { type: 'module' });
     const store = new AztecSQLiteOPFSStore(worker, dbName, log, ephemeral);
-    await store.#sendRequest({ type: 'init', id: store.#allocId(), dbName, ephemeral, poolDirectory });
+    // Transfer (not clone) the key buffer to the worker so we don't leave a
+    // second copy on the main thread. Caveat: this detaches the caller's
+    // encryptionKey.buffer — subsequent reads from the same Uint8Array are empty.
+    const transfer = encryptionKey ? [encryptionKey.buffer as ArrayBuffer] : undefined;
+    try {
+      await store.#sendRequest(
+        { type: 'init', id: store.#allocId(), dbName, ephemeral, poolDirectory, encryptionKey },
+        transfer,
+      );
+    } catch (err) {
+      worker.terminate();
+      throw err;
+    }
     return store;
   }
 
@@ -230,7 +260,7 @@ export class AztecSQLiteOPFSStore implements AztecAsyncKVStore {
     this.#pending.clear();
   }
 
-  #sendRequest(req: WorkerRequest): Promise<WorkerResponse> {
+  #sendRequest(req: WorkerRequest, transfer?: Transferable[]): Promise<WorkerResponse> {
     return new Promise<WorkerResponse>((resolve, reject) => {
       this.#pending.set(req.id, {
         resolve: resp => {
@@ -242,7 +272,11 @@ export class AztecSQLiteOPFSStore implements AztecAsyncKVStore {
         },
         reject,
       });
-      this.#worker.postMessage(req);
+      if (transfer && transfer.length > 0) {
+        this.#worker.postMessage(req, transfer);
+      } else {
+        this.#worker.postMessage(req);
+      }
     });
   }
 }