fix: respect wasm load config params when instantiating sqlite (#24937)

The sqlite3mc we vendor clobbers config params used to find the sqlite
wasm file to load.

As a result, the bundler-visible default wasm URL was silently discarded
on the first call in a given env and wasmBinary was never honored at
all.

We introduce an index.ts wrapper that takes care of ensuring options
survive the first call and every call after. It honors `locateFile`,
`wasmBinary`, and `instantiateWasm`, and it defaults to a static
`SQLITE3_WASM_URL` that bundlers rewrite to their emitted asset.

(We need to follow this up with some simplification of aztec-kit)

Closes TRIAGE-200
Fixes #24895

Author: mverzilli

yarn-project/sqlite3mc-wasm/package.json

@@ -59,6 +59,7 @@
       ]
     },
     "moduleNameMapper": {
+      "^\\.\\./vendor/jswasm/(.*)$": "<rootDir>/../vendor/jswasm/$1",
       "^(\\.{1,2}/.*)\\.[cm]?js$": "$1"
     },
     "reporters": [

yarn-project/sqlite3mc-wasm/package.local.json

@@ -1 +1,8 @@
-{}
+{
+  "jest": {
+    "moduleNameMapper": {
+      "^\\.\\./vendor/jswasm/(.*)$": "<rootDir>/../vendor/jswasm/$1",
+      "^(\\.{1,2}/.*)\\.[cm]?js$": "$1"
+    }
+  }
+}

yarn-project/sqlite3mc-wasm/src/index.test.ts

@@ -0,0 +1,171 @@
+import type { SqlValue } from '@sqlite.org/sqlite-wasm';
+import { readFile } from 'node:fs/promises';
+
+import sqlite3InitModule, { type Sqlite3Static } from './index.js';
+
+/**
+ * These are regression tests that simulate a bundled app: `sqlite3.wasm` is NOT fetchable relative to the module's
+ * `import.meta.url` (bundlers relocate chunks, so the wasm asset never sits next to them), which is why callers must
+ * supply the wasm through the standard Emscripten options `wasmBinary` or `locateFile`. The vendored sqlite3mc build
+ * routes those options through `globalThis.sqlite3InitModuleState`, a handoff that never forwards `wasmBinary` and is
+ * deleted after the first init call, so the wrapper in index.ts we're testing here must compensate for both.
+ */
+describe('sqlite3InitModule', () => {
+  const BUNDLED_WASM_URL = 'https://bundled.example/assets/sqlite3-4f2a.wasm';
+  const realFetch = globalThis.fetch;
+  let wasmBinary: Uint8Array<ArrayBuffer>;
+  let fetchCalls: string[];
+
+  beforeAll(async () => {
+    // Copy into a fresh Uint8Array so the bytes are plain-ArrayBuffer-backed (BufferSource), not a Buffer.
+    wasmBinary = new Uint8Array(await readFile(new URL('../vendor/jswasm/sqlite3.wasm', import.meta.url)));
+  });
+
+  beforeEach(() => {
+    fetchCalls = [];
+    globalThis.fetch = ((input: RequestInfo | URL) => {
+      const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
+      fetchCalls.push(url);
+      if (url === BUNDLED_WASM_URL) {
+        return Promise.resolve(new Response(wasmBinary, { headers: { 'content-type': 'application/wasm' } }));
+      }
+      return Promise.reject(new Error(`unexpected fetch of ${url}: the wasm is not addressable by URL when bundled`));
+    }) as typeof fetch;
+  });
+
+  afterEach(() => {
+    globalThis.fetch = realFetch;
+  });
+
+  /**
+   * The wrapper's built-in load paths reject on failure, but a regression (or a caller-supplied `instantiateWasm`,
+   * whose failures Emscripten's hook contract cannot report) hangs forever, so bound the wait and fail with the
+   * observed fetch calls as evidence instead of hitting the jest timeout.
+   */
+  const settleWithin = <T>(promise: Promise<T>, ms = 20_000): Promise<T> =>
+    Promise.race([
+      promise,
+      new Promise<never>((_, reject) =>
+        setTimeout(
+          () =>
+            reject(
+              new Error(`sqlite3InitModule did not settle within ${ms}ms; fetch calls: ${JSON.stringify(fetchCalls)}`),
+            ),
+          ms,
+        ),
+      ),
+    ]);
+
+  const expectWorkingDb = (sqlite3: Sqlite3Static) => {
+    // Open a disposable in-memory SQLite
+    const db = new sqlite3.oo1.DB(':memory:');
+    try {
+      db.exec('CREATE TABLE t(a INTEGER); INSERT INTO t VALUES (40), (2);');
+      const rows: SqlValue[][] = [];
+      db.exec({ sql: 'SELECT SUM(a) FROM t', rowMode: 'array', resultRows: rows });
+      expect(rows).toEqual([[42]]);
+    } finally {
+      db.close();
+    }
+  };
+
+  it('resolves the wasm via a bundler-visible static URL on a bare first call', async () => {
+    // Must match the literal `new URL('../vendor/jswasm/sqlite3.wasm', import.meta.url)` in src/index.ts, the static
+    // form bundlers detect and rewrite. This test file sits next to index.ts, so the same expression yields the URL
+    // the wrapper must produce.
+    const expectedUrl = new URL('../vendor/jswasm/sqlite3.wasm', import.meta.url).href;
+    globalThis.fetch = ((input: RequestInfo | URL) => {
+      const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
+      fetchCalls.push(url);
+      if (url === expectedUrl) {
+        return Promise.resolve(new Response(wasmBinary, { headers: { 'content-type': 'application/wasm' } }));
+      }
+      return Promise.reject(new Error(`unexpected fetch of ${url}`));
+    }) as typeof fetch;
+
+    // In this unbundled test env the vendored fallback (`new URL(path, import.meta.url)` relative to sqlite3.mjs)
+    // resolves to the same file URL as the static default, so the fetched URL alone cannot distinguish them. Capture
+    // the state object the wrapper installs to assert the resolution came from an installed locateFile, which is the
+    // only mechanism bundlers can rewrite.
+    let installedState: Record<string, unknown> | undefined;
+    let stateSlot: unknown;
+    Object.defineProperty(globalThis, 'sqlite3InitModuleState', {
+      configurable: true,
+      get: () => stateSlot,
+      set: value => {
+        installedState = value as Record<string, unknown>;
+        stateSlot = value;
+      },
+    });
+    try {
+      const sqlite3 = await settleWithin(sqlite3InitModule());
+      expectWorkingDb(sqlite3);
+      expect(fetchCalls).toEqual([expectedUrl]);
+      const locate = installedState?.emscriptenLocateFile as ((path: string, prefix: string) => string) | undefined;
+      expect(typeof locate).toBe('function');
+      expect(locate!('sqlite3.wasm', '')).toBe(expectedUrl);
+    } finally {
+      delete (globalThis as { sqlite3InitModuleState?: unknown }).sqlite3InitModuleState;
+    }
+  });
+
+  it('honors wasmBinary on the first call instead of fetching the wasm by URL', async () => {
+    const sqlite3 = await settleWithin(sqlite3InitModule({ wasmBinary }));
+    expect(fetchCalls).toEqual([]);
+    expectWorkingDb(sqlite3);
+  });
+
+  it('honors locateFile on the first call', async () => {
+    const locateCalls: string[] = [];
+    const sqlite3 = await settleWithin(
+      sqlite3InitModule({
+        locateFile: (path: string) => {
+          locateCalls.push(path);
+          return BUNDLED_WASM_URL;
+        },
+      }),
+    );
+    expect(locateCalls).toEqual(['sqlite3.wasm']);
+    expect(fetchCalls).toEqual([BUNDLED_WASM_URL]);
+    expectWorkingDb(sqlite3);
+  });
+
+  it('rejects when the wasm cannot be fetched', async () => {
+    // Bare call: the default locateFile resolves a file:// URL, which the stub rejects like a bundled app whose asset
+    // pipeline is broken.
+    await expect(settleWithin(sqlite3InitModule(), 5_000)).rejects.toThrow(/sqlite3 wasm instantiation failed/);
+  });
+
+  it('rejects with the HTTP status when the wasm URL returns an error response', async () => {
+    globalThis.fetch = ((input: RequestInfo | URL) => {
+      const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
+      fetchCalls.push(url);
+      return Promise.resolve(new Response('not found', { status: 404, statusText: 'Not Found' }));
+    }) as typeof fetch;
+    await expect(settleWithin(sqlite3InitModule({ locateFile: () => BUNDLED_WASM_URL }), 5_000)).rejects.toThrow(
+      /HTTP 404/,
+    );
+  });
+
+  it('rejects on corrupt wasmBinary bytes', async () => {
+    await expect(settleWithin(sqlite3InitModule({ wasmBinary: new Uint8Array([0, 1, 2, 3]) }), 5_000)).rejects.toThrow(
+      /sqlite3 wasm instantiation failed \(wasmBinary\)/,
+    );
+  });
+
+  it('keeps honoring caller options on calls after the first', async () => {
+    await settleWithin(sqlite3InitModule({ wasmBinary }));
+    const locateCalls: string[] = [];
+    const sqlite3 = await settleWithin(
+      sqlite3InitModule({
+        locateFile: (path: string) => {
+          locateCalls.push(path);
+          return BUNDLED_WASM_URL;
+        },
+      }),
+    );
+    expect(locateCalls).toEqual(['sqlite3.wasm']);
+    expect(fetchCalls).toEqual([BUNDLED_WASM_URL]);
+    expectWorkingDb(sqlite3);
+  });
+});

yarn-project/sqlite3mc-wasm/src/index.ts

@@ -1,40 +1,134 @@
+import type { Sqlite3Static } from '@sqlite.org/sqlite-wasm';
+
+import vendoredInit from '../vendor/jswasm/sqlite3.mjs';
+
+export type { Database, SAHPoolUtil, Sqlite3Static } from '@sqlite.org/sqlite-wasm';
+
 /**
- * Re-exports sqlite3mc's ES module default (sqlite3InitModule) and the TypeScript types expected by downstream
- * consumers. Mirrors the `@sqlite.org/sqlite-wasm` package default export — sqlite3mc is a strict API-compatible
- * superset, so upstream types apply unchanged.
+ * Bundler-visible static reference to the wasm binary. Because the URL argument is a string literal, bundlers detect
+ * the expression, emit the wasm as an asset, and rewrite the URL, so the default `locateFile` below resolves to the
+ * emitted asset instead of guessing a path relative to the (relocated) output chunk at runtime.
+ */
+export const SQLITE3_WASM_URL = new URL('../vendor/jswasm/sqlite3.wasm', import.meta.url);
+
+/**
+ * Emscripten module-loader options honored by {@link sqlite3InitModule}. Any further options are passed through to the
+ * vendored module unchanged.
+ */
+export interface Sqlite3InitOptions {
+  /** Resolves the URL from which a runtime asset (in practice always `sqlite3.wasm`) is fetched. */
+  locateFile?: (path: string, prefix: string) => string;
+  /** Pre-fetched wasm bytes. When set, the wasm is instantiated directly and never fetched by URL. */
+  wasmBinary?: BufferSource;
+  /** Custom wasm instantiation hook (standard Emscripten contract). Takes precedence over `wasmBinary`. */
+  instantiateWasm?: (
+    imports: WebAssembly.Imports,
+    onSuccess: (instance: WebAssembly.Instance, module: WebAssembly.Module) => void,
+  ) => object;
+  [key: string]: unknown;
+}
+
+/**
+ * Initializes the sqlite3mc wasm module.
+ *
+ * With no options, the wasm is fetched from {@link SQLITE3_WASM_URL}, which bundlers rewrite to their emitted asset,
+ * so bundled consumers work by default. Pass `locateFile`, `wasmBinary`, or `instantiateWasm` to override.
  *
- * BUNDLER COMPATIBILITY: SQLite3MultipleCiphers 2.3.5 stopped shipping the `-bundler-friendly` build variant, so
- * this entry re-exports the plain `sqlite3.mjs`. That loader always resolves `sqlite3.wasm` through its
- * `Module['locateFile']` hook, which computes `new URL(path, import.meta.url)` with a *dynamic* `path` — invisible
- * to bundlers, so bundled consumers request an unhashed `sqlite3.wasm` relative to the emitted chunk and 404 at
- * runtime (pre-2.3.5, the bundler-friendly variant carried a statically-analyzable reference instead). To stay
- * bundler-friendly we wrap the init and inject `emscriptenLocateFile` — the vendored loader's supported escape
- * hatch (`Module['locateFile']` defers to it when present on the init-module state) — resolving the wasm via a
- * static `new URL('…/sqlite3.wasm', import.meta.url)` that bundlers detect, emit, and rewrite. Unbundled usage is
- * unaffected: the static URL resolves to the real vendored path.
+ * If loading the wasm fails (unreachable URL, HTTP error, corrupt bytes), the returned promise rejects with the cause.
+ * Exception: failures inside a caller-supplied `instantiateWasm` cannot be observed (Emscripten's hook contract has no
+ * error channel), so with a custom hook the promise never settles on failure.
  */
-import sqlite3InitModuleUnwrapped from '../vendor/jswasm/sqlite3.mjs';
-
-/** Statically analyzable wasm reference: bundlers emit the asset and rewrite this URL to its final location. */
-const SQLITE3_WASM_URL = new URL('../vendor/jswasm/sqlite3.wasm', import.meta.url);
-
-type SqliteInitModuleState = {
-  emscriptenLocateFile?: (path: string, prefix: string) => string;
-  debugModule?: (...args: unknown[]) => void;
-};
-
-const sqlite3InitModule: typeof sqlite3InitModuleUnwrapped = (...args) => {
-  // `sqlite3.mjs` creates `globalThis.sqlite3InitModuleState` at import time and the inner Emscripten factory
-  // captures it (and deletes the global) on init, binding it as `this` of `Module['locateFile']`. Mutate the live
-  // object; recreate it if a previous init already consumed it (`debugModule` must exist — the factory calls it).
-  const g = globalThis as { sqlite3InitModuleState?: SqliteInitModuleState };
-  const state = (g.sqlite3InitModuleState ??= Object.assign(Object.create(null), {
-    debugModule: () => {},
-  }));
-  state.emscriptenLocateFile = (path: string, prefix: string) =>
-    path === 'sqlite3.wasm' ? SQLITE3_WASM_URL.href : new URL(path, prefix || import.meta.url).href;
-  return sqlite3InitModuleUnwrapped(...args);
-};
-
-export default sqlite3InitModule;
-export type { Database, SAHPoolUtil, Sqlite3Static } from '@sqlite.org/sqlite-wasm';
+export default function sqlite3InitModule(options: Sqlite3InitOptions = {}): Promise<Sqlite3Static> {
+  return new Promise((resolve, reject) => {
+    const instantiateWasm =
+      options.instantiateWasm ??
+      (options.wasmBinary
+        ? wasmBinaryInstantiator(options.wasmBinary, reject)
+        : urlInstantiator(options.locateFile ?? defaultLocateFile, reject));
+    const callOptions = { ...options, instantiateWasm };
+    installInitModuleState(callOptions);
+    // The vendored init consumes the installed state synchronously (its pre-js runs before the first await), so
+    // interleaved calls cannot observe each other's state. On instantiation failure the vendored promise never
+    // settles (the hook has no error channel), so the instantiators report failure through `reject` instead.
+    vendoredInit(callOptions).then(resolve, reject);
+  });
+}
+
+/** Builds an Emscripten `instantiateWasm` hook that instantiates the given bytes instead of fetching by URL. */
+function wasmBinaryInstantiator(
+  wasmBinary: BufferSource,
+  onFailure: (error: Error) => void,
+): Required<Sqlite3InitOptions>['instantiateWasm'] {
+  return (imports, onSuccess) => {
+    void WebAssembly.instantiate(wasmBinary, imports).then(
+      ({ instance, module }) => onSuccess(instance, module),
+      error => onFailure(instantiationError('wasmBinary', error)),
+    );
+    return {};
+  };
+}
+
+/**
+ * Builds an Emscripten `instantiateWasm` hook that fetches and instantiates the wasm from the located URL, replacing
+ * the vendored fallback (which reports failures nowhere). Prefers streaming compilation, falling back to
+ * buffer-based instantiation when streaming is unavailable or fails (e.g. a server responding without the
+ * `application/wasm` MIME type, which `instantiateStreaming` rejects).
+ */
+function urlInstantiator(
+  locate: (path: string, prefix: string) => string,
+  onFailure: (error: Error) => void,
+): Required<Sqlite3InitOptions>['instantiateWasm'] {
+  return (imports, onSuccess) => {
+    const url = locate('sqlite3.wasm', '');
+    const streaming = WebAssembly.instantiateStreaming
+      ? WebAssembly.instantiateStreaming(fetch(url, { credentials: 'same-origin' }), imports).catch(() =>
+          fetchAndInstantiate(url, imports),
+        )
+      : fetchAndInstantiate(url, imports);
+    void streaming.then(
+      ({ instance, module }) => onSuccess(instance, module),
+      error => onFailure(instantiationError(url, error)),
+    );
+    return {};
+  };
+}
+
+/** Fetches the wasm and instantiates it from a buffer, surfacing HTTP errors that streaming instantiation obscures. */
+async function fetchAndInstantiate(
+  url: string,
+  imports: WebAssembly.Imports,
+): Promise<WebAssembly.WebAssemblyInstantiatedSource> {
+  const response = await fetch(url, { credentials: 'same-origin' });
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status} ${response.statusText}`.trimEnd());
+  }
+  return WebAssembly.instantiate(await response.arrayBuffer(), imports);
+}
+
+function instantiationError(source: string, cause: unknown): Error {
+  const detail = cause instanceof Error ? cause.message : String(cause);
+  return new Error(`sqlite3 wasm instantiation failed (${source}): ${detail}`, { cause });
+}
+
+/**
+ * Installs the global state object the vendored module's pre-js binds its `Module.locateFile` and
+ * `Module.instantiateWasm` wrappers to.
+ */
+function installInitModuleState(options: Sqlite3InitOptions): void {
+  const urlParams = globalThis.location?.href ? new URL(globalThis.location.href).searchParams : new URLSearchParams();
+  const debugModule = urlParams.has('sqlite3.debugModule')
+    ? // eslint-disable-next-line no-console -- mirrors the vendored module's own console-based debug channel
+      (...args: unknown[]) => console.warn('sqlite3.debugModule:', ...args)
+    : () => {};
+  (globalThis as { sqlite3InitModuleState?: object }).sqlite3InitModuleState = Object.assign(Object.create(null), {
+    debugModule,
+    wasmFilename: 'sqlite3.wasm',
+    emscriptenLocateFile: options.locateFile ?? defaultLocateFile,
+    emscriptenInstantiateWasm: options.instantiateWasm,
+  });
+}
+
+/** Resolves the wasm to {@link SQLITE3_WASM_URL} so bundled consumers load the bundler-emitted asset by default. */
+function defaultLocateFile(path: string, prefix: string): string {
+  return path === 'sqlite3.wasm' ? SQLITE3_WASM_URL.href : new URL(path, prefix || import.meta.url).href;
+}