Skip to content

Commit 8634734

Browse files
committed
feat: split bundler-friendly bundle in to individual VFSs to reduce import size
1 parent 91ea478 commit 8634734

39 files changed

Lines changed: 22415 additions & 9 deletions

README.md

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,27 @@ initializeSQLite();
110110
The `db` object above implements the
111111
[Object-Oriented API #1](https://sqlite.org/wasm/doc/trunk/api-oo1.md).
112112

113+
### Importing only selected optional support:
114+
115+
The default `@sqlite.org/sqlite-wasm` entry keeps backwards compatibility and includes all bundled
116+
VFS support, virtual table helpers, and Worker API #1. To let bundlers include only the optional
117+
code you need, import the core runtime and one or more optional modules before calling
118+
`sqlite3InitModule()`:
119+
120+
```js
121+
import '@sqlite.org/sqlite-wasm/vfs/kvvfs';
122+
import sqlite3InitModule from '@sqlite.org/sqlite-wasm/core';
123+
124+
const sqlite3 = await sqlite3InitModule();
125+
```
126+
127+
Available optional modules are `@sqlite.org/sqlite-wasm/vfs/kvvfs`,
128+
`@sqlite.org/sqlite-wasm/vfs/opfs`, `@sqlite.org/sqlite-wasm/vfs/opfs-sahpool`,
129+
`@sqlite.org/sqlite-wasm/vfs/opfs-wl`, and `@sqlite.org/sqlite-wasm/vtab`.
130+
131+
The core entry intentionally does not include Worker API #1. Use the default
132+
`@sqlite.org/sqlite-wasm` entry if you need `sqlite3.initWorker1API()` or `sqlite3Worker1Promiser`.
133+
113134
## Usage with vite
114135

115136
If you are using [vite](https://vitejs.dev/), you need to add the following config option in

package.json

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,13 @@
2424
"opfs",
2525
"origin-private-file-system"
2626
],
27+
"sideEffects": [
28+
"./dist/index.mjs",
29+
"./dist/vtab.mjs",
30+
"./dist/vfs/*.mjs",
31+
"./dist/sqlite3-vfs-*.mjs",
32+
"./dist/sqlite3-worker1-api-*.mjs"
33+
],
2734
"publishConfig": {
2835
"access": "public"
2936
},
@@ -38,6 +45,36 @@
3845
"main": "./dist/index.mjs",
3946
"browser": "./dist/index.mjs"
4047
},
48+
"./core": {
49+
"types": "./dist/core.d.mts",
50+
"import": "./dist/core.mjs",
51+
"browser": "./dist/core.mjs"
52+
},
53+
"./vtab": {
54+
"types": "./dist/vtab.d.mts",
55+
"import": "./dist/vtab.mjs",
56+
"browser": "./dist/vtab.mjs"
57+
},
58+
"./vfs/kvvfs": {
59+
"types": "./dist/vfs/kvvfs.d.mts",
60+
"import": "./dist/vfs/kvvfs.mjs",
61+
"browser": "./dist/vfs/kvvfs.mjs"
62+
},
63+
"./vfs/opfs": {
64+
"types": "./dist/vfs/opfs.d.mts",
65+
"import": "./dist/vfs/opfs.mjs",
66+
"browser": "./dist/vfs/opfs.mjs"
67+
},
68+
"./vfs/opfs-sahpool": {
69+
"types": "./dist/vfs/opfs-sahpool.d.mts",
70+
"import": "./dist/vfs/opfs-sahpool.mjs",
71+
"browser": "./dist/vfs/opfs-sahpool.mjs"
72+
},
73+
"./vfs/opfs-wl": {
74+
"types": "./dist/vfs/opfs-wl.d.mts",
75+
"import": "./dist/vfs/opfs-wl.mjs",
76+
"browser": "./dist/vfs/opfs-wl.mjs"
77+
},
4178
"./package.json": "./package.json",
4279
"./sqlite3.wasm": "./dist/sqlite3.wasm"
4380
},
@@ -49,6 +86,8 @@
4986
"README.md"
5087
],
5188
"scripts": {
89+
"split:vfs": "node scripts/split-bundler-friendly-vfss.mjs",
90+
"prebuild": "npm run split:vfs",
5291
"test": "vitest",
5392
"test:node": "vitest --project node",
5493
"test:browser": "vitest --project browser",
Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
import { mkdir, readFile, writeFile } from 'node:fs/promises';
2+
import path from 'node:path';
3+
import { fileURLToPath } from 'node:url';
4+
5+
const rootDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
6+
const sourcePath = path.join(rootDir, 'src/bin/sqlite3-bundler-friendly.mjs');
7+
const outputDir = path.join(rootDir, 'src/bin');
8+
9+
const generatedHeader = (
10+
sourceFile,
11+
) => `// Generated by scripts/split-bundler-friendly-vfss.mjs from ${sourceFile}.
12+
// Do not edit this file directly.
13+
14+
`;
15+
16+
const initRegistry = `const sqlite3BundlerFriendlyOptionalState =
17+
(globalThis.__sqlite3BundlerFriendlyOptional ??= {
18+
initializers: Object.create(null),
19+
});
20+
21+
const sqlite3BundlerFriendlyRegisterOptionalInitializer = (slot, initializer) => {
22+
if (globalThis.sqlite3ApiBootstrap?.initializers) {
23+
globalThis.sqlite3ApiBootstrap.initializers.push(initializer);
24+
} else {
25+
(sqlite3BundlerFriendlyOptionalState.initializers[slot] ??= []).push(initializer);
26+
}
27+
};
28+
29+
`;
30+
31+
const baseHook = `const sqlite3BundlerFriendlyInstallOptionalInitializers = (slot) => {
32+
const sqlite3BundlerFriendlyOptionalState = globalThis.__sqlite3BundlerFriendlyOptional;
33+
const initializers = sqlite3BundlerFriendlyOptionalState?.initializers?.[slot];
34+
if (initializers?.length) {
35+
globalThis.sqlite3ApiBootstrap.initializers.push(
36+
...initializers,
37+
);
38+
}
39+
};
40+
41+
`;
42+
43+
const markerSpecs = {
44+
worker1: 'This file implements the initializer for SQLite\'s "Worker API #1"',
45+
helper: 'This file installs sqlite3.vfs, a namespace of helpers',
46+
vtab: 'This file installs sqlite3.vtab, a namespace of helpers',
47+
kvvfs: 'This file houses the "kvvfs" pieces of the SQLite3 JS API',
48+
opfsShared: 'This file holds code shared by sqlite3-vfs-opfs{,-wl}.c-pp.js',
49+
opfs: 'This file holds the synchronous half of an sqlite3_vfs',
50+
sahpool: 'This file holds a sqlite3_vfs backed by OPFS storage',
51+
opfsWl: 'This file is a reimplementation of the "opfs" VFS',
52+
tail: 'This file is the tail end of the sqlite3-api.js constellation',
53+
};
54+
55+
const findCommentStart = (source, marker) => {
56+
const markerIndex = source.indexOf(marker);
57+
if (markerIndex < 0) {
58+
throw new Error(`Unable to find marker: ${marker}`);
59+
}
60+
const commentStart = source.lastIndexOf('/*', markerIndex);
61+
if (commentStart < 0) {
62+
throw new Error(`Unable to find comment start for marker: ${marker}`);
63+
}
64+
return commentStart;
65+
};
66+
67+
const replaceAll = (value, replacements) => {
68+
let result = value;
69+
for (const [from, to] of replacements) {
70+
result = result.split(from).join(to);
71+
}
72+
return result;
73+
};
74+
75+
const source = await readFile(sourcePath, 'utf8');
76+
const markers = Object.fromEntries(
77+
Object.entries(markerSpecs).map(([name, marker]) => [name, findCommentStart(source, marker)]),
78+
);
79+
80+
const ranges = {
81+
worker1: [markers.worker1, markers.helper],
82+
helper: [markers.helper, markers.vtab],
83+
vtab: [markers.vtab, markers.kvvfs],
84+
kvvfs: [markers.kvvfs, markers.opfsShared],
85+
opfsShared: [markers.opfsShared, markers.opfs],
86+
opfs: [markers.opfs, markers.sahpool],
87+
sahpool: [markers.sahpool, markers.opfsWl],
88+
opfsWl: [markers.opfsWl, markers.tail],
89+
};
90+
91+
for (const [name, [start, end]] of Object.entries(ranges)) {
92+
if (start >= end) {
93+
throw new Error(`Invalid ${name} range: ${start} >= ${end}`);
94+
}
95+
}
96+
97+
const bootstrapAsyncMarker = 'globalThis.sqlite3ApiBootstrap.initializersAsync = [];\n';
98+
if (!source.includes(bootstrapAsyncMarker)) {
99+
throw new Error(`Unable to find bootstrap hook marker: ${bootstrapAsyncMarker.trim()}`);
100+
}
101+
102+
const strippedSource = Object.entries(ranges)
103+
.sort((a, b) => b[1][0] - a[1][0])
104+
.reduce(
105+
(current, [name, [start, end]]) =>
106+
current.slice(0, start) +
107+
`\n/* Optional initializer slot removed by scripts/split-bundler-friendly-vfss.mjs: ${name} */\n` +
108+
`sqlite3BundlerFriendlyInstallOptionalInitializers('${name}');\n` +
109+
current.slice(end),
110+
source,
111+
)
112+
.replace(bootstrapAsyncMarker, bootstrapAsyncMarker + baseHook);
113+
114+
const outputFileNames = {
115+
worker1: 'sqlite3-worker1-api.mjs',
116+
};
117+
118+
const toOptionalModule = (slot, chunk) =>
119+
generatedHeader('sqlite3-bundler-friendly.mjs') +
120+
initRegistry +
121+
replaceAll(chunk, [
122+
[
123+
'globalThis.sqlite3ApiBootstrap.initializers.push(',
124+
`sqlite3BundlerFriendlyRegisterOptionalInitializer('${slot}', `,
125+
],
126+
]);
127+
128+
await mkdir(outputDir, { recursive: true });
129+
130+
await writeFile(
131+
path.join(outputDir, 'sqlite3-bundler-friendly.core.mjs'),
132+
generatedHeader('sqlite3-bundler-friendly.mjs') + strippedSource,
133+
);
134+
135+
await Promise.all(
136+
Object.entries(ranges).map(([name, [start, end]]) =>
137+
writeFile(
138+
path.join(outputDir, outputFileNames[name] ?? `sqlite3-vfs-${name}.mjs`),
139+
toOptionalModule(name, source.slice(start, end)),
140+
),
141+
),
142+
);

src/__tests__/sqlite3-oo1.browser.test.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@ import type { SqlValue } from '../index';
55
test('Bundler-friendly OO1 API sanity check (browser)', async () => {
66
const sqlite3 = await sqlite3InitModule();
77

8+
expect(sqlite3.initWorker1API).toBeTypeOf('function');
9+
expect(sqlite3.vtab).toBeDefined();
10+
811
// 1. Create a database
912
const db = new sqlite3.oo1.DB(':memory:');
1013
expect(db.isOpen()).toBe(true);

src/__tests__/sqlite3-opfs.browser.test.ts

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,17 @@ const runWorker = async (workerUrl: URL): Promise<void> => {
4242
};
4343

4444
describe('opfs persistence APIs', () => {
45-
test('OpfsDb sanity check in Worker (browser)', async () => {
46-
await runWorker(new URL('./workers/sqlite3-opfs.worker.ts', import.meta.url));
45+
test.each([
46+
['default entry', './workers/sqlite3-opfs.worker.ts'],
47+
['treeshakable entry', './workers/sqlite3-opfs-treeshakable.worker.ts'],
48+
])('OpfsDb sanity check in Worker (browser, %s)', async (_label, workerPath) => {
49+
await runWorker(new URL(workerPath, import.meta.url));
4750
});
4851

49-
test('OpfsWlDb sanity check in Worker (browser)', async () => {
50-
await runWorker(new URL('./workers/sqlite3-opfs-wl.worker.ts', import.meta.url));
52+
test.each([
53+
['default entry', './workers/sqlite3-opfs-wl.worker.ts'],
54+
['treeshakable entry', './workers/sqlite3-opfs-wl-treeshakable.worker.ts'],
55+
])('OpfsWlDb sanity check in Worker (browser, %s)', async (_label, workerPath) => {
56+
await runWorker(new URL(workerPath, import.meta.url));
5157
});
5258
});

src/__tests__/sqlite3-sahpool-vfs.browser.test.ts

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,11 @@ type WorkerSuccessMessage = {
44
type: 'success';
55
};
66

7-
test('OpfsSAHPoolVfs sanity check in Worker (browser)', async () => {
8-
const worker = new Worker(new URL('./workers/sqlite3-sahpool.worker.ts', import.meta.url), {
9-
type: 'module',
10-
});
7+
test.each([
8+
['default entry', './workers/sqlite3-sahpool.worker.ts'],
9+
['treeshakable entry', './workers/sqlite3-sahpool-treeshakable.worker.ts'],
10+
])('OpfsSAHPoolVfs sanity check in Worker (browser, %s)', async (_label, workerPath) => {
11+
const worker = new Worker(new URL(workerPath, import.meta.url), { type: 'module' });
1112

1213
try {
1314
const result = await new Promise<WorkerSuccessMessage>((resolve, reject) => {
Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
import { expect, test } from 'vitest';
2+
3+
type WorkerMessage = {
4+
type: 'success';
5+
};
6+
7+
const runWorker = async (workerUrl: URL): Promise<void> => {
8+
const worker = new Worker(workerUrl, { type: 'module' });
9+
10+
try {
11+
const result = await new Promise<WorkerMessage>((resolve, reject) => {
12+
worker.onmessage = (e) => {
13+
if (e.data.type === 'success') {
14+
resolve(e.data);
15+
} else {
16+
reject(new Error(e.data.message || 'Unknown worker error'));
17+
}
18+
};
19+
worker.onerror = (e) => {
20+
reject(new Error('Worker error: ' + e.message));
21+
};
22+
worker.postMessage({ type: 'start' });
23+
});
24+
25+
expect(result.type).toBe('success');
26+
} finally {
27+
worker.terminate();
28+
}
29+
};
30+
31+
test('core runtime initializes without optional VFS modules', async () => {
32+
await runWorker(new URL('./workers/sqlite3-core.worker.ts', import.meta.url));
33+
});
34+
35+
test('core runtime can opt into only kvvfs', async () => {
36+
await runWorker(new URL('./workers/sqlite3-core-kvvfs.worker.ts', import.meta.url));
37+
});
38+
39+
test('core runtime can opt into only vtab helpers', async () => {
40+
await runWorker(new URL('./workers/sqlite3-core-vtab.worker.ts', import.meta.url));
41+
});
Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
import { describe, test } from 'vitest';
2+
import { execFileSync } from 'node:child_process';
3+
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
4+
import path from 'node:path';
5+
6+
const runTypeCheck = (source: string): void => {
7+
const dir = mkdtempSync(path.join(process.cwd(), '.tmp-treeshakable-types-'));
8+
const file = path.join(dir, 'fixture.ts');
9+
10+
try {
11+
writeFileSync(file, source);
12+
execFileSync(
13+
process.execPath,
14+
[
15+
'node_modules/typescript/bin/tsc',
16+
'--ignoreConfig',
17+
'--noEmit',
18+
'--module',
19+
'esnext',
20+
'--target',
21+
'es2023',
22+
'--moduleResolution',
23+
'bundler',
24+
'--strict',
25+
'--skipLibCheck',
26+
'--lib',
27+
'esnext,dom',
28+
file,
29+
],
30+
{ cwd: process.cwd(), stdio: 'inherit' },
31+
);
32+
} finally {
33+
rmSync(dir, { recursive: true, force: true });
34+
}
35+
};
36+
37+
describe('treeshakable entry types', () => {
38+
test('core types omit optional APIs', () => {
39+
runTypeCheck(`
40+
import init from '@sqlite.org/sqlite-wasm/core';
41+
42+
async function checkCore() {
43+
const sqlite3 = await init();
44+
sqlite3.oo1.DB;
45+
// @ts-expect-error core does not include kvvfs
46+
sqlite3.kvvfs;
47+
// @ts-expect-error core does not include vtab helpers
48+
sqlite3.vtab;
49+
// @ts-expect-error core does not include Worker API #1
50+
sqlite3.initWorker1API;
51+
// @ts-expect-error core does not include OPFS DB constructor
52+
sqlite3.oo1.OpfsDb;
53+
}
54+
`);
55+
});
56+
57+
test('vfs imports augment core types only with their installed APIs', () => {
58+
runTypeCheck(`
59+
import '@sqlite.org/sqlite-wasm/vfs/kvvfs';
60+
import init from '@sqlite.org/sqlite-wasm/core';
61+
62+
async function checkKvvfs() {
63+
const sqlite3 = await init();
64+
sqlite3.kvvfs.unlink('x');
65+
sqlite3.vfs.installVfs;
66+
sqlite3.oo1.JsStorageDb;
67+
// @ts-expect-error kvvfs import does not include OPFS DB constructor
68+
sqlite3.oo1.OpfsDb;
69+
// @ts-expect-error kvvfs import does not include Worker API #1
70+
sqlite3.initWorker1API;
71+
}
72+
`);
73+
});
74+
});

0 commit comments

Comments
 (0)