Skip to content

Commit 97358d0

Browse files
simolus3simolus3
authored
Web: Support in-memory databases (#1029)
1 parent efb8121 commit 97358d0

5 files changed

Lines changed: 131 additions & 61 deletions

File tree

‎.changeset/brown-suits-push.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@powersync/web': minor
3+
---
4+
5+
Allow using in-memory databases with `WASQLiteVFS.InMemoryVFS`.

‎packages/web/src/db/adapters/wa-sqlite/RawSqliteConnection.ts‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { Factory as WaSqliteFactory, SQLITE_ROW } from '@journeyapps/wa-sqlite';
22

3-
import { DEFAULT_MODULE_FACTORIES, WASQLiteModuleFactory, WASQLiteVFS } from './vfs.js';
3+
import { loadModuleAndVfs, WASQLiteVFS } from './vfs.js';
44
import { TemporaryStorageOption } from '../options.js';
55
import { RawQueryResult, SqliteValue } from '@powersync/common';
66

@@ -37,11 +37,8 @@ export class RawSqliteConnection {
3737
* The `sqlite3*` connection pointer.
3838
*/
3939
private db: number = 0;
40-
private _moduleFactory: WASQLiteModuleFactory;
4140

42-
constructor(readonly options: RawWaSqliteDatabaseOptions) {
43-
this._moduleFactory = DEFAULT_MODULE_FACTORIES[this.options.vfs];
44-
}
41+
constructor(readonly options: RawWaSqliteDatabaseOptions) {}
4542

4643
get isOpen(): boolean {
4744
return this.db != 0;
@@ -64,10 +61,7 @@ export class RawSqliteConnection {
6461
}
6562

6663
private async openSQLiteAPI(): Promise<SQLiteAPI> {
67-
const { module, vfs } = await this._moduleFactory({
68-
dbFileName: this.options.filename,
69-
encryptionKey: this.options.encryptionKey
70-
});
64+
const { module, vfs } = await loadModuleAndVfs(this.options);
7165
const sqlite3 = WaSqliteFactory(module);
7266
sqlite3.vfs_register(vfs, true);
7367
/**

‎packages/web/src/db/adapters/wa-sqlite/vfs.ts‎

Lines changed: 67 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import type * as SQLite from '@journeyapps/wa-sqlite';
2+
import { RawWaSqliteDatabaseOptions } from './RawSqliteConnection.js';
23

34
/**
45
* List of currently tested virtual filesystems
@@ -7,11 +8,28 @@ export enum WASQLiteVFS {
78
IDBBatchAtomicVFS = 'IDBBatchAtomicVFS',
89
OPFSCoopSyncVFS = 'OPFSCoopSyncVFS',
910
AccessHandlePoolVFS = 'AccessHandlePoolVFS',
10-
OPFSWriteAheadVFS = 'OPFSWriteAheadVFS'
11+
OPFSWriteAheadVFS = 'OPFSWriteAheadVFS',
12+
/**
13+
* A virtual file system storing data in-memory only, without persistence.
14+
*
15+
* This file system can be used in three configurations:
16+
*
17+
* 1. In shared workers (the default when available): All tabs share the same in-memory database, which is cleared
18+
* once the last tab is closed.
19+
* 2. In dedicated workers (used when `enableMultiTabs` is disabled). Each tab has its own in-memory database cleared
20+
* when the tab is closed. Queries are offloaded to a dedicated worker.
21+
* 3. In the context of the tab itself (used when both `enableMultiTabs` and `useWebWorker` are disabled). The per-tab
22+
* database is hosted in the tab itself, and queries run synchronously. This is _a lot_ faster than any other
23+
* single-threadedVFS, but can block JavaScript for computationally-intensive queries.
24+
*
25+
* This VFS primarily intended for development, but it also useful for online-first deployments not syncing large
26+
* amounts of data, as it is quicker to start up.
27+
*/
28+
InMemoryVfs = 'InMemoryVFS'
1129
}
1230

1331
export function vfsRequiresDedicatedWorkers(vfs: WASQLiteVFS) {
14-
return vfs != WASQLiteVFS.IDBBatchAtomicVFS;
32+
return vfs != WASQLiteVFS.IDBBatchAtomicVFS && vfs != WASQLiteVFS.InMemoryVfs;
1533
}
1634

1735
/**
@@ -24,14 +42,7 @@ export type WASQLiteModuleFactoryOptions = { dbFileName: string; encryptionKey?:
2442
*/
2543
export type SQLiteModule = Parameters<typeof SQLite.Factory>[0];
2644

27-
/**
28-
* @internal
29-
*/
30-
export type WASQLiteModuleFactory = (
31-
options: WASQLiteModuleFactoryOptions
32-
) => Promise<{ module: SQLiteModule; vfs: SQLiteVFS }>;
33-
34-
async function asyncModuleFactory(encryptionKey: unknown) {
45+
async function asyncModuleFactory(encryptionKey: string | undefined): Promise<SQLiteModule> {
3546
if (encryptionKey) {
3647
const { default: factory } = await import('@journeyapps/wa-sqlite/dist/mc-wa-sqlite-async.mjs');
3748
return factory();
@@ -41,7 +52,7 @@ async function asyncModuleFactory(encryptionKey: unknown) {
4152
}
4253
}
4354

44-
async function syncModuleFactory(encryptionKey: unknown) {
55+
async function syncModuleFactory(encryptionKey: string | undefined): Promise<SQLiteModule> {
4556
if (encryptionKey) {
4657
const { default: factory } = await import('@journeyapps/wa-sqlite/dist/mc-wa-sqlite.mjs');
4758
return factory();
@@ -54,43 +65,50 @@ async function syncModuleFactory(encryptionKey: unknown) {
5465
/**
5566
* @internal
5667
*/
57-
export const DEFAULT_MODULE_FACTORIES = {
58-
[WASQLiteVFS.IDBBatchAtomicVFS]: async (options: WASQLiteModuleFactoryOptions) => {
59-
const module = await asyncModuleFactory(options.encryptionKey);
60-
const { IDBBatchAtomicVFS } = await import('@journeyapps/wa-sqlite/src/examples/IDBBatchAtomicVFS.js');
61-
return {
62-
module,
68+
export async function loadModuleAndVfs({
69+
vfs,
70+
filename,
71+
encryptionKey
72+
}: RawWaSqliteDatabaseOptions): Promise<{ module: SQLiteModule; vfs: SQLiteVFS }> {
73+
let moduleFactory = syncModuleFactory;
74+
let resolveVfs: (module: any) => Promise<SQLiteVFS>;
75+
76+
switch (vfs) {
77+
case WASQLiteVFS.IDBBatchAtomicVFS: {
78+
moduleFactory = asyncModuleFactory;
79+
const { IDBBatchAtomicVFS } = await import('@journeyapps/wa-sqlite/src/examples/IDBBatchAtomicVFS.js');
80+
resolveVfs = (module) => {
81+
// @ts-expect-error The types for this static method are missing upstream
82+
return IDBBatchAtomicVFS.create(filename, module, { lockPolicy: 'exclusive' });
83+
};
84+
break;
85+
}
86+
case WASQLiteVFS.AccessHandlePoolVFS: {
87+
// @ts-expect-error The types for this import are missing upstream
88+
const { AccessHandlePoolVFS } = await import('@journeyapps/wa-sqlite/src/examples/AccessHandlePoolVFS.js');
89+
resolveVfs = (module) => AccessHandlePoolVFS.create(filename, module);
90+
break;
91+
}
92+
case WASQLiteVFS.OPFSCoopSyncVFS: {
93+
// @ts-expect-error The types for this import are missing upstream
94+
const { OPFSCoopSyncVFS } = await import('@journeyapps/wa-sqlite/src/examples/OPFSCoopSyncVFS.js');
95+
resolveVfs = (module) => OPFSCoopSyncVFS.create(filename, module);
96+
break;
97+
}
98+
case WASQLiteVFS.OPFSWriteAheadVFS: {
99+
// @ts-expect-error The types for this import are missing upstream
100+
const { OPFSWriteAheadVFS } = await import('@journeyapps/wa-sqlite/src/examples/OPFSWriteAheadVFS.js');
101+
resolveVfs = (module) => OPFSWriteAheadVFS.create(filename, module, {});
102+
break;
103+
}
104+
case WASQLiteVFS.InMemoryVfs: {
105+
const { MemoryVFS } = await import('@journeyapps/wa-sqlite/src/examples/MemoryVFS.js');
63106
// @ts-expect-error The types for this static method are missing upstream
64-
vfs: await IDBBatchAtomicVFS.create(options.dbFileName, module, { lockPolicy: 'exclusive' })
65-
};
66-
},
67-
[WASQLiteVFS.AccessHandlePoolVFS]: async (options: WASQLiteModuleFactoryOptions) => {
68-
const module = await syncModuleFactory(options.encryptionKey);
69-
// @ts-expect-error The types for this static method are missing upstream
70-
const { AccessHandlePoolVFS } = await import('@journeyapps/wa-sqlite/src/examples/AccessHandlePoolVFS.js');
71-
return {
72-
module,
73-
vfs: await AccessHandlePoolVFS.create(options.dbFileName, module)
74-
};
75-
},
76-
[WASQLiteVFS.OPFSCoopSyncVFS]: async (options: WASQLiteModuleFactoryOptions) => {
77-
const module = await syncModuleFactory(options.encryptionKey);
78-
// @ts-expect-error The types for this static method are missing upstream
79-
const { OPFSCoopSyncVFS } = await import('@journeyapps/wa-sqlite/src/examples/OPFSCoopSyncVFS.js');
80-
const vfs = await OPFSCoopSyncVFS.create(options.dbFileName, module);
81-
return {
82-
module,
83-
vfs
84-
};
85-
},
86-
[WASQLiteVFS.OPFSWriteAheadVFS]: async (options: WASQLiteModuleFactoryOptions) => {
87-
const module = await syncModuleFactory(options.encryptionKey);
88-
// @ts-expect-error The types for this static method are missing upstream
89-
const { OPFSWriteAheadVFS } = await import('@journeyapps/wa-sqlite/src/examples/OPFSWriteAheadVFS.js');
90-
const vfs = await OPFSWriteAheadVFS.create(options.dbFileName, module, {});
91-
return {
92-
module,
93-
vfs
94-
};
107+
resolveVfs = (module) => MemoryVFS.create(filename, module);
108+
break;
109+
}
95110
}
96-
};
111+
112+
const module = await moduleFactory(encryptionKey);
113+
return { module, vfs: await resolveVfs(module) };
114+
}

‎packages/web/src/worker/db/MultiDatabaseServer.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { ClientConnectionView, DatabaseServer } from '../../db/adapters/wa-sqlit
44
import { getNavigatorLocks } from '../../shared/navigator.js';
55
import { RawSqliteConnection, RawWaSqliteDatabaseOptions } from '../../db/adapters/wa-sqlite/RawSqliteConnection.js';
66
import { ConcurrentSqliteConnection } from '../../db/adapters/wa-sqlite/ConcurrentConnection.js';
7+
import { WASQLiteVFS } from '../../db/adapters/wa-sqlite/vfs.js';
78

89
const OPEN_DB_LOCK = 'open-wasqlite-db';
910

@@ -75,14 +76,15 @@ export class MultiDatabaseServer {
7576
options: RawWaSqliteDatabaseOptions
7677
): Promise<DatabaseServer> {
7778
return getNavigatorLocks().request(OPEN_DB_LOCK, async () => {
78-
const { filename } = options;
79+
const { filename, readonly, vfs } = options;
7980

8081
let server: DatabaseServer | undefined = this.activeDatabases.get(filename);
8182
if (server == null) {
8283
// We don't need navigator locks for shared workers because all queries run in this shared worker exclusively.
8384
// For read-only connections, we use a VFS that supports concurrent reads (so a single lock on the connection is
84-
// fine).
85-
const needsNavigatorLocks = !(isSharedWorker || options.readonly);
85+
// fine). In-memory databases either run in a shared worker or aren't shared across tabs at all, so the internal
86+
// lock is enough.
87+
const needsNavigatorLocks = !(isSharedWorker || readonly || vfs == WASQLiteVFS.InMemoryVfs);
8688
const connection = new RawSqliteConnection(options);
8789
const withSafeConcurrency = new ConcurrentSqliteConnection(connection, needsNavigatorLocks);
8890

‎packages/web/tests/main.test.ts‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,57 @@ describe(
4545
)
4646
);
4747

48+
describe('Basic - with in-memory', () => {
49+
const defaultOptions = {
50+
dbFilename: 'in-memory.db',
51+
vfs: WASQLiteVFS.InMemoryVfs,
52+
databaseWorkerLogLevel: defaultLogLevel
53+
};
54+
55+
describe(
56+
'in shared worker',
57+
{ sequential: true },
58+
describeBasicTests(() =>
59+
generateTestDb({
60+
schema: TEST_SCHEMA,
61+
logger: defaultTestLogger,
62+
database: {
63+
...defaultOptions
64+
}
65+
})
66+
)
67+
);
68+
69+
describe(
70+
'in dedicated worker',
71+
describeBasicTests(() =>
72+
generateTestDb({
73+
schema: TEST_SCHEMA,
74+
logger: defaultTestLogger,
75+
database: {
76+
...defaultOptions,
77+
enableMultiTabs: false
78+
}
79+
})
80+
)
81+
);
82+
83+
describe(
84+
'in local tab',
85+
describeBasicTests(() =>
86+
generateTestDb({
87+
schema: TEST_SCHEMA,
88+
logger: defaultTestLogger,
89+
database: {
90+
...defaultOptions,
91+
enableMultiTabs: false,
92+
useWebWorker: false
93+
}
94+
})
95+
)
96+
);
97+
});
98+
4899
function describeBasicTests(generateDB: () => PowerSyncDatabase) {
49100
return () => {
50101
it('should execute a select query using getAll', async () => {

0 commit comments

Comments
 (0)