Skip to content

Commit 6081789

Browse files
pyramationpyramation
committed
feat: generic module loader with pgpm-based integration tests
- ModuleConfigLoader<T> base class: TTL caching, scope-aware resolution, AmbiguousScopeError / ModuleNotProvisionedError (no fallback defaults) - 5 concrete loaders: function, namespace, secrets, storage, invocation - ModuleLoader facade composing all sub-loaders - Queryable interface (compatible with pg.Pool and PgTestClient) - Integration tests use seed.pgpm() to deploy real metaschema-modules schema - CI workflow with constructiveio/postgres-plus:18
1 parent 59cb03c commit 6081789

18 files changed

Lines changed: 868 additions & 984 deletions

‎.github/workflows/test-module-loader.yaml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
name: Module Loader Tests
2+
3+
on:
4+
push:
5+
branches: [develop]
6+
pull_request:
7+
branches: [develop]
8+
workflow_dispatch: {}
9+
10+
concurrency:
11+
group: ${{ github.workflow }}-${{ github.ref }}
12+
cancel-in-progress: true
13+
14+
jobs:
15+
test:
16+
name: Module loader integration tests
17+
runs-on: ubuntu-latest
18+
19+
env:
20+
PGHOST: localhost
21+
PGPORT: 5432
22+
PGUSER: postgres
23+
PGPASSWORD: password
24+
PGDATABASE: postgres
25+
26+
services:
27+
pg_db:
28+
image: constructiveio/postgres-plus:18
29+
env:
30+
POSTGRES_USER: postgres
31+
POSTGRES_PASSWORD: password
32+
ports:
33+
- 5432:5432
34+
options: >-
35+
--health-cmd "pg_isready -U postgres"
36+
--health-interval 10s
37+
--health-timeout 5s
38+
--health-retries 5
39+
40+
steps:
41+
- uses: actions/checkout@v5
42+
43+
- uses: pnpm/action-setup@v6
44+
45+
- uses: actions/setup-node@v5
46+
with:
47+
node-version: '22'
48+
cache: 'pnpm'
49+
50+
- name: Install pgpm CLI
51+
run: npm install -g pgpm@4.7.4
52+
53+
- name: Bootstrap pgpm admin users
54+
run: pgpm admin-users bootstrap --yes
55+
56+
- name: Install pgpm module dependencies
57+
run: |
58+
cd extensions/@pgpm/metaschema-modules && npm install --legacy-peer-deps && cd ../../..
59+
60+
- name: Generate function packages
61+
run: node --experimental-strip-types scripts/generate.ts
62+
63+
- name: Install dependencies
64+
run: pnpm install --frozen-lockfile
65+
66+
- name: Build module-loader
67+
run: pnpm --filter @constructive-io/module-loader build
68+
69+
- name: Run module-loader tests
70+
run: pnpm --filter @constructive-io/module-loader test

‎extensions/@pgpm/metaschema-modules/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
},
2323
"dependencies": {
2424
"@pgpm/metaschema-schema": "0.28.0",
25+
"@pgpm/services": "file:../services",
2526
"@pgpm/verify": "0.28.0"
2627
},
2728
"devDependencies": {

‎packages/module-loader/README.md‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# @constructive-io/module-loader
2+
3+
Generic scope-aware MetaSchema module resolution for constructive-functions.
4+
5+
Queries `metaschema_modules_public.*_module` tables to resolve schema/table locations dynamically. No hardcoded defaults — throws explicitly if a module is not provisioned.
6+
7+
## Usage
8+
9+
```typescript
10+
import { ModuleLoader } from '@constructive-io/module-loader';
11+
12+
const loader = new ModuleLoader({ pool });
13+
14+
// Resolve function module config for a database
15+
const fnConfig = await loader.function.load(databaseId);
16+
// → { scope, publicSchema, privateSchema, definitionsTable, secretDefinitionsTable }
17+
18+
// With explicit scope (when multiple instances exist)
19+
const orgConfig = await loader.function.load(databaseId, 'org');
20+
21+
// Enumerate all instances
22+
const all = await loader.function.loadAll(databaseId);
23+
```
24+
25+
## Scope Resolution
26+
27+
- `scope` provided → filter by it; throw `ModuleNotProvisionedError` if not found
28+
- `scope` null + 1 instance → return it (unambiguous)
29+
- `scope` null + 0 instances → throw `ModuleNotProvisionedError`
30+
- `scope` null + 2+ instances → throw `AmbiguousScopeError`

‎packages/module-loader/__tests__/module-loader.test.ts‎

Lines changed: 289 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,289 @@
1+
import { getConnections, seed } from 'pgsql-test';
2+
import type { SeedAdapter } from 'pgsql-test/seed/types';
3+
import type { PgTestClient } from 'pgsql-test/test-client';
4+
import path from 'path';
5+
6+
import {
7+
AmbiguousScopeError,
8+
ModuleLoader,
9+
ModuleNotProvisionedError,
10+
} from '../src';
11+
12+
/**
13+
* Path to the metaschema-modules pgpm extension which creates all module tables
14+
* in metaschema_modules_public (function_module, namespace_module, etc.)
15+
* and their dependencies (metaschema_public.database, schema, table).
16+
*/
17+
const METASCHEMA_MODULES_PATH = path.resolve(
18+
__dirname,
19+
'../../../extensions/@pgpm/metaschema-modules'
20+
);
21+
22+
/**
23+
* Seed adapter that inserts test module data into the pgpm-deployed tables.
24+
* This is runtime data (rows), not schema DDL — the DDL comes from pgpm above.
25+
*/
26+
const testDataSeed: SeedAdapter = {
27+
async seed(ctx) {
28+
const pool = ctx.pg;
29+
30+
// Create a test database entry
31+
await pool.query(`
32+
INSERT INTO metaschema_public.database (id, name)
33+
VALUES ('aaaaaaaa-0000-0000-0000-000000000001', 'test-database')
34+
`);
35+
36+
// Create schema references
37+
await pool.query(`
38+
INSERT INTO metaschema_public.schema (id, database_id, name, schema_name)
39+
VALUES
40+
('bbbbbbbb-0000-0000-0000-000000000001', 'aaaaaaaa-0000-0000-0000-000000000001', 'compute_public', 'constructive_compute_public'),
41+
('bbbbbbbb-0000-0000-0000-000000000002', 'aaaaaaaa-0000-0000-0000-000000000001', 'compute_private', 'constructive_compute_private'),
42+
('bbbbbbbb-0000-0000-0000-000000000003', 'aaaaaaaa-0000-0000-0000-000000000001', 'infra_public', 'constructive_infra_public'),
43+
('bbbbbbbb-0000-0000-0000-000000000004', 'aaaaaaaa-0000-0000-0000-000000000001', 'infra_private', 'constructive_infra_private'),
44+
('bbbbbbbb-0000-0000-0000-000000000005', 'aaaaaaaa-0000-0000-0000-000000000001', 'store_public', 'constructive_store_public'),
45+
('bbbbbbbb-0000-0000-0000-000000000006', 'aaaaaaaa-0000-0000-0000-000000000001', 'store_private', 'constructive_store_private'),
46+
('bbbbbbbb-0000-0000-0000-000000000007', 'aaaaaaaa-0000-0000-0000-000000000001', 'objects_public', 'constructive_objects_public'),
47+
('bbbbbbbb-0000-0000-0000-000000000008', 'aaaaaaaa-0000-0000-0000-000000000001', 'objects_private', 'constructive_objects_private')
48+
`);
49+
50+
// Create table entries (required by module FK constraints)
51+
await pool.query(`
52+
INSERT INTO metaschema_public.table (id, database_id, schema_id, name)
53+
VALUES
54+
('cccccccc-0000-0000-0000-000000000001', 'aaaaaaaa-0000-0000-0000-000000000001', 'bbbbbbbb-0000-0000-0000-000000000001', 'function_definitions'),
55+
('cccccccc-0000-0000-0000-000000000002', 'aaaaaaaa-0000-0000-0000-000000000001', 'bbbbbbbb-0000-0000-0000-000000000002', 'secret_definitions'),
56+
('cccccccc-0000-0000-0000-000000000003', 'aaaaaaaa-0000-0000-0000-000000000001', 'bbbbbbbb-0000-0000-0000-000000000003', 'namespaces'),
57+
('cccccccc-0000-0000-0000-000000000004', 'aaaaaaaa-0000-0000-0000-000000000001', 'bbbbbbbb-0000-0000-0000-000000000004', 'namespace_events'),
58+
('cccccccc-0000-0000-0000-000000000005', 'aaaaaaaa-0000-0000-0000-000000000001', 'bbbbbbbb-0000-0000-0000-000000000005', 'secrets'),
59+
('cccccccc-0000-0000-0000-000000000006', 'aaaaaaaa-0000-0000-0000-000000000001', 'bbbbbbbb-0000-0000-0000-000000000007', 'buckets'),
60+
('cccccccc-0000-0000-0000-000000000007', 'aaaaaaaa-0000-0000-0000-000000000001', 'bbbbbbbb-0000-0000-0000-000000000008', 'files'),
61+
('cccccccc-0000-0000-0000-000000000008', 'aaaaaaaa-0000-0000-0000-000000000001', 'bbbbbbbb-0000-0000-0000-000000000001', 'function_invocations'),
62+
('cccccccc-0000-0000-0000-000000000009', 'aaaaaaaa-0000-0000-0000-000000000001', 'bbbbbbbb-0000-0000-0000-000000000002', 'function_execution_logs')
63+
`);
64+
65+
// Register module instances (runtime data — same as constructive-infra-services does)
66+
await pool.query(`
67+
INSERT INTO metaschema_modules_public.function_module
68+
(database_id, schema_id, private_schema_id, definitions_table_id, secret_definitions_table_id, scope, prefix, definitions_table_name, secret_definitions_table_name)
69+
VALUES
70+
('aaaaaaaa-0000-0000-0000-000000000001', 'bbbbbbbb-0000-0000-0000-000000000001', 'bbbbbbbb-0000-0000-0000-000000000002', 'cccccccc-0000-0000-0000-000000000001', 'cccccccc-0000-0000-0000-000000000002', 'app', 'platform_', 'function_definitions', 'secret_definitions')
71+
`);
72+
73+
await pool.query(`
74+
INSERT INTO metaschema_modules_public.namespace_module
75+
(database_id, schema_id, private_schema_id, namespaces_table_id, namespace_events_table_id, scope, prefix, namespaces_table_name, namespace_events_table_name)
76+
VALUES
77+
('aaaaaaaa-0000-0000-0000-000000000001', 'bbbbbbbb-0000-0000-0000-000000000003', 'bbbbbbbb-0000-0000-0000-000000000004', 'cccccccc-0000-0000-0000-000000000003', 'cccccccc-0000-0000-0000-000000000004', 'app', 'platform_', 'namespaces', 'namespace_events')
78+
`);
79+
80+
await pool.query(`
81+
INSERT INTO metaschema_modules_public.config_secrets_module
82+
(database_id, schema_id, private_schema_id, table_id, scope, prefix, table_name)
83+
VALUES
84+
('aaaaaaaa-0000-0000-0000-000000000001', 'bbbbbbbb-0000-0000-0000-000000000005', 'bbbbbbbb-0000-0000-0000-000000000006', 'cccccccc-0000-0000-0000-000000000005', 'app', 'platform_', 'secrets')
85+
`);
86+
87+
await pool.query(`
88+
INSERT INTO metaschema_modules_public.storage_module
89+
(database_id, schema_id, private_schema_id, buckets_table_id, files_table_id, scope, prefix, buckets_table_name, files_table_name)
90+
VALUES
91+
('aaaaaaaa-0000-0000-0000-000000000001', 'bbbbbbbb-0000-0000-0000-000000000007', 'bbbbbbbb-0000-0000-0000-000000000008', 'cccccccc-0000-0000-0000-000000000006', 'cccccccc-0000-0000-0000-000000000007', 'app', 'platform_', 'buckets', 'files')
92+
`);
93+
94+
await pool.query(`
95+
INSERT INTO metaschema_modules_public.function_invocation_module
96+
(database_id, schema_id, private_schema_id, invocations_table_id, execution_logs_table_id, scope, prefix, invocations_table_name, execution_logs_table_name)
97+
VALUES
98+
('aaaaaaaa-0000-0000-0000-000000000001', 'bbbbbbbb-0000-0000-0000-000000000001', 'bbbbbbbb-0000-0000-0000-000000000002', 'cccccccc-0000-0000-0000-000000000008', 'cccccccc-0000-0000-0000-000000000009', 'app', 'platform_', 'function_invocations', 'function_execution_logs')
99+
`);
100+
},
101+
};
102+
103+
const TEST_DB_ID = 'aaaaaaaa-0000-0000-0000-000000000001';
104+
105+
let pg: PgTestClient;
106+
let teardown: () => Promise<void>;
107+
108+
beforeAll(async () => {
109+
const conn = await getConnections(
110+
{},
111+
[seed.pgpm(METASCHEMA_MODULES_PATH), testDataSeed]
112+
);
113+
pg = conn.pg;
114+
teardown = conn.teardown;
115+
}, 60_000);
116+
117+
afterAll(async () => {
118+
await teardown();
119+
});
120+
121+
beforeEach(async () => {
122+
await pg.beforeEach();
123+
});
124+
125+
afterEach(async () => {
126+
await pg.afterEach();
127+
});
128+
129+
describe('ModuleLoader', () => {
130+
let loader: ModuleLoader;
131+
132+
beforeEach(() => {
133+
loader = new ModuleLoader({ pool: pg, ttlMs: 0 });
134+
});
135+
136+
describe('function module', () => {
137+
it('loads a provisioned function module', async () => {
138+
const config = await loader.function.load(TEST_DB_ID);
139+
expect(config.scope).toBe('app');
140+
expect(config.publicSchema).toBe('constructive_compute_public');
141+
expect(config.privateSchema).toBe('constructive_compute_private');
142+
expect(config.definitionsTable).toBe('function_definitions');
143+
expect(config.secretDefinitionsTable).toBe('secret_definitions');
144+
});
145+
146+
it('returns all instances via loadAll', async () => {
147+
const all = await loader.function.loadAll(TEST_DB_ID);
148+
expect(all).toHaveLength(1);
149+
expect(all[0].scope).toBe('app');
150+
});
151+
});
152+
153+
describe('namespace module', () => {
154+
it('loads a provisioned namespace module', async () => {
155+
const config = await loader.namespace.load(TEST_DB_ID);
156+
expect(config.publicSchema).toBe('constructive_infra_public');
157+
expect(config.privateSchema).toBe('constructive_infra_private');
158+
expect(config.namespacesTable).toBe('namespaces');
159+
expect(config.namespaceEventsTable).toBe('namespace_events');
160+
});
161+
});
162+
163+
describe('secrets module', () => {
164+
it('loads a provisioned secrets module', async () => {
165+
const config = await loader.secrets.load(TEST_DB_ID);
166+
expect(config.publicSchema).toBe('constructive_store_public');
167+
expect(config.privateSchema).toBe('constructive_store_private');
168+
expect(config.secretsTable).toBe('secrets');
169+
});
170+
});
171+
172+
describe('storage module', () => {
173+
it('loads a provisioned storage module', async () => {
174+
const config = await loader.storage.load(TEST_DB_ID);
175+
expect(config.publicSchema).toBe('constructive_objects_public');
176+
expect(config.privateSchema).toBe('constructive_objects_private');
177+
expect(config.bucketsTable).toBe('buckets');
178+
expect(config.filesTable).toBe('files');
179+
});
180+
});
181+
182+
describe('invocation module', () => {
183+
it('loads a provisioned invocation module', async () => {
184+
const config = await loader.invocation.load(TEST_DB_ID);
185+
expect(config.publicSchema).toBe('constructive_compute_public');
186+
expect(config.invocationsTable).toBe('function_invocations');
187+
expect(config.executionLogsTable).toBe('function_execution_logs');
188+
});
189+
});
190+
191+
describe('scope resolution', () => {
192+
it('throws ModuleNotProvisionedError for non-existent database', async () => {
193+
const fakeDbId = '00000000-0000-0000-0000-999999999999';
194+
await expect(loader.function.load(fakeDbId)).rejects.toThrow(
195+
ModuleNotProvisionedError
196+
);
197+
});
198+
199+
it('throws ModuleNotProvisionedError for wrong scope', async () => {
200+
await expect(
201+
loader.function.load(TEST_DB_ID, 'nonexistent_scope')
202+
).rejects.toThrow(ModuleNotProvisionedError);
203+
});
204+
205+
it('throws AmbiguousScopeError when multiple instances exist without scope', async () => {
206+
// Insert org-scoped table entries for the second module instance
207+
await pg.query(`
208+
INSERT INTO metaschema_public.table (id, database_id, schema_id, name)
209+
VALUES
210+
('dddddddd-0000-0000-0000-000000000001', $1, 'bbbbbbbb-0000-0000-0000-000000000001', 'org_function_definitions'),
211+
('dddddddd-0000-0000-0000-000000000002', $1, 'bbbbbbbb-0000-0000-0000-000000000002', 'org_secret_definitions')
212+
`, [TEST_DB_ID]);
213+
214+
await pg.query(`
215+
INSERT INTO metaschema_modules_public.function_module
216+
(database_id, schema_id, private_schema_id, definitions_table_id, secret_definitions_table_id, scope, prefix, definitions_table_name, secret_definitions_table_name)
217+
VALUES ($1, 'bbbbbbbb-0000-0000-0000-000000000001', 'bbbbbbbb-0000-0000-0000-000000000002', 'dddddddd-0000-0000-0000-000000000001', 'dddddddd-0000-0000-0000-000000000002', 'org', 'org_', 'org_function_definitions', 'org_secret_definitions')
218+
`, [TEST_DB_ID]);
219+
220+
loader.invalidate(TEST_DB_ID);
221+
222+
await expect(loader.function.load(TEST_DB_ID)).rejects.toThrow(
223+
AmbiguousScopeError
224+
);
225+
});
226+
227+
it('resolves by scope when multiple instances exist', async () => {
228+
await pg.query(`
229+
INSERT INTO metaschema_public.table (id, database_id, schema_id, name)
230+
VALUES
231+
('dddddddd-0000-0000-0000-000000000001', $1, 'bbbbbbbb-0000-0000-0000-000000000001', 'org_function_definitions'),
232+
('dddddddd-0000-0000-0000-000000000002', $1, 'bbbbbbbb-0000-0000-0000-000000000002', 'org_secret_definitions')
233+
`, [TEST_DB_ID]);
234+
235+
await pg.query(`
236+
INSERT INTO metaschema_modules_public.function_module
237+
(database_id, schema_id, private_schema_id, definitions_table_id, secret_definitions_table_id, scope, prefix, definitions_table_name, secret_definitions_table_name)
238+
VALUES ($1, 'bbbbbbbb-0000-0000-0000-000000000001', 'bbbbbbbb-0000-0000-0000-000000000002', 'dddddddd-0000-0000-0000-000000000001', 'dddddddd-0000-0000-0000-000000000002', 'org', 'org_', 'org_function_definitions', 'org_secret_definitions')
239+
`, [TEST_DB_ID]);
240+
241+
loader.invalidate(TEST_DB_ID);
242+
243+
const appConfig = await loader.function.load(TEST_DB_ID, 'app');
244+
expect(appConfig.scope).toBe('app');
245+
246+
const orgConfig = await loader.function.load(TEST_DB_ID, 'org');
247+
expect(orgConfig.scope).toBe('org');
248+
});
249+
});
250+
251+
describe('caching', () => {
252+
it('caches results within TTL', async () => {
253+
const loaderWithTtl = new ModuleLoader({ pool: pg, ttlMs: 60_000 });
254+
255+
const first = await loaderWithTtl.function.loadAll(TEST_DB_ID);
256+
const second = await loaderWithTtl.function.loadAll(TEST_DB_ID);
257+
expect(first).toBe(second); // same reference = cache hit
258+
});
259+
260+
it('does not cache when ttlMs is 0', async () => {
261+
const first = await loader.function.loadAll(TEST_DB_ID);
262+
const second = await loader.function.loadAll(TEST_DB_ID);
263+
expect(first).not.toBe(second); // different reference = fresh query
264+
});
265+
266+
it('invalidate clears cache for specific database', async () => {
267+
const loaderWithTtl = new ModuleLoader({ pool: pg, ttlMs: 60_000 });
268+
269+
const first = await loaderWithTtl.function.loadAll(TEST_DB_ID);
270+
loaderWithTtl.function.invalidate(TEST_DB_ID);
271+
const second = await loaderWithTtl.function.loadAll(TEST_DB_ID);
272+
expect(first).not.toBe(second);
273+
});
274+
275+
it('facade invalidate clears all sub-loader caches', async () => {
276+
const loaderWithTtl = new ModuleLoader({ pool: pg, ttlMs: 60_000 });
277+
278+
await loaderWithTtl.function.loadAll(TEST_DB_ID);
279+
await loaderWithTtl.namespace.loadAll(TEST_DB_ID);
280+
281+
loaderWithTtl.invalidate(TEST_DB_ID);
282+
283+
const fn = await loaderWithTtl.function.loadAll(TEST_DB_ID);
284+
const ns = await loaderWithTtl.namespace.loadAll(TEST_DB_ID);
285+
expect(fn).toHaveLength(1);
286+
expect(ns).toHaveLength(1);
287+
});
288+
});
289+
});

0 commit comments

Comments
 (0)