Skip to content

Commit 700e7a4

Browse files
pyramationpyramation
authored
Merge pull request #111 from constructive-io/feat/module-loader-v2-dev
feat: generic scope-aware module loader with pgpm-based tests
2 parents 59cb03c + 9b88992 commit 700e7a4

63 files changed

Lines changed: 2803 additions & 1925 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/test-module-loader.yaml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
name: Module Loader Tests
2+
3+
on:
4+
push:
5+
branches: [develop]
6+
pull_request:
7+
branches: [develop]
8+
workflow_dispatch: {}
9+
10+
concurrency:
11+
group: ${{ github.workflow }}-${{ github.ref }}
12+
cancel-in-progress: true
13+
14+
jobs:
15+
test:
16+
name: Module loader integration tests
17+
runs-on: ubuntu-latest
18+
19+
env:
20+
PGHOST: localhost
21+
PGPORT: 5432
22+
PGUSER: postgres
23+
PGPASSWORD: password
24+
PGDATABASE: postgres
25+
26+
services:
27+
pg_db:
28+
image: constructiveio/postgres-plus:18
29+
env:
30+
POSTGRES_USER: postgres
31+
POSTGRES_PASSWORD: password
32+
ports:
33+
- 5432:5432
34+
options: >-
35+
--health-cmd "pg_isready -U postgres"
36+
--health-interval 10s
37+
--health-timeout 5s
38+
--health-retries 5
39+
40+
steps:
41+
- uses: actions/checkout@v5
42+
43+
- uses: pnpm/action-setup@v6
44+
45+
- uses: actions/setup-node@v5
46+
with:
47+
node-version: '22'
48+
cache: 'pnpm'
49+
50+
- name: Install pgpm CLI
51+
run: npm install -g pgpm@4.7.4
52+
53+
- name: Bootstrap pgpm admin users
54+
run: pgpm admin-users bootstrap --yes
55+
56+
- name: Generate function packages
57+
run: node --experimental-strip-types scripts/generate.ts
58+
59+
- name: Install dependencies
60+
run: pnpm install --frozen-lockfile
61+
62+
- name: Build module-loader
63+
run: pnpm --filter @constructive-io/module-loader build
64+
65+
- name: Run module-loader tests
66+
run: pnpm --filter @constructive-io/module-loader test

‎extensions/@pgpm/metaschema-modules/Makefile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
EXTENSION = metaschema-modules
2-
DATA = sql/metaschema-modules--0.26.5.sql
2+
DATA = sql/metaschema-modules--0.28.0.sql
33

44
PG_CONFIG = pg_config
55
PGXS := $(shell $(PG_CONFIG) --pgxs)
66
include $(PGXS)
7+

‎extensions/@pgpm/metaschema-modules/deploy/schemas/metaschema_modules_public/tables/graph_execution_module/table.sql‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
-- Deploy schemas/metaschema_modules_public/tables/graph_execution_module/table to pg
2+
3+
-- requires: schemas/metaschema_modules_public/schema
4+
-- requires: schemas/metaschema_modules_public/tables/graph_module/table
5+
6+
BEGIN;
7+
8+
CREATE TABLE metaschema_modules_public.graph_execution_module (
9+
id uuid PRIMARY KEY DEFAULT uuidv7(),
10+
database_id uuid NOT NULL,
11+
12+
-- Schema references (if uuid_nil, resolved from schema name or default)
13+
schema_id uuid NOT NULL DEFAULT uuid_nil(),
14+
private_schema_id uuid NOT NULL DEFAULT uuid_nil(),
15+
16+
-- Optional schema name overrides (used when schema IDs are not provided)
17+
public_schema_name text,
18+
private_schema_name text,
19+
20+
-- Reference to the graph module this execution module operates against.
21+
-- The execution module resolves definition tables (graphs, merkle store)
22+
-- from the linked graph_module at provision time.
23+
graph_module_id uuid NOT NULL,
24+
25+
-- Scope: determines the security level for this module instance.
26+
-- Can differ from graph_module scope (e.g., platform definitions + entity executions).
27+
scope text NOT NULL DEFAULT 'app',
28+
29+
-- Table name prefix. Auto-derived from scope by the trigger when empty.
30+
prefix text NOT NULL DEFAULT '',
31+
32+
-- Generated table IDs (populated by BEFORE INSERT trigger)
33+
-- Execution state tables (partitioned by time)
34+
executions_table_id uuid NOT NULL DEFAULT uuid_nil(),
35+
outputs_table_id uuid NOT NULL DEFAULT uuid_nil(),
36+
node_states_table_id uuid NOT NULL DEFAULT uuid_nil(),
37+
38+
39+
-- Configurable table names (bare names without scope prefix).
40+
-- The trigger prepends the scope prefix automatically.
41+
executions_table_name text NOT NULL DEFAULT 'function_graph_executions',
42+
outputs_table_name text NOT NULL DEFAULT 'function_graph_execution_outputs',
43+
node_states_table_name text NOT NULL DEFAULT 'function_graph_execution_node_states',
44+
45+
-- API routing (get-or-create: if set, schema is added to this API; if NULL, no API is added)
46+
api_name text,
47+
private_api_name text,
48+
49+
-- Entity table for RLS and billing attribution.
50+
-- When set, executions are scoped to the entity (org, app) for billing/metering.
51+
entity_table_id uuid NULL,
52+
53+
-- Configurable security policies (NULL = use defaults based on scope).
54+
policies jsonb NULL,
55+
56+
-- Per-table provisions overrides from blueprint config.
57+
-- Keys are table keys (executions, outputs, node_states).
58+
provisions jsonb NULL,
59+
60+
-- Default permissions: permission names auto-granted to new members.
61+
default_permissions text[] DEFAULT NULL,
62+
63+
-- Timestamps
64+
created_at timestamptz NOT NULL DEFAULT now(),
65+
66+
-- Constraints
67+
CONSTRAINT graph_execution_module_db_fkey FOREIGN KEY (database_id) REFERENCES metaschema_public.database (id) ON DELETE CASCADE,
68+
CONSTRAINT graph_execution_module_schema_fkey FOREIGN KEY (schema_id) REFERENCES metaschema_public.schema (id) ON DELETE CASCADE,
69+
CONSTRAINT graph_execution_module_private_schema_fkey FOREIGN KEY (private_schema_id) REFERENCES metaschema_public.schema (id) ON DELETE CASCADE,
70+
CONSTRAINT graph_execution_module_graph_module_fkey FOREIGN KEY (graph_module_id) REFERENCES metaschema_modules_public.graph_module (id) ON DELETE CASCADE,
71+
CONSTRAINT graph_execution_module_executions_table_fkey FOREIGN KEY (executions_table_id) REFERENCES metaschema_public.table (id) ON DELETE CASCADE,
72+
CONSTRAINT graph_execution_module_outputs_table_fkey FOREIGN KEY (outputs_table_id) REFERENCES metaschema_public.table (id) ON DELETE CASCADE,
73+
CONSTRAINT graph_execution_module_node_states_table_fkey FOREIGN KEY (node_states_table_id) REFERENCES metaschema_public.table (id) ON DELETE CASCADE,
74+
75+
CONSTRAINT graph_execution_module_entity_table_fkey FOREIGN KEY (entity_table_id) REFERENCES metaschema_public.table (id) ON DELETE CASCADE
76+
);
77+
78+
CREATE INDEX graph_execution_module_database_id_idx ON metaschema_modules_public.graph_execution_module ( database_id );
79+
80+
-- One execution module per (database, scope, prefix).
81+
CREATE UNIQUE INDEX graph_execution_module_unique_scope ON metaschema_modules_public.graph_execution_module ( database_id, scope, prefix );
82+
83+
COMMIT;

‎extensions/@pgpm/metaschema-modules/metaschema-modules.control‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# metaschema-modules extension
22
comment = 'metaschema-modules extension'
3-
default_version = '0.26.5'
3+
default_version = '0.28.0'
44
module_pathname = '$libdir/metaschema-modules'
5-
requires = 'plpgsql,uuid-ossp,metaschema-schema,services,pgpm-verify'
5+
requires = 'metaschema-schema,pgpm-database-jobs,pgpm-inflection,pgpm-jwt-claims,pgpm-types,pgpm-verify,plpgsql,services,uuid-ossp'
66
relocatable = false
77
superuser = false

‎extensions/@pgpm/metaschema-modules/package.json‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,6 @@
2020
"test": "jest",
2121
"test:watch": "jest --watch"
2222
},
23-
"dependencies": {
24-
"@pgpm/metaschema-schema": "0.28.0",
25-
"@pgpm/verify": "0.28.0"
26-
},
27-
"devDependencies": {
28-
"pgpm": "^4.23.2"
29-
},
3023
"repository": {
3124
"type": "git",
3225
"url": "https://github.com/constructive-io/pgpm-modules"
@@ -35,5 +28,13 @@
3528
"bugs": {
3629
"url": "https://github.com/constructive-io/pgpm-modules/issues"
3730
},
38-
"gitHead": "d2ab7ca810ded086eb742eb8f0ca362b6212b97e"
39-
}
31+
"gitHead": "d2ab7ca810ded086eb742eb8f0ca362b6212b97e",
32+
"dependencies": {
33+
"@pgpm/metaschema-schema": "0.28.0",
34+
"@pgpm/services": "0.28.0",
35+
"@pgpm/verify": "0.28.0"
36+
},
37+
"devDependencies": {
38+
"pgpm": "^4.23.2"
39+
}
40+
}

‎extensions/@pgpm/metaschema-modules/pgpm.plan‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,3 +66,4 @@ schemas/metaschema_modules_public/tables/user_credentials_module/table [schemas/
6666
schemas/metaschema_modules_public/tables/user_settings_module/table [schemas/metaschema_modules_public/schema] 2026-05-28T00:00:00Z devin <devin@cognition.ai> # add user_settings_module for extensible per-user preferences (1:1 with users)
6767

6868
schemas/metaschema_modules_public/tables/i18n_module/table [schemas/metaschema_modules_public/schema] 2026-05-28T00:00:00Z devin <devin@cognition.ai> # add i18n_module config table for internationalization settings
69+
schemas/metaschema_modules_public/tables/graph_execution_module/table [schemas/metaschema_modules_public/schema schemas/metaschema_modules_public/tables/graph_module/table] 2026-06-12T00:00:00Z devin <devin@cognition.ai> # add graph_execution_module config table for partitioned execution state + merkle tree time-travel debugging

‎extensions/@pgpm/metaschema-modules/revert/schemas/metaschema_modules_public/tables/graph_execution_module/table.sql‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
-- Revert schemas/metaschema_modules_public/tables/graph_execution_module/table from pg
2+
3+
BEGIN;
4+
5+
DROP TABLE metaschema_modules_public.graph_execution_module;
6+
7+
COMMIT;

‎extensions/@pgpm/metaschema-modules/verify/schemas/metaschema_modules_public/tables/graph_execution_module/table.sql‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
-- Verify schemas/metaschema_modules_public/tables/graph_execution_module/table on pg
2+
3+
BEGIN;
4+
5+
SELECT verify_table ('metaschema_modules_public.graph_execution_module');
6+
7+
ROLLBACK;

‎job/compute-service/src/index.ts‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
* 4. A Scheduler for cron-like scheduled jobs
1313
*/
1414

15-
import ComputeWorker, { ComputeModuleLoader } from '@constructive-io/compute-worker';
15+
import ComputeWorker, { ModuleLoader } from '@constructive-io/compute-worker';
1616
import poolManager from '@constructive-io/job-pg';
1717
import Scheduler from '@constructive-io/job-scheduler';
1818
import {
@@ -399,17 +399,9 @@ export const waitForComputePrereqs = async (): Promise<void> => {
399399
database: cfg.database,
400400
max: 1,
401401
});
402-
const loader = new ComputeModuleLoader(pool, 0);
403-
const config = await loader.load(databaseId);
404-
405-
if (config.functionModule) {
406-
const { publicSchema, definitionsTable } = config.functionModule;
407-
await client.query(`SELECT count(*) FROM "${publicSchema}"."${definitionsTable}" LIMIT 1`);
408-
} else {
409-
// Metaschema not populated — check the compute schema directly
410-
log.info('function_module not in metaschema, checking constructive_compute_public directly');
411-
await client.query('SELECT count(*) FROM constructive_compute_public.platform_function_definitions LIMIT 1');
412-
}
402+
const loader = new ModuleLoader({ pool, ttlMs: 0 });
403+
const fnConfig = await loader.function.loadDefault(databaseId);
404+
await client.query(`SELECT count(*) FROM "${fnConfig.publicSchema}"."${fnConfig.definitionsTable}" LIMIT 1`);
413405

414406
log.info('compute prereqs satisfied (jobs table + compute module present)');
415407
} catch (error) {

‎job/compute-worker/src/billing.ts‎

Lines changed: 52 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,59 @@
11
/**
2-
* BillingTracker — re-exports BillingLoader from @constructive-io/module-loader
3-
* under the legacy name `BillingTracker`.
2+
* BillingTracker — quota checks and usage recording via billing_module.
3+
*
4+
* Resolves billing config dynamically via ModuleLoader. Gracefully no-ops
5+
* when billing is not provisioned (standalone dev mode).
46
*/
57

6-
import { BillingLoader } from '@constructive-io/module-loader';
8+
import type { BillingModuleConfig } from '@constructive-io/module-loader';
9+
import { AmbiguousScopeError, ModuleLoader, ModuleNotProvisionedError } from '@constructive-io/module-loader';
710
import type { Pool } from 'pg';
811

9-
export class BillingTracker extends BillingLoader {
10-
constructor(pool: Pool, databaseId: string, cacheTtlMs?: number) {
11-
super(pool, databaseId, cacheTtlMs);
12+
export class BillingTracker {
13+
private loader: ModuleLoader;
14+
private pool: Pool;
15+
private databaseId: string;
16+
17+
constructor(pool: Pool, databaseId: string) {
18+
this.pool = pool;
19+
this.databaseId = databaseId;
20+
this.loader = new ModuleLoader({ pool });
21+
}
22+
23+
async load(databaseId?: string): Promise<BillingModuleConfig | null> {
24+
try {
25+
return await this.loader.billing.load(databaseId ?? this.databaseId, null);
26+
} catch (err) {
27+
if (err instanceof ModuleNotProvisionedError) return null;
28+
if (err instanceof AmbiguousScopeError) {
29+
return await this.loader.billing.loadDefault(databaseId ?? this.databaseId);
30+
}
31+
return null;
32+
}
33+
}
34+
35+
async checkQuota(entityId: string, meterSlug: string, amount = 1, databaseId?: string): Promise<boolean> {
36+
const config = await this.load(databaseId);
37+
if (!config) return true;
38+
try {
39+
const sql = `SELECT "${config.privateSchema}"."check_billing_quota"($1, $2::uuid, $3) AS allowed`;
40+
const { rows } = await this.pool.query(sql, [meterSlug, entityId, amount]);
41+
return rows[0]?.allowed !== false;
42+
} catch {
43+
return true;
44+
}
45+
}
46+
47+
async recordUsage(entityId: string, meterSlug: string, amount: number, metadata: Record<string, unknown>, databaseId?: string): Promise<void> {
48+
const config = await this.load(databaseId);
49+
if (!config) return;
50+
try {
51+
const sql = `SELECT "${config.privateSchema}"."${config.recordUsageFunction}"($1, $2::uuid, $3, $4::jsonb)`;
52+
await this.pool.query(sql, [meterSlug, entityId, amount, JSON.stringify(metadata)]);
53+
} catch { /* non-fatal */ }
54+
}
55+
56+
invalidate(databaseId?: string): void {
57+
this.loader.billing.invalidate(databaseId);
1258
}
1359
}

0 commit comments

Comments
 (0)