Skip to content

Commit 55460b8

Browse files
pyramationpyramation
authored
Merge pull request #1554 from constructive-io/feat/pgpm-export-dials
feat(export): granularity dial — wire pgpm export to the dials pipeline
2 parents 5a8642d + 5803afe commit 55460b8

8 files changed

Lines changed: 483 additions & 11 deletions

File tree

‎pgpm/cli/src/commands/export.ts‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { PgpmPackage } from '@pgpmjs/core';
2-
import { exportMigrations, exportGraphQL, GraphQLClient, graphqlRowToPostgresRow } from '@pgpmjs/export';
2+
import { exportMigrations, exportGraphQL, GraphQLClient, graphqlRowToPostgresRow, isExportGranularity, EXPORT_GRANULARITIES } from '@pgpmjs/export';
33
import { getEnvOptions } from '@pgpmjs/env';
44
import { getGitConfigInfo } from '@pgpmjs/types';
55
import { CLIOptions, Inquirerer } from 'inquirerer';
@@ -25,11 +25,17 @@ Options:
2525
--metaExtensionName <name> Meta extension name (default: svc)
2626
--exclude-categories <list> Comma-separated sql_actions categories to omit
2727
(e.g. security,permissions,auth,memberships). Works in SQL and GraphQL modes.
28+
--granularity <level> Restructure the exported database module through the
29+
dials pipeline: atomic | object | consolidated.
30+
Change paths are derived from the naming spec and
31+
requires from the statement graph. When omitted,
32+
sql_actions rows are exported unchanged.
2833
--cwd <directory> Working directory (default: current directory)
2934
3035
Examples:
3136
pgpm export Export migrations from selected database (SQL mode)
3237
pgpm export --exclude-categories security,permissions,auth,memberships
38+
pgpm export --granularity consolidated
3339
pgpm export --graphql-endpoint 'http://[::1]:3002/graphql' --migrate-endpoint 'http://[::1]:3000/graphql' --migrate-host db_migrate.localhost:3000
3440
`;
3541

@@ -64,6 +70,14 @@ export default async (
6470
? excludeCategoriesRaw
6571
: undefined;
6672

73+
const granularityRaw = argv.granularity;
74+
if (granularityRaw !== undefined && !isExportGranularity(granularityRaw)) {
75+
console.error(`Invalid --granularity "${granularityRaw}". Expected one of: ${EXPORT_GRANULARITIES.join(', ')}.`);
76+
prompter.close();
77+
return;
78+
}
79+
const granularity = granularityRaw;
80+
6781
if (graphqlEndpoint) {
6882
// =========================================================================
6983
// GraphQL export mode
@@ -180,7 +194,8 @@ export default async (
180194
prompter,
181195
argv,
182196
username,
183-
excludeCategories
197+
excludeCategories,
198+
granularity
184199
});
185200
} else {
186201
// =========================================================================
@@ -318,7 +333,8 @@ export default async (
318333
prompter,
319334
argv,
320335
username,
321-
excludeCategories
336+
excludeCategories,
337+
granularity
322338
});
323339
}
324340

‎pgpm/export/__tests__/export-granularity.test.ts‎

Lines changed: 328 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,328 @@
1+
/**
2+
* Granularity dial for `pgpm export` (Phase 4 of the three-dial plan).
3+
*
4+
* Seeds db_migrate.sql_actions with ad-hoc `migrate/*` change chains (the
5+
* pre-dials export shape), exports with `granularity` set, and verifies:
6+
* - change paths are derived from the naming spec (identityOf + pathFor),
7+
* not from the recorded ad-hoc names;
8+
* - `requires` are derived from the statement graph, not the hand-chained
9+
* deps recorded at action time;
10+
* - the exported module deploys cleanly (round-trip) at both the atomic and
11+
* consolidated granularities.
12+
*
13+
* PREREQUISITES: a running PostgreSQL instance via standard PG* env vars.
14+
*/
15+
16+
import { PgpmMigrate, PgpmPackage } from '@pgpmjs/core';
17+
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'fs';
18+
import { tmpdir } from 'os';
19+
import { join } from 'path';
20+
import type { PgConfig } from 'pg-env';
21+
import type { PgTestClient } from 'pgsql-test';
22+
import { getConnections, seed } from 'pgsql-test';
23+
24+
import { exportMigrations } from '../src/export-migrations';
25+
26+
jest.setTimeout(180000);
27+
28+
const DATABASE_ID = 'a1b2c3d4-e5f6-4708-b250-000000000009';
29+
30+
const SHIMS_SQL = `
31+
CREATE SCHEMA IF NOT EXISTS metaschema_public;
32+
CREATE SCHEMA IF NOT EXISTS db_migrate;
33+
34+
CREATE TABLE IF NOT EXISTS metaschema_public.database (
35+
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
36+
owner_id uuid,
37+
name text,
38+
hash uuid
39+
);
40+
41+
CREATE TABLE IF NOT EXISTS metaschema_public.schema (
42+
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
43+
database_id uuid REFERENCES metaschema_public.database(id),
44+
name text,
45+
schema_name text,
46+
description text
47+
);
48+
49+
CREATE TABLE IF NOT EXISTS db_migrate.sql_actions (
50+
id SERIAL PRIMARY KEY,
51+
name text,
52+
database_id uuid,
53+
deploy text,
54+
deps text[],
55+
payload json,
56+
content text,
57+
revert text,
58+
verify text,
59+
created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
60+
UNIQUE(database_id, deploy)
61+
);
62+
63+
INSERT INTO metaschema_public.database (id, owner_id, name, hash) VALUES
64+
('${DATABASE_ID}', '00000000-0000-0000-0000-000000000001', 'pets', 'f1e2d3c4-b5a6-5c2e-9a07-000000000009');
65+
66+
INSERT INTO metaschema_public.schema (id, database_id, name, schema_name, description) VALUES
67+
('aaaa0001-0000-0000-0000-000000000009', '${DATABASE_ID}', 'public', 'pets_public', 'Public-facing tables');
68+
69+
-- Ad-hoc migrate/* chain, atomic machine-emitted shape: bare CREATE TABLE
70+
-- plus one ALTER per column/constraint, hand-chained deps.
71+
INSERT INTO db_migrate.sql_actions (name, database_id, deploy, deps, content, revert, verify) VALUES
72+
(
73+
'create schema',
74+
'${DATABASE_ID}',
75+
'migrate/0001-schema',
76+
ARRAY[]::text[],
77+
'CREATE SCHEMA pets_public;',
78+
'DROP SCHEMA pets_public;',
79+
'SELECT 1;'
80+
),
81+
(
82+
'create owners',
83+
'${DATABASE_ID}',
84+
'migrate/0002-owners',
85+
ARRAY['migrate/0001-schema']::text[],
86+
E'CREATE TABLE pets_public.owners ();\nALTER TABLE pets_public.owners ADD COLUMN id uuid;\nALTER TABLE pets_public.owners ADD CONSTRAINT owners_pkey PRIMARY KEY (id);\nALTER TABLE pets_public.owners ADD COLUMN name text;',
87+
'DROP TABLE pets_public.owners;',
88+
'SELECT 1;'
89+
),
90+
(
91+
'create pets',
92+
'${DATABASE_ID}',
93+
'migrate/0003-pets',
94+
ARRAY['migrate/0002-owners']::text[],
95+
E'CREATE TABLE pets_public.pets ();\nALTER TABLE pets_public.pets ADD COLUMN id uuid;\nALTER TABLE pets_public.pets ADD COLUMN owner_id uuid;',
96+
'DROP TABLE pets_public.pets;',
97+
'SELECT 1;'
98+
),
99+
(
100+
'pets fk',
101+
'${DATABASE_ID}',
102+
'migrate/0004-pets-fk',
103+
ARRAY['migrate/0003-pets']::text[],
104+
'ALTER TABLE pets_public.pets ADD CONSTRAINT pets_owner_fk FOREIGN KEY (owner_id) REFERENCES pets_public.owners (id);',
105+
'ALTER TABLE pets_public.pets DROP CONSTRAINT pets_owner_fk;',
106+
'SELECT 1;'
107+
);
108+
`;
109+
110+
describe('Export granularity dial', () => {
111+
let tempDir: string;
112+
let exportWorkspaceDir: string;
113+
let pg: PgTestClient;
114+
let dbConfig: PgConfig;
115+
let teardown: () => Promise<void>;
116+
117+
const scaffoldModule = (name: string, description: string): string => {
118+
const moduleDir = join(exportWorkspaceDir, 'packages', name);
119+
mkdirSync(moduleDir, { recursive: true });
120+
writeFileSync(join(moduleDir, 'package.json'), JSON.stringify({
121+
name,
122+
version: '1.0.0',
123+
description
124+
}, null, 2));
125+
writeFileSync(join(moduleDir, `${name}.control`), `# ${name} extension
126+
comment = '${description}'
127+
default_version = '1.0.0'
128+
relocatable = false
129+
`);
130+
writeFileSync(join(moduleDir, 'pgpm.plan'), `%syntax-version=1.0.0
131+
%project=${name}
132+
%uri=https://github.com/test/${name}
133+
`);
134+
return moduleDir;
135+
};
136+
137+
const runExport = async (extensionName: string, granularity: 'atomic' | 'object' | 'consolidated'): Promise<void> => {
138+
scaffoldModule(extensionName, `Exported pets database schema (${granularity})`);
139+
scaffoldModule(`${extensionName}-svc`, `Exported pets service metadata (${granularity})`);
140+
141+
const project = new PgpmPackage(exportWorkspaceDir);
142+
143+
await exportMigrations({
144+
project,
145+
options: {
146+
pg: dbConfig
147+
},
148+
dbInfo: {
149+
dbname: dbConfig.database,
150+
databaseName: 'pets',
151+
database_ids: [DATABASE_ID]
152+
},
153+
author: 'test <test@test.local>',
154+
outdir: join(exportWorkspaceDir, 'packages'),
155+
schema_names: ['pets_public'],
156+
extensionName,
157+
extensionDesc: `Exported pets database schema (${granularity})`,
158+
metaExtensionName: `${extensionName}-svc`,
159+
metaExtensionDesc: `Exported pets service metadata (${granularity})`,
160+
granularity
161+
});
162+
};
163+
164+
beforeAll(async () => {
165+
tempDir = mkdtempSync(join(tmpdir(), 'pgpm-export-granularity-'));
166+
exportWorkspaceDir = join(tempDir, 'export-workspace');
167+
mkdirSync(join(exportWorkspaceDir, 'packages'), { recursive: true });
168+
mkdirSync(join(exportWorkspaceDir, 'extensions'), { recursive: true });
169+
writeFileSync(join(exportWorkspaceDir, 'pgpm.json'), JSON.stringify({
170+
packages: ['packages/*', 'extensions/*']
171+
}, null, 2));
172+
writeFileSync(join(exportWorkspaceDir, 'package.json'), JSON.stringify({
173+
name: 'export-granularity-workspace',
174+
version: '1.0.0',
175+
private: true
176+
}, null, 2));
177+
178+
({ pg, teardown } = await getConnections({}, [
179+
seed.fn(async ({ pg, config }) => {
180+
dbConfig = config;
181+
const migrate = new PgpmMigrate(config);
182+
await migrate.initialize();
183+
await pg.query(SHIMS_SQL);
184+
})
185+
]));
186+
});
187+
188+
afterAll(async () => {
189+
await teardown();
190+
rmSync(tempDir, { recursive: true, force: true });
191+
});
192+
193+
describe('consolidated granularity', () => {
194+
const EXTENSION_NAME = 'pets-consolidated';
195+
196+
beforeAll(async () => {
197+
await runExport(EXTENSION_NAME, 'consolidated');
198+
});
199+
200+
it('derives change paths from the naming spec, not the ad-hoc migrate/* names', () => {
201+
const planPath = join(exportWorkspaceDir, 'packages', EXTENSION_NAME, 'pgpm.plan');
202+
const plan = readFileSync(planPath, 'utf-8');
203+
204+
expect(plan).not.toContain('migrate/');
205+
expect(plan).toContain('schemas/pets_consolidated_public/schema');
206+
expect(plan).toContain('schemas/pets_consolidated_public/tables/owners/table');
207+
expect(plan).toContain('schemas/pets_consolidated_public/tables/pets/table');
208+
});
209+
210+
it('derives requires from the statement graph', () => {
211+
const planPath = join(exportWorkspaceDir, 'packages', EXTENSION_NAME, 'pgpm.plan');
212+
const plan = readFileSync(planPath, 'utf-8');
213+
214+
const tableLine = plan.split('\n').find(line => line.startsWith('schemas/pets_consolidated_public/tables/pets/table'));
215+
expect(tableLine).toBeDefined();
216+
// pets requires its schema; the FK on owners is folded or edged via the graph
217+
expect(tableLine).toContain('schemas/pets_consolidated_public/schema');
218+
});
219+
220+
it('consolidates the table changes (columns folded into CREATE TABLE)', () => {
221+
const tableSqlPath = join(
222+
exportWorkspaceDir, 'packages', EXTENSION_NAME,
223+
'deploy', 'schemas', 'pets_consolidated_public', 'tables', 'owners', 'table.sql'
224+
);
225+
expect(existsSync(tableSqlPath)).toBe(true);
226+
const sql = readFileSync(tableSqlPath, 'utf-8');
227+
expect(sql).toContain('-- Deploy: schemas/pets_consolidated_public/tables/owners/table');
228+
// columns are folded into the CREATE TABLE, not separate ALTERs
229+
expect(sql).toMatch(/CREATE TABLE pets_consolidated_public\.owners \([\s\S]*id uuid/);
230+
expect(sql).not.toMatch(/ALTER TABLE pets_consolidated_public\.owners ADD COLUMN/);
231+
});
232+
233+
it('deploys the exported module cleanly (round-trip)', async () => {
234+
const moduleDir = join(exportWorkspaceDir, 'packages', EXTENSION_NAME);
235+
const deployer = new PgpmMigrate(dbConfig);
236+
await deployer.deploy({ modulePath: moduleDir });
237+
238+
const tables = await pg.query(`
239+
SELECT table_name FROM information_schema.tables
240+
WHERE table_schema = 'pets_consolidated_public'
241+
ORDER BY table_name
242+
`);
243+
expect(tables.rows.map((r: any) => r.table_name)).toEqual(['owners', 'pets']);
244+
245+
const fk = await pg.query(`
246+
SELECT conname FROM pg_constraint
247+
WHERE conname = 'pets_owner_fk'
248+
`);
249+
expect(fk.rows).toHaveLength(1);
250+
});
251+
});
252+
253+
describe('atomic granularity', () => {
254+
const EXTENSION_NAME = 'pets-atomic';
255+
256+
beforeAll(async () => {
257+
await runExport(EXTENSION_NAME, 'atomic');
258+
});
259+
260+
it('derives spec paths while keeping the atomic statement shape', () => {
261+
const planPath = join(exportWorkspaceDir, 'packages', EXTENSION_NAME, 'pgpm.plan');
262+
const plan = readFileSync(planPath, 'utf-8');
263+
264+
expect(plan).not.toContain('migrate/');
265+
expect(plan).toContain('schemas/pets_atomic_public/tables/owners/table');
266+
267+
const tableSqlPath = join(
268+
exportWorkspaceDir, 'packages', EXTENSION_NAME,
269+
'deploy', 'schemas', 'pets_atomic_public', 'tables', 'owners', 'table.sql'
270+
);
271+
const sql = readFileSync(tableSqlPath, 'utf-8');
272+
// atomic keeps the bare CREATE TABLE + one ALTER per column/constraint
273+
expect(sql).toMatch(/CREATE TABLE pets_atomic_public\.owners \(\s*\)/);
274+
expect(sql).toMatch(/ALTER TABLE pets_atomic_public\.owners\s+ADD COLUMN id uuid/);
275+
});
276+
277+
it('deploys the exported module cleanly (round-trip)', async () => {
278+
const moduleDir = join(exportWorkspaceDir, 'packages', EXTENSION_NAME);
279+
const deployer = new PgpmMigrate(dbConfig);
280+
await deployer.deploy({ modulePath: moduleDir });
281+
282+
const tables = await pg.query(`
283+
SELECT table_name FROM information_schema.tables
284+
WHERE table_schema = 'pets_atomic_public'
285+
ORDER BY table_name
286+
`);
287+
expect(tables.rows.map((r: any) => r.table_name)).toEqual(['owners', 'pets']);
288+
});
289+
});
290+
291+
describe('no granularity (default)', () => {
292+
const EXTENSION_NAME = 'pets-passthrough';
293+
294+
beforeAll(async () => {
295+
scaffoldModule(EXTENSION_NAME, 'Exported pets database schema (passthrough)');
296+
scaffoldModule(`${EXTENSION_NAME}-svc`, 'Exported pets service metadata (passthrough)');
297+
298+
const project = new PgpmPackage(exportWorkspaceDir);
299+
300+
await exportMigrations({
301+
project,
302+
options: {
303+
pg: dbConfig
304+
},
305+
dbInfo: {
306+
dbname: dbConfig.database,
307+
databaseName: 'pets',
308+
database_ids: [DATABASE_ID]
309+
},
310+
author: 'test <test@test.local>',
311+
outdir: join(exportWorkspaceDir, 'packages'),
312+
schema_names: ['pets_public'],
313+
extensionName: EXTENSION_NAME,
314+
extensionDesc: 'Exported pets database schema (passthrough)',
315+
metaExtensionName: `${EXTENSION_NAME}-svc`,
316+
metaExtensionDesc: 'Exported pets service metadata (passthrough)'
317+
});
318+
});
319+
320+
it('keeps the recorded ad-hoc change names when granularity is omitted', () => {
321+
const planPath = join(exportWorkspaceDir, 'packages', EXTENSION_NAME, 'pgpm.plan');
322+
const plan = readFileSync(planPath, 'utf-8');
323+
324+
expect(plan).toContain('migrate/0001-schema');
325+
expect(plan).toContain('migrate/0002-owners');
326+
});
327+
});
328+
});

‎pgpm/export/package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@
4747
"@constructive-io/query-builder": "workspace:^",
4848
"@pgpmjs/core": "workspace:^",
4949
"@pgpmjs/migrate-client": "workspace:^",
50+
"@pgpmjs/naming-spec": "workspace:^",
51+
"@pgpmjs/transform": "workspace:^",
5052
"@pgpmjs/types": "workspace:^",
5153
"csv-to-pg": "workspace:^",
5254
"glob": "^13.0.6",

0 commit comments

Comments
 (0)