Skip to content

Commit 45b9191

Browse files
pyramationpyramation
authored
Merge pull request #1572 from constructive-io/feat/pgpm-diff
feat(pgpm): pgpm diff — identity-keyed semantic diff + migration generation
2 parents 8aa7f39 + 32f7626 commit 45b9191

12 files changed

Lines changed: 3609 additions & 7203 deletions

File tree

‎pgpm/cli/__tests__/diff-e2e.test.ts‎

Lines changed: 255 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,255 @@
1+
/**
2+
* e2e for `pgpm diff` against live Postgres.
3+
*
4+
* v1 -> v2 covers: new table, new column, dropped column, changed function
5+
* body, new policy, dropped index, changed constraint. The emitted migration
6+
* deploys on top of v1 to a catalog equivalent to v2, its verify passes, and
7+
* its revert returns to v1. Dial-invariance is proven by diffing v1 against
8+
* its consolidated transform (empty diff), and --verify exercises the oracle.
9+
*
10+
* PREREQUISITES: a running PostgreSQL instance via standard PG* env vars.
11+
*/
12+
import { diffCatalogSnapshots, loadDiffSideFromDisk, snapshotCatalog, withoutColumnOrder } from '@pgpmjs/export';
13+
import { diffChangeSets, loadModule } from '@pgpmjs/transform';
14+
import * as fs from 'fs';
15+
import * as path from 'path';
16+
import { teardownPgPools } from 'pg-cache';
17+
18+
import { CLIDeployTestFixture } from '../test-utils';
19+
20+
jest.setTimeout(180000);
21+
22+
afterAll(async () => {
23+
await teardownPgPools();
24+
});
25+
26+
const WS = 'diff-ws';
27+
28+
interface ChangeSpec {
29+
name: string;
30+
deps?: string[];
31+
deploy: string;
32+
}
33+
34+
const V1: ChangeSpec[] = [
35+
{ name: 'schemas/dfx/schema', deploy: 'CREATE SCHEMA dfx;' },
36+
{
37+
name: 'schemas/dfx/tables/products/table',
38+
deps: ['schemas/dfx/schema'],
39+
deploy: [
40+
'CREATE TABLE dfx.products (',
41+
' id serial PRIMARY KEY,',
42+
' price int NOT NULL,',
43+
' CONSTRAINT price_positive CHECK (price > 0),',
44+
' nickname text',
45+
');'
46+
].join('\n')
47+
},
48+
{
49+
name: 'schemas/dfx/procedures/product_count',
50+
deps: ['schemas/dfx/tables/products/table'],
51+
deploy:
52+
'CREATE FUNCTION dfx.product_count() RETURNS bigint LANGUAGE sql AS $$ SELECT count(*) FROM dfx.products $$;'
53+
},
54+
{
55+
name: 'schemas/dfx/tables/products/indexes/products_price_idx',
56+
deps: ['schemas/dfx/tables/products/table'],
57+
deploy: 'CREATE INDEX products_price_idx ON dfx.products (price);'
58+
}
59+
];
60+
61+
const V2: ChangeSpec[] = [
62+
{ name: 'schemas/dfx/schema', deploy: 'CREATE SCHEMA dfx;' },
63+
{
64+
// changed constraint, dropped column (nickname), new column (sku)
65+
name: 'schemas/dfx/tables/products/table',
66+
deps: ['schemas/dfx/schema'],
67+
deploy: [
68+
'CREATE TABLE dfx.products (',
69+
' id serial PRIMARY KEY,',
70+
' price int NOT NULL,',
71+
' CONSTRAINT price_positive CHECK (price >= 0),',
72+
' sku text',
73+
');'
74+
].join('\n')
75+
},
76+
{
77+
// changed function body
78+
name: 'schemas/dfx/procedures/product_count',
79+
deps: ['schemas/dfx/tables/products/table'],
80+
deploy:
81+
'CREATE FUNCTION dfx.product_count() RETURNS bigint LANGUAGE sql AS $$ SELECT count(1) FROM dfx.products $$;'
82+
},
83+
{
84+
// new table
85+
name: 'schemas/dfx/tables/orders/table',
86+
deps: ['schemas/dfx/tables/products/table'],
87+
deploy: 'CREATE TABLE dfx.orders (id serial PRIMARY KEY, product_id int NOT NULL REFERENCES dfx.products (id));'
88+
},
89+
{
90+
// new policy
91+
name: 'schemas/dfx/tables/products/policies/products_read',
92+
deps: ['schemas/dfx/tables/products/table'],
93+
deploy: [
94+
'ALTER TABLE dfx.products ENABLE ROW LEVEL SECURITY;',
95+
'CREATE POLICY products_read ON dfx.products FOR SELECT USING (true);'
96+
].join('\n')
97+
}
98+
// dropped index: products_price_idx exists only in v1
99+
];
100+
101+
const writeModule = (moduleDir: string, name: string, changes: ChangeSpec[]) => {
102+
fs.mkdirSync(moduleDir, { recursive: true });
103+
const planLines = changes.map((c, i) => {
104+
const deps = c.deps && c.deps.length > 0 ? ` [${c.deps.join(' ')}]` : '';
105+
return `${c.name}${deps} 2024-01-0${(i % 9) + 1}T00:00:00Z test <test@example.com> # add ${c.name}`;
106+
});
107+
fs.writeFileSync(
108+
path.join(moduleDir, 'pgpm.plan'),
109+
`%syntax-version=1.0.0\n%project=${name}\n%uri=https://github.com/test/${name}\n\n${planLines.join('\n')}\n`
110+
);
111+
fs.writeFileSync(
112+
path.join(moduleDir, `${name}.control`),
113+
`comment = 'Diff e2e module'\ndefault_version = '0.0.1'\nrelocatable = false\nsuperuser = false\n`
114+
);
115+
for (const c of changes) {
116+
const filePath = path.join(moduleDir, 'deploy', `${c.name}.sql`);
117+
fs.mkdirSync(path.dirname(filePath), { recursive: true });
118+
fs.writeFileSync(filePath, `-- Deploy ${c.name} to pg\n\nBEGIN;\n\n${c.deploy}\n\nCOMMIT;\n`);
119+
const revertPath = path.join(moduleDir, 'revert', `${c.name}.sql`);
120+
fs.mkdirSync(path.dirname(revertPath), { recursive: true });
121+
fs.writeFileSync(revertPath, `-- Revert ${c.name} from pg\n\nBEGIN;\n\n-- noop\n\nCOMMIT;\n`);
122+
const verifyPath = path.join(moduleDir, 'verify', `${c.name}.sql`);
123+
fs.mkdirSync(path.dirname(verifyPath), { recursive: true });
124+
fs.writeFileSync(verifyPath, `-- Verify ${c.name} on pg\n\nBEGIN;\n\nROLLBACK;\n`);
125+
}
126+
};
127+
128+
describe('pgpm diff e2e', () => {
129+
let fixture: CLIDeployTestFixture;
130+
let wsDir: string;
131+
let v1Db: any;
132+
let v2Db: any;
133+
134+
beforeAll(async () => {
135+
await loadModule();
136+
fixture = new CLIDeployTestFixture();
137+
wsDir = path.join(fixture.tempFixtureDir, WS);
138+
fs.mkdirSync(wsDir, { recursive: true });
139+
fs.writeFileSync(path.join(wsDir, 'pgpm.json'), '{\n "packages": [\n "*"\n ]\n}');
140+
writeModule(path.join(wsDir, 'diff-v1'), 'diff-v1', V1);
141+
writeModule(path.join(wsDir, 'diff-v2'), 'diff-v2', V2);
142+
143+
v1Db = await fixture.setupTestDatabase();
144+
v2Db = await fixture.setupTestDatabase();
145+
await fixture.runTerminalCommands(
146+
`
147+
cd ${WS}/diff-v1
148+
pgpm deploy --database ${v1Db.name} --package diff-v1 --yes
149+
`,
150+
{ database: v1Db.name }
151+
);
152+
await fixture.runTerminalCommands(
153+
`
154+
cd ${WS}/diff-v2
155+
pgpm deploy --database ${v2Db.name} --package diff-v2 --yes
156+
`,
157+
{ database: v2Db.name }
158+
);
159+
});
160+
161+
afterAll(async () => {
162+
await fixture.cleanup();
163+
});
164+
165+
it('diffs a module against itself as identical, dial-invariantly', async () => {
166+
const v1 = loadDiffSideFromDisk(path.join(wsDir, 'diff-v1'));
167+
expect(diffChangeSets(v1.changes, v1.changes).identical).toBe(true);
168+
169+
await fixture.runTerminalCommands(
170+
`
171+
cd ${WS}/diff-v1
172+
pgpm transform --granularity consolidated
173+
`,
174+
{}
175+
);
176+
const consolidated = loadDiffSideFromDisk(path.join(wsDir, 'diff-v1-consolidated'));
177+
expect(diffChangeSets(v1.changes, consolidated.changes).identical).toBe(true);
178+
});
179+
180+
it('emits a migration that deploys v1 to v2, verifies, and reverts back to v1', async () => {
181+
await fixture.runTerminalCommands(
182+
`
183+
cd ${WS}
184+
pgpm diff diff-v1 diff-v2 --emit-migration . --pkg diff-migration
185+
`,
186+
{}
187+
);
188+
const migrationDir = path.join(wsDir, 'diff-migration');
189+
expect(fs.existsSync(path.join(migrationDir, 'pgpm.plan'))).toBe(true);
190+
191+
// deploy v1 fresh, then the migration on top -> catalog equivalent to v2
192+
const testDb = await fixture.setupTestDatabase();
193+
await fixture.runTerminalCommands(
194+
`
195+
cd ${WS}/diff-v1
196+
pgpm deploy --database ${testDb.name} --package diff-v1 --yes
197+
cd ../diff-migration
198+
pgpm deploy --database ${testDb.name} --package diff-migration --yes
199+
`,
200+
{ database: testDb.name }
201+
);
202+
203+
const snapMigrated = await snapshotCatalog(testDb);
204+
const snapV2 = await snapshotCatalog(v2Db);
205+
// column order is physical: drop+add cannot reproduce fresh ordinals
206+
expect(diffCatalogSnapshots(withoutColumnOrder(snapMigrated), withoutColumnOrder(snapV2))).toEqual([]);
207+
208+
// the migration's own verify passes
209+
await fixture.runTerminalCommands(
210+
`
211+
cd ${WS}/diff-migration
212+
pgpm verify --database ${testDb.name} --package diff-migration --yes
213+
`,
214+
{ database: testDb.name }
215+
);
216+
217+
// revert returns to v1
218+
await fixture.runTerminalCommands(
219+
`
220+
cd ${WS}/diff-migration
221+
pgpm revert --database ${testDb.name} --package diff-migration --yes
222+
`,
223+
{ database: testDb.name }
224+
);
225+
const snapReverted = await snapshotCatalog(testDb);
226+
const snapV1 = await snapshotCatalog(v1Db);
227+
expect(diffCatalogSnapshots(withoutColumnOrder(snapReverted), withoutColumnOrder(snapV1))).toEqual([]);
228+
});
229+
230+
it('--verify proves the emitted migration is a catalog-level oracle match', async () => {
231+
await fixture.runTerminalCommands(
232+
`
233+
cd ${WS}
234+
pgpm diff diff-v1 diff-v2 --verify
235+
`,
236+
{}
237+
);
238+
});
239+
240+
it('diffs a module against a raw .sql file', async () => {
241+
const flat = V2.map(c => c.deploy).join('\n\n');
242+
fs.writeFileSync(path.join(wsDir, 'v2-flat.sql'), flat);
243+
const v1 = loadDiffSideFromDisk(path.join(wsDir, 'diff-v1'));
244+
const v2File = loadDiffSideFromDisk(path.join(wsDir, 'v2-flat.sql'));
245+
const v2Module = loadDiffSideFromDisk(path.join(wsDir, 'diff-v2'));
246+
247+
// module-vs-file and module-vs-module agree
248+
expect(diffChangeSets(v2Module.changes, v2File.changes).identical).toBe(true);
249+
const result = diffChangeSets(v1.changes, v2File.changes);
250+
expect(result.identical).toBe(false);
251+
expect(result.objects.map(o => `${o.delta} ${o.path}`).sort()).toEqual(
252+
diffChangeSets(v1.changes, v2Module.changes).objects.map(o => `${o.delta} ${o.path}`).sort()
253+
);
254+
});
255+
});

‎pgpm/cli/src/commands.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import analyze from './commands/analyze';
88
import cache from './commands/cache';
99
import clear from './commands/clear';
1010
import deploy from './commands/deploy';
11+
import diff from './commands/diff';
1112
import docker from './commands/docker';
1213
import doctor from './commands/doctor';
1314
import dump from './commands/dump';
@@ -90,6 +91,7 @@ export const createPgpmCommandMap = (skipPgTeardown: boolean = false): Record<st
9091
'admin-users': pgt(adminUsers),
9192
clear: pgt(clear),
9293
deploy: pgt(deploy),
94+
diff: pgt(diff),
9395
docker,
9496
doctor,
9597
dump: pgt(dump),

0 commit comments

Comments
 (0)