From 46d62ee746f5bc5225ac5828c17110423217380d Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Fri, 31 Jul 2026 22:47:50 +0000 Subject: [PATCH 1/6] =?UTF-8?q?feat(pgpm):=20pgpm=20import=20=E2=80=94=20p?= =?UTF-8?q?gpm-itize=20a=20SQL=20dump=20through=20the=20dials=20pipeline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pgpm/cli/__tests__/import-e2e.test.ts | 209 ++++++++++++++++++++++ pgpm/cli/src/commands.ts | 2 + pgpm/cli/src/commands/import.ts | 192 ++++++++++++++++++++ pgpm/cli/src/commands/transform.ts | 2 +- pgpm/cli/src/utils/display.ts | 1 + pgpm/export/__tests__/dump-source.test.ts | 115 ++++++++++++ pgpm/export/src/dump-source.ts | 78 ++++++++ pgpm/export/src/index.ts | 2 + 8 files changed, 600 insertions(+), 1 deletion(-) create mode 100644 pgpm/cli/__tests__/import-e2e.test.ts create mode 100644 pgpm/cli/src/commands/import.ts create mode 100644 pgpm/export/__tests__/dump-source.test.ts create mode 100644 pgpm/export/src/dump-source.ts diff --git a/pgpm/cli/__tests__/import-e2e.test.ts b/pgpm/cli/__tests__/import-e2e.test.ts new file mode 100644 index 000000000..1d148b084 --- /dev/null +++ b/pgpm/cli/__tests__/import-e2e.test.ts @@ -0,0 +1,209 @@ +/** + * e2e for `pgpm import` against live Postgres: a pg_dump-style example dump + * (psql \\restrict/\\unrestrict meta-commands, session SET noise, OWNER TO) is + * imported at object granularity, deployed, verified, and reverted clean. + * + * PREREQUISITES: a running PostgreSQL instance via standard PG* env vars. + */ +import * as fs from 'fs'; +import * as path from 'path'; +import { teardownPgPools } from 'pg-cache'; + +import { CLIDeployTestFixture } from '../test-utils'; + +jest.setTimeout(120000); + +afterAll(async () => { + await teardownPgPools(); +}); + +const WS = 'import-ws'; +const MODULE_NAME = 'import-e2e'; + +/** Example pg_dump --schema-only output shape (plain format). */ +const DUMP_SQL = `-- +-- PostgreSQL database dump +-- + +\\restrict 8fKz01Lm3q + +-- Dumped from database version 18.0 +-- Dumped by pg_dump version 18.0 + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: imp_app; Type: SCHEMA; Schema: -; Owner: postgres +-- + +CREATE SCHEMA imp_app; + +ALTER SCHEMA imp_app OWNER TO postgres; + +-- +-- Name: users; Type: TABLE; Schema: imp_app; Owner: postgres +-- + +CREATE TABLE imp_app.users ( + id integer NOT NULL, + email text NOT NULL +); + +ALTER TABLE imp_app.users OWNER TO postgres; + +-- +-- Name: notes; Type: TABLE; Schema: imp_app; Owner: postgres +-- + +CREATE TABLE imp_app.notes ( + id integer NOT NULL, + user_id integer NOT NULL, + body text +); + +ALTER TABLE imp_app.notes OWNER TO postgres; + +-- +-- Name: users users_pkey; Type: CONSTRAINT; Schema: imp_app; Owner: postgres +-- + +ALTER TABLE ONLY imp_app.users + ADD CONSTRAINT users_pkey PRIMARY KEY (id); + +-- +-- Name: notes notes_pkey; Type: CONSTRAINT; Schema: imp_app; Owner: postgres +-- + +ALTER TABLE ONLY imp_app.notes + ADD CONSTRAINT notes_pkey PRIMARY KEY (id); + +-- +-- Name: notes notes_user_id_fkey; Type: FK CONSTRAINT; Schema: imp_app; Owner: postgres +-- + +ALTER TABLE ONLY imp_app.notes + ADD CONSTRAINT notes_user_id_fkey FOREIGN KEY (user_id) REFERENCES imp_app.users(id); + +-- +-- Name: notes_user_id_idx; Type: INDEX; Schema: imp_app; Owner: postgres +-- + +CREATE INDEX notes_user_id_idx ON imp_app.notes USING btree (user_id); + +\\unrestrict 8fKz01Lm3q + +-- +-- PostgreSQL database dump complete +-- +`; + +describe('pgpm import e2e', () => { + let fixture: CLIDeployTestFixture; + let wsDir: string; + + beforeAll(async () => { + fixture = new CLIDeployTestFixture(); + wsDir = path.join(fixture.tempFixtureDir, WS); + fs.mkdirSync(wsDir, { recursive: true }); + fs.writeFileSync(path.join(wsDir, 'pgpm.json'), '{\n "packages": [\n "*"\n ]\n}'); + fs.writeFileSync(path.join(wsDir, `${MODULE_NAME}.sql`), DUMP_SQL); + }); + + afterAll(async () => { + await fixture.cleanup(); + }); + + it('--dry-run prints derived changes without writing', async () => { + await fixture.runTerminalCommands( + ` + cd ${WS} + pgpm import ${MODULE_NAME}.sql --granularity object --dry-run + `, + {} + ); + expect(fs.existsSync(path.join(wsDir, MODULE_NAME))).toBe(false); + }); + + it('imports the dump into a complete pgpm module', async () => { + await fixture.runTerminalCommands( + ` + cd ${WS} + pgpm import ${MODULE_NAME}.sql --granularity object + `, + {} + ); + + const moduleDir = path.join(wsDir, MODULE_NAME); + expect(fs.existsSync(path.join(moduleDir, 'pgpm.plan'))).toBe(true); + expect(fs.existsSync(path.join(moduleDir, `${MODULE_NAME}.control`))).toBe(true); + + const plan = fs.readFileSync(path.join(moduleDir, 'pgpm.plan'), 'utf-8'); + expect(plan).toContain('schemas/imp_app/schema'); + expect(plan).toContain('tables/users/table'); + expect(plan).toContain('tables/notes/table'); + + // psql/session/ownership noise must not leak into deploy scripts + const deployDir = path.join(moduleDir, 'deploy'); + const walk = (dir: string): string[] => + fs.readdirSync(dir, { withFileTypes: true }).flatMap(entry => + entry.isDirectory() ? walk(path.join(dir, entry.name)) : [path.join(dir, entry.name)] + ); + for (const file of walk(deployDir)) { + const sql = fs.readFileSync(file, 'utf-8'); + expect(sql).not.toContain('\\restrict'); + expect(sql).not.toContain('OWNER TO'); + expect(sql).not.toContain('set_config'); + } + }); + + it('deploys, verifies, and reverts the imported module cleanly', async () => { + const testDb = await fixture.setupTestDatabase(); + + await fixture.runTerminalCommands( + ` + cd ${WS}/${MODULE_NAME} + pgpm deploy --database ${testDb.name} --package ${MODULE_NAME} --yes + `, + { database: testDb.name } + ); + + expect(await testDb.exists('schema', 'imp_app')).toBe(true); + expect(await testDb.exists('table', 'imp_app.users')).toBe(true); + expect(await testDb.exists('table', 'imp_app.notes')).toBe(true); + + const fk = await testDb.query( + `SELECT conname FROM pg_constraint WHERE conname = 'notes_user_id_fkey'` + ); + expect(fk.rows).toHaveLength(1); + + const idx = await testDb.query( + `SELECT indexname FROM pg_indexes WHERE schemaname = 'imp_app' AND indexname = 'notes_user_id_idx'` + ); + expect(idx.rows).toHaveLength(1); + + await fixture.runTerminalCommands( + ` + cd ${WS}/${MODULE_NAME} + pgpm verify --database ${testDb.name} --package ${MODULE_NAME} --yes + pgpm revert --database ${testDb.name} --package ${MODULE_NAME} --yes + `, + { database: testDb.name } + ); + + expect(await testDb.exists('schema', 'imp_app')).toBe(false); + const remaining = await testDb.query( + `SELECT COUNT(*)::int AS count FROM pgpm_migrate.changes WHERE package = $1`, + [MODULE_NAME] + ); + expect(remaining.rows[0].count).toBe(0); + }); +}); diff --git a/pgpm/cli/src/commands.ts b/pgpm/cli/src/commands.ts index b120752ca..dc12ba02d 100644 --- a/pgpm/cli/src/commands.ts +++ b/pgpm/cli/src/commands.ts @@ -14,6 +14,7 @@ import dump from './commands/dump'; import env from './commands/env'; import _export from './commands/export'; import extension from './commands/extension'; +import importDump from './commands/import'; import init from './commands/init'; import install from './commands/install'; import kill from './commands/kill'; @@ -113,6 +114,7 @@ export const createPgpmCommandMap = (skipPgTeardown: boolean = false): Record --granularity [OPTIONS] + + pgpm-itize a SQL dump (pg_dump plain format or any plain SQL file) through + the dials pipeline: psql meta-commands and session/ownership noise are + stripped, then the statements are re-projected at the requested granularity + with spec-derived change paths, graph-derived requires, and generated + revert/verify scripts — emitting a complete pgpm module. + +Options: + --help, -h Show this help message + --granularity Target granularity: atomic | object | consolidated (required) + --name Module name (default: dump file name without extension) + --partition Partition config (JSON: rules/defaultPackage/splitRiders) + splitting the import into multiple pgpm packages with + derived cross-package requires. + --naming