diff --git a/pgpm/cli/__tests__/fixtures/import/dump.sql b/pgpm/cli/__tests__/fixtures/import/dump.sql new file mode 100644 index 000000000..62054e85f --- /dev/null +++ b/pgpm/cli/__tests__/fixtures/import/dump.sql @@ -0,0 +1,363 @@ +-- +-- PostgreSQL database dump +-- + +\restrict Rhcdwy2WGDhj9Q1hKGNyv7beFEsbwg6bGgxjVYIVY9511ICWws8BXfu2kIXtqFa + +-- Dumped from database version 18.4 +-- Dumped by pg_dump version 18.4 + +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET transaction_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: imp_app; Type: SCHEMA; Schema: -; Owner: postgres +-- + +CREATE SCHEMA imp_app; + + +ALTER SCHEMA imp_app OWNER TO postgres; + +-- +-- Name: SCHEMA imp_app; Type: COMMENT; Schema: -; Owner: postgres +-- + +COMMENT ON SCHEMA imp_app IS 'Application schema'; + + +-- +-- Name: imp_audit; Type: SCHEMA; Schema: -; Owner: postgres +-- + +CREATE SCHEMA imp_audit; + + +ALTER SCHEMA imp_audit OWNER TO postgres; + +-- +-- Name: pgcrypto; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public; + + +-- +-- Name: EXTENSION pgcrypto; Type: COMMENT; Schema: -; Owner: +-- + +COMMENT ON EXTENSION pgcrypto IS 'cryptographic functions'; + + +-- +-- Name: touch(); Type: FUNCTION; Schema: imp_app; Owner: postgres +-- + +CREATE FUNCTION imp_app.touch() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN + NEW.created_at := now(); + RETURN NEW; +END; +$$; + + +ALTER FUNCTION imp_app.touch() OWNER TO postgres; + +-- +-- Name: FUNCTION touch(); Type: COMMENT; Schema: imp_app; Owner: postgres +-- + +COMMENT ON FUNCTION imp_app.touch() IS 'touch trigger fn'; + + +-- +-- Name: order_seq; Type: SEQUENCE; Schema: imp_app; Owner: postgres +-- + +CREATE SEQUENCE imp_app.order_seq + START WITH 100 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER SEQUENCE imp_app.order_seq OWNER TO postgres; + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: orders; Type: TABLE; Schema: imp_app; Owner: postgres +-- + +CREATE TABLE imp_app.orders ( + id integer DEFAULT nextval('imp_app.order_seq'::regclass) NOT NULL, + user_id integer NOT NULL, + note text +); + + +ALTER TABLE imp_app.orders OWNER TO postgres; + +-- +-- Name: users; Type: TABLE; Schema: imp_app; Owner: postgres +-- + +CREATE TABLE imp_app.users ( + id integer NOT NULL, + email text NOT NULL, + created_at timestamp with time zone DEFAULT now() +); + + +ALTER TABLE imp_app.users OWNER TO postgres; + +-- +-- Name: TABLE users; Type: COMMENT; Schema: imp_app; Owner: postgres +-- + +COMMENT ON TABLE imp_app.users IS 'App users'; + + +-- +-- Name: COLUMN users.email; Type: COMMENT; Schema: imp_app; Owner: postgres +-- + +COMMENT ON COLUMN imp_app.users.email IS 'Login email'; + + +-- +-- Name: users_id_seq; Type: SEQUENCE; Schema: imp_app; Owner: postgres +-- + +CREATE SEQUENCE imp_app.users_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER SEQUENCE imp_app.users_id_seq OWNER TO postgres; + +-- +-- Name: users_id_seq; Type: SEQUENCE OWNED BY; Schema: imp_app; Owner: postgres +-- + +ALTER SEQUENCE imp_app.users_id_seq OWNED BY imp_app.users.id; + + +-- +-- Name: events; Type: TABLE; Schema: imp_audit; Owner: postgres +-- + +CREATE TABLE imp_audit.events ( + id integer NOT NULL, + user_id integer, + detail text +); + + +ALTER TABLE imp_audit.events OWNER TO postgres; + +-- +-- Name: events_id_seq; Type: SEQUENCE; Schema: imp_audit; Owner: postgres +-- + +CREATE SEQUENCE imp_audit.events_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER SEQUENCE imp_audit.events_id_seq OWNER TO postgres; + +-- +-- Name: events_id_seq; Type: SEQUENCE OWNED BY; Schema: imp_audit; Owner: postgres +-- + +ALTER SEQUENCE imp_audit.events_id_seq OWNED BY imp_audit.events.id; + + +-- +-- Name: users id; Type: DEFAULT; Schema: imp_app; Owner: postgres +-- + +ALTER TABLE ONLY imp_app.users ALTER COLUMN id SET DEFAULT nextval('imp_app.users_id_seq'::regclass); + + +-- +-- Name: events id; Type: DEFAULT; Schema: imp_audit; Owner: postgres +-- + +ALTER TABLE ONLY imp_audit.events ALTER COLUMN id SET DEFAULT nextval('imp_audit.events_id_seq'::regclass); + + +-- +-- Data for Name: orders; Type: TABLE DATA; Schema: imp_app; Owner: postgres +-- + +COPY imp_app.orders (id, user_id, note) FROM stdin; +100 1 first\torder +101 2 \N +\. + + +-- +-- Data for Name: users; Type: TABLE DATA; Schema: imp_app; Owner: postgres +-- + +COPY imp_app.users (id, email, created_at) FROM stdin; +1 a@example.com 2026-07-31 22:53:49.879981+00 +2 b@example.com 2026-07-31 22:53:49.879981+00 +\. + + +-- +-- Data for Name: events; Type: TABLE DATA; Schema: imp_audit; Owner: postgres +-- + +COPY imp_audit.events (id, user_id, detail) FROM stdin; +\. + + +-- +-- Name: order_seq; Type: SEQUENCE SET; Schema: imp_app; Owner: postgres +-- + +SELECT pg_catalog.setval('imp_app.order_seq', 101, true); + + +-- +-- Name: users_id_seq; Type: SEQUENCE SET; Schema: imp_app; Owner: postgres +-- + +SELECT pg_catalog.setval('imp_app.users_id_seq', 2, true); + + +-- +-- Name: events_id_seq; Type: SEQUENCE SET; Schema: imp_audit; Owner: postgres +-- + +SELECT pg_catalog.setval('imp_audit.events_id_seq', 1, false); + + +-- +-- Name: orders orders_pkey; Type: CONSTRAINT; Schema: imp_app; Owner: postgres +-- + +ALTER TABLE ONLY imp_app.orders + ADD CONSTRAINT orders_pkey PRIMARY KEY (id); + + +-- +-- Name: users users_email_key; Type: CONSTRAINT; Schema: imp_app; Owner: postgres +-- + +ALTER TABLE ONLY imp_app.users + ADD CONSTRAINT users_email_key UNIQUE (email); + + +-- +-- Name: users users_pkey; Type: CONSTRAINT; Schema: imp_app; Owner: postgres +-- + +ALTER TABLE ONLY imp_app.users + ADD CONSTRAINT users_pkey PRIMARY KEY (id); + + +-- +-- Name: events events_pkey; Type: CONSTRAINT; Schema: imp_audit; Owner: postgres +-- + +ALTER TABLE ONLY imp_audit.events + ADD CONSTRAINT events_pkey PRIMARY KEY (id); + + +-- +-- Name: orders_user_idx; Type: INDEX; Schema: imp_app; Owner: postgres +-- + +CREATE INDEX orders_user_idx ON imp_app.orders USING btree (user_id); + + +-- +-- Name: users users_touch; Type: TRIGGER; Schema: imp_app; Owner: postgres +-- + +CREATE TRIGGER users_touch BEFORE UPDATE ON imp_app.users FOR EACH ROW EXECUTE FUNCTION imp_app.touch(); + + +-- +-- Name: orders orders_user_fk; Type: FK CONSTRAINT; Schema: imp_app; Owner: postgres +-- + +ALTER TABLE ONLY imp_app.orders + ADD CONSTRAINT orders_user_fk FOREIGN KEY (user_id) REFERENCES imp_app.users(id); + + +-- +-- Name: events events_user_id_fkey; Type: FK CONSTRAINT; Schema: imp_audit; Owner: postgres +-- + +ALTER TABLE ONLY imp_audit.events + ADD CONSTRAINT events_user_id_fkey FOREIGN KEY (user_id) REFERENCES imp_app.users(id); + + +-- +-- Name: users; Type: ROW SECURITY; Schema: imp_app; Owner: postgres +-- + +ALTER TABLE imp_app.users ENABLE ROW LEVEL SECURITY; + +-- +-- Name: users users_select; Type: POLICY; Schema: imp_app; Owner: postgres +-- + +CREATE POLICY users_select ON imp_app.users FOR SELECT USING (true); + + +-- +-- Name: SCHEMA imp_app; Type: ACL; Schema: -; Owner: postgres +-- + +GRANT USAGE ON SCHEMA imp_app TO PUBLIC; + + +-- +-- Name: TABLE orders; Type: ACL; Schema: imp_app; Owner: postgres +-- + +GRANT SELECT,INSERT ON TABLE imp_app.orders TO PUBLIC; + + +-- +-- Name: TABLE users; Type: ACL; Schema: imp_app; Owner: postgres +-- + +GRANT SELECT ON TABLE imp_app.users TO PUBLIC; + + +-- +-- PostgreSQL database dump complete +-- + +\unrestrict Rhcdwy2WGDhj9Q1hKGNyv7beFEsbwg6bGgxjVYIVY9511ICWws8BXfu2kIXtqFa + diff --git a/pgpm/cli/__tests__/import-e2e.test.ts b/pgpm/cli/__tests__/import-e2e.test.ts new file mode 100644 index 000000000..e686d63b5 --- /dev/null +++ b/pgpm/cli/__tests__/import-e2e.test.ts @@ -0,0 +1,197 @@ +/** + * e2e for `pgpm import` against live Postgres. + * + * Fixture: a real `pg_dump` output (committed at fixtures/import/dump.sql) + * with its preamble (`\restrict`, SET ..., set_config), two schemas, tables + * with a late-FK `ALTER TABLE ONLY ... ADD CONSTRAINT`, serial sequences + + * OWNED BY, a standalone sequence, function, trigger, policy, grants, + * comments, `CREATE EXTENSION pgcrypto`, and COPY data blocks. + * + * The source schema is applied directly to a scratch DB; the dump is + * imported at each granularity, deployed into a fresh DB, and the catalogs + * are asserted equivalent. Then verify passes and revert leaves the DB + * clean. `--with-data` additionally proves the seed fixtures land. + * + * PREREQUISITES: a running PostgreSQL instance via standard PG* env vars. + */ +import { copyBlockToInsert, diffCatalogSnapshots, preprocessDumpText, snapshotCatalog } from '@pgpmjs/export'; +import * as fs from 'fs'; +import * as path from 'path'; +import { teardownPgPools } from 'pg-cache'; + +import { CLIDeployTestFixture } from '../test-utils'; + +jest.setTimeout(180000); + +afterAll(async () => { + await teardownPgPools(); +}); + +const WS = 'import-ws'; +const DUMP_PATH = path.join(__dirname, 'fixtures', 'import', 'dump.sql'); + +describe('pgpm import e2e', () => { + let fixture: CLIDeployTestFixture; + let originalDb: any; + let wsDir: string; + + beforeAll(async () => { + fixture = new CLIDeployTestFixture(); + wsDir = path.join(fixture.tempFixtureDir, WS); + fs.mkdirSync(wsDir, { recursive: true }); + fs.writeFileSync(path.join(wsDir, 'pgpm.json'), '{\n "packages": [\n "*"\n ]\n}'); + fs.copyFileSync(DUMP_PATH, path.join(wsDir, 'dump.sql')); + + // Apply the source schema directly so we can assert catalog equivalence. + originalDb = await fixture.setupTestDatabase(); + const { sql, copyBlocks } = preprocessDumpText(fs.readFileSync(DUMP_PATH, 'utf-8')); + await originalDb.query(sql); + // COPY blocks appear in pg_dump's data order (not FK order), so load + // them the way pg_restore would: with FK triggers disabled. + const inserts = copyBlocks.map(copyBlockToInsert).filter(Boolean).join('\n'); + if (inserts) { + await originalDb.query( + `BEGIN;\nSET LOCAL session_replication_role = replica;\n${inserts}\nCOMMIT;` + ); + } + }); + + afterAll(async () => { + await fixture.cleanup(); + }); + + it('--dry-run prints the plan without writing anything', async () => { + await fixture.runTerminalCommands( + ` + cd ${WS} + pgpm import dump.sql --pkg imp-dry --dry-run + `, + {} + ); + expect(fs.existsSync(path.join(wsDir, 'imp-dry'))).toBe(false); + }); + + it.each(['atomic', 'object', 'consolidated'] as const)( + 'import --granularity %s deploys with an equivalent catalog, verifies, and reverts clean', + async granularity => { + const pkg = `imp-${granularity}`; + await fixture.runTerminalCommands( + ` + cd ${WS} + pgpm import dump.sql --pkg ${pkg} --granularity ${granularity} + `, + {} + ); + + const outDir = path.join(wsDir, pkg); + expect(fs.existsSync(path.join(outDir, 'pgpm.plan'))).toBe(true); + const control = fs.readFileSync(path.join(outDir, `${pkg}.control`), 'utf-8'); + expect(control).toMatch(/requires = '.*pgcrypto.*'/); + + const testDb = await fixture.setupTestDatabase(); + await fixture.runTerminalCommands( + ` + cd ${WS}/${pkg} + pgpm deploy --database ${testDb.name} --package ${pkg} --yes + `, + { database: testDb.name } + ); + + const snapOriginal = await snapshotCatalog(originalDb); + const snapImported = await snapshotCatalog(testDb); + expect(diffCatalogSnapshots(snapOriginal, snapImported)).toEqual([]); + + await fixture.runTerminalCommands( + ` + cd ${WS}/${pkg} + pgpm verify --database ${testDb.name} --package ${pkg} --yes + pgpm revert --database ${testDb.name} --package ${pkg} --yes + `, + { database: testDb.name } + ); + + expect(await testDb.exists('schema', 'imp_app')).toBe(false); + expect(await testDb.exists('schema', 'imp_audit')).toBe(false); + const remaining = await testDb.query( + `SELECT COUNT(*)::int AS count FROM pgpm_migrate.changes WHERE package = $1`, + [pkg] + ); + expect(remaining.rows[0].count).toBe(0); + } + ); + + it('--partition splits the import into multiple packages with cross-package requires', async () => { + fs.writeFileSync( + path.join(wsDir, 'partition.json'), + JSON.stringify({ + defaultPackage: 'imp-core', + rules: [{ package: 'imp-audit-pkg', select: [{ schema: 'imp_audit' }] }] + }) + ); + + await fixture.runTerminalCommands( + ` + cd ${WS} + pgpm import dump.sql --pkg imp-part --partition partition.json + `, + {} + ); + + expect(fs.existsSync(path.join(wsDir, 'imp-core', 'pgpm.plan'))).toBe(true); + expect(fs.existsSync(path.join(wsDir, 'imp-audit-pkg', 'pgpm.plan'))).toBe(true); + const auditPlan = fs.readFileSync(path.join(wsDir, 'imp-audit-pkg', 'pgpm.plan'), 'utf-8'); + expect(auditPlan).toContain('schemas/imp_audit'); + const corePlan = fs.readFileSync(path.join(wsDir, 'imp-core', 'pgpm.plan'), 'utf-8'); + expect(corePlan).not.toContain('schemas/imp_audit/'); + // extension requires + cross-package requires land in the .control files + const coreControl = fs.readFileSync(path.join(wsDir, 'imp-core', 'imp-core.control'), 'utf-8'); + expect(coreControl).toMatch(/requires = '.*pgcrypto.*'/); + const auditControl = fs.readFileSync(path.join(wsDir, 'imp-audit-pkg', 'imp-audit-pkg.control'), 'utf-8'); + expect(auditControl).toMatch(/requires = '.*imp-core.*'/); + + const testDb = await fixture.setupTestDatabase(); + await fixture.runTerminalCommands( + ` + cd ${WS}/imp-core + pgpm deploy --database ${testDb.name} --package imp-core --yes + cd ../imp-audit-pkg + pgpm deploy --database ${testDb.name} --package imp-audit-pkg --yes + `, + { database: testDb.name } + ); + const snapOriginal = await snapshotCatalog(originalDb); + const snapPartitioned = await snapshotCatalog(testDb); + expect(diffCatalogSnapshots(snapOriginal, snapPartitioned)).toEqual([]); + }); + + it('--with-data deploys COPY/INSERT data as seed fixtures', async () => { + await fixture.runTerminalCommands( + ` + cd ${WS} + pgpm import dump.sql --pkg imp-data --with-data + `, + {} + ); + + const plan = fs.readFileSync(path.join(wsDir, 'imp-data', 'pgpm.plan'), 'utf-8'); + expect(plan).toContain('schemas/imp_app/tables/users/fixtures/seed'); + expect(plan).toContain('schemas/imp_app/tables/orders/fixtures/seed'); + + const testDb = await fixture.setupTestDatabase(); + await fixture.runTerminalCommands( + ` + cd ${WS}/imp-data + pgpm deploy --database ${testDb.name} --package imp-data --yes + `, + { database: testDb.name } + ); + + const users = await testDb.query('SELECT COUNT(*)::int AS count FROM imp_app.users'); + expect(users.rows[0].count).toBe(2); + const orders = await testDb.query('SELECT note FROM imp_app.orders ORDER BY id'); + expect(orders.rows.map((r: any) => r.note)).toEqual(['first\torder', null]); + // setval rode the sequence changes: nextval continues after the seeds + const next = await testDb.query("SELECT nextval('imp_app.order_seq')::int AS v"); + expect(next.rows[0].v).toBeGreaterThan(101); + }); +}); diff --git a/pgpm/cli/src/commands.ts b/pgpm/cli/src/commands.ts index c2725544d..44bc9a571 100644 --- a/pgpm/cli/src/commands.ts +++ b/pgpm/cli/src/commands.ts @@ -15,6 +15,7 @@ import dump from './commands/dump'; import env from './commands/env'; import _export from './commands/export'; import extension from './commands/extension'; +import _import from './commands/import'; import init from './commands/init'; import install from './commands/install'; import kill from './commands/kill'; @@ -58,6 +59,7 @@ const ENGINE_EXEMPT_COMMANDS = new Set([ 'doctor', 'env', 'extension', + 'import', 'init', 'install', 'package', @@ -104,6 +106,7 @@ export const createPgpmCommandMap = (skipPgTeardown: boolean = false): Record --pkg [OPTIONS] + + pgpm-itize an arbitrary SQL dump (e.g. pg_dump --schema-only output): parse + it, classify every statement, and emit a complete deployable pgpm module — + module dir, pgpm.plan, and deploy/revert/verify trees with spec-derived + change paths, graph-derived requires, and generated revert/verify scripts. + + Directory inputs concatenate their .sql files in sorted (lexicographic) + filename order. + + Dump handling: + - pg_dump preamble noise (SET ..., SELECT set_config(...), psql backslash + commands) is skipped; COMMENT ON and GRANT statements are kept, attached + to their host object's change. + - CREATE EXTENSION statements become .control requires (pgpm owns + extension creation at deploy time). + - COPY ... FROM stdin blocks and INSERTs are skipped with a warning unless + --with-data is passed, which emits them as seed fixture changes (COPY + data is converted to INSERTs). + - Statements that classify to no object land in a misc/statements change + with a warning — never dropped silently. + +Options: + --help, -h Show this help message + --pkg Module name for the generated package (required) + --granularity Granularity dial: atomic | object | consolidated + (default: object) + --naming