refactor(pgpm): split the dials sources into @pgpmjs/import and @pgpmjs/diff; engine glue moves to @pgpmjs/transform

pgpm/cli/__tests__/diff-e2e.test.ts

@@ -9,8 +9,8 @@
  *
  * PREREQUISITES: a running PostgreSQL instance via standard PG* env vars.
  */
-import { diffCatalogSnapshots, loadDiffSideFromDisk, snapshotCatalog, withoutColumnOrder } from '@pgpmjs/export';
-import { diffChangeSets, loadModule } from '@pgpmjs/transform';
+import { loadDiffSideFromDisk } from '@pgpmjs/diff';
+import { diffCatalogSnapshots, diffChangeSets, loadModule, snapshotCatalog, withoutColumnOrder } from '@pgpmjs/transform';
 import * as fs from 'fs';
 import * as path from 'path';
 import { teardownPgPools } from 'pg-cache';

pgpm/cli/__tests__/import-e2e.test.ts

@@ -14,7 +14,8 @@
  *
  * PREREQUISITES: a running PostgreSQL instance via standard PG* env vars.
  */
-import { copyBlockToInsert, diffCatalogSnapshots, preprocessDumpText, snapshotCatalog } from '@pgpmjs/export';
+import { copyBlockToInsert, preprocessDumpText } from '@pgpmjs/import';
+import { diffCatalogSnapshots, snapshotCatalog } from '@pgpmjs/transform';
 import * as fs from 'fs';
 import * as path from 'path';
 import { teardownPgPools } from 'pg-cache';

pgpm/cli/__tests__/transform-e2e.test.ts

@@ -9,7 +9,7 @@
  *
  * PREREQUISITES: a running PostgreSQL instance via standard PG* env vars.
  */
-import { diffCatalogSnapshots, snapshotCatalog } from '@pgpmjs/export';
+import { diffCatalogSnapshots, snapshotCatalog } from '@pgpmjs/transform';
 import * as fs from 'fs';
 import * as path from 'path';
 import { teardownPgPools } from 'pg-cache';

pgpm/cli/package.json

@@ -47,8 +47,10 @@
   "dependencies": {
     "@inquirerer/utils": "^3.3.9",
     "@pgpmjs/core": "workspace:^",
+    "@pgpmjs/diff": "workspace:^",
     "@pgpmjs/env": "workspace:^",
     "@pgpmjs/export": "workspace:^",
+    "@pgpmjs/import": "workspace:^",
     "@pgpmjs/logger": "workspace:^",
     "@pgpmjs/slice": "workspace:^",
     "@pgpmjs/transform": "workspace:^",

pgpm/cli/src/commands/diff.ts

@@ -2,21 +2,25 @@ import { PgpmMigrate, PgpmRow, SqlWriteOptions, writePgpmFiles, writePgpmPlan }
 import {
   deltaChangesToRows,
   DiffSide,
+  loadDiffSideFromDisk,
+  sqlToDiffChanges
+} from '@pgpmjs/diff';
+import { Logger } from '@pgpmjs/logger';
+import {
   diffCatalogSnapshots,
+  diffChangeSets,
   EXPORT_GRANULARITIES,
   ExportGranularity,
   isExportGranularity,
-  loadDiffSideFromDisk,
-  resolveDiffSideKind,
+  loadModule,
+  SemanticDiffResult,
+  SemanticObjectDiff,
   snapshotCatalog,
-  sqlToDiffChanges,
   withoutColumnOrder
-} from '@pgpmjs/export';
-import { Logger } from '@pgpmjs/logger';
-import { diffChangeSets, loadModule, SemanticDiffResult, SemanticObjectDiff } from '@pgpmjs/transform';
+} from '@pgpmjs/transform';
 import { spawn } from 'child_process';
 import * as fs from 'fs';
-import { CLIOptions, cliExitWithError, extractFirst, Inquirerer, ParsedArgs } from 'inquirerer';
+import { cliExitWithError, CLIOptions, extractFirst, Inquirerer, ParsedArgs } from 'inquirerer';
 import * as os from 'os';
 import * as path from 'path';
 import { getPgPool } from 'pg-cache';

pgpm/cli/src/commands/export.ts

@@ -1,6 +1,7 @@
 import { PgpmPackage } from '@pgpmjs/core';
 import { getEnvOptions } from '@pgpmjs/env';
-import { EXPORT_GRANULARITIES, exportGraphQL, exportMigrations, GraphQLClient, graphqlRowToPostgresRow, isExportGranularity, parsePartitionConfig, PartitionConfig } from '@pgpmjs/export';
+import { exportGraphQL, exportMigrations, GraphQLClient, graphqlRowToPostgresRow } from '@pgpmjs/export';
+import { EXPORT_GRANULARITIES, isExportGranularity, parsePartitionConfig, PartitionConfig } from '@pgpmjs/transform';
 import { getGitConfigInfo } from '@pgpmjs/types';
 import { CLIOptions, Inquirerer } from 'inquirerer';
 import { resolve } from 'path';

pgpm/cli/src/commands/import.ts

@@ -1,17 +1,19 @@
 import {
   dumpCompatibilityWarnings,
+  importDumpRows,
+  linkTextualDeps,
+  loadDumpSource
+} from '@pgpmjs/import';
+import { Logger } from '@pgpmjs/logger';
+import {
   EXPORT_GRANULARITIES,
   ExportGranularity,
-  importDumpRows,
   isExportGranularity,
-  linkTextualDeps,
-  loadDumpSource,
   parsePartitionConfig,
   PartitionConfig,
+  PartitionCycleError,
   partitionExportRows
-} from '@pgpmjs/export';
-import { Logger } from '@pgpmjs/logger';
-import { PartitionCycleError } from '@pgpmjs/transform';
+} from '@pgpmjs/transform';
 import { cliExitWithError, CLIOptions, Inquirerer, ParsedArgs } from 'inquirerer';
 import * as path from 'path';
 

pgpm/cli/src/commands/transform.ts

@@ -1,4 +1,5 @@
 import { PgpmMigrate, PgpmPackage, PgpmRow } from '@pgpmjs/core';
+import { Logger } from '@pgpmjs/logger';
 import {
   diffCatalogSnapshots,
   EXPORT_GRANULARITIES,
@@ -7,13 +8,12 @@ import {
   loadModuleSource,
   parsePartitionConfig,
   PartitionConfig,
+  PartitionCycleError,
   PartitionedPackageRows,
   partitionExportRows,
   restructureExportRows,
   snapshotCatalog
-} from '@pgpmjs/export';
-import { Logger } from '@pgpmjs/logger';
-import { PartitionCycleError } from '@pgpmjs/transform';
+} from '@pgpmjs/transform';
 import * as fs from 'fs';
 import { cliExitWithError, CLIOptions, Inquirerer, ParsedArgs } from 'inquirerer';
 import * as path from 'path';

pgpm/diff/README.md

@@ -0,0 +1,7 @@
+# @pgpmjs/diff
+
+`pgpm diff` engine: identity-keyed semantic schema diff between any two SQL sources — pgpm modules, SQL dumps, or live databases (dumped via `pg_dump` by the CLI).
+
+Every side normalizes to the same `DiffInputChange[]` seam consumed by the semantic diff driver in `@pgpmjs/transform`, so the comparison is source- and dial-invariant. `deltaChangesToRows` lifts the resulting deltas back onto deployable `PgpmRow`s.
+
+Used by the `pgpm diff` CLI command (`pgpm/cli`). Sibling front doors: `@pgpmjs/export` (live database), `@pgpmjs/import` (dumps).

pgpm/export/__tests__/diff-source.test.ts → pgpm/diff/__tests__/diff-source.test.ts

@@ -55,8 +55,8 @@ describe('stripDumpPreamble', () => {
       '-- PostgreSQL database dump',
       '--',
       '\\restrict abcdef',
-      "SET statement_timeout = 0;",
-      "SET lock_timeout = 0;",
+      'SET statement_timeout = 0;',
+      'SET lock_timeout = 0;',
       "SELECT pg_catalog.set_config('search_path', '', false);",
       "SET default_tablespace = '';",
       '\\connect mydb',

pgpm/diff/jest.config.js

@@ -0,0 +1,18 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  transform: {
+    '^.+\\.tsx?$': [
+      'ts-jest',
+      {
+        babelConfig: false,
+        tsconfig: 'tsconfig.json',
+      },
+    ],
+  },
+  transformIgnorePatterns: [`/node_modules/*`],
+  testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$',
+  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
+  modulePathIgnorePatterns: ['dist/*']
+};

pgpm/diff/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@pgpmjs/diff",
+  "version": "0.1.0",
+  "author": "Constructive <developers@constructive.io>",
+  "description": "pgpm diff — identity-keyed semantic schema diff between pgpm modules, SQL dumps, and live databases",
+  "main": "index.js",
+  "module": "esm/index.js",
+  "types": "index.d.ts",
+  "homepage": "https://github.com/constructive-io/constructive",
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public",
+    "directory": "dist"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/constructive-io/constructive"
+  },
+  "bugs": {
+    "url": "https://github.com/constructive-io/constructive/issues"
+  },
+  "scripts": {
+    "clean": "makage clean",
+    "prepack": "npm run build",
+    "build": "makage build",
+    "build:dev": "makage build --dev",
+    "lint": "eslint . --fix",
+    "test": "jest",
+    "test:watch": "jest --watch"
+  },
+  "keywords": [
+    "sql",
+    "postgres",
+    "postgresql",
+    "pgpm",
+    "pgpmjs",
+    "diff",
+    "schema",
+    "migration"
+  ],
+  "devDependencies": {
+    "makage": "^0.3.0"
+  },
+  "dependencies": {
+    "@pgpmjs/ast": "workspace:^",
+    "@pgpmjs/naming-spec": "workspace:^",
+    "@pgpmjs/transform": "workspace:^"
+  }
+}

pgpm/export/src/diff-source.ts → pgpm/diff/src/diff-source.ts

@@ -7,14 +7,13 @@
  * `DiffInputChange[]` seam the semantic diff driver consumes, so the
  * comparison is source- and dial-invariant.
  */
-import { PgpmRow } from '@pgpmjs/core';
+import { PgpmRow } from '@pgpmjs/ast';
 import { alterationPathFor } from '@pgpmjs/naming-spec';
 import type { DiffInputChange, SemanticDeltaChange } from '@pgpmjs/transform';
+import { loadModuleSource, stripTransactionWrapper } from '@pgpmjs/transform';
 import * as fs from 'fs';
 import * as path from 'path';
 
-import { loadModuleSource, stripTransactionWrapper } from './module-source';
-
 /** What a diff side spec resolves to. */
 export type DiffSideKind = 'module' | 'sql' | 'database';
 

pgpm/diff/src/index.ts

@@ -0,0 +1,2 @@
+export type { DiffSide, DiffSideKind } from './diff-source';
+export { deltaChangesToRows, loadDiffSideFromDisk, resolveDiffSideKind, sqlToDiffChanges, stripDumpPreamble } from './diff-source';

pgpm/diff/tsconfig.esm.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "dist/esm",
+    "module": "es2022",
+    "rootDir": "src/",
+    "declaration": false
+  }
+}

pgpm/diff/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "outDir": "dist",
+    "rootDir": "src/"
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["dist", "node_modules", "**/*.spec.*", "**/*.test.*"]
+}

pgpm/export/src/export-graphql.ts

@@ -9,6 +9,7 @@
  */
 import { PgpmPackage, PgpmRow, SqlWriteOptions, writePgpmFiles, writePgpmPlan } from '@pgpmjs/core';
 import { createClient } from '@pgpmjs/migrate-client';
+import { ExportGranularity, PartitionConfig, partitionExportRows, restructureExportRows } from '@pgpmjs/transform';
 import { Inquirerer } from 'inquirerer';
 
 import { exportGraphQLMeta } from './export-graphql-meta';
@@ -26,8 +27,6 @@ import {
   SERVICE_REQUIRED_EXTENSIONS} from './export-utils';
 import { GraphQLClient } from './graphql-client';
 import { graphqlRowToPostgresRow } from './graphql-naming';
-import { PartitionConfig, partitionExportRows } from './partition';
-import { ExportGranularity, restructureExportRows } from './restructure';
 
 // =============================================================================
 // Public API

pgpm/export/src/export-migrations.ts

@@ -1,4 +1,5 @@
 import { PgpmPackage, PgpmRow, SqlWriteOptions, writePgpmFiles, writePgpmPlan } from '@pgpmjs/core';
+import { ExportGranularity, PartitionConfig, partitionExportRows, restructureExportRows } from '@pgpmjs/transform';
 import { PgpmOptions } from '@pgpmjs/types';
 import { Inquirerer } from 'inquirerer';
 import { getPgPool } from 'pg-cache';
@@ -15,8 +16,6 @@ import {
   normalizeOutdir,
   preparePackage,
   SERVICE_REQUIRED_EXTENSIONS} from './export-utils';
-import { PartitionConfig, partitionExportRows } from './partition';
-import { ExportGranularity, restructureExportRows } from './restructure';
 
 interface ExportMigrationsToDiskOptions {
   project: PgpmPackage;

pgpm/export/src/index.ts

@@ -1,9 +1,3 @@
-export type { CatalogQueryable, CatalogSnapshot } from './catalog-check';
-export { diffCatalogSnapshots, snapshotCatalog, withoutColumnOrder } from './catalog-check';
-export type { DiffSide, DiffSideKind } from './diff-source';
-export { deltaChangesToRows, loadDiffSideFromDisk, resolveDiffSideKind, sqlToDiffChanges, stripDumpPreamble } from './diff-source';
-export type { CopyBlock, CopyTarget, DumpSource } from './dump-source';
-export { copyBlockToInsert, copyTargetOf, dumpCompatibilityWarnings, loadDumpSource, preprocessDumpText } from './dump-source';
 export * from './export-data';
 export * from './export-graphql';
 export * from './export-graphql-meta';
@@ -34,13 +28,5 @@ export {
   SERVICE_REQUIRED_EXTENSIONS} from './export-utils';
 export { GraphQLClient } from './graphql-client';
 export { buildFieldsFragment, getGraphQLQueryName, getGraphQLTypeName, graphqlRowToPostgresRow, GraphQLTypeInfo,mapGraphQLTypeToFieldType, unwrapGraphQLType } from './graphql-naming';
-export type { ImportDumpRowsOptions, ImportDumpRowsResult, ImportDumpSummary } from './import';
-export { importDumpRows, linkTextualDeps, MISC_CHANGE_PATH } from './import';
 export { intervalToPostgres, parsePgInterval, PgInterval } from './interval-utils';
-export type { ModuleSource, ModuleSourceChange } from './module-source';
-export { loadModuleSource, stripTransactionWrapper } from './module-source';
-export type { PartitionConfig, PartitionedPackageRows, PartitionExportRowsResult } from './partition';
-export { parsePartitionConfig, partitionExportRows } from './partition';
-export type { ExportGranularity, RestructureExportRowsOptions, RestructureExportRowsResult } from './restructure';
-export { EXPORT_GRANULARITIES, isExportGranularity, restructureExportRows } from './restructure';
 export { lookupByGqlType,lookupByPgUdt, PG_TYPE_MAP, TypeMapEntry } from './type-map';

pgpm/import/README.md

@@ -0,0 +1,8 @@
+# @pgpmjs/import
+
+`pgpm import` engine: pgpm-itize arbitrary SQL dumps (plain `pg_dump` output or raw `.sql` files) through the dials pipeline.
+
+- `dump-source` — preprocess dump text: strip psql backslash commands, extract `COPY ... FROM stdin` blocks, convert COPY data to INSERTs.
+- `import` — `importDumpRows`: classify statements, attach riders (grants/comments/ownership) to their host objects, route through the shared restructure seam (`@pgpmjs/transform`), and emit deployable `PgpmRow`s with generated revert/verify.
+
+Used by the `pgpm import` CLI command (`pgpm/cli`). Sibling front doors: `@pgpmjs/export` (live database), `@pgpmjs/diff` (semantic diff).

pgpm/import/jest.config.js

@@ -0,0 +1,18 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  transform: {
+    '^.+\\.tsx?$': [
+      'ts-jest',
+      {
+        babelConfig: false,
+        tsconfig: 'tsconfig.json',
+      },
+    ],
+  },
+  transformIgnorePatterns: [`/node_modules/*`],
+  testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$',
+  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
+  modulePathIgnorePatterns: ['dist/*']
+};

pgpm/import/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@pgpmjs/import",
+  "version": "0.1.0",
+  "author": "Constructive <developers@constructive.io>",
+  "description": "pgpm import — pgpm-itize arbitrary SQL dumps (pg_dump output, raw .sql) through the dials pipeline",
+  "main": "index.js",
+  "module": "esm/index.js",
+  "types": "index.d.ts",
+  "homepage": "https://github.com/constructive-io/constructive",
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public",
+    "directory": "dist"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/constructive-io/constructive"
+  },
+  "bugs": {
+    "url": "https://github.com/constructive-io/constructive/issues"
+  },
+  "scripts": {
+    "clean": "makage clean",
+    "prepack": "npm run build",
+    "build": "makage build",
+    "build:dev": "makage build --dev",
+    "lint": "eslint . --fix",
+    "test": "jest",
+    "test:watch": "jest --watch"
+  },
+  "keywords": [
+    "sql",
+    "postgres",
+    "postgresql",
+    "pgpm",
+    "pgpmjs",
+    "import",
+    "pg_dump",
+    "migration"
+  ],
+  "devDependencies": {
+    "makage": "^0.3.0"
+  },
+  "dependencies": {
+    "@pgpmjs/ast": "workspace:^",
+    "@pgpmjs/naming-spec": "workspace:^",
+    "@pgpmjs/transform": "workspace:^"
+  }
+}