Compile and run will both replace includes now. (#248)

Author: benjie

README.md

@@ -217,22 +217,23 @@ Commands:
   graphile-migrate reset           Drops and re-creates the database, re-running
                                    all committed migrations from the start.
                                    **HIGHLY DESTRUCTIVE**.
-  graphile-migrate compile [file]  Compiles a SQL file, inserting all the
-                                   placeholders and returning the result to
-                                   STDOUT
-  graphile-migrate run [file]      Compiles a SQL file, inserting all the
-                                   placeholders, and then runs it against the
-                                   database. Useful for seeding. If called from
-                                   an action will automatically run against the
-                                   same database (via GM_DBURL envvar) unless
-                                   --shadow or --rootDatabase are supplied.
+  graphile-migrate compile [file]  Compiles a SQL file (resolving `--!includes`,
+                                   replacing :PLACEHOLDERs, etc) and outputs the
+                                   result to STDOUT
+  graphile-migrate run [file]      Compiles a SQL file (resolving `--!includes`,
+                                   replacing :PLACEHOLDERs, etc) and then runs
+                                   it against the database. Useful for seeding.
+                                   If called from an action will automatically
+                                   run against the same database (via GM_DBURL
+                                   envvar) unless --shadow or --rootDatabase are
+                                   supplied.
   graphile-migrate completion      Generate shell completion script.
 
 Options:
       --help    Show help                                              [boolean]
   -c, --config  Optional path to gmrc file   [string] [default: .gmrc[.js|.cjs]]
 
-You are running graphile-migrate v2.0.0-rc.1.
+You are running graphile-migrate v2.0.0-rc.2.
 ```
 
 
@@ -373,8 +374,8 @@ Options:
 ```
 graphile-migrate compile [file]
 
-Compiles a SQL file, inserting all the placeholders and returning the result to
-STDOUT
+Compiles a SQL file (resolving `--!includes`, replacing :PLACEHOLDERs, etc) and
+outputs the result to STDOUT
 
 Options:
       --help    Show help                                              [boolean]
@@ -389,10 +390,10 @@ Options:
 ```
 graphile-migrate run [file]
 
-Compiles a SQL file, inserting all the placeholders, and then runs it against
-the database. Useful for seeding. If called from an action will automatically
-run against the same database (via GM_DBURL envvar) unless --shadow or
---rootDatabase are supplied.
+Compiles a SQL file (resolving `--!includes`, replacing :PLACEHOLDERs, etc) and
+then runs it against the database. Useful for seeding. If called from an action
+will automatically run against the same database (via GM_DBURL envvar) unless
+--shadow or --rootDatabase are supplied.
 
 Options:
       --help          Show help                                        [boolean]
@@ -767,8 +768,9 @@ drop policy if exists access_by_numbers on mytable;
 create policy access_by_numbers on mytable for update using (myfunction(4, 2) < 42);
 ```
 
-and when the migration is committed or watched, the contents of `myfunction.sql`
-will be included in the result, such that the following SQL is executed:
+and when the migration is committed, watched, run, or compiled, the contents of
+`myfunction.sql` will be included in the result, such that the following SQL is
+executed:
 
 ```sql
 create or replace function myfunction(a int, b int)

__tests__/compile.test.ts

@@ -1,9 +1,15 @@
 import "./helpers";
 
-import * as mockFs from "mock-fs";
+import mockFs from "mock-fs";
 
 import { compile } from "../src";
 let old: string | undefined;
+const settings = {
+  connectionString: "postgres://dbowner:dbpassword@dbhost:1221/dbname",
+  placeholders: {
+    ":DATABASE_AUTHENTICATOR": "!ENV",
+  },
+};
 beforeAll(() => {
   old = process.env.DATABASE_AUTHENTICATOR;
   process.env.DATABASE_AUTHENTICATOR = "dbauth";
@@ -19,12 +25,7 @@ afterEach(() => {
 it("compiles SQL with settings", async () => {
   expect(
     await compile(
-      {
-        connectionString: "postgres://dbowner:dbpassword@dbhost:1221/dbname",
-        placeholders: {
-          ":DATABASE_AUTHENTICATOR": "!ENV",
-        },
-      },
+      settings,
       `\
 BEGIN;
 GRANT CONNECT ON DATABASE :DATABASE_NAME TO :DATABASE_OWNER;
@@ -53,3 +54,24 @@ CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA public;
 COMMIT;
 `);
 });
+
+it("will compile included files", async () => {
+  mockFs({
+    "migrations/fixtures/foo.sql": "select * from foo;",
+  });
+  expect(
+    await compile(
+      settings,
+      `\
+select 1;
+--!include foo.sql
+select 2;
+`,
+      { filename: `${process.cwd()}/migrations/current.sql` },
+    ),
+  ).toEqual(`\
+select 1;
+select * from foo;
+select 2;
+`);
+});

src/commands/_common.ts

@@ -119,6 +119,23 @@ export async function getSettings(options: Options = {}): Promise<Settings> {
   }
 }
 
+export async function readFileOrStdin(file: unknown): Promise<{
+  filename: string;
+  content: string;
+}> {
+  if (file != null) {
+    if (typeof file === "string") {
+      const filename = resolve(file);
+      const content = await fsp.readFile(filename, "utf8");
+      return { filename, content };
+    } else {
+      throw new Error(`Unexpected value for "file" flag`);
+    }
+  } else {
+    return { filename: "stdin", content: await readStdin() };
+  }
+}
+
 export function readStdin(): Promise<string> {
   return new Promise((resolve, reject) => {
     let data = "";

src/commands/compile.ts

@@ -1,20 +1,34 @@
-import * as fsp from "fs/promises";
 import { CommandModule } from "yargs";
 
-import { compilePlaceholders } from "../migration";
+import { compileIncludes, compilePlaceholders } from "../migration";
 import { parseSettings, Settings } from "../settings";
-import { CommonArgv, getSettings, readStdin } from "./_common";
+import { CommonArgv, getSettings, readFileOrStdin } from "./_common";
 
 interface CompileArgv extends CommonArgv {
   shadow?: boolean;
 }
 
+interface CompileOptions {
+  shadow?: boolean;
+  filename?: string;
+}
+
+function resolveOptions(options: CompileOptions | boolean) {
+  return typeof options === "boolean" ? { shadow: options } : options;
+}
+
 export async function compile(
   settings: Settings,
-  content: string,
-  shadow = false,
+  rawContent: string,
+  options: boolean | CompileOptions = false,
 ): Promise<string> {
+  const { shadow = false, filename = "unknown" } = resolveOptions(options);
   const parsedSettings = await parseSettings(settings, shadow);
+  const content = await compileIncludes(
+    parsedSettings,
+    rawContent,
+    new Set([filename]),
+  );
   return compilePlaceholders(parsedSettings, content, shadow);
 }
 
@@ -25,7 +39,7 @@ export const compileCommand: CommandModule<
   command: "compile [file]",
   aliases: [],
   describe: `\
-Compiles a SQL file, inserting all the placeholders and returning the result to STDOUT`,
+Compiles a SQL file (resolving \`--!includes\`, replacing :PLACEHOLDERs, etc) and outputs the result to STDOUT`,
   builder: {
     shadow: {
       type: "boolean",
@@ -35,13 +49,11 @@ Compiles a SQL file, inserting all the placeholders and returning the result to
   },
   handler: async (argv) => {
     const settings = await getSettings({ configFile: argv.config });
-    const content =
-      typeof argv.file === "string"
-        ? await fsp.readFile(argv.file, "utf8")
-        : await readStdin();
-
-    const compiled = await compile(settings, content, argv.shadow);
-
+    const { content, filename } = await readFileOrStdin(argv.file);
+    const compiled = await compile(settings, content, {
+      shadow: argv.shadow,
+      filename,
+    });
     // eslint-disable-next-line no-console
     console.log(compiled);
   },

src/commands/run.ts

@@ -1,17 +1,17 @@
-import * as fsp from "fs/promises";
 import { QueryResultRow } from "pg";
 import { CommandModule } from "yargs";
 
 import { DO_NOT_USE_DATABASE_URL } from "../actions";
 import { runQueryWithErrorInstrumentation } from "../instrumentation";
-import { compilePlaceholders } from "../migration";
+import { compileIncludes, compilePlaceholders } from "../migration";
 import { withClient } from "../pgReal";
 import {
   makeRootDatabaseConnectionString,
   parseSettings,
   Settings,
 } from "../settings";
-import { CommonArgv, getDatabaseName, getSettings, readStdin } from "./_common";
+import type { CommonArgv } from "./_common";
+import { getDatabaseName, getSettings, readFileOrStdin } from "./_common";
 
 interface RunArgv extends CommonArgv {
   shadow?: boolean;
@@ -22,7 +22,7 @@ interface RunArgv extends CommonArgv {
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export async function run<T extends QueryResultRow = QueryResultRow>(
   settings: Settings,
-  content: string,
+  rawContent: string,
   filename: string,
   {
     shadow = false,
@@ -35,6 +35,11 @@ export async function run<T extends QueryResultRow = QueryResultRow>(
   } = {},
 ): Promise<T[] | undefined> {
   const parsedSettings = await parseSettings(settings, shadow);
+  const content = await compileIncludes(
+    parsedSettings,
+    rawContent,
+    new Set([filename]),
+  );
   const sql = compilePlaceholders(parsedSettings, content, shadow);
   const baseConnectionString = rootDatabase
     ? parsedSettings.rootConnectionString
@@ -62,7 +67,7 @@ export const runCommand: CommandModule<Record<string, never>, RunArgv> = {
   command: "run [file]",
   aliases: [],
   describe: `\
-Compiles a SQL file, inserting all the placeholders, and then runs it against the database. Useful for seeding. If called from an action will automatically run against the same database (via GM_DBURL envvar) unless --shadow or --rootDatabase are supplied.`,
+Compiles a SQL file (resolving \`--!includes\`, replacing :PLACEHOLDERs, etc) and then runs it against the database. Useful for seeding. If called from an action will automatically run against the same database (via GM_DBURL envvar) unless --shadow or --rootDatabase are supplied.`,
   builder: {
     shadow: {
       type: "boolean",
@@ -102,14 +107,7 @@ Compiles a SQL file, inserting all the placeholders, and then runs it against th
             connectionString: process.env.GM_DBURL,
           };
 
-    const { content, filename } =
-      typeof argv.file === "string"
-        ? {
-            filename: argv.file,
-            content: await fsp.readFile(argv.file, "utf8"),
-          }
-        : { filename: "stdin", content: await readStdin() };
-
+    const { content, filename } = await readFileOrStdin(argv.file);
     const rows = await run(settings, content, filename, argv);
 
     if (rows) {