fix(cli): migrate .env on mon update / mon update-config (closes #203)

NikolayS · Sarumyan · commit cab81b3f2311 · 2026-05-25T14:23:23.000Z
diff --git a/README.md b/README.md
@@ -294,7 +294,7 @@ This will:
 
 > **Note:** The `.env` file contains configuration for the monitoring stack, including `PGAI_TAG` (the Docker image version tag), `REPLICATOR_PASSWORD` (generated password for the demo standby replication user), `VM_AUTH_USERNAME`, `VM_AUTH_PASSWORD`, and optionally `GF_SECURITY_ADMIN_PASSWORD` (Grafana admin password) and `PGAI_REGISTRY` (custom Docker registry). `postgresai mon local-install` preserves existing `REPLICATOR_PASSWORD` and `VM_AUTH_*` values or generates new ones when they are missing; Docker Compose requires these values and does not use known default passwords.
 
-> **Manual upgrade note:** If you run `docker compose` directly or maintain `.env` yourself, add `VM_AUTH_USERNAME=vmauth` and a non-empty `VM_AUTH_PASSWORD` before upgrading. Rotate VictoriaMetrics auth with `VM_AUTH_PASSWORD="$(openssl rand -base64 18)" ./scripts/rotate-vm-auth.sh` from the monitoring directory; the script updates `.env` and recreates `sink-prometheus` plus `grafana` together so datasource provisioning cannot reinsert stale credentials on restart.
+> **In-place upgrade note:** Newer stack versions can require additional `.env` keys (e.g., `VM_AUTH_USERNAME` / `VM_AUTH_PASSWORD` were added in 0.15 for VictoriaMetrics basic auth). Both `postgresai mon local-install -y` and `postgresai mon update` perform a purely-additive `.env` migration on every run: existing values are preserved verbatim, and any newly-required keys are appended with safe random defaults. If you run `docker compose` directly and maintain `.env` yourself, add `VM_AUTH_USERNAME=vmauth` and a non-empty `VM_AUTH_PASSWORD` before upgrading, or run `postgresai mon update-config` once to have the CLI fill them in for you. To rotate the VictoriaMetrics auth password, run `VM_AUTH_PASSWORD="$(openssl rand -base64 18)" ./scripts/rotate-vm-auth.sh` from the monitoring directory; the script updates `.env` and recreates `sink-prometheus` plus `grafana` together so datasource provisioning cannot reinsert stale credentials on restart.
 
 **Alternative: Manual upgrade**
 
@@ -305,6 +305,9 @@ If you prefer more control:
 postgresai --version  # check your CLI version
 # Edit .env and set PGAI_TAG to the version number
 
+# Migrate .env to add any newly-required keys (e.g. VM_AUTH_* for 0.15+)
+postgresai mon update-config
+
 # Pull new images
 docker compose pull
 
diff --git a/cli/bin/postgres-ai.ts b/cli/bin/postgres-ai.ts
@@ -74,6 +74,71 @@ function stripMatchingQuotes(value: string): string {
   return trimmed;
 }
 
+/**
+ * Required env vars contract for the monitoring stack.
+ *
+ * Keys listed here are required by the docker-compose stack and must exist in
+ * `.env` for the stack to start cleanly. Each entry knows how to mint a safe
+ * default if the key is missing. Existing values are always preserved
+ * verbatim - this function is purely additive.
+ *
+ * This is the spine of the in-place upgrade story: when a user upgrades from
+ * a version that didn't require a key (e.g. 0.14, pre-VM-auth) to one that
+ * does (0.15), `ensureRequiredEnvVars` appends what's missing so the next
+ * `docker compose up` doesn't fail with `missing "<KEY>" env var`.
+ */
+type EnvKeyDefault = {
+  key: string;
+  /** Default value or factory for green-field installs / first upgrade. */
+  defaultValue: () => string;
+  /** Key was introduced in this CLI version - used in human-readable migration logs. */
+  introducedIn: string;
+};
+
+const REQUIRED_ENV_KEYS: EnvKeyDefault[] = [
+  { key: "REPLICATOR_PASSWORD", defaultValue: () => crypto.randomBytes(32).toString("hex"), introducedIn: "0.13" },
+  { key: "VM_AUTH_USERNAME", defaultValue: () => "vmauth", introducedIn: "0.15" },
+  { key: "VM_AUTH_PASSWORD", defaultValue: () => crypto.randomBytes(18).toString("base64"), introducedIn: "0.15" },
+];
+
+/**
+ * Read `.env` (if present), append any required keys that are missing, write
+ * back atomically with 0600 perms, and return the list of keys that were added.
+ *
+ * Idempotent: a second call is a no-op once all keys are present.
+ *
+ * Used by `mon local-install`, `mon update`, and `mon update-config` so the
+ * in-place upgrade path picks up newly-required env vars without surprising
+ * the user with a silent boot failure on `sink-prometheus` / `grafana`.
+ */
+function ensureRequiredEnvVars(projectDir: string): string[] {
+  const envFile = path.resolve(projectDir, ".env");
+  const existing = fs.existsSync(envFile) ? fs.readFileSync(envFile, "utf8") : "";
+
+  const added: string[] = [];
+  const appendLines: string[] = [];
+
+  for (const spec of REQUIRED_ENV_KEYS) {
+    const re = new RegExp(`^${spec.key}=`, "m");
+    if (!re.test(existing)) {
+      appendLines.push(`${spec.key}=${spec.defaultValue()}`);
+      added.push(spec.key);
+    }
+  }
+
+  if (appendLines.length === 0) {
+    return added;
+  }
+
+  // Append (don't overwrite) so we preserve order and any comments the user
+  // may have added to their .env. Make sure we have a trailing newline first.
+  const needsTrailingNewline = existing.length > 0 && !existing.endsWith("\n");
+  const newContent = existing + (needsTrailingNewline ? "\n" : "") + appendLines.join("\n") + "\n";
+  fs.writeFileSync(envFile, newContent, { encoding: "utf8", mode: 0o600 });
+
+  return added;
+}
+
 // Helper functions for spawning processes - use Node.js child_process for compatibility
 async function execFilePromise(file: string, args: string[]): Promise<{ stdout: string; stderr: string }> {
   return new Promise((resolve, reject) => {
@@ -2970,41 +3035,83 @@ mon
   });
 mon
   .command("update-config")
-  .description("apply monitoring services configuration (generate sources)")
+  .description("apply monitoring services configuration (generate sources, migrate .env)")
   .action(async () => {
+    let projectDir: string;
+    try {
+      ({ projectDir } = await resolveOrInitPaths());
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      console.error(message);
+      process.exitCode = 1;
+      return;
+    }
+
+    // Migrate .env first: append any required keys introduced by newer stack
+    // versions (e.g. VM_AUTH_* added in 0.15). This is what makes in-place
+    // upgrades from older deployments not break with `missing "VM_AUTH_USERNAME"
+    // env var` when sink-prometheus boots.
+    const added = ensureRequiredEnvVars(projectDir);
+    if (added.length > 0) {
+      console.log(`Added missing .env keys for this stack version: ${added.join(", ")}`);
+      console.log("(existing values were preserved; missing keys filled with safe defaults)\n");
+    }
+
     const code = await runCompose(["run", "--rm", "sources-generator"]);
     if (code !== 0) process.exitCode = code;
   });
 mon
   .command("update")
-  .description("update monitoring stack")
+  .description("update monitoring stack (migrate .env, pull images)")
   .action(async () => {
     console.log("Updating PostgresAI monitoring stack...\n");
 
     try {
-      // Check if we're in a git repo
-      const gitDir = path.resolve(process.cwd(), ".git");
-      if (!fs.existsSync(gitDir)) {
-        console.error("Not a git repository. Cannot update.");
+      let projectDir: string;
+      try {
+        ({ projectDir } = await resolveOrInitPaths());
+      } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        console.error(message);
         process.exitCode = 1;
         return;
       }
 
-      // Fetch latest changes
-      console.log("Fetching latest changes...");
-      await execFilePromise("git", ["fetch", "origin"]);
-
-      // Check current branch
-      const { stdout: branch } = await execFilePromise("git", ["rev-parse", "--abbrev-ref", "HEAD"]);
-      const currentBranch = branch.trim();
-      console.log(`Current branch: ${currentBranch}`);
-
-      // Pull latest changes
-      console.log("Pulling latest changes...");
-      const { stdout: pullOut } = await execFilePromise("git", ["pull", "origin", currentBranch]);
-      console.log(pullOut);
+      // Step 1: migrate .env so newer stack versions that require additional
+      // env vars (e.g. VM_AUTH_USERNAME / VM_AUTH_PASSWORD introduced in 0.15)
+      // don't make `docker compose up` fail silently for users who installed
+      // before those vars existed. Purely additive: existing values are kept.
+      console.log("Checking .env for newly-required keys...");
+      const added = ensureRequiredEnvVars(projectDir);
+      if (added.length > 0) {
+        console.log(`✓ Added missing .env keys: ${added.join(", ")}`);
+        console.log("  (existing values preserved; missing keys filled with safe defaults)");
+      } else {
+        console.log("✓ .env is up to date");
+      }
+      console.log();
+
+      // Step 2: refresh repo if this is a git-based deployment. Some users
+      // upgrade purely via `npm install -g postgresai@latest` and don't have a
+      // git checkout - in that case we skip git operations and still do the
+      // env migration + docker pull.
+      const gitDir = path.resolve(projectDir, ".git");
+      if (fs.existsSync(gitDir)) {
+        console.log("Fetching latest changes...");
+        await execFilePromise("git", ["fetch", "origin"]);
+
+        const { stdout: branch } = await execFilePromise("git", ["rev-parse", "--abbrev-ref", "HEAD"]);
+        const currentBranch = branch.trim();
+        console.log(`Current branch: ${currentBranch}`);
+
+        console.log("Pulling latest changes...");
+        const { stdout: pullOut } = await execFilePromise("git", ["pull", "origin", currentBranch]);
+        console.log(pullOut);
+      } else {
+        console.log("(not a git checkout — skipping git fetch/pull and going straight to image pull)");
+      }
 
-      // Update Docker images
+      // Step 3: pull new images.
       console.log("\nUpdating Docker images...");
       const code = await runCompose(["pull"]);
 
diff --git a/cli/test/upgrade.test.ts b/cli/test/upgrade.test.ts
@@ -420,3 +420,126 @@ describe("upgrade CLI commands", () => {
     expect(stdout).toMatch(/health/i);
   }, { timeout: TEST_TIMEOUT });
 });
+
+describe("in-place upgrade env migration (mon update / update-config)", () => {
+  /**
+   * Regression tests for the 0.14 -> 0.15 in-place upgrade gap (#203).
+   *
+   * Before this fix, a user who installed at 0.14 and ran the documented
+   * upgrade flow (`pgai mon update`) ended up with a .env file that lacked
+   * VM_AUTH_USERNAME / VM_AUTH_PASSWORD, so sink-prometheus exited with:
+   *
+   *   fatal cannot read "/postgres_ai_configs/prometheus/prometheus.yml":
+   *   cannot expand environment variables: missing "VM_AUTH_USERNAME" env var
+   *
+   * `mon update` and `mon update-config` now migrate .env additively before
+   * doing anything else.
+   */
+
+  let tempDir: string;
+
+  beforeAll(() => {
+    tempDir = fs.mkdtempSync(resolve(os.tmpdir(), "pgai-upgrade-env-migration-"));
+  });
+
+  afterAll(() => {
+    if (tempDir && fs.existsSync(tempDir)) {
+      fs.rmSync(tempDir, { recursive: true, force: true });
+    }
+  });
+
+  test("mon update-config appends missing VM_AUTH_USERNAME / VM_AUTH_PASSWORD to a 0.14-shaped .env", () => {
+    const testDir = resolve(tempDir, "update-config-0.14-env");
+    fs.mkdirSync(testDir, { recursive: true });
+
+    // 0.14-shaped .env: PGAI_TAG present, VM_AUTH_* absent.
+    fs.writeFileSync(resolve(testDir, ".env"), "PGAI_TAG=0.14.0\nGF_SECURITY_ADMIN_PASSWORD=user-set-grafana-pw\n");
+    fs.writeFileSync(resolve(testDir, "docker-compose.yml"), "version: '3'\nservices: {}\n");
+    fs.writeFileSync(resolve(testDir, "instances.yml"), "# instances\n");
+
+    // The compose run will fail (no Docker in CI), but env migration runs first.
+    runCliInDir(["mon", "update-config"], testDir, { PGAI_TAG: undefined });
+
+    const envContent = fs.readFileSync(resolve(testDir, ".env"), "utf8");
+
+    // Existing values must be preserved verbatim.
+    expect(envContent).toMatch(/^PGAI_TAG=0\.14\.0$/m);
+    expect(envContent).toMatch(/^GF_SECURITY_ADMIN_PASSWORD=user-set-grafana-pw$/m);
+
+    // New required keys must be appended (vmauth username + non-empty base64 password).
+    expect(envContent).toMatch(/^VM_AUTH_USERNAME=vmauth$/m);
+    expect(envContent).toMatch(/^VM_AUTH_PASSWORD=[A-Za-z0-9+/]+={0,2}$/m);
+
+    // REPLICATOR_PASSWORD was introduced earlier and is also part of the contract.
+    expect(envContent).toMatch(/^REPLICATOR_PASSWORD=[a-f0-9]{64}$/m);
+  }, { timeout: TEST_TIMEOUT });
+
+  test("mon update appends missing VM_AUTH_USERNAME / VM_AUTH_PASSWORD to a 0.14-shaped .env", () => {
+    const testDir = resolve(tempDir, "update-0.14-env");
+    fs.mkdirSync(testDir, { recursive: true });
+
+    fs.writeFileSync(resolve(testDir, ".env"), "PGAI_TAG=0.14.0\n");
+    fs.writeFileSync(resolve(testDir, "docker-compose.yml"), "version: '3'\nservices: {}\n");
+    fs.writeFileSync(resolve(testDir, "instances.yml"), "# instances\n");
+
+    // mon update will fail (no Docker in CI, no git repo), but env migration runs first.
+    const result = runCliInDir(["mon", "update"], testDir, { PGAI_TAG: undefined });
+
+    const envContent = fs.readFileSync(resolve(testDir, ".env"), "utf8");
+
+    expect(envContent).toMatch(/^PGAI_TAG=0\.14\.0$/m);
+    expect(envContent).toMatch(/^VM_AUTH_USERNAME=vmauth$/m);
+    expect(envContent).toMatch(/^VM_AUTH_PASSWORD=[A-Za-z0-9+/]+={0,2}$/m);
+
+    // The migration step should print what it added so the user can see it.
+    expect(result.stdout).toMatch(/Added missing \.env keys/);
+    expect(result.stdout).toMatch(/VM_AUTH_USERNAME/);
+    expect(result.stdout).toMatch(/VM_AUTH_PASSWORD/);
+  }, { timeout: TEST_TIMEOUT });
+
+  test("mon update preserves existing VM_AUTH_* values (no rotation)", () => {
+    const testDir = resolve(tempDir, "update-preserve-vm-auth");
+    fs.mkdirSync(testDir, { recursive: true });
+
+    // User already has VM auth configured (e.g. set up via rotate-vm-auth.sh).
+    fs.writeFileSync(
+      resolve(testDir, ".env"),
+      "PGAI_TAG=0.15.0\nVM_AUTH_USERNAME=custom-user\nVM_AUTH_PASSWORD=custom-pw-do-not-rotate\nREPLICATOR_PASSWORD=" +
+        "a".repeat(64) +
+        "\n",
+    );
+    fs.writeFileSync(resolve(testDir, "docker-compose.yml"), "version: '3'\nservices: {}\n");
+    fs.writeFileSync(resolve(testDir, "instances.yml"), "# instances\n");
+
+    const result = runCliInDir(["mon", "update"], testDir, { PGAI_TAG: undefined });
+
+    const envContent = fs.readFileSync(resolve(testDir, ".env"), "utf8");
+
+    expect(envContent).toMatch(/^VM_AUTH_USERNAME=custom-user$/m);
+    expect(envContent).toMatch(/^VM_AUTH_PASSWORD=custom-pw-do-not-rotate$/m);
+    expect(envContent).toMatch(/^REPLICATOR_PASSWORD=a{64}$/m);
+
+    // When nothing is missing, the migration step should say so.
+    expect(result.stdout).toMatch(/\.env is up to date/);
+  }, { timeout: TEST_TIMEOUT });
+
+  test("mon update-config handles a .env that doesn't end with a newline", () => {
+    const testDir = resolve(tempDir, "update-config-no-trailing-newline");
+    fs.mkdirSync(testDir, { recursive: true });
+
+    // No trailing newline - migration must add one before appending new keys
+    // or we'd produce e.g. `PGAI_TAG=0.14.0VM_AUTH_USERNAME=vmauth`.
+    fs.writeFileSync(resolve(testDir, ".env"), "PGAI_TAG=0.14.0");
+    fs.writeFileSync(resolve(testDir, "docker-compose.yml"), "version: '3'\nservices: {}\n");
+    fs.writeFileSync(resolve(testDir, "instances.yml"), "# instances\n");
+
+    runCliInDir(["mon", "update-config"], testDir, { PGAI_TAG: undefined });
+
+    const envContent = fs.readFileSync(resolve(testDir, ".env"), "utf8");
+
+    expect(envContent).toMatch(/^PGAI_TAG=0\.14\.0$/m);
+    expect(envContent).toMatch(/^VM_AUTH_USERNAME=vmauth$/m);
+    // No key should be glued onto the previous line.
+    expect(envContent).not.toMatch(/PGAI_TAG=0\.14\.0VM_AUTH_USERNAME/);
+  }, { timeout: TEST_TIMEOUT });
+});
