fix blue-green PM2 startup on alternate ports

devalade · devalade · commit 5f842bc26224 · 2026-07-15T01:19:01.000+02:00
diff --git a/docs/adr/0003-source-env-instead-of-pm2-env-file.md b/docs/adr/0003-source-env-instead-of-pm2-env-file.md
@@ -7,12 +7,12 @@ We now drop `env_file` from the ecosystem and emit each app as a `bash` script w
 ```js
 {
   script: 'bash',
-  args: ['-c', "set -a && . '/var/www/app/shared/.env.production' && set +a && exec node dist/server.js"],
+  args: ['-c', "set -a && . '/var/www/app/shared/.env.production' && set +a && export NODE_ENV='production' PORT='3000' && exec node dist/server.js"],
   env: { NODE_ENV: 'production', PORT: 3000 }
 }
 ```
 
-`set -a` exports everything sourced. `exec` replaces the wrapper shell so PM2 supervises the real process directly — signals, reloads, and crash detection behave exactly as before. Per-app `env:` overrides (e.g. `WORKER_QUEUE`) still apply because PM2 layers them on top of the inherited env. Secrets stay in the chmod-600 `.env` file; they do **not** leak into the world-readable `ecosystem.config.cjs`.
+`set -a` exports everything sourced. The wrapper then reapplies the ecosystem's explicit values so `NODE_ENV`, per-app overrides, and especially a blue-green target `PORT` take precedence over stale values in the shared dotenv file. `exec` replaces the wrapper shell so PM2 supervises the real process directly — signals, reloads, and crash detection behave exactly as before. Secrets from the dotenv file stay in the chmod-600 shared file; they are not duplicated into the world-readable ecosystem configuration.
 
 `args` is emitted as an array because PM2 word-splits string args on whitespace, which would shred the inline shell snippet.
 
diff --git a/docs/adr/0005-blue-green-zero-downtime.md b/docs/adr/0005-blue-green-zero-downtime.md
@@ -9,15 +9,15 @@ Two ports, named **blue** and **green**. Blue is the web app's configured `port`
 Each deploy:
 
 1. Stages the release and switches the `current` symlink (as always). The previously-started colour keeps running the old code in memory — the symlink move doesn't touch a running process.
-2. Boots the **idle** colour on its own port (`pm2 start ecosystem.web.cjs`), reaping any stale same-colour instance first. The active colour is never touched. Workers are **not** reloaded yet.
+2. Boots the **idle** colour on its own port (`pm2 start ecosystem.web.config.cjs`), reaping any stale same-colour instance first. The active colour is never touched. Workers are **not** reloaded yet.
 3. Health-checks the new colour on **its** port and colour-suffixed pm2 name (`api-green`).
 4. Only on success: reloads the single worker set against the new release, then rewrites the Caddy site to the new colour's port and `systemctl reload caddy` — a graceful reload that drains in-flight requests. Then persists the new active colour. On the first migration only, the legacy uncoloured process is removed after this sequence.
 
 A failed health check throws before step 4: Caddy is untouched, workers stay on the previous release, the old colour is still serving, `current` is reverted to the previous release when one exists, and the failed colour is reaped by the next deploy. Zero user impact on a bad release — the main prize.
 
 ## Why the web app is duplicated but workers are not
 
-Blue-green needs two copies of the *web* app (old + new, different ports) resident at once. Workers have no port and aren't behind Caddy; running two copies would double-process their queues. So the ecosystem is split: `ecosystem.web.cjs` holds only the colour-suffixed web app; `ecosystem.workers.cjs` holds the workers as a single set, reloaded in place **after** health passes (a brief worker blip on a successful deploy is acceptable — they're not serving HTTP). Reloading before health would leave workers on a bad release while traffic stayed on the old colour.
+Blue-green needs two copies of the *web* app (old + new, different ports) resident at once. Workers have no port and aren't behind Caddy; running two copies would double-process their queues. So the ecosystem is split: `ecosystem.web.config.cjs` holds only the colour-suffixed web app; `ecosystem.workers.config.cjs` holds the workers as a single set, reloaded in place **after** health passes (a brief worker blip on a successful deploy is acceptable — they're not serving HTTP). The `.config.cjs` suffix is required so PM2 parses these files as ecosystem configuration instead of launching the file itself as an application. Reloading before health would leave workers on a bad release while traffic stayed on the old colour.
 
 ## Rollback is a flip, not a restart
 
diff --git a/src/domain/deploy/backend-strategy.ts b/src/domain/deploy/backend-strategy.ts
@@ -13,6 +13,10 @@ function escapeSingleQuotes(s: string): string {
   return s.replace(/'/g, "\\'");
 }
 
+function shellSingleQuote(s: string): string {
+  return `'${s.replace(/'/g, `'"'"'`)}'`;
+}
+
 // Q5: `command` is shell-style — split on whitespace into script + args.
 // Omitted command falls back to `<pkgManager> start` (the pre-multi-process default).
 function parseCommand(command: string | undefined, pkgManager: string): { script: string; args: string } {
@@ -216,13 +220,13 @@ export class BackendStrategy implements DeploymentStrategy {
     const webEco = this.generateEcosystemForApps([webApp], pkgManager, (a) =>
       a.name === webApp.name ? { nameSuffix: `-${target.color}`, portOverride: target.port } : {},
     );
-    const webRuntimePath = `${this.appPath}/current/ecosystem.web.cjs`;
-    await this.writeEcosystem(ctx, `${ctx.workDir}/ecosystem.web.cjs`, webEco);
+    const webRuntimePath = `${this.appPath}/current/ecosystem.web.config.cjs`;
+    await this.writeEcosystem(ctx, `${ctx.workDir}/ecosystem.web.config.cjs`, webEco);
 
     // Write workers ecosystem now so afterHealthy can reload it; do not start yet.
     if (workers.length > 0) {
       const workersEco = this.generateEcosystemForApps(workers, pkgManager);
-      await this.writeEcosystem(ctx, `${ctx.workDir}/ecosystem.workers.cjs`, workersEco);
+      await this.writeEcosystem(ctx, `${ctx.workDir}/ecosystem.workers.config.cjs`, workersEco);
     }
 
     await this.relinkPackages(ctx, pkgManager, cdPath, mise);
@@ -253,7 +257,7 @@ export class BackendStrategy implements DeploymentStrategy {
 
     const cdPath = `${this.appPath}/current`;
     const mise = `export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:$PATH"`;
-    const workersRuntimePath = `${this.appPath}/current/ecosystem.workers.cjs`;
+    const workersRuntimePath = `${this.appPath}/current/ecosystem.workers.config.cjs`;
 
     await ctx.executor.execOrThrow(
       `cd "${cdPath}" && ${mise} && ` +
@@ -381,7 +385,14 @@ ${appBlocks.join(',\n')}
 
     if (useWrapper) {
       const tail = origArgs ? `${origScript} ${origArgs}` : origScript;
-      const inner = `set -a && . '${escapeSingleQuotes(envFilePath)}' && set +a && exec ${tail}`;
+      // Sourcing the dotenv file happens inside the PM2-launched shell, so it
+      // can overwrite inline PM2 values such as the blue-green port. Re-apply
+      // the ecosystem's explicit values afterwards to preserve their normal
+      // precedence over the shared environment.
+      const explicitEnv = Object.entries(env)
+        .map(([key, value]) => `${key}=${shellSingleQuote(String(value))}`)
+        .join(' ');
+      const inner = `set -a && . '${escapeSingleQuotes(envFilePath)}' && set +a && export ${explicitEnv} && exec ${tail}`;
       scriptLine = `script: 'bash',`;
       argsLine = `\n      args: ['-c', '${escapeSingleQuotes(inner)}'],`;
     } else {
diff --git a/tests/unit/backend-strategy.test.ts b/tests/unit/backend-strategy.test.ts
@@ -424,6 +424,10 @@ describe('BackendStrategy.startApp', () => {
     expect(writeCmd).toContain(`script: '"'"'bash'"'"',`);
     expect(writeCmd).toContain('shared/.env.production');
     expect(writeCmd).toContain('set -a');
+    expect(writeCmd).toContain('export NODE_ENV=');
+    expect(writeCmd).toContain('PORT=');
+    expect(writeCmd).toContain('3000');
+    expect(writeCmd.indexOf('export NODE_ENV=')).toBeGreaterThan(writeCmd.indexOf('set +a'));
     expect(writeCmd).toContain('exec');
   });
 
@@ -507,11 +511,11 @@ describe('BackendStrategy.startApp — blue-green', () => {
     await strategy.startApp!(makeCtx(executor, { deployTarget: bgTarget() }));
 
     const cmds = executor.getHistory().map((h) => h.command);
-    const webEco = cmds.find((c) => c.includes('ecosystem.web.cjs') && c.includes('echo'));
+    const webEco = cmds.find((c) => c.includes('ecosystem.web.config.cjs') && c.includes('echo'));
     expect(webEco).toContain('myapp-green');
     expect(webEco).toContain('PORT: 3001');
 
-    const start = cmds.find((c) => c.includes('pm2 start') && c.includes('ecosystem.web.cjs'));
+    const start = cmds.find((c) => c.includes('pm2 start') && c.includes('ecosystem.web.config.cjs'));
     expect(start).toBeDefined();
     // reaps any stale same-colour instance before starting
     expect(start).toContain('pm2 delete "myapp-green"');
@@ -522,7 +526,7 @@ describe('BackendStrategy.startApp — blue-green', () => {
     const executor = new FakeRemoteExecutor();
     await strategy.startApp!(makeCtx(executor, { deployTarget: bgTarget({ previousColor: 'blue' }) }));
 
-    const start = executor.getHistory().map((h) => h.command).find((c) => c.includes('pm2 start') && c.includes('ecosystem.web.cjs'))!;
+    const start = executor.getHistory().map((h) => h.command).find((c) => c.includes('pm2 start') && c.includes('ecosystem.web.config.cjs'))!;
     expect(start).not.toContain('pm2 delete "myapp"');
   });
 
@@ -532,7 +536,7 @@ describe('BackendStrategy.startApp — blue-green', () => {
     const ctx = makeCtx(executor, { deployTarget: bgTarget({ color: 'green', port: 3001, previousColor: null }) });
     await strategy.startApp!(ctx);
 
-    const start = executor.getHistory().map((h) => h.command).find((c) => c.includes('pm2 start') && c.includes('ecosystem.web.cjs'))!;
+    const start = executor.getHistory().map((h) => h.command).find((c) => c.includes('pm2 start') && c.includes('ecosystem.web.config.cjs'))!;
     expect(start).not.toContain('pm2 delete "myapp"');
 
     await strategy.afterTrafficSwitch!(ctx);
@@ -558,17 +562,17 @@ describe('BackendStrategy.startApp — blue-green', () => {
 
     const cmds = executor.getHistory().map((h) => h.command);
     // web ecosystem holds only the coloured web app
-    const webEco = cmds.find((c) => c.includes('ecosystem.web.cjs') && c.includes('echo'))!;
+    const webEco = cmds.find((c) => c.includes('ecosystem.web.config.cjs') && c.includes('echo'))!;
     expect(webEco).toContain('api-green');
     expect(webEco).not.toContain('worker');
     // workers ecosystem written during start, but not reloaded yet
-    const workersEco = cmds.find((c) => c.includes('ecosystem.workers.cjs') && c.includes('echo'))!;
+    const workersEco = cmds.find((c) => c.includes('ecosystem.workers.config.cjs') && c.includes('echo'))!;
     expect(workersEco).toContain('api-worker');
-    const start = cmds.find((c) => c.includes('pm2 start') && c.includes('ecosystem.web.cjs'))!;
-    expect(start).not.toContain('ecosystem.workers.cjs');
+    const start = cmds.find((c) => c.includes('pm2 start') && c.includes('ecosystem.web.config.cjs'))!;
+    expect(start).not.toContain('ecosystem.workers.config.cjs');
 
     await strategy.afterHealthy!(ctx);
     const afterCmds = executor.getHistory().map((h) => h.command);
-    expect(afterCmds.some((c) => c.includes('pm2 reload') && c.includes('ecosystem.workers.cjs'))).toBe(true);
+    expect(afterCmds.some((c) => c.includes('pm2 reload') && c.includes('ecosystem.workers.config.cjs'))).toBe(true);
   });
 });
diff --git a/tests/unit/blue-green-orchestrator.test.ts b/tests/unit/blue-green-orchestrator.test.ts
@@ -69,7 +69,7 @@ describe('blue-green deploy (orchestrator)', () => {
     const cmds = history.map((h) => h.command);
 
     // green avoids interrupting a legacy uncoloured process on the blue port
-    const webEco = cmds.find((c) => c.includes('ecosystem.web.cjs') && c.includes('echo'));
+    const webEco = cmds.find((c) => c.includes('ecosystem.web.config.cjs') && c.includes('echo'));
     expect(webEco).toBeDefined();
     expect(webEco).toContain('app-green');
     expect(webEco).toContain('PORT: 3001');
@@ -101,7 +101,7 @@ describe('blue-green deploy (orchestrator)', () => {
     await orchestrator.deploy({ cwd: '/test', skipBuild: false });
 
     const cmds = executor.getHistory().map((h) => h.command);
-    const webEco = cmds.find((c) => c.includes('ecosystem.web.cjs') && c.includes('echo'));
+    const webEco = cmds.find((c) => c.includes('ecosystem.web.config.cjs') && c.includes('echo'));
     expect(webEco).toContain('app-green');
     expect(webEco).toContain('PORT: 3001');
     expect(cmds.some((c) => c.includes('curl') && c.includes('localhost:3001/health'))).toBe(true);
@@ -195,8 +195,8 @@ describe('blue-green deploy (orchestrator)', () => {
     await orchestrator.deploy({ cwd: '/test', skipBuild: false });
 
     const cmds = executor.getHistory().map((h) => h.command);
-    const webStartIdx = cmds.findIndex((c) => c.includes('pm2 start') && c.includes('ecosystem.web.cjs'));
-    const workersIdx = cmds.findIndex((c) => c.includes('ecosystem.workers.cjs') && (c.includes('pm2 reload') || c.includes('pm2 start')));
+    const webStartIdx = cmds.findIndex((c) => c.includes('pm2 start') && c.includes('ecosystem.web.config.cjs'));
+    const workersIdx = cmds.findIndex((c) => c.includes('ecosystem.workers.config.cjs') && (c.includes('pm2 reload') || c.includes('pm2 start')));
     const curlIdx = cmds.findIndex((c) => c.includes('curl') && c.includes('/health'));
     const caddyIdx = cmds.findIndex((c) => c.includes('systemctl reload caddy'));
 
@@ -219,7 +219,7 @@ describe('blue-green deploy (orchestrator)', () => {
 
     const commands = executor.getHistory().map((entry) => entry.command);
     expect(commands.some((command) => command.includes('ecosystem.config.cjs') && command.includes('pm2 start'))).toBe(true);
-    expect(commands.some((command) => command.includes('ecosystem.web.cjs'))).toBe(false);
+    expect(commands.some((command) => command.includes('ecosystem.web.config.cjs'))).toBe(false);
     expect(commands.some((command) => command.includes('deploy-state.json'))).toBe(false);
   });
 });

Original file line number	Diff line number	Diff line change
@@ -7,12 +7,12 @@ We now drop `env_file` from the ecosystem and emit each app as a `bash` script w
`7`	`7`	```js
`8`	`8`	`{`
`9`	`9`	`script: 'bash',`
`10`		`- args: ['-c', "set -a && . '/var/www/app/shared/.env.production' && set +a && exec node dist/server.js"],`
	`10`	`+ args: ['-c', "set -a && . '/var/www/app/shared/.env.production' && set +a && export NODE_ENV='production' PORT='3000' && exec node dist/server.js"],`
`11`	`11`	`env: { NODE_ENV: 'production', PORT: 3000 }`
`12`	`12`	`}`
`13`	`13`	```
`14`	`14`
`15`		-`set -a` exports everything sourced. `exec` replaces the wrapper shell so PM2 supervises the real process directly — signals, reloads, and crash detection behave exactly as before. Per-app `env:` overrides (e.g. `WORKER_QUEUE`) still apply because PM2 layers them on top of the inherited env. Secrets stay in the chmod-600 `.env` file; they do not leak into the world-readable `ecosystem.config.cjs`.
	`15`	+`set -a` exports everything sourced. The wrapper then reapplies the ecosystem's explicit values so `NODE_ENV`, per-app overrides, and especially a blue-green target `PORT` take precedence over stale values in the shared dotenv file. `exec` replaces the wrapper shell so PM2 supervises the real process directly — signals, reloads, and crash detection behave exactly as before. Secrets from the dotenv file stay in the chmod-600 shared file; they are not duplicated into the world-readable ecosystem configuration.
`16`	`16`
`17`	`17`	`args` is emitted as an array because PM2 word-splits string args on whitespace, which would shred the inline shell snippet.
`18`	`18`