Skip to content

Commit e4bb725

Browse files
devaladedevalade
committed
Roll rollbacks across the fleet instead of one server
rollback --app api went through runRemoteCommandForConfig, which opens exactly one connection, so on a three-replica fleet it rolled back one server and left the other two on the bad release. It now drives the same roller a deploy does. deployFleet becomes rollFleet, taking the per-replica operation as a callback, so a rollback inherits identical batching, drain timing, partial-failure reporting and mixed-version messaging rather than reimplementing them. rollFleet no longer mints a release id — a deploy passes one so every replica lands on the same release directory, and an instant blue-green rollback has no new release to name. Each replica re-reads its own state and rolls itself back one step rather than applying a plan computed from the first server. After a roll that died halfway the fleet is deliberately not uniform, and "everyone flip to green" would put the already-correct replicas back on the bad release. The confirmation is hoisted above the roll: prompting inside the loop would ask again with half the fleet already rolled back. rollback also takes --on to roll back a single replica.
1 parent 9c30de7 commit e4bb725

6 files changed

Lines changed: 391 additions & 124 deletions

File tree

‎src/cli/commands/deploy.ts‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@ import { appStateDir } from '../../domain/deploy/drain.js';
1414
import { type ServerTargetError } from '../../shared/result-errors.js';
1515
import { runDeployWatch } from './deploy-watch.js';
1616
import type { BuildLocation } from '../../domain/deploy/hot-sync.js';
17-
import { deployFleet, type FleetEvent } from '../../domain/deploy/fleet.js';
17+
import { rollFleet, type FleetEvent } from '../../domain/deploy/fleet.js';
18+
import { newReleaseId } from '../../domain/deploy/orchestrator.js';
1819
import { SshConnection } from '../../infrastructure/ssh/connection.js';
1920

2021
export async function cmdDeploy(cwd: string, options: { dryRun?: boolean; skipBuild?: boolean; app?: string; config?: string; watch?: boolean; build?: string; on?: string }): Promise<void> {
@@ -169,10 +170,13 @@ async function rollFleetApp(
169170
for (const warning of renderFleetWarnings(app, replicas)) ui.warn(`${app.name}: ${warning}`);
170171
for (const warning of renderDependencyWarnings(config, app)) ui.warn(warning);
171172

172-
const result = await deployFleet({
173+
const result = await rollFleet({
173174
app,
174175
fleet: app.fleet!,
175176
replicas,
177+
// One release id for the whole roll, so `status` can tell a converged fleet
178+
// from a half-rolled one.
179+
releaseId: newReleaseId(),
176180
// From the full list, not the narrowed one: `--on web-b` must not promote
177181
// web-b to primary and start a second copy of the scheduler alongside web-a's.
178182
primary: allReplicas[0],
@@ -182,7 +186,7 @@ async function rollFleetApp(
182186
await ssh.connect(configForServer(appConfig, serverName).ssh);
183187
return { executor: ssh, close: () => ssh.disconnect() };
184188
},
185-
deployReplica: async ({ serverName, executor, releaseId, role }) => {
189+
applyToReplica: async ({ serverName, executor, releaseId, role }) => {
186190
const replicaConfig = { ...configForServer(appConfig, serverName), apps: [app], accessories: {} };
187191
const deployer = new DeployService(new LoggingExecutor(executor), replicaConfig);
188192
await deployer.execute(cwd, options.skipBuild ?? false, releaseId, role);
@@ -193,7 +197,7 @@ async function rollFleetApp(
193197
if (result.failed) {
194198
ui.error(
195199
`${app.name}: ${result.failed.server} failed and is out of rotation. ` +
196-
`${result.deployed.length ? `${result.deployed.join(', ')} now on ${result.releaseId}; ` : ''}` +
200+
`${result.applied.length ? `${result.applied.join(', ')} now on ${result.releaseId}; ` : ''}` +
197201
`${result.skipped.length ? `${result.skipped.join(', ')} still on the previous release. ` : ''}` +
198202
`The fleet is running mixed versions.`,
199203
);
@@ -202,7 +206,7 @@ async function rollFleetApp(
202206
}
203207

204208
ui.note(
205-
[`release ${result.releaseId}`, `servers ${result.deployed.join(', ')}`, ...(app.domain ? [`url https://${app.domain}`] : [])].join('\n'),
209+
[`release ${result.releaseId}`, `servers ${result.applied.join(', ')}`, ...(app.domain ? [`url https://${app.domain}`] : [])].join('\n'),
206210
`${app.name} rolled`,
207211
);
208212
}
@@ -212,7 +216,7 @@ function reportFleetEvent(app: ShipnodeApp, event: FleetEvent): void {
212216
case 'drained':
213217
ui.info(`${event.servers.join(', ')} draining — waiting ${event.waitSeconds}s for the load balancer`);
214218
break;
215-
case 'deploying':
219+
case 'applying':
216220
ui.step(`${app.name} → ${event.server}`);
217221
break;
218222
case 'undrained':

‎src/cli/commands/rollback.ts‎

Lines changed: 190 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { runRemoteCommandForConfig } from '../runner.js';
22
import { ReleaseManager } from '../../domain/release/manager.js';
33
import { HealthCheckService } from '../../services/health.service.js';
44
import { CaddyService } from '../../services/caddy.service.js';
5+
import { SshConnection } from '../../infrastructure/ssh/connection.js';
56
import { ui } from '../ui.js';
67
import { confirm } from '../prompt.js';
78
import { loadConfig } from '../../config/loader.js';
@@ -14,13 +15,26 @@ import {
1415
portFor,
1516
coloredWebName,
1617
} from '../../domain/deploy/blue-green.js';
18+
import { rollFleet, type FleetEvent } from '../../domain/deploy/fleet.js';
1719
import type { RemoteExecutor } from '../../domain/remote/executor.js';
1820
import type { ShipnodeConfig, ShipnodeApp } from '../../shared/types.js';
19-
import { configForAppResult } from '../../domain/servers.js';
21+
import { configForAppResult, configForServer, getServerTargets } from '../../domain/servers.js';
22+
23+
/**
24+
* Asked once, answered once.
25+
*
26+
* The single-server path prompts with the concrete release it is about to
27+
* restore. A fleet asks up front, before the roll starts, because prompting
28+
* inside the per-replica callback would ask N times — and the second prompt
29+
* would arrive with half the fleet already rolled back.
30+
*/
31+
type Confirmer = (message: string) => Promise<boolean>;
32+
33+
const alreadyConfirmed: Confirmer = async () => true;
2034

2135
export async function cmdRollback(
2236
cwd: string,
23-
options: { steps?: number; app?: string; config?: string },
37+
options: { steps?: number; app?: string; config?: string; on?: string },
2438
): Promise<void> {
2539
if (!options.app) {
2640
throw new Error(
@@ -35,77 +49,179 @@ export async function cmdRollback(
3549
return;
3650
}
3751

38-
await runRemoteCommandForConfig(
39-
selectedConfig.value,
40-
async ({ config, executor }) => {
41-
const app = getActiveApp(config, options.app);
42-
const appPath = `${config.remotePath}/${app.name}`;
43-
const stepsBack = options.steps ?? 1;
44-
45-
// Blue-green apps roll back by flipping Caddy back to the previous colour,
46-
// which is still running — instant and zero-drop. No pm2 restart.
47-
if (app.appType === 'backend' && app.zeroDowntime) {
48-
await rollbackBlueGreen(executor, config, app, appPath, stepsBack);
49-
return;
50-
}
51-
52-
const releases = new ReleaseManager(executor, appPath, app.keepReleases);
53-
54-
ui.info('Fetching release history...');
55-
const allReleases = await releases.listReleases();
56-
57-
if (allReleases.length < 2) {
58-
throw new Error('No previous release to roll back to.');
59-
}
60-
61-
const targetIdx = allReleases.length - 1 - stepsBack;
62-
if (targetIdx < 0) {
63-
throw new Error(
64-
`Cannot go back ${stepsBack} step(s) — only ${allReleases.length} release(s) recorded.`,
65-
);
66-
}
67-
68-
const current = allReleases[allReleases.length - 1];
69-
const target = allReleases[targetIdx];
70-
71-
ui.warn(`Current release: ${current.timestamp}`);
72-
ui.warn(`Target release: ${target.timestamp}`);
73-
74-
const ok = await confirm('Proceed with rollback?');
75-
if (!ok) {
76-
ui.info('Rollback cancelled.');
77-
return;
78-
}
79-
80-
const targetPath = `${appPath}/releases/${target.timestamp}`;
81-
await releases.switchSymlink(targetPath);
82-
ui.success('Symlink switched');
83-
84-
const namespace = getDeploymentName(config);
85-
if (app.appType === 'backend' && namespace) {
86-
const nodeVersion = config.nodeVersion === 'lts' ? '24' : config.nodeVersion;
87-
const mise = `export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:$PATH"`;
88-
// Prefer reloading from the rolled-back release's ecosystem file (ADR-0001 — it
89-
// restores the exact process set that was active for that release). Fall back to
90-
// namespace reload if the target release predates per-release ecosystem files.
91-
const ecosystem = getEcosystemPath(config, app.name);
92-
await executor.execOrThrow(
93-
`${mise}; mise exec node@${nodeVersion} -- ` +
94-
`(pm2 reload "${ecosystem}" --update-env 2>/dev/null || pm2 reload ${namespace} --update-env)`,
95-
);
96-
ui.success('PM2 reloaded');
97-
}
98-
99-
if (app.appType === 'backend' && app.healthCheck.enabled) {
100-
ui.info('Running health check...');
101-
const health = new HealthCheckService(executor, config);
102-
await health.perform(app);
103-
ui.success('Health check passed');
104-
}
105-
106-
ui.success(`Rolled back to ${target.timestamp}`);
107-
},
52+
const appConfig = selectedConfig.value;
53+
const app = getActiveApp(appConfig, options.app);
54+
const stepsBack = options.steps ?? 1;
55+
56+
if (app.fleet) {
57+
await rollbackFleet(appConfig, app, stepsBack, options.on);
58+
return;
59+
}
60+
61+
await runRemoteCommandForConfig(appConfig, async ({ config, executor }) => {
62+
await rollbackReplica(executor, config, app, stepsBack, confirm);
63+
});
64+
}
65+
66+
/**
67+
* Roll a fleet back one replica at a time, through the same drain/wait/undrain
68+
* machinery a deploy uses — a rollback that restarts PM2 has the same
69+
* traffic-dropping window a deploy does.
70+
*
71+
* Each replica re-reads its own state and rolls itself back one step rather than
72+
* being told what to do by a plan computed elsewhere. After a roll that failed
73+
* halfway the fleet is deliberately not uniform, and "everyone flip to green"
74+
* would put the already-correct replicas back on the bad release.
75+
*/
76+
async function rollbackFleet(
77+
appConfig: ShipnodeConfig,
78+
app: ShipnodeApp,
79+
stepsBack: number,
80+
on: string | undefined,
81+
): Promise<void> {
82+
const allReplicas = getServerTargets(appConfig).map((target) => target.name);
83+
let replicas = allReplicas;
84+
if (on) {
85+
if (!replicas.includes(on)) {
86+
ui.error(`App '${app.name}' does not run on '${on}'. It runs on: ${replicas.join(', ')}`);
87+
process.exit(1);
88+
return;
89+
}
90+
replicas = [on];
91+
}
92+
93+
ui.warn(
94+
`Rolling ${app.name} back ${stepsBack} release(s) across ${replicas.join(', ')}, ` +
95+
`${app.fleet!.batch} at a time.`,
10896
);
97+
if (!(await confirm('Proceed with rollback?'))) {
98+
ui.info('Rollback cancelled.');
99+
return;
100+
}
101+
102+
const result = await rollFleet({
103+
app,
104+
fleet: app.fleet!,
105+
replicas,
106+
primary: allReplicas[0],
107+
remotePath: appConfig.remotePath,
108+
connect: async (serverName) => {
109+
const ssh = new SshConnection();
110+
await ssh.connect(configForServer(appConfig, serverName).ssh);
111+
return { executor: ssh, close: () => ssh.disconnect() };
112+
},
113+
applyToReplica: async ({ serverName, executor }) => {
114+
const replicaConfig = { ...configForServer(appConfig, serverName), apps: [app] };
115+
await rollbackReplica(executor, replicaConfig, app, stepsBack, alreadyConfirmed);
116+
},
117+
onEvent: (event) => reportRollbackEvent(app, event),
118+
});
119+
120+
if (result.failed) {
121+
ui.error(
122+
`${app.name}: ${result.failed.server} failed to roll back and is out of rotation. ` +
123+
`${result.applied.length ? `${result.applied.join(', ')} rolled back; ` : ''}` +
124+
`${result.skipped.length ? `${result.skipped.join(', ')} untouched. ` : ''}` +
125+
`The fleet is running mixed versions.`,
126+
);
127+
process.exit(1);
128+
return;
129+
}
130+
131+
ui.success(`${app.name} rolled back on ${result.applied.join(', ')}`);
132+
}
133+
134+
function reportRollbackEvent(app: ShipnodeApp, event: FleetEvent): void {
135+
switch (event.type) {
136+
case 'drained':
137+
ui.info(`${event.servers.join(', ')} draining — waiting ${event.waitSeconds}s for the load balancer`);
138+
break;
139+
case 'applying':
140+
ui.step(`Rolling back ${app.name} on ${event.server}`);
141+
break;
142+
case 'undrained':
143+
ui.success(`${event.server} back in rotation`);
144+
break;
145+
case 'failed':
146+
ui.error(`${event.server}: ${event.message}`);
147+
break;
148+
default:
149+
break;
150+
}
151+
}
152+
153+
/** Roll one server back. `config` must already be scoped to that server. */
154+
async function rollbackReplica(
155+
executor: RemoteExecutor,
156+
config: ShipnodeConfig,
157+
app: ShipnodeApp,
158+
stepsBack: number,
159+
ask: Confirmer,
160+
): Promise<void> {
161+
const appPath = `${config.remotePath}/${app.name}`;
162+
163+
// Blue-green apps roll back by flipping Caddy back to the previous colour,
164+
// which is still running — instant and zero-drop. No pm2 restart.
165+
if (app.appType === 'backend' && app.zeroDowntime) {
166+
await rollbackBlueGreen(executor, config, app, appPath, stepsBack, ask);
167+
return;
168+
}
169+
170+
const releases = new ReleaseManager(executor, appPath, app.keepReleases);
171+
172+
ui.info('Fetching release history...');
173+
const allReleases = await releases.listReleases();
174+
175+
if (allReleases.length < 2) {
176+
throw new Error('No previous release to roll back to.');
177+
}
178+
179+
const targetIdx = allReleases.length - 1 - stepsBack;
180+
if (targetIdx < 0) {
181+
throw new Error(
182+
`Cannot go back ${stepsBack} step(s) — only ${allReleases.length} release(s) recorded.`,
183+
);
184+
}
185+
186+
const current = allReleases[allReleases.length - 1];
187+
const target = allReleases[targetIdx];
188+
189+
ui.warn(`Current release: ${current.timestamp}`);
190+
ui.warn(`Target release: ${target.timestamp}`);
191+
192+
const ok = await ask('Proceed with rollback?');
193+
if (!ok) {
194+
ui.info('Rollback cancelled.');
195+
return;
196+
}
197+
198+
const targetPath = `${appPath}/releases/${target.timestamp}`;
199+
await releases.switchSymlink(targetPath);
200+
ui.success('Symlink switched');
201+
202+
const namespace = getDeploymentName(config);
203+
if (app.appType === 'backend' && namespace) {
204+
const nodeVersion = config.nodeVersion === 'lts' ? '24' : config.nodeVersion;
205+
const mise = `export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:$PATH"`;
206+
// Prefer reloading from the rolled-back release's ecosystem file (ADR-0001 — it
207+
// restores the exact process set that was active for that release). Fall back to
208+
// namespace reload if the target release predates per-release ecosystem files.
209+
const ecosystem = getEcosystemPath(config, app.name);
210+
await executor.execOrThrow(
211+
`${mise}; mise exec node@${nodeVersion} -- ` +
212+
`(pm2 reload "${ecosystem}" --update-env 2>/dev/null || pm2 reload ${namespace} --update-env)`,
213+
);
214+
ui.success('PM2 reloaded');
215+
}
216+
217+
if (app.appType === 'backend' && app.healthCheck.enabled) {
218+
ui.info('Running health check...');
219+
const health = new HealthCheckService(executor, config);
220+
await health.perform(app);
221+
ui.success('Health check passed');
222+
}
223+
224+
ui.success(`Rolled back to ${target.timestamp}`);
109225
}
110226

111227
/**
@@ -122,6 +238,7 @@ async function rollbackBlueGreen(
122238
app: ShipnodeApp,
123239
appPath: string,
124240
stepsBack: number,
241+
ask: Confirmer,
125242
): Promise<void> {
126243
if (app.blueGreenRetention === 'none') {
127244
throw new Error(
@@ -173,7 +290,7 @@ async function rollbackBlueGreen(
173290
ui.warn(`Active colour: ${state.activeColor} (port ${portFor(state.activeColor, state)})`);
174291
ui.warn(`Rollback target: ${previous} (port ${previousPort})`);
175292

176-
const ok = await confirm('Flip traffic back to the previous colour?');
293+
const ok = await ask('Flip traffic back to the previous colour?');
177294
if (!ok) {
178295
ui.info('Rollback cancelled.');
179296
return;

‎src/cli/index.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,9 @@ program
101101
.description('Roll back to a previous release')
102102
.option('--steps <n>', 'Number of releases to go back', '1')
103103
.option('--app <name>', 'App to roll back (required)')
104+
.option('--on <server>', 'Roll back one replica of a fleet instead of all of them')
104105
.option('--config <path>', 'Use a specific config file')
105-
.action((opts) => cmdRollback(process.cwd(), { steps: parseInt(opts.steps, 10), app: opts.app, config: opts.config }));
106+
.action((opts) => cmdRollback(process.cwd(), { steps: parseInt(opts.steps, 10), app: opts.app, on: opts.on, config: opts.config }));
106107

107108
program
108109
.command('migrate')

0 commit comments

Comments
 (0)