Add fleet config: server groups, plural targets, per-server isolation

Groundwork for running one app on several servers. Nothing rolls yet — this
is the config and resolution layer the rolling deploy will sit on.

An app's `on` becomes a server name, a group name, or a list of either.
Groups are declared workspace-level and expand at resolution time, so
`.group('web', ['web-a','web-b'])` + `.on('web')` puts one app on two boxes.
Resolving to more than one server is what makes an app a fleet: assembly
fills in rolling defaults (batch 1, port 80, 30s drain, /_shipnode/ready)
so nothing downstream has to ask "is this a fleet?" twice and get two
answers. Declaring `fleet` explicitly opts a single-server app into the same
contract, which is how you get behind a load balancer before scaling out.

servers.ts goes plural. resolveServerName -> resolveServerNames returning a
list, with resolveSingleServerName kept for the things that genuinely cannot
be replicated — an accessory's host, a `run` invocation, a monitor session.
That one errors naming the candidates instead of silently picking the first,
because operating on the wrong replica is worse than being asked which.
getAppsForServer stops being a partition: a fleet app appears under each of
its replicas, or deploy would skip all but one.

Two things the schema now refuses. An accessory, database or redis pointed at
a group — shipnode does not replicate managed services, and doing so would
start two unrelated databases rather than a cluster. And a fleet whose
servers have no privateHost, since the load balancer has to reach each
replica directly and the public SSH host is not necessarily that address.

The fan-out no longer abandons the remaining servers when one fails. The
try/catch moves inside the loop, failures are collected per server and
reported together, and the process exits non-zero if any failed — a
half-finished fan-out with no account of what happened is worse than a slow
one. `--app` now also narrows which servers are visited rather than only
validating the name, so an unrelated unreachable server stops sinking a
command scoped to one app.

Author: devalade

src/cli/commands/cloudflare.ts

@@ -2,7 +2,7 @@ import { runRemoteCommandForTargets } from '../runner.js';
 import { ui } from '../ui.js';
 import { CloudflareOrchestrator } from '../../services/cloudflare.orchestrator.js';
 import { loadConfig } from '../../config/loader.js';
-import { getServerTargets, configForServer, resolveServerName } from '../../domain/servers.js';
+import { getServerTargets, configForServer, expandTarget } from '../../domain/servers.js';
 import type { ShipnodeConfig } from '../../shared/types.js';
 
 function requireToken(): string {
@@ -49,8 +49,10 @@ export async function cmdCloudflareInit(
   const workspace = await loadConfig(cwd, options.config);
   assertTunnelNameIsUnambiguous(workspace);
 
-  // Only one tunnel can own the workspace's ssh hostname.
-  const sshHostnameOwner = resolveServerName(workspace);
+  // Only one tunnel can own the workspace's ssh hostname: the default server,
+  // or the first declared when there is no server called `default`.
+  const sshHostnameOwner = expandTarget(workspace, undefined)[0]
+    ?? getServerTargets(workspace)[0]?.name;
 
   await runRemoteCommandForTargets(
     cwd,

src/cli/commands/deploy.ts

@@ -8,7 +8,7 @@ import { ui } from '../ui.js';
 import type { AccessoryConfig, ShipnodeConfig, ShipnodeApp } from '../../shared/types.js';
 import { accessoryMounts } from '../../services/accessory.service.js';
 import { getPm2Name } from '../../domain/pm2/apps.js';
-import { configForAppResult, configForServer, getServerTargets, resolveServerName, resolveServerNameResult } from '../../domain/servers.js';
+import { configForAppResult, configForServer, getServerTargets, resolveServerNames, resolveSingleServerNameResult } from '../../domain/servers.js';
 import { generateBackendCaddyfile, generateFrontendCaddyfile } from '../../services/caddy.service.js';
 import { type ServerTargetError } from '../../shared/result-errors.js';
 import { runDeployWatch } from './deploy-watch.js';
@@ -167,13 +167,19 @@ function renderAppPlan(config: ShipnodeConfig, app: ShipnodeApp, skipBuild: bool
   const namespace = app.pm2?.apps[0]?.name;
   const web = app.pm2?.apps.find((a) => a.port !== undefined);
 
+  const replicas = resolveServerNames(config, app.on);
   const serverRows: [string, string][] = [
     ['App type', app.appType],
-    ['Server', resolveServerName(config, app.on)],
+    [replicas.length > 1 ? 'Servers' : 'Server', replicas.join(', ')],
     ['App root', app.appRoot ?? '(repo root)'],
     ['Keep releases', String(app.keepReleases)],
   ];
 
+  if (app.fleet) {
+    serverRows.push(['Rolling', `${app.fleet.batch} at a time, ${app.fleet.drainWait}s drain`]);
+    serverRows.push(['Ready path', `:${app.fleet.port}${app.fleet.readyPath}`]);
+  }
+
   if (app.appType === 'backend') {
     const pm2Apps = app.pm2?.apps ?? [];
     if (pm2Apps.length && namespace) {
@@ -243,14 +249,19 @@ function renderDependencyWarnings(config: ShipnodeConfig, app: ShipnodeApp): str
   const dependencies = app.dependsOn ?? [];
   if (dependencies.length === 0) return [];
 
-  const appServer = resolveServerName(config, app.on);
+  const appServers = resolveServerNames(config, app.on);
   const warnings: string[] = [];
   for (const name of dependencies) {
     const accessory = config.accessories?.[name];
     if (!accessory) continue;
-    const accessoryServer = resolveServerName(config, accessory.on);
-    if (appServer !== accessoryServer) {
-      warnings.push(`${name} runs on ${accessoryServer}; ${app.name} runs on ${appServer}. Confirm reachable networking.`);
+    const [accessoryServer] = resolveServerNames(config, accessory.on);
+    if (accessoryServer === undefined) continue;
+    const strangers = appServers.filter((server) => server !== accessoryServer);
+    if (strangers.length > 0) {
+      warnings.push(
+        `${name} runs on ${accessoryServer}; ${app.name} runs on ${strangers.join(', ')}. ` +
+        `Confirm reachable networking.`,
+      );
     }
   }
   return warnings;
@@ -315,7 +326,7 @@ function resolveAccessoryServer(
   config: ShipnodeConfig,
   accessory: AccessoryConfig,
 ): ResultType<string, ServerTargetError> {
-  return resolveServerNameResult(config, accessory.on);
+  return resolveSingleServerNameResult(config, accessory.on, 'This accessory');
 }
 
 function renderAccessoryPlan(

src/cli/monitor/monitor-session.ts

@@ -1,5 +1,5 @@
 import { Result, type Result as ResultType } from 'better-result';
-import { getServerTargetResult, resolveServerNameResult, type ServerTarget } from '../../domain/servers.js';
+import { getServerTargetResult, resolveServerNamesResult, type ServerTarget } from '../../domain/servers.js';
 import type { ShipnodeApp, ShipnodeConfig } from '../../shared/types.js';
 import { UnknownAppError, type AppTargetError, type ServerTargetError } from '../../shared/result-errors.js';
 
@@ -19,7 +19,9 @@ export function resolveMonitorSession(
 
   if (app === undefined) return Result.err(new UnknownAppError({ name: appName ?? '(default)' }));
 
-  const target = getServerTargetResult(config, app.on);
+  // The monitor holds one live connection, so a fleet app must be narrowed to
+  // one replica first (`monitor --on <server>`).
+  const target = getServerTargetResult(config, app.on, `App '${app.name}'`);
   if (target.isErr()) return Result.err(target.error);
 
   return Result.ok({ config, app, target: target.value });
@@ -31,9 +33,9 @@ export function getAppsForMonitorTarget(
 ): ResultType<ShipnodeApp[], ServerTargetError> {
   const apps: ShipnodeApp[] = [];
   for (const app of config.apps) {
-    const serverName = resolveServerNameResult(config, app.on);
-    if (serverName.isErr()) return Result.err(serverName.error);
-    if (serverName.value === targetName) apps.push(app);
+    const serverNames = resolveServerNamesResult(config, app.on);
+    if (serverNames.isErr()) return Result.err(serverNames.error);
+    if (serverNames.value.includes(targetName)) apps.push(app);
   }
   return Result.ok(apps);
 }
@@ -44,9 +46,9 @@ export function getAccessoriesForMonitorTarget(
 ): ResultType<string[], ServerTargetError> {
   const names: string[] = [];
   for (const [name, accessory] of Object.entries(config.accessories ?? {})) {
-    const serverName = resolveServerNameResult(config, accessory.on);
-    if (serverName.isErr()) return Result.err(serverName.error);
-    if (serverName.value === targetName) names.push(name);
+    const serverNames = resolveServerNamesResult(config, accessory.on);
+    if (serverNames.isErr()) return Result.err(serverNames.error);
+    if (serverNames.value.includes(targetName)) names.push(name);
   }
   return Result.ok(names);
 }

src/cli/runner.ts

@@ -42,39 +42,75 @@ export async function runRemoteCommand(
   }
 }
 
+/**
+ * Run a command against every server the workspace touches.
+ *
+ * Servers are independent: one failing does not stop the rest, because a
+ * half-finished fan-out with no account of what happened is worse than a slow
+ * one. Failures are collected and reported together at the end, and the process
+ * exits non-zero if any server failed.
+ *
+ * `appName` both validates the app exists and narrows the traversal to the
+ * servers that app actually runs on — otherwise an unrelated server being
+ * unreachable would sink a command scoped to one app.
+ */
 export async function runRemoteCommandForTargets(
   cwd: string,
   command: (ctx: { config: ShipnodeConfig; executor: RemoteExecutor; serverName: string }) => Promise<void>,
-  options: { configPath?: string; includeEmpty?: boolean; appName?: string } = {},
+  options: { configPath?: string; includeEmpty?: boolean; appName?: string; serverName?: string } = {},
 ): Promise<void> {
-  const config = await loadConfig(cwd, options.configPath);
+  const workspace = await loadConfig(cwd, options.configPath);
+
+  let config = workspace;
   if (options.appName) {
-    const selected = configForAppResult(config, options.appName);
+    const selected = configForAppResult(workspace, options.appName);
     if (selected.isErr()) {
       ui.error(selected.error.message);
       process.exit(1);
       return;
     }
+    config = selected.value;
   }
 
-  try {
-    for (const target of getServerTargets(config)) {
-      const targetConfig = configForServer(config, target.name);
-      if (!options.includeEmpty && targetConfig.apps.length === 0 && Object.keys(targetConfig.accessories ?? {}).length === 0) {
-        continue;
-      }
+  let targets = getServerTargets(config);
+  if (options.serverName) {
+    targets = targets.filter((target) => target.name === options.serverName);
+    if (targets.length === 0) {
+      const known = getServerTargets(config).map((target) => target.name).join(', ') || '(none)';
+      ui.error(`Unknown server target '${options.serverName}'. Known targets: ${known}`);
+      process.exit(1);
+      return;
+    }
+  }
+
+  const failures: { serverName: string; message: string }[] = [];
+  let visited = 0;
 
-      const ssh = new SshConnection();
-      try {
-        await ssh.connect(target.ssh);
-        await command({ config: targetConfig, executor: ssh, serverName: target.name });
-      } finally {
-        ssh.disconnect();
-      }
+  for (const target of targets) {
+    const targetConfig = configForServer(config, target.name);
+    if (!options.includeEmpty && targetConfig.apps.length === 0 && Object.keys(targetConfig.accessories ?? {}).length === 0) {
+      continue;
     }
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error);
-    ui.error(message);
+
+    visited += 1;
+    const ssh = new SshConnection();
+    try {
+      await ssh.connect(target.ssh);
+      await command({ config: targetConfig, executor: ssh, serverName: target.name });
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      ui.error(`${target.name}: ${message}`);
+      failures.push({ serverName: target.name, message });
+    } finally {
+      ssh.disconnect();
+    }
+  }
+
+  if (failures.length > 0) {
+    ui.error(
+      `Failed on ${failures.length} of ${visited} servers: ` +
+      failures.map((failure) => failure.serverName).join(', '),
+    );
     process.exit(1);
   }
 }

src/config/assembly.ts

@@ -13,6 +13,7 @@ type LegacyPm2Input = {
 type AssembleInput = {
   ssh?: ShipnodeConfig['ssh'];
   servers?: ShipnodeConfig['servers'];
+  groups?: ShipnodeConfig['groups'];
   remotePath?: string;
   nodeVersion?: string;
   pkgManager?: ShipnodeConfig['pkgManager'];
@@ -26,7 +27,7 @@ type AssembleInput = {
   accessories?: ShipnodeConfig['accessories'];
   // Legacy input fields — synthesized to apps[0] by z.preprocess
   app?: string;
-  on?: string;
+  on?: string | string[];
   domain?: string;
   caddy?: ShipnodeApp['caddy'];
   pm2?: LegacyPm2Input | Pm2Config;

src/config/builder.ts

@@ -18,6 +18,7 @@ import { assembleConfig } from './assembly.js';
 type BuilderState = {
   ssh?: SshConfig;
   servers?: Record<string, SshConfig>;
+  groups?: Record<string, string[]>;
   remotePath?: string;
   nodeVersion?: string;
   pkgManager?: PkgManager;
@@ -31,7 +32,7 @@ type BuilderState = {
   accessories?: Record<string, AccessoryConfig>;
   // Legacy per-app input fields (synthesized to apps[0] by z.preprocess)
   app?: string;
-  on?: string;
+  on?: string | string[];
   pm2?: { apps: Pm2App[] };
   domain?: string;
   keepReleases?: number;
@@ -49,8 +50,10 @@ type BuilderState = {
   apps?: Partial<ShipnodeApp>[];
 };
 
-type AppBuilderState = Omit<Partial<ShipnodeApp>, 'pm2'> & {
+type AppBuilderState = Omit<Partial<ShipnodeApp>, 'pm2' | 'fleet'> & {
   pm2?: { apps: Pm2App[] };
+  // Partial: the schema fills in batch/port/drainWait/readyPath.
+  fleet?: Partial<ShipnodeApp['fleet']>;
 };
 
 export type WorkerOptions = Omit<Pm2App, 'port'>;
@@ -84,7 +87,21 @@ export class ShipnodeBuilder {
     return this;
   }
 
-  on(target: string): this {
+  /**
+   * Name a set of servers, usable anywhere a single server name is.
+   * `.group('web', ['web-a', 'web-b'])` then `.on('web')` runs an app on both.
+   */
+  group(name: string, servers: string[]): this {
+    this.config.groups = { ...this.config.groups, [name]: servers };
+    return this;
+  }
+
+  groups(groups: Record<string, string[]>): this {
+    this.config.groups = { ...this.config.groups, ...groups };
+    return this;
+  }
+
+  on(target: string | string[]): this {
     this.config.on = target;
     return this;
   }
@@ -343,11 +360,26 @@ export class ShipnodeAppBuilder {
     return this;
   }
 
-  on(target: string): this {
+  /** A server name, a group name, or a list of either. More than one is a fleet. */
+  on(target: string | string[]): this {
     this.state.on = target;
     return this;
   }
 
+  /**
+   * Rolling-deploy settings. Implied for any app that resolves to more than one
+   * server; declare it explicitly to put a single-server app behind the same
+   * drain contract, or to tune the roll.
+   *
+   * `drainWait` (seconds) must match your load balancer's health check — it is
+   * how long shipnode waits after flipping the readiness endpoint to 503 before
+   * it touches the app. Too low and in-flight requests are dropped.
+   */
+  fleet(opts: Partial<ShipnodeApp['fleet']> = {}): this {
+    this.state.fleet = { ...this.state.fleet, ...opts };
+    return this;
+  }
+
   caddy(opts: { append?: string }): this {
     this.state.caddy = opts;
     return this;