fix: stop multi-server workspaces provisioning and routing over each other

Three bugs made a workspace with more than one server unsafe. Each is
independently reachable today; together they make the multi-server path
unusable, and replication would multiply all three.

Managed services were installed on every host. `database` and `redis` are
workspace-level, and configForServer spread the whole config onto each
scoped copy, so setup's localhost guard fired once per server: a 3-server
workspace installed and initialised Postgres three times, same user, same
database name. Both now carry an `on` target and are stripped from the
servers that do not host them.

Cloudflare init repointed every domain at whichever host ran last. The
orchestrator iterated the entire workspace app list, so DNS, ingress,
firewall lockdown and audit all described apps that live elsewhere; the
final CNAME upsert won and the other server's apps 404'd through the
tunnel catch-all. It now requires a server-scoped config and the commands
fan out. Two consequences fall out of that: sshHostname is one workspace
name, so exactly one tunnel may claim it, and an explicit tunnelName shared
across several tunnel-needing servers is rejected up front rather than
failing halfway through on the missing-credentials guard.

Servers were visited in the order they appear in the `servers` literal, and
each starts only its own accessories. Declare the app server before the
database server and the app deploys and health-checks before the database
it depends on exists — reordering two lines silently fixed it. getServerTargets
now sorts by dependsOn, declaration order breaking ties. Cycles fall back to
declaration order instead of throwing, since two servers can legitimately
host accessories the other consumes. The cross-server reachability warning
also moves out of --dry-run into the real deploy path, resolved against the
whole workspace because a per-server config cannot see other servers'
accessories.

runRemoteCommandForTargets had no test coverage at all — its mocked config
did not even carry a `servers` key. Traversal order, per-server scoping, the
includeEmpty skip, connection lifecycle and --app validation are now pinned.
The abort-on-failure test documents current behaviour: the try/catch sits
outside the loop, so one server failing abandons the rest with no account of
what succeeded. That is left deliberately visible for the fleet work to change.

One behaviour change beyond the fix: backup reads config.database, so a
database is now dumped only on the server that hosts it, instead of every
server attempting a pg_dump of a database only one of them has.

Author: devalade

src/cli/commands/cloudflare.ts

@@ -1,6 +1,9 @@
-import { runRemoteCommand } from '../runner.js';
+import { runRemoteCommandForTargets } from '../runner.js';
 import { ui } from '../ui.js';
 import { CloudflareOrchestrator } from '../../services/cloudflare.orchestrator.js';
+import { loadConfig } from '../../config/loader.js';
+import { getServerTargets, configForServer, resolveServerName } from '../../domain/servers.js';
+import type { ShipnodeConfig } from '../../shared/types.js';
 
 function requireToken(): string {
   const token = process.env['CLOUDFLARE_API_TOKEN'] ?? process.env['CF_API_TOKEN'];
@@ -11,40 +14,91 @@ function requireToken(): string {
   return token;
 }
 
+/** Servers that actually host an app with a domain, and so need a tunnel. */
+function tunnelledServers(config: ShipnodeConfig): string[] {
+  return getServerTargets(config)
+    .map((target) => target.name)
+    .filter((name) => configForServer(config, name).apps.some((app) => app.domain));
+}
+
+/**
+ * A tunnel name identifies one host's connector, and its credentials live on
+ * that host. Sharing an explicit name across servers makes the second `init`
+ * fail on the missing-credentials guard, so reject it up front with a fix
+ * rather than halfway through the fan-out.
+ */
+function assertTunnelNameIsUnambiguous(config: ShipnodeConfig): void {
+  const explicit = config.cloudflare?.tunnelName;
+  if (!explicit) return;
+
+  const servers = tunnelledServers(config);
+  if (servers.length <= 1) return;
+
+  ui.error(
+    `cloudflare.tunnelName is set to "${explicit}", but ${servers.length} servers host apps with domains ` +
+    `(${servers.join(', ')}). A tunnel belongs to one host. Remove tunnelName to get a per-host default ` +
+    `(shipnode-<host>), or run cloudflare init against one server at a time.`,
+  );
+  process.exit(1);
+}
+
 export async function cmdCloudflareInit(
   cwd: string,
   options: { config?: string },
 ): Promise<void> {
-  await runRemoteCommand(cwd, async ({ config, executor }) => {
-    const orchestrator = new CloudflareOrchestrator(executor, config, requireToken());
-    await orchestrator.init();
-    ui.success(`Cloudflare tunnel initialized.`);
-  }, { configPath: options.config });
+  const workspace = await loadConfig(cwd, options.config);
+  assertTunnelNameIsUnambiguous(workspace);
+
+  // Only one tunnel can own the workspace's ssh hostname.
+  const sshHostnameOwner = resolveServerName(workspace);
+
+  await runRemoteCommandForTargets(
+    cwd,
+    async ({ config, executor, serverName }) => {
+      ui.step(`Cloudflare: ${serverName} (${config.ssh.user}@${config.ssh.host})`);
+      const orchestrator = new CloudflareOrchestrator(executor, config, requireToken(), {
+        manageSshHostname: serverName === sshHostnameOwner,
+      });
+      await orchestrator.init();
+      ui.success(`Cloudflare tunnel initialized on ${serverName}.`);
+    },
+    { configPath: options.config },
+  );
 }
 
 export async function cmdCloudflareAudit(
   cwd: string,
   options: { config?: string },
 ): Promise<void> {
-  await runRemoteCommand(cwd, async ({ config, executor }) => {
-    const orchestrator = new CloudflareOrchestrator(executor, config, requireToken());
-    const result = await orchestrator.audit();
-    console.log(`  Zone: ${result.zone.name} (${result.zone.id}) — ${result.zone.status}`);
-    for (const app of result.apps) {
-      console.log(`  DNS ${app.domain} → localhost:${app.port}`);
-    }
-    console.log(`  cloudflared tunnels:\n${result.tunnelList}`);
-    console.log(`  cloudflared service: ${result.service}`);
-  }, { configPath: options.config });
+  await runRemoteCommandForTargets(
+    cwd,
+    async ({ config, executor, serverName }) => {
+      const orchestrator = new CloudflareOrchestrator(executor, config, requireToken());
+      const result = await orchestrator.audit();
+      ui.heading(`Server: ${serverName} (${config.ssh.user}@${config.ssh.host})`);
+      console.log(`  Zone: ${result.zone.name} (${result.zone.id}) — ${result.zone.status}`);
+      for (const app of result.apps) {
+        console.log(`  DNS ${app.domain} → localhost:${app.port}`);
+      }
+      console.log(`  cloudflared tunnels:\n${result.tunnelList}`);
+      console.log(`  cloudflared service: ${result.service}`);
+    },
+    { configPath: options.config },
+  );
 }
 
 export async function cmdCloudflareStatus(
   cwd: string,
   options: { config?: string },
 ): Promise<void> {
-  await runRemoteCommand(cwd, async ({ config, executor }) => {
-    const orchestrator = new CloudflareOrchestrator(executor, config, requireToken());
-    const output = await orchestrator.status();
-    console.log(output);
-  }, { configPath: options.config });
+  await runRemoteCommandForTargets(
+    cwd,
+    async ({ config, executor, serverName }) => {
+      const orchestrator = new CloudflareOrchestrator(executor, config, requireToken());
+      const output = await orchestrator.status();
+      ui.heading(`Server: ${serverName} (${config.ssh.user}@${config.ssh.host})`);
+      console.log(output);
+    },
+    { configPath: options.config },
+  );
 }

src/cli/commands/deploy.ts

@@ -52,6 +52,11 @@ export async function cmdDeploy(cwd: string, options: { dryRun?: boolean; skipBu
     return;
   }
 
+  // The per-server config the callback receives only carries that server's own
+  // accessories, so cross-server dependencies are invisible from inside it.
+  // Dependency warnings have to be resolved against the whole workspace.
+  const workspaceConfig = config;
+
   await runRemoteCommandForTargets(
     cwd,
     async ({ config, executor, serverName }) => {
@@ -66,6 +71,12 @@ export async function cmdDeploy(cwd: string, options: { dryRun?: boolean; skipBu
       const label = names || Object.keys(deployConfig.accessories ?? {}).join(', ');
       ui.step(`Deploying ${chalk.bold(label)} → ${serverName} (${config.ssh.user}@${config.ssh.host})`);
 
+      for (const deployingApp of deployConfig.apps) {
+        for (const warning of renderDependencyWarnings(workspaceConfig, deployingApp)) {
+          ui.warn(warning);
+        }
+      }
+
       const deployer = new DeployService(new LoggingExecutor(executor), deployConfig);
       await deployer.execute(cwd, options.skipBuild ?? false);
 

src/config/schema.ts

@@ -79,6 +79,7 @@ export const HealthCheckConfigSchema = z.object({
 }).default({});
 
 const networkDbFields = {
+  on: z.string().min(1).optional(),
   host: z.string().min(1, 'Database host is required'),
   port: z.number().int().min(1).max(65535),
   name: z.string().min(1, 'Database name is required'),
@@ -87,13 +88,18 @@ const networkDbFields = {
 };
 
 export const DatabaseConfigSchema = z.discriminatedUnion('type', [
-  z.object({ type: z.literal('sqlite'), name: z.string().min(1, 'SQLite file path is required') }),
+  z.object({
+    type: z.literal('sqlite'),
+    on: z.string().min(1).optional(),
+    name: z.string().min(1, 'SQLite file path is required'),
+  }),
   z.object({ type: z.literal('postgres'), ...networkDbFields }),
   z.object({ type: z.literal('mysql'), ...networkDbFields }),
   z.object({ type: z.literal('mongodb'), ...networkDbFields }),
 ]).optional();
 
 export const RedisConfigSchema = z.object({
+  on: z.string().min(1).optional(),
   host: z.string().default('localhost'),
   port: z.number().int().min(1).max(65535).default(6379),
   password: z.string().optional(),
@@ -264,6 +270,8 @@ const ShipnodeConfigBaseSchema = z.object({
   Object.entries(cfg.accessories ?? {}).forEach(([name, accessory]) => {
     validateTarget(accessory.on, ['accessories', name, 'on']);
   });
+  if (cfg.database) validateTarget(cfg.database.on, ['database', 'on']);
+  if (cfg.redis) validateTarget(cfg.redis.on, ['redis', 'on']);
   const accessoryNames = new Set(Object.keys(cfg.accessories ?? {}));
   cfg.apps.forEach((app, index) => {
     for (const name of app.dependsOn ?? []) {

src/domain/servers.ts

@@ -49,8 +49,69 @@ export function getServerTargetResult(
   return Result.ok({ name, ssh });
 }
 
+/**
+ * Which servers must be visited before which, derived from `dependsOn`.
+ *
+ * Maps a server to the set of *other* servers it needs first. An app declaring
+ * `dependsOn: ['postgres']` where postgres lives elsewhere means that
+ * accessory's host has to be up before this one is deployed and health-checked.
+ * Same-server dependencies impose no ordering — the orchestrator already
+ * starts a server's own accessories before its apps.
+ */
+function serverPrerequisites(config: ShipnodeConfig): Map<string, Set<string>> {
+  const needs = new Map<string, Set<string>>();
+
+  for (const app of config.apps) {
+    const appServer = resolveServerName(config, app.on);
+    for (const name of app.dependsOn ?? []) {
+      const accessory = config.accessories?.[name];
+      if (!accessory) continue;
+      const accessoryServer = resolveServerName(config, accessory.on);
+      if (accessoryServer === appServer) continue;
+
+      const existing = needs.get(appServer) ?? new Set<string>();
+      existing.add(accessoryServer);
+      needs.set(appServer, existing);
+    }
+  }
+
+  return needs;
+}
+
+/**
+ * Every server, ordered so that a server hosting an accessory comes before the
+ * servers whose apps depend on it.
+ *
+ * Declaration order is the tiebreak, so a workspace without cross-server
+ * `dependsOn` is traversed exactly as written. Without this, ordering is purely
+ * the order of the `servers` literal: declare the app server first and its
+ * health check runs before the database it needs has been started.
+ *
+ * A dependency cycle is not an error — two servers can legitimately host
+ * accessories the other's apps consume. Cycle members fall back to declaration
+ * order rather than throwing.
+ */
 export function getServerTargets(config: ShipnodeConfig): ServerTarget[] {
-  return Object.entries(config.servers).map(([name, ssh]) => ({ name, ssh }));
+  const declared = Object.entries(config.servers).map(([name, ssh]) => ({ name, ssh }));
+  const needs = serverPrerequisites(config);
+  if (needs.size === 0) return declared;
+
+  const ordered: ServerTarget[] = [];
+  const placed = new Set<string>();
+  const remaining = [...declared];
+
+  while (remaining.length > 0) {
+    const ready = remaining.findIndex((target) =>
+      [...(needs.get(target.name) ?? [])].every((dep) => placed.has(dep) || !config.servers[dep]),
+    );
+    // -1 means everything left is in a cycle; take the first to stay deterministic.
+    const [next] = remaining.splice(ready === -1 ? 0 : ready, 1);
+    if (!next) break;
+    ordered.push(next);
+    placed.add(next.name);
+  }
+
+  return ordered;
 }
 
 export function getAppsForServer(config: ShipnodeConfig, serverName: string): ShipnodeApp[] {
@@ -63,13 +124,29 @@ export function getAccessoriesForServer(config: ShipnodeConfig, serverName: stri
   return Object.fromEntries(entries);
 }
 
+/**
+ * Managed services are provisioned on exactly one server. Returning the config
+ * verbatim would hand `database`/`redis` to every scoped config, and `setup`
+ * would then install Postgres once per host — same user, same database name.
+ */
+function serviceForServer<T extends { on?: string }>(
+  config: ShipnodeConfig,
+  service: T | undefined,
+  serverName: string,
+): T | undefined {
+  if (!service) return undefined;
+  return resolveServerName(config, service.on) === serverName ? service : undefined;
+}
+
 export function configForServer(config: ShipnodeConfig, serverName: string): ShipnodeConfig {
   const ssh = getServerTarget(config, serverName).ssh;
   return {
     ...config,
     ssh,
     apps: getAppsForServer(config, serverName),
     accessories: getAccessoriesForServer(config, serverName),
+    database: serviceForServer(config, config.database, serverName),
+    redis: serviceForServer(config, config.redis, serverName),
   };
 }
 

src/services/cloudflare.orchestrator.ts

@@ -9,10 +9,20 @@ const CFD_INGRESS_TARGET = '.cfargotunnel.com';
 export class CloudflareOrchestrator {
   private api: CloudflareApi;
 
+  /**
+   * `config` must be scoped to the host `executor` is connected to (see
+   * `configForServer`). Every app in it gets a DNS record pointing at *this*
+   * host's tunnel, so handing it the whole workspace in a multi-server setup
+   * repoints other servers' domains at the wrong tunnel.
+   *
+   * `manageSshHostname` is false for every server but one: `cloudflare.sshHostname`
+   * is a single workspace-level name and only one tunnel can own it.
+   */
   constructor(
     private executor: RemoteExecutor,
     private config: ShipnodeConfig,
     apiToken: string,
+    private options: { manageSshHostname?: boolean } = {},
   ) {
     this.api = new CloudflareApi(apiToken);
   }
@@ -85,7 +95,7 @@ export class CloudflareOrchestrator {
       }
     }
 
-    if (cf.sshHostname) {
+    if (cf.sshHostname && this.options.manageSshHostname !== false) {
       tunnel.addIngress(cf.sshHostname, 'ssh://localhost:22');
       await this.api.upsertDnsRecord(zoneId, {
         type: 'CNAME',

src/shared/types.ts

@@ -7,11 +7,19 @@ export type DatabaseType = 'postgres' | 'mysql' | 'sqlite' | 'mongodb';
 
 export interface SqliteDatabaseConfig {
   type: 'sqlite';
+  /**
+   * Which server hosts this database. Provisioning (`shipnode setup`) only runs
+   * on that server; every other server's scoped config drops the field. Omitted,
+   * it resolves the same way an app's `on` does — `default`, or the sole server.
+   */
+  on?: string;
   name: string;
 }
 
 export interface NetworkDatabaseConfig {
   type: 'postgres' | 'mysql' | 'mongodb';
+  /** Which server hosts this database. See {@link SqliteDatabaseConfig.on}. */
+  on?: string;
   host: string;
   port: number;
   name: string;
@@ -96,6 +104,8 @@ export interface HealthCheckConfig {
 export type DatabaseConfig = SqliteDatabaseConfig | NetworkDatabaseConfig;
 
 export interface RedisConfig {
+  /** Which server hosts Redis. See {@link SqliteDatabaseConfig.on}. */
+  on?: string;
   host: string;
   port: number;
   password?: string;