feat(service-cluster): split-brain startup guard for memory driver (ADR-0010 path A) (#1841)

The memory cluster driver keeps PubSub/Lock/KV/Counter state per-process.
Across multiple replicas each process holds its own state → locks grant
to everyone, counters/KV versions diverge, pub/sub doesn't fan out. This
fails silently: single-node tests/dev pass, only production multi-replica
corrupts.

Add a conservative startup guard (ADR-0010 path A): when a multi-node
topology is *declared* (OS_EXPECT_MULTI_NODE=true or OS_CLUSTER_REPLICAS>1)
but the resolved driver is in-process "memory", ClusterServicePlugin throws
at init (before registering the service) with an actionable message. Escape
hatch OS_ALLOW_MEMORY_CLUSTER_MULTINODE=true for the rare "replicas declared
but cluster primitives unused cross-node" case.

Pure, unit-tested decision (declaresMultiNode / assertClusterDriverSafeForTopology),
exported for reuse. Path B (active DB-presence detection) is deferred per ADR-0010.

Refs: cloud ADR-0010 (D6).

Co-authored-by: Jack Zhuang <277994282+os-zhuang@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: 3 people

packages/services/service-cluster/src/cluster-service-plugin.ts

@@ -4,6 +4,7 @@ import type { Plugin, PluginContext } from '@objectstack/core';
 import type { IClusterService } from '@objectstack/spec/contracts';
 import type { ClusterCapabilityConfigInput } from '@objectstack/spec/kernel';
 import { defineCluster } from './cluster.js';
+import { assertClusterDriverSafeForTopology } from './split-brain-guard.js';
 
 /**
  * Options for `ClusterServicePlugin`.
@@ -59,6 +60,11 @@ export class ClusterServicePlugin implements Plugin {
             this.cluster = defineCluster(this.options.config ?? {});
             this.owned = true;
         }
+        // Split-brain guard (ADR-0010): fail fast if a multi-node topology is
+        // declared but the resolved driver is in-process. Runs before
+        // registration so a misconfiguration never half-wires the service.
+        assertClusterDriverSafeForTopology(this.cluster.driver);
+
         ctx.registerService('cluster', this.cluster);
         ctx.logger.info(
             `ClusterServicePlugin: registered "${this.cluster.driver}" driver (node=${this.cluster.nodeId})`,

packages/services/service-cluster/src/index.ts

@@ -39,6 +39,12 @@ export {
     type ClusterServicePluginOptions,
 } from './cluster-service-plugin.js';
 
+export {
+    assertClusterDriverSafeForTopology,
+    declaresMultiNode,
+    type SplitBrainGuardEnv,
+} from './split-brain-guard.js';
+
 export { MetadataClusterBridgePlugin } from './metadata-cluster-bridge-plugin.js';
 
 // Re-export contracts for convenience.

packages/services/service-cluster/src/split-brain-guard.test.ts

@@ -0,0 +1,61 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+import { describe, it, expect } from 'vitest';
+import {
+    declaresMultiNode,
+    assertClusterDriverSafeForTopology,
+} from './split-brain-guard.js';
+
+describe('declaresMultiNode', () => {
+    it('false when nothing declared', () => {
+        expect(declaresMultiNode({})).toBe(false);
+    });
+    it('true for OS_EXPECT_MULTI_NODE=true (case-insensitive, trimmed)', () => {
+        expect(declaresMultiNode({ OS_EXPECT_MULTI_NODE: 'true' })).toBe(true);
+        expect(declaresMultiNode({ OS_EXPECT_MULTI_NODE: ' TRUE ' })).toBe(true);
+    });
+    it('false for OS_EXPECT_MULTI_NODE=false / other', () => {
+        expect(declaresMultiNode({ OS_EXPECT_MULTI_NODE: 'false' })).toBe(false);
+        expect(declaresMultiNode({ OS_EXPECT_MULTI_NODE: '1' })).toBe(false);
+    });
+    it('replicas: >1 true, <=1 false, non-numeric false', () => {
+        expect(declaresMultiNode({ OS_CLUSTER_REPLICAS: '3' })).toBe(true);
+        expect(declaresMultiNode({ OS_CLUSTER_REPLICAS: '1' })).toBe(false);
+        expect(declaresMultiNode({ OS_CLUSTER_REPLICAS: '0' })).toBe(false);
+        expect(declaresMultiNode({ OS_CLUSTER_REPLICAS: 'abc' })).toBe(false);
+    });
+});
+
+describe('assertClusterDriverSafeForTopology', () => {
+    it('no throw: single-node + memory (the common case)', () => {
+        expect(() => assertClusterDriverSafeForTopology('memory', {})).not.toThrow();
+    });
+    it('no throw: multi-node + remote driver', () => {
+        expect(() =>
+            assertClusterDriverSafeForTopology('redis', { OS_EXPECT_MULTI_NODE: 'true' }),
+        ).not.toThrow();
+    });
+    it('THROWS: multi-node (OS_EXPECT_MULTI_NODE) + memory', () => {
+        expect(() =>
+            assertClusterDriverSafeForTopology('memory', { OS_EXPECT_MULTI_NODE: 'true' }),
+        ).toThrow(/split-brain/);
+    });
+    it('THROWS: multi-node (replicas>1) + memory, cites ADR-0010', () => {
+        expect(() =>
+            assertClusterDriverSafeForTopology('memory', { OS_CLUSTER_REPLICAS: '2' }),
+        ).toThrow(/ADR-0010/);
+    });
+    it('escape hatch: multi-node + memory + OS_ALLOW_MEMORY_CLUSTER_MULTINODE=true', () => {
+        expect(() =>
+            assertClusterDriverSafeForTopology('memory', {
+                OS_EXPECT_MULTI_NODE: 'true',
+                OS_ALLOW_MEMORY_CLUSTER_MULTINODE: 'true',
+            }),
+        ).not.toThrow();
+    });
+    it('does not guard unknown custom drivers (author responsibility)', () => {
+        expect(() =>
+            assertClusterDriverSafeForTopology('mycustom', { OS_EXPECT_MULTI_NODE: 'true' }),
+        ).not.toThrow();
+    });
+});

packages/services/service-cluster/src/split-brain-guard.ts

@@ -0,0 +1,78 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+/**
+ * Split-brain guard (ADR-0010, path A).
+ *
+ * The `memory` cluster driver keeps PubSub/Lock/KV/Counter state **per
+ * process**. On a single node that is correct; across multiple replicas
+ * each process holds its own state, so:
+ *   - `ILock` is acquired locally by every replica -> no mutual exclusion;
+ *   - `ICounter`/`IKV` versions diverge per replica;
+ *   - `IPubSub` does not fan out across processes.
+ *
+ * This fails silently -- single-node tests and dev pass, only production
+ * multi-replica corrupts. The guard turns that silent corruption into a
+ * loud startup error when the operator has *declared* a multi-node
+ * topology yet wired an in-process driver.
+ *
+ * Detection is deliberately conservative (path A): it keys off an
+ * explicit operator declaration, not active peer discovery (that is the
+ * optional path B in ADR-0010). It cannot know at boot whether the
+ * cluster primitives are actually used cross-node, so it offers an
+ * escape hatch for the rare "replicas declared but primitives unused"
+ * case.
+ */
+
+/** In-process drivers whose state does not coordinate across replicas. */
+const IN_PROCESS_DRIVERS = new Set(['memory']);
+
+/** Environment inputs the guard reads. */
+export interface SplitBrainGuardEnv {
+    /** `'true'` -> operator declares a multi-node deployment. */
+    OS_EXPECT_MULTI_NODE?: string;
+    /** Replica count; `> 1` -> multi-node. */
+    OS_CLUSTER_REPLICAS?: string;
+    /** `'true'` -> bypass the guard (replicas declared but primitives unused cross-node). */
+    OS_ALLOW_MEMORY_CLUSTER_MULTINODE?: string;
+}
+
+function isTrue(v: string | undefined): boolean {
+    return String(v).trim().toLowerCase() === 'true';
+}
+
+/**
+ * True when the operator has declared a multi-node topology via
+ * `OS_EXPECT_MULTI_NODE=true` or `OS_CLUSTER_REPLICAS` greater than 1.
+ */
+export function declaresMultiNode(env: SplitBrainGuardEnv = process.env): boolean {
+    if (isTrue(env.OS_EXPECT_MULTI_NODE)) return true;
+    const replicas = Number(env.OS_CLUSTER_REPLICAS);
+    return Number.isFinite(replicas) && replicas > 1;
+}
+
+/**
+ * Throw if a multi-node topology is declared while the resolved cluster
+ * `driver` is in-process (`memory`), unless explicitly allowed via
+ * `OS_ALLOW_MEMORY_CLUSTER_MULTINODE=true`.
+ *
+ * Call this at startup *before* registering the cluster service so a
+ * misconfiguration fails fast.
+ */
+export function assertClusterDriverSafeForTopology(
+    driver: string,
+    env: SplitBrainGuardEnv = process.env,
+): void {
+    if (!declaresMultiNode(env)) return;
+    if (!IN_PROCESS_DRIVERS.has(driver)) return;
+    if (isTrue(env.OS_ALLOW_MEMORY_CLUSTER_MULTINODE)) return;
+
+    throw new Error(
+        `ClusterServicePlugin: multi-node deployment declared ` +
+            `(OS_EXPECT_MULTI_NODE / OS_CLUSTER_REPLICAS>1) but the cluster driver is ` +
+            `in-process "${driver}" -- its locks/counters/pub-sub do not coordinate across ` +
+            `processes, so multiple replicas silently split-brain. Configure a remote cluster ` +
+            `driver (e.g. @objectstack/service-cluster-redis) or a DB-backed driver. To override ` +
+            `(replicas declared but cluster primitives unused cross-node), set ` +
+            `OS_ALLOW_MEMORY_CLUSTER_MULTINODE=true. See cloud ADR-0010.`,
+    );
+}