Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import type { Plugin, PluginContext } from '@objectstack/core';
import type { IClusterService } from '@objectstack/spec/contracts';
import type { ClusterCapabilityConfigInput } from '@objectstack/spec/kernel';
import { defineCluster } from './cluster.js';
import { assertClusterDriverSafeForTopology } from './split-brain-guard.js';

/**
* Options for `ClusterServicePlugin`.
Expand Down Expand Up @@ -59,6 +60,11 @@ export class ClusterServicePlugin implements Plugin {
this.cluster = defineCluster(this.options.config ?? {});
this.owned = true;
}
// Split-brain guard (ADR-0010): fail fast if a multi-node topology is
// declared but the resolved driver is in-process. Runs before
// registration so a misconfiguration never half-wires the service.
assertClusterDriverSafeForTopology(this.cluster.driver);

ctx.registerService('cluster', this.cluster);
ctx.logger.info(
`ClusterServicePlugin: registered "${this.cluster.driver}" driver (node=${this.cluster.nodeId})`,
Expand Down
6 changes: 6 additions & 0 deletions packages/services/service-cluster/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,12 @@ export {
type ClusterServicePluginOptions,
} from './cluster-service-plugin.js';

export {
assertClusterDriverSafeForTopology,
declaresMultiNode,
type SplitBrainGuardEnv,
} from './split-brain-guard.js';

export { MetadataClusterBridgePlugin } from './metadata-cluster-bridge-plugin.js';

// Re-export contracts for convenience.
Expand Down
61 changes: 61 additions & 0 deletions packages/services/service-cluster/src/split-brain-guard.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.

import { describe, it, expect } from 'vitest';
import {
declaresMultiNode,
assertClusterDriverSafeForTopology,
} from './split-brain-guard.js';

describe('declaresMultiNode', () => {
it('false when nothing declared', () => {
expect(declaresMultiNode({})).toBe(false);
});
it('true for OS_EXPECT_MULTI_NODE=true (case-insensitive, trimmed)', () => {
expect(declaresMultiNode({ OS_EXPECT_MULTI_NODE: 'true' })).toBe(true);
expect(declaresMultiNode({ OS_EXPECT_MULTI_NODE: ' TRUE ' })).toBe(true);
});
it('false for OS_EXPECT_MULTI_NODE=false / other', () => {
expect(declaresMultiNode({ OS_EXPECT_MULTI_NODE: 'false' })).toBe(false);
expect(declaresMultiNode({ OS_EXPECT_MULTI_NODE: '1' })).toBe(false);
});
it('replicas: >1 true, <=1 false, non-numeric false', () => {
expect(declaresMultiNode({ OS_CLUSTER_REPLICAS: '3' })).toBe(true);
expect(declaresMultiNode({ OS_CLUSTER_REPLICAS: '1' })).toBe(false);
expect(declaresMultiNode({ OS_CLUSTER_REPLICAS: '0' })).toBe(false);
expect(declaresMultiNode({ OS_CLUSTER_REPLICAS: 'abc' })).toBe(false);
});
});

describe('assertClusterDriverSafeForTopology', () => {
it('no throw: single-node + memory (the common case)', () => {
expect(() => assertClusterDriverSafeForTopology('memory', {})).not.toThrow();
});
it('no throw: multi-node + remote driver', () => {
expect(() =>
assertClusterDriverSafeForTopology('redis', { OS_EXPECT_MULTI_NODE: 'true' }),
).not.toThrow();
});
it('THROWS: multi-node (OS_EXPECT_MULTI_NODE) + memory', () => {
expect(() =>
assertClusterDriverSafeForTopology('memory', { OS_EXPECT_MULTI_NODE: 'true' }),
).toThrow(/split-brain/);
});
it('THROWS: multi-node (replicas>1) + memory, cites ADR-0010', () => {
expect(() =>
assertClusterDriverSafeForTopology('memory', { OS_CLUSTER_REPLICAS: '2' }),
).toThrow(/ADR-0010/);
});
it('escape hatch: multi-node + memory + OS_ALLOW_MEMORY_CLUSTER_MULTINODE=true', () => {
expect(() =>
assertClusterDriverSafeForTopology('memory', {
OS_EXPECT_MULTI_NODE: 'true',
OS_ALLOW_MEMORY_CLUSTER_MULTINODE: 'true',
}),
).not.toThrow();
});
it('does not guard unknown custom drivers (author responsibility)', () => {
expect(() =>
assertClusterDriverSafeForTopology('mycustom', { OS_EXPECT_MULTI_NODE: 'true' }),
).not.toThrow();
});
});
78 changes: 78 additions & 0 deletions packages/services/service-cluster/src/split-brain-guard.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.

/**
* Split-brain guard (ADR-0010, path A).
*
* The `memory` cluster driver keeps PubSub/Lock/KV/Counter state **per
* process**. On a single node that is correct; across multiple replicas
* each process holds its own state, so:
* - `ILock` is acquired locally by every replica -> no mutual exclusion;
* - `ICounter`/`IKV` versions diverge per replica;
* - `IPubSub` does not fan out across processes.
*
* This fails silently -- single-node tests and dev pass, only production
* multi-replica corrupts. The guard turns that silent corruption into a
* loud startup error when the operator has *declared* a multi-node
* topology yet wired an in-process driver.
*
* Detection is deliberately conservative (path A): it keys off an
* explicit operator declaration, not active peer discovery (that is the
* optional path B in ADR-0010). It cannot know at boot whether the
* cluster primitives are actually used cross-node, so it offers an
* escape hatch for the rare "replicas declared but primitives unused"
* case.
*/

/** In-process drivers whose state does not coordinate across replicas. */
const IN_PROCESS_DRIVERS = new Set(['memory']);

/** Environment inputs the guard reads. */
export interface SplitBrainGuardEnv {
/** `'true'` -> operator declares a multi-node deployment. */
OS_EXPECT_MULTI_NODE?: string;
/** Replica count; `> 1` -> multi-node. */
OS_CLUSTER_REPLICAS?: string;
/** `'true'` -> bypass the guard (replicas declared but primitives unused cross-node). */
OS_ALLOW_MEMORY_CLUSTER_MULTINODE?: string;
}

function isTrue(v: string | undefined): boolean {
return String(v).trim().toLowerCase() === 'true';
}

/**
* True when the operator has declared a multi-node topology via
* `OS_EXPECT_MULTI_NODE=true` or `OS_CLUSTER_REPLICAS` greater than 1.
*/
export function declaresMultiNode(env: SplitBrainGuardEnv = process.env): boolean {
if (isTrue(env.OS_EXPECT_MULTI_NODE)) return true;
const replicas = Number(env.OS_CLUSTER_REPLICAS);
return Number.isFinite(replicas) && replicas > 1;
}

/**
* Throw if a multi-node topology is declared while the resolved cluster
* `driver` is in-process (`memory`), unless explicitly allowed via
* `OS_ALLOW_MEMORY_CLUSTER_MULTINODE=true`.
*
* Call this at startup *before* registering the cluster service so a
* misconfiguration fails fast.
*/
export function assertClusterDriverSafeForTopology(
driver: string,
env: SplitBrainGuardEnv = process.env,
): void {
if (!declaresMultiNode(env)) return;
if (!IN_PROCESS_DRIVERS.has(driver)) return;
if (isTrue(env.OS_ALLOW_MEMORY_CLUSTER_MULTINODE)) return;

throw new Error(
`ClusterServicePlugin: multi-node deployment declared ` +
`(OS_EXPECT_MULTI_NODE / OS_CLUSTER_REPLICAS>1) but the cluster driver is ` +
`in-process "${driver}" -- its locks/counters/pub-sub do not coordinate across ` +
`processes, so multiple replicas silently split-brain. Configure a remote cluster ` +
`driver (e.g. @objectstack/service-cluster-redis) or a DB-backed driver. To override ` +
`(replicas declared but cluster primitives unused cross-node), set ` +
`OS_ALLOW_MEMORY_CLUSTER_MULTINODE=true. See cloud ADR-0010.`,
);
}