feat(cluster): spec-5.57 cross-instance CR read-path coordinator boundary

Make the "cross-node table content read is unavailable" wall an explicit,
observable, frozen read-path coordinator boundary (CR-9) instead of six
scattered fail-closed + forward-link sites.  Contract + hardening +
observability only; the cross-instance data plane stays fail-closed and
forwards to Stage 6 (#119) -- the value gate proves it is unreachable in
Stage 5.5 (legal NO-GO).

- D2 (Q11-A): CR-side remote-UBA pre-check classifies the segment owner BEFORE
  the undo read; a runtime-warm cross-instance origin (class③) fails closed with
  the canonical 53R9G via a shared refusal routine, instead of being conflated
  with own-instance retention-recycled undo (53R9F).  NON-DEGRADABLE: the
  ereport fires under any GUC value; cluster_undo_get_record's return-0 contract
  (recovery / SRF callers) is preserved, not blanket-converted to ERROR.
- D3: independent observability region "pgrac cluster cr coordinator" + 4
  counters + pg_cluster_state category 'cr_coord'; GUCs
  cluster.cross_instance_cr_coordinator (off/boundary/forward) +
  cluster.cross_instance_cr_probe.  The GUC gates only the observability
  surface, never the fail-closed boundary (rule 8.A).
- D1: unify the W1/W2/W3 (+ stale GCS) forward-links onto Spec: spec-5.57.
- D4/D5/D6/D7/D8: freeze the version-selection + requester-side fabrication
  contract, the cross-instance key/epoch identity contract (origin is a routing
  dimension, not a CR key field), the C4 class③ invalidation contract, and the
  wire payload byte-layout (spec text only, no struct/enum -- AD-013).
- D0: scripts/perf/run-cross-instance-cr-probe.sh measure-leg (static value gate
  + soundness-gate checklist + self-test).
- D9: cluster_unit test_cluster_cr_coordinator (classifier / counters /
  namespace / GUC defaults) + cluster_tap t/318 (real 2-node fail-closed
  boundary e2e: 53R9G + counters, probe, forward LOG-once, off-mode
  non-degradability, SRF return-0 audit, measure-leg).

catversion not bumped (no catalog / SRF / wire ABI).  Region +1 / category +1
baselines regrounded.

Spec: spec-5.57-cross-instance-cr-current-coordinator.md

Author: SqlRush

scripts/perf/run-cross-instance-cr-probe.sh

@@ -0,0 +1,167 @@
+#!/bin/bash
+#-------------------------------------------------------------------------
+#
+# run-cross-instance-cr-probe.sh -- spec-5.57 D0 measure-leg (value gate +
+#    soundness gate) for the cross-instance CR/current read-path coordinator
+#    boundary.
+#
+# This is a MEASURE-ONLY tool (zero product code).  It answers the spec-5.57
+# value-gate question: "is the class③ runtime-warm-remote CR data plane reachable
+# / worth opening in Stage 5.5?"  The answer is determined by STATIC evidence that
+# survives in CI (no harness needed): the codebase has NO over-the-wire remote
+# undo fetch path and the CR walker fail-closes class③ with 53R9G.  Therefore the
+# value-gate verdict is NO-GO-in-5.5 (data plane -> Stage 6 #119) -- a LEGAL NO-GO
+# (據實 not降 scope), same paradigm as spec-5.50/5.55/5.56.
+#
+# It ALSO prints the L257 soundness-gate checklist (§3.4) with a per-item ruling:
+# 5.57 verifies failure-direction (GREEN, the fail-closed boundary) here; the
+# other four gates are Stage 6 data-plane ship gates (listed, owned by Stage 6).
+#
+# An OPTIONAL --dynamic 2-node leg (off by default; needs the local IPC::Run TAP
+# harness) confirms at runtime that cross-node reads fail closed.
+#
+# Author: SqlRush <sqlrush@gmail.com>
+# Portions Copyright (c) 2026, pgrac contributors
+#
+# Spec: spec-5.57-cross-instance-cr-current-coordinator.md (FROZEN v1.0)
+#
+# IDENTIFICATION
+#    scripts/perf/run-cross-instance-cr-probe.sh
+#
+# NOTES
+#    pgrac-original.  --self-test parses the --static output (L223: validate
+#    CONTENT, not mere file existence) and is the CI-safe entry point.
+#
+#-------------------------------------------------------------------------
+set -euo pipefail
+
+PROGNAME="run-cross-instance-cr-probe.sh"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SRCROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+CR_C="$SRCROOT/src/backend/cluster/cluster_cr.c"
+UNDO_C="$SRCROOT/src/backend/cluster/cluster_undo_record.c"
+WAIT_H="$SRCROOT/src/include/utils/wait_event.h"
+
+usage() {
+	cat <<EOF
+$PROGNAME -- spec-5.57 D0 cross-instance CR measure-leg.
+
+Usage:
+  $PROGNAME --static       Emit the static value-gate + soundness-gate evidence.
+  $PROGNAME --self-test    Run --static, parse the output, assert the verdict (CI-safe).
+  $PROGNAME --dynamic      (optional) Additionally run a 2-node fail-closed probe.
+  $PROGNAME --help
+EOF
+}
+
+# ---- static value gate -------------------------------------------------
+# Each GATE line is machine-parseable: "GATE <name> <PASS|FAIL> <detail>".
+emit_static() {
+	echo "# spec-5.57 D0 measure-leg: cross-instance CR value gate (STATIC evidence)"
+	echo "# srcroot=$SRCROOT"
+
+	# E1: no over-the-wire remote undo fetch -- cluster_undo_get_record returns 0
+	#     for a non-own, non-materialized (runtime-warm) origin.
+	if grep -q "runtime cross-instance undo read not supported" "$UNDO_C" 2>/dev/null; then
+		echo "GATE no_remote_undo_fetch PASS cluster_undo_get_record returns 0 for runtime cross-instance (no wire fetch)"
+	else
+		echo "GATE no_remote_undo_fetch FAIL marker not found in cluster_undo_record.c"
+	fi
+
+	# E2: the CR walker fail-closes class③ with the canonical 53R9G (not a silent
+	#     fallback, not 53R9F snapshot-too-old) at the remote-undo-read leg.
+	if grep -q "remote-undo-read \+\"" "$CR_C" 2>/dev/null \
+		|| grep -q "remote-undo-read" "$CR_C" 2>/dev/null; then
+		echo "GATE class3_fail_closed PASS CR walker fail-closes class③ with ERRCODE_CLUSTER_CR_CROSS_INSTANCE_UNSUPPORTED (53R9G)"
+	else
+		echo "GATE class3_fail_closed FAIL pre-check marker not found in cluster_cr.c"
+	fi
+
+	# E3: GCS ships only current blocks; the CR-ship wait events are reserved /
+	#     dormant (declared but no producer) -- no CR-image wire path.
+	if grep -q "WAIT_EVENT_BUFFER_SHIP_CR_BUILD" "$WAIT_H" 2>/dev/null \
+		&& ! grep -rq "WAIT_EVENT_BUFFER_SHIP_CR_BUILD" "$SRCROOT/src/backend/cluster/cluster_gcs_block.c" 2>/dev/null; then
+		echo "GATE no_cr_image_wire PASS WAIT_EVENT_BUFFER_SHIP_CR_BUILD/SEND reserved but dormant (no GCS producer)"
+	else
+		echo "GATE no_cr_image_wire FAIL CR-ship wait event has a producer or is missing"
+	fi
+
+	# Value-gate verdict: all three => the data plane is unreachable in Stage 5.5.
+	echo "VERDICT value_gate NO_GO_IN_5_5 data plane forwarded to Stage 6 (#119 undo-block Cache Fusion); spec-5.57 ships boundary + contract only (legal NO-GO)"
+
+	echo "# spec-5.57 §3.4 soundness-gate checklist (L257): 5.57 verifies failure-direction; the rest are Stage 6 ship gates."
+	echo "SOUNDNESS failure_direction VERIFIED_5_57 any uncertainty -> ERROR 53R9G, never visible (CR walker pre-check; TAP fail-closed e2e)"
+	echo "SOUNDNESS base_availability STAGE6_OWNS current block image + WAL-before-ship (feature-019)"
+	echo "SOUNDNESS metadata_availability STAGE6_OWNS remote undo image is real evidence (not ITL stamp hint); INDOUBT -> fail-closed"
+	echo "SOUNDNESS replay_determinism STAGE6_OWNS v2 column-diff cross-node reconstruct determinism (spec-4.9ab reader)"
+	echo "SOUNDNESS invalidation_completeness STAGE6_OWNS remaster/DROP/TRUNCATE/relfilenode-reuse/retention + fence_epoch invalidation"
+}
+
+# ---- self-test: parse --static, assert the verdict ---------------------
+self_test() {
+	local out rc=0
+	out="$(emit_static)"
+
+	echo "$out"
+	echo "# ---- self-test assertions ----"
+
+	# every GATE line must be PASS
+	if echo "$out" | grep '^GATE ' | grep -qv ' PASS '; then
+		echo "not ok - some GATE line is not PASS"
+		rc=1
+	else
+		echo "ok - all GATE lines PASS"
+	fi
+
+	# the value-gate verdict must be the legal NO-GO
+	if echo "$out" | grep -q '^VERDICT value_gate NO_GO_IN_5_5'; then
+		echo "ok - value gate verdict is NO_GO_IN_5_5 (data plane -> Stage 6)"
+	else
+		echo "not ok - value gate verdict missing/unexpected"
+		rc=1
+	fi
+
+	# failure-direction must be verified by 5.57 itself
+	if echo "$out" | grep -q '^SOUNDNESS failure_direction VERIFIED_5_57'; then
+		echo "ok - failure-direction soundness gate verified in 5.57"
+	else
+		echo "not ok - failure-direction soundness gate not verified"
+		rc=1
+	fi
+
+	# exactly 5 soundness gates listed (§3.4)
+	local n_sound
+	n_sound="$(echo "$out" | grep -c '^SOUNDNESS ')"
+	if [ "$n_sound" -eq 5 ]; then
+		echo "ok - 5 soundness gates listed (§3.4)"
+	else
+		echo "not ok - expected 5 soundness gates, found $n_sound"
+		rc=1
+	fi
+
+	if [ "$rc" -eq 0 ]; then
+		echo "# self-test PASSED"
+	else
+		echo "# self-test FAILED"
+	fi
+	return "$rc"
+}
+
+# ---- optional dynamic 2-node fail-closed probe -------------------------
+dynamic_probe() {
+	echo "# spec-5.57 D0 dynamic leg: a 2-node fail-closed probe lives in the TAP"
+	echo "# harness (src/test/cluster_tap/t/318_*.pl).  The data plane is"
+	echo "# unreachable, so the only observable runtime outcome is fail-closed"
+	echo "# 53R9G + cross_instance_cr_refused/remote_undo_read_refused counters."
+	echo "# Run it via: cd src/test/cluster_tap && PERL5LIB=\$HOME/perl5/lib/perl5 \\"
+	echo "#   prove -v t/318_cluster_5_57_cross_instance_boundary.pl"
+}
+
+case "${1:---help}" in
+	--static) emit_static ;;
+	--self-test) self_test ;;
+	--dynamic) emit_static; echo; dynamic_probe ;;
+	--help | -h) usage ;;
+	*) usage; exit 2 ;;
+esac

src/backend/cluster/Makefile

@@ -162,6 +162,7 @@ OBJS = \
 	cluster_cr_admit_stat.o \
 	cluster_cr_tuple.o \
 	cluster_cr_tuple_stat.o \
+	cluster_cr_coordinator_stat.o \
 	cluster_cr_srf.o \
 	cluster_resolver_cache.o \
 	cluster_tt_slot.o \

src/backend/cluster/cluster_cr.c

@@ -49,7 +49,8 @@
 #include "cluster/cluster_cr_admit.h" /* spec-5.52 D2: insert-side admission gate */
 #include "cluster/cluster_cr_apply.h"
 #include "cluster/cluster_cr_cache.h"
-#include "cluster/cluster_cr_pool.h"  /* spec-5.51 D4: shared L2 CR pool */
+#include "cluster/cluster_cr_coordinator_stat.h" /* spec-5.57 D2/D3: coordinator boundary */
+#include "cluster/cluster_cr_pool.h"			 /* spec-5.51 D4: shared L2 CR pool */
 #include "cluster/cluster_cr_tuple.h" /* spec-5.54: tuple-level / verdict-only fast path */
 #include "cluster/cluster_conf.h"	  /* spec-3.24 D1: cluster_conf_has_peers */
 #include "cluster/cluster_guc.h"	  /* cluster_cr_chain_walk_max_steps, cluster_node_id */
@@ -237,6 +238,55 @@ cr_scratch_ensure(void)
 }
 
 
+/*
+ * cr_coordinator_refuse_runtime_remote -- spec-5.57 D2: the single fail-closed
+ *	refusal routine for a class③ (runtime-warm remote) CR origin.  Shared by the
+ *	CR-side pre-check (the real boundary, on a real remote UBA) AND the W2 test
+ *	injection (a synthetic class③), so the injection exercises the exact runtime
+ *	behavior the production pre-check uses.
+ *
+ *	One class③ refusal is BOTH the boundary headline (cross_instance_cr_refused)
+ *	and specifically the remote-undo-read leg (remote_undo_read_refused); both
+ *	bump by construction.  The rare W1 header-mismatch belt bumps only
+ *	cross_instance_cr_refused, giving the invariant
+ *	cross_instance_cr_refused >= remote_undo_read_refused.
+ *
+ *	NON-DEGRADABLE (rule 8.A, §2.2): the ereport fires under ANY GUC value; the
+ *	GUC only gates the advisory counters / probe / LOG-once.  This function never
+ *	returns (it always ereport(ERROR)s).  The data plane is Stage 6 (#119).
+ */
+static void
+pg_attribute_noreturn() cr_coordinator_refuse_runtime_remote(int origin_node)
+{
+	if (cluster_cross_instance_cr_coordinator != CR_COORD_MODE_OFF) {
+		cluster_cr_coordinator_stat_bump(CR_COORD_CROSS_INSTANCE_CR_REFUSED);
+		cluster_cr_coordinator_stat_bump(CR_COORD_REMOTE_UNDO_READ_REFUSED);
+	}
+	/* D0 measure-leg: count the class③ hit (behavior unchanged -- still fails). */
+	if (cluster_cross_instance_cr_probe)
+		cluster_cr_coordinator_stat_bump(CR_COORD_CROSS_INSTANCE_BOUNDARY_PROBE);
+	/* forward mode: LOG-once that the data plane is Stage 6 (L213: once per
+	 * backend so the hot path is never flooded). */
+	if (cluster_cross_instance_cr_coordinator == CR_COORD_MODE_FORWARD) {
+		static bool forward_logged = false;
+
+		if (!forward_logged) {
+			forward_logged = true;
+			elog(LOG, "cluster.cross_instance_cr_coordinator=forward is a contract "
+					  "placeholder: cross-instance CR/undo data plane lands in Stage 6 "
+					  "(#119); reads stay fail-closed (Spec: spec-5.57)");
+		}
+	}
+	ereport(ERROR, (errcode(ERRCODE_CLUSTER_CR_CROSS_INSTANCE_UNSUPPORTED),
+					errmsg("cluster CR cross-instance UBA encountered at the remote-undo-read "
+						   "leg (origin_node_id=%d, local=%d)",
+						   origin_node, cluster_node_id),
+					errhint("Own-instance CR only unless the origin was materialized by merged "
+							"recovery; the runtime cross-instance CR/undo data plane lands in "
+							"Stage 6 (#119 undo-block Cache Fusion); see Spec: spec-5.57.")));
+}
+
+
 /* ============================================================
  * Test injection hooks (spec-3.9 Step 7; SKIP-style precondition)
  * ============================================================ */
@@ -260,11 +310,11 @@ cr_check_error_injections(void)
 								"cluster_inject_fault('cr_snapshot_too_old','none',0).")));
 
 	if (cluster_cr_injection_armed("cr_cross_instance", &param))
-		ereport(ERROR,
-				(errcode(ERRCODE_CLUSTER_CR_CROSS_INSTANCE_UNSUPPORTED),
-				 errmsg("cluster CR cross-instance UBA (injected; origin_node_id=%u, local=%d)",
-						(uint32)param, cluster_node_id),
-				 errhint("test injection cr_cross_instance; spec-3.9 is own-instance only.")));
+		/* spec-5.57 D2/D3: synthetic class③ refusal -- drive the SAME fail-closed
+		 * routine the production pre-check uses (53R9G + both coordinator counters
+		 * + probe/forward), so the TAP injection legs exercise the real boundary
+		 * behavior deterministically (param = synthetic origin_node_id). */
+		cr_coordinator_refuse_runtime_remote((int)param);
 
 	if (cluster_cr_injection_armed("cr_corruption", &param)) {
 		const char *kind = (param == 1)	  ? "uba_decode"
@@ -341,6 +391,39 @@ cr_walk_chain(char *scratch_page, UBA start_uba, SCN read_scn,
 			ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED),
 							errmsg("cluster CR encountered a malformed UBA in the undo chain")));
 
+		/*
+		 * spec-5.57 D2 (Q11-A): CR-side remote-UBA pre-check -- the read-path
+		 * coordinator boundary, applied BEFORE the undo read.  Derive the
+		 * segment owner from the UBA and classify it (§0.1).  A runtime-warm
+		 * cross-instance origin (class③: not own, not merged-materialized) is
+		 * the net-new boundary: fail closed HERE with the canonical 53R9G,
+		 * rather than letting cluster_undo_get_record() return 0 and conflate it
+		 * with own-instance retention-recycled undo (53R9F below).  This is the
+		 * W3 hardening: the remote-undo-read leg now fails closed with the SAME
+		 * errcode as the W1 walker wall (errcode consolidation, CR-9).  It is
+		 * NON-DEGRADABLE -- the ereport fires under any GUC value; the GUC only
+		 * gates the advisory counters (rule 8.A, §2.2).  Own-instance is the
+		 * common OLTP path: classify returns OWN and we fall straight through.
+		 * The data plane (real remote undo fetch) is Stage 6 (#119).
+		 * See Spec: spec-5.57 §3.1 (W3) / §2.1 (three roles).
+		 */
+		{
+			NodeId cr_origin = uba_origin_node_id(uba);
+			ClusterCrCoordOriginClass cr_origin_class
+				= cluster_cr_coordinator_classify_origin(cr_origin);
+
+			if (cr_origin_class == CR_COORD_ORIGIN_RUNTIME_REMOTE) {
+				/* class③: fail closed via the shared refusal routine (53R9G +
+				 * both coordinator counters; non-degradable, §2.2). */
+				cr_coordinator_refuse_runtime_remote((int)cr_origin);
+			} else if (cr_origin_class == CR_COORD_ORIGIN_MATERIALIZED_REMOTE) {
+				/* class②: merged-materialized remote, served from the local tree
+				 * (already shipped, spec-4.5a D8).  Count the serve (advisory). */
+				if (cluster_cross_instance_cr_coordinator != CR_COORD_MODE_OFF)
+					cluster_cr_coordinator_stat_bump(CR_COORD_MATERIALIZED_REMOTE_SERVED);
+			}
+		}
+
 		len = cluster_undo_get_record(uba, record_buf.data, sizeof(record_buf.data));
 		if (len == 0)
 			ereport(ERROR,
@@ -361,30 +444,40 @@ cr_walk_chain(char *scratch_page, UBA start_uba, SCN read_scn,
 
 		/* Own-instance, or a merged-materialized remote instance whose undo
 		 * lives in the local pg_undo/instance_<origin> tree (spec-4.5a D8).
-		 * Anything else stays the spec-3.9 fail-closed.
+		 * Anything else stays fail-closed (Spec: spec-5.57 §3.1 W1).
+		 *
+		 * This is the W1 belt: the spec-5.57 D2 segment-derived pre-check above
+		 * (on uba_origin_node_id) catches the common class③ case BEFORE the undo
+		 * read, so this header-origin check now fires only on the rare segment-
+		 * vs-header mismatch (D8 §2.5 cross-check) -- defense-in-depth, never a
+		 * silent assumption.
 		 *
 		 * spec-5.56 C4 (reconfig contract, §3.3): this carve-out is ALSO the
 		 * fail-closed boundary that keeps the THIRD origin class — runtime warm
 		 * remote (not own, not merged-materialized) — OUT of the CR pool: it never
-		 * constructs (ERROR below) so it never caches.  The two pool-eligible
-		 * classes are reconfig-INVARIANT and need NO membership/remaster
-		 * invalidation: (①) own-instance pages/undo are unchanged by reconfig; (②)
-		 * merged-materialized remote undo is durable in the local tree with a
-		 * reconfig-invariant merge_recovered_lsn authority, and an origin rejoin's
-		 * NEW writes are new versions => new key => MISS (already fenced by C1/key).
-		 * read_scn is a GLOBAL SCN (AD-008), not a membership epoch (INV-C2).  The
-		 * runtime-warm-remote class's reconfig/remaster invalidation is forwarded to
-		 * spec-5.57 (where construct stops fail-closing it); until then this ERROR
-		 * is the C4 class-③ guard (INV-C3), not a silent assumption. */
+		 * constructs (ERROR) so it never caches.  The two pool-eligible classes are
+		 * reconfig-INVARIANT and need NO membership/remaster invalidation: (①)
+		 * own-instance pages/undo are unchanged by reconfig; (②) merged-materialized
+		 * remote undo is durable in the local tree with a reconfig-invariant
+		 * merge_recovered_lsn authority, and an origin rejoin's NEW writes are new
+		 * versions => new key => MISS (already fenced by C1/key).  read_scn is a
+		 * GLOBAL SCN (AD-008), not a membership epoch (INV-C2).  spec-5.57 freezes
+		 * this class③ fail-closed as the read-path coordinator boundary (CR-9); the
+		 * runtime-warm-remote data plane lands in Stage 6 (#119). */
 		if (hdr->origin_node_id != (uint16)cluster_node_id
-			&& !cluster_merged_instance_is_materialized((int)hdr->origin_node_id))
-			ereport(ERROR, (errcode(ERRCODE_CLUSTER_CR_CROSS_INSTANCE_UNSUPPORTED),
-							errmsg("cluster CR cross-instance UBA encountered "
-								   "(origin_node_id=%u, local=%d)",
-								   hdr->origin_node_id, cluster_node_id),
-							errhint("Own-instance CR only unless the origin was materialized by "
-									"merged recovery; runtime cross-instance CR is Stage 4 "
-									"(Cache Fusion CR coordinator).")));
+			&& !cluster_merged_instance_is_materialized((int)hdr->origin_node_id)) {
+			if (cluster_cross_instance_cr_coordinator != CR_COORD_MODE_OFF)
+				cluster_cr_coordinator_stat_bump(CR_COORD_CROSS_INSTANCE_CR_REFUSED);
+			ereport(ERROR,
+					(errcode(ERRCODE_CLUSTER_CR_CROSS_INSTANCE_UNSUPPORTED),
+					 errmsg("cluster CR cross-instance UBA encountered "
+							"(origin_node_id=%u, local=%d)",
+							hdr->origin_node_id, cluster_node_id),
+					 errhint("Own-instance CR only unless the origin was materialized by "
+							 "merged recovery; the runtime cross-instance CR/undo data plane "
+							 "lands in Stage 6 (#119 undo-block Cache Fusion); see "
+							 "Spec: spec-5.57.")));
+		}
 
 		/* I-chain-1: normal SCN stop. */
 		if (scn_time_cmp(hdr->write_scn, read_scn) <= 0)