fix(cluster): spec-5.15 Hardening v1.3 — cold-bootstrap proof requires fresh-alive, not just a valid slot

v1.2 anchored the co-boot quorum on a valid voting-disk slot (generation > 0 at
epoch INITIAL), but a generation > 0 slot alone is not liveness: a CRASHED peer
leaves a stale leftover slot at epoch INITIAL.  decide_quorum_view's P2.1
heartbeat-freshness gate already excludes such stale slots from the alive_bitmap,
but the v1.2 bootstrap proof read the raw observed slot (no freshness) — so a node
could reach quorum with self + a stale peer slot and fail-open (latch BOOTSTRAP
without a live co-boot quorum).

Fix: publish the per-node FRESH-ALIVE signal (decide_quorum_view's alive_bitmap)
into the reconfig region, and require it in the cold-bootstrap proof: fresh-alive
AND generation > 0 AND epoch INITIAL.  Anchored on the durable voting-disk
heartbeat (not live CSSD), so it rejects stale slots WITHOUT reintroducing the
v1.2 IC-churn race (the disk heartbeat keeps flowing while CSSD/tier1 churns).
Quorum threshold and rejoiner predicate unchanged.

New ReconfigShmem field observed_fresh_alive[CLUSTER_MAX_NODES] (atomic, default
0 = fail-closed); qvotec publishes it each poll from decide_quorum_view.

Unit: test_cluster_reconfig U21 (stale slot must not count, TDD red->green) +
U19/U20 updated to fresh-alive semantics; UT_PLAN 43->45 (also fixes a pre-existing
plan/count mismatch).  No on-disk/wire/catalog/GUC change; no catversion bump.

Spec: spec-5.15-online-declared-node-join-membership.md (Hardening v1.3)

Author: SqlRush

src/backend/cluster/cluster_qvotec.c

@@ -969,6 +969,25 @@ qvotec_poll_once(void)
 							 CLUSTER_MAX_NODES, (uint32)cluster_node_id, qvotec_self_incarnation,
 							 now_us, heartbeat_timeout_us, &decision);
 
+	/*
+	 * spec-5.15 Hardening v1.3 (INV-J14 stale-slot fail-open) — publish the
+	 * per-node FRESH-ALIVE liveness from decide_quorum_view's alive_bitmap (the
+	 * P2.1 heartbeat-freshness gate that already excludes a crashed peer's stale
+	 * leftover slot) into the reconfig region.  The cold-bootstrap proof reads it
+	 * so it counts only genuinely live co-booting peers, never a stale gen > 0
+	 * leftover (which would fail-open).  Anchored on the durable voting-disk
+	 * heartbeat, so it is robust to CSSD / tier1 churn — the v1.2 race fix stands.
+	 */
+	{
+		uint32 node;
+
+		for (node = 0; node < CLUSTER_MAX_NODES; node++) {
+			bool fresh = (decision.alive_bitmap[node / 8] & (uint8)(1u << (node % 8))) != 0;
+
+			cluster_reconfig_record_observed_fresh_alive((int32)node, fresh);
+		}
+	}
+
 	/*
 	 * Hardening v0.4 P1.1:  Q6 v0.2 newer-self-FATAL.  decide_quorum_
 	 * view observed an OK-disk fresh slot at our node_id offset with

src/backend/cluster/cluster_reconfig.c

@@ -666,6 +666,30 @@ cluster_reconfig_get_observed_slot(int32 node_id, uint64 *incarnation, uint64 *g
 	return gen > 0;
 }
 
+/*
+ * spec-5.15 Hardening v1.3 (INV-J14 stale-slot fail-open) — publish / read the
+ * per-node FRESH-ALIVE liveness qvotec derived from decide_quorum_view's
+ * heartbeat-freshness gate (P2.1).  The cold-bootstrap proof counts a peer only
+ * when it is fresh-alive at epoch INITIAL — a generation > 0 slot alone may be a
+ * crashed peer's stale leftover.  Anchored on the durable voting-disk heartbeat,
+ * not live CSSD, so the v1.2 IC-churn race fix is preserved.
+ */
+void
+cluster_reconfig_record_observed_fresh_alive(int32 node_id, bool fresh_alive)
+{
+	if (ReconfigShmem == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES)
+		return;
+	pg_atomic_write_u64(&ReconfigShmem->observed_fresh_alive[node_id], fresh_alive ? 1 : 0);
+}
+
+bool
+cluster_reconfig_get_observed_fresh_alive(int32 node_id)
+{
+	if (ReconfigShmem == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES)
+		return false;
+	return pg_atomic_read_u64(&ReconfigShmem->observed_fresh_alive[node_id]) != 0;
+}
+
 
 /*
  * Read snapshot of last_applied.event_id under shared lock.  Used by
@@ -1252,12 +1276,20 @@ cluster_reconfig_bootstrap_quorum_at_initial(void)
 		if (ep > CLUSTER_EPOCH_INITIAL)
 			return false;
 		/*
-		 * Count a peer only on a VALID durable co-boot slot: a real observed
-		 * voting-disk slot (generation > 0) at epoch INITIAL.  Never count a
-		 * default-0 placeholder (generation 0) nor rely on live CSSD state.
+		 * Count a peer only on a FRESH-ALIVE co-boot slot: a real observed
+		 * voting-disk slot (generation > 0) that qvotec's decide_quorum_view saw
+		 * FRESH-ALIVE this poll (heartbeat_ts_us recent, the P2.1 freshness gate),
+		 * at epoch INITIAL.  Hardening v1.3: the generation > 0 test alone is NOT
+		 * liveness — a CRASHED peer leaves a stale leftover slot (gen > 0, epoch
+		 * INITIAL) that v1.2 wrongly counted, letting a node fail-open (latch
+		 * BOOTSTRAP on self + a stale peer slot, with no live co-boot quorum).  The
+		 * fresh-alive signal is anchored on the durable voting-disk heartbeat (NOT
+		 * live CSSD), so it rejects stale slots WITHOUT reintroducing the v1.2
+		 * IC-churn race (the disk heartbeat keeps flowing while CSSD/tier1 churns).
+		 * A default-0 placeholder (generation 0) never counts either.
 		 */
 		if (cluster_reconfig_get_observed_slot(i, &inc, &gen) && gen > 0
-			&& ep == CLUSTER_EPOCH_INITIAL)
+			&& cluster_reconfig_get_observed_fresh_alive(i) && ep == CLUSTER_EPOCH_INITIAL)
 			proven_at_initial++;
 	}
 	if (declared == 0)
@@ -2869,6 +2901,17 @@ cluster_reconfig_get_observed_epoch(int32 node_id pg_attribute_unused())
 	return 0;
 }
 
+void
+cluster_reconfig_record_observed_fresh_alive(int32 node_id pg_attribute_unused(),
+											 bool fresh_alive pg_attribute_unused())
+{}
+
+bool
+cluster_reconfig_get_observed_fresh_alive(int32 node_id pg_attribute_unused())
+{
+	return false;
+}
+
 ClusterJoinMarkerSubmitResult
 cluster_reconfig_submit_join_marker(int32 target_node pg_attribute_unused(),
 									const ClusterJoinCommitMarker *m pg_attribute_unused())

src/include/cluster/cluster_reconfig.h

@@ -338,6 +338,20 @@ typedef struct ClusterReconfigState {
 	 */
 	pg_atomic_uint64 observed_epoch[CLUSTER_MAX_NODES];
 
+	/*
+	 * spec-5.15 Hardening v1.3 (INV-J14 stale-slot fail-open) — per declared node,
+	 * whether qvotec's decide_quorum_view saw that node FRESH-ALIVE this poll (its
+	 * voting-disk heartbeat_ts_us recent, per the P2.1 freshness gate).  This is the
+	 * liveness signal the cold-bootstrap proof needs: a generation > 0 slot alone
+	 * may be a CRASHED peer's stale leftover at epoch INITIAL — counting it would
+	 * fail-open (latch BOOTSTRAP without a live co-boot quorum).  Anchored on the
+	 * durable voting-disk heartbeat (NOT live CSSD), so it is robust to IC/tier1
+	 * heartbeat churn — the v1.2 race fix is preserved.  1 = fresh-alive, 0 = stale /
+	 * absent (default 0 = fail-closed).  pg_atomic — qvotec (writer) and LMON (reader)
+	 * are different processes.
+	 */
+	pg_atomic_uint64 observed_fresh_alive[CLUSTER_MAX_NODES];
+
 	/*
 	 * spec-5.15 D4 — join-commit-marker submit mailbox (§2.6).  The coordinator
 	 * stages a marker for the joiner (join_marker_target_node_id), bumps
@@ -482,6 +496,16 @@ extern bool cluster_reconfig_get_observed_slot(int32 node_id, uint64 *incarnatio
 											   uint64 *generation);
 extern uint64 cluster_reconfig_get_observed_epoch(int32 node_id);
 
+/*
+ * spec-5.15 Hardening v1.3 — publish / read the per-node FRESH-ALIVE liveness
+ * qvotec's decide_quorum_view derived from the durable voting-disk heartbeat
+ * (the P2.1 freshness gate).  The cold-bootstrap proof counts a peer only when
+ * it is fresh-alive AND at epoch INITIAL — never on a generation > 0 slot alone
+ * (a crashed peer's stale leftover).  get returns false when absent (fail-closed).
+ */
+extern void cluster_reconfig_record_observed_fresh_alive(int32 node_id, bool fresh_alive);
+extern bool cluster_reconfig_get_observed_fresh_alive(int32 node_id);
+
 /*
  * spec-5.15 Hardening v1.1 (HF-1 / INV-J9): true iff a majority of the current
  * MEMBER survivors have advanced their durable observed epoch to >=

src/test/cluster_unit/test_cluster_qvotec.c

@@ -447,6 +447,12 @@ cluster_reconfig_record_observed_slot(int32 node_id pg_attribute_unused(),
 									  uint64 generation pg_attribute_unused(),
 									  uint64 epoch pg_attribute_unused())
 {}
+/* spec-5.15 Hardening v1.3: qvotec.o now also publishes per-node fresh-alive. */
+void cluster_reconfig_record_observed_fresh_alive(int32 node_id, bool fresh_alive);
+void
+cluster_reconfig_record_observed_fresh_alive(int32 node_id pg_attribute_unused(),
+											 bool fresh_alive pg_attribute_unused())
+{}
 bool cluster_reconfig_join_qvotec_poll_pending(int32 *out_target_node, void *out_slot512);
 bool
 cluster_reconfig_join_qvotec_poll_pending(int32 *out_target_node,

src/test/cluster_unit/test_cluster_reconfig.c

@@ -1414,10 +1414,12 @@ UT_TEST(test_reconfig_bootstrap_quorum_epoch_proof)
 	ut_declared_set[1] = true; /* 3 declared nodes */
 	ut_declared_set[2] = true;
 
-	/* quorum of declared on VALID co-boot slots at INITIAL -> bootstrap proven
-	 * (v1.2: anchored on the durable voting-disk slot, not live CSSD). */
+	/* quorum of declared FRESH-ALIVE co-boot slots at INITIAL -> bootstrap proven
+	 * (v1.3: durable voting-disk heartbeat freshness + valid slot, not live CSSD). */
 	cluster_reconfig_record_observed_slot(1, 1, 1, 0);
 	cluster_reconfig_record_observed_slot(2, 1, 1, 0);
+	cluster_reconfig_record_observed_fresh_alive(1, true);
+	cluster_reconfig_record_observed_fresh_alive(2, true);
 	UT_ASSERT(cluster_reconfig_bootstrap_quorum_at_initial());
 
 	/* a peer past INITIAL (running cluster) -> NOT a bootstrap (fail-closed). */
@@ -1448,15 +1450,18 @@ UT_TEST(test_reconfig_bootstrap_quorum_epoch_proof)
  * ====================================================================== */
 UT_TEST(test_reconfig_bootstrap_proof_valid_slot_not_cssd)
 {
-	/* --- A. valid co-boot slots at INITIAL but peers CSSD-DEAD -> still a
-	 *        proven bootstrap (durable slot, not live CSSD).  v1.1 returned
-	 *        false here (the race window); v1.2 returns true. --- */
+	/* --- A. FRESH-ALIVE co-boot slots at INITIAL but peers CSSD-DEAD -> still a
+	 *        proven bootstrap (durable voting-disk heartbeat, not live CSSD).  v1.1
+	 *        returned false here (the race window); v1.3 returns true because the
+	 *        liveness is the voting-disk fresh-alive signal, immune to CSSD churn. --- */
 	ut_join_setup();		   /* self = node 0 */
 	ut_declared_set[1] = true; /* 3 declared nodes */
 	ut_declared_set[2] = true;
-	cluster_reconfig_record_observed_slot(1, 7, 1, 0); /* valid slot, INITIAL */
-	cluster_reconfig_record_observed_slot(2, 7, 1, 0); /* valid slot, INITIAL */
-	ut_peer_state[1] = CLUSTER_CSSD_PEER_DEAD;		   /* live CSSD churned down */
+	cluster_reconfig_record_observed_slot(1, 7, 1, 0);	   /* valid slot, INITIAL */
+	cluster_reconfig_record_observed_slot(2, 7, 1, 0);	   /* valid slot, INITIAL */
+	cluster_reconfig_record_observed_fresh_alive(1, true); /* voting-disk fresh */
+	cluster_reconfig_record_observed_fresh_alive(2, true);
+	ut_peer_state[1] = CLUSTER_CSSD_PEER_DEAD; /* live CSSD churned down */
 	ut_peer_state[2] = CLUSTER_CSSD_PEER_DEAD;
 	UT_ASSERT(cluster_reconfig_bootstrap_quorum_at_initial());
 
@@ -1481,14 +1486,50 @@ UT_TEST(test_reconfig_bootstrap_proof_valid_slot_not_cssd)
 }
 
 
+/* ======================================================================
+ * U21 (spec-5.15 Hardening v1.3 / INV-J14 stale-slot fail-open) -- a valid
+ * generation > 0 slot at epoch INITIAL is NOT proof of co-booting: it may be a
+ * CRASHED peer's stale leftover (decide_quorum_view's P2.1 freshness gate marks
+ * it not-fresh).  The cold-bootstrap proof must additionally require the per-node
+ * FRESH-ALIVE signal (durable voting-disk heartbeat), not slot existence alone —
+ * else a node with self + a stale peer slot fail-opens (latches BOOTSTRAP without
+ * a live co-boot quorum).  v1.2 counted such a stale slot (the regression this
+ * guards); v1.3 fail-closes on it.
+ * ====================================================================== */
+UT_TEST(test_reconfig_bootstrap_proof_stale_slot_failclosed)
+{
+	/* --- A. valid slots at INITIAL but STALE heartbeat (fresh_alive=false) must
+	 *        NOT count -> only self proven -> below quorum -> false.  v1.2 (no
+	 *        freshness) counted them = fail-open; v1.3 fail-closes. --- */
+	ut_join_setup();		   /* self = node 0 */
+	ut_declared_set[1] = true; /* 3 declared nodes */
+	ut_declared_set[2] = true;
+	cluster_reconfig_record_observed_slot(1, 7, 1, 0);		/* valid slot, INITIAL */
+	cluster_reconfig_record_observed_slot(2, 7, 1, 0);		/* valid slot, INITIAL */
+	cluster_reconfig_record_observed_fresh_alive(1, false); /* crashed peer: stale hb */
+	cluster_reconfig_record_observed_fresh_alive(2, false);
+	UT_ASSERT(!cluster_reconfig_bootstrap_quorum_at_initial());
+
+	/* --- B. the same slots but FRESH-ALIVE -> genuine co-boot -> proven. --- */
+	cluster_reconfig_record_observed_fresh_alive(1, true);
+	cluster_reconfig_record_observed_fresh_alive(2, true);
+	UT_ASSERT(cluster_reconfig_bootstrap_quorum_at_initial());
+
+	/* --- C. one fresh + one stale -> self + the one fresh = quorum (3-node) ->
+	 *        proven; the stale peer simply does not contribute (no over-reject). --- */
+	cluster_reconfig_record_observed_fresh_alive(2, false);
+	UT_ASSERT(cluster_reconfig_bootstrap_quorum_at_initial());
+}
+
+
 /* ============================================================
  * Main — register + run all tests.
  * ============================================================ */
 
 int
 main(void)
 {
-	UT_PLAN(43);
+	UT_PLAN(45);
 
 	/* T-reconfig-1 */
 	UT_RUN(test_reconfig_dead_bitmap_bytes_eq_16);
@@ -1559,6 +1600,7 @@ main(void)
 	UT_RUN(test_reconfig_join_publish_proven_no_member_failclosed);
 	UT_RUN(test_reconfig_bootstrap_quorum_epoch_proof);
 	UT_RUN(test_reconfig_bootstrap_proof_valid_slot_not_cssd);
+	UT_RUN(test_reconfig_bootstrap_proof_stale_slot_failclosed);
 
 	UT_DONE();
 	return ut_failed_count == 0 ? 0 : 1;