bootjp
diff --git a/‎adapter/encryption_admin.go‎
Lines changed: 347 additions & 3 deletions b/‎adapter/encryption_admin.go‎
Lines changed: 347 additions & 3 deletions
diff --git a/‎adapter/encryption_admin_test.go‎
Lines changed: 580 additions & 4 deletions b/‎adapter/encryption_admin_test.go‎
Lines changed: 580 additions & 4 deletions
diff --git a/‎docs/design/2026_05_31_partial_6e_enable_raft_envelope.md‎
Lines changed: 6 additions & 1 deletion b/‎docs/design/2026_05_31_partial_6e_enable_raft_envelope.md‎
Lines changed: 6 additions & 1 deletion
diff --git a/‎internal/raftengine/engine.go‎
Lines changed: 14 additions & 0 deletions b/‎internal/raftengine/engine.go‎
Lines changed: 14 additions & 0 deletions
diff --git a/‎kv/raft_payload_wrapper.go‎
Lines changed: 237 additions & 0 deletions b/‎kv/raft_payload_wrapper.go‎
Lines changed: 237 additions & 0 deletions
@@ -14,7 +14,12 @@
 |---|---|---|
 | 6E-1a — FSM apply machinery (`applyEnableRaftEnvelope`, sidecar field plumbing, wire sub-tag whitelist) | shipped | #899 (3bffd344) |
 | 6E-1b — `EnableRaftEnvelope` admin RPC + `elastickv-admin enable-raft-envelope` CLI subcommand (server method **gated** until 6E-2; see §3.3 below) | shipped | #907 |
-| 6E-2 — engine unwrap-on-apply + coordinator wrap-on-propose + §7.1 proposal-quiescence barrier (atomic 3-piece flip; also flips the 6E-1b gate to true) | not started | — |
+| 6E-2a — typed cutover sentinel + sidecar `RaftEnvelopeCutoverIndex` apply seam | shipped | (rolled into earlier slices) |
+| 6E-2b — `ProposeAdmin` sibling on `raftengine.Proposer` (barrier-exempt by interface contract) | shipped | (rolled into earlier slices) |
+| 6E-2c — Coordinator `dynamicWrappedProposer` + `ShardGroup.raftPayloadWrap` hot-swap + `Proposer()` accessor + Internal.Forward wrap-aware proposer + fail-closed startup guard on active cutover | shipped | #922 (eb371ca6) |
+| 6E-2d — §7.1 6-step quiescence barrier on `dynamicWrappedProposer.Propose` + `ShardGroup` barrier forwarders + `CutoverBarrierController` option + state-machine in `EnableRaftEnvelope` handler (gated behind `raftEnvelopeWrapEnabled = false`; flipped in 6E-2f) | shipped | this PR |
+| 6E-2e — `main.go` wiring: `OpenConfig.RaftCipher` + `RaftCutoverIndex` → `CutoverBarrierController` implementation fanning out over participating `ShardGroup`s. **BLOCKER (a):** route admin RPCs (RotateDEK, RegisterEncryptionWriter) through the wrap-aware proposer so post-cutover admin entries are wrapped — the raw-engine ProposeAdmin path leaves cleartext admin entries at `index > cutoverIdx` and §6.3 halts the cluster (codex P1 #1 round-2 on PR933). **BLOCKER (b):** auto-install the wrap on every replica's FSM-apply of the cutover marker so a leader failover between cutover commit and `InstallWrap` doesn't admit cleartext writes on the newly-elected leader (codex P1 round-3 on PR933). Both blockers MUST land before 6E-2f flips the gate. | not started | — |
+| 6E-2f — atomic flip of `raftEnvelopeWrapEnabled` to `true` (the §3.3 6E-1b gate release) | not started | — |
 | 6E-3 — §6C-4 fail-closed guards (`ErrEnvelopeCutoverDivergence`, `ErrEncryptionNotBootstrapped`, `ErrLocalEpochOutOfRange`) | not started | — |
 
 With 6E-1 (both sub-milestones) complete, the wire-format and
 
@@ -23,6 +23,20 @@ var (
 	// ErrLeadershipTransferInProgress indicates a leadership transfer
 	// is under way and proposals are being held back.
 	ErrLeadershipTransferInProgress = errors.New("raft engine: leadership transfer in progress")
+	// ErrEnvelopeCutoverInProgress indicates the §7.1 raft-envelope
+	// cutover barrier is open on this leader and rejecting fresh
+	// USER proposals on the Propose path. The error is the step-1
+	// gate-rejection surface for coordinator clients while the
+	// EnableRaftEnvelope handler runs the 6-step barrier sequence
+	// (block intake → drain → propose cutover via ProposeAdmin →
+	// wait apply → flip wrap → unblock). Callers should treat it
+	// as a transient back-off: a healthy cluster spends single-
+	// digit milliseconds inside the barrier. ProposeAdmin is the
+	// design's barrier exemption (admin entries that MUST remain
+	// admissible across the barrier — the cutover marker itself,
+	// ConfChange-time RegisterEncryptionWriter) and never observes
+	// this error.
+	ErrEnvelopeCutoverInProgress = errors.New("raft engine: envelope cutover in progress")
 )
 
 type State string
 
@@ -2,6 +2,7 @@ package kv
 
 import (
 	"context"
+	"sync"
 	"sync/atomic"
 
 	"github.com/bootjp/elastickv/internal/raftengine"
@@ -131,9 +132,44 @@ func (p *wrappedProposer) ProposeAdmin(ctx context.Context, data []byte) (*rafte
 // stores nil to express "wrap inactive". This is so the call site
 // (typically a ShardGroup) owns the storage and the proposer just
 // reads.
+//
+// §7.1 quiescence-barrier state (Stage 6E-2d) lives here too — see
+// the BeginCutoverBarrier / WaitInflightDrained / EndCutoverBarrier
+// trio below for the 6-step state machine's mechanics. The barrier
+// gates only the user-Propose path; ProposeAdmin is exempt by
+// interface contract so the EnableRaftEnvelope handler can propose
+// the cutover marker itself across its own barrier.
 type dynamicWrappedProposer struct {
 	inner   raftengine.Proposer
 	wrapPtr *atomic.Pointer[RaftPayloadWrapper]
+
+	// barrierMu guards barrierOpen, inflightUser, and drainSig. Held
+	// only briefly on Propose entry/exit (the engine.Propose call
+	// itself runs without the mutex), so contention is bounded by
+	// the per-Propose acquire/release pair. The 6E-2d handler holds
+	// it longer on the BeginCutoverBarrier / EndCutoverBarrier
+	// transitions, but those are once per cutover (~ms-scale).
+	//
+	// Why a mutex rather than three atomics: the barrier-open check
+	// and the in-flight counter increment MUST be observed as one
+	// atomic transition, or a Propose that read barrierOpen=false
+	// just before the cutover handler stores true could increment
+	// the counter AFTER BeginCutoverBarrier observed it as 0,
+	// leaving WaitInflightDrained returning success while a fresh
+	// user proposal slips through to engine.Propose. The mutex is
+	// the simplest way to make Propose's (read-barrier, inc-counter)
+	// pair atomic with the handler's (open-barrier, sample-counter)
+	// pair on the other side.
+	barrierMu    sync.Mutex
+	barrierOpen  bool
+	inflightUser int64
+	// drainSig is freshly allocated each BeginCutoverBarrier call
+	// and closed when inflightUser drops to 0 after the barrier has
+	// opened (either at BeginCutoverBarrier time if no Propose is
+	// in flight, or at the last proposeExit that hits 0). nil
+	// outside a barrier cycle so WaitInflightDrained called without
+	// an active barrier degrades gracefully to immediate success.
+	drainSig chan struct{}
 }
 
 // newDynamicWrappedProposer wires a proposer that consults wrapPtr
@@ -174,7 +210,24 @@ func (p *dynamicWrappedProposer) currentWrap() RaftPayloadWrapper {
 	return nil
 }
 
+// Propose runs the §7.1 quiescence-barrier gate, then forwards the
+// (optionally wrapped) payload to the inner Propose. The gate
+// rejects with ErrEnvelopeCutoverInProgress while the barrier is
+// open — the 6E-2d EnableRaftEnvelope handler holds the barrier
+// open for the few-ms it takes to commit the cutover entry and
+// publish the active wrap closure. Callers should treat the error
+// as a transient back-off (retry on a new leader-issued ts).
+//
+// The barrierMu + inflight-counter pair makes (read-barrier,
+// inc-counter) atomic with the handler's (open-barrier,
+// sample-counter); see the type comment for the race that motivates
+// it.
 func (p *dynamicWrappedProposer) Propose(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) {
+	if err := p.beginUserPropose(); err != nil {
+		return nil, err
+	}
+	defer p.endUserPropose()
+
 	wrapped, err := applyRaftPayloadWrap(p.currentWrap(), data)
 	if err != nil {
 		return nil, err
@@ -186,6 +239,190 @@ func (p *dynamicWrappedProposer) Propose(ctx context.Context, data []byte) (*raf
 	return res, nil
 }
 
+// beginUserPropose increments the in-flight counter under
+// barrierMu, returning ErrEnvelopeCutoverInProgress if the barrier
+// is open. Returning the error without incrementing is intentional:
+// a rejected Propose must not count toward in-flight (the handler
+// would otherwise wait on a fictional in-flight that will never
+// drain). Pair with endUserPropose.
+func (p *dynamicWrappedProposer) beginUserPropose() error {
+	p.barrierMu.Lock()
+	defer p.barrierMu.Unlock()
+	if p.barrierOpen {
+		return raftengine.ErrEnvelopeCutoverInProgress
+	}
+	p.inflightUser++
+	return nil
+}
+
+// endUserPropose pairs with beginUserPropose: decrement the
+// in-flight counter under barrierMu and, if the barrier is open
+// and we just dropped to 0, close drainSig so the handler's
+// WaitInflightDrained unblocks. Idempotent close via the
+// select/default pattern so concurrent EndCutoverBarrier doesn't
+// race a late proposeExit on a freshly nil-ed drainSig (the nil
+// check covers EndCutoverBarrier having already torn down the
+// channel; the closed-recv guard covers the BeginCutoverBarrier
+// path that closed an empty-inflight channel at open time).
+func (p *dynamicWrappedProposer) endUserPropose() {
+	p.barrierMu.Lock()
+	defer p.barrierMu.Unlock()
+	p.inflightUser--
+	if p.barrierOpen && p.inflightUser == 0 && p.drainSig != nil {
+		select {
+		case <-p.drainSig:
+			// already closed at BeginCutoverBarrier-with-no-inflight
+			// or by a sibling exit; treat as success.
+		default:
+			close(p.drainSig)
+		}
+	}
+}
+
+// BeginCutoverBarrier opens the §7.1 step-1 quiescence barrier.
+// After return, every fresh dynamicWrappedProposer.Propose call
+// fails with ErrEnvelopeCutoverInProgress until EndCutoverBarrier
+// runs. ProposeAdmin is unaffected (the cutover marker proposes
+// through it).
+//
+// HAZARDS — per-leader scope of the barrier (codex P1 round-2 and
+// round-3 on PR933): the barrier is an in-memory data structure
+// owned by THIS leader's dynamicWrappedProposer. It does not
+// coordinate across the cluster, and it does not survive leadership
+// transfer. Two related future-state failure modes follow from
+// that scope and MUST be closed by 6E-2e before 6E-2f flips the
+// gate; 6E-2d ships them inert by leaving raftEnvelopeWrapEnabled
+// false so production never opens the cutover window.
+//
+//	(a) Wrap-gap admin RPCs (codex P1 #1 round-2):
+//	    Other admin RPCs that route through ProposeAdmin (RotateDEK,
+//	    RegisterEncryptionWriter) are barrier-exempt and currently
+//	    reach the engine via the raw-engine s.proposer in
+//	    adapter/encryption_admin.go. Between the cutover marker's
+//	    commit and the handler's InstallWrap call, an admin RPC
+//	    that lands at `index > raftEnvelopeCutoverIndex` would be
+//	    cleartext, and the §6.3 strict-`>` apply hook on every
+//	    follower would treat it as a wrapped envelope and halt
+//	    apply cluster-wide.
+//
+//	    Remediation options for 6E-2e:
+//	      Option A (preferred): route RotateDEK /
+//	                            RegisterEncryptionWriter through the
+//	                            wrap-aware proposer so post-cutover
+//	                            admin entries are wrapped. The
+//	                            cutover marker itself remains on a
+//	                            separate raw-engine reference held
+//	                            by the EnableRaftEnvelope handler.
+//	      Option B: extend cutoverSem to serialize RotateDEK and
+//	                RegisterEncryptionWriter against the
+//	                EnableRaftEnvelope handler so no admin RPC can
+//	                race the barrier window.
+//	    See main_encryption_registration.go's call-site comment for
+//	    the 7c §3.1 wiring that Option A would extend.
+//
+//	(b) Leader failover mid-cutover (codex P1 round-3):
+//	    If leadership transfers from L1 to L2 between the cutover
+//	    marker's commit and L1's InstallWrap call, L2 has its own
+//	    barrierOpen=false and a nil wrap pointer. L2 admits a fresh
+//	    user proposal through Propose without wrapping; it lands at
+//	    `index > raftEnvelopeCutoverIndex` in cleartext, and once
+//	    L2 (or any follower) applies the cutover marker the §6.3
+//	    strict-`>` hook treats every subsequent cleartext proposal
+//	    as a wrapped envelope and halts.
+//
+//	    Remediation options for 6E-2e:
+//	      Option A (preferred): auto-install the wrap on every
+//	                            replica's FSM-apply of the cutover
+//	                            marker so L2's
+//	                            dynamicWrappedProposer publishes the
+//	                            same wrap closure independently of
+//	                            leadership state. The handler's
+//	                            InstallWrap call then becomes a
+//	                            redundant convenience (matches the
+//	                            state every follower will reach via
+//	                            the apply path).
+//	      Option B: make dynamicWrappedProposer.Propose consult the
+//	                sidecar's RaftEnvelopeCutoverIndex on every call
+//	                and refuse when the wrap pointer is nil but the
+//	                sidecar already reflects a cutover. Trades a
+//	                sidecar load per propose for a closed gap.
+//
+// Both hazards (a) and (b) share a single shape: post-cutover
+// cleartext entries land in Raft at indexes that the §6.3 apply
+// hook treats as wrapped. The gate (`raftEnvelopeWrapEnabled =
+// false`) is the only thing keeping either from triggering today.
+//
+// Idempotent against double-Begin: a second call freshens drainSig
+// and leaves barrierOpen true. CALLER SAFETY: a goroutine that was
+// blocked on a prior cycle's drainSig from WaitInflightDrained is
+// orphaned by the freshen (it never observes the close because the
+// channel reference was discarded). This is safe in practice
+// because the EncryptionAdminServer serializes EnableRaftEnvelope
+// calls via cutoverSem, so only one handler goroutine ever drives
+// the BeginCutoverBarrier → WaitInflightDrained → EndCutoverBarrier
+// sequence at a time. A future caller that drives the barrier
+// outside that semaphore MUST not rely on Begin's idempotency to
+// rescue an orphaned WaitInflightDrained — explicit End/Begin
+// ordering on a single goroutine is the only correct usage.
+//
+// Returns the channel that closes when in-flight drains to 0 so
+// callers MAY block on it directly; the recommended pattern is to
+// use WaitInflightDrained which composes context cancellation.
+func (p *dynamicWrappedProposer) BeginCutoverBarrier() <-chan struct{} {
+	p.barrierMu.Lock()
+	defer p.barrierMu.Unlock()
+	p.drainSig = make(chan struct{})
+	p.barrierOpen = true
+	if p.inflightUser == 0 {
+		// Fast path: no in-flight, drain is already complete.
+		close(p.drainSig)
+	}
+	return p.drainSig
+}
+
+// WaitInflightDrained blocks until the in-flight Propose counter
+// drops to 0 after BeginCutoverBarrier was called, or ctx fires.
+// Returns nil on drain, an error wrapping ctx.Err() with a
+// domain-only prefix on cancellation (the ShardGroup forwarder
+// adds the package prefix so operator logs don't carry a
+// redundant "kv: ... kv: ..." chain, claude r2 finding B), and
+// nil also if no barrier is currently open (degraded fast-path so
+// out-of-sequence calls don't deadlock callers).
+func (p *dynamicWrappedProposer) WaitInflightDrained(ctx context.Context) error {
+	p.barrierMu.Lock()
+	ch := p.drainSig
+	p.barrierMu.Unlock()
+	if ch == nil {
+		// No active barrier — drain is trivially complete.
+		return nil
+	}
+	select {
+	case <-ch:
+		return nil
+	case <-ctx.Done():
+		return errors.Wrap(ctx.Err(), "drain await canceled")
+	}
+}
+
+// EndCutoverBarrier closes the §7.1 step-6 barrier. After return,
+// fresh Propose calls succeed again. Idempotent against a
+// no-barrier-active state. Callers MUST call this once for each
+// BeginCutoverBarrier (the EnableRaftEnvelope handler uses defer).
+func (p *dynamicWrappedProposer) EndCutoverBarrier() {
+	p.barrierMu.Lock()
+	defer p.barrierMu.Unlock()
+	p.barrierOpen = false
+	// Drop the drainSig reference so a stale WaitInflightDrained
+	// caller that reads the channel after EndCutoverBarrier sees
+	// nil (immediate-success degraded path) rather than blocking
+	// on a closed channel from a previous cycle. Whether the
+	// channel was closed or not at this point depends on whether
+	// in-flight drained; both shapes are acceptable transient
+	// states because no new BeginCutoverBarrier has run yet to
+	// allocate a fresh channel.
+	p.drainSig = nil
+}
+
 // ProposeAdmin mirrors Propose's wrap-applies semantics. See
 // wrappedProposer.ProposeAdmin for the design rationale (the wrap
 // layer is NOT a barrier exemption; the EnableRaftEnvelope cutover