fix(consul-postgres-ha): mesh-conn PacketConn shim must delegate deadlines

Discovered while verifying terraform's beta.3 env-update path against
the live cluster on 2026-05-04: when terraform recreates a CVM, the
peers' QUIC links to it die, but the *redial* path can hang.

Specifically: dialICE returns a Connected ice.Conn, dialAndPump enters
quic.Dial, ICE later goes Failed (peer went away again, hairpin lost,
etc.). quic.Dial's context times out and quic-go calls
SetReadDeadline(past) to interrupt the blocked ReadFrom in our
iceConnPacketConn shim. The shim was returning nil from
SetReadDeadline, so the call had no effect on the underlying
ice.Conn.Read, and the goroutine hung forever. The surrounding
runPeerLink retry loop never got to retry, leaving the peer slot
permanently dead until the entire mesh-conn process was restarted.

Fix: delegate SetDeadline / SetReadDeadline / SetWriteDeadline to the
underlying conn (pion/ice.Conn implements net.Conn deadlines properly).
Same fix applied to the stage4/quic-on-ice smoke test so future
debugging stays trustworthy.

Adds a regression test using net.Pipe (which honors deadlines) that
asserts ReadFrom returns a Timeout-flagged net.Error within ~50ms of
SetReadDeadline.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: h4x3rotab

consul-postgres-ha/stage4/mesh-conn/main.go

@@ -697,11 +697,20 @@ func (p *iceConnPacketConn) WriteTo(b []byte, _ net.Addr) (int, error) {
 	return p.conn.Write(b)
 }
 
-func (p *iceConnPacketConn) Close() error                       { return p.conn.Close() }
-func (p *iceConnPacketConn) LocalAddr() net.Addr                { return p.conn.LocalAddr() }
-func (p *iceConnPacketConn) SetDeadline(_ time.Time) error      { return nil }
-func (p *iceConnPacketConn) SetReadDeadline(_ time.Time) error  { return nil }
-func (p *iceConnPacketConn) SetWriteDeadline(_ time.Time) error { return nil }
+func (p *iceConnPacketConn) Close() error        { return p.conn.Close() }
+func (p *iceConnPacketConn) LocalAddr() net.Addr { return p.conn.LocalAddr() }
+
+// Deadline methods delegate to ice.Conn (via the embedded net.Conn on
+// countingConn) instead of being a no-op. quic-go relies on
+// SetReadDeadline to interrupt a blocked ReadFrom when its context
+// cancels — without this delegation, a quic.Dial whose context times
+// out (e.g. because ICE went Failed mid-handshake) would hang forever
+// in our shim, and the surrounding runPeerLink retry loop never gets
+// to retry. Pion's ice.Conn implements the deadline methods, so this
+// is the natural place to wire them through.
+func (p *iceConnPacketConn) SetDeadline(t time.Time) error      { return p.conn.SetDeadline(t) }
+func (p *iceConnPacketConn) SetReadDeadline(t time.Time) error  { return p.conn.SetReadDeadline(t) }
+func (p *iceConnPacketConn) SetWriteDeadline(t time.Time) error { return p.conn.SetWriteDeadline(t) }
 
 // reportLinkStats logs a periodic summary per peer link. Once a minute,
 // and only when bytes actually moved since the last tick, so an idle

consul-postgres-ha/stage4/mesh-conn/validate_test.go

@@ -1,8 +1,10 @@
 package main
 
 import (
+	"net"
 	"strings"
 	"testing"
+	"time"
 )
 
 func TestValidatePeers_OK(t *testing.T) {
@@ -129,3 +131,41 @@ func TestValidatePeers_DigestDiffersWithDifferentPorts(t *testing.T) {
 		t.Fatalf("digest collides for different ports")
 	}
 }
+
+// iceConnPacketConn must delegate deadline methods to the underlying
+// conn so quic-go can interrupt blocked reads on context cancel.
+// Returning nil from these methods (the previous behavior) leaves
+// quic.Dial hung when ICE goes Failed mid-handshake — the surrounding
+// runPeerLink retry loop then never gets to retry. Verified once at
+// 2026-05-04 against the live cluster; this test pins the behavior so
+// a future refactor doesn't regress.
+func TestIceConnPacketConn_DeadlinesPropagate(t *testing.T) {
+	a, b := net.Pipe()
+	defer a.Close()
+	defer b.Close()
+
+	pkt := &iceConnPacketConn{conn: newCountingConn(a, "test")}
+
+	deadline := time.Now().Add(50 * time.Millisecond)
+	if err := pkt.SetReadDeadline(deadline); err != nil {
+		t.Fatalf("SetReadDeadline: %v", err)
+	}
+
+	buf := make([]byte, 100)
+	start := time.Now()
+	_, _, err := pkt.ReadFrom(buf)
+	elapsed := time.Since(start)
+
+	if err == nil {
+		t.Fatal("ReadFrom returned nil error past the deadline")
+	}
+	netErr, ok := err.(net.Error)
+	if !ok || !netErr.Timeout() {
+		t.Fatalf("expected timeout net.Error, got %v (%T)", err, err)
+	}
+	// Generous bounds: net.Pipe's deadline implementation is precise
+	// enough that 40-300ms covers test-VM jitter without flakes.
+	if elapsed < 40*time.Millisecond || elapsed > 300*time.Millisecond {
+		t.Errorf("ReadFrom returned in %v, expected ~50ms", elapsed)
+	}
+}

consul-postgres-ha/stage4/quic-on-ice/main.go

@@ -100,11 +100,14 @@ func (p *iceConnPacketConn) WriteTo(b []byte, _ net.Addr) (int, error) {
 	return p.conn.Write(b)
 }
 
-func (p *iceConnPacketConn) Close() error                       { return p.conn.Close() }
-func (p *iceConnPacketConn) LocalAddr() net.Addr                { return p.conn.LocalAddr() }
-func (p *iceConnPacketConn) SetDeadline(t time.Time) error      { return nil }
-func (p *iceConnPacketConn) SetReadDeadline(t time.Time) error  { return nil }
-func (p *iceConnPacketConn) SetWriteDeadline(t time.Time) error { return nil }
+func (p *iceConnPacketConn) Close() error        { return p.conn.Close() }
+func (p *iceConnPacketConn) LocalAddr() net.Addr { return p.conn.LocalAddr() }
+
+// Deadlines delegate to the ICE conn so quic-go can interrupt a
+// blocked ReadFrom on context cancel; mirrors mesh-conn/main.go.
+func (p *iceConnPacketConn) SetDeadline(t time.Time) error      { return p.conn.SetDeadline(t) }
+func (p *iceConnPacketConn) SetReadDeadline(t time.Time) error  { return p.conn.SetReadDeadline(t) }
+func (p *iceConnPacketConn) SetWriteDeadline(t time.Time) error { return p.conn.SetWriteDeadline(t) }
 
 // =============================================================================
 // QUIC client / server over the PacketConn