refactor(consul-postgres-ha): mesh-conn post-QUIC cleanup pass

Drops the MESH_CONN_TCP_ONLY env knob entirely (from dialICE, both
compose templates, and reportLinkStats's tick cadence). The flag was
investigated as a yamux-era escape hatch and proven non-helpful — pion
still selects relay-UDP candidates regardless because the relay
candidate's transport comes from the TURN allocation's relayed leg
(always UDP unless RFC 6062 TCP-allocation requested), not from the
client→TURN leg. With the QUIC switch, the underlying loss is handled
by the transport layer, so the knob has no remaining purpose and was
becoming misleading.

Also quiets reportLinkStats: tick 10s → 60s and skip the log line
entirely when bytes haven't moved since the last tick. Idle peer pairs
no longer spam every 10 seconds. Final-stats line on stop is unchanged
so postmortems still get a summary regardless of activity.

Drops the unused *quic.Conn parameter from reportLinkStats, refreshes
the stale "log every 10s" banner, and tightens the MESH_CONN_RELAY_ONLY
comment in worker.yaml so the rationale ("flip on if worker-to-worker
direct pairs fail") doesn't contradict itself.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: h4x3rotab

consul-postgres-ha/stage4/compose/coordinator.yaml

@@ -49,7 +49,6 @@ services:
       - TURN_SHARED_SECRET=${TURN_SHARED_SECRET}
       # See worker.yaml for the rationale on MESH_CONN_RELAY_ONLY.
       - MESH_CONN_RELAY_ONLY=${MESH_CONN_RELAY_ONLY:-}
-      - MESH_CONN_TCP_ONLY=${MESH_CONN_TCP_ONLY:-}
     volumes:
       - /tmp/dstack-runtime/secrets:/run/secrets:ro
       - /tmp/dstack-runtime/instance:/run/instance:ro

consul-postgres-ha/stage4/compose/worker.yaml

@@ -50,17 +50,12 @@ services:
       - TURN_HOST=${TURN_HOST}
       - TURN_SHARED_SECRET=${TURN_SHARED_SECRET}
       # MESH_CONN_RELAY_ONLY=1 forces ICE to gather only Relay candidates,
-      # routing all peer traffic through the coturn server. Trade-off:
-      # adds the Vultr round-trip to every peer hop, but avoids the
-      # direct-pair connectivity-check failures we hit between dstack
-      # workers (host/srflx/prflx pairs never establish, while relay
-      # always works). Set when DIRECT_FAILS_FOR_THIS_DEPLOYMENT.
+      # routing all peer traffic through the coturn server. Default off
+      # because coordinator-to-worker direct candidates do work; flip on
+      # if you hit a deployment where worker-to-worker direct pairs fail
+      # (host/srflx/prflx never establish, only relay does), trading some
+      # latency through coturn for guaranteed reachability.
       - MESH_CONN_RELAY_ONLY=${MESH_CONN_RELAY_ONLY:-}
-      # MESH_CONN_TCP_ONLY=1 restricts ICE to TCP candidates only. The
-      # underlying conn becomes a TCP byte stream that yamux can survive
-      # on (UDP loss kills yamux keepalives — proven by trace at
-      # 268KB pg_basebackup transfers).
-      - MESH_CONN_TCP_ONLY=${MESH_CONN_TCP_ONLY:-}
     volumes:
       - /tmp/dstack-runtime/secrets:/run/secrets:ro
       - /tmp/dstack-runtime/instance:/run/instance:ro

consul-postgres-ha/stage4/mesh-conn/main.go

@@ -435,7 +435,7 @@ func dialAndPump(cfg *Config, self, peer Peer) error {
 	// the underlying ice.Conn (i.e. wire bytes including QUIC overhead).
 	// QUIC's StreamCount isn't directly exposed, so we report just bytes.
 	stopStats := make(chan struct{})
-	go reportLinkStats(peer.ID, counted, qconn, stopStats)
+	go reportLinkStats(peer.ID, counted, stopStats)
 	defer close(stopStats)
 
 	// 3. Bind localhost UDP+TCP listeners for every one of peer's ports.
@@ -640,8 +640,8 @@ func pumpUDPStreamToSock(s *quic.Stream, sock *net.UDPConn, dst *net.UDPAddr) er
 // =============================================================================
 // Per-link instrumentation: count bytes through the ICE conn (i.e. the
 // raw wire bytes including QUIC framing/encryption overhead) and log
-// every 10s. Useful for diagnosing whether a link drop happens after
-// 0 bytes, 1KB, or 100MB.
+// a periodic summary. Useful for diagnosing whether a link drop happens
+// after 0 bytes, 1KB, or 100MB.
 // =============================================================================
 
 type countingConn struct {
@@ -703,8 +703,12 @@ func (p *iceConnPacketConn) SetDeadline(_ time.Time) error      { return nil }
 func (p *iceConnPacketConn) SetReadDeadline(_ time.Time) error  { return nil }
 func (p *iceConnPacketConn) SetWriteDeadline(_ time.Time) error { return nil }
 
-func reportLinkStats(peerID string, conn *countingConn, qconn *quic.Conn, stop <-chan struct{}) {
-	t := time.NewTicker(10 * time.Second)
+// reportLinkStats logs a periodic summary per peer link. Once a minute,
+// and only when bytes actually moved since the last tick, so an idle
+// mesh stays quiet. Always logs the final summary on stop, regardless
+// of activity, since that's what postmortems read.
+func reportLinkStats(peerID string, conn *countingConn, stop <-chan struct{}) {
+	t := time.NewTicker(60 * time.Second)
 	defer t.Stop()
 	var lastIn, lastOut uint64
 	for {
@@ -716,7 +720,10 @@ func reportLinkStats(peerID string, conn *countingConn, qconn *quic.Conn, stop <
 			return
 		case <-t.C:
 			in, out := conn.bytesIn.Load(), conn.bytesOut.Load()
-			log.Printf("[%s] link: in=%d (+%d B/10s) out=%d (+%d B/10s) reads=%d writes=%d",
+			if in == lastIn && out == lastOut {
+				continue
+			}
+			log.Printf("[%s] link: in=%d (+%d B/min) out=%d (+%d B/min) reads=%d writes=%d",
 				peerID, in, in-lastIn, out, out-lastOut,
 				conn.reads.Load(), conn.writes.Load())
 			lastIn, lastOut = in, out
@@ -831,27 +838,9 @@ func dialICE(cfg *Config, remoteID string) (*ice.Conn, error) {
 	if os.Getenv("MESH_CONN_RELAY_ONLY") == "1" {
 		candidateTypes = []ice.CandidateType{ice.CandidateTypeRelay}
 	}
-	// MESH_CONN_TCP_ONLY=1 restricts ICE to TCP NetworkTypes AND drops
-	// UDP-transport TURN URLs (pion's NetworkTypes only filters HOST
-	// candidates; Relay candidates inherit transport from their TURN
-	// URL's Proto, regardless of NetworkTypes). Without dropping UDP
-	// URLs, pion still picks `relay ... (proto=udp)` and we get the
-	// same UDP-loss behavior that kills yamux keepalives.
-	netTypes := []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeTCP4}
-	if os.Getenv("MESH_CONN_TCP_ONLY") == "1" {
-		netTypes = []ice.NetworkType{ice.NetworkTypeTCP4}
-		var tcpURLs []*stun.URI
-		for _, u := range urls {
-			if u.Proto == stun.ProtoTypeTCP {
-				tcpURLs = append(tcpURLs, u)
-			}
-		}
-		urls = tcpURLs
-		log.Printf("[%s] TCP-only mode: %d URLs after UDP filter", remoteID, len(urls))
-	}
 	agent, err := ice.NewAgent(&ice.AgentConfig{
 		Urls:           urls,
-		NetworkTypes:   netTypes,
+		NetworkTypes:   []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeTCP4},
 		CandidateTypes: candidateTypes,
 	})
 	if err != nil {