chore(consul-postgres-ha): env-gated pion ICE debug logs (MESH_CONN_DEBUG_ICE)

Plumbed `mesh_conn_debug_ice` from terraform.tfvars through cluster.tf
+ both compose templates + entrypoint into mesh-conn, where setting
the env to "1" wires `logging.NewDefaultLoggerFactory()` at LogLevelTrace
into pion's ice.AgentConfig. Used to diagnose the post-hot-patch
worker↔coord ICE-failure mode (see findings below). Off by default —
the trace output is very chatty.

Findings from one full instrumented run on the live cluster:

- pion's outbound STUN BINDING_REQUESTs are sent normally over the
  TURN allocation (~1Hz to each candidate pair).
- ZERO inbound STUN ever arrives at the workers' pion after a sidecar
  restart. `agent.go:1092 Inbound STUN` log line never appears in
  worker-3's log; eventually pion logs `Maximum requests reached for
  pair ... marking it as failed` and the agent gives up.
- Coordinators' pion DOES receive inbound STUN (from workers, in
  fact) and successfully NOMINATE candidate pairs — coord-0 logged
  `selected pair: relay 155.138.146.255:49666 <-> relay
  155.138.146.255:49456 (proto=udp)` for [worker-3] within ~3s of
  worker-3 republishing post-restart. So the path is genuinely
  half-open: worker→coord traffic flows, coord→worker doesn't.
- coturn shows zero rejections / quota / permission errors throughout.
  Allocation count rises and falls cleanly.
- All 6 CVMs share the same public IP (66.220.6.107 = dstack provider
  NAT). Different per-CVM source ports.

Conclusion: the asymmetric inbound-UDP failure is at the dstack
edge NAT, not at mesh-conn or coturn. Workers post-restart can't
receive inbound UDP from the relay on coturn's high port range
(155.138.146.255:49152-49999), even though coordinators on the same
cluster can. This is likely a NAT-mapping lifecycle issue (the new
post-restart mapping permits coturn:3478 source but not the
coturn:relay-port source), but proving that needs platform-side
inspection that we don't have access to.

Approach 2 (the partial fix in 5548bab) catches the stale-auth race
when it does fire, but the underlying inbound-UDP-blocked condition
prevents convergence regardless. The fix stays in (catches a real
race, doesn't make anything worse); the dstack-platform UDP behavior
needs a separate investigation and fix.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: h4x3rotab

consul-postgres-ha/cluster-example/cluster.tf

@@ -110,6 +110,14 @@ variable "mesh_conn_relay_only" {
   default = ""
 }
 
+# Set to "1" to enable pion's verbose ICE debug logs (connectivity
+# checks, STUN attribute parsing). Useful when debugging hot-patch
+# / re-handshake flakiness; off by default because it's very chatty.
+variable "mesh_conn_debug_ice" {
+  type    = string
+  default = ""
+}
+
 # ---------- Cluster topology + VIP allocation ----------
 
 locals {
@@ -188,6 +196,7 @@ resource "phala_app" "coordinator" {
     # Stage-1 WORKAROUND — see `random_bytes` block at top of file.
     GOSSIP_KEY           = random_bytes.gossip_key.base64
     MESH_CONN_RELAY_ONLY = var.mesh_conn_relay_only
+    MESH_CONN_DEBUG_ICE  = var.mesh_conn_debug_ice
   }
 
   listed         = false
@@ -233,6 +242,7 @@ resource "phala_app" "worker" {
     PATRONI_SUPERUSER_PW   = random_bytes.patroni_superuser_pw.hex
     PATRONI_REPLICATION_PW = random_bytes.patroni_replication_pw.hex
     MESH_CONN_RELAY_ONLY   = var.mesh_conn_relay_only
+    MESH_CONN_DEBUG_ICE    = var.mesh_conn_debug_ice
   }
 
   listed         = false

consul-postgres-ha/compose/coordinator.yaml

@@ -60,6 +60,7 @@ services:
       - GOSSIP_KEY=${GOSSIP_KEY}
       # See worker.yaml for the rationale on MESH_CONN_RELAY_ONLY.
       - MESH_CONN_RELAY_ONLY=${MESH_CONN_RELAY_ONLY:-}
+      - MESH_CONN_DEBUG_ICE=${MESH_CONN_DEBUG_ICE:-}
     volumes:
       - /var/run/dstack.sock:/var/run/dstack.sock:ro
       - /tmp/dstack-runtime/secrets:/run/secrets

consul-postgres-ha/compose/worker.yaml

@@ -48,6 +48,7 @@ services:
       # Default off because direct candidates work; flip on if a
       # deployment hits worker↔worker direct-pair instability.
       - MESH_CONN_RELAY_ONLY=${MESH_CONN_RELAY_ONLY:-}
+      - MESH_CONN_DEBUG_ICE=${MESH_CONN_DEBUG_ICE:-}
     volumes:
       - /var/run/dstack.sock:/var/run/dstack.sock:ro
       - /tmp/dstack-runtime/secrets:/run/secrets

consul-postgres-ha/mesh-conn/go.mod

@@ -4,6 +4,7 @@ go 1.24
 
 require (
 	github.com/pion/ice/v2 v2.3.25
+	github.com/pion/logging v0.2.2
 	github.com/pion/stun v0.6.1
 	github.com/quic-go/quic-go v0.59.0
 )
@@ -12,7 +13,6 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/google/uuid v1.3.1 // indirect
 	github.com/pion/dtls/v2 v2.2.7 // indirect
-	github.com/pion/logging v0.2.2 // indirect
 	github.com/pion/mdns v0.0.12 // indirect
 	github.com/pion/randutil v0.1.0 // indirect
 	github.com/pion/transport/v2 v2.2.2 // indirect

consul-postgres-ha/mesh-conn/main.go

@@ -81,6 +81,7 @@ import (
 	"time"
 
 	"github.com/pion/ice/v2"
+	"github.com/pion/logging"
 	"github.com/pion/stun"
 	"github.com/quic-go/quic-go"
 )
@@ -919,11 +920,20 @@ func dialICE(cfg *Config, remoteID string) (*ice.Conn, error) {
 	if os.Getenv("MESH_CONN_RELAY_ONLY") == "1" {
 		candidateTypes = []ice.CandidateType{ice.CandidateTypeRelay}
 	}
-	agent, err := ice.NewAgent(&ice.AgentConfig{
+	// MESH_CONN_DEBUG_ICE=1 turns on pion's verbose ICE-level logging
+	// (connectivity-check requests/responses, STUN attribute parsing,
+	// candidate-pair scoring). Off by default because it's chatty.
+	agentCfg := &ice.AgentConfig{
 		Urls:           urls,
 		NetworkTypes:   []ice.NetworkType{ice.NetworkTypeUDP4, ice.NetworkTypeTCP4},
 		CandidateTypes: candidateTypes,
-	})
+	}
+	if os.Getenv("MESH_CONN_DEBUG_ICE") == "1" {
+		lf := logging.NewDefaultLoggerFactory()
+		lf.DefaultLogLevel = logging.LogLevelTrace
+		agentCfg.LoggerFactory = lf
+	}
+	agent, err := ice.NewAgent(agentCfg)
 	if err != nil {
 		return nil, fmt.Errorf("NewAgent: %w", err)
 	}