fix(consul-postgres-ha): live-test fallout — sidecar registration + self-VIP dispatch

Three issues surfaced from the first end-to-end run on a real
6-CVM dstack cluster:

1. **sidecar registration JSON format**: the brief's plan to use
   `consul services register` failed because that CLI takes
   HCL-style files with lowercase keys wrapped in `{"service": ...}`,
   not the flat capital-letter struct the agent HTTP API takes.
   Switched to `curl PUT /v1/agent/service/register` for consistency
   with how webdemo registers itself.

2. **service-resolver service-name mismatch**: Patroni auto-registers
   the parent service under `<scope>` (= `CLUSTER_NAME`), not under
   the literal string `postgres`. Service-resolver entries +
   intentions + the postgres-sidecar's `DestinationServiceName` now
   parameterize on `${CLUSTER_NAME}` so consumer-facing names
   (`postgres-master`/`postgres-replica`) redirect to the actual
   Patroni-tagged scope.

3. **self-VIP dispatch**: Consul's "server health" probes self-dial
   `127.50.0.<self-vip>:8300`, and mesh-conn dispatched inbound to
   `127.0.0.1:port` while Consul (per the brief) also listened on
   `127.0.0.1`. Self-dial then failed with connection-refused.
   Fixed by (a) `-bind=127.50.0.${SELF_VIP}` so Consul listens on
   the self-VIP (which is local-aliased on lo) and (b) mesh-conn's
   accept-loop dispatches to `self.vipAddr():port` on the receive
   side. Both directions now land on the same listener.

Also adds `MESH_CONN_RELAY_ONLY` as a Terraform variable + plumbs
it through to both coord and worker env so the documented
ICE-flakiness workaround can be flipped without a code change.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: h4x3rotab

consul-postgres-ha/cluster-example/cluster.tf

@@ -101,6 +101,15 @@ variable "external_turn_secret" {
   sensitive = true
 }
 
+# Force ICE to gather Relay candidates only — routes all peer traffic
+# through coturn instead of attempting NAT-hairpin direct paths. Set
+# this when worker↔worker direct-pair ICE handshakes are unstable
+# (the dstack provider NAT path is known-flaky for these pairs).
+variable "mesh_conn_relay_only" {
+  type    = string
+  default = ""
+}
+
 # ---------- Cluster topology + VIP allocation ----------
 
 locals {
@@ -177,7 +186,8 @@ resource "phala_app" "coordinator" {
     TURN_SHARED_SECRET  = var.external_turn_secret
     MESH_SIDECAR_IMAGE  = var.mesh_sidecar_image
     # Stage-1 WORKAROUND — see `random_bytes` block at top of file.
-    GOSSIP_KEY = random_bytes.gossip_key.base64
+    GOSSIP_KEY           = random_bytes.gossip_key.base64
+    MESH_CONN_RELAY_ONLY = var.mesh_conn_relay_only
   }
 
   listed         = false
@@ -222,6 +232,7 @@ resource "phala_app" "worker" {
     GOSSIP_KEY             = random_bytes.gossip_key.base64
     PATRONI_SUPERUSER_PW   = random_bytes.patroni_superuser_pw.hex
     PATRONI_REPLICATION_PW = random_bytes.patroni_replication_pw.hex
+    MESH_CONN_RELAY_ONLY   = var.mesh_conn_relay_only
   }
 
   listed         = false
@@ -237,7 +248,8 @@ resource "phala_app" "worker" {
 output "coordinator_app_ids" { value = { for k, c in phala_app.coordinator : k => c.app_id } }
 output "worker_app_ids" { value = { for k, w in phala_app.worker : k => w.app_id } }
 output "consul_ui" {
-  # Coordinator-0's Consul HTTP API on the canonical 8500. The dstack
-  # gateway maps `<app_id>-<port>s.<gateway>` to that port on the CVM.
-  value = "https://${phala_app.coordinator["0"].app_id}-8500s.${var.gateway_domain}/ui"
+  # Coordinator-0's Consul HTTP API on the canonical 8500. Plain HTTP
+  # backend → use the no-`s` gateway form (gateway terminates TLS).
+  # See README "dstack gateway URL convention".
+  value = "https://${phala_app.coordinator["0"].app_id}-8500.${var.gateway_domain}/ui"
 }

consul-postgres-ha/mesh-conn/main.go

@@ -497,7 +497,7 @@ func dialAndPump(cfg *Config, self, peer Peer) error {
 
 	udpPortCount := len(udpSocks)
 	go func() {
-		errCh <- runAcceptLoop(connCtx, qconn, udpStreams, udpPortCount, allUDPReady)
+		errCh <- runAcceptLoop(connCtx, qconn, &self, udpStreams, udpPortCount, allUDPReady)
 	}()
 
 	if isClient {
@@ -539,7 +539,14 @@ func dialAndPump(cfg *Config, self, peer Peer) error {
 			st := udpStreams[port]
 			go func() { errCh <- pumpUDPSockToStream(us, st) }()
 			go func() {
-				udpDst := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port}
+				// Dispatch to the SELF VIP, not 127.0.0.1, so the
+				// inbound packet lands on whatever (Consul, Envoy, ...)
+				// is listening on this peer's own loopback alias. This
+				// also keeps the source-port-preservation invariant
+				// intact: the local listener's bound addr is the
+				// remote peer's VIP, so the receiving service sees
+				// "from peer-N" as the source.
+				udpDst := &net.UDPAddr{IP: self.vipAddr(), Port: port}
 				errCh <- pumpUDPStreamToSock(st, us, udpDst)
 			}()
 		}
@@ -554,10 +561,11 @@ func dialAndPump(cfg *Config, self, peer Peer) error {
 
 // runAcceptLoop handles every incoming QUIC stream from the peer.
 // streamUDP headers are matched into udpStreams keyed by port; streamTCP
-// triggers a Dial to 127.0.0.1:<port>. The receiver validates port
-// against the static allowlist; anything else is rejected — that's the
-// only port-knowledge the receiver needs.
-func runAcceptLoop(ctx context.Context, qconn *quic.Conn, udpStreams map[int]*quic.Stream, expectedUDP int, allUDPReady chan struct{}) error {
+// triggers a Dial to <self-vip>:<port> (so it lands on whatever the
+// local Consul / Envoy bound on the peer's own loopback alias). The
+// receiver validates port against the static allowlist; anything
+// else is rejected — that's the only port-knowledge the receiver needs.
+func runAcceptLoop(ctx context.Context, qconn *quic.Conn, self *Peer, udpStreams map[int]*quic.Stream, expectedUDP int, allUDPReady chan struct{}) error {
 	udpRegistered := 0
 	for {
 		s, err := qconn.AcceptStream(ctx)
@@ -592,7 +600,7 @@ func runAcceptLoop(ctx context.Context, qconn *quic.Conn, udpStreams map[int]*qu
 				close(allUDPReady)
 			}
 		case streamTCP:
-			go handleIncomingTCP(s, &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port})
+			go handleIncomingTCP(s, &net.TCPAddr{IP: self.vipAddr(), Port: port})
 		default:
 			log.Printf("unknown stream tag 0x%x", tag)
 			s.CancelRead(0)

consul-postgres-ha/mesh-sidecar/entrypoint.sh

@@ -94,8 +94,17 @@ CONSUL_ARGS=(
   agent
   -node="$PEER_ID"
   -datacenter="${CLUSTER_NAME}"
-  -bind=127.0.0.1
+  # bind on the self-VIP so serf + RPC listen there. Consumers (other
+  # peers AND self) reach this Consul at 127.50.0.<self-vip>:8301/8300:
+  #   - other-peer dials → that peer's mesh-conn forwards via QUIC →
+  #     this peer's mesh-conn dispatches to 127.50.0.<self-vip>:port →
+  #     Consul receives.
+  #   - self-dial (e.g. internal "server health" probe) → kernel
+  #     loopback → Consul receives directly. No mesh-conn hop.
+  -bind="127.50.0.${SELF_VIP}"
   -advertise="127.50.0.${SELF_VIP}"
+  # HTTP API + gRPC stay loopback-only — apps and Patroni use them
+  # from inside the same network namespace, never via the mesh.
   -client=127.0.0.1
   -serf-lan-port=8301
   -server-port=8300
@@ -154,39 +163,44 @@ if [ "$ROLE" = "worker" ]; then
   # filtered by the role tag Patroni stamps on each registration.
   POSTGRES_MASTER_VIP=$(jq -r '.[] | select(.name=="postgres-master") | .vip' <<<"$UPSTREAMS_JSON")
   POSTGRES_REPLICA_VIP=$(jq -r '.[] | select(.name=="postgres-replica") | .vip' <<<"$UPSTREAMS_JSON")
+  # Patroni's auto-registered parent service uses CLUSTER_NAME as the
+  # service name (Patroni-internal "scope"). The Connect sidecar
+  # terminates inbound mTLS for that same name; consumers reach it via
+  # the postgres-master/-replica service-resolver redirects below.
   cat > /tmp/postgres-sidecar.json <<EOF
 {
-  "kind": "connect-proxy",
-  "id": "postgres-sidecar-${PEER_ID}",
-  "name": "postgres-sidecar-proxy",
-  "address": "127.50.0.${SELF_VIP}",
-  "port": 21001,
-  "proxy": {
-    "destination_service_name": "postgres",
-    "destination_service_id": "postgres-${PEER_ID}",
-    "local_service_address": "127.0.0.1",
-    "local_service_port": 5432,
-    "upstreams": [
+  "Kind": "connect-proxy",
+  "ID": "postgres-sidecar-${PEER_ID}",
+  "Name": "postgres-sidecar-proxy",
+  "Address": "127.50.0.${SELF_VIP}",
+  "Port": 21001,
+  "Proxy": {
+    "DestinationServiceName": "${CLUSTER_NAME}",
+    "LocalServiceAddress": "127.0.0.1",
+    "LocalServicePort": 5432,
+    "Upstreams": [
       {
-        "destination_name": "postgres-master",
-        "local_bind_address": "127.10.0.${POSTGRES_MASTER_VIP}",
-        "local_bind_port": 5432
+        "DestinationName": "postgres-master",
+        "LocalBindAddress": "127.10.0.${POSTGRES_MASTER_VIP}",
+        "LocalBindPort": 5432
       },
       {
-        "destination_name": "postgres-replica",
-        "local_bind_address": "127.10.0.${POSTGRES_REPLICA_VIP}",
-        "local_bind_port": 5432
+        "DestinationName": "postgres-replica",
+        "LocalBindAddress": "127.10.0.${POSTGRES_REPLICA_VIP}",
+        "LocalBindPort": 5432
       }
     ]
   }
 }
 EOF
-  # Re-register on every boot; Consul's API is PUT-style so this is
-  # idempotent. Patroni's auto-registration of the regular `postgres`
-  # service may not have landed yet — that's fine, the sidecar can
-  # register before its destination service exists.
+  # Re-register on every boot via the agent HTTP API (PUT-idempotent).
+  # The `consul services register` CLI uses a different file format
+  # (HCL-style with a top-level `service` block); the HTTP API takes
+  # the flat capital-letter JSON above and is consistent with how
+  # webdemo registers itself.
   log "registering postgres-sidecar-proxy (port=21001)"
-  until consul services register /tmp/postgres-sidecar.json; do
+  until curl -fsS -X PUT --data-binary @/tmp/postgres-sidecar.json \
+           http://127.0.0.1:8500/v1/agent/service/register; do
     log "postgres-sidecar-proxy register failed; retrying"
     sleep 2
   done
@@ -195,7 +209,7 @@ EOF
   # need Envoy to form quorum (Raft RPC rides peer VIPs as plain TCP),
   # but workers can't start their sidecars until a server can issue.
   log "waiting for connect CA to be ready..."
-  until consul connect ca leaf -service=postgres >/dev/null 2>&1; do
+  until consul connect ca leaf -service="${CLUSTER_NAME}" >/dev/null 2>&1; do
     sleep 2
   done
   log "connect CA ready"
@@ -249,36 +263,37 @@ Config { protocol = "tcp" }
 HCL
 
     # Postgres subset definitions. Patroni auto-registers the parent
-    # `postgres` service with role tags (master|replica); subsets pick
-    # the right instances by tag, and the redirect resolvers expose
-    # consumer-facing names that are independent of Patroni's tagging.
-    consul config write - <<'HCL' || true
+    # service under its `scope` (= CLUSTER_NAME) with role tags
+    # (master|replica); subsets pick the right instances by tag, and
+    # the redirect resolvers expose consumer-facing names that are
+    # independent of Patroni's scope.
+    consul config write - <<HCL || true
 Kind    = "service-resolver"
-Name    = "postgres"
+Name    = "${CLUSTER_NAME}"
 Subsets = {
   master  = { Filter = "Service.Tags contains \"master\"" }
   replica = { Filter = "Service.Tags contains \"replica\"" }
 }
 HCL
 
-    consul config write - <<'HCL' || true
+    consul config write - <<HCL || true
 Kind     = "service-resolver"
 Name     = "postgres-master"
-Redirect { Service = "postgres", ServiceSubset = "master" }
+Redirect { Service = "${CLUSTER_NAME}", ServiceSubset = "master" }
 HCL
 
-    consul config write - <<'HCL' || true
+    consul config write - <<HCL || true
 Kind     = "service-resolver"
 Name     = "postgres-replica"
-Redirect { Service = "postgres", ServiceSubset = "replica" }
+Redirect { Service = "${CLUSTER_NAME}", ServiceSubset = "replica" }
 HCL
 
     # Default-allow intentions for now. Tightening this to per-pair
     # allow + default-deny is straightforward but out of scope here;
     # see design/attestation-admission.md for the longer-term shape.
-    consul config write - <<'HCL' || true
+    consul config write - <<HCL || true
 Kind = "service-intentions"
-Name = "postgres"
+Name = "${CLUSTER_NAME}"
 Sources = [{ Name = "*", Action = "allow" }]
 HCL
     consul config write - <<'HCL' || true