feat(consul-postgres-ha): declarative local.services in cluster.tf

Replace the hand-managed local.service_vips list with a single
local.services declaration. VIP octets (127.10.0.<n>) and per-backend
sidecar_ports (21000+n) are computed in HCL from declaration order;
entries sharing a canonical port collapse onto one backend (one Envoy,
one Connect-mTLS endpoint).

Workers now receive SERVICES_JSON — single source of truth for the
platform sidecar, patroni's role-watcher, and the consumer-side
/etc/hosts setup in compose/worker.yaml. Plan-time precondition
catches duplicate (name, subset) tuples.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: h4x3rotab

consul-postgres-ha/cluster-example/cluster.tf

@@ -153,18 +153,80 @@ locals {
   # COORDINATOR_VIPS — comma-separated for serf retry-join.
   coordinator_vips = join(",", [for p in local.peers : tostring(p.vip) if p.role == "coordinator"])
 
-  # Service VIPs: 127.10.0.<vip>/32 — one per Connect upstream a
-  # worker consumes. Three services in this template (extend by
-  # adding entries here + Connect upstreams in the sidecar config):
-  #   webdemo          for the cross-peer fan-out demo
-  #   postgres-master  the Patroni leader, leader-aware via subset filter
-  #   postgres-replica any Patroni replica
-  service_vips = [
-    { name = "webdemo", vip = 10, port = 8080 },
-    { name = "postgres-master", vip = 20, port = 5432 },
-    { name = "postgres-replica", vip = 21, port = 5432 },
+  # ---------- Service declarations ----------
+  #
+  # local.services is the single source of truth for every microservice
+  # on the mesh. Adding a service is one entry here; the platform
+  # sidecar reads SERVICES_JSON at startup and generates everything
+  # downstream (loopback aliases, /etc/hosts, Consul registrations,
+  # Envoy supervise loops, mesh-conn allowlist).
+  #
+  #   name    Consul service name (also the /etc/hosts alias).
+  #   port    Canonical app port. App binds 127.0.0.1:port; the local
+  #           Envoy upstream listener binds 127.10.0.<vip>:port.
+  #   subset  Optional. When set, this entry is a subset filter on a
+  #           shared backend (the postgres-master/-replica split is the
+  #           only case today). Entries sharing the same `port` collapse
+  #           into one producer-side sidecar — same Envoy public
+  #           listener, same Connect-mTLS endpoint, just different
+  #           service-resolver names with different subset filters.
+  services_raw = [
+    { name = "webdemo", port = 8080, subset = null },
+    { name = "postgres-master", port = 5432, subset = "master" },
+    { name = "postgres-replica", port = 5432, subset = "replica" },
   ]
-  upstreams_json = jsonencode(local.service_vips)
+
+  # Backends = unique canonical ports in declaration order. Each
+  # backend gets ONE producer-side sidecar at sidecar_port=21000+idx.
+  # Logical names sharing a backend (e.g. postgres-master + -replica)
+  # ride the same Envoy public listener; their distinct names live
+  # purely in service-resolver redirects on the Consul side.
+  service_ports = distinct([for s in local.services_raw : s.port])
+
+  port_sidecar = {
+    for idx, p in local.service_ports : tostring(p) => 21000 + idx
+  }
+
+  # Parent service name per backend. If at least one entry at this
+  # port has subset=null, that entry's `name` is the parent (the
+  # platform sidecar registers a Consul service with Connect.SidecarService
+  # inline — pattern A, webdemo case). If every entry has a subset,
+  # the parent is var.cluster_name (pattern B — Patroni-style: the
+  # parent service is auto-registered by the workload itself under
+  # its `scope`, so the platform only registers a standalone
+  # connect-proxy pointing at it).
+  port_parent = {
+    for p in local.service_ports : tostring(p) => try(
+      [for s in local.services_raw : s.name if s.port == p && s.subset == null][0],
+      var.cluster_name,
+    )
+  }
+
+  # Whether the platform registers the parent service itself.
+  # False = the workload (e.g. Patroni) auto-registers its parent.
+  port_registers_parent = {
+    for p in local.service_ports : tostring(p) =>
+    length([for s in local.services_raw : s if s.port == p && s.subset == null]) > 0
+  }
+
+  # Final per-service descriptors. VIP octets allocated 10+ordinal so
+  # subset entries each get their own 127.10.0.<vip>/32 loopback alias
+  # + /etc/hosts entry — that's what makes `postgres-master:5432` and
+  # `postgres-replica:5432` resolve to different consumer-side Envoy
+  # listeners even when they share a producer-side sidecar.
+  services = [
+    for idx, s in local.services_raw : {
+      name             = s.name
+      port             = s.port
+      subset           = s.subset
+      vip              = 10 + idx
+      sidecar_port     = local.port_sidecar[tostring(s.port)]
+      parent           = local.port_parent[tostring(s.port)]
+      registers_parent = local.port_registers_parent[tostring(s.port)]
+    }
+  ]
+
+  services_json = jsonencode(local.services)
 }
 
 # ---------- Coordinator ----------
@@ -228,7 +290,7 @@ resource "phala_app" "worker" {
   env = {
     CLUSTER_NAME       = var.cluster_name
     PEERS_JSON         = local.peers_json
-    UPSTREAMS_JSON     = local.upstreams_json
+    SERVICES_JSON      = local.services_json
     WORKER_ORDINAL     = tostring(each.value)
     EXPECTED_REPLICAS  = var.worker_replicas + var.coordinator_replicas
     COORDINATOR_VIPS   = local.coordinator_vips
@@ -253,6 +315,20 @@ resource "phala_app" "worker" {
   wait_for_ready       = true
   wait_timeout_seconds = 600
 
+  # Plan-time check: every (name, subset) tuple in local.services must
+  # be unique. Catches typos like declaring two `webdemo` entries, or
+  # two `postgres-master` entries with the same subset. Backends
+  # (services sharing a canonical port) intentionally collapse onto
+  # one sidecar_port; that's not flagged.
+  lifecycle {
+    precondition {
+      condition = length(local.services) == length(distinct([
+        for s in local.services : "${s.name}/${s.subset == null ? "" : s.subset}"
+      ]))
+      error_message = "local.services has duplicate (name, subset) entries — each logical service name (with its subset) must be unique."
+    }
+  }
+
   depends_on = [phala_app.coordinator]
 }
 

consul-postgres-ha/compose/worker.yaml

@@ -3,11 +3,14 @@
 # A "worker" CVM ends up running exactly three containers:
 #
 #   sidecar   bundled platform plumbing — bootstrap-secrets, mesh-conn,
-#             consul (client), and TWO envoys (one per Connect sidecar:
-#             webdemo on 21000, postgres on 21001). Provisions per-peer
-#             VIP loopback aliases on 127.50.0.0/24 and per-service VIP
-#             aliases on 127.10.0.0/24 at startup, so needs NET_ADMIN.
-#   patroni   the workload — Postgres + Patroni leader/replica.
+#             consul (client), and one Envoy per backend declared in
+#             SERVICES_JSON (services sharing a canonical port share a
+#             backend). Provisions per-peer VIP loopback aliases on
+#             127.50.0.0/24 and per-service VIP aliases on
+#             127.10.0.0/24 at startup, so needs NET_ADMIN.
+#   patroni   the workload — Postgres + Patroni leader/replica. Also
+#             hosts the role-watcher that mirrors Patroni's current
+#             role onto the postgres sidecar's Service.Tags.
 #   webdemo   tiny example app sitting on the mesh; swap for your
 #             own service when adapting this template.
 
@@ -28,12 +31,16 @@ services:
       - WORKER_ORDINAL=${WORKER_ORDINAL}
       - EXPECTED_REPLICAS=${EXPECTED_REPLICAS}
       # PEERS_JSON shape: [{id, vip}] — vip is the last octet in
-      # 127.50.0.0/24. mesh-conn binds the static infra-port allowlist
-      # on every other peer's VIP; entrypoint provisions the aliases.
+      # 127.50.0.0/24. mesh-conn binds the runtime allowlist on every
+      # other peer's VIP; entrypoint provisions the aliases.
       - PEERS_JSON=${PEERS_JSON}
-      # UPSTREAMS_JSON shape: [{name, vip, port}] — service VIPs the
-      # local Envoy will bind for its declared Connect upstreams.
-      - UPSTREAMS_JSON=${UPSTREAMS_JSON}
+      # SERVICES_JSON shape: [{name, port, subset, vip, sidecar_port,
+      # parent, registers_parent}] — single source of truth for every
+      # microservice on the mesh. The sidecar generates Consul
+      # registrations, Envoy supervise loops, /etc/hosts aliases, and
+      # mesh-conn's allowlist from this one env. Editing local.services
+      # in cluster.tf is the only place adding a service touches.
+      - SERVICES_JSON=${SERVICES_JSON}
       # COORDINATOR_VIPS: comma-separated list of coordinators' peer
       # VIPs. Workers retry-join 127.50.0.<vip>:8301 (serf) for each.
       - COORDINATOR_VIPS=${COORDINATOR_VIPS}
@@ -71,18 +78,25 @@ services:
     # CLUSTER_NAME drives Patroni's `scope` — every peer's patroni
     # must use the same value to land in the same cluster. Postgres
     # binds the canonical 127.0.0.1:5432; Patroni REST on 127.0.0.1:8008.
+    # The Patroni entrypoint also hosts the role-watcher loop that
+    # mirrors Patroni's current role into the postgres-sidecar
+    # registration's Service.Tags (workload-specific, lives here
+    # instead of in the platform sidecar).
     # WORKAROUND: superuser + replication passwords generated in
     # Terraform and broadcast identically to every worker. See
     # cluster.tf.
     environment:
       - CLUSTER_NAME=${CLUSTER_NAME}
       - PATRONI_SUPERUSER_PW=${PATRONI_SUPERUSER_PW}
       - PATRONI_REPLICATION_PW=${PATRONI_REPLICATION_PW}
-      # Service VIPs for /etc/hosts — patroni needs to resolve
-      # postgres-master to the local service VIP so its replication
-      # source string ("primary_conninfo") routes through the local
-      # Envoy → mesh → leader's Postgres.
-      - UPSTREAMS_JSON=${UPSTREAMS_JSON}
+      # SERVICES_JSON: service VIPs for /etc/hosts (patroni needs to
+      # resolve postgres-master to its local Envoy upstream so
+      # primary_conninfo routes through the mesh), AND the sidecar
+      # registration shape the role-watcher re-PUTs with tags.
+      - SERVICES_JSON=${SERVICES_JSON}
+      # PEERS_JSON: role-watcher needs SELF_VIP to set Address on the
+      # sidecar registration.
+      - PEERS_JSON=${PEERS_JSON}
     volumes:
       - /tmp/dstack-runtime/instance:/run/instance:ro
       - /tmp/dstack-runtime/secrets:/run/secrets:ro
@@ -97,17 +111,15 @@ services:
     restart: unless-stopped
     environment:
       - PEERS_JSON=${PEERS_JSON}
-      - UPSTREAMS_JSON=${UPSTREAMS_JSON}
+      - SERVICES_JSON=${SERVICES_JSON}
     entrypoint: ["/bin/sh", "-c"]
     command:
       - |
         set -e
         export PEER_ID=$$(jq -r '.role + "-" + (.ordinal|tostring)' /run/instance/info.json)
-        export SELF_VIP=$$(echo "$$PEERS_JSON" | jq -r --arg id "$$PEER_ID" '.[] | select(.id == $$id) | .vip')
         # Append service-VIP /etc/hosts entries — each container has
         # its own /etc/hosts (network_mode: host doesn't share it).
-        echo "$$UPSTREAMS_JSON" | jq -r '.[] | "127.10.0.\(.vip)  \(.name)"' >> /etc/hosts
-        export CONSUL_HTTP_ADDR=127.0.0.1:8500
+        echo "$$SERVICES_JSON" | jq -r '.[] | "127.10.0.\(.vip)  \(.name)"' >> /etc/hosts
         exec webdemo
     volumes:
       - /tmp/dstack-runtime/instance:/run/instance:ro