Fix envoy configuration for gateway high availability (#131)

* envoy admin, experiments

* nginx, further experiments

* envoy sticky sessions

* configure hc timings for all timing paths

* leave jitter settings unmodified

* cleanup

Author: j-chmielewski

docker-compose2.0/docker-compose.yaml

@@ -68,14 +68,31 @@ services:
 
   gateway-lb:
     image: envoyproxy/envoy:v1.33-latest
+    command: ["envoy", "-c", "/etc/envoy/envoy.yaml", "-l", "debug"]
     ports:
       - "50051:50051/udp"
+      - "9901:9901"
     volumes:
       - ./envoy/envoy.yaml:/etc/envoy/envoy.yaml:ro
     depends_on:
       - gateway1
       - gateway2
 
+  # NGINX can be used for LB but not HA since it does not support healthchecks
+  # gateway-lb-nginx:
+  #   image: nginx:1.25-alpine
+  #   command:
+  #     - /bin/sh
+  #     - -ec
+  #     - until getent hosts gateway1 gateway2 >/dev/null 2>&1; do sleep 0.2; done; exec nginx -g 'daemon off;'
+  #   ports:
+  #     - "50051:50051/udp"
+  #   volumes:
+  #     - ./nginx/gateway-lb.conf:/etc/nginx/nginx.conf:ro
+  #   depends_on:
+  #     - gateway1
+  #     - gateway2
+
   db:
     image: postgres:18-alpine
     environment:

docker-compose2.0/envoy/envoy.yaml

@@ -1,3 +1,9 @@
+admin:
+  address:
+    socket_address:
+      address: 0.0.0.0
+      port_value: 9901
+
 static_resources:
   listeners:
     - name: udp_listener
@@ -19,20 +25,27 @@ static_resources:
             "@type": type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.UdpProxyConfig
             stat_prefix: udp_lb
             cluster: defguard_gateway_cluster
+            hash_policies:
+              - source_ip: true
             idle_timeout: 60s
 
   clusters:
     - name: defguard_gateway_cluster
       type: STRICT_DNS
       connect_timeout: 1s
-      lb_policy: ROUND_ROBIN
+      lb_policy: RING_HASH
       dns_lookup_family: V4_ONLY
 
       health_checks:
-        - timeout: 2s
-          interval: 5s
-          unhealthy_threshold: 2
-          healthy_threshold: 2
+        - timeout: 1s
+          interval: 1s
+          no_traffic_interval: 1s
+          no_traffic_healthy_interval: 1s
+          unhealthy_interval: 1s
+          healthy_edge_interval: 1s
+          unhealthy_edge_interval: 1s
+          unhealthy_threshold: 1
+          healthy_threshold: 1
           http_health_check:
             path: /health
             host: gateway_health
@@ -57,4 +70,4 @@ static_resources:
                       address: gateway2
                       port_value: 50051
                   health_check_config:
-                    port_value: 55003
+                    port_value: 55003

docker-compose2.0/nginx/gateway-lb.conf

@@ -0,0 +1,28 @@
+worker_processes auto;
+
+error_log /dev/stderr info;
+pid /tmp/nginx.pid;
+
+events {
+  worker_connections 1024;
+}
+
+stream {
+  log_format udp '$remote_addr:$remote_port [$time_local] '
+                 '$protocol $status bytes_sent=$bytes_sent bytes_received=$bytes_received '
+                 'session_time=$session_time upstream=$upstream_addr';
+
+  access_log /dev/stdout udp;
+
+  upstream defguard_gateways_udp {
+    server gateway1:50051 max_fails=1 fail_timeout=1s;
+    server gateway2:50051 max_fails=1 fail_timeout=1s;
+  }
+
+  server {
+    listen 50051 udp reuseport;
+    proxy_pass defguard_gateways_udp;
+    proxy_connect_timeout 1s;
+    proxy_timeout 60s;
+  }
+}