Skip to content

Commit a31525c

Browse files
alexghralexghr
authored
refactor: sticky session based on IP (#24120)
Pin session by client ID rather than cookie, add Kong healtchecks and clean up namespaces.
1 parent 05942c3 commit a31525c

8 files changed

Lines changed: 70 additions & 62 deletions

File tree

‎spartan/terraform/deploy-rpc/environments/mainnet/outputs.tf‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@ output "kong_routes" {
88
value = module.environment.kong_routes
99
}
1010

11-
output "kong_sticky_session_policy_name" {
12-
description = "Kong sticky session policy name, or null when disabled."
13-
value = module.environment.kong_sticky_session_policy_name
11+
output "kong_upstream_policy_name" {
12+
description = "Kong upstream policy name, or null when disabled."
13+
value = module.environment.kong_upstream_policy_name
1414
}
1515

1616
output "kong_metrics_service" {

‎spartan/terraform/deploy-rpc/environments/testnet/outputs.tf‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@ output "kong_routes" {
88
value = module.environment.kong_routes
99
}
1010

11-
output "kong_sticky_session_policy_name" {
12-
description = "Kong sticky session policy name, or null when disabled."
13-
value = module.environment.kong_sticky_session_policy_name
11+
output "kong_upstream_policy_name" {
12+
description = "Kong upstream policy name, or null when disabled."
13+
value = module.environment.kong_upstream_policy_name
1414
}
1515

1616
output "kong_metrics_service" {

‎spartan/terraform/deploy-rpc/modules/environment/main.tf‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,6 @@ terraform {
1313
}
1414

1515
locals {
16-
# route requests from the same client to the same RPC node in order to have a consisten view of the chain
17-
sticky_policy_name = "${var.RELEASE_PREFIX}-rpc-sticky-sessions"
18-
1916
irm_alloy_name = "${var.RELEASE_PREFIX}-rpc-irm-alloy"
2017
irm_secret_name = "${local.irm_alloy_name}-grafana-cloud"
2118
irm_metric_regex = "up|kong_http_requests_total|kong_request_latency_ms_bucket|kong_latency_bucket|kong_upstream_target_health"
@@ -86,8 +83,7 @@ module "rpc_gateway" {
8683
RELEASE_PREFIX = var.RELEASE_PREFIX
8784
CONSUMER_NAMESPACE = var.NAMESPACE
8885

89-
STICKY_SESSIONS_ENABLED = true
90-
STICKY_SESSION_POLICY_NAME = local.sticky_policy_name
86+
KONG_TRUSTED_IP_RANGES = ["35.191.0.0/16", "130.211.0.0/22"] # Google LB IP ranges https://docs.cloud.google.com/load-balancing/docs/firewall-rules
9187

9288
ROUTES = local.rpc_routes
9389
CONSUMERS = var.CONSUMERS

‎spartan/terraform/deploy-rpc/modules/environment/outputs.tf‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@ output "kong_routes" {
1515
value = module.rpc_gateway.route_names
1616
}
1717

18-
output "kong_sticky_session_policy_name" {
19-
description = "Kong sticky session policy name."
20-
value = module.rpc_gateway.sticky_session_policy_name
18+
output "kong_upstream_policy_name" {
19+
description = "Kong upstream policy name."
20+
value = module.rpc_gateway.upstream_policy_name
2121
}
2222

2323
output "kong_metrics_service" {

‎spartan/terraform/deploy-rpc/modules/rpc/main.tf‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,10 @@ locals {
1414
aztec_image_repository = join(":", slice(local.aztec_image_parts, 0, length(local.aztec_image_parts) - 1))
1515
aztec_image_tag = local.aztec_image_parts[length(local.aztec_image_parts) - 1]
1616

17-
workload_name = "${var.RELEASE_NAME}-aztec-node"
18-
l1_secret_name = "${var.RELEASE_NAME}-l1"
19-
otel_secret_name = "${var.RELEASE_NAME}-otel"
20-
sticky_policy_name = "${var.RELEASE_PREFIX}-rpc-sticky-sessions"
17+
workload_name = "${var.RELEASE_NAME}-aztec-node"
18+
l1_secret_name = "${var.RELEASE_NAME}-l1"
19+
otel_secret_name = "${var.RELEASE_NAME}-otel"
20+
upstream_policy_name = "${var.RELEASE_PREFIX}-rpc-upstream-policy"
2121
}
2222

2323
resource "helm_release" "rpc" {
@@ -29,7 +29,7 @@ resource "helm_release" "rpc" {
2929
force_update = true
3030
recreate_pods = true
3131
reuse_values = false
32-
timeout = 600
32+
timeout = 3600 # 1h to sync
3333
wait = true
3434
wait_for_jobs = true
3535
take_ownership = true
@@ -150,6 +150,9 @@ resource "helm_release" "rpc" {
150150

151151
node = {
152152
logLevel = "info"
153+
startupProbe = {
154+
failureThreshold = 120
155+
}
153156
env = {
154157
OTEL_SERVICE_NAME = var.RELEASE_NAME
155158
}
@@ -194,7 +197,7 @@ resource "helm_release" "rpc" {
194197
port = 8080
195198
type = "ClusterIP"
196199
annotations = {
197-
"konghq.com/upstream-policy" = local.sticky_policy_name
200+
"konghq.com/upstream-policy" = local.upstream_policy_name
198201
}
199202
}
200203
admin = {

‎spartan/terraform/modules/rpc-gateway/main.tf‎

Lines changed: 41 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,17 +13,18 @@ terraform {
1313
}
1414

1515
locals {
16-
kong_namespace = var.KONG_NAMESPACE != "" ? var.KONG_NAMESPACE : "${var.RELEASE_PREFIX}-rpc-kong"
16+
kong_namespace = var.KONG_NAMESPACE != "" ? var.KONG_NAMESPACE : var.CONSUMER_NAMESPACE
1717
kong_helm_release_name = var.KONG_HELM_RELEASE_NAME != "" ? var.KONG_HELM_RELEASE_NAME : "${var.RELEASE_PREFIX}-rpc-kong"
1818
kong_ingress_class = var.KONG_INGRESS_CLASS != "" ? var.KONG_INGRESS_CLASS : "${var.RELEASE_PREFIX}-rpc-kong"
1919

20-
sticky_session_policy_name = var.STICKY_SESSION_POLICY_NAME != "" ? var.STICKY_SESSION_POLICY_NAME : "${var.RELEASE_PREFIX}-rpc-sticky-sessions"
20+
upstream_policy_name = "${var.RELEASE_PREFIX}-rpc-upstream-policy"
2121
frontend_static_ip_name = var.FRONTEND_STATIC_IP_NAME != "" ? var.FRONTEND_STATIC_IP_NAME : "${var.RELEASE_PREFIX}-rpc-frontend"
2222
frontend_service_name = var.FRONTEND_SERVICE_NAME != "" ? var.FRONTEND_SERVICE_NAME : "${local.kong_helm_release_name}-gateway-proxy"
2323
managed_certificate_name = var.GCP_MANAGED_CERTIFICATE_NAME != "" ? var.GCP_MANAGED_CERTIFICATE_NAME : "${var.RELEASE_PREFIX}-rpc-cert"
2424
frontend_backend_config_name = "${var.RELEASE_PREFIX}-rpc-kong-backend"
2525
frontend_hosts = toset(flatten([for _, route in var.ROUTES : route.hosts]))
2626
frontend_load_balancer_ip = var.FRONTEND_ENABLED && var.FRONTEND_STATIC_IP_ENABLED ? try(google_compute_global_address.frontend[0].address, "") : ""
27+
kong_trusted_ips = concat(var.KONG_TRUSTED_IP_RANGES, var.FRONTEND_ENABLED && var.FRONTEND_STATIC_IP_ENABLED ? [local.frontend_load_balancer_ip] : [])
2728

2829
kong_proxy_service_annotations = merge(
2930
var.FRONTEND_ENABLED ? {
@@ -87,9 +88,16 @@ resource "helm_release" "kong" {
8788
values = concat([
8889
yamlencode({
8990
gateway = {
90-
env = {
91-
database = "off"
92-
}
91+
env = merge(
92+
{
93+
database = "off"
94+
real_ip_header = "X-Forwarded-For"
95+
real_ip_recursive = "on"
96+
},
97+
length(local.kong_trusted_ips) > 0 ? {
98+
trusted_ips = join(",", local.kong_trusted_ips)
99+
} : {}
100+
)
93101
proxy = {
94102
type = var.KONG_PROXY_SERVICE_TYPE
95103
annotations = local.kong_proxy_service_annotations
@@ -151,6 +159,9 @@ resource "kubernetes_manifest" "frontend_backend_config" {
151159
requestPath = "/status"
152160
port = 8100
153161
}
162+
customRequestHeaders = {
163+
headers = ["X-Forwarded-For:{client_ip_address},{server_ip_address}"]
164+
}
154165
}
155166
}
156167

@@ -536,27 +547,43 @@ resource "kubernetes_manifest" "otel_collector_deployment" {
536547
]
537548
}
538549

539-
resource "kubernetes_manifest" "sticky_session_policy" {
540-
for_each = var.STICKY_SESSIONS_ENABLED ? toset(distinct([for _, route in var.ROUTES : route.route_namespace])) : toset([])
550+
resource "kubernetes_manifest" "upstream_policy" {
551+
for_each = toset(distinct([for _, route in var.ROUTES : route.route_namespace]))
541552

542553
manifest = {
543554
apiVersion = "configuration.konghq.com/v1beta1"
544555
kind = "KongUpstreamPolicy"
545556
metadata = {
546-
name = local.sticky_session_policy_name
557+
name = local.upstream_policy_name
547558
namespace = each.value
548559
annotations = {
549560
"kubernetes.io/ingress.class" = local.kong_ingress_class
550561
}
551562
}
552563
spec = {
553-
algorithm = "sticky-sessions"
564+
algorithm = "consistent-hashing"
554565
hashOn = {
555-
input = "none"
566+
input = "ip"
556567
}
557-
stickySessions = {
558-
cookie = var.STICKY_SESSION_COOKIE_NAME
559-
cookiePath = var.STICKY_SESSION_COOKIE_PATH
568+
healthchecks = {
569+
active = {
570+
type = "http"
571+
httpPath = "/status"
572+
timeout = 3
573+
concurrency = 10
574+
healthy = {
575+
interval = 10
576+
successes = 1
577+
httpStatuses = [200]
578+
}
579+
unhealthy = {
580+
interval = 5
581+
httpFailures = 3
582+
tcpFailures = 3
583+
timeouts = 3
584+
httpStatuses = [404, 429, 500, 501, 502, 503, 504, 505]
585+
}
586+
}
560587
}
561588
}
562589
}
@@ -807,6 +834,6 @@ resource "kubernetes_manifest" "rpc_route" {
807834
depends_on = [
808835
kubernetes_manifest.key_auth_plugin,
809836
kubernetes_manifest.prometheus_plugin,
810-
kubernetes_manifest.sticky_session_policy,
837+
kubernetes_manifest.upstream_policy,
811838
]
812839
}

‎spartan/terraform/modules/rpc-gateway/outputs.tf‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,9 @@ output "kong_namespace" {
3333
value = local.kong_namespace
3434
}
3535

36-
output "sticky_session_policy_name" {
37-
description = "KongUpstreamPolicy name for sticky sessions, or null when disabled."
38-
value = var.STICKY_SESSIONS_ENABLED ? local.sticky_session_policy_name : null
36+
output "upstream_policy_name" {
37+
description = "KongUpstreamPolicy name for RPC upstream balancing, or null when disabled."
38+
value = local.upstream_policy_name
3939
}
4040

4141
output "metrics_service_name" {

‎spartan/terraform/modules/rpc-gateway/variables.tf‎

Lines changed: 7 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ variable "INSTALL_KONG" {
1515
}
1616

1717
variable "KONG_NAMESPACE" {
18-
description = "Namespace for the Kong Helm release. Defaults to RELEASE_PREFIX-rpc-kong when empty."
18+
description = "Namespace for the Kong Helm release. Defaults to CONSUMER_NAMESPACE when empty."
1919
type = string
2020
default = ""
2121
}
@@ -62,6 +62,12 @@ variable "KONG_PROXY_SERVICE_LOAD_BALANCER_SOURCE_RANGES" {
6262
default = []
6363
}
6464

65+
variable "KONG_TRUSTED_IP_RANGES" {
66+
description = "Trusted proxy CIDR ranges for Kong real IP handling. The frontend load balancer IP is appended automatically when allocated by this module."
67+
type = list(string)
68+
default = []
69+
}
70+
6571
variable "KONG_EXTRA_HELM_VALUES" {
6672
description = "Additional YAML values passed to the Kong Helm chart."
6773
type = list(string)
@@ -170,30 +176,6 @@ variable "KONG_OTEL_METRICS_COLLECTOR_RESOURCES" {
170176
}
171177
}
172178

173-
variable "STICKY_SESSIONS_ENABLED" {
174-
description = "Whether to create KongUpstreamPolicy resources for RPC backend pods."
175-
type = bool
176-
default = false
177-
}
178-
179-
variable "STICKY_SESSION_POLICY_NAME" {
180-
description = "Optional KongUpstreamPolicy name. Defaults to RELEASE_PREFIX-rpc-sticky-sessions."
181-
type = string
182-
default = ""
183-
}
184-
185-
variable "STICKY_SESSION_COOKIE_NAME" {
186-
description = "Cookie name used by Kong sticky-sessions upstream balancing."
187-
type = string
188-
default = "aztec_rpc_backend"
189-
}
190-
191-
variable "STICKY_SESSION_COOKIE_PATH" {
192-
description = "Cookie path used by Kong sticky-sessions upstream balancing."
193-
type = string
194-
default = "/"
195-
}
196-
197179
variable "API_KEY_HEADER_NAME" {
198180
description = "Header checked by Kong's key-auth plugin."
199181
type = string

0 commit comments

Comments
 (0)