fix(sdk): ban rate-limited node for Envoy-advertised reset window (#3951)

Co-authored-by: Lukasz Klimek <842586+lklimek@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: 3 people

.gitignore

@@ -13,6 +13,7 @@
 
 # Env file
 .env
+.env.*.bak
 
 # NYC test runner
 .nyc_output

packages/dashmate/configs/defaults/getBaseConfigFactory.js

@@ -241,6 +241,9 @@ export default function getBaseConfigFactory() {
             blacklist: [],
             whitelist: [],
             enabled: true,
+            responseHeaders: {
+              enabled: true,
+            },
           },
           ssl: {
             enabled: false,

packages/dashmate/configs/getConfigFileMigrationsFactory.js

@@ -1520,6 +1520,25 @@ export default function getConfigFileMigrationsFactory(homeDir, defaultConfigs)
 
         return configFile;
       },
+      '4.0.0-rc.3': (configFile) => {
+        Object.entries(configFile.configs)
+          .forEach(([, options]) => {
+            // Add responseHeaders toggle to rate limiter (default true so existing
+            // deployments keep emitting RateLimit-* headers; rs-dapi-client depends
+            // on RateLimit-Reset to apply precise ban windows instead of the
+            // exponential health-ban ladder).
+            // Keyed at the next release (4.0.0-rc.3), not the already-released
+            // rc.2: the runner skips fromVersion===toVersion, so a key equal to
+            // an operator's current version never fires. Backfill runs once the
+            // package bumps to rc.3 (mirrors the 3.1.0 migration added at 3.1.0-dev.1).
+            if (options.platform?.gateway?.rateLimiter
+              && typeof options.platform.gateway.rateLimiter.responseHeaders === 'undefined') {
+              options.platform.gateway.rateLimiter.responseHeaders = base.get('platform.gateway.rateLimiter.responseHeaders');
+            }
+          });
+
+        return configFile;
+      },
     };
   }
 

packages/dashmate/docker-compose.rate_limiter.yml

@@ -42,6 +42,11 @@ services:
       - GRPC_MAX_CONNECTION_AGE=1h
       - GRPC_MAX_CONNECTION_AGE_GRACE=10m
       - GRPC_PORT=8081
+      # Emit RateLimit-Limit / RateLimit-Remaining / RateLimit-Reset response
+      # headers so rs-dapi-client can read the exact reset window and ban the
+      # node for that duration instead of the exponential health-ban ladder.
+      # Controlled by platform.gateway.rateLimiter.responseHeaders.enabled.
+      - LIMIT_RESPONSE_HEADERS_ENABLED=${PLATFORM_GATEWAY_RATE_LIMITER_RESPONSE_HEADERS_ENABLED:?err}
     expose:
       - 8081
     profiles:

packages/dashmate/docs/config/gateway.md

@@ -159,6 +159,7 @@ The rate limiter protects the Platform from excessive requests:
 | `platform.gateway.rateLimiter.unit` | Time unit for rate limiting | `minute` | `hour` |
 | `platform.gateway.rateLimiter.whitelist` | IPs exempt from rate limiting | `[]` | `["192.168.1.1"]` |
 | `platform.gateway.rateLimiter.blacklist` | IPs blocked from all requests | `[]` | `["10.0.0.1"]` |
+| `platform.gateway.rateLimiter.responseHeaders.enabled` | Emit `RateLimit-Limit`, `RateLimit-Remaining`, and `RateLimit-Reset` response headers. `rs-dapi-client` reads the Reset header to apply a precise ban window instead of the exponential health-ban ladder. Disable only for privacy reasons. | `true` | `false` |
 
 Available time units:
 - `second`: Per-second rate limiting

packages/dashmate/src/config/configJsonSchema.js

@@ -690,8 +690,22 @@ export default {
                 enabled: {
                   type: 'boolean',
                 },
+                responseHeaders: {
+                  type: 'object',
+                  description: 'Control emission of RateLimit-* response headers (RateLimit-Limit, '
+                    + 'RateLimit-Remaining, RateLimit-Reset). When enabled, rs-dapi-client reads '
+                    + 'the Reset header to ban the node for the server-advertised window instead '
+                    + 'of the exponential health-ban ladder. Disable only for privacy reasons.',
+                  properties: {
+                    enabled: {
+                      type: 'boolean',
+                    },
+                  },
+                  additionalProperties: false,
+                  required: ['enabled'],
+                },
               },
-              required: ['docker', 'enabled', 'unit', 'requestsPerUnit', 'blacklist', 'whitelist', 'metrics'],
+              required: ['docker', 'enabled', 'unit', 'requestsPerUnit', 'blacklist', 'whitelist', 'metrics', 'responseHeaders'],
               additionalProperties: false,
             },
             ssl: {

packages/dashmate/templates/platform/gateway/envoy.yaml.dot

@@ -33,6 +33,52 @@
         max_concurrent_streams: {{= it.platform.gateway.listeners.dapiAndDrive.http2.maxConcurrentStreams }}
       # Stream idle timeout
       stream_idle_timeout: 15s
+{{? it.platform.gateway.rateLimiter.enabled }}
+      # Make the grpc-web over-limit reply readable as a rate-limit BAN by browser
+      # (grpc-web/wasm) clients. The native gRPC over-limit local reply is HTTP 200
+      # + grpc-status:8 (trailers-only) because Envoy detects content-type
+      # application/grpc; the grpc-web request (application/grpc-web+proto) is NOT
+      # gRPC-detected, so `rate_limited_as_resource_exhausted` never tags it and the
+      # browser sees a bare HTTP 429 with no grpc-status -> tonic-web maps it to
+      # Unavailable, not ResourceExhausted, so rate_limit_ban_duration() returns None.
+      #
+      # This mapper rewrites ONLY the grpc-web over-limit local reply (status 429 AND
+      # request carried `x-grpc-web`, which the native and JSON-RPC paths never send)
+      # to status 200 + grpc-status:8. Because the reply is headers-only (empty body,
+      # end_stream on headers) the grpc_web encoder passes it through untouched
+      # (encodeHeaders returns Continue when end_stream=true), so grpc-status:8 and
+      # the existing ratelimit-reset header stay CO-LOCATED in the HTTP response
+      # headers. tonic-web's create_response then builds the Status (and its metadata,
+      # incl. ratelimit-reset) from those headers via Status::from_header_map without
+      # reading the body, giving browser clients the same ResourceExhausted +
+      # ratelimit-reset node-backoff the native client gets. CORS expose_headers
+      # already lists both grpc-status and ratelimit-reset (see virtual host below).
+      local_reply_config:
+        mappers:
+          - filter:
+              and_filter:
+                filters:
+                  - status_code_filter:
+                      comparison:
+                        op: EQ
+                        value:
+                          default_value: 429
+                          runtime_key: ratelimit_grpc_web_local_reply_status
+                  - header_filter:
+                      header:
+                        name: x-grpc-web
+                        present_match: true
+            status_code: 200
+            headers_to_add:
+              - header:
+                  key: grpc-status
+                  value: "8"
+                append_action: OVERWRITE_IF_EXISTS_OR_ADD
+              - header:
+                  key: grpc-message
+                  value: "rate limited"
+                append_action: OVERWRITE_IF_EXISTS_OR_ADD
+{{?}}
 {{? it.platform.gateway.log.accessLogs }}
       access_log:
       {{~ it.platform.gateway.log.accessLogs :log }}
@@ -78,6 +124,24 @@
 {{?}}
       http_filters:
         # TODO: Introduce when stable https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/adaptive_concurrency_filter.html
+        # Filter order matters: cors and grpc_web MUST precede ratelimit so the
+        # over-limit RESOURCE_EXHAUSTED *local reply* gets CORS headers + grpc-web
+        # content-type framing on encode, letting browser (grpc-web) clients read
+        # RateLimit-Reset for the same node-backoff the native client does. A local
+        # reply only traverses encoder filters positioned ABOVE its generating
+        # filter (Envoy #11776), so ratelimit must come last (before router).
+        # Live-verified on the pinned Envoy build (PR #3951): the grpc-web over-limit
+        # reply is trailers-only (HTTP 200 + grpc-status:8 + ratelimit-reset as HTTP
+        # headers, empty body — grpc_web leaves it untouched, no body trailer frame).
+        # The grpc-status:8 itself is synthesized by the local_reply_config mapper
+        # above (Envoy's gRPC detection ignores application/grpc-web), giving browser
+        # clients the same ResourceExhausted + ratelimit-reset node-backoff as native.
+        - name: envoy.filters.http.cors
+          typed_config:
+            "@type": type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors
+        - name: envoy.filters.http.grpc_web
+          typed_config:
+            "@type": type.googleapis.com/envoy.extensions.filters.http.grpc_web.v3.GrpcWeb
         {{? it.platform.gateway.rateLimiter.enabled}}
         - name: envoy.filters.http.ratelimit
           typed_config:
@@ -96,12 +160,6 @@
                 timeout: 0.5s
               transport_api_version: V3
         {{?}}
-        - name: envoy.filters.http.grpc_web
-          typed_config:
-            "@type": type.googleapis.com/envoy.extensions.filters.http.grpc_web.v3.GrpcWeb
-        - name: envoy.filters.http.cors
-          typed_config:
-            "@type": type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors
         - name: envoy.filters.http.router
           typed_config:
             "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
@@ -198,7 +256,7 @@
                 allow_methods: GET, PUT, DELETE, POST, OPTIONS
                 allow_headers: keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,custom-header-1,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout
                 max_age: "1728000"
-                expose_headers: custom-header-1,grpc-status,grpc-message,code,drive-error-data-bin,dash-serialized-consensus-error-bin,stack-bin
+                expose_headers: custom-header-1,grpc-status,grpc-message,code,drive-error-data-bin,dash-serialized-consensus-error-bin,stack-bin,ratelimit-reset,ratelimit-limit,ratelimit-remaining
 
 static_resources:
   listeners: