feat(extension-server): branded data-plane error pages

config/extension-server/deployment.yaml

@@ -123,6 +123,13 @@ spec:
             - name: tls-ca
               mountPath: /tls-ca
               readOnly: true
+            # Branded data-plane error pages. Backed by an OPTIONAL ConfigMap so
+            # the pod starts even when the ConfigMap is absent — the extension
+            # server then serves the page compiled into the operator image.
+            # The operator config's gateway.errorPage.bodyPath points here.
+            - name: error-pages
+              mountPath: /etc/datum/error-pages
+              readOnly: true
       terminationGracePeriodSeconds: 20
       volumes:
         # issuer-name and dns-names are placeholders — an overlay must patch them.
@@ -144,3 +151,14 @@ spec:
             items:
               - key: ca.crt
                 path: ca.crt
+        # Optional branded error-page content. The ConfigMap is created in the
+        # infrastructure (GitOps) repo and carries the key error-5xx.html. With
+        # optional: true the pod starts cleanly when it is absent, in which case
+        # the extension server serves the embedded default page.
+        - name: error-pages
+          configMap:
+            name: envoy-error-pages
+            optional: true
+            items:
+              - key: error-5xx.html
+                path: error-5xx.html

config/manager/config.yaml

@@ -13,3 +13,18 @@ gateway:
   #
   # Default: false. Set to true in environments where dns-operator is deployed.
   enableDNSIntegration: false
+  # errorPage configures the branded data-plane error page served by the
+  # extension server for edge-generated 5xx responses on the downstream /
+  # Connector data plane (e.g. an offline Connector tunnel). When enabled, the
+  # extension server attaches an Envoy local_reply_config to every customer-
+  # facing HCM so visitors see a branded "temporarily unavailable" page instead
+  # of a raw body like "no healthy upstream".
+  #
+  # bodyPath points at the optional ConfigMap mount (volume "error-pages" in
+  # config/extension-server/deployment.yaml). When the file is absent or empty,
+  # the page compiled into the operator image is used as the fallback — content
+  # problems never block xDS or fail startup.
+  errorPage:
+    enabled: true
+    bodyPath: /etc/datum/error-pages/error-5xx.html
+    minStatusCode: 500

internal/config/config.go

@@ -633,6 +633,11 @@ type GatewayConfig struct {
 	// Coraza specifies configuration for the Coraza WAF.
 	Coraza CorazaConfig `json:"coraza,omitempty"`
 
+	// ErrorPage specifies configuration for the branded data-plane error page
+	// served for edge-generated 5xx responses on the downstream / Connector
+	// data plane.
+	ErrorPage ErrorPageConfig `json:"errorPage,omitempty"`
+
 	// ValidPortNumbers is a list of port numbers that are permitted on gateway
 	// listeners.
 	//
@@ -790,6 +795,45 @@ type CorazaConfig struct {
 
 // +k8s:deepcopy-gen=true
 
+// ErrorPageConfig configures the branded data-plane error page. When enabled,
+// the extension server attaches an Envoy local_reply_config to every
+// customer-facing HCM so edge-generated 5xx responses render a branded HTML
+// page instead of a raw body like "no healthy upstream".
+//
+// The page content is sourced from BodyPath (a mounted ConfigMap) when present
+// and readable, otherwise from the page compiled into the operator image. A
+// missing or unreadable override never fails startup and never blocks xDS — it
+// falls back to the embedded default.
+type ErrorPageConfig struct {
+	// Enabled toggles branded error-page injection. Defaults to false; the
+	// extension server only attaches local_reply_config when this is true.
+	Enabled bool `json:"enabled,omitempty"`
+
+	// BodyPath is an optional path to a file (typically a mounted ConfigMap key)
+	// containing the branded HTML. When empty, unreadable, or empty-on-disk, the
+	// embedded default page is used instead.
+	BodyPath string `json:"bodyPath,omitempty"`
+
+	// MinStatusCode is the inclusive lower bound for response status codes that
+	// receive the branded body. The original status code is always preserved.
+	//
+	// +default=500
+	MinStatusCode uint32 `json:"minStatusCode,omitempty"`
+
+	// RuntimeKey is the Envoy runtime key gating the branded reply, allowing it
+	// to be disabled at runtime without a redeploy.
+	//
+	// +default="local_reply_5xx"
+	RuntimeKey string `json:"runtimeKey,omitempty"`
+
+	// ContentType is the Content-Type set on the branded response body.
+	//
+	// +default="text/html; charset=UTF-8"
+	ContentType string `json:"contentType,omitempty"`
+}
+
+// +k8s:deepcopy-gen=true
+
 type ExtensionAPIValidationOptions struct {
 	// BackendTrafficPolicies specifies validation options for BackendTrafficPolicy resources.
 	BackendTrafficPolicies BackendTrafficPolicyValidationOptions `json:"backendTrafficPolicies"`

internal/extensionserver/assets/assets.go

@@ -0,0 +1,16 @@
+// Package assets holds static content compiled into the extension server.
+//
+// The default branded 5xx error page is embedded here so the extension server
+// always has a valid page to serve, even when no override ConfigMap is mounted.
+// The canonical content is expected to come from a mounted ConfigMap (managed in
+// the infrastructure GitOps repo); this embed is the always-valid fallback.
+package assets
+
+import _ "embed"
+
+// DefaultError5xxHTML is the compiled-in branded HTML served for edge-generated
+// 5xx responses when no override page is mounted. Keep it small and
+// self-contained (inline CSS, no external assets) so it renders standalone.
+//
+//go:embed error-5xx-default.html
+var DefaultError5xxHTML string

internal/extensionserver/assets/error-5xx-default.html

@@ -0,0 +1,85 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1" />
+  <meta name="robots" content="noindex" />
+  <title>Temporarily unavailable</title>
+  <style>
+    :root { color-scheme: dark; }
+    * { box-sizing: border-box; }
+    html, body { height: 100%; margin: 0; }
+    body {
+      display: flex;
+      align-items: center;
+      justify-content: center;
+      min-height: 100vh;
+      padding: 1.5rem;
+      font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
+      background: radial-gradient(120% 120% at 50% 0%, #14151a 0%, #0a0b0d 60%, #060708 100%);
+      color: #e8eaed;
+    }
+    .card {
+      width: 100%;
+      max-width: 30rem;
+      text-align: center;
+      background: rgba(255, 255, 255, 0.03);
+      border: 1px solid rgba(255, 255, 255, 0.08);
+      border-radius: 16px;
+      padding: 2.5rem 2rem;
+      box-shadow: 0 1px 0 rgba(255, 255, 255, 0.04) inset, 0 20px 60px rgba(0, 0, 0, 0.45);
+    }
+    .badge {
+      display: inline-block;
+      font-size: 0.72rem;
+      letter-spacing: 0.18em;
+      text-transform: uppercase;
+      color: #9aa3b2;
+      margin-bottom: 1.25rem;
+    }
+    .dot {
+      display: inline-block;
+      width: 0.5rem;
+      height: 0.5rem;
+      border-radius: 50%;
+      background: #f0a23c;
+      margin-right: 0.5rem;
+      vertical-align: middle;
+      box-shadow: 0 0 12px rgba(240, 162, 60, 0.7);
+    }
+    h1 {
+      font-size: 1.5rem;
+      line-height: 1.3;
+      margin: 0 0 0.75rem;
+      font-weight: 600;
+    }
+    p {
+      margin: 0 auto;
+      max-width: 24rem;
+      color: #aab2c0;
+      font-size: 0.95rem;
+      line-height: 1.55;
+    }
+    .meta {
+      margin-top: 1.75rem;
+      font-size: 0.78rem;
+      color: #6b7280;
+    }
+    .code {
+      font-variant-numeric: tabular-nums;
+      color: #8b93a1;
+    }
+  </style>
+</head>
+<body>
+  <main class="card">
+    <span class="badge"><span class="dot"></span>Datum Cloud</span>
+    <h1>This site is temporarily unavailable</h1>
+    <p>
+      The site you're trying to reach can't be served right now. This is usually
+      temporary &mdash; please try again in a few moments.
+    </p>
+    <div class="meta">Error <span class="code">%RESPONSE_CODE%</span> &middot; served by the Datum edge</div>
+  </main>
+</body>
+</html>

internal/extensionserver/cmd/run.go

@@ -36,6 +36,7 @@ import (
 	networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha"
 	networkingv1alpha1 "go.datum.net/network-services-operator/api/v1alpha1"
 	"go.datum.net/network-services-operator/internal/config"
+	extassets "go.datum.net/network-services-operator/internal/extensionserver/assets"
 	extcache "go.datum.net/network-services-operator/internal/extensionserver/cache"
 	extmetrics "go.datum.net/network-services-operator/internal/extensionserver/metrics"
 	"go.datum.net/network-services-operator/internal/extensionserver/mutate"
@@ -110,6 +111,44 @@ func NewCommand() *cobra.Command {
 	return cmd
 }
 
+// buildLocalReplyConfig assembles the branded error-page configuration for the
+// extension server from the operator's ErrorPageConfig.
+//
+// Content sourcing is fail-safe by design: it starts from the page compiled
+// into the operator image and, only if ErrorPage.BodyPath points at a readable
+// non-empty file, swaps in that override. A missing, unreadable, or empty
+// override logs a warning and keeps the embedded default — it NEVER fails
+// startup. This guarantees the downstream EG extension hook (failOpen:false)
+// always has a valid body to inject and is never stalled by a content problem.
+//
+// Injection itself is gated on ErrorPage.Enabled via the Disabled flag.
+func buildLocalReplyConfig(cfg config.ErrorPageConfig, log *slog.Logger) mutate.LocalReplyConfig {
+	body := extassets.DefaultError5xxHTML
+
+	if cfg.BodyPath != "" {
+		data, err := os.ReadFile(cfg.BodyPath)
+		switch {
+		case err != nil:
+			log.Warn("read error-page body override; using embedded default",
+				"path", cfg.BodyPath, "err", err)
+		case len(data) == 0:
+			log.Warn("error-page body override is empty; using embedded default",
+				"path", cfg.BodyPath)
+		default:
+			body = string(data)
+			log.Info("using error-page body override", "path", cfg.BodyPath, "bytes", len(data))
+		}
+	}
+
+	return mutate.LocalReplyConfig{
+		Disabled:      !cfg.Enabled,
+		MinStatusCode: cfg.MinStatusCode,
+		RuntimeKey:    cfg.RuntimeKey,
+		BodyHTML:      body,
+		ContentType:   cfg.ContentType,
+	}
+}
+
 // run starts the extension server with the given options. It owns cache startup,
 // gRPC server lifecycle, and signal handling.
 func run(o options) {
@@ -205,6 +244,7 @@ func run(o options) {
 		},
 		ConnectorInternalListener: serverConfig.Gateway.ConnectorTunnelListenerName(),
 		CorazaRouteBaseDirectives: coraza.RouteBaseDirectives,
+		LocalReply:                buildLocalReplyConfig(serverConfig.Gateway.ErrorPage, log),
 	}
 
 	// --- gRPC panic recovery interceptor ---

internal/extensionserver/metrics/metrics.go

@@ -101,6 +101,17 @@ var (
 		},
 	)
 
+	// LocalReplyMutationsTotal counts total branded error-page (local_reply_config)
+	// injections across all hook invocations. Each increment = one HCM that had a
+	// branded local_reply_config attached. Use rate()/sum() to derive per-build
+	// averages.
+	LocalReplyMutationsTotal = promauto.NewCounter(
+		prometheus.CounterOpts{
+			Name: "nso_extension_local_reply_mutations_total",
+			Help: "Total branded error-page (local_reply_config) injections across all hook invocations.",
+		},
+	)
+
 	// WAFRouteMutationsTotal counts total per-route TrafficProtectionPolicy (WAF)
 	// config applications across all hook invocations.
 	WAFRouteMutationsTotal = promauto.NewCounter(