adobe
diff --git a/‎src/controllers/redirects.js‎
Lines changed: 82 additions & 6 deletions b/‎src/controllers/redirects.js‎
Lines changed: 82 additions & 6 deletions
diff --git a/‎src/support/aso-overlay-key-handler.js‎
Lines changed: 41 additions & 4 deletions b/‎src/support/aso-overlay-key-handler.js‎
Lines changed: 41 additions & 4 deletions
diff --git a/‎src/support/aso-overlay-metrics.js‎
Lines changed: 83 additions & 0 deletions b/‎src/support/aso-overlay-metrics.js‎
Lines changed: 83 additions & 0 deletions
@@ -20,6 +20,10 @@ import { Entitlement as EntitlementModel } from '@adobe/spacecat-shared-data-acc
 import { isNonEmptyObject } from '@adobe/spacecat-shared-utils';
 
 import { getHeader } from '../support/http-headers.js';
+import { emitMetric, resolveEnvironment } from '../support/metrics-emf.js';
+import {
+  ASO_OVERLAY_NAMESPACE, OUTCOME, S3_RESULT, INM_INVALID_REASON,
+} from '../support/aso-overlay-metrics.js';
 
 // Cloud Manager service identifier, e.g. cm-p154709-e1629980. Digit runs are
 // capped to a realistic length to avoid arbitrarily long lookups; the capture
@@ -134,13 +138,44 @@ function RedirectsController(ctx) {
    */
   async function getRedirects(context) {
     const { service } = context.params;
+    // Capture start time up front so every terminal branch reports duration.
+    const startedAt = Date.now();
+    const emitOpts = {
+      environment: resolveEnvironment(env),
+      namespace: ASO_OVERLAY_NAMESPACE,
+    };
+
+    // Single call-site helper: emits RequestTotal + RequestDurationMs together
+    // so on-call always has both a rate and a latency view for the same outcome.
+    // Note: no `Tier` dimension — the origin route is `/config/:service/...`
+    // (Fastly strips the public `/config/<tier>/...` prefix before proxying), so
+    // any tier extraction from the origin path would be either wrong or
+    // per-tenant, blowing up CloudWatch cardinality. Environment (auto-added by
+    // emitMetric from AWS_ENV) already carries dev-vs-prod.
+    const emitFinal = (outcome) => {
+      emitMetric(
+        { name: 'AsoOverlayRequestTotal', dimensions: { Outcome: outcome } },
+        emitOpts,
+      );
+      emitMetric(
+        {
+          name: 'AsoOverlayRequestDurationMs',
+          value: Date.now() - startedAt,
+          unit: 'Milliseconds',
+          dimensions: { Outcome: outcome },
+        },
+        emitOpts,
+      );
+    };
 
     const match = SERVICE_RE.exec(service);
     if (!match) {
+      emitFinal(OUTCOME.BAD_REQUEST_400);
       return badRequest('Invalid service identifier', NO_STORE_HEADERS);
     }
     if (!bucketName) {
       log.error('[aso-overlay] S3_ASO_OVERLAYS_BUCKET is not configured');
+      emitFinal(OUTCOME.BUCKET_NOT_CONFIGURED);
       return internalServerError('Overlay endpoint not configured', NO_STORE_HEADERS);
     }
 
@@ -155,6 +190,7 @@ function RedirectsController(ctx) {
     );
     if (!site) {
       log.info('[aso-overlay] no site resolves for service', { service });
+      emitFinal(OUTCOME.AUTHZ_NO_SITE);
       return notFound('No redirect overlay found', NO_STORE_HEADERS);
     }
 
@@ -166,20 +202,36 @@ function RedirectsController(ctx) {
     );
     if (!entitlement) {
       log.info('[aso-overlay] site org not ASO-entitled', { siteId: site.getId() });
+      emitFinal(OUTCOME.AUTHZ_NO_ENTITLEMENT);
       return notFound('No redirect overlay found', NO_STORE_HEADERS);
     }
     const enrollments = await SiteEnrollment.allBySiteId(site.getId());
     const enrolled = enrollments.some((se) => se.getEntitlementId() === entitlement.getId());
     if (!enrolled) {
       log.info('[aso-overlay] site not enrolled for ASO', { siteId: site.getId() });
+      emitFinal(OUTCOME.AUTHZ_NOT_ENROLLED);
       return notFound('No redirect overlay found', NO_STORE_HEADERS);
     }
 
     // Read the overlay with the Lambda's own execution role (no SigV4 from caller).
     const key = `config/${service}/redirects.txt`;
+    const s3StartedAt = Date.now();
+    // Emits AsoOverlayS3ReadDurationMs with the S3-scoped result. Kept separate
+    // from `emitFinal`'s request-level Outcome so dashboards don't ambiguously
+    // slice S3 latency by request-level codes (e.g. "S3 success on a 304").
+    const emitS3Duration = (s3Result) => emitMetric(
+      {
+        name: 'AsoOverlayS3ReadDurationMs',
+        value: Date.now() - s3StartedAt,
+        unit: 'Milliseconds',
+        dimensions: { S3Result: s3Result },
+      },
+      emitOpts,
+    );
     try {
       const command = new GetObjectCommand({ Bucket: bucketName, Key: key });
       const response = await s3Client.send(command);
+      emitS3Duration(S3_RESULT.SUCCESS);
 
       // S3 returns the object's ETag already quoted (RFC 7232 opaque-tag form),
       // e.g. `"d41d8cd98f00b204e9800998ecf8427e"`. Passthrough gives the client
@@ -191,12 +243,26 @@ function RedirectsController(ctx) {
       // a plain 200 rather than breaking.
       const etag = response.ETag;
       const ifNoneMatch = getHeader(context, 'if-none-match');
-      if (ifNoneMatchMatches(ifNoneMatch, etag)) {
-        // Deliberately no per-request 304 log — this is the *expected* path at
-        // steady state and would dominate log volume across the dispatcher fleet.
-        // If 304-rate visibility becomes necessary, emit a metric rather than a
-        // log line.
-        //
+      const inmMatched = ifNoneMatchMatches(ifNoneMatch, etag);
+      // Track shell-stripped / malformed If-None-Match separately: a spike here
+      // means a specific sidecar rollout parses ETags wrong (RFC 7232 §2.3
+      // requires quoted opaque-tag; unquoted → rejected → falls through to 200).
+      if (ifNoneMatch !== null && !inmMatched) {
+        const trimmed = ifNoneMatch.trim();
+        // Every token failed the opaque-tag check — likely unquoted. Non-matching
+        // but well-formed validators (client's stale copy) are normal cache
+        // misses and don't warrant a metric.
+        if (trimmed !== '*'
+            && !trimmed.split(',').some((tok) => normalizeValidator(tok) !== null)) {
+          emitMetric(
+            { name: 'AsoOverlayIfNoneMatchInvalid', dimensions: { Reason: INM_INVALID_REASON.UNQUOTED } },
+            emitOpts,
+          );
+        }
+      }
+      if (inmMatched) {
+        emitMetric({ name: 'AsoOverlayConditionalGet304' }, emitOpts);
+        emitFinal(OUTCOME.NOT_MODIFIED_304);
         // 304 MUST NOT include a message body (RFC 7230 §3.3.3); MUST carry any
         // Cache-Control / ETag we would have sent on a 200 (RFC 7232 §4.1).
         // Content-type is set explicitly because createResponse would otherwise
@@ -215,6 +281,10 @@ function RedirectsController(ctx) {
       }
 
       const body = await response.Body.transformToString();
+      if (etag) {
+        emitMetric({ name: 'AsoOverlayEtagPresent' }, emitOpts);
+      }
+      emitFinal(OUTCOME.OK_200);
       return createResponse(body, 200, {
         'content-type': 'text/plain; charset=utf-8',
         // Cache-friendly for Fastly request-collapsing; edge TTL also set in VCL.
@@ -231,6 +301,8 @@ function RedirectsController(ctx) {
     } catch (err) {
       const code = err.$metadata?.httpStatusCode;
       if (err.name === 'NoSuchKey' || code === 404) {
+        emitS3Duration(S3_RESULT.NO_SUCH_KEY);
+        emitFinal(OUTCOME.S3_NO_SUCH_KEY);
         return notFound('No redirect overlay found', NO_STORE_HEADERS);
       }
       // The reader role intentionally lacks s3:ListBucket (least privilege, no key
@@ -245,9 +317,13 @@ function RedirectsController(ctx) {
           + '— missing object or missing s3:GetObject grant',
           err,
         );
+        emitS3Duration(S3_RESULT.ACCESS_DENIED);
+        emitFinal(OUTCOME.S3_ACCESS_DENIED);
         return notFound('No redirect overlay found', NO_STORE_HEADERS);
       }
       log.error(`[aso-overlay] failed to read ${key} from ${bucketName}`, err);
+      emitS3Duration(S3_RESULT.UNEXPECTED);
+      emitFinal(OUTCOME.S3_UNEXPECTED);
       return internalServerError('Failed to retrieve redirect overlay', NO_STORE_HEADERS);
     }
   }
 
@@ -13,6 +13,12 @@
 import crypto from 'crypto';
 import AbstractHandler from '@adobe/spacecat-shared-http-utils/src/auth/handlers/abstract.js';
 import AuthInfo from '@adobe/spacecat-shared-http-utils/src/auth/auth-info.js';
+import { emitMetric, resolveEnvironment } from './metrics-emf.js';
+import {
+  ASO_OVERLAY_NAMESPACE,
+  AUTH_FAIL_REASON,
+  AUTH_KEY_SLOT,
+} from './aso-overlay-metrics.js';
 
 // Path-scoped to the ASO redirect-overlay read route. The service shape mirrors
 // the controller's own validation (RedirectsController SERVICE_RE), so only
@@ -61,28 +67,59 @@ class AsoOverlayKeyHandler extends AbstractHandler {
     const suffix = context.pathInfo?.suffix || '';
 
     // Path-scoped: only the overlay read route. Return null otherwise so the
-    // remaining handlers run.
+    // remaining handlers run. No metric emitted for non-overlay paths — this
+    // handler runs on every request and we'd flood the namespace.
     if (method !== 'GET' || !OVERLAY_ROUTE.test(suffix)) {
       return null;
     }
 
+    // From here on, we're on the overlay route — every outcome is worth a metric.
+    const emitOpts = {
+      environment: resolveEnvironment(context.env),
+      namespace: ASO_OVERLAY_NAMESPACE,
+    };
+
     const providedKey = request.headers.get('x-aso-api-key');
     // No key supplied: not an overlay-key request. Fall through (→ 401 if no
-    // other handler authenticates).
+    // other handler authenticates). Track separately from an invalid key.
     if (!providedKey) {
+      emitMetric(
+        { name: 'AsoOverlayAuthFailed', dimensions: { Reason: AUTH_FAIL_REASON.MISSING } },
+        emitOpts,
+      );
       return null;
     }
 
     const expectedKey = context.env?.ASO_OVERLAY_API_KEY;
     if (!expectedKey) {
       this.log('ASO_OVERLAY_API_KEY is not configured', 'error');
+      emitMetric(
+        { name: 'AsoOverlayAuthFailed', dimensions: { Reason: AUTH_FAIL_REASON.CONFIG_MISSING } },
+        emitOpts,
+      );
       return null;
     }
 
     const previousKey = context.env?.ASO_OVERLAY_API_KEY_PREVIOUS;
-    if (!safeEqual(providedKey, expectedKey)
-        && !(previousKey && safeEqual(providedKey, previousKey))) {
+    // Track which slot matched — `previous` non-zero signals rotation in-flight,
+    // and sustained non-zero across > 24h signals the sidecar fleet hasn't picked
+    // up the new key.
+    if (safeEqual(providedKey, expectedKey)) {
+      emitMetric(
+        { name: 'AsoOverlayAuthKeyUsed', dimensions: { Slot: AUTH_KEY_SLOT.CURRENT } },
+        emitOpts,
+      );
+    } else if (previousKey && safeEqual(providedKey, previousKey)) {
+      emitMetric(
+        { name: 'AsoOverlayAuthKeyUsed', dimensions: { Slot: AUTH_KEY_SLOT.PREVIOUS } },
+        emitOpts,
+      );
+    } else {
       this.log('invalid X-ASO-API-Key', 'warn');
+      emitMetric(
+        { name: 'AsoOverlayAuthFailed', dimensions: { Reason: AUTH_FAIL_REASON.INVALID } },
+        emitOpts,
+      );
       return null;
     }
 
 
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2026 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+/**
+ * Canonical catalog of CloudWatch EMF metrics emitted by the ASO dispatcher
+ * overlay read path (`GET /config/:service/redirects.txt`) and its auth handler.
+ * Drift guard: `RedirectsController` and `AsoOverlayKeyHandler` must reference
+ * exactly these names, and the Grafana dashboard queries against them via the
+ * `Mysticat/AsoOverlay` CloudWatch namespace.
+ *
+ * Cardinality note: no per-tenant dimensions here. CloudWatch charges per unique
+ * dimension combination; 10k+ CM services would blow the budget. Tenant-level
+ * views come from Fastly access logs in Splunk instead.
+ */
+
+export const ASO_OVERLAY_NAMESPACE = 'Mysticat/AsoOverlay';
+
+export const ASO_OVERLAY_METRICS = Object.freeze([
+  'AsoOverlayRequestTotal', // Count · dims: Environment, Outcome
+  'AsoOverlayRequestDurationMs', // Milliseconds · dims: Environment, Outcome
+  'AsoOverlayEtagPresent', // Count · dims: Environment (subset of 200)
+  'AsoOverlayConditionalGet304', // Count · dims: Environment (INM matched)
+  'AsoOverlayIfNoneMatchInvalid', // Count · dims: Environment, Reason
+  'AsoOverlayS3ReadDurationMs', // Milliseconds · dims: Environment, S3Result
+  'AsoOverlayAuthKeyUsed', // Count · dims: Environment, Slot (rotation-in-flight signal)
+  'AsoOverlayAuthFailed', // Count · dims: Environment, Reason
+]);
+
+// Outcome enum for AsoOverlayRequestTotal / AsoOverlayRequestDurationMs.
+// Distinguishes 404 sub-reasons so on-call can tell authz-fail from
+// S3-object-missing (indistinguishable to clients by design).
+export const OUTCOME = Object.freeze({
+  OK_200: '200',
+  NOT_MODIFIED_304: '304',
+  BAD_REQUEST_400: '400',
+  AUTHZ_NO_SITE: '404-authz-nosite',
+  AUTHZ_NO_ENTITLEMENT: '404-authz-noent',
+  AUTHZ_NOT_ENROLLED: '404-authz-noenroll',
+  S3_NO_SUCH_KEY: '404-s3-nosuchkey',
+  S3_ACCESS_DENIED: '404-s3-accessdenied',
+  BUCKET_NOT_CONFIGURED: '500-config',
+  S3_UNEXPECTED: '500-s3',
+});
+
+// S3Result enum for AsoOverlayS3ReadDurationMs. Kept separate from OUTCOME so
+// dashboards querying "filter by Outcome" don't ambiguously match S3-scoped
+// metrics — the two share the axis name only, not the semantic domain.
+export const S3_RESULT = Object.freeze({
+  SUCCESS: 'success',
+  NO_SUCH_KEY: 'nosuchkey',
+  ACCESS_DENIED: 'accessdenied',
+  UNEXPECTED: 'unexpected',
+});
+
+// Reason enum for AsoOverlayAuthFailed.
+export const AUTH_FAIL_REASON = Object.freeze({
+  MISSING: 'missing', // Overlay route, X-ASO-API-Key header absent
+  INVALID: 'invalid', // Header present but doesn't match either key slot
+  CONFIG_MISSING: 'config-missing', // Server-side ASO_OVERLAY_API_KEY env var unset (deploy misconfig)
+});
+
+// Reason enum for AsoOverlayIfNoneMatchInvalid. Whitespace-only values are
+// normalized to null by `getHeader` and reach the controller as "absent" — so
+// we don't track them separately.
+export const INM_INVALID_REASON = Object.freeze({
+  UNQUOTED: 'unquoted', // Client sent bare token without RFC 7232 §2.3 quotes
+});
+
+// Slot enum for AsoOverlayAuthKeyUsed. `previous` non-zero signals rotation
+// in-flight; sustained non-zero across > 24h signals rotation didn't complete.
+export const AUTH_KEY_SLOT = Object.freeze({
+  CURRENT: 'current',
+  PREVIOUS: 'previous',
+});