[codex] Enrich relay authorization diagnostics (#2977)

juliusmarminge · codex · web-flow · commit 3ea6adf17d55 · 2026-06-05T18:01:34.000-07:00
Co-authored-by: codex &lt;codex@users.noreply.github.com&gt;
diff --git a/infra/relay/src/environments/EnvironmentConnector.test.ts b/infra/relay/src/environments/EnvironmentConnector.test.ts
@@ -22,6 +22,7 @@ import * as Redacted from "effect/Redacted";
 import * as Result from "effect/Result";
 import * as Schema from "effect/Schema";
 import * as TestClock from "effect/testing/TestClock";
+import * as Tracer from "effect/Tracer";
 import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http";
 
 import * as EnvironmentLinks from "./EnvironmentLinks.ts";
@@ -50,6 +51,9 @@ const decodeHealthRequestBody = Schema.decodeUnknownSync(
 const decodeMintRequestBody = Schema.decodeUnknownSync(
   Schema.fromJsonString(RelayCloudMintCredentialRequest),
 );
+const isEnvironmentConnectNotAuthorized = Schema.is(
+  EnvironmentConnector.EnvironmentConnectNotAuthorized,
+);
 
 function requestBodyText(request: HttpClientRequest.HttpClientRequest): string {
   return request.body._tag === "Uint8Array" ? new TextDecoder().decode(request.body.body) : "{}";
@@ -280,7 +284,13 @@ describe("EnvironmentConnector", () => {
 
       expect(Result.isFailure(result)).toBe(true);
       if (Result.isFailure(result)) {
-        expect(result.failure).toBeInstanceOf(EnvironmentConnector.EnvironmentConnectNotAuthorized);
+        expect(isEnvironmentConnectNotAuthorized(result.failure)).toBe(true);
+        if (isEnvironmentConnectNotAuthorized(result.failure)) {
+          expect(result.failure).toMatchObject({
+            operation: "status",
+            reason: "endpoint_provider_not_managed",
+          });
+        }
       }
       expect(requestCount).toBe(0);
     }).pipe(
@@ -300,6 +310,14 @@ describe("EnvironmentConnector", () => {
 
   it.effect("rejects stale managed endpoints before sending a mint request", () => {
     let requestCount = 0;
+    const spans: Array<Tracer.NativeSpan> = [];
+    const tracer = Tracer.make({
+      span: (options) => {
+        const span = new Tracer.NativeSpan(options);
+        spans.push(span);
+        return span;
+      },
+    });
     const execute = () =>
       Effect.sync(() => {
         requestCount += 1;
@@ -318,8 +336,27 @@ describe("EnvironmentConnector", () => {
 
       expect(Result.isFailure(result)).toBe(true);
       if (Result.isFailure(result)) {
-        expect(result.failure).toBeInstanceOf(EnvironmentConnector.EnvironmentConnectNotAuthorized);
+        expect(isEnvironmentConnectNotAuthorized(result.failure)).toBe(true);
+        if (isEnvironmentConnectNotAuthorized(result.failure)) {
+          expect(result.failure).toMatchObject({
+            operation: "connect",
+            reason: "managed_endpoint_mismatch",
+          });
+        }
       }
+      const resolutionSpan = spans.find(
+        (span) => span.name === "relay.environment_connector.resolve_managed_endpoint",
+      );
+      expect(Object.fromEntries(resolutionSpan?.attributes ?? [])).toMatchObject({
+        "relay.authorization.allocation_hostname": "env.example.test",
+        "relay.authorization.allocation_has_ready_at": true,
+        "relay.authorization.allocation_has_tunnel_id": true,
+        "relay.authorization.allocation_has_dns_record_id": true,
+        "relay.authorization.linked_http_base_url": "https://attacker.example.test/",
+        "relay.authorization.linked_ws_base_url": "wss://attacker.example.test/ws",
+        "relay.authorization.resolved_http_base_url": "https://env.example.test/",
+        "relay.authorization.resolved_ws_base_url": "wss://env.example.test/ws",
+      });
       expect(requestCount).toBe(0);
     }).pipe(
       Effect.provide(
@@ -333,6 +370,7 @@ describe("EnvironmentConnector", () => {
           }),
         }),
       ),
+      Effect.provideService(Tracer.Tracer, tracer),
     );
   });
 
@@ -355,7 +393,13 @@ describe("EnvironmentConnector", () => {
 
       expect(Result.isFailure(result)).toBe(true);
       if (Result.isFailure(result)) {
-        expect(result.failure).toBeInstanceOf(EnvironmentConnector.EnvironmentConnectNotAuthorized);
+        expect(isEnvironmentConnectNotAuthorized(result.failure)).toBe(true);
+        if (isEnvironmentConnectNotAuthorized(result.failure)) {
+          expect(result.failure).toMatchObject({
+            operation: "status",
+            reason: "managed_endpoint_allocation_not_ready",
+          });
+        }
       }
       expect(requestCount).toBe(0);
     }).pipe(
diff --git a/infra/relay/src/environments/EnvironmentConnector.ts b/infra/relay/src/environments/EnvironmentConnector.ts
@@ -40,15 +40,54 @@ import { FetchHttpClient, HttpClient } from "effect/unstable/http";
 import * as EnvironmentLinks from "./EnvironmentLinks.ts";
 import * as ManagedEndpointAllocations from "./ManagedEndpointAllocations.ts";
 import * as RelayConfiguration from "../Config.ts";
+import { isManagedEndpointHostname } from "../deploymentConfig.ts";
+
+export const EnvironmentConnectNotAuthorizedReason = Schema.Literals([
+  "client_proof_key_thumbprint_missing",
+  "environment_link_not_found",
+  "endpoint_provider_not_managed",
+  "managed_endpoint_allocation_not_found",
+  "managed_endpoint_base_domain_not_configured",
+  "managed_endpoint_allocation_not_ready",
+  "managed_endpoint_hostname_invalid",
+  "managed_endpoint_mismatch",
+]);
+export type EnvironmentConnectNotAuthorizedReason =
+  typeof EnvironmentConnectNotAuthorizedReason.Type;
+
+function environmentConnectNotAuthorizedReasonMessage(
+  reason: EnvironmentConnectNotAuthorizedReason,
+): string {
+  switch (reason) {
+    case "client_proof_key_thumbprint_missing":
+      return "the client proof key thumbprint is missing";
+    case "environment_link_not_found":
+      return "no active environment link was found";
+    case "endpoint_provider_not_managed":
+      return "the linked endpoint is not relay-managed";
+    case "managed_endpoint_allocation_not_found":
+      return "no managed endpoint allocation was found";
+    case "managed_endpoint_base_domain_not_configured":
+      return "the managed endpoint base domain is not configured";
+    case "managed_endpoint_allocation_not_ready":
+      return "the managed endpoint allocation is incomplete";
+    case "managed_endpoint_hostname_invalid":
+      return "the managed endpoint hostname is invalid";
+    case "managed_endpoint_mismatch":
+      return "the linked endpoint does not match its managed allocation";
+  }
+}
 
 export class EnvironmentConnectNotAuthorized extends Schema.TaggedErrorClass<EnvironmentConnectNotAuthorized>()(
   "EnvironmentConnectNotAuthorized",
   {
     environmentId: Schema.String,
+    operation: Schema.Literals(["connect", "status"]),
+    reason: EnvironmentConnectNotAuthorizedReason,
   },
 ) {
   override get message(): string {
-    return `Environment '${this.environmentId}' is not authorized to connect`;
+    return `Environment '${this.environmentId}' is not authorized for ${this.operation}: ${environmentConnectNotAuthorizedReasonMessage(this.reason)}`;
   }
 }
 
@@ -257,30 +296,91 @@ const make = Effect.gen(function* () {
   const resolveManagedEndpoint = Effect.fn("relay.environment_connector.resolve_managed_endpoint")(
     function* (input: {
       readonly userId: string;
+      readonly operation: "connect" | "status";
       readonly link: EnvironmentLinks.RelayLinkedEnvironmentRecord;
     }) {
       if (input.link.endpoint.providerKind !== "cloudflare_tunnel") {
+        yield* Effect.annotateCurrentSpan({
+          "relay.authorization.endpoint_provider_kind": input.link.endpoint.providerKind,
+        });
         return yield* new EnvironmentConnectNotAuthorized({
           environmentId: input.link.environmentId,
+          operation: input.operation,
+          reason: "endpoint_provider_not_managed",
         });
       }
       const allocation = yield* allocations.get({
         userId: input.userId,
         environmentId: input.link.environmentId,
       });
-      const endpoint = allocation
-        ? ManagedEndpointAllocations.resolveReadyManagedEndpoint({
-            allocation,
-            baseDomain: settings.managedEndpointBaseDomain,
-          })
-        : null;
+      if (!allocation) {
+        return yield* new EnvironmentConnectNotAuthorized({
+          environmentId: input.link.environmentId,
+          operation: input.operation,
+          reason: "managed_endpoint_allocation_not_found",
+        });
+      }
+      const allocationAttributes = {
+        "relay.authorization.allocation_hostname": allocation.hostname,
+        "relay.authorization.allocation_has_ready_at": allocation.readyAt !== null,
+        "relay.authorization.allocation_has_tunnel_id": allocation.tunnelId !== null,
+        "relay.authorization.allocation_has_dns_record_id": allocation.dnsRecordId !== null,
+      } as const;
+      if (!settings.managedEndpointBaseDomain) {
+        yield* Effect.annotateCurrentSpan(allocationAttributes);
+        return yield* new EnvironmentConnectNotAuthorized({
+          environmentId: input.link.environmentId,
+          operation: input.operation,
+          reason: "managed_endpoint_base_domain_not_configured",
+        });
+      }
+      if (
+        allocation.readyAt === null ||
+        allocation.tunnelId === null ||
+        allocation.dnsRecordId === null
+      ) {
+        yield* Effect.annotateCurrentSpan(allocationAttributes);
+        return yield* new EnvironmentConnectNotAuthorized({
+          environmentId: input.link.environmentId,
+          operation: input.operation,
+          reason: "managed_endpoint_allocation_not_ready",
+        });
+      }
+      if (!isManagedEndpointHostname(allocation.hostname, settings.managedEndpointBaseDomain)) {
+        yield* Effect.annotateCurrentSpan({
+          ...allocationAttributes,
+          "relay.authorization.managed_endpoint_base_domain": settings.managedEndpointBaseDomain,
+        });
+        return yield* new EnvironmentConnectNotAuthorized({
+          environmentId: input.link.environmentId,
+          operation: input.operation,
+          reason: "managed_endpoint_hostname_invalid",
+        });
+      }
+      const endpoint = ManagedEndpointAllocations.resolveReadyManagedEndpoint({
+        allocation,
+        baseDomain: settings.managedEndpointBaseDomain,
+      });
       if (
         endpoint === null ||
         endpoint.httpBaseUrl !== input.link.endpoint.httpBaseUrl ||
         endpoint.wsBaseUrl !== input.link.endpoint.wsBaseUrl
       ) {
+        yield* Effect.annotateCurrentSpan({
+          ...allocationAttributes,
+          "relay.authorization.linked_http_base_url": input.link.endpoint.httpBaseUrl,
+          "relay.authorization.linked_ws_base_url": input.link.endpoint.wsBaseUrl,
+          ...(endpoint
+            ? {
+                "relay.authorization.resolved_http_base_url": endpoint.httpBaseUrl,
+                "relay.authorization.resolved_ws_base_url": endpoint.wsBaseUrl,
+              }
+            : {}),
+        });
         return yield* new EnvironmentConnectNotAuthorized({
           environmentId: input.link.environmentId,
+          operation: input.operation,
+          reason: "managed_endpoint_mismatch",
         });
       }
       return endpoint;
@@ -295,9 +395,17 @@ const make = Effect.gen(function* () {
       });
       const link = yield* links.getForUser(input);
       if (!link) {
-        return yield* new EnvironmentConnectNotAuthorized({ environmentId: input.environmentId });
+        return yield* new EnvironmentConnectNotAuthorized({
+          environmentId: input.environmentId,
+          operation: "status",
+          reason: "environment_link_not_found",
+        });
       }
-      const endpoint = yield* resolveManagedEndpoint({ userId: input.userId, link });
+      const endpoint = yield* resolveManagedEndpoint({
+        userId: input.userId,
+        operation: "status",
+        link,
+      });
       const now = yield* DateTime.now;
       const expiresAt = DateTime.add(now, { minutes: 2 });
       const nonce = yield* crypto.randomUUIDv4.pipe(
@@ -404,13 +512,25 @@ const make = Effect.gen(function* () {
         ...(input.deviceId ? { "relay.mobile.device_id": input.deviceId } : {}),
       });
       if (input.clientProofKeyThumbprint.trim().length === 0) {
-        return yield* new EnvironmentConnectNotAuthorized({ environmentId: input.environmentId });
+        return yield* new EnvironmentConnectNotAuthorized({
+          environmentId: input.environmentId,
+          operation: "connect",
+          reason: "client_proof_key_thumbprint_missing",
+        });
       }
       const link = yield* links.getForUser(input);
       if (!link) {
-        return yield* new EnvironmentConnectNotAuthorized({ environmentId: input.environmentId });
+        return yield* new EnvironmentConnectNotAuthorized({
+          environmentId: input.environmentId,
+          operation: "connect",
+          reason: "environment_link_not_found",
+        });
       }
-      const endpoint = yield* resolveManagedEndpoint({ userId: input.userId, link });
+      const endpoint = yield* resolveManagedEndpoint({
+        userId: input.userId,
+        operation: "connect",
+        link,
+      });
       const now = yield* DateTime.now;
       const expiresAt = DateTime.add(now, { minutes: 2 });
       const nonce = yield* crypto.randomUUIDv4.pipe(
diff --git a/infra/relay/src/http/Api.ts b/infra/relay/src/http/Api.ts
@@ -830,7 +830,7 @@ const currentTraceId = Effect.currentParentSpan.pipe(
   Effect.orElseSucceed(() => "unavailable"),
 );
 
-const COMMON_AUTH_INVALID_REASONS = [
+const RelayCommonPersistenceError = Schema.Union([
   Devices.DeviceRegistrationPersistenceError,
   Devices.DeviceUnregistrationPersistenceError,
   Devices.DeviceListPersistenceError,
@@ -850,9 +850,9 @@ const COMMON_AUTH_INVALID_REASONS = [
   AgentActivityRows.AgentActivityRowListPersistenceError,
   LiveActivities.LiveActivityDeliveryMarkPersistenceError,
   DeliveryAttempts.DeliveryAttemptRecordPersistenceError,
-] as const;
-type RelayCommonPersistenceError = InstanceType<(typeof COMMON_AUTH_INVALID_REASONS)[number]>;
-const isRelayCommonPersistenceError = Schema.is(Schema.Union(COMMON_AUTH_INVALID_REASONS));
+]);
+type RelayCommonPersistenceError = typeof RelayCommonPersistenceError.Type;
+const isRelayCommonPersistenceError = Schema.is(RelayCommonPersistenceError);
 
 type MapRelayCommonApiError<E> =
   | Exclude<E, HttpApiError.Unauthorized | RelayCommonPersistenceError>
diff --git a/infra/relay/src/observability.test.ts b/infra/relay/src/observability.test.ts
@@ -9,7 +9,7 @@ import * as HttpServerRequest from "effect/unstable/http/HttpServerRequest";
 import * as HttpServerResponse from "effect/unstable/http/HttpServerResponse";
 import type { OtlpTracer } from "effect/unstable/observability";
 
-import { EnvironmentMintRequestFailed } from "./environments/EnvironmentConnector.ts";
+import { EnvironmentConnectNotAuthorized } from "./environments/EnvironmentConnector.ts";
 import { makeRelayTraceLayer } from "./observability.ts";
 
 interface ExportedRequest {
@@ -43,10 +43,10 @@ it.effect("exports schema error fields as span attributes", () =>
     );
 
     yield* Effect.fail(
-      new EnvironmentMintRequestFailed({
+      new EnvironmentConnectNotAuthorized({
         environmentId: "environment-1",
         operation: "connect",
-        cause: new Error("upstream unavailable"),
+        reason: "managed_endpoint_allocation_not_ready",
       }),
     ).pipe(
       Effect.withSpan("relay.test.schema_error"),
@@ -76,11 +76,10 @@ it.effect("exports schema error fields as span attributes", () =>
     expect(request.authorization).toBe("Bearer test-token");
     expect(request.dataset).toBe("relay-test-traces");
     expect(attributes).toMatchObject({
-      "error.type": "EnvironmentMintRequestFailed",
+      "error.type": "EnvironmentConnectNotAuthorized",
       "error.environmentId": "environment-1",
       "error.operation": "connect",
-      "error.cause.name": "Error",
-      "error.cause.message": "upstream unavailable",
+      "error.reason": "managed_endpoint_allocation_not_ready",
     });
   }).pipe(Effect.provide(NodeHttpServer.layerTest), Effect.scoped),
 );
