Detect OAuth MCP servers that omit the 401 challenge (#819)

* Detect OAuth MCP servers that omit the 401 challenge

Some MCP servers return a bare 401 with no usable WWW-Authenticate
challenge, so the wire-shape probe could not distinguish them from an
unrelated OAuth-protected API and the user landed on the manual
credentials prompt instead of an OAuth sign-in. When a 401 would
otherwise be rejected, fall back to fetching the RFC 9728 path-scoped
protected-resource metadata; a document whose resource matches the
endpoint is enough to classify it as MCP and start the OAuth flow.

* Add changeset for MCP OAuth probe fallback

Author: RhysSullivan

.changeset/mcp-oauth-probe-prm-fallback.md

@@ -0,0 +1,14 @@
+---
+"@executor-js/plugin-mcp": patch
+"executor": patch
+---
+
+Fix: remote MCP servers that return a bare `401` without a usable
+`WWW-Authenticate` challenge (e.g. Datadog's `mcp.datadoghq.com`) now trigger
+the OAuth sign-in flow instead of falling back to the manual-credentials
+prompt. When a `401` would otherwise be rejected, the endpoint probe also
+checks for RFC 9728 protected-resource metadata at the path-scoped
+well-known URL; a document there whose `resource` matches the endpoint is
+enough to classify it as an OAuth-protected MCP server. The check stays
+narrow — path-scoped metadata plus a `resource`-to-endpoint match — so a
+generic OAuth-protected API is not misclassified as MCP.

packages/plugins/mcp/src/sdk/probe-shape.test.ts

@@ -182,6 +182,52 @@ describe("probeMcpEndpointShape", () => {
     ),
   );
 
+  // Datadog shape: bare `401 {"errors":["Unauthorized"]}` with no
+  // WWW-Authenticate header at all, but the server still publishes
+  // RFC 9728 protected-resource metadata at the path-scoped well-known
+  // URL. The metadata probe is what lets us classify it as MCP+auth.
+  it.effect("classifies 401 w/o WWW-Authenticate but with RFC 9728 metadata as MCP+auth", () =>
+    withServer(
+      (request) => {
+        if (request.url.includes("/.well-known/oauth-protected-resource")) {
+          const host = request.headers.host ?? "127.0.0.1";
+          return HttpServerResponse.jsonUnsafe({
+            resource: `http://${host}/probe`,
+            authorization_servers: [`http://${host}`],
+          });
+        }
+        return HttpServerResponse.jsonUnsafe({ errors: ["Unauthorized"] }, { status: 401 });
+      },
+      (endpoint) =>
+        Effect.gen(function* () {
+          const result = yield* probeMcpEndpointShape(endpoint);
+          expect(result).toEqual({ kind: "mcp", requiresAuth: true });
+        }),
+    ),
+  );
+
+  // The metadata probe must not rescue a 401 when the published
+  // `resource` describes some unrelated resource on the same host.
+  it.effect("rejects 401 when RFC 9728 metadata resource doesn't match endpoint", () =>
+    withServer(
+      (request) => {
+        if (request.url.includes("/.well-known/oauth-protected-resource")) {
+          const host = request.headers.host ?? "127.0.0.1";
+          return HttpServerResponse.jsonUnsafe({
+            resource: `http://${host}/some-other-api`,
+            authorization_servers: [`http://${host}`],
+          });
+        }
+        return HttpServerResponse.jsonUnsafe({ errors: ["Unauthorized"] }, { status: 401 });
+      },
+      (endpoint) =>
+        Effect.gen(function* () {
+          const result = yield* probeMcpEndpointShape(endpoint);
+          expect(result).toMatchObject({ kind: "not-mcp", category: "auth-required" });
+        }),
+    ),
+  );
+
   it.effect("rejects 401 + Bearer with empty body as auth-required", () =>
     withServer(
       () =>

packages/plugins/mcp/src/sdk/probe-shape.ts

@@ -108,6 +108,78 @@ const isOAuthErrorBody = (body: string): boolean => {
   return typeof obj.error === "string";
 };
 
+/** RFC 9728 protected-resource-metadata document. We only need the two
+ *  fields that prove the document genuinely describes an OAuth-protected
+ *  resource: `resource` (the resource identifier) and a non-empty
+ *  `authorization_servers` list. */
+const ProtectedResourceMetadata = Schema.Struct({
+  resource: Schema.String,
+  authorization_servers: Schema.Array(Schema.String),
+});
+const decodeProtectedResourceMetadata = Schema.decodeUnknownOption(
+  Schema.fromJsonString(ProtectedResourceMetadata),
+);
+
+/** RFC 9728 §3.1 path-scoped well-known URL: insert
+ *  `/.well-known/oauth-protected-resource` before the resource's path
+ *  component. `https://host/api/mcp` → `https://host/.well-known/oauth-
+ *  protected-resource/api/mcp`. This is exactly the URL the MCP
+ *  authorization spec tells clients to construct. */
+const protectedResourceMetadataUrl = (endpoint: URL): string => {
+  const path = endpoint.pathname === "/" ? "" : endpoint.pathname;
+  return `${endpoint.origin}/.well-known/oauth-protected-resource${path}`;
+};
+
+/** The RFC 9728 `resource` value must actually describe this endpoint
+ *  before we trust the document — an exact URL match, or a same-origin
+ *  parent whose path is a prefix of the endpoint's. Guards against a
+ *  shared host serving protected-resource metadata for some unrelated
+ *  resource. */
+const resourceMatchesEndpoint = (resource: string, endpoint: URL): boolean => {
+  if (!URL.canParse(resource)) return false;
+  const parsed = new URL(resource);
+  if (parsed.origin !== endpoint.origin) return false;
+  const resourcePath = parsed.pathname.replace(/\/+$/, "");
+  const endpointPath = endpoint.pathname.replace(/\/+$/, "");
+  return endpointPath === resourcePath || endpointPath.startsWith(`${resourcePath}/`);
+};
+
+/** Workaround for MCP servers that omit (or under-specify) the
+ *  `WWW-Authenticate` challenge on their 401 — e.g. Datadog's
+ *  `mcp.datadoghq.com` returns a bare `401 {"errors":["Unauthorized"]}`
+ *  with no header at all, so the wire-shape gate above can't tell it
+ *  apart from an unrelated OAuth-protected API and the user lands on the
+ *  manual-credentials prompt instead of an OAuth sign-in.
+ *
+ *  The MCP authorization spec still requires such servers to publish
+ *  RFC 9728 metadata at the path-scoped well-known URL. A document there
+ *  whose `resource` matches this endpoint is a deliberate, MCP-spec-
+ *  specific signal a generic OAuth API would not emit — strong enough to
+ *  classify the endpoint as MCP so the OAuth flow can start. */
+const probeProtectedResourceMetadata = (
+  client: HttpClient.HttpClient,
+  endpoint: URL,
+  timeoutMs: number,
+): Effect.Effect<boolean> =>
+  Effect.gen(function* () {
+    const response = yield* client
+      .execute(
+        HttpClientRequest.get(protectedResourceMetadataUrl(endpoint)).pipe(
+          HttpClientRequest.setHeader("accept", "application/json"),
+        ),
+      )
+      .pipe(Effect.timeout(Duration.millis(timeoutMs)));
+    if (response.status < 200 || response.status >= 300) return false;
+    const body = yield* response.text.pipe(
+      Effect.timeout(Duration.millis(timeoutMs)),
+      Effect.catch(() => Effect.succeed("")),
+    );
+    const metadata = decodeProtectedResourceMetadata(body);
+    if (Option.isNone(metadata)) return false;
+    if (metadata.value.authorization_servers.length === 0) return false;
+    return resourceMatchesEndpoint(metadata.value.resource, endpoint);
+  }).pipe(Effect.catch(() => Effect.succeed(false)));
+
 const ErrorMessageShape = Schema.Struct({ message: Schema.String });
 const decodeErrorMessageShape = Schema.decodeUnknownOption(ErrorMessageShape);
 
@@ -203,6 +275,13 @@ export const probeMcpEndpointShape = (
           if (response.status === 401) {
             const wwwAuth = readHeader(response.headers, "www-authenticate");
             if (!wwwAuth || !/^\s*bearer\b/i.test(wwwAuth)) {
+              // Spec-non-compliant 401 (no `Bearer` challenge). Before
+              // giving up, check whether the server still publishes
+              // RFC 9728 protected-resource metadata for this path —
+              // some real MCP servers (Datadog) do exactly this.
+              if (yield* probeProtectedResourceMetadata(client, url, timeoutMs)) {
+                return { kind: "mcp", requiresAuth: true } as const;
+              }
               return {
                 kind: "not-mcp",
                 category: "auth-required",
@@ -245,6 +324,11 @@ export const probeMcpEndpointShape = (
             // arrays or other shapes that fail both checks.
             const body = yield* readBody(response);
             if (!isJsonRpcEnvelope(body) && !isOAuthErrorBody(body)) {
+              // Bearer challenge with no usable accept signal. Same
+              // RFC 9728 fallback as the no-`Bearer` case above.
+              if (yield* probeProtectedResourceMetadata(client, url, timeoutMs)) {
+                return { kind: "mcp", requiresAuth: true } as const;
+              }
               return {
                 kind: "not-mcp",
                 category: "auth-required",