feat: connection health checks (liveness) with OpenAPI backing

A connection can declare ONE authenticated operation that answers "is this
credential still alive?". The probe runs the operation, maps the HTTP status to
a health state (2xx healthy, 401/403 expired, other degraded, unset unknown),
and reports it. Built for short-lived OAuth tokens (Google's 7-day dev-token
revocation): a credential that authenticated yesterday and now 401s reads as
expired.

Core owns the shared vocabulary (HealthStatus, HealthCheckSpec, HealthCheckResult,
HealthCheckCandidate, classifyHttpStatus, candidate ranking); plugins own which
operation runs, stored in their opaque integration config and picked the same way
auth methods are. Dispatch adds integrations.healthCheck.{get,candidates,set} and
connections.{checkHealth,validate}.

OpenAPI backing: list ranked candidates (non-destructive GET first), persist the
chosen spec (merging over the raw config so provider supersets keep their keys),
and run the probe against a resolved credential.

React surfaces:
- a per-connection status dot + "Check now",
- a self-hiding operation editor (a searchable freeform combobox over the whole
  spec),
- and "Check the key works" in the Add Connection modal: probe the pasted key
  before saving. When no health check is configured yet, the user picks a
  read-only operation inline; a healthy probe saves it as the integration's
  health check.

Covered by e2e: a saved connection flips healthy -> expired when its key stops
working; an unconfigured integration reports unknown; the operation picker
filters a hundreds-long candidate list on both the edit sheet and the add screen;
and Add Connection checks the key against an inline-picked operation and persists
it.

Author: RhysSullivan

e2e/scenarios/health-checks-ui.test.ts

@@ -0,0 +1,315 @@
+// Cross-target: connection health checks, the feature that answers "has this
+// credential expired?" (the Google 7-day dev-token case) in one declared probe.
+// Entirely through the typed client:
+//
+//   1. register an OpenAPI integration whose `GET /me` is auth-gated,
+//   2. CONFIGURE a health check by picking that operation (the same flow the
+//      user drives in the editor: list candidates, ranked GET-first, then set),
+//   3. CHECK a SAVED connection and watch its status flip healthy -> expired
+//      when the stored key stops working,
+//   4. confirm a connection with no configured check reports `unknown`.
+//
+// The upstream API is a real node:http server started inside the scenario on
+// 127.0.0.1 that gates `GET /me` on a bearer token: a generic "bring your own
+// OpenAPI" integration, so the generic openapi health-check path is exercised
+// rather than any one provider's quirks.
+import { randomBytes } from "node:crypto";
+import { createServer } from "node:http";
+
+import { expect } from "@effect/vitest";
+import { Effect } from "effect";
+import type { HttpApiClient } from "effect/unstable/httpapi";
+import { composePluginApi } from "@executor-js/api/server";
+import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api";
+import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared";
+
+import { scenario } from "../src/scenario";
+import { Api, Target } from "../src/services";
+
+const api = composePluginApi([openApiHttpPlugin()] as const);
+type Client = HttpApiClient.ForApi<typeof api>;
+
+const TEMPLATE = AuthTemplateSlug.make("apiKey");
+const ACCOUNT_EMAIL = "alice@example.com";
+
+const newSlug = (prefix: string) =>
+  IntegrationSlug.make(`${prefix}-${randomBytes(4).toString("hex")}`);
+
+/** OpenAPI 3 spec with an auth-gated GET (`/me`, the obvious health check) plus a
+ *  destructive POST so the candidate ranking has something to sort the GET ahead
+ *  of. */
+const identitySpec = (baseUrl: string): string =>
+  JSON.stringify({
+    openapi: "3.0.3",
+    info: { title: "Identity API", version: "1.0.0" },
+    servers: [{ url: baseUrl }],
+    paths: {
+      "/me": {
+        get: {
+          operationId: "getMe",
+          summary: "The current account",
+          responses: {
+            "200": {
+              description: "The authenticated account",
+              content: {
+                "application/json": {
+                  schema: {
+                    type: "object",
+                    properties: { email: { type: "string" }, login: { type: "string" } },
+                  },
+                },
+              },
+            },
+          },
+        },
+      },
+      "/messages": {
+        post: {
+          operationId: "sendMessage",
+          summary: "Send a message",
+          responses: { "201": { description: "created" } },
+        },
+      },
+    },
+  });
+
+/** A real node:http identity API on 127.0.0.1. `GET /me` returns the account
+ *  JSON only when the bearer token matches `validToken`; any other token is a
+ *  401 (the "the dev token got revoked" case the health check classifies as
+ *  expired). Closed by the scope's finalizer. */
+const serveIdentityApi = (validToken: string) =>
+  Effect.acquireRelease(
+    Effect.callback<{ readonly url: string; readonly close: () => void }>((resume) => {
+      const server = createServer((request, response) => {
+        const authorized = request.headers["authorization"] === `Bearer ${validToken}`;
+        if (request.method === "GET" && (request.url ?? "").startsWith("/me")) {
+          if (!authorized) {
+            response.writeHead(401, { "content-type": "application/json" });
+            response.end(JSON.stringify({ error: "invalid_token" }));
+            return;
+          }
+          response.writeHead(200, { "content-type": "application/json" });
+          response.end(JSON.stringify({ email: ACCOUNT_EMAIL, login: "alice" }));
+          return;
+        }
+        response.writeHead(404, { "content-type": "application/json" });
+        response.end(JSON.stringify({ error: "not_found" }));
+      });
+      server.listen(0, "127.0.0.1", () => {
+        const address = server.address();
+        const port = typeof address === "object" && address ? address.port : 0;
+        resume(
+          Effect.succeed({
+            url: `http://127.0.0.1:${port}`,
+            close: () => {
+              server.close();
+              server.closeAllConnections();
+            },
+          }),
+        );
+      });
+    }),
+    (server) => Effect.sync(server.close),
+  );
+
+/** Register the identity integration against `baseUrl` with a bearer-token auth
+ *  method (single `token` input → connection `value`). Returns the slug. */
+const registerIdentityIntegration = (client: Client, slug: IntegrationSlug, baseUrl: string) =>
+  client.openapi.addSpec({
+    payload: {
+      spec: { kind: "blob", value: identitySpec(baseUrl) },
+      slug,
+      baseUrl,
+      authenticationTemplate: [
+        {
+          slug: "apiKey",
+          type: "apiKey",
+          headers: { authorization: ["Bearer ", { type: "variable", name: "token" }] },
+        },
+      ],
+    },
+  });
+
+/** The stored operation name for the GET probe (openapi prefixes it by tag, e.g.
+ *  `me.getMe`), discovered the same way the editor does: from the ranked
+ *  candidate list. */
+const getMeOperation = (client: Client, slug: IntegrationSlug) =>
+  Effect.gen(function* () {
+    const candidates = yield* client.integrations.healthCheckCandidates({ params: { slug } });
+    const getMe = candidates.find((candidate) => candidate.method === "get");
+    if (!getMe) return yield* Effect.die("identity spec exposed no GET candidate");
+    return getMe.operation;
+  });
+
+scenario(
+  "Health checks · the editor ranks the non-destructive GET ahead of the destructive POST",
+  {},
+  Effect.scoped(
+    Effect.gen(function* () {
+      const target = yield* Target;
+      const { client: makeClient } = yield* Api;
+      const identity = yield* target.newIdentity();
+      const client = yield* makeClient(api, identity);
+      const goodToken = `gk_${randomBytes(8).toString("hex")}`;
+      const server = yield* serveIdentityApi(goodToken);
+      const slug = newSlug("hc-rank");
+
+      yield* Effect.ensuring(
+        Effect.gen(function* () {
+          yield* registerIdentityIntegration(client, slug, server.url);
+
+          // The editor offers the integration's operations, ranked so the
+          // non-destructive GET endpoint floats to the top.
+          const candidates = yield* client.integrations.healthCheckCandidates({
+            params: { slug },
+          });
+          const get = candidates.find((candidate) => candidate.method === "get");
+          const post = candidates.find((candidate) => candidate.method === "post");
+          if (!get || !post) {
+            return yield* Effect.die("identity spec should expose a GET and a POST candidate");
+          }
+          // Operations are stored tag-prefixed (e.g. `me.getMe`); match the suffix.
+          expect(get.operation.split(".").at(-1), "the GET is offered").toBe("getMe");
+          expect(post.operation.split(".").at(-1), "the destructive POST is offered").toBe(
+            "sendMessage",
+          );
+          expect(
+            candidates[0]?.operation,
+            "the non-destructive GET ranks ahead of the destructive POST",
+          ).toBe(get.operation);
+          expect(get.destructive, "the GET probe is non-destructive").toBe(false);
+          expect(post.destructive, "the POST is flagged destructive").toBe(true);
+
+          // Pick it: just the operation (a pure liveness probe).
+          yield* client.integrations.healthCheckSet({
+            params: { slug },
+            payload: { spec: { operation: get.operation } },
+          });
+          const stored = yield* client.integrations.healthCheckGet({ params: { slug } });
+          expect(stored, "the chosen health check round-trips").toEqual({
+            operation: get.operation,
+          });
+        }),
+        client.openapi.removeSpec({ params: { slug } }).pipe(Effect.ignore),
+      );
+    }),
+  ),
+);
+
+scenario(
+  "Health checks · a saved connection reports healthy, then expired when its key stops working",
+  {},
+  Effect.scoped(
+    Effect.gen(function* () {
+      const target = yield* Target;
+      const { client: makeClient } = yield* Api;
+      const identity = yield* target.newIdentity();
+      const client = yield* makeClient(api, identity);
+      const goodToken = `gk_${randomBytes(8).toString("hex")}`;
+      const server = yield* serveIdentityApi(goodToken);
+      const slug = newSlug("hc-saved");
+      const name = ConnectionName.make("main");
+
+      yield* Effect.ensuring(
+        Effect.gen(function* () {
+          yield* registerIdentityIntegration(client, slug, server.url);
+          const operation = yield* getMeOperation(client, slug);
+          yield* client.integrations.healthCheckSet({
+            params: { slug },
+            payload: { spec: { operation } },
+          });
+
+          // A connection holding the live key checks out healthy.
+          yield* client.connections.create({
+            payload: {
+              owner: "org",
+              name,
+              integration: slug,
+              template: TEMPLATE,
+              value: goodToken,
+            },
+          });
+          const healthy = yield* client.connections.checkHealth({
+            params: { owner: "org", integration: slug, name },
+          });
+          expect(healthy.status, "the saved connection's live key is healthy").toBe("healthy");
+          expect(healthy.httpStatus, "the saved probe saw the 200").toBe(200);
+
+          // Re-creating the same (owner, integration, name) replaces the stored
+          // key in place: now the connection holds a key the server rejects.
+          yield* client.connections.create({
+            payload: {
+              owner: "org",
+              name,
+              integration: slug,
+              template: TEMPLATE,
+              value: "rotated-away",
+            },
+          });
+          const expired = yield* client.connections.checkHealth({
+            params: { owner: "org", integration: slug, name },
+          });
+          expect(expired.status, "the same connection now reads as expired").toBe("expired");
+          expect(expired.httpStatus, "the saved probe saw the 401").toBe(401);
+        }),
+        Effect.gen(function* () {
+          yield* client.connections
+            .remove({ params: { owner: "org", integration: slug, name } })
+            .pipe(Effect.ignore);
+          yield* client.openapi.removeSpec({ params: { slug } }).pipe(Effect.ignore);
+        }),
+      );
+    }),
+  ),
+);
+
+scenario(
+  "Health checks · a connection with no configured check reports unknown, not a failure",
+  {},
+  Effect.scoped(
+    Effect.gen(function* () {
+      const target = yield* Target;
+      const { client: makeClient } = yield* Api;
+      const identity = yield* target.newIdentity();
+      const client = yield* makeClient(api, identity);
+      const goodToken = `gk_${randomBytes(8).toString("hex")}`;
+      const server = yield* serveIdentityApi(goodToken);
+      const slug = newSlug("hc-unknown");
+      const name = ConnectionName.make("main");
+
+      yield* Effect.ensuring(
+        Effect.gen(function* () {
+          // No healthCheckSet: the integration declares no probe.
+          yield* registerIdentityIntegration(client, slug, server.url);
+          expect(
+            yield* client.integrations.healthCheckGet({ params: { slug } }),
+            "an integration with no configured check reports none",
+          ).toBeNull();
+
+          yield* client.connections.create({
+            payload: {
+              owner: "org",
+              name,
+              integration: slug,
+              template: TEMPLATE,
+              value: goodToken,
+            },
+          });
+          const result = yield* client.connections.checkHealth({
+            params: { owner: "org", integration: slug, name },
+          });
+          expect(result.status, "with no check configured the status is unknown").toBe("unknown");
+          expect(result.detail ?? "", "the result explains why it is unknown").toContain(
+            "No health check configured",
+          );
+        }),
+        Effect.gen(function* () {
+          yield* client.connections
+            .remove({ params: { owner: "org", integration: slug, name } })
+            .pipe(Effect.ignore);
+          yield* client.openapi.removeSpec({ params: { slug } }).pipe(Effect.ignore);
+        }),
+      );
+    }),
+  ),
+);

e2e/scenarios/health-checks.test.ts

@@ -16,6 +16,8 @@ import {
   ConnectionName,
   ConnectionNotFoundError,
   CredentialProviderNotRegisteredError,
+  HealthCheckResult,
+  HealthCheckSpec,
   IntegrationNotFoundError,
   IntegrationSlug,
   InternalError,
@@ -108,6 +110,31 @@ const CreateConnectionPayload = Schema.Struct({
   ),
 );
 
+// Validate an in-flight credential WITHOUT saving it (the key-first connect
+// flow). Same origin shape as create, plus an optional `spec` override so the
+// editor can preview a candidate against a live key. No `name` — the point is
+// to derive one from the identity the probe returns.
+const ValidateConnectionPayload = Schema.Struct({
+  owner: Owner,
+  integration: IntegrationSlug,
+  template: AuthTemplateSlug,
+  spec: Schema.optional(HealthCheckSpec),
+  value: Schema.optional(Schema.String),
+  values: Schema.optional(Schema.Record(Schema.String, Schema.String)),
+  from: Schema.optional(
+    Schema.Struct({
+      provider: ProviderKey,
+      id: ProviderItemId,
+    }),
+  ),
+}).check(
+  Schema.makeFilter((payload) =>
+    [payload.value, payload.values, payload.from].filter(Predicate.isNotUndefined).length === 1
+      ? undefined
+      : "Expected exactly one credential origin",
+  ),
+);
+
 // ---------------------------------------------------------------------------
 // Query — optional list filters.
 // ---------------------------------------------------------------------------
@@ -186,4 +213,25 @@ export const ConnectionsApi = HttpApiGroup.make("connections")
       success: Schema.Array(ToolResponse),
       error: [InternalError, ConnectionNotFound, IntegrationNotFound],
     }),
+  )
+  // Run the integration's declared health check against a SAVED connection: is
+  // this credential still alive (Google's 7-day dev-token revocation), and whose
+  // account is it? Returns a classified status + optional identity, never an
+  // error for an auth wall (that surfaces as `status: "expired"`).
+  .add(
+    HttpApiEndpoint.post("checkHealth", "/connections/:owner/:integration/:name/health", {
+      params: ConnectionParams,
+      success: HealthCheckResult,
+      error: [InternalError, ConnectionNotFound, IntegrationNotFound],
+    }),
+  )
+  // Run the health check against an IN-FLIGHT credential without saving it (the
+  // key-first connect flow): confirm the pasted key works and surface the
+  // identity the UI derives a connection name from before anything persists.
+  .add(
+    HttpApiEndpoint.post("validate", "/connections/validate", {
+      payload: ValidateConnectionPayload,
+      success: HealthCheckResult,
+      error: [InternalError, IntegrationNotFound],
+    }),
   );