Cache compiled specs for request-path OpenAPI fallbacks

The health-check endpoints, candidate listing, and the tools/invoke
fallbacks recompiled the full OpenAPI document on every request. The UI
now auto-fires health checks on page mounts, so a large spec was parsed
into a fresh multi-MB object graph over and over, growing the dev server
heap until the process hit the V8 limit and every subsequent request
failed (the CI shard wedge behind the login 500 cascade).

Compile through a small module-level LRU instead, keyed by the config's
content-addressed specHash: same hash means byte-identical text, and a
spec update writes a new hash so stale entries age out. Capacity is four
compiled documents; legacy configs without a hash bypass the cache.
Add and update paths keep compiling fresh input directly.

Author: RhysSullivan

packages/plugins/openapi/src/sdk/backing.ts

@@ -248,6 +248,56 @@ export const compileOpenApiSpec = (
     return yield* compileOpenApiDocument(doc);
   });
 
+/**
+ * Bounded reuse of compiled specs for the serve-side fallback paths (health
+ * checks, candidate listing, tools/invoke legacy fallbacks). These paths run
+ * per request, and the UI auto-fires the health endpoints on every page mount,
+ * so recompiling a multi-MB spec on each call grows the dev server heap until
+ * the process hits the V8 limit. The stored spec is content-addressed: the
+ * config's `specHash` keys the blob the text is loaded from, so the hash is a
+ * free, already-computed identity (same hash means byte-identical text means
+ * identical compile output), and a spec update writes a new hash so stale
+ * entries age out naturally.
+ *
+ * An insertion-ordered Map serves as a tiny LRU: hits are re-inserted to
+ * refresh recency and the oldest entry is evicted past capacity. Capacity 4
+ * covers the handful of integrations a page's probes touch at once while
+ * keeping worst-case retention to a few compiled documents.
+ */
+const COMPILED_SPEC_CACHE_CAPACITY = 4;
+const compiledSpecCache = new Map<string, CompiledOpenApiSpec>();
+
+/** Test-only reset so unit tests can observe cold-cache behavior. */
+export const clearCompiledOpenApiSpecCache = (): void => {
+  compiledSpecCache.clear();
+};
+
+/** Compile through the module-level LRU. `specHash` is the content hash the
+ *  spec text was loaded by (`OpenApiIntegrationConfig.specHash`). A missing
+ *  hash (legacy config shape) bypasses the cache rather than paying to hash
+ *  multi-MB text on the request path. */
+export const compileOpenApiSpecCached = (
+  specHash: string | null | undefined,
+  specText: string,
+): Effect.Effect<CompiledOpenApiSpec, OpenApiParseError | OpenApiExtractionError> =>
+  Effect.gen(function* () {
+    if (specHash == null) return yield* compileOpenApiSpec(specText);
+    const hit = compiledSpecCache.get(specHash);
+    if (hit !== undefined) {
+      compiledSpecCache.delete(specHash);
+      compiledSpecCache.set(specHash, hit);
+      return hit;
+    }
+    const compiled = yield* compileOpenApiSpec(specText);
+    compiledSpecCache.set(specHash, compiled);
+    while (compiledSpecCache.size > COMPILED_SPEC_CACHE_CAPACITY) {
+      const oldest = compiledSpecCache.keys().next().value;
+      if (oldest === undefined) break;
+      compiledSpecCache.delete(oldest);
+    }
+    return compiled;
+  });
+
 export const openApiToolDefsFromCompiled = (compiled: CompiledOpenApiSpec): readonly ToolDef[] =>
   compiled.definitions.map((def): ToolDef => {
     const returnsFile = Option.match(def.operation.responseBody, {
@@ -543,7 +593,7 @@ export const resolveOpenApiBackedTools = ({
     }
     const specText = yield* loadOpenApiSpecText(storage, openApiConfig);
     if (specText == null) return { tools: [], definitions: {} };
-    const compiled = yield* compileOpenApiSpec(specText).pipe(
+    const compiled = yield* compileOpenApiSpecCached(openApiConfig.specHash, specText).pipe(
       Effect.catch(() => Effect.succeed(null)),
     );
     if (!compiled) return { tools: [], definitions: {} };
@@ -577,7 +627,9 @@ export const invokeOpenApiBackedTool = (input: {
       const compiled =
         specText == null
           ? null
-          : yield* compileOpenApiSpec(specText).pipe(Effect.catch(() => Effect.succeed(null)));
+          : yield* compileOpenApiSpecCached(config.specHash, specText).pipe(
+              Effect.catch(() => Effect.succeed(null)),
+            );
       binding = compiled
         ? openApiStoredOperationsFromCompiled(integration, compiled).find(
             (op) => op.toolName === input.toolRow.name,
@@ -716,7 +768,7 @@ const resolveHealthCheckBinding = (
       Effect.catch(() => Effect.succeed(null)),
     );
     if (specText == null) return undefined;
-    const compiled = yield* compileOpenApiSpec(specText).pipe(
+    const compiled = yield* compileOpenApiSpecCached(config.specHash, specText).pipe(
       Effect.catch(() => Effect.succeed(null)),
     );
     if (!compiled) return undefined;
@@ -878,7 +930,9 @@ export const listHealthCheckCandidatesOpenApi = (input: {
       const compiled =
         specText == null
           ? null
-          : yield* compileOpenApiSpec(specText).pipe(Effect.catch(() => Effect.succeed(null)));
+          : yield* compileOpenApiSpecCached(config.specHash, specText).pipe(
+              Effect.catch(() => Effect.succeed(null)),
+            );
       if (compiled) {
         for (const def of compiled.definitions) {
           const summary =

packages/plugins/openapi/src/sdk/compiled-spec-cache.test.ts

@@ -0,0 +1,66 @@
+import { describe, expect, it } from "@effect/vitest";
+import { Effect } from "effect";
+
+import { clearCompiledOpenApiSpecCache, compileOpenApiSpecCached } from "./backing";
+
+const specText = (title: string): string =>
+  JSON.stringify({
+    openapi: "3.0.3",
+    info: { title, version: "1.0.0" },
+    servers: [{ url: "https://api.example.com" }],
+    paths: {
+      "/me": {
+        get: {
+          operationId: "getMe",
+          responses: {
+            "200": {
+              description: "ok",
+              content: {
+                "application/json": {
+                  schema: { type: "object", properties: { email: { type: "string" } } },
+                },
+              },
+            },
+          },
+        },
+      },
+    },
+  });
+
+describe("compileOpenApiSpecCached", () => {
+  it("reuses the compiled result for the same spec hash", () => {
+    clearCompiledOpenApiSpecCache();
+    const text = specText("Cached API");
+    const first = Effect.runSync(compileOpenApiSpecCached("hash-a", text));
+    const second = Effect.runSync(compileOpenApiSpecCached("hash-a", text));
+    // Reference equality proves the second call skipped recompilation.
+    expect(second).toBe(first);
+  });
+
+  it("recompiles when the spec hash changes", () => {
+    clearCompiledOpenApiSpecCache();
+    const first = Effect.runSync(compileOpenApiSpecCached("hash-a", specText("V1")));
+    const second = Effect.runSync(compileOpenApiSpecCached("hash-b", specText("V2")));
+    expect(second).not.toBe(first);
+    expect(second.title).toBe("V2");
+  });
+
+  it("bypasses the cache when no hash is available", () => {
+    clearCompiledOpenApiSpecCache();
+    const text = specText("No hash");
+    const first = Effect.runSync(compileOpenApiSpecCached(undefined, text));
+    const second = Effect.runSync(compileOpenApiSpecCached(undefined, text));
+    expect(second).not.toBe(first);
+  });
+
+  it("evicts the oldest entry past capacity", () => {
+    clearCompiledOpenApiSpecCache();
+    const first = Effect.runSync(compileOpenApiSpecCached("hash-0", specText("Zero")));
+    for (let index = 1; index <= 4; index++) {
+      Effect.runSync(compileOpenApiSpecCached(`hash-${index}`, specText(`Spec ${index}`)));
+    }
+    // hash-0 was evicted by the four newer entries, so it recompiles.
+    const again = Effect.runSync(compileOpenApiSpecCached("hash-0", specText("Zero")));
+    expect(again).not.toBe(first);
+  });
+});