Retry transient Durable Object relocation on MCP session init (#1034)

Cloudflare may relocate a live Durable Object between machines; an
in-flight `init` then throws "cannot access storage because object has
moved to a different machine". The worker collapsed that rejection into a
defect and surfaced it to the client as a JSON-RPC -32603, which a
reconnect cleared on the next attempt.

Retry that specific transient class on the create/init path with a
bounded, jittered backoff, annotating the recovered-retry count on the
span. If it never clears (or any non-relocation rejection occurs) the
original value is re-raised unchanged, preserving the prior
Effect.promise -> -32603 behavior. The forward path (handleRequest) is
intentionally left alone: its Request body is a one-shot stream that
cannot be safely replayed.

Covered by an injection test at the McpSessionDOStub seam: recovery
after a relocation, the bounded-attempt give-up, and no retry of a
non-relocation failure.

Author: RhysSullivan

packages/hosts/cloudflare/src/mcp/session-store.test.ts

@@ -0,0 +1,148 @@
+// ---------------------------------------------------------------------------
+// Regression coverage for the transient DO-relocation retry in the worker-side
+// session-store dispatcher.
+//
+// Cloudflare may relocate a live Durable Object between machines; an in-flight
+// `init` then throws "cannot access storage because object has moved to a
+// different machine". Before the retry, that rejection became an unrecoverable
+// defect the envelope rendered as a -32603 — exactly the prod incident on
+// 2026-06-15 (one `mcp.do.init` failure a reconnect cleared). These tests inject
+// that rejection at the `McpSessionDOStub` seam and pin the recovery behavior.
+// ---------------------------------------------------------------------------
+
+import { describe, expect, it } from "@effect/vitest";
+import { Cause, Effect, Exit } from "effect";
+
+import { McpSessionStore, type McpDispatchResult, type Principal } from "@executor-js/host-mcp";
+
+import {
+  DO_RELOCATION_MAX_RETRIES,
+  makeDurableObjectMcpSessionStore,
+  type McpSessionDOStub,
+} from "./session-store";
+
+const RELOCATION_ERROR = "cannot access storage because object has moved to a different machine";
+
+const TEST_PRINCIPAL: Principal = {
+  accountId: "user_test",
+  organizationId: "org_test",
+  organizationName: "Test Org",
+  email: "test@example.com",
+  name: "Test",
+  avatarUrl: null,
+  roles: ["user"],
+};
+
+/** A POST with no session id — routes dispatch through the create/init path. */
+const initializeRequest = (): Request =>
+  new Request("https://mcp.test/mcp", {
+    method: "POST",
+    headers: { authorization: "Bearer x", "content-type": "application/json" },
+    body: JSON.stringify({ jsonrpc: "2.0", id: 0, method: "initialize" }),
+  });
+
+/** The JSON-RPC body the faked DO returns once `init` succeeds. */
+const okResponse = (): Response =>
+  new Response(JSON.stringify({ jsonrpc: "2.0", id: 0, result: {} }), {
+    status: 200,
+    headers: { "content-type": "application/json" },
+  });
+
+/** Drive `dispatch` for a create over a store built from one fake stub. */
+const dispatchCreate = (stub: McpSessionDOStub): Effect.Effect<McpDispatchResult> =>
+  Effect.gen(function* () {
+    const store = yield* McpSessionStore;
+    return yield* store.dispatch({
+      request: initializeRequest(),
+      principal: TEST_PRINCIPAL,
+      sessionId: null,
+      method: "POST",
+    });
+  }).pipe(
+    Effect.provide(makeDurableObjectMcpSessionStore({ newStub: () => stub, getStub: () => stub })),
+  );
+
+/** Rendered cause of a failed dispatch (empty for a success) — keeps the
+ *  message assertion unconditional, off the `Exit.isFailure` branch. */
+const failureText = (exit: Exit.Exit<McpDispatchResult>): string =>
+  Exit.isFailure(exit) ? Cause.pretty(exit.cause) : "";
+
+describe("makeDurableObjectMcpSessionStore — DO-relocation retry", () => {
+  it.live("retries mcp.do.init past a relocation, then returns the DO response", () =>
+    Effect.gen(function* () {
+      let initCalls = 0;
+      let handleCalls = 0;
+      const stub: McpSessionDOStub = {
+        init: () => {
+          initCalls += 1;
+          if (initCalls === 1) {
+            // oxlint-disable-next-line executor/no-promise-reject, executor/no-error-constructor -- boundary: fake DO stub rejects to simulate a Cloudflare relocation throw
+            return Promise.reject(new Error(RELOCATION_ERROR));
+          }
+          return Promise.resolve();
+        },
+        handleRequest: () => {
+          handleCalls += 1;
+          return Promise.resolve(okResponse());
+        },
+        clearSession: () => Promise.resolve(),
+      };
+
+      const result = yield* dispatchCreate(stub);
+
+      expect(initCalls, "init retried once after the relocation").toBe(2);
+      expect(handleCalls, "handleRequest runs once, after init recovers").toBe(1);
+      expect(result).toBeInstanceOf(Response);
+      expect((result as Response).status).toBe(200);
+    }),
+  );
+
+  it.live("gives up after the retry budget when relocation never clears", () =>
+    Effect.gen(function* () {
+      let initCalls = 0;
+      let handleCalls = 0;
+      const stub: McpSessionDOStub = {
+        init: () => {
+          initCalls += 1;
+          // oxlint-disable-next-line executor/no-promise-reject, executor/no-error-constructor -- boundary: fake DO stub rejects to simulate a Cloudflare relocation throw
+          return Promise.reject(new Error(RELOCATION_ERROR));
+        },
+        handleRequest: () => {
+          handleCalls += 1;
+          return Promise.resolve(okResponse());
+        },
+        clearSession: () => Promise.resolve(),
+      };
+
+      const exit = yield* Effect.exit(dispatchCreate(stub));
+
+      expect(Exit.isFailure(exit), "exhausted retries surface as a defect").toBe(true);
+      expect(initCalls, "one initial attempt plus the full retry budget").toBe(
+        1 + DO_RELOCATION_MAX_RETRIES,
+      );
+      expect(handleCalls, "handleRequest is never reached when init never succeeds").toBe(0);
+      expect(failureText(exit), "the relocation cause is preserved").toContain(RELOCATION_ERROR);
+    }),
+  );
+
+  it.live("does not retry a non-relocation failure — surfaces it immediately", () =>
+    Effect.gen(function* () {
+      let initCalls = 0;
+      const stub: McpSessionDOStub = {
+        init: () => {
+          initCalls += 1;
+          // oxlint-disable-next-line executor/no-promise-reject, executor/no-error-constructor -- boundary: fake DO stub rejects to simulate a non-transient init failure
+          return Promise.reject(new Error("schema spread bug — not transient"));
+        },
+        handleRequest: () => Promise.resolve(okResponse()),
+        clearSession: () => Promise.resolve(),
+      };
+
+      const exit = yield* Effect.exit(dispatchCreate(stub));
+
+      expect(Exit.isFailure(exit), "a non-transient rejection still dies").toBe(true);
+      expect(initCalls, "non-relocation errors are not retried").toBe(1);
+      expect(failureText(exit), "the original failure is preserved").toContain("schema spread bug");
+    }),
+  );
+});

packages/hosts/cloudflare/src/mcp/session-store.ts

@@ -19,7 +19,7 @@
 // and DELETE (204) before dispatch, so dispatch only sees create or forward.
 // ---------------------------------------------------------------------------
 
-import { Effect, Layer } from "effect";
+import { Data, Effect, Layer, Schedule } from "effect";
 
 import {
   McpSessionStore,
@@ -49,6 +49,87 @@ export interface DurableObjectStoreConfig {
   readonly onInternalError?: OnInternalJsonRpcError;
 }
 
+// ---------------------------------------------------------------------------
+// Transient DO-relocation retry.
+//
+// Cloudflare may relocate a live Durable Object between machines; an in-flight
+// stub call against the now-stale instance throws "cannot access storage
+// because object has moved to a different machine". The runtime throws this
+// BEFORE the object touched storage, so no work and no writes happened on that
+// instance — a re-issued stub call resolves the DO's new location and succeeds.
+// (Observed in prod 2026-06-15: a single `mcp.do.init` failure surfaced to the
+// client as a -32603 "Internal server error" that a reconnect cleared.)
+//
+// We retry ONLY this relocation class, and ONLY `init` — init takes a
+// structured `meta` object with no request body, so replaying it is safe. The
+// forward path (`handleRequest`) is deliberately excluded: its `Request` body is
+// a one-shot stream that cannot be re-sent on a retry without buffering.
+// ---------------------------------------------------------------------------
+
+const RELOCATED_DO_ERROR_FRAGMENT = "moved to a different machine";
+
+/** A rejected DO stub call, normalized at the boundary: the original thrown
+ *  value in `cause` (so the final `Effect.die` re-raises it verbatim) plus a
+ *  pre-computed `relocated` flag so the retry gate never re-inspects `unknown`. */
+class DoStubError extends Data.TaggedError("DoStubError")<{
+  readonly cause: unknown;
+  readonly relocated: boolean;
+}> {}
+
+/** Normalize a rejected DO stub call, flagging the Cloudflare relocation class —
+ *  the only safe-to-retry error, and detectable only via the thrown message. */
+const toDoStubError = (cause: unknown): DoStubError =>
+  new DoStubError({
+    cause,
+    // oxlint-disable-next-line executor/no-instanceof-error, executor/no-unknown-error-message -- boundary: Cloudflare signals DO relocation only via the thrown Error's message
+    relocated: cause instanceof Error && cause.message.includes(RELOCATED_DO_ERROR_FRAGMENT),
+  });
+
+/** Up to 4 attempts total (1 + 3 retries) at ~25/50/100ms jittered backoff. */
+export const DO_RELOCATION_MAX_RETRIES = 3;
+const DO_RELOCATION_RETRY_SCHEDULE = Schedule.jittered(Schedule.exponential("25 millis"));
+
+/**
+ * Run a replayable DO stub call under one `withSpan`, retrying the DO-relocation
+ * error a bounded number of times. A call that still fails after retries (or any
+ * non-relocation rejection) becomes an unrecoverable defect — exactly the prior
+ * `Effect.promise` semantics, which the envelope's top-level `catchCause` renders
+ * as a -32603 and reports to the error reporter (cloud: Sentry).
+ */
+const withDoRelocationRetry = <A>(
+  spanName: string,
+  attributes: Record<string, string | number | boolean>,
+  call: () => Promise<A>,
+): Effect.Effect<A> =>
+  Effect.suspend(() => {
+    let attempts = 0;
+    return Effect.tryPromise({
+      try: () => {
+        attempts += 1;
+        return call();
+      },
+      catch: toDoStubError,
+    }).pipe(
+      // Gate retries on BOTH the relocation class and a hard attempt budget. The
+      // `attempts` counter (bumped in `try`) bounds this directly rather than via
+      // `retry`'s `times`, whose interaction with `while` is unreliable here.
+      Effect.retry({
+        schedule: DO_RELOCATION_RETRY_SCHEDULE,
+        while: (error) => error.relocated && attempts <= DO_RELOCATION_MAX_RETRIES,
+      }),
+      Effect.tap(() =>
+        attempts > 1
+          ? Effect.annotateCurrentSpan({ "mcp.do.relocation_retries": attempts - 1 })
+          : Effect.void,
+      ),
+      // A call that still fails (or any non-relocation rejection) re-raises the
+      // original thrown value as an unrecoverable defect — the prior
+      // `Effect.promise` semantics the envelope renders as a -32603.
+      // oxlint-disable-next-line executor/no-effect-escape-hatch -- boundary: dispatch's contract is E=never; an exhausted DO RPC re-raises as the defect the envelope renders as -32603
+      Effect.catch((error) => Effect.die(error.cause)),
+    );
+  }).pipe(Effect.withSpan(spanName, { attributes }));
+
 /**
  * Forward a request to an existing session DO. `peek` tees the body for
  * telemetry on POST/DELETE; GET (SSE) streams through untouched. Returns the DO
@@ -91,7 +172,7 @@ const createSession = (
   Effect.gen(function* () {
     const stub = config.newStub();
     const propagation = yield* currentPropagationHeaders(request);
-    yield* Effect.promise(() =>
+    yield* withDoRelocationRetry("mcp.do.init", { "mcp.request.session_id_present": false }, () =>
       stub.init(
         {
           organizationId: token.organizationId,
@@ -104,10 +185,6 @@ const createSession = (
         },
         propagation,
       ),
-    ).pipe(
-      Effect.withSpan("mcp.do.init", {
-        attributes: { "mcp.request.session_id_present": false },
-      }),
     );
     const propagated = withPropagationHeaders(
       withVerifiedIdentityHeaders(request, token),