fix(supervisor): retry transient instance create failures in compute workload manager (#3902)

`ComputeWorkloadManager.create` swallows gateway errors currently, so a
cold start that fails placement (e.g. a netns slot with a busy tap, a
full node disk) silently abandons the dequeued run until the run
engine's `PENDING_EXECUTING` heartbeat timeout redrives it via stall
detection.

### Changes

- Retry `instances.create` with short backoff (default 3 attempts, 250ms
backoff), recording `createAttempts` in the wide event.
- **Only statuses where the create definitely did not commit are
retried**: 500 (agent/fcrun create failed) and 503 (no placement).
502/504 are excluded — the gateway emits those when it fails to reach
the node or read its response, which can happen *after* the agent
committed the create; the gateway only records the instance name on a
clean 201, so a same-name retry would miss the collision check and could
double-create the VM on another node. Network-level fetch failures are
retried (if the gateway processed the create, its name index is
populated and the retry 409s harmlessly). Timeouts are not retried.
- **Retry attempts after a 5xx use a deterministic `-rN` name suffix**:
a failed create can leave its name registered until async cleanup runs.
Attempt 1 keeps the unsuffixed name.

Author: myftija

.server-changes/retry-transient-instance-create-failures.md

@@ -0,0 +1,6 @@
+---
+area: supervisor
+type: fix
+---
+
+Retry transient instance create failures during cold starts instead of waiting minutes for the run to be requeued.

apps/supervisor/src/env.ts

@@ -114,6 +114,14 @@ const Env = z
     COMPUTE_TRACE_OTLP_ENDPOINT: z.string().url().optional(), // Override for span export (derived from TRIGGER_API_URL if unset)
     COMPUTE_SNAPSHOT_DELAY_MS: z.coerce.number().int().min(0).max(60_000).default(5_000),
     COMPUTE_SNAPSHOT_DISPATCH_LIMIT: z.coerce.number().int().min(1).max(100).default(10),
+    // Instance create retries for transient placement failures (1 = no retries)
+    COMPUTE_INSTANCE_CREATE_MAX_ATTEMPTS: z.coerce.number().int().min(1).max(10).default(3),
+    COMPUTE_INSTANCE_CREATE_RETRY_BASE_DELAY_MS: z.coerce
+      .number()
+      .int()
+      .min(0)
+      .max(10_000)
+      .default(250),
 
     // Kubernetes settings
     KUBERNETES_FORCE_ENABLED: BoolEnv.default(false),

apps/supervisor/src/index.ts

@@ -144,6 +144,10 @@ class ManagedSupervisor {
           otelEndpoint: env.OTEL_EXPORTER_OTLP_ENDPOINT,
           prettyLogs: env.RUNNER_PRETTY_LOGS,
         },
+        createRetry: {
+          maxAttempts: env.COMPUTE_INSTANCE_CREATE_MAX_ATTEMPTS,
+          baseDelayMs: env.COMPUTE_INSTANCE_CREATE_RETRY_BASE_DELAY_MS,
+        },
       });
       this.computeManager = computeManager;
       this.workloadManager = computeManager;

apps/supervisor/src/workloadManager/compute.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it } from "vitest";
+import { ComputeClientError } from "@internal/compute";
+import { isRetryableCreateError, runnerNameForAttempt } from "./compute.js";
+
+describe("runnerNameForAttempt", () => {
+  it("keeps the unsuffixed name for the first attempt", () => {
+    expect(runnerNameForAttempt("runner-abc123", 1)).toBe("runner-abc123");
+  });
+
+  it("suffixes retry attempts deterministically", () => {
+    expect(runnerNameForAttempt("runner-abc123", 2)).toBe("runner-abc123-r2");
+    expect(runnerNameForAttempt("runner-abc123", 3)).toBe("runner-abc123-r3");
+  });
+});
+
+describe("isRetryableCreateError", () => {
+  it("retries statuses where the create definitely did not commit", () => {
+    expect(isRetryableCreateError(new ComputeClientError(500, "tap busy", "http://gw"))).toBe(
+      true
+    );
+    expect(isRetryableCreateError(new ComputeClientError(503, "no placement", "http://gw"))).toBe(
+      true
+    );
+  });
+
+  it("does not retry lost-response statuses (create may have committed)", () => {
+    expect(isRetryableCreateError(new ComputeClientError(502, "bad gateway", "http://gw"))).toBe(
+      false
+    );
+    expect(
+      isRetryableCreateError(new ComputeClientError(504, "gateway timeout", "http://gw"))
+    ).toBe(false);
+  });
+
+  it("does not retry 4xx responses", () => {
+    expect(isRetryableCreateError(new ComputeClientError(400, "bad request", "http://gw"))).toBe(
+      false
+    );
+    expect(isRetryableCreateError(new ComputeClientError(409, "conflict", "http://gw"))).toBe(
+      false
+    );
+  });
+
+  it("does not retry timeouts (instance may still be provisioning)", () => {
+    expect(isRetryableCreateError(new DOMException("timed out", "TimeoutError"))).toBe(false);
+  });
+
+  it("retries network-level fetch failures", () => {
+    expect(isRetryableCreateError(new TypeError("fetch failed"))).toBe(true);
+  });
+
+  it("does not retry unknown errors", () => {
+    expect(isRetryableCreateError(new Error("something else"))).toBe(false);
+    expect(isRetryableCreateError("string error")).toBe(false);
+  });
+});

apps/supervisor/src/workloadManager/compute.ts

@@ -6,12 +6,58 @@ import {
   type WorkloadManagerCreateOptions,
   type WorkloadManagerOptions,
 } from "./types.js";
-import { ComputeClient, stripImageDigest } from "@internal/compute";
+import { ComputeClient, ComputeClientError, stripImageDigest } from "@internal/compute";
+import { setTimeout as sleep } from "node:timers/promises";
 import { extractTraceparent, getRunnerId } from "../util.js";
 import type { OtlpTraceService } from "../services/otlpTraceService.js";
 import { tryCatch } from "@trigger.dev/core";
 import { encodeBaggage, fromContext } from "../wideEvents/index.js";
 
+const DEFAULT_CREATE_MAX_ATTEMPTS = 3;
+const DEFAULT_CREATE_RETRY_BASE_DELAY_MS = 250;
+
+/**
+ * TEMPORARY (TRI-10293): a failed create can leave its instance name
+ * registered gateway/fcrun-side until async cleanup runs, so a same-name
+ * retry can 409 against our own residue. Until the gateway cleans up
+ * failed-create registrations properly, retry attempts get a deterministic
+ * suffix. Attempt 1 keeps the unsuffixed name so the non-retry path is
+ * unchanged; the suffixed name flows into both the instance name and
+ * TRIGGER_RUNNER_ID, which downstream flows treat as one opaque
+ * self-reported token. Only attempts following a ComputeClientError are
+ * suffixed - network-failure retries keep the same name on purpose, because
+ * the gateway's name-collision 409 is their safety net against
+ * double-creating an instance whose create response was lost.
+ */
+export function runnerNameForAttempt(runnerId: string, attempt: number): string {
+  return attempt === 1 ? runnerId : `${runnerId}-r${attempt}`;
+}
+
+/**
+ * Whether a failed instance create is worth retrying. Only statuses where
+ * the create definitely did NOT commit are retried: 500 means the agent or
+ * fcrun returned a create error (e.g. a netns slot holding the tap busy, a
+ * full node disk - placement may differ on retry), 503 means the gateway
+ * had nowhere to place it. 502/504 are excluded: the gateway emits those
+ * when it fails to reach the node or read its response, which can happen
+ * AFTER the agent committed the create - and the gateway only records the
+ * instance name on a clean 201, so a same-name retry would miss the
+ * collision check and could double-create the VM on another node. 4xx won't
+ * heal on retry, and timeouts may still be provisioning. Network-level
+ * fetch failures are safe: if the gateway processed the create, its name
+ * index is populated and the retry 409s harmlessly.
+ */
+export function isRetryableCreateError(error: unknown): boolean {
+  if (error instanceof ComputeClientError) {
+    return error.status === 500 || error.status === 503;
+  }
+  if (error instanceof DOMException && error.name === "TimeoutError") {
+    return false;
+  }
+  // Network-level fetch failures (gateway briefly unreachable)
+  return error instanceof TypeError;
+}
+
 type ComputeWorkloadManagerOptions = WorkloadManagerOptions & {
   gateway: {
     url: string;
@@ -30,13 +76,23 @@ type ComputeWorkloadManagerOptions = WorkloadManagerOptions & {
     otelEndpoint: string;
     prettyLogs: boolean;
   };
+  createRetry?: {
+    maxAttempts: number;
+    baseDelayMs: number;
+  };
 };
 
 export class ComputeWorkloadManager implements WorkloadManager {
   private readonly logger = new SimpleStructuredLogger("compute-workload-manager");
   private readonly compute: ComputeClient;
+  private readonly createMaxAttempts: number;
+  private readonly createRetryBaseDelayMs: number;
 
   constructor(private opts: ComputeWorkloadManagerOptions) {
+    this.createMaxAttempts = opts.createRetry?.maxAttempts ?? DEFAULT_CREATE_MAX_ATTEMPTS;
+    this.createRetryBaseDelayMs =
+      opts.createRetry?.baseDelayMs ?? DEFAULT_CREATE_RETRY_BASE_DELAY_MS;
+
     if (opts.workloadApiDomain) {
       this.logger.warn("⚠️ Custom workload API domain", {
         domain: opts.workloadApiDomain,
@@ -163,27 +219,71 @@ export class ComputeWorkloadManager implements WorkloadManager {
     const startMs = performance.now();
 
     try {
-      const [error, data] = await tryCatch(
-        this.compute.instances.create({
-          name: runnerId,
-          image: imageRef,
-          env: envVars,
-          cpu: opts.machine.cpu,
-          memory_gb: opts.machine.memory,
-          metadata: {
-            runId: opts.runFriendlyId,
-            envId: opts.envId,
-            envType: opts.envType,
-            orgId: opts.orgId,
-            projectId: opts.projectId,
-            deploymentVersion: opts.deploymentVersion,
-            machine: opts.machine.name,
-          },
-          ...(Object.keys(labels).length > 0 ? { labels } : {}),
-        })
-      );
+      const createRequest = {
+        name: runnerId,
+        image: imageRef,
+        env: envVars,
+        cpu: opts.machine.cpu,
+        memory_gb: opts.machine.memory,
+        metadata: {
+          runId: opts.runFriendlyId,
+          envId: opts.envId,
+          envType: opts.envType,
+          orgId: opts.orgId,
+          projectId: opts.projectId,
+          deploymentVersion: opts.deploymentVersion,
+          machine: opts.machine.name,
+        },
+        ...(Object.keys(labels).length > 0 ? { labels } : {}),
+      };
+
+      // Retry transient placement failures instead of abandoning the run: a
+      // swallowed create error leaves the run waiting for the run engine's
+      // PENDING_EXECUTING timeout (minutes) before it is redriven, while a
+      // retried create typically succeeds in under a second (TRI-10293).
+      let error: unknown;
+      let data: Awaited<ReturnType<typeof this.compute.instances.create>> | null | undefined;
+      let attempt = 1;
+      // Set after a ComputeClientError: the failed create may have left its
+      // name registered, so subsequent attempts use a suffixed name.
+      let suffixAttempts = false;
+      for (; attempt <= this.createMaxAttempts; attempt++) {
+        const attemptRunnerId = suffixAttempts
+          ? runnerNameForAttempt(runnerId, attempt)
+          : runnerId;
+        [error, data] = await tryCatch(
+          this.compute.instances.create(
+            attemptRunnerId === runnerId
+              ? createRequest
+              : {
+                  ...createRequest,
+                  name: attemptRunnerId,
+                  env: { ...envVars, TRIGGER_RUNNER_ID: attemptRunnerId },
+                }
+          )
+        );
+
+        if (!error) {
+          event.runnerId = attemptRunnerId;
+          break;
+        }
+
+        if (error instanceof ComputeClientError) {
+          suffixAttempts = true;
+        }
+
+        this.logger.warn("create instance attempt failed", {
+          runnerId: attemptRunnerId,
+          attempt,
+          error: error instanceof Error ? error.message : String(error),
+        });
+
+        if (!isRetryableCreateError(error) || attempt === this.createMaxAttempts) break;
+        await sleep(this.createRetryBaseDelayMs * attempt);
+      }
+      event.createAttempts = attempt;
 
-      if (error) {
+      if (error || !data) {
         event.error = error instanceof Error ? error.message : String(error);
         event.errorType =
           error instanceof DOMException && error.name === "TimeoutError" ? "timeout" : "fetch";