Stop Codex harness retry loops on TPM exhaustion and unfinished-goal errors (#42420)

Author: Copilot

actions/setup/js/codex_harness.cjs

@@ -64,6 +64,7 @@ const MAX_DELAY_MS = 60000;
 // the human-readable message Codex emits inside "Reconnecting..." / error lines:
 // "Rate limit reached for <model> in organization <org> on tokens per min (TPM): ..."
 const RATE_LIMIT_ERROR_PATTERN = /rate_limit_exceeded|429 Too Many Requests|RateLimitError|Rate limit reached for [^\s]+(?: in organization [^\s]+)? on tokens per min/i;
+const TOKEN_PER_MIN_RATE_LIMIT_PATTERN = /Rate limit reached for [^\s]+(?: in organization [^\s]+)? on tokens per min/i;
 
 // Pattern to detect when Codex's internal stream-reconnect budget is fully spent.
 // Codex emits "Reconnecting... N/N (reason)" where both numbers are the same when
@@ -106,6 +107,17 @@ function isRateLimitError(output) {
   return RATE_LIMIT_ERROR_PATTERN.test(output);
 }
 
+/**
+ * Determines if the collected output indicates OpenAI token-per-minute exhaustion.
+ * This limit is workload-dependent and immediate fresh-run retries can consume
+ * additional budget without making progress.
+ * @param {string} output - Collected stdout+stderr from the process
+ * @returns {boolean}
+ */
+function isTokenPerMinuteRateLimitError(output) {
+  return TOKEN_PER_MIN_RATE_LIMIT_PATTERN.test(output);
+}
+
 /**
  * Determines if the collected output contains an authentication failed error.
  * @param {string} output - Collected stdout+stderr from the process
@@ -436,6 +448,7 @@ async function main() {
     }
 
     const isRateLimit = isRateLimitError(result.output);
+    const isTokenPerMinuteRateLimit = isTokenPerMinuteRateLimitError(result.output);
     const isAuthenticationFailed = isAuthenticationFailedError(result.output);
     const isMissingApiKey = isMissingApiKeyError(result.output);
     const isServer = isServerError(result.output);
@@ -446,6 +459,7 @@ async function main() {
       `attempt ${attempt + 1} failed:` +
         ` exitCode=${result.exitCode}` +
         ` isRateLimitError=${isRateLimit}` +
+        ` isTokenPerMinuteRateLimitError=${isTokenPerMinuteRateLimit}` +
         ` isAuthenticationFailedError=${isAuthenticationFailed}` +
         ` isMissingApiKeyError=${isMissingApiKey}` +
         ` isServerError=${isServer}` +
@@ -506,6 +520,14 @@ async function main() {
       break;
     }
 
+    // Token-per-minute limits indicate exhausted budget for the current workload profile.
+    // Fresh-run retries immediately repeat the same prompt workload and can quickly
+    // drain available credits without making forward progress.
+    if (isTokenPerMinuteRateLimit) {
+      log(`attempt ${attempt + 1}: token-per-minute rate limit detected — not retrying (fresh runs can further drain token budget)`);
+      break;
+    }
+
     // Codex's internal stream-reconnect retries are exhausted and the root cause is a
     // rate-limit error.  Each reconnect attempt immediately failed with the same limit,
     // so a fresh harness run will encounter the same rate-limit at the same point in the
@@ -545,6 +567,7 @@ if (typeof module !== "undefined" && module.exports) {
     resolveCodexPromptFileArgs,
     injectJsonFlag,
     isRateLimitError,
+    isTokenPerMinuteRateLimitError,
     isAuthenticationFailedError,
     isMissingApiKeyError,
     isServerError,

actions/setup/js/codex_harness.test.cjs

@@ -10,6 +10,7 @@ const {
   resolveCodexPromptFileArgs,
   injectJsonFlag,
   isRateLimitError,
+  isTokenPerMinuteRateLimitError,
   isAuthenticationFailedError,
   isMissingApiKeyError,
   isServerError,
@@ -90,6 +91,20 @@ describe("codex_harness.cjs", () => {
       expect(isRateLimitError("Error: rate_limit_exceeded")).toBe(true);
     });
 
+    describe("isTokenPerMinuteRateLimitError", () => {
+      it("returns true for OpenAI TPM-limit wording", () => {
+        expect(isTokenPerMinuteRateLimitError("Rate limit reached for gpt-4o-mini in organization org-xxx on tokens per min (TPM): Limit 200000, Used 166655, Requested 35398. Please try again in 615ms.")).toBe(true);
+      });
+
+      it("returns false for generic rate-limit wording", () => {
+        expect(isTokenPerMinuteRateLimitError("rate_limit_exceeded")).toBe(false);
+      });
+
+      it("returns false for unrelated mention of tokens per min", () => {
+        expect(isTokenPerMinuteRateLimitError("rate_limit_exceeded while printing docs about 'on tokens per min'")).toBe(false);
+      });
+    });
+
     it("returns true for 429 Too Many Requests", () => {
       expect(isRateLimitError("429 Too Many Requests")).toBe(true);
     });
@@ -425,6 +440,8 @@ env_key = "OPENAI_API_KEY"
       const nonRetryableGuard = detectNonRetryableHarnessGuard(result.output);
       if (nonRetryableGuard.aiCreditsExceeded || nonRetryableGuard.awfAPIProxyBlockingRequests || nonRetryableGuard.goalAlreadyActive || nonRetryableGuard.maxRunsExceeded) return false;
       const isRateLimit = isRateLimitError(result.output);
+      const isTokenPerMinuteRateLimit = isTokenPerMinuteRateLimitError(result.output);
+      if (isTokenPerMinuteRateLimit) return false;
       if (isRateLimit && isReconnectExhaustedError(result.output)) return false;
       const isTransient = isRateLimit || isServerError(result.output);
       return attempt < MAX_RETRIES && (result.hasOutput || isTransient);
@@ -494,24 +511,33 @@ env_key = "OPENAI_API_KEY"
       expect(shouldRetry(result, 0)).toBe(false);
     });
 
-    it("retries on rate limit with format 'Rate limit reached for' without exhausted reconnects", () => {
+    it("does not retry on token-per-minute rate limit wording", () => {
       const result = {
         exitCode: 1,
         hasOutput: false,
         output: '{"type":"error","message":"Rate limit reached for gpt-4o-mini in organization org-xxx on tokens per min (TPM): Limit 200000, Used 50000, Requested 35000. Please try again in 615ms."}',
       };
-      expect(shouldRetry(result, 0)).toBe(true);
+      expect(shouldRetry(result, 0)).toBe(false);
+    });
+
+    it("does not retry on token-per-minute rate limit wording even with partial output", () => {
+      const result = {
+        exitCode: 1,
+        hasOutput: true,
+        output: '{"type":"error","message":"Rate limit reached for gpt-4o-mini in organization org-xxx on tokens per min (TPM): Limit 200000, Used 50000, Requested 35000. Please try again in 615ms."}',
+      };
+      expect(shouldRetry(result, 0)).toBe(false);
     });
 
-    it("does not retry when rate-limit reconnects are exhausted (N/N pattern)", () => {
+    it("does not retry when rate-limit reconnects are exhausted (non-TPM rate limit)", () => {
       // Simulates the real log format: multiple Reconnecting... lines appear in
       // the output as codex retries the stream. The final "5/5" line is what
       // triggers the exhausted-reconnect detection; intermediate lines (1/5, 2/5)
       // confirm that the function ignores non-final attempts.
       const output =
-        '{"type":"error","message":"Reconnecting... 1/5 (stream disconnected before completion: Rate limit reached for gpt-4o-mini on tokens per min (TPM): Limit 200000, Used 166655, Requested 35398. Please try again in 615ms.)"}\n' +
-        '{"type":"error","message":"Reconnecting... 2/5 (stream disconnected before completion: Rate limit reached for gpt-4o-mini on tokens per min (TPM): Limit 200000, Used 166655, Requested 35398. Please try again in 615ms.)"}\n' +
-        '{"type":"error","message":"Reconnecting... 5/5 (stream disconnected before completion: Rate limit reached for gpt-4o-mini on tokens per min (TPM): Limit 200000, Used 166655, Requested 35398. Please try again in 615ms.)"}';
+        '{"type":"error","message":"Reconnecting... 1/5 (stream disconnected before completion: RateLimitError)"}\n' +
+        '{"type":"error","message":"Reconnecting... 2/5 (stream disconnected before completion: RateLimitError)"}\n' +
+        '{"type":"error","message":"Reconnecting... 5/5 (stream disconnected before completion: RateLimitError)"}';
       const result = { exitCode: 1, hasOutput: true, output };
       expect(shouldRetry(result, 0)).toBe(false);
     });

actions/setup/js/harness_retry_guard.cjs

@@ -5,7 +5,7 @@
 const AI_CREDITS_EXCEEDED_PATTERNS = [/\bmax[\s_-]*ai[\s_-]*credits[\s_-]*exceeded\b/i, /\bai[\s_-]*credits[\s_-]*rate[\s_-]*limit[\s_-]*error\b/i, /ai[\s_-]*credits?.*(?:rate[\s-]*limit|limit exceeded|budget exceeded|exceeded)/i];
 
 const AWF_API_PROXY_BLOCKING_REQUESTS_PATTERNS = [/\bawf\b.*\bapi[\s_-]*proxy\b.*\bblocking requests\b/i, /\bapi[\s_-]*proxy\b.*\bblocking requests\b/i, /\bapi[\s_-]*proxy\b.*\bblocked requests?\b/i, /\bDIFC_FILTERED\b/];
-const GOAL_ALREADY_ACTIVE_PATTERNS = [/\bthis thread already has a goal\b[\s\S]*?\buse update_goal\b/i];
+const GOAL_ALREADY_ACTIVE_PATTERNS = [/\bthis thread already has a goal\b[\s\S]*?\buse update_goal\b/i, /\bcannot create a new goal because this thread has an unfinished goal\b;\s*\bcomplete the existing goal first\b/i];
 
 // Patterns to detect Anthropic "max_runs_exceeded" (HTTP 403).
 // This occurs when the per-session LLM invocation quota is exhausted.

actions/setup/js/harness_retry_guard.test.cjs

@@ -92,6 +92,16 @@ describe("harness_retry_guard.cjs", () => {
     expect(result.goalAlreadyActive).toBe(true);
   });
 
+  it("detects goal already active for unfinished-goal wording", () => {
+    const result = detectNonRetryableHarnessGuard("cannot create a new goal because this thread has an unfinished goal; complete the existing goal first");
+    expect(result.goalAlreadyActive).toBe(true);
+  });
+
+  it("detects goal already active for unfinished-goal wording (JSON-wrapped)", () => {
+    const result = detectNonRetryableHarnessGuard('{"type":"error","message":"cannot create a new goal because this thread has an unfinished goal; complete the existing goal first"}');
+    expect(result.goalAlreadyActive).toBe(true);
+  });
+
   it("detects max_runs_exceeded by JSON error type", () => {
     const result = detectNonRetryableHarnessGuard('{"error":{"type":"max_runs_exceeded","message":"Maximum LLM invocations exceeded (20 / 20).","invocation_count":20,"max_runs":20}}');
     expect(result.maxRunsExceeded).toBe(true);

docs/src/content/docs/reference/engines.md

@@ -296,6 +296,12 @@ The value must be a bare filename — no directory separators, no `..`, and no s
 > [!NOTE]
 > `engine.harness` is currently only applied during Copilot engine execution. Setting it on other engines has no effect.
 
+### Harness Retry Count
+
+Built-in harness scripts (`copilot_harness.cjs`, `claude_harness.cjs`, `codex_harness.cjs`) currently use a fixed retry budget of **3 retries** after the initial run (4 total attempts).
+
+To specify a different retry count, provide a custom harness script and implement your own retry policy there. At present, `engine.harness` customization is only applied for the Copilot engine.
+
 **Validation rules:**
 
 | Rule | Valid example | Invalid example |