Add --retry flag to re-run only failed or timed-out agents (#115)

Loads latest result, identifies failed/timed-out agents, creates new
worktrees only for those, re-runs with same prompt, merges results
back into original result set. Saves money by not re-running successful
agents.

Generated by thinktank Opus (5 agents, 1 completed just under timeout).

Closes #77

Co-authored-by: unknown <that-github-user@github.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: 3 people

src/cli.ts

@@ -7,7 +7,7 @@ import { compare } from "./commands/compare.js";
 import { type ConfigAction, config } from "./commands/config.js";
 import { evaluate } from "./commands/evaluate.js";
 import { list } from "./commands/list.js";
-import { run } from "./commands/run.js";
+import { retry, run } from "./commands/run.js";
 import { stats } from "./commands/stats.js";
 import { undo } from "./commands/undo.js";
 import { loadConfig } from "./utils/config.js";
@@ -48,15 +48,8 @@ program
   .option("--no-color", "Disable colored output")
   .option("--output-format <format>", "Output format: text (default) or json", "text")
   .option("--verbose", "Show detailed output from each agent")
+  .option("--retry", "Re-run only failed/timed-out agents from the last run")
   .action(async (promptArg: string | undefined, opts) => {
-    const prompt = resolvePrompt(promptArg, opts.file);
-
-    const attempts = parseInt(opts.attempts, 10);
-    if (Number.isNaN(attempts) || attempts < 1 || attempts > 20) {
-      console.error("Error: --attempts must be a number between 1 and 20");
-      process.exit(1);
-    }
-
     const testTimeout = parseInt(opts.testTimeout, 10);
     if (Number.isNaN(testTimeout) || testTimeout < 10 || testTimeout > 600) {
       console.error("Error: --test-timeout must be a number between 10 and 600 seconds");
@@ -92,6 +85,33 @@ program
       process.exit(1);
     }
 
+    // --retry: re-run only failed agents from last run, ignore --attempts and prompt
+    if (opts.retry) {
+      await retry({
+        prompt: "", // ignored — loaded from previous result
+        attempts: 0, // ignored — determined by failed agent count
+        testCmd: opts.testCmd,
+        testTimeout,
+        timeout,
+        model: opts.model,
+        threshold,
+        runner: opts.runner,
+        scoring: opts.scoring,
+        verbose: opts.verbose ?? false,
+        outputFormat: opts.outputFormat,
+        retry: true,
+      });
+      return;
+    }
+
+    const prompt = resolvePrompt(promptArg, opts.file);
+
+    const attempts = parseInt(opts.attempts, 10);
+    if (Number.isNaN(attempts) || attempts < 1 || attempts > 20) {
+      console.error("Error: --attempts must be a number between 1 and 20");
+      process.exit(1);
+    }
+
     const knownModels = ["sonnet", "opus", "haiku"];
     if (!knownModels.includes(opts.model) && !opts.model.startsWith("claude-")) {
       console.warn(

src/commands/run.test.ts

@@ -1,7 +1,12 @@
 import assert from "node:assert/strict";
 import { afterEach, describe, it } from "node:test";
-import type { RunOptions } from "../types.js";
-import { makeResultFilename, preflightValidation } from "./run.js";
+import type { AgentResult, EnsembleResult, RunOptions } from "../types.js";
+import {
+  findFailedAgents,
+  makeResultFilename,
+  mergeRetryResults,
+  preflightValidation,
+} from "./run.js";
 
 function makeOpts(overrides: Partial<RunOptions> = {}): RunOptions {
   return {
@@ -117,3 +122,135 @@ describe("NO_COLOR environment variable", () => {
     assert.equal(colors.bold("test"), "test");
   });
 });
+
+function makeAgent(overrides: Partial<AgentResult> = {}): AgentResult {
+  return {
+    id: 1,
+    worktree: "/tmp/thinktank-agent-1",
+    status: "success",
+    exitCode: 0,
+    duration: 5000,
+    output: "",
+    diff: "diff --git a/file.ts b/file.ts\n+added line",
+    filesChanged: ["file.ts"],
+    linesAdded: 1,
+    linesRemoved: 0,
+    ...overrides,
+  };
+}
+
+function makeResult(overrides: Partial<EnsembleResult> = {}): EnsembleResult {
+  return {
+    prompt: "fix the bug",
+    model: "sonnet",
+    timestamp: "2026-03-28T10:00:00.000Z",
+    scoring: "copeland",
+    agents: [
+      makeAgent({ id: 1, status: "success" }),
+      makeAgent({ id: 2, status: "error", exitCode: 1, diff: "", filesChanged: [] }),
+      makeAgent({ id: 3, status: "timeout", exitCode: 1, diff: "", filesChanged: [] }),
+    ],
+    tests: [],
+    convergence: [],
+    recommended: 1,
+    scores: [],
+    ...overrides,
+  };
+}
+
+describe("findFailedAgents", () => {
+  it("returns agents with error status", () => {
+    const result = makeResult();
+    const failed = findFailedAgents(result);
+    const ids = failed.map((a) => a.id);
+    assert.ok(ids.includes(2));
+  });
+
+  it("returns agents with timeout status", () => {
+    const result = makeResult();
+    const failed = findFailedAgents(result);
+    const ids = failed.map((a) => a.id);
+    assert.ok(ids.includes(3));
+  });
+
+  it("does not return successful agents", () => {
+    const result = makeResult();
+    const failed = findFailedAgents(result);
+    const ids = failed.map((a) => a.id);
+    assert.ok(!ids.includes(1));
+  });
+
+  it("returns empty array when all agents succeeded", () => {
+    const result = makeResult({
+      agents: [makeAgent({ id: 1, status: "success" }), makeAgent({ id: 2, status: "success" })],
+    });
+    const failed = findFailedAgents(result);
+    assert.equal(failed.length, 0);
+  });
+
+  it("returns all agents when all failed", () => {
+    const result = makeResult({
+      agents: [
+        makeAgent({ id: 1, status: "error" }),
+        makeAgent({ id: 2, status: "timeout" }),
+        makeAgent({ id: 3, status: "error" }),
+      ],
+    });
+    const failed = findFailedAgents(result);
+    assert.equal(failed.length, 3);
+  });
+});
+
+describe("mergeRetryResults", () => {
+  it("replaces failed agents with retried results", () => {
+    const original = makeResult();
+    const retried = [
+      makeAgent({ id: 2, status: "success", diff: "new diff for 2", filesChanged: ["a.ts"] }),
+      makeAgent({ id: 3, status: "success", diff: "new diff for 3", filesChanged: ["b.ts"] }),
+    ];
+
+    const merged = mergeRetryResults(original, retried);
+
+    assert.equal(merged.length, 3);
+    assert.equal(merged[0].id, 1);
+    assert.equal(merged[0].status, "success");
+    assert.equal(merged[1].id, 2);
+    assert.equal(merged[1].status, "success");
+    assert.equal(merged[1].diff, "new diff for 2");
+    assert.equal(merged[2].id, 3);
+    assert.equal(merged[2].status, "success");
+    assert.equal(merged[2].diff, "new diff for 3");
+  });
+
+  it("preserves successful agents unchanged", () => {
+    const original = makeResult();
+    const retried = [makeAgent({ id: 2, status: "success" })];
+
+    const merged = mergeRetryResults(original, retried);
+
+    assert.equal(merged[0].id, 1);
+    assert.equal(merged[0].status, "success");
+    assert.equal(merged[0].diff, original.agents[0].diff);
+  });
+
+  it("handles retry where agent still fails", () => {
+    const original = makeResult();
+    const retried = [makeAgent({ id: 2, status: "error", diff: "" })];
+
+    const merged = mergeRetryResults(original, retried);
+
+    assert.equal(merged[1].id, 2);
+    assert.equal(merged[1].status, "error");
+  });
+
+  it("returns same count as original agents", () => {
+    const original = makeResult();
+    const retried = [
+      makeAgent({ id: 2, status: "success" }),
+      makeAgent({ id: 3, status: "success" }),
+    ];
+
+    const merged = mergeRetryResults(original, retried);
+    assert.equal(merged.length, original.agents.length);
+  });
+});