Improve convergence analysis with diff-content comparison (#17)

- Add diff parser: extract added/removed lines per file from unified diffs
- Compute Jaccard similarity on added lines for pairwise agent comparison
- Single-linkage clustering groups agents by diff similarity (threshold 0.3)
- Convergence score now combines group ratio (50%) + avg diff similarity (50%)
- 10 new tests for diff parsing, similarity, pairwise comparison
- Updated convergence tests to use realistic diff content

Closes #4

Co-authored-by: unknown <that-github-user@github.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: 3 people

src/scoring/convergence.test.ts

@@ -3,15 +3,34 @@ import { describe, it } from "node:test";
 import type { AgentResult } from "../types.js";
 import { analyzeConvergence, recommend } from "./convergence.js";
 
+const DIFF_A = `diff --git a/a.ts b/a.ts
+--- a/a.ts
++++ b/a.ts
+@@ -1 +1 @@
++const x = 1;`;
+
+const DIFF_A_VARIANT = `diff --git a/a.ts b/a.ts
+--- a/a.ts
++++ b/a.ts
+@@ -1 +1 @@
++const x = 1;
++const extra = true;`;
+
+const DIFF_B = `diff --git a/b.ts b/b.ts
+--- a/b.ts
++++ b/b.ts
+@@ -1 +1 @@
++const y = 2;`;
+
 function makeAgent(overrides: Partial<AgentResult> & { id: number }): AgentResult {
   return {
     worktree: `/tmp/agent-${overrides.id}`,
     status: "success",
     exitCode: 0,
     duration: 5000,
     output: "",
-    diff: "some diff",
-    filesChanged: ["src/index.ts"],
+    diff: DIFF_A,
+    filesChanged: ["a.ts"],
     linesAdded: 10,
     linesRemoved: 5,
     ...overrides,
@@ -33,44 +52,41 @@ describe("analyzeConvergence", () => {
     assert.deepEqual(result, []);
   });
 
-  it("groups agents that changed the same files", () => {
+  it("groups agents with similar diffs together", () => {
     const agents = [
-      makeAgent({ id: 1, filesChanged: ["a.ts", "b.ts"] }),
-      makeAgent({ id: 2, filesChanged: ["a.ts", "b.ts"] }),
-      makeAgent({ id: 3, filesChanged: ["c.ts"] }),
+      makeAgent({ id: 1, diff: DIFF_A, filesChanged: ["a.ts"] }),
+      makeAgent({ id: 2, diff: DIFF_A, filesChanged: ["a.ts"] }),
+      makeAgent({ id: 3, diff: DIFF_B, filesChanged: ["b.ts"] }),
     ];
     const groups = analyzeConvergence(agents);
 
     assert.equal(groups.length, 2);
-    assert.deepEqual(groups[0]!.agents, [1, 2]);
-    assert.ok(groups[0]!.similarity > groups[1]!.similarity);
-    assert.deepEqual(groups[1]!.agents, [3]);
+    // Agents 1,2 have identical diffs — should be in the same group
+    const largestGroup = groups[0]!;
+    assert.ok(largestGroup.agents.includes(1));
+    assert.ok(largestGroup.agents.includes(2));
+    assert.ok(largestGroup.similarity > groups[1]!.similarity);
   });
 
-  it("handles file order differences", () => {
-    const agents = [
-      makeAgent({ id: 1, filesChanged: ["b.ts", "a.ts"] }),
-      makeAgent({ id: 2, filesChanged: ["a.ts", "b.ts"] }),
-    ];
+  it("clusters agents with identical diffs", () => {
+    const agents = [makeAgent({ id: 1, diff: DIFF_A }), makeAgent({ id: 2, diff: DIFF_A })];
     const groups = analyzeConvergence(agents);
 
     assert.equal(groups.length, 1);
-    assert.deepEqual(groups[0]!.agents, [1, 2]);
-    assert.equal(groups[0]!.similarity, 1);
+    assert.deepEqual(groups[0]!.agents.sort(), [1, 2]);
   });
 
-  it("labels strong consensus at 80%+", () => {
+  it("labels strong consensus correctly", () => {
     const agents = [
-      makeAgent({ id: 1, filesChanged: ["a.ts"] }),
-      makeAgent({ id: 2, filesChanged: ["a.ts"] }),
-      makeAgent({ id: 3, filesChanged: ["a.ts"] }),
-      makeAgent({ id: 4, filesChanged: ["a.ts"] }),
-      makeAgent({ id: 5, filesChanged: ["b.ts"] }),
+      makeAgent({ id: 1, diff: DIFF_A }),
+      makeAgent({ id: 2, diff: DIFF_A }),
+      makeAgent({ id: 3, diff: DIFF_A }),
+      makeAgent({ id: 4, diff: DIFF_A }),
+      makeAgent({ id: 5, diff: DIFF_B, filesChanged: ["b.ts"] }),
     ];
     const groups = analyzeConvergence(agents);
 
     assert.ok(groups[0]!.description.includes("Strong consensus"));
-    assert.ok(groups[1]!.description.includes("Divergent"));
   });
 });
 
@@ -82,8 +98,8 @@ describe("recommend", () => {
 
   it("prefers agents that pass tests", () => {
     const agents = [
-      makeAgent({ id: 1, linesAdded: 20, linesRemoved: 10 }),
-      makeAgent({ id: 2, linesAdded: 5, linesRemoved: 2 }),
+      makeAgent({ id: 1, diff: DIFF_A, linesAdded: 20, linesRemoved: 10 }),
+      makeAgent({ id: 2, diff: DIFF_B, linesAdded: 5, linesRemoved: 2, filesChanged: ["b.ts"] }),
     ];
     const tests = [
       { agentId: 1, passed: true },
@@ -96,9 +112,15 @@ describe("recommend", () => {
 
   it("prefers agents in larger convergence group when tests are equal", () => {
     const agents = [
-      makeAgent({ id: 1, filesChanged: ["a.ts"], linesAdded: 10, linesRemoved: 5 }),
-      makeAgent({ id: 2, filesChanged: ["a.ts"], linesAdded: 10, linesRemoved: 5 }),
-      makeAgent({ id: 3, filesChanged: ["b.ts"], linesAdded: 10, linesRemoved: 5 }),
+      makeAgent({ id: 1, diff: DIFF_A, filesChanged: ["a.ts"], linesAdded: 10, linesRemoved: 5 }),
+      makeAgent({ id: 2, diff: DIFF_A, filesChanged: ["a.ts"], linesAdded: 10, linesRemoved: 5 }),
+      makeAgent({
+        id: 3,
+        diff: DIFF_B,
+        filesChanged: ["b.ts"],
+        linesAdded: 10,
+        linesRemoved: 5,
+      }),
     ];
     const tests = [
       { agentId: 1, passed: true },
@@ -114,12 +136,11 @@ describe("recommend", () => {
 
   it("prefers smaller diffs as tiebreaker", () => {
     const agents = [
-      makeAgent({ id: 1, filesChanged: ["a.ts"], linesAdded: 50, linesRemoved: 20 }),
-      makeAgent({ id: 2, filesChanged: ["a.ts"], linesAdded: 5, linesRemoved: 2 }),
+      makeAgent({ id: 1, diff: DIFF_A, linesAdded: 50, linesRemoved: 20 }),
+      makeAgent({ id: 2, diff: DIFF_A, linesAdded: 5, linesRemoved: 2 }),
     ];
     const convergence = analyzeConvergence(agents);
 
-    // No test results — convergence is equal, so diff size decides
     assert.equal(recommend(agents, [], convergence), 2);
   });
 });

src/scoring/convergence.ts

@@ -1,54 +1,124 @@
 import type { AgentResult, ConvergenceGroup } from "../types.js";
+import { pairwiseSimilarity } from "./diff-parser.js";
 
 /**
- * Analyze convergence across agent results by comparing which files
- * each agent changed. Agents that changed the same set of files are
- * grouped together — a larger group = higher confidence.
+ * Analyze convergence across agent results using two levels:
+ * 1. File-level: which files did each agent change?
+ * 2. Diff-level: how similar are the actual changes? (Jaccard on added lines)
+ *
+ * Agents are clustered by diff similarity using single-linkage clustering
+ * with a 0.5 similarity threshold.
  */
 export function analyzeConvergence(agents: AgentResult[]): ConvergenceGroup[] {
   const completed = agents.filter((a) => a.status === "success" && a.diff.length > 0);
 
   if (completed.length === 0) return [];
 
-  // Group by file-change fingerprint
-  const fingerprints = new Map<string, number[]>();
-
-  for (const agent of completed) {
-    const key = [...agent.filesChanged].sort().join("|");
-    const group = fingerprints.get(key) ?? [];
-    group.push(agent.id);
-    fingerprints.set(key, group);
-  }
-
-  // Convert to convergence groups, sorted by size (largest first)
-  const groups: ConvergenceGroup[] = [];
+  // Compute pairwise diff similarity
+  const similarities = pairwiseSimilarity(completed.map((a) => ({ id: a.id, diff: a.diff })));
+
+  // Single-linkage clustering: merge agents with similarity >= threshold
+  const SIMILARITY_THRESHOLD = 0.3;
+  const clusters = clusterAgents(
+    completed.map((a) => a.id),
+    similarities,
+    SIMILARITY_THRESHOLD,
+  );
+
+  // Convert clusters to convergence groups
+  const groups: ConvergenceGroup[] = clusters.map((cluster) => {
+    // Average similarity within cluster
+    let totalSim = 0;
+    let pairs = 0;
+    for (let i = 0; i < cluster.length; i++) {
+      for (let j = i + 1; j < cluster.length; j++) {
+        const key = `${Math.min(cluster[i]!, cluster[j]!)}-${Math.max(cluster[i]!, cluster[j]!)}`;
+        totalSim += similarities.get(key) ?? 0;
+        pairs++;
+      }
+    }
+    const avgSimilarity = pairs > 0 ? totalSim / pairs : 1;
+    const groupRatio = cluster.length / completed.length;
+
+    // Collect files changed by this cluster
+    const filesSet = new Set<string>();
+    for (const agentId of cluster) {
+      const agent = completed.find((a) => a.id === agentId);
+      if (agent) {
+        for (const f of agent.filesChanged) filesSet.add(f);
+      }
+    }
 
-  for (const [key, agentIds] of fingerprints) {
-    const files = key.split("|").filter(Boolean);
-    const similarity = agentIds.length / completed.length;
+    // Combine file-level ratio with diff-level similarity for final score
+    const similarity = groupRatio * 0.5 + avgSimilarity * 0.5;
 
     let description: string;
-    if (similarity >= 0.8) {
-      description = `Strong consensus — ${agentIds.length}/${completed.length} agents changed the same files`;
-    } else if (similarity >= 0.5) {
-      description = `Moderate agreement — ${agentIds.length}/${completed.length} agents took a similar approach`;
+    if (groupRatio >= 0.8 && avgSimilarity >= 0.5) {
+      description = `Strong consensus — ${cluster.length}/${completed.length} agents made similar changes`;
+    } else if (groupRatio >= 0.5 || avgSimilarity >= 0.5) {
+      description = `Moderate agreement — ${cluster.length}/${completed.length} agents took a similar approach`;
     } else {
-      description = `Divergent approach — ${agentIds.length}/${completed.length} agents went a different direction`;
+      description = `Divergent approach — ${cluster.length}/${completed.length} agents went a different direction`;
     }
 
-    groups.push({
-      agents: agentIds,
+    return {
+      agents: cluster,
       similarity,
-      filesChanged: files,
+      filesChanged: [...filesSet].sort(),
       description,
-    });
-  }
+    };
+  });
 
   groups.sort((a, b) => b.similarity - a.similarity);
-
   return groups;
 }
 
+/**
+ * Single-linkage clustering. Two agents are in the same cluster if
+ * ANY pair within the cluster has similarity >= threshold.
+ */
+function clusterAgents(
+  agentIds: number[],
+  similarities: Map<string, number>,
+  threshold: number,
+): number[][] {
+  // Union-Find
+  const parent = new Map<number, number>();
+  for (const id of agentIds) parent.set(id, id);
+
+  function find(x: number): number {
+    let root = x;
+    while (parent.get(root) !== root) root = parent.get(root)!;
+    parent.set(x, root); // path compression
+    return root;
+  }
+
+  function union(a: number, b: number): void {
+    const ra = find(a);
+    const rb = find(b);
+    if (ra !== rb) parent.set(ra, rb);
+  }
+
+  // Merge agents with similarity >= threshold
+  for (const [key, sim] of similarities) {
+    if (sim >= threshold) {
+      const [a, b] = key.split("-").map(Number) as [number, number];
+      union(a, b);
+    }
+  }
+
+  // Collect clusters
+  const clusters = new Map<number, number[]>();
+  for (const id of agentIds) {
+    const root = find(id);
+    const cluster = clusters.get(root) ?? [];
+    cluster.push(id);
+    clusters.set(root, cluster);
+  }
+
+  return [...clusters.values()];
+}
+
 /**
  * Recommend the best agent based on test results and convergence.
  * Priority: passing tests > convergence group size > smaller diff.
@@ -61,7 +131,6 @@ export function recommend(
   const completed = agents.filter((a) => a.status === "success" && a.diff.length > 0);
   if (completed.length === 0) return null;
 
-  // Score each agent
   const scores = new Map<number, number>();
 
   for (const agent of completed) {
@@ -83,7 +152,6 @@ export function recommend(
     scores.set(agent.id, score);
   }
 
-  // Return highest-scoring agent
   let bestId: number | null = null;
   let bestScore = -1;
   for (const [id, score] of scores) {