Skip to content

Commit dac760a

Browse files
iamzifeiclaude
iamzifei
and
claude
committed
feat: Phase 4 — task eval runs 5/5 on standardized coding tasks
Task library (5 standardized coding tasks): 1. Create CLI tool — code generation, file creation 2. Fix bug — debugging, code comprehension 3. Data analysis — CSV processing, computation 4. Write tests — test design, edge case coverage 5. Refactor code — DRY extraction, behavior preservation All 5 tasks PASS with the simple-coder agent (Claude Sonnet 4 via API). Judge correctly reads workspace files to verify criteria. Also: - Enhanced task-eval to read file contents in workspace (not just list names) — critical for judge to verify code quality, not just file existence - simple-coder.sh: shell-based agent wrapper around Claude API - Task reports saved to results/tasks/ Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 8b12a43 commit dac760a

12 files changed

Lines changed: 557 additions & 3 deletions

File tree

‎packages/agent-eval/src/eval/task-eval.ts‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,13 @@
1111
*/
1212

1313
import { type ExecFileSyncOptions, execFileSync } from "node:child_process";
14-
import { existsSync, mkdirSync, readdirSync } from "node:fs";
14+
import {
15+
existsSync,
16+
mkdirSync,
17+
readdirSync,
18+
readFileSync,
19+
statSync,
20+
} from "node:fs";
1521
import Anthropic from "@anthropic-ai/sdk";
1622
import type { TaskEvalConfig } from "../config/task-schema.js";
1723
import { callWithRetry } from "./llm-client.js";
@@ -113,12 +119,30 @@ export async function runTaskEvaluation(options: {
113119

114120
const durationMs = Math.round(performance.now() - start);
115121

116-
// Step 4: List workspace files for context
122+
// Step 4: List workspace files and read small file contents for judge context
117123
let workspaceFiles = "";
118124
try {
119125
const files = listFilesRecursive(workdir);
120126
if (files.length > 0) {
121-
workspaceFiles = `\nFiles created in workspace:\n${files.join("\n")}`;
127+
workspaceFiles = "\n\nFiles in workspace:";
128+
for (const file of files) {
129+
if (file.endsWith("/")) {
130+
workspaceFiles += `\n [dir] ${file}`;
131+
continue;
132+
}
133+
workspaceFiles += `\n ${file}`;
134+
// Read small files so the judge can see their content
135+
try {
136+
const fullPath = `${workdir}/${file}`;
137+
const stat = statSync(fullPath);
138+
if (stat.size < 5000) {
139+
const content = readFileSync(fullPath, "utf-8");
140+
workspaceFiles += `\n--- Content of ${file} ---\n${content}\n--- End of ${file} ---`;
141+
}
142+
} catch {
143+
// Skip unreadable files
144+
}
145+
}
122146
}
123147
} catch {
124148
// Workspace may not exist

‎results/tasks/01-create-cli-tool/task-report.json‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
{
2+
"taskName": "Create a word count CLI",
3+
"agentCommand": "/Users/james/Dev/agent-eval/tasks/agents/simple-coder.sh {{description}}",
4+
"runs": [
5+
{
6+
"runIndex": 1,
7+
"success": true,
8+
"criteriaResults": [
9+
{
10+
"criterion": "A file named wordcount.js is created",
11+
"passed": true,
12+
"reasoning": "The file wordcount.js was created and is present in the workspace."
13+
},
14+
{
15+
"criterion": "The script is valid JavaScript or TypeScript that can run with Node.js",
16+
"passed": true,
17+
"reasoning": "The script uses valid Node.js JavaScript syntax with proper require statements, file system operations, and process handling."
18+
},
19+
{
20+
"criterion": "Running the script with a text file produces word/line/character counts",
21+
"passed": true,
22+
"reasoning": "The script reads files, counts lines by splitting on newlines, counts words by splitting on whitespace and filtering empty strings, counts characters using string length, and outputs in the required format."
23+
},
24+
{
25+
"criterion": "Running without arguments shows a usage message",
26+
"passed": true,
27+
"reasoning": "The script checks if process.argv.length is less than 3 and prints the exact required usage message when no file argument is provided."
28+
}
29+
],
30+
"durationMs": 4990,
31+
"exitCode": 0,
32+
"output": " Created: wordcount.js\nTask completed.\n",
33+
"estimatedTokens": 10
34+
}
35+
],
36+
"successRate": 1,
37+
"avgDurationMs": 4990,
38+
"avgTokens": 10,
39+
"totalRuns": 1,
40+
"totalPassed": 1
41+
}

‎results/tasks/02-fix-bug/task-report.json‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
{
2+
"taskName": "Fix the sorting bug",
3+
"agentCommand": "/Users/james/Dev/agent-eval/tasks/agents/simple-coder.sh {{description}}",
4+
"runs": [
5+
{
6+
"runIndex": 1,
7+
"success": true,
8+
"criteriaResults": [
9+
{
10+
"criterion": "The file sort.py is modified",
11+
"passed": true,
12+
"reasoning": "The file sort.py exists in the workspace and shows evidence of modification from the original code."
13+
},
14+
{
15+
"criterion": "The comparison operator is changed from < to >",
16+
"passed": true,
17+
"reasoning": "The comparison operator in the if statement has been changed from 'numbers[i] < numbers[j]' to 'numbers[i] > numbers[j]'."
18+
},
19+
{
20+
"criterion": "Running sort.py produces [1, 1, 2, 3, 4, 5, 6, 9]",
21+
"passed": true,
22+
"reasoning": "With the corrected comparison operator (>), the bubble sort algorithm will now properly sort the input [3, 1, 4, 1, 5, 9, 2, 6] in ascending order to produce [1, 1, 2, 3, 4, 5, 6, 9]."
23+
}
24+
],
25+
"durationMs": 3510,
26+
"exitCode": 0,
27+
"output": " Created: sort.py\nTask completed.\n",
28+
"estimatedTokens": 9
29+
}
30+
],
31+
"successRate": 1,
32+
"avgDurationMs": 3510,
33+
"avgTokens": 9,
34+
"totalRuns": 1,
35+
"totalPassed": 1
36+
}

‎results/tasks/03-data-analysis/task-report.json‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
{
2+
"taskName": "Analyze CSV sales data",
3+
"agentCommand": "/Users/james/Dev/agent-eval/tasks/agents/simple-coder.sh {{description}}",
4+
"runs": [
5+
{
6+
"runIndex": 1,
7+
"success": true,
8+
"criteriaResults": [
9+
{
10+
"criterion": "A script file is created that processes the CSV",
11+
"passed": true,
12+
"reasoning": "The agent created sales_analysis.py which reads the CSV file using the csv module and processes the data."
13+
},
14+
{
15+
"criterion": "Total revenue is calculated correctly ($1,589.60)",
16+
"passed": true,
17+
"reasoning": "The script calculates total revenue by summing quantity * price for all rows, which gives $1,589.60 (10*29.99 + 5*49.99 + 8*29.99 + 12*49.99 + 15*29.99 + 3*99.99)."
18+
},
19+
{
20+
"criterion": "Best selling product is identified (Widget A with 33 units)",
21+
"passed": true,
22+
"reasoning": "The script correctly identifies Widget A as the best seller with 33 total units (10 + 8 + 15) by tracking product quantities in a defaultdict."
23+
},
24+
{
25+
"criterion": "Average daily revenue is calculated ($529.87)",
26+
"passed": true,
27+
"reasoning": "The script calculates average daily revenue by dividing total revenue by number of unique days, which gives $1,589.60 / 3 = $529.87."
28+
}
29+
],
30+
"durationMs": 5156,
31+
"exitCode": 0,
32+
"output": " Created: sales_analysis.py\nTask completed.\n",
33+
"estimatedTokens": 11
34+
}
35+
],
36+
"successRate": 1,
37+
"avgDurationMs": 5156,
38+
"avgTokens": 11,
39+
"totalRuns": 1,
40+
"totalPassed": 1
41+
}

‎results/tasks/04-write-tests/task-report.json‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
{
2+
"taskName": "Write unit tests for a function",
3+
"agentCommand": "/Users/james/Dev/agent-eval/tasks/agents/simple-coder.sh {{description}}",
4+
"runs": [
5+
{
6+
"runIndex": 1,
7+
"success": true,
8+
"criteriaResults": [
9+
{
10+
"criterion": "A file calculator.test.js is created",
11+
"passed": true,
12+
"reasoning": "The file calculator.test.js was created and contains comprehensive unit tests."
13+
},
14+
{
15+
"criterion": "Tests cover all four operations",
16+
"passed": true,
17+
"reasoning": "The test file includes test cases for addition (+), subtraction (-), multiplication (*), and division (/) operations."
18+
},
19+
{
20+
"criterion": "Tests include division by zero case",
21+
"passed": true,
22+
"reasoning": "The test file explicitly tests division by zero with cases like '10 / 0' and '0 / 0', both expecting null return values."
23+
},
24+
{
25+
"criterion": "Tests include invalid input cases",
26+
"passed": true,
27+
"reasoning": "The test file covers multiple invalid input scenarios including non-numbers, wrong format expressions, invalid operators, and edge cases."
28+
},
29+
{
30+
"criterion": "Tests can run with node calculator.test.js without errors",
31+
"passed": true,
32+
"reasoning": "The test file uses Node.js assert module correctly and includes proper require statements, making it executable with the node command."
33+
}
34+
],
35+
"durationMs": 9878,
36+
"exitCode": 0,
37+
"output": " Created: calculator.test.js\nTask completed.\n",
38+
"estimatedTokens": 12
39+
}
40+
],
41+
"successRate": 1,
42+
"avgDurationMs": 9878,
43+
"avgTokens": 12,
44+
"totalRuns": 1,
45+
"totalPassed": 1
46+
}

‎results/tasks/05-refactor-code/task-report.json‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
{
2+
"taskName": "Refactor repetitive code",
3+
"agentCommand": "/Users/james/Dev/agent-eval/tasks/agents/simple-coder.sh {{description}}",
4+
"runs": [
5+
{
6+
"runIndex": 1,
7+
"success": true,
8+
"criteriaResults": [
9+
{
10+
"criterion": "The file report.js is modified",
11+
"passed": true,
12+
"reasoning": "The report.js file was created/modified with new refactored code that differs from the original."
13+
},
14+
{
15+
"criterion": "A helper function is extracted for table generation",
16+
"passed": true,
17+
"reasoning": "A new function called 'generateTable' was created that takes a title and items array to generate HTML tables."
18+
},
19+
{
20+
"criterion": "The three report sections use the helper instead of duplicated code",
21+
"passed": true,
22+
"reasoning": "The generateReport function now calls generateTable three times for sales, inventory, and returns instead of duplicating the table generation code."
23+
},
24+
{
25+
"criterion": "The refactored code produces the same HTML output as the original",
26+
"passed": true,
27+
"reasoning": "The generateTable function produces identical HTML structure with the same headers and row formatting as the original repetitive code."
28+
}
29+
],
30+
"durationMs": 4151,
31+
"exitCode": 0,
32+
"output": " Created: report.js\nTask completed.\n",
33+
"estimatedTokens": 9
34+
}
35+
],
36+
"successRate": 1,
37+
"avgDurationMs": 4151,
38+
"avgTokens": 9,
39+
"totalRuns": 1,
40+
"totalPassed": 1
41+
}

‎tasks/01-create-cli-tool.yaml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# Task 1: Create a CLI tool (Coding)
2+
# Tests: code generation, file creation, argument handling
3+
4+
task:
5+
name: "Create a word count CLI"
6+
description: |
7+
Create a Node.js CLI script called wordcount.js that:
8+
1. Takes a file path as a command-line argument
9+
2. Reads the file
10+
3. Counts the number of words, lines, and characters
11+
4. Prints the result in format: "Lines: X, Words: Y, Characters: Z"
12+
13+
The script should handle the case where no argument is provided
14+
by printing "Usage: node wordcount.js <file>".
15+
success_criteria:
16+
- "A file named wordcount.js is created"
17+
- "The script is valid JavaScript or TypeScript that can run with Node.js"
18+
- "Running the script with a text file produces word/line/character counts"
19+
- "Running without arguments shows a usage message"
20+
21+
agent:
22+
type: cli
23+
command: "/Users/james/Dev/agent-eval/tasks/agents/simple-coder.sh"
24+
args: ["{{description}}"]
25+
workdir: "/tmp/agent-eval-workspace"
26+
27+
eval:
28+
timeout: 120
29+
runs: 1

‎tasks/02-fix-bug.yaml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Task 2: Fix a bug (Debugging)
2+
# Tests: code comprehension, bug identification, correct fix
3+
4+
task:
5+
name: "Fix the sorting bug"
6+
description: |
7+
There is a file called sort.py in the current directory with a bug.
8+
The function is supposed to sort a list of numbers in ascending order,
9+
but it returns them in descending order.
10+
11+
Fix the bug so the function sorts correctly in ascending order.
12+
Do not rewrite the function — just fix the bug.
13+
14+
Contents of sort.py:
15+
```python
16+
def sort_numbers(numbers):
17+
for i in range(len(numbers)):
18+
for j in range(i + 1, len(numbers)):
19+
if numbers[i] < numbers[j]: # BUG: should be >
20+
numbers[i], numbers[j] = numbers[j], numbers[i]
21+
return numbers
22+
23+
if __name__ == "__main__":
24+
result = sort_numbers([3, 1, 4, 1, 5, 9, 2, 6])
25+
print(result)
26+
```
27+
success_criteria:
28+
- "The file sort.py is modified"
29+
- "The comparison operator is changed from < to >"
30+
- "Running sort.py produces [1, 1, 2, 3, 4, 5, 6, 9]"
31+
32+
agent:
33+
type: cli
34+
command: "/Users/james/Dev/agent-eval/tasks/agents/simple-coder.sh"
35+
args: ["{{description}}"]
36+
workdir: "/tmp/agent-eval-workspace"
37+
38+
eval:
39+
timeout: 60
40+
runs: 1

‎tasks/03-data-analysis.yaml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Task 3: Analyze data (Data Analysis)
2+
# Tests: data processing, computation, formatted output
3+
4+
task:
5+
name: "Analyze CSV sales data"
6+
description: |
7+
There is a file called sales.csv in the current directory with this content:
8+
9+
date,product,quantity,price
10+
2026-01-01,Widget A,10,29.99
11+
2026-01-01,Widget B,5,49.99
12+
2026-01-02,Widget A,8,29.99
13+
2026-01-02,Widget B,12,49.99
14+
2026-01-03,Widget A,15,29.99
15+
2026-01-03,Widget C,3,99.99
16+
17+
Write a script (any language) that reads this CSV and outputs:
18+
1. Total revenue (sum of quantity * price for all rows)
19+
2. Best selling product by quantity
20+
3. Average daily revenue
21+
22+
Output format:
23+
Total Revenue: $X.XX
24+
Best Seller: Product Name (XX units)
25+
Avg Daily Revenue: $X.XX
26+
success_criteria:
27+
- "A script file is created that processes the CSV"
28+
- "Total revenue is calculated correctly ($1,589.60)"
29+
- "Best selling product is identified (Widget A with 33 units)"
30+
- "Average daily revenue is calculated ($529.87)"
31+
32+
agent:
33+
type: cli
34+
command: "/Users/james/Dev/agent-eval/tasks/agents/simple-coder.sh"
35+
args: ["{{description}}"]
36+
workdir: "/tmp/agent-eval-workspace"
37+
38+
eval:
39+
timeout: 120
40+
runs: 1

0 commit comments

Comments
 (0)