Skip to content

Commit ff2abd2

Browse files
KRRT7KRRT7
committed
chore: add eval scenarios for codeflash-skills tile
5 scenarios testing: sequential debugging, Result type + effort config, test patterns, domain type conventions, and deduplication/repair mechanics. Also adds tessl-labs/tessl-skill-eval-scenarios dev dependency.
1 parent 289b75c commit ff2abd2

19 files changed

Lines changed: 408 additions & 0 deletions

File tree

‎tessl.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,9 @@
7272
},
7373
"codeflash/codeflash-skills": {
7474
"version": "0.2.0"
75+
},
76+
"tessl-labs/tessl-skill-eval-scenarios": {
77+
"version": "0.0.5"
7578
}
7679
}
7780
}

‎tiles/codeflash-skills/evals/capabilities.json‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
{
2+
"package_name": "codeflash-skills",
3+
"total_capabilities": 14,
4+
"capabilities": [
5+
{
6+
"id": 0,
7+
"name": "sequential-pipeline-debugging",
8+
"description": "Debug optimization failures by walking through pipeline stages sequentially and stopping at the first failure found",
9+
"complexity": "intermediate",
10+
"api_elements": ["discovery", "ranking", "context", "AI service", "verification", "deduplication", "repair"]
11+
},
12+
{
13+
"id": 1,
14+
"name": "token-limit-awareness",
15+
"description": "Know that OPTIMIZATION_CONTEXT_TOKEN_LIMIT and TESTGEN_CONTEXT_TOKEN_LIMIT are both 16000 tokens and that exceeding them causes function rejection",
16+
"complexity": "basic",
17+
"api_elements": ["OPTIMIZATION_CONTEXT_TOKEN_LIMIT", "TESTGEN_CONTEXT_TOKEN_LIMIT", "encoded_tokens_len()"]
18+
},
19+
{
20+
"id": 2,
21+
"name": "improvement-threshold",
22+
"description": "Know that MIN_IMPROVEMENT_THRESHOLD is 0.05 (5%) and candidates below this speedup are rejected",
23+
"complexity": "basic",
24+
"api_elements": ["MIN_IMPROVEMENT_THRESHOLD", "STABILITY_WINDOW_SIZE"]
25+
},
26+
{
27+
"id": 3,
28+
"name": "ast-deduplication",
29+
"description": "Know that candidates are deduplicated via AST normalization using normalize_code() and CandidateEvaluationContext.ast_code_to_id",
30+
"complexity": "intermediate",
31+
"api_elements": ["normalize_code()", "CandidateEvaluationContext.ast_code_to_id", "code_utils/deduplicate_code.py"]
32+
},
33+
{
34+
"id": 4,
35+
"name": "repair-trigger-conditions",
36+
"description": "Know that repair only triggers when fewer than MIN_CORRECT_CANDIDATES=2 pass, and is skipped when REPAIR_UNMATCHED_PERCENTAGE_LIMIT is exceeded",
37+
"complexity": "advanced",
38+
"api_elements": ["MIN_CORRECT_CANDIDATES", "REPAIR_UNMATCHED_PERCENTAGE_LIMIT", "AIServiceCodeRepairRequest"]
39+
},
40+
{
41+
"id": 5,
42+
"name": "ai-service-error-patterns",
43+
"description": "Know specific log patterns to search for when AI service fails: 'Error generating optimized candidates', 'cli-optimize-error-caught', 'cli-optimize-error-response'",
44+
"complexity": "intermediate",
45+
"api_elements": ["AiServiceClient", "api/aiservice.py"]
46+
},
47+
{
48+
"id": 6,
49+
"name": "behavioral-vs-benchmark-failures",
50+
"description": "Distinguish between behavioral test failures (return value/stdout/pass-fail mismatches via TestDiffScope) and benchmark failures (speedup below threshold)",
51+
"complexity": "intermediate",
52+
"api_elements": ["TestDiffScope", "RETURN_VALUE", "STDOUT", "DID_PASS"]
53+
},
54+
{
55+
"id": 7,
56+
"name": "result-type-pattern",
57+
"description": "Use Result[L, R] from either.py with Success/Failure constructors and is_successful() checks before unwrap()",
58+
"complexity": "basic",
59+
"api_elements": ["Result", "Success", "Failure", "is_successful", "unwrap()", "either.py"]
60+
},
61+
{
62+
"id": 8,
63+
"name": "effort-config-pattern",
64+
"description": "Add effort-dependent config via EffortKeys enum, EFFORT_VALUES dict with LOW/MEDIUM/HIGH levels, and get_effort_value()",
65+
"complexity": "intermediate",
66+
"api_elements": ["EffortKeys", "EffortLevel", "EFFORT_VALUES", "get_effort_value()", "config_consts.py"]
67+
},
68+
{
69+
"id": 9,
70+
"name": "module-to-feature-mapping",
71+
"description": "Know which codeflash module to modify for different feature types (optimization/ for strategies, api/ for endpoints, languages/ for language support, etc.)",
72+
"complexity": "basic",
73+
"api_elements": ["MODULE_REFERENCE.md"]
74+
},
75+
{
76+
"id": 10,
77+
"name": "domain-type-conventions",
78+
"description": "Use @dataclass(frozen=True) for immutable data, BaseModel for serializable models, and keep function_types.py dependency-free",
79+
"complexity": "intermediate",
80+
"api_elements": ["@dataclass(frozen=True)", "BaseModel", "models/models.py", "models/function_types.py"]
81+
},
82+
{
83+
"id": 11,
84+
"name": "test-patterns",
85+
"description": "Use tmp_path fixture, .resolve() on Paths, .as_posix() for string conversion, full string equality assertions, and awareness of deterministic patches",
86+
"complexity": "basic",
87+
"api_elements": ["tmp_path", ".resolve()", ".as_posix()", "pytest_plugin.py"]
88+
},
89+
{
90+
"id": 12,
91+
"name": "quality-check-commands",
92+
"description": "Run uv run prek run for formatting/linting, uv run mypy for type checking, and uv run pytest for tests",
93+
"complexity": "basic",
94+
"api_elements": ["uv run prek run", "uv run mypy", "uv run pytest"]
95+
},
96+
{
97+
"id": 13,
98+
"name": "language-support-patterns",
99+
"description": "Use @register_language decorator, get_language_support() for lookup, singleton pattern via set_current_language()/current_language(), and is_python()/is_javascript() guards",
100+
"complexity": "advanced",
101+
"api_elements": ["@register_language", "get_language_support()", "set_current_language()", "is_python()", "is_javascript()"]
102+
}
103+
]
104+
}

‎tiles/codeflash-skills/evals/scenario-1/capability.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Sequential pipeline debugging with specific thresholds

‎tiles/codeflash-skills/evals/scenario-1/criteria.json‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{
2+
"context": "Tests whether the agent follows the sequential debugging workflow from the skill, checking pipeline stages in order and using correct threshold values when diagnosing an optimization that produced no results.",
3+
"type": "weighted_checklist",
4+
"checklist": [
5+
{
6+
"name": "Sequential stage order",
7+
"description": "Investigates pipeline stages in order: discovery before ranking before context before AI service before test failures. Does NOT jump to later stages without checking earlier ones first.",
8+
"max_score": 25
9+
},
10+
{
11+
"name": "Token limit value",
12+
"description": "References the specific token limit of 16000 for OPTIMIZATION_CONTEXT_TOKEN_LIMIT or TESTGEN_CONTEXT_TOKEN_LIMIT when checking context extraction",
13+
"max_score": 25
14+
},
15+
{
16+
"name": "Importance threshold",
17+
"description": "References DEFAULT_IMPORTANCE_THRESHOLD=0.001 when checking function ranking",
18+
"max_score": 25
19+
},
20+
{
21+
"name": "Stops at failure",
22+
"description": "Identifies the failing stage and focuses investigation there rather than continuing through all remaining stages",
23+
"max_score": 25
24+
}
25+
]
26+
}

‎tiles/codeflash-skills/evals/scenario-1/task.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Diagnose Silent Optimization Skip
2+
3+
## Context
4+
5+
A user reports that when running codeflash on their project, a specific function `calculate_metrics` in `analytics/processor.py` never appears in the optimization results. The function exists in the module root, is not in the exclude list, and has not been previously optimized. Trace data shows the function is called frequently but with very short execution times (averaging 0.0005 seconds total addressable time). The function has moderate dependencies.
6+
7+
## Task
8+
9+
Write a diagnostic report explaining why this function is being skipped and at which stage in the pipeline the function is filtered out. Include the specific threshold or condition that causes the skip.
10+
11+
## Expected Outputs
12+
13+
A markdown file `diagnostic-report.md` explaining the root cause.

‎tiles/codeflash-skills/evals/scenario-2/capability.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Result type pattern and effort-dependent configuration

‎tiles/codeflash-skills/evals/scenario-2/criteria.json‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
{
2+
"context": "Tests whether the agent uses the codeflash Result type pattern from either.py and the effort-dependent configuration pattern when implementing a new pipeline feature.",
3+
"type": "weighted_checklist",
4+
"checklist": [
5+
{
6+
"name": "Imports from either.py",
7+
"description": "Imports Success, Failure, and is_successful from codeflash.either (NOT from a different error handling module)",
8+
"max_score": 20
9+
},
10+
{
11+
"name": "Result return type",
12+
"description": "Function returns Result type using Success() for success and Failure() for errors, not exceptions or None",
13+
"max_score": 20
14+
},
15+
{
16+
"name": "is_successful check",
17+
"description": "Calls is_successful() or .is_successful() before calling unwrap() on the result",
18+
"max_score": 20
19+
},
20+
{
21+
"name": "EffortKeys enum entry",
22+
"description": "Adds a new entry to the EffortKeys enum in config_consts.py",
23+
"max_score": 20
24+
},
25+
{
26+
"name": "Three effort levels",
27+
"description": "Adds values for all three EffortLevel variants (LOW, MEDIUM, HIGH) in EFFORT_VALUES dict",
28+
"max_score": 20
29+
}
30+
]
31+
}

‎tiles/codeflash-skills/evals/scenario-2/task.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Add Candidate Timeout Feature
2+
3+
## Context
4+
5+
The codeflash optimization engine currently has no per-candidate timeout. Some candidates take too long during verification, wasting the optimization budget. A new feature is needed to skip candidates that exceed a configurable time limit during behavioral testing.
6+
7+
The timeout should vary based on the optimization effort setting — shorter timeouts for low effort runs (to save time) and longer for high effort runs (to allow more complex optimizations).
8+
9+
## Task
10+
11+
Implement a `check_candidate_timeout` function in `codeflash/optimization/function_optimizer.py` that:
12+
1. Takes a candidate runtime and returns whether the candidate should be skipped
13+
2. Uses a configurable timeout threshold that scales with optimization effort
14+
3. Handles the error case where the runtime measurement is unavailable
15+
16+
Also add the necessary configuration constant to `codeflash/code_utils/config_consts.py`.
17+
18+
## Expected Outputs
19+
20+
- Modified `function_optimizer.py` with the new function
21+
- Modified `config_consts.py` with the new configuration

‎tiles/codeflash-skills/evals/scenario-3/capability.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Test patterns and deterministic patch awareness

‎tiles/codeflash-skills/evals/scenario-3/criteria.json‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{
2+
"context": "Tests whether the agent follows codeflash test conventions when writing tests, including path handling, temp directory patterns, and awareness of the deterministic patching system.",
3+
"type": "weighted_checklist",
4+
"checklist": [
5+
{
6+
"name": "Uses tmp_path fixture",
7+
"description": "Test function uses pytest tmp_path fixture parameter, NOT tempfile.NamedTemporaryFile or tempfile.mkdtemp",
8+
"max_score": 25
9+
},
10+
{
11+
"name": "Calls resolve on paths",
12+
"description": "Calls .resolve() on Path objects before using them in assertions or function calls",
13+
"max_score": 25
14+
},
15+
{
16+
"name": "Full string equality",
17+
"description": "Uses exact equality assertions (== or assert_equal) for code string comparisons, NOT substring checks like 'in' or assertIn or contains",
18+
"max_score": 25
19+
},
20+
{
21+
"name": "No real time dependency",
22+
"description": "Test does NOT depend on real time.time(), datetime.now(), random values, or uuid generation for correctness. Acknowledges or accounts for deterministic patches if time/random values are involved.",
23+
"max_score": 25
24+
}
25+
]
26+
}

0 commit comments

Comments
 (0)