Skip to content

Commit e329d52

Browse files
Codeflash Botclaude
andcommitted
fix: Improve error messaging for files excluded from Vitest coverage
## Problem When a file is excluded from coverage by vitest.config.ts (e.g., via `coverage.exclude: ["src/agents/**"]`), Codeflash reports misleading "Test coverage is 0.0%" messages even though tests run successfully. This happens because: - Vitest doesn't include excluded files in coverage-final.json - Codeflash detects this (status = NOT_FOUND) but shows generic 0% message - Users don't know the file is excluded from coverage collection ## Solution Detect when coverage status is NOT_FOUND and provide a clear, actionable error message explaining: 1. No coverage data was found for the file 2. It may be excluded by test framework configuration 3. Where to check (coverage.exclude in vitest.config.ts, etc.) ## Changes - function_optimizer.py: Check CoverageStatus.NOT_FOUND before reporting 0% - Added clear warning log and user-facing error message - New test file: test_vitest_coverage_exclusions.py ## Testing - All existing JavaScript tests pass - New tests verify NOT_FOUND status is returned correctly - Manual verification with openclaw logs (trace: 2a84fe6b-9871-4916-96da-bdd79bca508a) Fixes #BUG-1 (from autoresearch:debug workflow) Trace IDs affected: All 10 log files showing 0% coverage in /workspace/logs Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
1 parent f86fe2d commit e329d52

2 files changed

Lines changed: 144 additions & 0 deletions

File tree

codeflash/languages/function_optimizer.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2787,6 +2787,25 @@ def establish_original_code_baseline(
27872787
did_pass_all_tests = all(result.did_pass for result in behavioral_results)
27882788
if not did_pass_all_tests:
27892789
return Failure("Tests failed to pass for the original code.")
2790+
2791+
# Check if coverage data was not found (file excluded from coverage)
2792+
from codeflash.models.models import CoverageStatus
2793+
2794+
if coverage_results and coverage_results.status == CoverageStatus.NOT_FOUND:
2795+
# File was not found in coverage data - likely excluded by test framework config
2796+
logger.warning(
2797+
f"No coverage data found for {self.function_to_optimize.source_file_path}. "
2798+
f"This file may be excluded from coverage collection by your test framework configuration "
2799+
f"(e.g., coverage.exclude in vitest.config.ts for Vitest, or testMatch/coveragePathIgnorePatterns "
2800+
f"for Jest). Tests ran successfully but coverage cannot be measured."
2801+
)
2802+
return Failure(
2803+
f"Coverage data not found for {self.function_to_optimize.source_file_path}. "
2804+
f"The file may be excluded from coverage by your test framework config. "
2805+
f"Check coverage.exclude patterns in vitest.config.ts or jest.config.js."
2806+
)
2807+
2808+
# Normal coverage failure (tests ran but coverage below threshold)
27902809
coverage_pct = coverage_results.coverage if coverage_results else 0
27912810
return Failure(
27922811
f"Test coverage is {coverage_pct}%, which is below the required threshold of {COVERAGE_THRESHOLD}%."
Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
"""Tests for handling Vitest coverage exclusions.
2+
3+
These tests verify that Codeflash correctly detects and handles files
4+
that are excluded from coverage by vitest.config.ts, preventing false
5+
0% coverage reports.
6+
"""
7+
8+
from __future__ import annotations
9+
10+
import json
11+
import tempfile
12+
from pathlib import Path
13+
14+
import pytest
15+
16+
from codeflash.models.models import CodeOptimizationContext, CoverageStatus
17+
from codeflash.verification.coverage_utils import JestCoverageUtils
18+
19+
20+
class TestVitestCoverageExclusions:
21+
"""Tests for Vitest coverage exclusion handling."""
22+
23+
def test_missing_coverage_returns_not_found_status(self) -> None:
24+
"""Should return NOT_FOUND status when file is not in coverage data.
25+
26+
When a file is excluded from Vitest coverage (via coverage.exclude),
27+
it won't appear in coverage-final.json. Codeflash should return
28+
NOT_FOUND status (not PARSED_SUCCESSFULLY).
29+
30+
This test verifies the current behavior is correct at the coverage
31+
parsing level. The issue is at a higher level (function_optimizer.py)
32+
where NOT_FOUND status needs better handling.
33+
"""
34+
with tempfile.TemporaryDirectory() as tmp_dir:
35+
tmp_path = Path(tmp_dir)
36+
37+
# Create mock coverage-final.json that's missing the target file
38+
coverage_file = tmp_path / "coverage-final.json"
39+
coverage_data = {
40+
"/workspace/project/src/utils/helpers.ts": {
41+
"fnMap": {},
42+
"s": {},
43+
},
44+
# src/agents/sandbox/fs-paths.ts is NOT here (excluded by Vitest)
45+
}
46+
with coverage_file.open("w") as f:
47+
json.dump(coverage_data, f)
48+
49+
# Try to load coverage for a missing file
50+
missing_file = Path("/workspace/project/src/agents/sandbox/fs-paths.ts")
51+
from codeflash.models.models import CodeStringsMarkdown
52+
53+
mock_context = CodeOptimizationContext(
54+
testgen_context=CodeStringsMarkdown(language="typescript"),
55+
read_writable_code=CodeStringsMarkdown(language="typescript"),
56+
helper_functions=[],
57+
preexisting_objects=set(),
58+
)
59+
60+
result = JestCoverageUtils.load_from_jest_json(
61+
coverage_json_path=coverage_file,
62+
function_name="parseSandboxBindMount",
63+
code_context=mock_context,
64+
source_code_path=missing_file,
65+
)
66+
67+
# Should return NOT_FOUND when file not in coverage
68+
assert result.status == CoverageStatus.NOT_FOUND, (
69+
f"Expected NOT_FOUND for missing file, got {result.status}"
70+
)
71+
assert result.coverage == 0.0
72+
73+
def test_handles_included_file_normally(self) -> None:
74+
"""Should handle files that ARE included in coverage normally.
75+
76+
This test verifies that the fix doesn't break normal coverage parsing
77+
for files that are NOT excluded.
78+
"""
79+
with tempfile.TemporaryDirectory() as tmp_dir:
80+
tmp_path = Path(tmp_dir)
81+
82+
# Create mock coverage-final.json with a valid file
83+
coverage_file = tmp_path / "coverage-final.json"
84+
test_file = "/workspace/project/src/utils/helpers.ts"
85+
coverage_data = {
86+
test_file: {
87+
"fnMap": {
88+
"0": {"name": "someHelper", "loc": {"start": {"line": 1}, "end": {"line": 5}}}
89+
},
90+
"statementMap": {
91+
"0": {"start": {"line": 2}, "end": {"line": 2}},
92+
"1": {"start": {"line": 3}, "end": {"line": 3}},
93+
},
94+
"s": {"0": 5, "1": 5}, # Both statements executed
95+
"branchMap": {},
96+
"b": {},
97+
}
98+
}
99+
with coverage_file.open("w") as f:
100+
json.dump(coverage_data, f)
101+
102+
source_file = Path(test_file)
103+
from codeflash.models.models import CodeStringsMarkdown
104+
105+
mock_context = CodeOptimizationContext(
106+
testgen_context=CodeStringsMarkdown(language="typescript"),
107+
read_writable_code=CodeStringsMarkdown(language="typescript"),
108+
helper_functions=[],
109+
preexisting_objects=set(),
110+
)
111+
112+
result = JestCoverageUtils.load_from_jest_json(
113+
coverage_json_path=coverage_file,
114+
function_name="someHelper",
115+
code_context=mock_context,
116+
source_code_path=source_file,
117+
)
118+
119+
# Should parse successfully for non-excluded files
120+
assert result.status == CoverageStatus.PARSED_SUCCESSFULLY
121+
assert result.coverage > 0.0 # Should have actual coverage
122+
123+
124+
if __name__ == "__main__":
125+
pytest.main([__file__, "-v"])

0 commit comments

Comments
 (0)