Skip to content

Commit 933a260

Browse files
Fix: Make coverage error messages framework-agnostic
Error messages in coverage_utils.py hardcoded "Jest" even when the test framework was Vitest. This caused confusion in logs when Vitest tests failed (e.g., "Jest coverage file not found" when using Vitest). The JestCoverageUtils class is used for both Jest and Vitest since they share the same Istanbul/v8 coverage format. Error messages should be framework-agnostic. Changes: - "Jest coverage file not found" → "JavaScript coverage file not found" - "Failed to parse Jest coverage file" → "Failed to parse JavaScript coverage file" - "No coverage data found for X in Jest coverage" → "No coverage data found for X in JavaScript coverage" - "Function X not found in Jest fnMap" → "Function X not found in JavaScript fnMap" Affected trace IDs: 37e5a406, 735555fa, 940dfe80, c1e1de0e, dbec6c33, de96b1ab, fcf08c6b (7 logs from Apr 4 00:50 batch) Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
1 parent 8d51e2d commit 933a260

2 files changed

Lines changed: 95 additions & 4 deletions

File tree

codeflash/verification/coverage_utils.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,14 +43,14 @@ def load_from_jest_json(
4343
4444
"""
4545
if not coverage_json_path or not coverage_json_path.exists():
46-
logger.debug(f"Jest coverage file not found: {coverage_json_path}")
46+
logger.debug(f"JavaScript coverage file not found: {coverage_json_path}")
4747
return CoverageData.create_empty(source_code_path, function_name, code_context)
4848

4949
try:
5050
with coverage_json_path.open(encoding="utf-8") as f:
5151
coverage_data = json.load(f)
5252
except (json.JSONDecodeError, OSError) as e:
53-
logger.warning(f"Failed to parse Jest coverage file: {e}")
53+
logger.warning(f"Failed to parse JavaScript coverage file: {e}")
5454
return CoverageData.create_empty(source_code_path, function_name, code_context)
5555

5656
# Find the file entry in coverage data
@@ -66,7 +66,7 @@ def load_from_jest_json(
6666
break
6767

6868
if not file_coverage:
69-
logger.debug(f"No coverage data found for {source_code_path} in Jest coverage")
69+
logger.debug(f"No coverage data found for {source_code_path} in JavaScript coverage")
7070
return CoverageData.create_empty(source_code_path, function_name, code_context)
7171

7272
# Extract line coverage from statement map and execution counts
@@ -94,7 +94,7 @@ def load_from_jest_json(
9494
# If function not found in fnMap, use entire file
9595
fn_start_line = 1
9696
fn_end_line = 999999
97-
logger.debug(f"Function {function_name} not found in Jest fnMap, using file coverage")
97+
logger.debug(f"Function {function_name} not found in JavaScript fnMap, using file coverage")
9898

9999
# Calculate executed and unexecuted lines within the function
100100
executed_lines = []
Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
"""Test that coverage error messages are framework-agnostic."""
2+
3+
import tempfile
4+
from pathlib import Path
5+
from unittest.mock import MagicMock
6+
7+
import pytest
8+
9+
from codeflash.languages.language_enum import Language
10+
from codeflash.models.models import CodeOptimizationContext
11+
from codeflash.verification.coverage_utils import JestCoverageUtils
12+
13+
14+
class TestCoverageUtilsFrameworkAgnostic:
15+
"""Test that error messages don't hardcode 'Jest' when used for Vitest."""
16+
17+
def test_missing_coverage_file_message_is_framework_agnostic(self, caplog):
18+
"""When coverage file is missing, error message should not say 'Jest' specifically.
19+
20+
This class is used for both Jest and Vitest (they use the same Istanbul/v8 format).
21+
Error messages should be generic, not hardcode 'Jest'.
22+
"""
23+
# Set log level to DEBUG to capture all messages
24+
caplog.set_level("DEBUG")
25+
26+
# Create minimal context
27+
context = MagicMock(spec=CodeOptimizationContext)
28+
context.language = Language.JAVASCRIPT
29+
context.target_code = "export function test() {}"
30+
context.helper_functions = []
31+
32+
nonexistent_path = Path("/tmp/nonexistent_coverage_12345.json")
33+
34+
# Load coverage from non-existent file
35+
result = JestCoverageUtils.load_from_jest_json(
36+
coverage_json_path=nonexistent_path,
37+
function_name="testFunc",
38+
code_context=context,
39+
source_code_path=Path("/tmp/test.ts")
40+
)
41+
42+
# Should return empty coverage data
43+
assert result.status.name in ("NOT_FOUND", "EMPTY")
44+
45+
# Error message should NOT hardcode "Jest" - it should be framework-agnostic
46+
# since this util is used for both Jest and Vitest
47+
log_messages = [record.message for record in caplog.records]
48+
49+
# Check that if there's a message about coverage file, it doesn't say "Jest"
50+
coverage_messages = [msg for msg in log_messages if "coverage file not found" in msg.lower()]
51+
if coverage_messages:
52+
# The message should NOT contain "Jest" specifically
53+
# It should say something like "Coverage file not found" or "JavaScript coverage file not found"
54+
for msg in coverage_messages:
55+
assert "Jest" not in msg, (
56+
f"Error message should not hardcode 'Jest' since this util is used for Vitest too. "
57+
f"Got: {msg}"
58+
)
59+
60+
def test_parse_error_message_is_framework_agnostic(self, tmp_path, caplog):
61+
"""When coverage file is malformed, error should not say 'Jest' specifically."""
62+
# Set log level to capture all messages
63+
caplog.set_level("DEBUG")
64+
65+
# Create invalid JSON file
66+
coverage_file = tmp_path / "invalid_coverage.json"
67+
coverage_file.write_text("{invalid json")
68+
69+
context = MagicMock(spec=CodeOptimizationContext)
70+
context.language = Language.JAVASCRIPT
71+
context.target_code = "export function test() {}"
72+
context.helper_functions = []
73+
74+
result = JestCoverageUtils.load_from_jest_json(
75+
coverage_json_path=coverage_file,
76+
function_name="testFunc",
77+
code_context=context,
78+
source_code_path=Path("/tmp/test.ts")
79+
)
80+
81+
# Should return empty coverage
82+
assert result.status.name in ("NOT_FOUND", "EMPTY")
83+
84+
# Check log messages don't hardcode "Jest"
85+
log_messages = [record.message for record in caplog.records]
86+
parse_error_messages = [msg for msg in log_messages if "parse" in msg.lower() and "coverage" in msg.lower()]
87+
88+
for msg in parse_error_messages:
89+
assert "Jest" not in msg, (
90+
f"Parse error message should not hardcode 'Jest'. Got: {msg}"
91+
)

0 commit comments

Comments
 (0)