Skip to content

Commit 29e42ae

Browse files
Codeflash Botclaude
andcommitted
Fix false positive test discovery from substring matching
Issue: Test discovery incorrectly matched test files with source functions when the function name appeared anywhere in the test file, including in mocks, comments, or unrelated code. This caused 'Failed to instrument test file' errors. Root cause: In javascript/support.py line 259, naive substring matching (func.function_name in source) matched function names even when they were only mentioned in mocks like: vi.mock('./file.js', () => ({ funcName: ... })) Example: Function parseRestartRequestParams from restart-request.ts was wrongly matched with update.test.ts because the test file mocked it. Fix: Removed substring matching, now only matches explicitly imported functions. This is more reliable and avoids false positives. Trace ID: 0b575a96-62a8-4910-b163-1ad10e60ba79 Changes: - Removed naive substring check in discover_tests() - Only match functions that are explicitly imported - Added regression tests (2 test cases) Testing: - All 70 JavaScript tests pass - New tests verify fix works correctly - Linting/type checks pass (uv run prek) Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
1 parent f86fe2d commit 29e42ae

2 files changed

Lines changed: 113 additions & 1 deletion

File tree

codeflash/languages/javascript/support.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -256,7 +256,10 @@ def discover_tests(
256256

257257
# Match source functions to tests
258258
for func in source_functions:
259-
if func.function_name in imported_names or func.function_name in source:
259+
# Only match functions that are explicitly imported from the source module.
260+
# Avoid substring matching (e.g., "func.function_name in source") as it causes
261+
# false positives when functions are mentioned in mocks, comments, or unrelated code.
262+
if func.function_name in imported_names:
260263
if func.qualified_name not in result:
261264
result[func.qualified_name] = []
262265
for test_name in test_functions:
Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
"""Test for false positive test discovery bug (Bug #4)."""
2+
3+
from pathlib import Path
4+
from tempfile import TemporaryDirectory
5+
6+
import pytest
7+
8+
from codeflash.discovery.functions_to_optimize import FunctionToOptimize
9+
from codeflash.languages.javascript.support import TypeScriptSupport
10+
from codeflash.models.models import CodePosition
11+
12+
13+
def test_discover_tests_should_not_match_mocked_functions():
14+
"""Test that functions mentioned only in mocks are not matched as test targets.
15+
16+
Regression test for Bug #4: False positive test discovery due to substring matching.
17+
18+
When a test file mocks a function (e.g., vi.mock("./restart-request.js", () => ({...}))),
19+
that function should NOT be considered as tested by that file, since it's only mocked,
20+
not actually called or tested.
21+
"""
22+
support = TypeScriptSupport()
23+
24+
with TemporaryDirectory() as tmpdir:
25+
test_root = Path(tmpdir)
26+
27+
# Create a test file that MOCKS parseRestartRequestParams but doesn't test it
28+
test_file = test_root / "update.test.ts"
29+
test_file.write_text(
30+
'''
31+
import { updateSomething } from "./update.js";
32+
33+
vi.mock("./restart-request.js", () => ({
34+
parseRestartRequestParams: (params: any) => ({ sessionKey: undefined }),
35+
}));
36+
37+
describe("updateSomething", () => {
38+
it("should update successfully", () => {
39+
const result = updateSomething();
40+
expect(result).toBe(true);
41+
});
42+
});
43+
'''
44+
)
45+
46+
# Source function that is only mocked, not tested
47+
source_function = FunctionToOptimize(
48+
qualified_name="parseRestartRequestParams",
49+
function_name="parseRestartRequestParams",
50+
file_path=test_root / "restart-request.ts",
51+
starting_line=1,
52+
ending_line=10,
53+
function_signature="",
54+
code_position=CodePosition(line_no=1, col_no=0),
55+
file_path_relative_to_project_root="restart-request.ts",
56+
)
57+
58+
# Discover tests
59+
result = support.discover_tests(test_root, [source_function])
60+
61+
# The bug: discovers update.test.ts as a test for parseRestartRequestParams
62+
# because "parseRestartRequestParams" appears as a substring in the mock
63+
# Expected: should NOT match (empty result)
64+
assert (
65+
source_function.qualified_name not in result or len(result[source_function.qualified_name]) == 0
66+
), f"Should not match mocked function, but found: {result.get(source_function.qualified_name, [])}"
67+
68+
69+
def test_discover_tests_should_match_actually_imported_functions():
70+
"""Test that functions actually imported and tested ARE correctly matched.
71+
72+
This is the positive case to ensure we don't break legitimate test discovery.
73+
"""
74+
support = TypeScriptSupport()
75+
76+
with TemporaryDirectory() as tmpdir:
77+
test_root = Path(tmpdir)
78+
79+
# Create a test file that ACTUALLY imports and tests the function
80+
test_file = test_root / "restart-request.test.ts"
81+
test_file.write_text(
82+
'''
83+
import { parseRestartRequestParams } from "./restart-request.js";
84+
85+
describe("parseRestartRequestParams", () => {
86+
it("should parse valid params", () => {
87+
const result = parseRestartRequestParams({ sessionKey: "abc" });
88+
expect(result.sessionKey).toBe("abc");
89+
});
90+
});
91+
'''
92+
)
93+
94+
source_function = FunctionToOptimize(
95+
qualified_name="parseRestartRequestParams",
96+
function_name="parseRestartRequestParams",
97+
file_path=test_root / "restart-request.ts",
98+
starting_line=1,
99+
ending_line=10,
100+
function_signature="",
101+
code_position=CodePosition(line_no=1, col_no=0),
102+
file_path_relative_to_project_root="restart-request.ts",
103+
)
104+
105+
result = support.discover_tests(test_root, [source_function])
106+
107+
# Should match: function is imported and tested
108+
assert source_function.qualified_name in result, f"Should match imported function, but got: {result}"
109+
assert len(result[source_function.qualified_name]) > 0, "Should find at least one test"

0 commit comments

Comments
 (0)