Skip to content

Commit 986654b

Browse files
committed
fix: pin PYTHONHASHSEED=0 in test env and enhance diff diagnostics
Set PYTHONHASHSEED=0 in test subprocess environments so original and candidate runs use identical hash behavior, eliminating a source of non-deterministic return-value comparisons. Also upgrade diff logging from debug to info level with actual types and repr values for DID_PASS, RETURN_VALUE, and STDOUT diffs.
1 parent e191f74 commit 986654b

2 files changed

Lines changed: 24 additions & 7 deletions

File tree

codeflash/languages/function_optimizer.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3253,6 +3253,11 @@ def get_test_env(
32533253
test_env["CODEFLASH_TEST_ITERATION"] = str(codeflash_test_iteration)
32543254
test_env["CODEFLASH_TRACER_DISABLE"] = str(codeflash_tracer_disable)
32553255
test_env["CODEFLASH_LOOP_INDEX"] = str(codeflash_loop_index)
3256+
# Pin PYTHONHASHSEED so original and candidate test processes use the same hash seed.
3257+
# Without this, each subprocess gets a random seed, which can cause non-deterministic
3258+
# iteration order in sets/dicts and lead to flaky return-value comparisons.
3259+
if "PYTHONHASHSEED" not in test_env:
3260+
test_env["PYTHONHASHSEED"] = "0"
32563261
return test_env
32573262

32583263
def line_profiler_step(

codeflash/verification/equivalence.py

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,11 @@ def compare_test_results(
111111
original_pytest_error=original_pytest_error,
112112
)
113113
)
114+
logger.info(
115+
f"[DIFF] scope=DID_PASS test_id={test_id} "
116+
f"orig_pass={original_test_result.did_pass} cand_pass={cdd_test_result.did_pass} "
117+
f"test_type={original_test_result.test_type} cand_error={cdd_pytest_error[:200] if cdd_pytest_error else 'none'}"
118+
)
114119

115120
elif not pass_fail_only and not comparator(
116121
original_test_result.return_value, cdd_test_result.return_value, superset_obj=superset_obj
@@ -129,13 +134,15 @@ def compare_test_results(
129134
)
130135

131136
try:
132-
logger.debug(
133-
f"File Name: {original_test_result.file_name}\n"
134-
f"Test Type: {original_test_result.test_type}\n"
135-
f"Verification Type: {original_test_result.verification_type}\n"
136-
f"Invocation ID: {original_test_result.id}\n"
137-
f"Original return value: {original_test_result.return_value}\n"
138-
f"Candidate return value: {cdd_test_result.return_value}\n"
137+
_orig_rv = original_test_result.return_value
138+
_cand_rv = cdd_test_result.return_value
139+
logger.info(
140+
f"[DIFF] scope=RETURN_VALUE test_id={test_id} "
141+
f"orig_type={type(_orig_rv).__name__} cand_type={type(_cand_rv).__name__} "
142+
f"orig_pass={original_test_result.did_pass} cand_pass={cdd_test_result.did_pass} "
143+
f"test_type={original_test_result.test_type} "
144+
f"orig_repr={safe_repr(_orig_rv)[:200]} "
145+
f"cand_repr={safe_repr(_cand_rv)[:200]}"
139146
)
140147
except Exception as e:
141148
logger.error(e)
@@ -156,6 +163,11 @@ def compare_test_results(
156163
original_pytest_error=original_pytest_error,
157164
)
158165
)
166+
logger.info(
167+
f"[DIFF] scope=STDOUT test_id={test_id} "
168+
f"orig_stdout={str(original_test_result.stdout)[:200]} "
169+
f"cand_stdout={str(cdd_test_result.stdout)[:200]}"
170+
)
159171

160172
sys.setrecursionlimit(original_recursion_limit)
161173
logger.info(

0 commit comments

Comments
 (0)