Merge pull request #2150 from codeflash-ai/fix/pytohn-stdout-comparator

[FIX] Python stdout comparator when the stdout is empty

Author: Saga4

codeflash/verification/equivalence.py

@@ -129,16 +129,12 @@ def compare_test_results(
                 )
             except Exception as e:
                 logger.error(e)
-        elif (
-            not pass_fail_only
-            and (original_test_result.stdout and cdd_test_result.stdout)
-            and not comparator(original_test_result.stdout, cdd_test_result.stdout)
-        ):
+        elif not pass_fail_only and not comparator(original_test_result.stdout or "", cdd_test_result.stdout or ""):
             test_diffs.append(
                 TestDiff(
                     scope=TestDiffScope.STDOUT,
-                    original_value=str(original_test_result.stdout),
-                    candidate_value=str(cdd_test_result.stdout),
+                    original_value=original_test_result.stdout or "",
+                    candidate_value=cdd_test_result.stdout or "",
                     test_src_code=original_test_result.id.get_src_code(original_test_result.file_name),
                     candidate_pytest_error=cdd_pytest_error,
                     original_pass=original_test_result.did_pass,

tests/test_comparator.py

@@ -16,7 +16,7 @@
 import pytest
 
 from codeflash.either import Failure, Success
-from codeflash.models.models import FunctionTestInvocation, InvocationId, TestResults, TestType
+from codeflash.models.models import FunctionTestInvocation, InvocationId, TestDiffScope, TestResults, TestType
 from codeflash.verification.comparator import (
     PYTEST_TEMP_PATH_PATTERN,
     PYTHON_TEMPFILE_PATTERN,
@@ -2999,6 +2999,60 @@ def test_compare_results_fn():
     assert not match
 
 
+def test_compare_results_detects_stdout_mismatch_when_candidate_stdout_is_empty() -> None:
+    original_results = TestResults()
+    original_results.add(
+        FunctionTestInvocation(
+            id=InvocationId(
+                test_module_path="test_module_path",
+                test_class_name="test_class_name",
+                test_function_name="test_function_name",
+                function_getting_tested="function_getting_tested",
+                iteration_id="0",
+            ),
+            file_name=Path("file_name"),
+            did_pass=True,
+            runtime=5,
+            test_framework="pytest",
+            test_type=TestType.EXISTING_UNIT_TEST,
+            return_value=[0, 1, 2],
+            timed_out=False,
+            loop_index=1,
+            stdout="codeflash stdout: Sorting list\nresult: [0, 1, 2]\n",
+        )
+    )
+
+    candidate_results = TestResults()
+    candidate_results.add(
+        FunctionTestInvocation(
+            id=InvocationId(
+                test_module_path="test_module_path",
+                test_class_name="test_class_name",
+                test_function_name="test_function_name",
+                function_getting_tested="function_getting_tested",
+                iteration_id="0",
+            ),
+            file_name=Path("file_name"),
+            did_pass=True,
+            runtime=5,
+            test_framework="pytest",
+            test_type=TestType.EXISTING_UNIT_TEST,
+            return_value=[0, 1, 2],
+            timed_out=False,
+            loop_index=1,
+            stdout="",
+        )
+    )
+
+    match, diffs = compare_test_results(original_results, candidate_results)
+
+    assert not match
+    assert len(diffs) == 1
+    assert diffs[0].scope == TestDiffScope.STDOUT
+    assert diffs[0].original_value == "codeflash stdout: Sorting list\nresult: [0, 1, 2]\n"
+    assert diffs[0].candidate_value == ""
+
+
 def test_exceptions():
     type_error = TypeError("This is a type error")