Merge branch 'main' into jit-docs

Author: HeshamHM28

codeflash/api/cfapi.py

@@ -171,6 +171,8 @@ def suggest_changes(
     replay_tests: str = "",
     concolic_tests: str = "",
     optimization_review: str = "",
+    original_line_profiler: str | None = None,
+    optimized_line_profiler: str | None = None,
 ) -> Response:
     """Suggest changes to a pull request.
 
@@ -182,6 +184,8 @@ def suggest_changes(
     :param file_changes: A dictionary of file changes.
     :param pr_comment: The pull request comment object, containing the optimization explanation, best runtime, etc.
     :param generated_tests: The generated tests.
+    :param original_line_profiler: Line profiler results for original code (markdown format).
+    :param optimized_line_profiler: Line profiler results for optimized code (markdown format).
     :return: The response object.
     """
     payload = {
@@ -197,7 +201,10 @@ def suggest_changes(
         "replayTests": replay_tests,
         "concolicTests": concolic_tests,
         "optimizationReview": optimization_review,  # impact keyword left for legacy reasons, touches js/ts code
+        "originalLineProfiler": original_line_profiler,
+        "optimizedLineProfiler": optimized_line_profiler,
     }
+
     return make_cfapi_request(endpoint="/suggest-pr-changes", method="POST", payload=payload)
 
 
@@ -214,6 +221,8 @@ def create_pr(
     replay_tests: str = "",
     concolic_tests: str = "",
     optimization_review: str = "",
+    original_line_profiler: str | None = None,
+    optimized_line_profiler: str | None = None,
 ) -> Response:
     """Create a pull request, targeting the specified branch. (usually 'main').
 
@@ -223,6 +232,8 @@ def create_pr(
     :param file_changes: A dictionary of file changes.
     :param pr_comment: The pull request comment object, containing the optimization explanation, best runtime, etc.
     :param generated_tests: The generated tests.
+    :param original_line_profiler: Line profiler results for original code (markdown format).
+    :param optimized_line_profiler: Line profiler results for optimized code (markdown format).
     :return: The response object.
     """
     # convert Path objects to strings
@@ -239,7 +250,10 @@ def create_pr(
         "replayTests": replay_tests,
         "concolicTests": concolic_tests,
         "optimizationReview": optimization_review,  # Impact keyword left for legacy reasons, it touches js/ts codebase
+        "originalLineProfiler": original_line_profiler,
+        "optimizedLineProfiler": optimized_line_profiler,
     }
+
     return make_cfapi_request(endpoint="/create-pr", method="POST", payload=payload)
 
 
@@ -269,6 +283,8 @@ def create_staging(
     concolic_tests: str,
     root_dir: Path,
     optimization_review: str = "",
+    original_line_profiler: str | None = None,
+    optimized_line_profiler: str | None = None,
 ) -> Response:
     """Create a staging pull request, targeting the specified branch. (usually 'staging').
 
@@ -279,6 +295,8 @@ def create_staging(
     :param generated_original_test_source: Generated tests for the original function.
     :param function_trace_id: Unique identifier for this optimization trace.
     :param coverage_message: Coverage report or summary.
+    :param original_line_profiler: Line profiler results for original code (markdown format).
+    :param optimized_line_profiler: Line profiler results for optimized code (markdown format).
     :return: The response object from the backend.
     """
     relative_path = explanation.file_path.relative_to(root_dir).as_posix()
@@ -310,6 +328,8 @@ def create_staging(
         "replayTests": replay_tests,
         "concolicTests": concolic_tests,
         "optimizationReview": optimization_review,  # Impact keyword left for legacy reasons, it touches js/ts codebase
+        "originalLineProfiler": original_line_profiler,
+        "optimizedLineProfiler": optimized_line_profiler,
     }
 
     return make_cfapi_request(endpoint="/create-staging", method="POST", payload=payload)

codeflash/optimization/function_optimizer.py

@@ -2001,6 +2001,8 @@ def process_review(
             "coverage_message": coverage_message,
             "replay_tests": replay_tests,
             "concolic_tests": concolic_tests,
+            "original_line_profiler": original_code_baseline.line_profile_results.get("str_out", ""),
+            "optimized_line_profiler": best_optimization.line_profiler_test_results.get("str_out", ""),
         }
 
         raise_pr = not self.args.no_pr

codeflash/result/create_pr.py

@@ -186,6 +186,8 @@ def check_create_pr(
     root_dir: Path,
     git_remote: Optional[str] = None,
     optimization_review: str = "",
+    original_line_profiler: str | None = None,
+    optimized_line_profiler: str | None = None,
 ) -> None:
     pr_number: Optional[int] = env_utils.get_pr_number()
     git_repo = git.Repo(search_parent_directories=True)
@@ -230,6 +232,8 @@ def check_create_pr(
             replay_tests=replay_tests,
             concolic_tests=concolic_tests,
             optimization_review=optimization_review,
+            original_line_profiler=original_line_profiler,
+            optimized_line_profiler=optimized_line_profiler,
         )
         if response.ok:
             logger.info(f"Suggestions were successfully made to PR #{pr_number}")
@@ -282,6 +286,8 @@ def check_create_pr(
             replay_tests=replay_tests,
             concolic_tests=concolic_tests,
             optimization_review=optimization_review,
+            original_line_profiler=original_line_profiler,
+            optimized_line_profiler=optimized_line_profiler,
         )
         if response.ok:
             pr_id = response.text

codeflash/verification/comparator.py

@@ -30,20 +30,25 @@
 # These paths vary between test runs but are logically equivalent
 PYTEST_TEMP_PATH_PATTERN = re.compile(r"/tmp/pytest-of-[^/]+/pytest-\d+/")  # noqa: S108
 
+# Pattern to match Python tempfile directories: /tmp/tmp<random>/
+# Created by tempfile.mkdtemp() or tempfile.TemporaryDirectory()
+PYTHON_TEMPFILE_PATTERN = re.compile(r"/tmp/tmp[a-zA-Z0-9_]+/")  # noqa: S108
+
 
 def _normalize_temp_path(path: str) -> str:
     """Normalize temporary file paths by replacing session-specific components.
 
-    Pytest creates temp directories like /tmp/pytest-of-<user>/pytest-<N>/
-    where N is a session number that increments. When comparing return values
-    from different test runs, these paths should be considered equivalent.
+    Handles two types of temp paths:
+    - Pytest: /tmp/pytest-of-<user>/pytest-<N>/ -> /tmp/pytest-temp/
+    - Python tempfile: /tmp/tmp<random>/ -> /tmp/python-temp/
     """
-    return PYTEST_TEMP_PATH_PATTERN.sub("/tmp/pytest-temp/", path)  # noqa: S108
+    path = PYTEST_TEMP_PATH_PATTERN.sub("/tmp/pytest-temp/", path)  # noqa: S108
+    return PYTHON_TEMPFILE_PATTERN.sub("/tmp/python-temp/", path)  # noqa: S108
 
 
 def _is_temp_path(s: str) -> bool:
-    """Check if a string looks like a pytest temp path."""
-    return PYTEST_TEMP_PATH_PATTERN.search(s) is not None
+    """Check if a string looks like a temp path (pytest or Python tempfile)."""
+    return PYTEST_TEMP_PATH_PATTERN.search(s) is not None or PYTHON_TEMPFILE_PATTERN.search(s) is not None
 
 
 def _extract_exception_from_message(msg: str) -> Optional[BaseException]:  # noqa: FA100

tests/test_comparator.py

@@ -19,6 +19,7 @@
 from codeflash.models.models import FunctionTestInvocation, InvocationId, TestResults, TestType
 from codeflash.verification.comparator import (
     PYTEST_TEMP_PATH_PATTERN,
+    PYTHON_TEMPFILE_PATTERN,
     _extract_exception_from_message,
     _get_wrapped_exception,
     _is_temp_path,
@@ -3977,3 +3978,82 @@ def test_config_object_with_paths(self):
             "permanent_dir": "/home/user/data/"
         }
         assert comparator(config1, config2)
+
+
+class TestPythonTempfilePaths:
+    """Tests for Python tempfile paths (from tempfile.mkdtemp() or TemporaryDirectory())."""
+
+    def test_is_temp_path_detects_python_tempfile(self):
+        """Test that _is_temp_path detects Python tempfile paths."""
+        assert _is_temp_path("/tmp/tmpqtwy7hpf/special.txt")
+        assert _is_temp_path("/tmp/tmpp6wx3tz3/special.txt")
+        assert _is_temp_path("/tmp/tmpabcdef12/")
+        assert _is_temp_path("/tmp/tmp_underscore/file.txt")
+
+    def test_is_temp_path_various_tempfile_names(self):
+        """Test various tempfile naming patterns."""
+        assert _is_temp_path("/tmp/tmpABCDEF/file.txt")  # uppercase
+        assert _is_temp_path("/tmp/tmp123456/file.txt")  # numeric
+        assert _is_temp_path("/tmp/tmpaBc123/file.txt")  # mixed
+        assert _is_temp_path("/tmp/tmp_test_dir/subdir/file.txt")  # with underscore
+
+    def test_is_temp_path_non_tempfile(self):
+        """Test that non-tempfile paths are not detected."""
+        assert not _is_temp_path("/tmp/mydir/file.txt")  # doesn't start with tmp
+        assert not _is_temp_path("/tmp/temp/file.txt")  # temp, not tmp
+        assert not _is_temp_path("/home/user/tmp123/file.txt")  # not in /tmp/
+
+    def test_normalize_temp_path_python_tempfile(self):
+        """Test normalization of Python tempfile paths."""
+        path1 = _normalize_temp_path("/tmp/tmpqtwy7hpf/special.txt")
+        path2 = _normalize_temp_path("/tmp/tmpp6wx3tz3/special.txt")
+        assert path1 == path2 == "/tmp/python-temp/special.txt"
+
+    def test_normalize_temp_path_preserves_subdirs(self):
+        """Test that subdirectories are preserved during normalization."""
+        result = _normalize_temp_path("/tmp/tmpabcdef12/subdir/nested/file.txt")
+        assert result == "/tmp/python-temp/subdir/nested/file.txt"
+
+    def test_comparator_python_tempfile_paths_equal(self):
+        """Test that different tempfile paths with same content are equal."""
+        path1 = "/tmp/tmpqtwy7hpf/special.txt"
+        path2 = "/tmp/tmpp6wx3tz3/special.txt"
+        assert comparator(path1, path2)
+
+    def test_comparator_python_tempfile_different_filenames_not_equal(self):
+        """Test that different filenames in tempfile paths are not equal."""
+        path1 = "/tmp/tmpqtwy7hpf/special.txt"
+        path2 = "/tmp/tmpp6wx3tz3/different.txt"
+        assert not comparator(path1, path2)
+
+    def test_comparator_python_tempfile_in_tuple(self):
+        """Test tempfile paths in tuples."""
+        orig = ("/tmp/tmpqtwy7hpf/special.txt",)
+        new = ("/tmp/tmpp6wx3tz3/special.txt",)
+        assert comparator(orig, new)
+
+    def test_comparator_python_tempfile_in_list(self):
+        """Test tempfile paths in lists."""
+        orig = ["/tmp/tmpabcdef12/file1.txt", "/tmp/tmpabcdef12/file2.txt"]
+        new = ["/tmp/tmpxyz78901/file1.txt", "/tmp/tmpxyz78901/file2.txt"]
+        assert comparator(orig, new)
+
+    def test_comparator_python_tempfile_in_dict(self):
+        """Test tempfile paths in dictionaries."""
+        orig = {"output": "/tmp/tmpabcdef12/result.json"}
+        new = {"output": "/tmp/tmpxyz78901/result.json"}
+        assert comparator(orig, new)
+
+    def test_comparator_mixed_pytest_and_python_tempfile(self):
+        """Test that pytest and Python tempfile paths don't match each other."""
+        pytest_path = "/tmp/pytest-of-user/pytest-0/file.txt"
+        python_path = "/tmp/tmpabcdef12/file.txt"
+        # These should not be equal - they're different temp path types
+        assert not comparator(pytest_path, python_path)
+
+    def test_python_tempfile_pattern_regex(self):
+        """Test the PYTHON_TEMPFILE_PATTERN regex directly."""
+        assert PYTHON_TEMPFILE_PATTERN.search("/tmp/tmpabcdef/file.txt")
+        assert PYTHON_TEMPFILE_PATTERN.search("/tmp/tmp123456/")
+        assert not PYTHON_TEMPFILE_PATTERN.search("/tmp/mydir/file.txt")
+        assert not PYTHON_TEMPFILE_PATTERN.search("/home/tmp123/file.txt")