get the duration from the pytest overriden methods

Author: mohammedahmed18

codeflash/code_utils/config_consts.py

@@ -8,8 +8,7 @@
 MAX_CUMULATIVE_TEST_RUNTIME_NANOSECONDS = 100e6  # 100ms
 N_TESTS_TO_GENERATE = 2
 TOTAL_LOOPING_TIME = 10.0  # 10 second candidate benchmarking budget
-CONSISTENT_LOOP_COUNT = 5
-CONSISTENT_DURATION_TOLERANCE = 0.05
+CONSISTENT_LOOP_COUNT = 3
 COVERAGE_THRESHOLD = 60.0
 MIN_TESTCASE_PASSED_THRESHOLD = 6
 REPEAT_OPTIMIZATION_PROBABILITY = 0.1

codeflash/code_utils/env_utils.py

@@ -19,7 +19,6 @@
 def check_formatter_installed(formatter_cmds: list[str], exit_on_failure: bool = True) -> bool:  # noqa
     if not formatter_cmds or formatter_cmds[0] == "disabled":
         return True
-
     first_cmd = formatter_cmds[0]
     cmd_tokens = shlex.split(first_cmd) if isinstance(first_cmd, str) else [first_cmd]
 

codeflash/verification/pytest_plugin.py

@@ -2,8 +2,6 @@
 
 import contextlib
 import inspect
-
-# System Imports
 import logging
 import os
 import platform
@@ -13,6 +11,8 @@
 import time as _time_module
 import warnings
 from collections import deque
+
+# System Imports
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable
 from unittest import TestCase
@@ -21,7 +21,7 @@
 import pytest
 from pluggy import HookspecMarker
 
-from codeflash.code_utils.config_consts import CONSISTENT_DURATION_TOLERANCE, CONSISTENT_LOOP_COUNT
+from codeflash.code_utils.config_consts import CONSISTENT_LOOP_COUNT
 
 if TYPE_CHECKING:
     from _pytest.config import Config, Parser
@@ -272,21 +272,25 @@ def __init__(self, config: Config) -> None:
         level = logging.DEBUG if config.option.verbose > 1 else logging.INFO
         logging.basicConfig(level=level)
         self.logger = logging.getLogger(self.name)
+        self.current_loop_durations_in_seconds: list[float] = []
 
     def dynamic_tolerance(self, avg: float) -> float:
-        # (< 0.1 ms)
-        if avg < 0.0001:
-            return 0.7  # 70%
-
-        # (< 0.5 ms)
-        if avg < 0.0005:
-            return 0.4  # 40%
-
-        # (< 1 ms)
-        if avg < 0.001:
-            return 0.2  # 20%
-
-        return CONSISTENT_DURATION_TOLERANCE
+        if avg < 0.0001:  # < 100 µs
+            return 0.7
+        if avg < 0.0005:  # < 500 µs
+            return 0.5
+        if avg < 0.001:  # < 1 ms
+            return 0.4
+        if avg < 0.01:  # < 10 ms
+            return 0.2
+        if avg < 0.1:  # < 100 ms
+            return 0.1
+        return 0.03  # > 0.1 s
+
+    @pytest.hookimpl
+    def pytest_runtest_logreport(self, report: pytest.TestReport) -> None:
+        if report.when == "call" and report.outcome == "passed":
+            self.current_loop_durations_in_seconds.append(report.duration)
 
     @hookspec(firstresult=True)
     def pytest_runtestloop(self, session: Session) -> bool:
@@ -307,7 +311,7 @@ def pytest_runtestloop(self, session: Session) -> bool:
 
         while total_time >= SHORTEST_AMOUNT_OF_TIME:
             count += 1
-            loop_start = _ORIGINAL_TIME_TIME()
+            self.current_loop_durations_in_seconds.clear()
 
             for index, item in enumerate(session.items):
                 item: pytest.Item = item  # noqa: PLW0127, PLW2901
@@ -326,9 +330,12 @@ def pytest_runtestloop(self, session: Session) -> bool:
                 if session.shouldstop:
                     raise session.Interrupted(session.shouldstop)
 
-            loop_end = _ORIGINAL_TIME_TIME()
-            loop_duration = loop_end - loop_start
-            durations.append(loop_duration)
+            total_duration_in_seconds = sum(self.current_loop_durations_in_seconds)
+
+            if total_duration_in_seconds > 0:
+                durations.append(total_duration_in_seconds)
+            else:
+                durations.clear()
 
             # Consistency check
             if len(durations) == CONSISTENT_LOOP_COUNT:
@@ -338,6 +345,7 @@ def pytest_runtestloop(self, session: Session) -> bool:
                 else:
                     consistent = all(abs(d - avg) / avg <= self.dynamic_tolerance(avg) for d in durations)
                 if consistent:
+                    Path(f"/home/mohammed/Documents/test-results/break-{session.name}.txt").write_text(str(count))
                     break
 
             if self._timed_out(session, start_time, count):