test: add comprehensive Java run-and-parse integration tests

Add end-to-end tests for Java test instrumentation, execution, and result
parsing, covering both behavior and performance testing modes.

Key additions:
- PreciseWaiter: monotonic timing implementation with <2% variance
- 3 behavior tests: single/multiple test methods, return value validation
- 2 performance tests: timing accuracy, inner/outer loop counts
- Validation of total_passed_runtime() aggregation

Infrastructure improvements:
- Add inner_iterations parameter to benchmarking call chain
- Rename pytest_* parameters to language-agnostic names:
  - pytest_min_loops → min_outer_loops
  - pytest_max_loops → max_outer_loops
  - pytest_inner_iterations → inner_iterations
- Pass inner_iterations from tests through function_optimizer → test_runner → language_support

All tests validate timing accuracy (±2%), variance (<2% CV), and correct
result grouping by test case including iteration_id.

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

Author: misrasaurabh1

codeflash/optimization/function_optimizer.py

@@ -3064,8 +3064,9 @@ def run_and_parse_tests(
         testing_time: float = TOTAL_LOOPING_TIME_EFFECTIVE,
         *,
         enable_coverage: bool = False,
-        pytest_min_loops: int = 5,
-        pytest_max_loops: int = 250,
+        min_outer_loops: int = 5,
+        max_outer_loops: int = 250,
+        inner_iterations: int | None = None,
         code_context: CodeOptimizationContext | None = None,
         line_profiler_output_file: Path | None = None,
     ) -> tuple[TestResults | dict, CoverageData | None]:
@@ -3101,10 +3102,11 @@ def run_and_parse_tests(
                     cwd=self.project_root,
                     test_env=test_env,
                     pytest_cmd=self.test_cfg.pytest_cmd,
-                    pytest_timeout=INDIVIDUAL_TESTCASE_TIMEOUT,
-                    pytest_target_runtime_seconds=testing_time,
-                    pytest_min_loops=pytest_min_loops,
-                    pytest_max_loops=pytest_max_loops,
+                    timeout=INDIVIDUAL_TESTCASE_TIMEOUT,
+                    target_runtime_seconds=testing_time,
+                    min_outer_loops=min_outer_loops,
+                    max_outer_loops=max_outer_loops,
+                    inner_iterations=inner_iterations,
                     test_framework=self.test_cfg.test_framework,
                     js_project_root=self.test_cfg.js_project_root,
                 )
@@ -3368,8 +3370,8 @@ def run_concurrency_benchmark(
                 testing_time=5.0,  # Short benchmark time
                 enable_coverage=False,
                 code_context=code_context,
-                pytest_min_loops=1,
-                pytest_max_loops=3,
+                min_outer_loops=1,
+                max_outer_loops=3,
             )
         except Exception as e:
             logger.debug(f"Concurrency benchmark failed: {e}")

codeflash/verification/test_runner.py

@@ -345,10 +345,11 @@ def run_benchmarking_tests(
     cwd: Path,
     test_framework: str,
     *,
-    pytest_target_runtime_seconds: float = TOTAL_LOOPING_TIME_EFFECTIVE,
-    pytest_timeout: int | None = None,
-    pytest_min_loops: int = 5,
-    pytest_max_loops: int = 100_000,
+    target_runtime_seconds: float = TOTAL_LOOPING_TIME_EFFECTIVE,
+    timeout: int | None = None,
+    min_outer_loops: int = 5,
+    max_outer_loops: int = 100_000,
+    inner_iterations: int | None = None,
     js_project_root: Path | None = None,
 ) -> tuple[Path, subprocess.CompletedProcess]:
     logger.debug(f"run_benchmarking_tests called: framework={test_framework}, num_files={len(test_paths.test_files)}")
@@ -359,26 +360,30 @@ def run_benchmarking_tests(
         # Use Java-specific timeout if no explicit timeout provided
         from codeflash.code_utils.config_consts import JAVA_TESTCASE_TIMEOUT
 
-        effective_timeout = pytest_timeout
-        if test_framework in ("junit4", "junit5", "testng") and pytest_timeout is not None:
+        effective_timeout = timeout
+        if test_framework in ("junit4", "junit5", "testng") and timeout is not None:
             # For Java, use a minimum timeout to account for Maven overhead
-            effective_timeout = max(pytest_timeout, JAVA_TESTCASE_TIMEOUT)
-            if effective_timeout != pytest_timeout:
+            effective_timeout = max(timeout, JAVA_TESTCASE_TIMEOUT)
+            if effective_timeout != timeout:
                 logger.debug(
-                    f"Increased Java test timeout from {pytest_timeout}s to {effective_timeout}s "
+                    f"Increased Java test timeout from {timeout}s to {effective_timeout}s "
                     "to account for Maven startup overhead"
                 )
 
-        return language_support.run_benchmarking_tests(
-            test_paths=test_paths,
-            test_env=test_env,
-            cwd=cwd,
-            timeout=effective_timeout,
-            project_root=js_project_root,
-            min_loops=pytest_min_loops,
-            max_loops=pytest_max_loops,
-            target_duration_seconds=pytest_target_runtime_seconds,
-        )
+        kwargs = {
+            "test_paths": test_paths,
+            "test_env": test_env,
+            "cwd": cwd,
+            "timeout": effective_timeout,
+            "project_root": js_project_root,
+            "min_loops": min_outer_loops,
+            "max_loops": max_outer_loops,
+            "target_duration_seconds": target_runtime_seconds,
+        }
+        # Pass inner_iterations if specified (for Java/JavaScript)
+        if inner_iterations is not None:
+            kwargs["inner_iterations"] = inner_iterations
+        return language_support.run_benchmarking_tests(**kwargs)
     if is_python():  # pytest runs both pytest and unittest tests
         pytest_cmd_list = (
             shlex.split(f"{SAFE_SYS_EXECUTABLE} -m pytest", posix=IS_POSIX)
@@ -393,13 +398,13 @@ def run_benchmarking_tests(
             "--capture=tee-sys",
             "-q",
             "--codeflash_loops_scope=session",
-            f"--codeflash_min_loops={pytest_min_loops}",
-            f"--codeflash_max_loops={pytest_max_loops}",
-            f"--codeflash_seconds={pytest_target_runtime_seconds}",
+            f"--codeflash_min_loops={min_outer_loops}",
+            f"--codeflash_max_loops={max_outer_loops}",
+            f"--codeflash_seconds={target_runtime_seconds}",
             "--codeflash_stability_check=true",
         ]
-        if pytest_timeout is not None:
-            pytest_args.append(f"--timeout={pytest_timeout}")
+        if timeout is not None:
+            pytest_args.append(f"--timeout={timeout}")
 
         result_file_path = get_run_tmp_file(Path("pytest_results.xml"))
         result_args = [f"--junitxml={result_file_path.as_posix()}", "-o", "junit_logging=all"]

tests/test_async_run_and_parse_tests.py

@@ -118,8 +118,8 @@ async def test_async_sort():
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
 
@@ -244,8 +244,8 @@ async def test_async_class_sort():
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
 
@@ -369,8 +369,8 @@ async def test_async_perf():
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
 
@@ -489,8 +489,8 @@ async def async_error_function(lst):
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
 
@@ -594,8 +594,8 @@ async def test_async_multi():
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=2,
-            pytest_max_loops=5,
+            min_outer_loops=2,
+            max_outer_loops=5,
             testing_time=0.2,
         )
 
@@ -714,8 +714,8 @@ async def test_async_edge_cases():
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
 
@@ -860,8 +860,8 @@ def test_sync_sort():
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
 
@@ -1035,8 +1035,8 @@ async def test_mixed_sorting():
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
 

tests/test_codeflash_capture.py

@@ -475,8 +475,8 @@ def __init__(self, x=2):
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
         assert len(test_results) == 3
@@ -508,8 +508,8 @@ def __init__(self, x=2):
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
         match, _ = compare_test_results(test_results, test_results2)
@@ -598,8 +598,8 @@ def __init__(self, *args, **kwargs):
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
         assert len(test_results) == 3
@@ -632,8 +632,8 @@ def __init__(self, *args, **kwargs):
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
 
@@ -725,8 +725,8 @@ def __init__(self, x=2):
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
 
@@ -761,8 +761,8 @@ def __init__(self, x=2):
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
 
@@ -889,8 +889,8 @@ def another_helper(self):
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
 
@@ -910,8 +910,8 @@ def another_helper(self):
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
 
@@ -1049,8 +1049,8 @@ def another_helper(self):
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
 
@@ -1101,8 +1101,8 @@ def target_function(self):
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
         # Remove instrumentation
@@ -1140,8 +1140,8 @@ def target_function(self):
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
         # Remove instrumentation
@@ -1179,8 +1179,8 @@ def target_function(self):
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
         # Remove instrumentation
@@ -1471,8 +1471,8 @@ def calculate_portfolio_metrics(
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
 
@@ -1538,8 +1538,8 @@ def risk_adjusted_return(return_val, weight):
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
         # Remove instrumentation
@@ -1601,8 +1601,8 @@ def calculate_portfolio_metrics(
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )
         # Remove instrumentation
@@ -1687,8 +1687,8 @@ def __init__(self, x, y):
             test_env=test_env,
             test_files=func_optimizer.test_files,
             optimization_iteration=0,
-            pytest_min_loops=1,
-            pytest_max_loops=1,
+            min_outer_loops=1,
+            max_outer_loops=1,
             testing_time=0.1,
         )