Merge pull request #2020 from codeflash-ai/fix/js-benchmarking-calibrate

[JS] Improvements for benchmarking (reduce the noise) & calibration

Author: Saga4

codeflash/languages/javascript/support.py

@@ -2059,9 +2059,9 @@ def process_generated_test_strings(
             generated_test_source, project_module_system, test_cfg.tests_project_rootdir
         )
 
-        # Add .js extensions to relative imports for ESM projects
-        # TypeScript + ESM requires explicit .js extensions even for .ts source files
-        if project_module_system == ModuleSystem.ES_MODULE:
+        # Add .js extensions to relative imports for ESM projects — but NOT for Jest,
+        # which resolves .ts imports without .js extensions via its transform/resolver.
+        if project_module_system == ModuleSystem.ES_MODULE and test_cfg.test_framework != "jest":
             from codeflash.languages.javascript.module_system import add_js_extensions_to_relative_imports
 
             generated_test_source = add_js_extensions_to_relative_imports(generated_test_source)
@@ -2264,10 +2264,13 @@ def get_module_path(self, source_file: Path, project_root: Path, tests_root: Pat
             # (e.g. \t → tab, \n → newline) and would break imports on Windows.
             rel_path = os.path.relpath(str(source_without_ext), str(tests_root_abs)).replace("\\", "/")
 
-            # For ESM, add .js extension (TypeScript convention)
-            # TypeScript requires imports to reference the OUTPUT file extension (.js),
-            # even when the source file is .ts. This is required for Node.js ESM resolution.
-            if module_system == ModuleSystem.ES_MODULE:
+            # For ESM, add .js extension (TypeScript convention) — but only for Vitest/native ESM.
+            # Jest resolves .ts imports without .js extensions via its transform/resolver config,
+            # so adding .js breaks Jest module resolution (Cannot find module '../foo.js').
+            from codeflash.languages.test_framework import get_js_test_framework_or_default
+
+            test_framework = get_js_test_framework_or_default()
+            if module_system == ModuleSystem.ES_MODULE and test_framework != "jest":
                 rel_path = rel_path + ".js"
                 logger.debug(
                     f"!lsp|Module path (ESM): source={source_file_abs}, tests_root={tests_root_abs}, "
@@ -2286,7 +2289,7 @@ def get_module_path(self, source_file: Path, project_root: Path, tests_root: Pat
             # For fallback, also check module system
             module_system = detect_module_system(project_root, source_file)
             path_without_ext = "../" + rel_path.with_suffix("").as_posix()
-            if module_system == ModuleSystem.ES_MODULE:
+            if module_system == ModuleSystem.ES_MODULE and test_framework != "jest":
                 return path_without_ext + ".js"
             return path_without_ext
 

codeflash/languages/javascript/test_runner.py

@@ -1108,6 +1108,9 @@ def run_jest_benchmarking_tests(
     jest_env["CODEFLASH_PERF_TARGET_DURATION_MS"] = str(target_duration_ms)
     jest_env["CODEFLASH_PERF_STABILITY_CHECK"] = "true" if stability_check else "false"
     jest_env["CODEFLASH_LOOP_INDEX"] = "1"  # Initial value for compatibility
+    # Warmup and calibration for accurate benchmarking
+    jest_env["CODEFLASH_PERF_WARMUP_ITERATIONS"] = "5"
+    jest_env["CODEFLASH_PERF_MIN_TIME_NS"] = "5000"  # 5us minimum time for calibration
 
     # Enable console output for timing markers
     # Some projects mock console.log in test setup (e.g., based on LOG_LEVEL or DEBUG)
@@ -1123,10 +1126,13 @@ def run_jest_benchmarking_tests(
     # Configure ESM support if project uses ES Modules
     _configure_esm_environment(jest_env, effective_cwd)
 
-    # Increase Node.js heap size for large TypeScript projects
+    # Increase Node.js heap size and expose GC for accurate benchmarking
     existing_node_options = jest_env.get("NODE_OPTIONS", "")
     if "--max-old-space-size" not in existing_node_options:
-        jest_env["NODE_OPTIONS"] = f"{existing_node_options} --max-old-space-size=4096".strip()
+        existing_node_options = f"{existing_node_options} --max-old-space-size=4096".strip()
+    if "--expose-gc" not in existing_node_options:
+        existing_node_options = f"{existing_node_options} --expose-gc".strip()
+    jest_env["NODE_OPTIONS"] = existing_node_options
 
     # Subprocess timeout: target_duration + 120s headroom for Jest startup
     # and TS compilation.  capturePerf's time budget governs actual looping.

codeflash/languages/javascript/vitest_runner.py

@@ -681,6 +681,17 @@ def run_vitest_benchmarking_tests(
     vitest_env["CODEFLASH_PERF_TARGET_DURATION_MS"] = str(target_duration_ms)
     vitest_env["CODEFLASH_PERF_STABILITY_CHECK"] = "true" if stability_check else "false"
     vitest_env["CODEFLASH_LOOP_INDEX"] = "1"
+    # Warmup and calibration for accurate benchmarking
+    vitest_env["CODEFLASH_PERF_WARMUP_ITERATIONS"] = "5"
+    vitest_env["CODEFLASH_PERF_MIN_TIME_NS"] = "5000"  # 5us minimum time for calibration
+
+    # Expose GC for accurate benchmarking (allows capturePerf to force GC before timing)
+    existing_node_options = vitest_env.get("NODE_OPTIONS", "")
+    if "--expose-gc" not in existing_node_options:
+        existing_node_options = f"{existing_node_options} --expose-gc".strip()
+    if "--max-old-space-size" not in existing_node_options:
+        existing_node_options = f"{existing_node_options} --max-old-space-size=4096".strip()
+    vitest_env["NODE_OPTIONS"] = existing_node_options
 
     # Set test module for marker identification (use first test file as reference)
     if test_files: