Merge branch 'multi-language' of github.com:codeflash-ai/codeflash into multi-language

Author: mohammedahmed18

code_to_optimize_js/codeflash-jest-helper.js

@@ -193,7 +193,7 @@ function initDatabase() {
             )
         `);
     } catch (e) {
-        console.log('[codeflash] Failed to initialize SQLite:', e.message);
+        console.error('[codeflash] Failed to initialize SQLite:', e.message);
         useSqlite = false;
     }
 }

codeflash/discovery/discover_unit_tests.py

@@ -554,9 +554,7 @@ def filter_test_files_by_imports(
     return filtered_map
 
 
-def _detect_language_from_functions(
-    file_to_funcs: dict[Path, list[FunctionToOptimize]] | None,
-) -> str | None:
+def _detect_language_from_functions(file_to_funcs: dict[Path, list[FunctionToOptimize]] | None) -> str | None:
     """Detect language from the functions to optimize.
 
     Args:
@@ -576,9 +574,7 @@ def _detect_language_from_functions(
 
 
 def discover_tests_for_language(
-    cfg: TestConfig,
-    language: str,
-    file_to_funcs_to_optimize: dict[Path, list[FunctionToOptimize]] | None,
+    cfg: TestConfig, language: str, file_to_funcs_to_optimize: dict[Path, list[FunctionToOptimize]] | None
 ) -> tuple[dict[str, set[FunctionCalledInTest]], int, int]:
     """Discover tests using language-specific support.
 
@@ -656,6 +652,10 @@ def discover_unit_tests(
 
     # Route to language-specific test discovery for non-Python languages
     if language and language != "python":
+        # For JavaScript/TypeScript, tests_project_rootdir should be tests_root itself
+        # The Jest helper will be configured to NOT include "tests." prefix to match
+        if language in ("javascript", "typescript"):
+            cfg.tests_project_rootdir = cfg.tests_root
         return discover_tests_for_language(cfg, language, file_to_funcs_to_optimize)
 
     # Existing Python logic

codeflash/languages/javascript/runtime/codeflash-jest-helper.js

@@ -301,10 +301,12 @@ function capture(funcName, lineId, fn, ...args) {
         // Get relative path from cwd and convert to module-style path
         const path = require('path');
         const relativePath = path.relative(process.cwd(), currentTestPath);
-        // Convert to Python module-style path (e.g., "tests/test_foo.test.js" -> "tests.test_foo.test")
-        // This matches what Jest's junit XML produces
+        // Convert to module-style path (e.g., "tests/test_foo.test.js" -> "test_foo.test")
+        // Strip the leading "tests/" directory to match what Jest's junit XML produces
+        // and what module_name_from_file_path(test_file, tests_root) generates
         testModulePath = relativePath
             .replace(/\\/g, '/')       // Handle Windows paths
+            .replace(/^tests\//, '')   // Strip leading "tests/" directory
             .replace(/\.js$/, '')       // Remove .js extension
             .replace(/\.test$/, '.test') // Keep .test suffix
             .replace(/\//g, '.');       // Convert path separators to dots
@@ -404,9 +406,11 @@ function capturePerf(funcName, lineId, fn, ...args) {
         // Get relative path from cwd and convert to module-style path
         const path = require('path');
         const relativePath = path.relative(process.cwd(), currentTestPath);
-        // Convert to Python module-style path (e.g., "tests/test_foo.test.js" -> "tests.test_foo.test")
+        // Convert to module-style path (e.g., "tests/test_foo.test.js" -> "test_foo.test")
+        // Strip leading "tests/" to match XML module path format
         testModulePath = relativePath
             .replace(/\\/g, '/')
+            .replace(/^tests\//, '')   // Strip leading "tests/" directory
             .replace(/\.js$/, '')
             .replace(/\.test$/, '.test')
             .replace(/\//g, '.');
@@ -544,9 +548,11 @@ function capturePerfLooped(funcName, lineId, fn, ...args) {
         // Get relative path from cwd and convert to module-style path
         const path = require('path');
         const relativePath = path.relative(process.cwd(), currentTestPath);
-        // Convert to Python module-style path (e.g., "tests/test_foo.test.js" -> "tests.test_foo.test")
+        // Convert to module-style path (e.g., "tests/test_foo.test.js" -> "test_foo.test")
+        // Strip leading "tests/" to match XML module path format
         testModulePath = relativePath
             .replace(/\\/g, '/')
+            .replace(/^tests\//, '')   // Strip leading "tests/" directory
             .replace(/\.js$/, '')
             .replace(/\.test$/, '.test')
             .replace(/\//g, '.');
@@ -632,8 +638,8 @@ function capturePerfLooped(funcName, lineId, fn, ...args) {
             break;
         }
 
-        // Stop if we've reached min loops AND exceeded time limit
-        if (loopCount >= MIN_LOOPS && elapsedMs >= TARGET_DURATION_MS) {
+        // Stop if we've reached min loops AND exceeded a reasonable portion of time
+        if (loopCount >= MIN_LOOPS && elapsedMs >= TARGET_DURATION_MS * 0.8) {
             break;
         }
 

codeflash/optimization/function_optimizer.py

@@ -601,6 +601,24 @@ def generate_and_instrument_tests(
             for test_index in range(n_tests)
         ]
 
+        # For JavaScript/TypeScript, copy all runtime files to tests directory
+        if language in ("javascript", "typescript"):
+            import shutil
+
+            from codeflash.languages.javascript.runtime import get_all_runtime_files
+
+            # Copy all runtime files (helper, serializer, comparator, etc.)
+            for runtime_file_source in get_all_runtime_files():
+                runtime_file_dest = self.test_cfg.tests_root / runtime_file_source.name
+
+                # Copy file if it doesn't exist or is outdated
+                if (
+                    not runtime_file_dest.exists()
+                    or runtime_file_source.stat().st_mtime > runtime_file_dest.stat().st_mtime
+                ):
+                    shutil.copy2(runtime_file_source, runtime_file_dest)
+                    logger.debug(f"Copied {runtime_file_source.name} to {runtime_file_dest}")
+
         test_results = self.generate_tests(
             testgen_context=code_context.testgen_context,
             helper_functions=code_context.helper_functions,
@@ -2238,6 +2256,7 @@ def establish_original_code_baseline(
             for result in behavioral_results
             if (result.test_type == TestType.GENERATED_REGRESSION and not result.did_pass)
         ]
+
         if total_timing == 0:
             logger.warning("The overall summed benchmark runtime of the original function is 0, couldn't run tests.")
             console.rule()
@@ -2587,6 +2606,7 @@ def run_and_parse_tests(
 
         if testing_type in {TestingMode.BEHAVIOR, TestingMode.PERFORMANCE}:
             # For JavaScript behavior tests, skip SQLite cleanup - files needed for JS-native comparison
+            #TODO (ali): make sure it works fine
             is_js_for_original_code = self.is_js and optimization_iteration == 0
             is_js_behavior = (self.is_js and testing_type == TestingMode.BEHAVIOR) or is_js_for_original_code
 

codeflash/verification/parse_test_output.py

@@ -519,9 +519,11 @@ def parse_jest_test_xml(
                     timed_out = True
 
             # Find matching timing markers for this test
-            # Jest test names in markers match the full test name
-            # the test_name is the string in describe and test arguments, the group 2 is the same but with _ instead of spaces
-            matching_starts = [m for m in start_matches if m.group(2).replace("_", " ") in test_name]
+            # Jest test names in markers are sanitized (spaces → underscores)
+            # The marker format is: testModulePath:testFunctionName:funcName:loopIndex:lineId
+            # We need to check group(2) (testFunctionName) and handle sanitization
+            sanitized_test_name = re.sub(r"[!#: ()\[\]{}|\\/*?^$.+\-]", "_", test_name)
+            matching_starts = [m for m in start_matches if sanitized_test_name in m.group(2)]
 
             if not matching_starts:
                 # No timing markers found - add basic result
@@ -1024,6 +1026,7 @@ def parse_test_results(
     # (comparison happens in JavaScript land via compare_javascript_test_results)
     if not skip_sqlite_cleanup:
         get_run_tmp_file(Path(f"test_return_values_{optimization_iteration}.sqlite")).unlink(missing_ok=True)
+
     results = merge_test_results(test_results_xml, test_results_data, test_config.test_framework)
 
     all_args = False

codeflash/verification/test_runner.py

@@ -2,6 +2,7 @@
 
 import contextlib
 import shlex
+import shutil
 import subprocess
 import sys
 from pathlib import Path
@@ -13,6 +14,7 @@
 from codeflash.code_utils.config_consts import TOTAL_LOOPING_TIME_EFFECTIVE
 from codeflash.code_utils.coverage_utils import prepare_coverage_files
 from codeflash.code_utils.shell_utils import get_cross_platform_subprocess_run_args
+from codeflash.languages.javascript.runtime import get_all_runtime_files
 from codeflash.models.models import TestFiles, TestType
 
 if TYPE_CHECKING:
@@ -22,6 +24,23 @@
 BENCHMARKING_BLOCKLISTED_PLUGINS = ["codspeed", "cov", "benchmark", "profiling", "xdist", "sugar"]
 
 
+def _ensure_js_runtime_files(js_project_root: Path) -> None:
+    """Ensure JavaScript runtime files are present in the project root.
+
+    Copies codeflash-jest-helper.js and related files to the JS project root
+    if they don't already exist or are outdated.
+
+    Args:
+        js_project_root: The JavaScript project root directory.
+    """
+    for runtime_file in get_all_runtime_files():
+        dest_path = js_project_root / runtime_file.name
+        # Always copy to ensure we have the latest version
+        if not dest_path.exists() or dest_path.stat().st_mtime < runtime_file.stat().st_mtime:
+            shutil.copy2(runtime_file, dest_path)
+            logger.debug(f"Copied {runtime_file.name} to {js_project_root}")
+
+
 def _find_js_project_root(file_path: Path) -> Path | None:
     """Find the JavaScript/TypeScript project root by looking for package.json.
 
@@ -82,6 +101,9 @@ def run_jest_behavioral_tests(
     effective_cwd = js_project_root if js_project_root else cwd
     logger.debug(f"Jest working directory: {effective_cwd}")
 
+    # Ensure runtime files (codeflash-jest-helper.js, etc.) are present
+    _ensure_js_runtime_files(effective_cwd)
+
     # Coverage output directory
     coverage_dir = get_run_tmp_file(Path("jest_coverage"))
     coverage_json_path = coverage_dir / "coverage-final.json" if enable_coverage else None
@@ -372,11 +394,13 @@ def run_jest_benchmarking_tests(
         test_paths: TestFiles object containing test file information.
         test_env: Environment variables for the test run.
         cwd: Working directory for running tests.
-        timeout: Optional timeout in seconds.
+        timeout: Optional timeout in seconds for the subprocess.
         js_project_root: JavaScript project root (directory containing package.json).
         min_loops: Minimum number of loops to run for each test case.
         max_loops: Maximum number of loops to run for each test case.
-        target_duration_ms: Target duration in milliseconds for looping.
+        target_duration_ms: Target TOTAL duration in milliseconds for looping.
+            This is divided among test cases since JavaScript uses capturePerfLooped
+            which loops internally per test case, unlike Python's external looping.
         stability_check: Whether to enable stability-based early stopping.
 
     Returns:
@@ -388,6 +412,13 @@ def run_jest_benchmarking_tests(
     # Get performance test files
     test_files = [str(file.benchmarking_file_path) for file in test_paths.test_files if file.benchmarking_file_path]
 
+    # Count approximate number of test cases to divide time budget
+    # JavaScript's capturePerfLooped loops internally per test case, so we need to divide
+    # the total time budget among test cases to avoid timeout
+    num_test_cases = len(test_files) * 10  # Estimate ~10 test cases per file (conservative)
+    # Use at least 500ms per test case for fast functions, cap at 2 seconds
+    per_test_duration_ms = max(500, min(2000, target_duration_ms // max(1, num_test_cases)))
+
     # Use provided js_project_root, or detect it as fallback
     if js_project_root is None and test_files:
         first_test_file = Path(test_files[0])
@@ -396,6 +427,9 @@ def run_jest_benchmarking_tests(
     effective_cwd = js_project_root if js_project_root else cwd
     logger.debug(f"Jest benchmarking working directory: {effective_cwd}")
 
+    # Ensure runtime files (codeflash-jest-helper.js, etc.) are present
+    _ensure_js_runtime_files(effective_cwd)
+
     # Build Jest command for performance tests
     jest_cmd = ["npx", "jest", "--reporters=default", "--reporters=jest-junit", "--runInBand", "--forceExit"]
 
@@ -425,12 +459,21 @@ def run_jest_benchmarking_tests(
     # Looping configuration for stable performance measurements
     jest_env["CODEFLASH_MIN_LOOPS"] = str(min_loops)
     jest_env["CODEFLASH_MAX_LOOPS"] = str(max_loops)
-    jest_env["CODEFLASH_TARGET_DURATION_MS"] = str(target_duration_ms)
+    # Use per-test duration instead of total duration
+    jest_env["CODEFLASH_TARGET_DURATION_MS"] = str(per_test_duration_ms)
     jest_env["CODEFLASH_STABILITY_CHECK"] = "true" if stability_check else "false"
     # Seed random number generator for reproducible test runs across original and optimized code
     jest_env["CODEFLASH_RANDOM_SEED"] = "42"
 
+    # Calculate subprocess timeout based on expected benchmarking time
+    # For slow O(n²) functions, a single call might take seconds, so add generous buffer
+    # Allow for Jest startup overhead (10s) + per-test-case benchmarking + safety margin
+    expected_benchmarking_time_s = (num_test_cases * per_test_duration_ms) / 1000 + 10
+    # Use at least 60 seconds to handle slow functions, or calculated time with 2x margin
+    subprocess_timeout = max(60, int(expected_benchmarking_time_s * 2))
+
     logger.debug(f"Running Jest benchmarking tests: {' '.join(jest_cmd)}")
+    logger.debug(f"Jest benchmarking config: {num_test_cases} estimated test cases, {per_test_duration_ms}ms per test, min_loops={min_loops}, {subprocess_timeout}s subprocess timeout")
 
     # Calculate subprocess timeout: for Jest benchmarking, we need enough time for all tests to complete
     # Each test can run up to target_duration_ms (default 10s) for stable measurements

codeflash/verification/verifier.py

@@ -27,7 +27,7 @@ def generate_tests(
     test_index: int,
     test_path: Path,
     test_perf_path: Path,
-) -> tuple[str, str, Path] | None:
+) -> tuple[str, str, str, Path, Path] | None:
     # TODO: Sometimes this recreates the original Class definition. This overrides and messes up the original
     #  class import. Remove the recreation of the class definition
     start_time = time.perf_counter()