Merge pull request #1539 from codeflash-ai/faster-file-discovery

feat: replace rglob with os.walk and per-language directory pruning

Author: KRRT7

.codex/config.toml

@@ -0,0 +1,4 @@
+[mcp_servers.tessl]
+type = "stdio"
+command = "tessl"
+args = [ "mcp", "start" ]

.codex/skills/.gitignore

@@ -0,0 +1,12 @@
+{
+  "mcpServers": {
+    "tessl": {
+      "type": "stdio",
+      "command": "tessl",
+      "args": [
+        "mcp",
+        "start"
+      ]
+    }
+  }
+}

.gemini/settings.json

@@ -146,6 +146,27 @@ def generic_visit(self, node: ast.AST) -> None:
 # Multi-language support helpers
 # =============================================================================
 
+_VCS_EXCLUDES = frozenset({".git", ".hg", ".svn"})
+
+
+def parse_dir_excludes(patterns: frozenset[str]) -> tuple[frozenset[str], tuple[str, ...], tuple[str, ...]]:
+    """Split glob patterns into exact names, prefixes, and suffixes.
+
+    Patterns ending with ``*`` become prefix matches, patterns starting with ``*``
+    become suffix matches, and plain strings become exact matches.
+    """
+    exact: set[str] = set()
+    prefixes: list[str] = []
+    suffixes: list[str] = []
+    for p in patterns:
+        if p.endswith("*"):
+            prefixes.append(p[:-1])
+        elif p.startswith("*"):
+            suffixes.append(p[1:])
+        else:
+            exact.add(p)
+    return frozenset(exact), tuple(prefixes), tuple(suffixes)
+
 
 def get_files_for_language(
     module_root_path: Path, ignore_paths: list[Path] | None = None, language: Language | None = None
@@ -164,37 +185,44 @@ def get_files_for_language(
     if ignore_paths is None:
         ignore_paths = []
 
+    all_patterns: frozenset[str]
     if language is not None:
         support = get_language_support(language)
         extensions = support.file_extensions
+        all_patterns = support.dir_excludes | _VCS_EXCLUDES
     else:
         extensions = tuple(get_supported_extensions())
-
-    # Default directory patterns to always exclude for JS/TS
-    js_ts_default_excludes = {
-        "node_modules",
-        "dist",
-        "build",
-        ".next",
-        ".nuxt",
-        "coverage",
-        ".cache",
-        ".turbo",
-        ".vercel",
-        "__pycache__",
-    }
-
-    files = []
-    for ext in extensions:
-        pattern = f"*{ext}"
-        for file_path in module_root_path.rglob(pattern):
-            # Check explicit ignore paths
-            if any(file_path.is_relative_to(ignore_path) for ignore_path in ignore_paths):
-                continue
-            # Check default JS/TS excludes in path parts
-            if any(part in js_ts_default_excludes for part in file_path.parts):
-                continue
-            files.append(file_path)
+        all_patterns = _VCS_EXCLUDES
+        for lang in Language:
+            if is_language_supported(lang):
+                all_patterns = all_patterns | get_language_support(lang).dir_excludes
+
+    dir_excludes, prefixes, suffixes = parse_dir_excludes(all_patterns)
+
+    ignore_dirs: set[str] = set()
+    ignore_files: set[Path] = set()
+    for p in ignore_paths:
+        p = Path(p) if not isinstance(p, Path) else p
+        if p.is_file():
+            ignore_files.add(p)
+        else:
+            ignore_dirs.add(str(p))
+
+    files: list[Path] = []
+    for dirpath, dirnames, filenames in os.walk(module_root_path):
+        dirnames[:] = [
+            d
+            for d in dirnames
+            if d not in dir_excludes
+            and not (prefixes and d.startswith(prefixes))
+            and not (suffixes and d.endswith(suffixes))
+            and str(Path(dirpath) / d) not in ignore_dirs
+        ]
+        for fname in filenames:
+            if fname.endswith(extensions):
+                fpath = Path(dirpath, fname)
+                if fpath not in ignore_files:
+                    files.append(fpath)
     return files
 
 

.gemini/skills/.gitignore

@@ -254,6 +254,14 @@ def comment_prefix(self) -> str:
         """Like # or //."""
         ...
 
+    @property
+    def dir_excludes(self) -> frozenset[str]:
+        """Directory name patterns to skip during file discovery.
+
+        Supports glob wildcards: "name" for exact, "prefix*" for startswith, "*suffix" for endswith.
+        """
+        ...
+
     # === Discovery ===
 
     def discover_functions(

codeflash/discovery/functions_to_optimize.py

@@ -63,6 +63,10 @@ def test_framework(self) -> str:
     def comment_prefix(self) -> str:
         return "//"
 
+    @property
+    def dir_excludes(self) -> frozenset[str]:
+        return frozenset({"node_modules", "dist", "build", ".next", ".nuxt", "coverage", ".cache", ".turbo", ".vercel"})
+
     # === Discovery ===
 
     def discover_functions(

codeflash/languages/base.py

@@ -75,6 +75,37 @@ def test_framework(self) -> str:
     def comment_prefix(self) -> str:
         return "#"
 
+    @property
+    def dir_excludes(self) -> frozenset[str]:
+        return frozenset(
+            {
+                "__pycache__",
+                ".venv",
+                "venv",
+                ".tox",
+                ".nox",
+                ".eggs",
+                ".mypy_cache",
+                ".ruff_cache",
+                ".pytest_cache",
+                ".hypothesis",
+                "htmlcov",
+                ".pytype",
+                ".pyre",
+                ".pybuilder",
+                ".ipynb_checkpoints",
+                ".codeflash",
+                ".cache",
+                ".complexipy_cache",
+                "build",
+                "dist",
+                "sdist",
+                ".coverage*",
+                ".pyright*",
+                "*.egg-info",
+            }
+        )
+
     # === Discovery ===
 
     def discover_functions(

codeflash/languages/javascript/support.py

@@ -20,7 +20,7 @@
       "version": "0.13.0"
     },
     "tessl/pypi-pydantic": {
-      "version": "1.10.0"
+      "version": "2.11.0"
     },
     "tessl/pypi-humanize": {
       "version": "4.13.0"
@@ -35,7 +35,7 @@
       "version": "3.4.0"
     },
     "tessl/pypi-sentry-sdk": {
-      "version": "1.45.0"
+      "version": "2.36.0"
     },
     "tessl/pypi-parameterized": {
       "version": "0.9.0"
@@ -44,10 +44,10 @@
       "version": "0.4.0"
     },
     "tessl/pypi-rich": {
-      "version": "13.9.0"
+      "version": "14.1.0"
     },
     "tessl/pypi-lxml": {
-      "version": "5.4.0"
+      "version": "6.0.0"
     },
     "tessl/pypi-crosshair-tool": {
       "version": "0.0.0"
@@ -64,17 +64,20 @@
     "tessl/pypi-filelock": {
       "version": "3.19.0"
     },
-    "codeflash/codeflash-rules": {
-      "version": "0.1.0"
+    "tessl/pypi-ipython": {
+      "version": "9.5.0"
     },
-    "codeflash/codeflash-docs": {
-      "version": "0.1.0"
+    "tessl/pypi-mypy": {
+      "version": "1.17.0"
     },
-    "codeflash/codeflash-skills": {
-      "version": "0.2.0"
+    "tessl/pypi-ty": {
+      "version": "0.0.0"
+    },
+    "tessl/pypi-types-jsonschema": {
+      "version": "3.2.0"
     },
-    "tessl-labs/tessl-skill-eval-scenarios": {
-      "version": "0.0.5"
+    "tessl/pypi-uv": {
+      "version": "0.8.0"
     }
   }
 }