Performance: optimisation massive du démarrage du build (hashing métadonnées, élagage de scan et préparation asynchrone)

Author: raidos23

Core/Locking/__init__.py

@@ -77,17 +77,39 @@ def installed_distributions_snapshot() -> dict[str, str]:
 def included_workspace_files(
     workspace: Path, exclude_patterns: list[str]
 ) -> list[Path]:
+    """
+    Return a list of files to be included in the workspace snapshot.
+    Optimized to skip excluded directories early.
+    """
     included: list[Path] = []
-    for path in sorted(workspace.rglob("*")):
-        if not path.is_file():
-            continue
-        rel = path.relative_to(workspace).as_posix()
-        if rel.startswith(".ark/"):
-            continue
-        if any(_matches_exclude_pattern(rel, pattern) for pattern in exclude_patterns):
-            continue
-        included.append(path)
-    return included
+    ws_str = str(workspace.resolve())
+    
+    # Prune list for os.walk
+    prune_dirs = {".git", ".ark", "__pycache__", "venv", ".venv", "build", "dist"}
+    
+    import os
+    for root, dirs, files in os.walk(ws_str):
+        # 1. Early pruning of common heavy/system directories
+        dirs[:] = [d for d in dirs if d not in prune_dirs]
+        
+        # 2. Apply custom exclude patterns to directories
+        rel_root = os.path.relpath(root, ws_str)
+        if rel_root == ".":
+            rel_root = ""
+            
+        if rel_root:
+            if any(_matches_exclude_pattern(rel_root + "/", p) for p in exclude_patterns):
+                dirs[:] = [] # Stop recursion here
+                continue
+
+        # 3. Process files
+        for f in files:
+            rel_path = os.path.join(rel_root, f) if rel_root else f
+            if any(_matches_exclude_pattern(rel_path, p) for p in exclude_patterns):
+                continue
+            included.append(Path(root) / f)
+            
+    return sorted(included)
 
 
 def _matches_exclude_pattern(relative_path: str, pattern: str) -> bool:
@@ -106,14 +128,21 @@ def _matches_exclude_pattern(relative_path: str, pattern: str) -> bool:
 
 
 def compute_workspace_hash(workspace: Path, exclude_patterns: list[str]) -> str:
+    """
+    Compute a fast hash of the workspace using file metadata (path, size, mtime).
+    This is much faster than reading full content for large projects.
+    """
     digest = sha256()
     for path in included_workspace_files(workspace, exclude_patterns):
-        rel = path.relative_to(workspace).as_posix().encode("utf-8")
-        digest.update(rel)
-        digest.update(b"\0")
-        digest.update(path.read_bytes())
-        digest.update(b"\0")
-    return "sha256:" + digest.hexdigest()
+        try:
+            stat = path.stat()
+            # We hash the relative path, size, and mtime
+            rel = path.relative_to(workspace).as_posix()
+            entry = f"{rel}|{stat.st_size}|{stat.st_mtime}"
+            digest.update(entry.encode("utf-8"))
+        except Exception:
+            continue
+    return "metadata-sha256:" + digest.hexdigest()
 
 
 def get_git_commit_hash(workspace: Path) -> str | None:

Core/deps_analyser/analyser.py

@@ -708,6 +708,27 @@ def collect_project_dependencies(
     all_deps: set[str] = set()
 
     # 1. Scan configuration files (high priority, more accurate than imports)
+    
+    # requirements.txt / requirements.in (Very high priority, can skip full scan if found)
+    reqs_found = False
+    for req_file in ["requirements.txt", "requirements.in"]:
+        path = os.path.join(workspace_dir, req_file)
+        if os.path.isfile(path):
+            try:
+                with open(path, "r", encoding="utf-8") as f:
+                    for line in f:
+                        line = line.strip()
+                        if line and not line.startswith(("#", "-")):
+                            # Basic parsing of requirement line
+                            name = re.split(r"[<>=!~\s]", line)[0].strip()
+                            if name:
+                                all_deps.add(name)
+                                reqs_found = True
+            except Exception:
+                pass
+    
+    # If we found explicit requirements, we can potentially skip the expensive scan
+    # unless specific validation is requested. For now, we continue but mark it.
 
     # pyproject.toml
     pyproject_path = os.path.join(workspace_dir, "pyproject.toml")
@@ -793,6 +814,10 @@ def collect_project_dependencies(
             pass
 
     # 2. Scan imports in all Python files (fallback/validation)
+    # OPTIMIZATION: If we already found dependencies in config files, we skip the slow full scan
+    if reqs_found and all_deps:
+        return all_deps
+
     modules_from_imports = set()
     workspace_python_files = []
     for root, dirs, files in os.walk(workspace_dir):

Ui/Gui/Compilation/compiler.py

@@ -60,6 +60,8 @@ def __init__(
         engine_id: str,
         context: BuildContext,
         engine_config: Optional[Dict[str, Any]] = None,
+        ark_config: Optional[Dict[str, Any]] = None,
+        python_version: str | None = None,
     ):
         """
         Initialize the compilation thread.
@@ -69,6 +71,8 @@ def __init__(
         self.engine_id = engine_id
         self.context = context
         self.engine_config = engine_config
+        self.ark_config = ark_config
+        self.python_version = python_version
         self.cancel_requested = False
         self.start_time: Optional[datetime] = None
 
@@ -79,6 +83,30 @@ def run(self) -> None:
 
         self.progress_update.emit(0, "Process started")
 
+        # 1. Build Preparation (Background)
+        if self.ark_config:
+            try:
+                from Core.Locking import build_lock_payload, write_lock_files
+                from engine_sdk import build_context_object_from_ark_config
+                from Ui.Cli.helpers import engine_config_from_lock
+
+                self.output_ready.emit("🔒 Génération du verrou de compilation (lock file)...")
+                lock_payload = build_lock_payload(
+                    self.workspace,
+                    self.ark_config,
+                    engine_id=self.engine_id,
+                    python_version=self.python_version,
+                )
+                write_lock_files(self.workspace, lock_payload)
+                
+                # Update context and config from the fresh lock
+                self.context = build_context_object_from_ark_config(self.ark_config)
+                self.engine_config = engine_config_from_lock(lock_payload)
+            except Exception as e:
+                self.error_ready.emit(f"Error during preparation: {e}")
+                self.finished.emit(1)
+                return
+
         def _on_stdout(line: str):
             self.output_ready.emit(line)
             self._update_progress(line)
@@ -223,11 +251,13 @@ def compile_from_context(
         engine_id: str,
         context: BuildContext,
         engine_config: Optional[Dict[str, Any]] = None,
+        ark_config: Optional[Dict[str, Any]] = None,
+        python_version: str | None = None,
     ) -> bool:
         """
         Start an async compilation from a BuildContext.
         """
-        if self.is_running:
+        if self.is_running():
             self.log_message.emit("warning", "Compilation already in progress")
             return False
 
@@ -250,6 +280,8 @@ def compile_from_context(
             engine_id=engine_id,
             context=context,
             engine_config=engine_config,
+            ark_config=ark_config,
+            python_version=python_version,
         )
 
         # Connecter les signaux

Ui/Gui/Dialogs/CompilerDialog.py

@@ -227,16 +227,9 @@ def _t(fr: str, en: str) -> str:
     except Exception:
         pass
 
-    # 4. Generate Lock and Context (Exact CLI Code)
+    # 4. Preparation (Resolve context and engine config)
     try:
-        log_i18n_level(
-            self,
-            "info",
-            "🔒 Génération du verrou de compilation (lock file)...",
-            "🔒 Generating build lock file...",
-        )
-        
-        # Resolve Python version for locking
+        # Resolve Python version for locking (will be used in background thread)
         python_version = None
         try:
             from Core.Compiler.utils import get_interpreter_version_str
@@ -251,17 +244,10 @@ def _t(fr: str, en: str) -> str:
         except Exception:
             pass
 
-        lock_payload = build_lock_payload(
-            Path(self.workspace_dir),
-            validated.config,
-            engine_id=engine_id,
-            python_version=python_version,
-        )
-        write_lock_files(Path(self.workspace_dir), lock_payload)
-
-        # Context and Config from lock/config (source of truth)
+        # Use BuildContext helper to get initial context
         context = build_context_object_from_ark_config(validated.config)
-        engine_config = engine_config_from_lock(lock_payload)
+        # In UI mode, we use current overrides. The thread will update them from the fresh lock.
+        engine_config = getattr(self, "_config_overrides", {})
     except Exception as e:
         log_i18n_level(
             self,
@@ -280,7 +266,7 @@ def _t(fr: str, en: str) -> str:
         engine = None
     if engine is None:
         try:
-            engine = create(engine_id)
+            engine = engines_loader.create(engine_id)
         except Exception:
             pass
     engine_name = getattr(engine, "name", engine_id)
@@ -369,6 +355,8 @@ def _after_bcasl(_report=None) -> None:
                 engine_id=engine_id,
                 context=context,
                 engine_config=engine_config,
+                ark_config=validated.config, # Passing this triggers background lock generation
+                python_version=python_version,
             )
 
             if not success: