diff --git a/flytekit/image_spec/default_builder.py b/flytekit/image_spec/default_builder.py index 96ee5f80c7..acc2546b25 100644 --- a/flytekit/image_spec/default_builder.py +++ b/flytekit/image_spec/default_builder.py @@ -21,6 +21,7 @@ ImageSpec, ImageSpecBuilder, _find_git_root, + discover_nix_flake_lock_path_inputs, discover_nix_flake_path_inputs, ) from flytekit.tools.ignore import DockerIgnore, GitIgnore, IgnoreGroup, StandardIgnore @@ -389,23 +390,22 @@ def _copy_local_packages_and_update_lock(image_spec: ImageSpec, tmp_dir: Path): flake_path_replacements = {} inputs = discover_nix_flake_path_inputs(lock_dir, flake_content) - for inp in inputs: - full_spec = inp.full_spec - old_rel_path = inp.old_rel_path - src_path = inp.src_path + # Also discover transitive path deps from flake.lock (e.g. minos2_rust -> rust-exautils) + already_discovered = {str(inp.src_path) for inp in inputs} + transitive_inputs = discover_nix_flake_lock_path_inputs( + lock_dir, flake_lock_content, already_discovered + ) + def _copy_nix_input(inp): + src_path = inp.src_path if not src_path.exists(): raise ValueError(f"Nix flake path input does not exist: {src_path}") - git_root = _find_git_root(str(src_path)) if git_root is None: raise ValueError(f"Could not find git root for Nix flake path input: {src_path}") - rel_path = os.path.relpath(path=str(src_path), start=str(git_root)) target_path = local_packages_dir / rel_path target_path.parent.mkdir(parents=True, exist_ok=True) - - # Copy with ignore patterns if src_path.is_dir(): ignore_group_dep = IgnoreGroup(str(src_path), [GitIgnore, DockerIgnore, StandardIgnore]) files_to_copy_dep, _ = ls_files( @@ -421,14 +421,21 @@ def _copy_local_packages_and_update_lock(image_spec: ImageSpec, tmp_dir: Path): shutil.copy2(file_to_copy, file_dst) else: shutil.copy2(str(src_path), str(target_path)) + return rel_path + # Copy and rewrite direct inputs (referenced in flake.nix) + for inp in inputs: + rel_path = _copy_nix_input(inp) new_rel_path = f"local_packages/{rel_path}" - - # Update flake.nix content in-memory - flake_content = flake_content.replace(full_spec, f"path:{new_rel_path}") - - # Track mapping for flake.lock updates - flake_path_replacements[old_rel_path] = new_rel_path + flake_content = flake_content.replace(inp.full_spec, f"path:{new_rel_path}") + flake_path_replacements[inp.old_rel_path] = new_rel_path + + # Copy transitive inputs (only in flake.lock, resolved relative to parent) + # Only copy them - do NOT rewrite their paths in the top-level flake.lock + # because nix resolves transitive dep paths relative to the parent input, + # and the relative structure is preserved under local_packages/ + for inp in transitive_inputs: + _copy_nix_input(inp) # Update flake.lock JSON for nodes that reference local path inputs if flake_path_replacements: diff --git a/flytekit/image_spec/image_spec.py b/flytekit/image_spec/image_spec.py index 8f6addf76a..4ef71d5c62 100644 --- a/flytekit/image_spec/image_spec.py +++ b/flytekit/image_spec/image_spec.py @@ -82,6 +82,75 @@ def discover_nix_flake_path_inputs( return inputs +def discover_nix_flake_lock_path_inputs( + lock_dir: typing.Union[str, pathlib.Path], + flake_lock_content: typing.Optional[str] = None, + already_discovered: typing.Optional[typing.Set[str]] = None, +) -> List[NixFlakePathInput]: + """Discover transitive path inputs from flake.lock not already found in flake.nix. + + Nix flake.lock records all resolved inputs including transitive ones. When a + direct path input (e.g. minos2_rust) itself depends on another path input + (e.g. rust-exautils), that transitive dependency only appears in flake.lock, + not in the top-level flake.nix. This function discovers those transitive path + dependencies so they can be copied into the Docker build context. + + Args: + lock_dir: Directory containing flake.lock + flake_lock_content: Optional preloaded flake.lock JSON; if None, the file is read + already_discovered: Set of resolved src_path strings already discovered from + flake.nix (to avoid duplicates) + + Returns: + List of NixFlakePathInput entries for transitive path inputs. + """ + import json + + lock_dir_path = pathlib.Path(lock_dir) + if flake_lock_content is None: + flake_lock_path = lock_dir_path / "flake.lock" + if not flake_lock_path.exists(): + return [] + flake_lock_content = flake_lock_path.read_text() + + if already_discovered is None: + already_discovered = set() + + try: + lock_data = json.loads(flake_lock_content) + except (json.JSONDecodeError, ValueError): + return [] + + nodes = lock_data.get("nodes", {}) + inputs: List[NixFlakePathInput] = [] + + for node in nodes.values(): + locked = node.get("locked", {}) + if locked.get("type") != "path": + continue + + old_rel_path = locked.get("path") + if not old_rel_path: + continue + + if os.path.isabs(old_rel_path): + src_path = pathlib.Path(old_rel_path).resolve() + else: + src_path = (lock_dir_path / old_rel_path).resolve() + + if str(src_path) in already_discovered: + continue + + if not src_path.exists(): + continue + + full_spec = f"path:{old_rel_path}" + inputs.append(NixFlakePathInput(full_spec=full_spec, old_rel_path=old_rel_path, src_path=src_path)) + already_discovered.add(str(src_path)) + + return inputs + + def _compute_directory_digest(dir_path: pathlib.Path) -> str: """Compute a stable digest for a directory mirroring builder copy logic (respects .gitignore/.dockerignore).""" # Imports here to avoid circular imports at module load time @@ -418,6 +487,12 @@ def tag(self) -> str: try: flake_inputs = discover_nix_flake_path_inputs(lock_dir) + # Also discover transitive path deps from flake.lock + already_discovered = {str(inp.src_path) for inp in flake_inputs} + transitive_inputs = discover_nix_flake_lock_path_inputs( + lock_dir, None, already_discovered + ) + flake_inputs = flake_inputs + transitive_inputs except Exception: flake_inputs = []