diff --git a/py-rattler-build/rust/src/build.rs b/py-rattler-build/rust/src/build.rs index 78c973ce9..8c725bd4c 100644 --- a/py-rattler-build/rust/src/build.rs +++ b/py-rattler-build/rust/src/build.rs @@ -225,6 +225,7 @@ pub fn build_rendered_variant_py( finalized_sources: None, finalized_cache_dependencies: None, finalized_cache_sources: None, + cached_prefix_info: None, build_summary: Arc::new(Mutex::new(BuildSummary::default())), system_tools: SystemTools::default(), extra_meta: None, diff --git a/src/build.rs b/src/build.rs index 4752ac802..ffb466858 100644 --- a/src/build.rs +++ b/src/build.rs @@ -137,10 +137,12 @@ pub async fn run_build( // This will build or restore staging caches and return their dependencies/sources if inherited let staging_result = output.process_staging_caches(tool_configuration).await?; - // If we inherit from a staging cache, store its dependencies and sources - if let Some((deps, sources)) = staging_result { + // If we inherit from a staging cache, store its dependencies, sources, + // and cached prefix info (for linking checks) + if let Some((deps, sources, cached_prefix_info)) = staging_result { output.finalized_cache_dependencies = Some(deps); output.finalized_cache_sources = Some(sources); + output.cached_prefix_info = Some(cached_prefix_info); } // Fetch sources for this output diff --git a/src/lib.rs b/src/lib.rs index e9cf8baec..ab44116e4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -600,6 +600,7 @@ pub async fn get_build_output( finalized_sources: None, finalized_cache_dependencies: None, finalized_cache_sources: None, + cached_prefix_info: None, system_tools: SystemTools::new(), build_summary: Arc::new(Mutex::new(BuildSummary::default())), extra_meta: Some( diff --git a/src/post_process/checks.rs b/src/post_process/checks.rs index 5431fffcf..465881060 100644 --- a/src/post_process/checks.rs +++ b/src/post_process/checks.rs @@ -6,7 +6,10 @@ use std::{ path::{Path, PathBuf}, }; -use crate::post_process::{package_nature::PackageNature, relink}; +use crate::post_process::{ + package_nature::{CaseInsensitivePathBuf, PackageNature}, + relink, +}; use crate::{ metadata::Output, post_process::{package_nature::PrefixInfo, relink::RelinkError}, @@ -528,7 +531,15 @@ pub fn perform_linking_checks( let dynamic_linking = &output.recipe.build().dynamic_linking; let system_libs = find_system_libs(output)?; - let prefix_info = PrefixInfo::from_prefix(output.prefix())?; + let mut prefix_info = PrefixInfo::from_prefix(output.prefix())?; + + // Merge cached prefix info from the staging cache (if any). + // The staging cache's host packages may not be physically installed in + // the prefix, but we need their path-to-package and nature mappings + // for linking checks to attribute shared libraries correctly. + if let Some(cached) = &output.cached_prefix_info { + prefix_info.merge_cached(cached); + } let host_dso_packages = host_run_export_dso_packages(output, &prefix_info.package_to_nature); tracing::trace!("Host run_export DSO packages: {host_dso_packages:#?}",); @@ -576,18 +587,54 @@ pub fn perform_linking_checks( continue; } - let lib = resolved.as_ref().unwrap_or(lib); - if let Ok(libpath) = lib.strip_prefix(host_prefix) - && let Some(package) = prefix_info - .path_to_package - .get(&libpath.to_path_buf().into()) - && let Some(nature) = prefix_info.package_to_nature.get(package) - { - // Accept any package that provides shared objects (DSO libraries, - // interpreters like python providing python3XX.dll, plugin libraries, etc.) - if nature.provides_shared_objects() { - file_dsos.push((libpath.to_path_buf(), package.clone())); + let effective_lib = resolved.as_ref().unwrap_or(lib); + + // Try to attribute the library to a host package. + // First attempt: resolved path stripped of host prefix. + let attributed = + if let Ok(libpath) = effective_lib.strip_prefix(host_prefix) { + let key: CaseInsensitivePathBuf = libpath.to_path_buf().into(); + if let Some(package) = prefix_info.path_to_package.get(&key) + && let Some(nature) = prefix_info.package_to_nature.get(package) + && nature.provides_shared_objects() + { + Some((libpath.to_path_buf(), package.clone())) + } else { + None + } + } else { + None + }; + + // Second attempt: for unresolved @rpath/@loader_path libraries + // (host dep files may not be on disk due to staging cache + // optimization), resolve virtually against path_to_package. + let attributed = attributed.or_else(|| { + if resolved.is_some() { + return None; + } + let rel = lib + .strip_prefix("@rpath") + .or_else(|_| lib.strip_prefix("@loader_path")) + .ok()?; + // Try common library directories + for dir in &["lib", "bin", "Library/bin"] { + let candidate: CaseInsensitivePathBuf = + Path::new(dir).join(rel).into(); + if let Some(package) = prefix_info.path_to_package.get(&candidate) + && let Some(nature) = prefix_info.package_to_nature.get(package) + && nature.provides_shared_objects() + { + // Use the unresolved path as key so it matches + // the entry in shared_libraries downstream + return Some((lib.to_path_buf(), package.clone())); + } } + None + }); + + if let Some((libpath, package)) = attributed { + file_dsos.push((libpath, package)); } } @@ -669,6 +716,23 @@ pub fn perform_linking_checks( continue; } + // Check if the library is a build artifact from the staging cache + // (i.e. it will be packaged by a sibling output). + if let Some(cached) = &output.cached_prefix_info { + let lib_str = lib.to_string_lossy(); + if cached + .staging_prefix_files + .iter() + .any(|f| f == lib_str.as_ref() || Path::new(f) == lib) + { + link_info.linked_packages.push(LinkedPackage { + name: lib.to_path_buf(), + link_origin: LinkOrigin::PackageItself, + }); + continue; + } + } + // Check if we allow overlinking. if dynamic_linking.missing_dso_allowlist.is_match(lib) { tracing::info!( @@ -712,6 +776,16 @@ pub fn perform_linking_checks( // If there are any host packages with DSOs that we didn't link against, // it is "overdepending". for host_package in host_dso_packages.iter() { + // Skip overdepending check for packages inherited from the staging cache. + // These are transitive dependencies needed by the staging cache's build + // artifacts (sibling outputs) and are expected to not be directly linked + // by this output's files. + if let Some(cached) = &output.cached_prefix_info { + if cached.package_to_nature.contains_key(host_package) { + continue; + } + } + if !package_files .iter() .map(|package| { diff --git a/src/post_process/package_nature.rs b/src/post_process/package_nature.rs index f363c7eee..04a343ac3 100644 --- a/src/post_process/package_nature.rs +++ b/src/post_process/package_nature.rs @@ -18,10 +18,11 @@ use std::{ hash::Hash, ops::Sub, path::{Path, PathBuf}, + str::FromStr, }; /// The nature of a package -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone, serde::Serialize, serde::Deserialize)] pub enum PackageNature { /// Libraries RunExportsLibrary, @@ -213,6 +214,67 @@ impl PrefixInfo { Ok(prefix_info) } + + /// Merge cached prefix info (from a staging cache) into this PrefixInfo. + /// Cached entries are only added if they don't already exist. + pub fn merge_cached(&mut self, cached: &CachedPrefixInfo) { + for (name_str, nature) in &cached.package_to_nature { + if let Ok(name) = PackageName::from_str(name_str) { + self.package_to_nature.entry(name).or_insert(nature.clone()); + } + } + for (path_str, name_str) in &cached.path_to_package { + if let Ok(name) = PackageName::from_str(name_str) { + let path_buf: CaseInsensitivePathBuf = PathBuf::from(path_str).into(); + self.path_to_package.entry(path_buf).or_insert(name); + } + } + } +} + +/// Serializable prefix info for storing in staging cache metadata. +/// Maps file paths to their owning packages and packages to their nature, +/// so that linking checks can attribute libraries without needing the +/// original conda-meta records installed. +#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] +pub struct CachedPrefixInfo { + /// Maps file paths (relative to prefix) to package names + pub path_to_package: HashMap, + /// Maps package names to their nature + pub package_to_nature: HashMap, + /// Files produced by the staging cache build script (relative to prefix). + /// These are build artifacts that will be split across sibling outputs. + /// Used by linking checks to recognize libraries from sibling outputs. + #[serde(default)] + pub staging_prefix_files: Vec, +} + +impl CachedPrefixInfo { + /// Build a CachedPrefixInfo from a PrefixInfo and the staging cache's + /// prefix files (build artifacts). + pub(crate) fn from_prefix_info(info: &PrefixInfo, staging_prefix_files: &[PathBuf]) -> Self { + Self { + path_to_package: info + .path_to_package + .iter() + .map(|(path, name)| { + ( + path.path.to_string_lossy().to_string(), + name.as_normalized().to_string(), + ) + }) + .collect(), + package_to_nature: info + .package_to_nature + .iter() + .map(|(name, nature)| (name.as_normalized().to_string(), nature.clone())) + .collect(), + staging_prefix_files: staging_prefix_files + .iter() + .map(|p| p.to_string_lossy().to_string()) + .collect(), + } + } } #[cfg(test)] diff --git a/src/render/resolved_dependencies.rs b/src/render/resolved_dependencies.rs index 24d60291f..c46851dcf 100644 --- a/src/render/resolved_dependencies.rs +++ b/src/render/resolved_dependencies.rs @@ -1103,6 +1103,11 @@ impl Output { /// Install the environments of the outputs. Assumes that the dependencies /// for the environment have already been resolved. + /// + /// When this output inherits from a staging cache, the cache's host + /// dependencies are merged into the host environment so that their + /// shared libraries and `conda-meta` records are present during + /// post-processing (linking checks, relinking, etc.). pub async fn install_environments( &self, tool_configuration: &Configuration, diff --git a/src/staging.rs b/src/staging.rs index c4a36408b..047175b82 100644 --- a/src/staging.rs +++ b/src/staging.rs @@ -23,6 +23,7 @@ use crate::{ env_vars, metadata::{Output, build_reindexed_channels}, packaging::Files, + post_process::package_nature::{CachedPrefixInfo, PrefixInfo}, render::resolved_dependencies::{ FinalizedDependencies, RunExportsDownload, install_environments, resolve_dependencies, }, @@ -72,6 +73,13 @@ pub struct StagingCacheMetadata { /// The variant configuration that was used pub variant: BTreeMap, + + /// Cached prefix info (path-to-package and package nature mappings) + /// from the host environment at staging cache build time. + /// This allows linking checks to attribute shared libraries to their + /// providing packages without needing the conda-meta records installed. + #[serde(default)] + pub cached_prefix_info: CachedPrefixInfo, } impl Output { @@ -142,7 +150,7 @@ impl Output { /// 2. If yes, restore the cached files to the prefix /// 3. If no, build the staging cache and save it /// - /// Returns the finalized dependencies and sources from the staging cache + /// Returns the finalized dependencies, sources, and cached prefix info pub async fn build_or_restore_staging_cache( &self, staging: &StagingCache, @@ -151,6 +159,7 @@ impl Output { ( FinalizedDependencies, Vec, + CachedPrefixInfo, ), miette::Error, > { @@ -217,6 +226,7 @@ impl Output { ( FinalizedDependencies, Vec, + CachedPrefixInfo, ), miette::Error, > { @@ -368,6 +378,12 @@ impl Output { .run() .into_diagnostic()?; + // Capture prefix info (path-to-package and package nature mappings) + // from the host environment while conda-meta records are still present. + // This data is needed by linking checks in inheriting outputs. + let prefix_info = PrefixInfo::from_prefix(self.prefix()).into_diagnostic()?; + let cached_prefix_info = CachedPrefixInfo::from_prefix_info(&prefix_info, &copied_files); + // Save metadata let metadata = StagingCacheMetadata { name: staging.name.clone(), @@ -377,6 +393,7 @@ impl Output { work_dir_files: copied_work_dir.copied_paths().to_vec(), prefix: self.prefix().to_path_buf(), variant: staging.used_variant.clone(), + cached_prefix_info, }; let metadata_json = serde_json::to_string_pretty(&metadata).into_diagnostic()?; @@ -388,7 +405,11 @@ impl Output { metadata.work_dir_files.len() ); - Ok((finalized_dependencies, finalized_sources)) + Ok(( + finalized_dependencies, + finalized_sources, + metadata.cached_prefix_info, + )) } /// Restore a staging cache from disk @@ -400,6 +421,7 @@ impl Output { ( FinalizedDependencies, Vec, + CachedPrefixInfo, ), miette::Error, > { @@ -439,7 +461,11 @@ impl Output { metadata.name ); - Ok((metadata.finalized_dependencies, metadata.finalized_sources)) + Ok(( + metadata.finalized_dependencies, + metadata.finalized_sources, + metadata.cached_prefix_info, + )) } /// Process all staging caches for this output @@ -454,6 +480,7 @@ impl Output { Option<( FinalizedDependencies, Vec, + CachedPrefixInfo, )>, miette::Error, > { @@ -470,7 +497,7 @@ impl Output { "Building or restoring staging cache: {}", staging_cache.name ); - let (_deps, _sources) = self + let (_deps, _sources, _prefix_info) = self .build_or_restore_staging_cache(staging_cache, tool_configuration) .await?; } @@ -491,11 +518,11 @@ impl Output { })?; // Get or build the cache - let (deps, sources) = self + let (deps, sources, cached_prefix_info) = self .build_or_restore_staging_cache(staging, tool_configuration) .await?; - Ok(Some((deps, sources))) + Ok(Some((deps, sources, cached_prefix_info))) } else { Ok(None) } diff --git a/src/types/build_output.rs b/src/types/build_output.rs index f00b4762c..d3e50154d 100644 --- a/src/types/build_output.rs +++ b/src/types/build_output.rs @@ -20,6 +20,7 @@ use std::{ use crate::{ console_utils::github_integration_enabled, + post_process::package_nature::CachedPrefixInfo, render::resolved_dependencies::FinalizedDependencies, system_tools::SystemTools, types::{BuildConfiguration, BuildSummary, PlatformWithVirtualPackages}, @@ -50,6 +51,13 @@ pub struct BuildOutput { #[serde(skip_serializing_if = "Option::is_none")] pub finalized_cache_sources: Option>, + /// Cached prefix info from the staging cache's host environment. + /// Used by linking checks to attribute shared libraries to packages + /// that were installed during the staging cache build but are not + /// physically present in the current host prefix. + #[serde(skip_serializing_if = "Option::is_none")] + pub cached_prefix_info: Option, + /// Summary of the build #[serde(skip)] pub build_summary: Arc>,