Adds RmetaLinkCache a per-link cache that uses path as the key of decoded lib.rmeta-link archive members, and routes add_archive read through it so each rlib link metadata is decoded at most once per link. This is a demand that originated from the discussion in #156735 and we split it out as its own PR. It gives that PR a decode once path tp read instead of reparsing each rlib per crate once native_lib_filenames moves to a link time read.

Author: mehdiakiki

compiler/rustc_codegen_ssa/src/back/archive.rs

@@ -311,7 +311,7 @@ fn find_binutils_dlltool(sess: &Session) -> OsString {
 }
 
 pub enum AddArchiveKind<'a> {
-    Rlib(/*skip*/ &'a dyn Fn(&str, ArchiveEntryKind) -> bool),
+    Rlib { skip: &'a dyn Fn(&str, ArchiveEntryKind) -> bool, cache: &'a rmeta_link::RmetaLinkCache },
     Other,
 }
 
@@ -482,7 +482,9 @@ impl<'a> ArchiveBuilder for ArArchiveBuilder<'a> {
         let archive = ArchiveFile::parse(&*archive_map)
             .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
         let metadata_link = match ar_kind {
-            AddArchiveKind::Rlib(..) => rmeta_link::read(&archive, &archive_map, &archive_path),
+            AddArchiveKind::Rlib { cache, .. } => cache.get_or_insert_with(&archive_path, || {
+                rmeta_link::read(&archive, &archive_map, &archive_path)
+            }),
             AddArchiveKind::Other => None,
         };
         let archive_index = self.src_archives.len();
@@ -513,7 +515,7 @@ impl<'a> ArchiveBuilder for ArArchiveBuilder<'a> {
                 ArchiveEntryKind::Other
             };
             let drop = match ar_kind {
-                AddArchiveKind::Rlib(skip) => skip(&file_name, kind),
+                AddArchiveKind::Rlib { skip, .. } => skip(&file_name, kind),
                 AddArchiveKind::Other => false,
             };
             if !drop {

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -86,6 +86,7 @@ pub fn link_binary(
     let _timer = sess.timer("link_binary");
     let output_metadata = sess.opts.output_types.contains_key(&OutputType::Metadata);
     let mut tempfiles_for_stdout_output: Vec<PathBuf> = Vec::new();
+    let rmeta_link_cache = rmeta_link::RmetaLinkCache::default();
     for &crate_type in &crate_info.crate_types {
         // Ignore executable crates if we have -Z no-codegen, as they will error.
         if (sess.opts.unstable_opts.no_codegen || !sess.opts.output_types.should_codegen())
@@ -139,6 +140,7 @@ pub fn link_binary(
                     link_staticlib(
                         sess,
                         archive_builder_builder,
+                        &rmeta_link_cache,
                         &compiled_modules,
                         &crate_info,
                         &metadata,
@@ -150,6 +152,7 @@ pub fn link_binary(
                     link_natively(
                         sess,
                         archive_builder_builder,
+                        &rmeta_link_cache,
                         crate_type,
                         &out_filename,
                         &compiled_modules,
@@ -502,6 +505,7 @@ fn link_rlib<'a>(
 fn link_staticlib(
     sess: &Session,
     archive_builder_builder: &dyn ArchiveBuilderBuilder,
+    rmeta_link_cache: &rmeta_link::RmetaLinkCache,
     compiled_modules: &CompiledModules,
     crate_info: &CrateInfo,
     metadata: &EncodedMetadata,
@@ -531,24 +535,27 @@ fn link_staticlib(
         let bundled_libs: FxIndexSet<_> = native_libs.filter_map(|lib| lib.filename).collect();
         ab.add_archive(
             path,
-            AddArchiveKind::Rlib(&|fname: &str, entry_kind| {
-                // Ignore metadata and rmeta-link files.
-                if fname == METADATA_FILENAME || fname == rmeta_link::FILENAME {
-                    return true;
-                }
+            AddArchiveKind::Rlib {
+                skip: &|fname: &str, entry_kind| {
+                    // Ignore metadata and rmeta-link files.
+                    if fname == METADATA_FILENAME || fname == rmeta_link::FILENAME {
+                        return true;
+                    }
 
-                // Don't include Rust objects if LTO is enabled.
-                if lto && entry_kind == ArchiveEntryKind::RustObj {
-                    return true;
-                }
+                    // Don't include Rust objects if LTO is enabled.
+                    if lto && entry_kind == ArchiveEntryKind::RustObj {
+                        return true;
+                    }
 
-                // Skip objects for bundled libs.
-                if bundled_libs.contains(&Symbol::intern(fname)) {
-                    return true;
-                }
+                    // Skip objects for bundled libs.
+                    if bundled_libs.contains(&Symbol::intern(fname)) {
+                        return true;
+                    }
 
-                false
-            }),
+                    false
+                },
+                cache: rmeta_link_cache,
+            },
         )
         .unwrap();
 
@@ -939,6 +946,7 @@ fn report_linker_output(
 fn link_natively(
     sess: &Session,
     archive_builder_builder: &dyn ArchiveBuilderBuilder,
+    rmeta_link_cache: &rmeta_link::RmetaLinkCache,
     crate_type: CrateType,
     out_filename: &Path,
     compiled_modules: &CompiledModules,
@@ -965,6 +973,7 @@ fn link_natively(
         flavor,
         sess,
         archive_builder_builder,
+        rmeta_link_cache,
         crate_type,
         tmpdir,
         temp_filename,
@@ -2491,6 +2500,7 @@ fn linker_with_args(
     flavor: LinkerFlavor,
     sess: &Session,
     archive_builder_builder: &dyn ArchiveBuilderBuilder,
+    rmeta_link_cache: &rmeta_link::RmetaLinkCache,
     crate_type: CrateType,
     tmpdir: &Path,
     out_filename: &Path,
@@ -2619,6 +2629,7 @@ fn linker_with_args(
         cmd,
         sess,
         archive_builder_builder,
+        rmeta_link_cache,
         crate_info,
         crate_type,
         tmpdir,
@@ -3055,6 +3066,7 @@ fn add_upstream_rust_crates(
     cmd: &mut dyn Linker,
     sess: &Session,
     archive_builder_builder: &dyn ArchiveBuilderBuilder,
+    rmeta_link_cache: &rmeta_link::RmetaLinkCache,
     crate_info: &CrateInfo,
     crate_type: CrateType,
     tmpdir: &Path,
@@ -3107,6 +3119,7 @@ fn add_upstream_rust_crates(
                         cmd,
                         sess,
                         archive_builder_builder,
+                        rmeta_link_cache,
                         crate_info,
                         tmpdir,
                         cnum,
@@ -3238,6 +3251,7 @@ fn add_static_crate(
     cmd: &mut dyn Linker,
     sess: &Session,
     archive_builder_builder: &dyn ArchiveBuilderBuilder,
+    rmeta_link_cache: &rmeta_link::RmetaLinkCache,
     crate_info: &CrateInfo,
     tmpdir: &Path,
     cnum: CrateNum,
@@ -3268,33 +3282,36 @@ fn add_static_crate(
         let mut archive = archive_builder_builder.new_archive_builder(sess);
         if let Err(error) = archive.add_archive(
             cratepath,
-            AddArchiveKind::Rlib(&|f, entry_kind| {
-                if f == METADATA_FILENAME || f == rmeta_link::FILENAME {
-                    return true;
-                }
+            AddArchiveKind::Rlib {
+                skip: &|f, entry_kind| {
+                    if f == METADATA_FILENAME || f == rmeta_link::FILENAME {
+                        return true;
+                    }
 
-                // If we're performing LTO and this is a rust-generated object
-                // file, then we don't need the object file as it's part of the
-                // LTO module. Note that `#![no_builtins]` is excluded from LTO,
-                // though, so we let that object file slide.
-                if upstream_rust_objects_already_included
-                    && entry_kind == ArchiveEntryKind::RustObj
-                    && is_builtins
-                {
-                    return true;
-                }
+                    // If we're performing LTO and this is a rust-generated object
+                    // file, then we don't need the object file as it's part of the
+                    // LTO module. Note that `#![no_builtins]` is excluded from LTO,
+                    // though, so we let that object file slide.
+                    if upstream_rust_objects_already_included
+                        && entry_kind == ArchiveEntryKind::RustObj
+                        && is_builtins
+                    {
+                        return true;
+                    }
 
-                // We skip native libraries because:
-                // 1. This native libraries won't be used from the generated rlib,
-                //    so we can throw them away to avoid the copying work.
-                // 2. We can't allow it to be a single remaining entry in archive
-                //    as some linkers may complain on that.
-                if bundled_lib_file_names.contains(&Symbol::intern(f)) {
-                    return true;
-                }
+                    // We skip native libraries because:
+                    // 1. This native libraries won't be used from the generated rlib,
+                    //    so we can throw them away to avoid the copying work.
+                    // 2. We can't allow it to be a single remaining entry in archive
+                    //    as some linkers may complain on that.
+                    if bundled_lib_file_names.contains(&Symbol::intern(f)) {
+                        return true;
+                    }
 
-                false
-            }),
+                    false
+                },
+                cache: rmeta_link_cache,
+            },
         ) {
             sess.dcx()
                 .emit_fatal(errors::RlibArchiveBuildFailure { path: cratepath.clone(), error });

compiler/rustc_codegen_ssa/src/back/rmeta_link.rs

@@ -2,9 +2,12 @@
 //! and potentially other data collected and used when building or linking a rlib.
 //! See <https://github.com/rust-lang/rust/issues/138243>.
 
-use std::path::Path;
+use std::cell::RefCell;
+use std::path::{Path, PathBuf};
+use std::rc::Rc;
 
 use object::read::archive::ArchiveFile;
+use rustc_data_structures::fx::FxHashMap;
 use rustc_serialize::opaque::mem_encoder::MemEncoder;
 use rustc_serialize::opaque::{MAGIC_END_BYTES, MemDecoder};
 use rustc_serialize::{Decodable, Encodable};
@@ -54,3 +57,23 @@ pub fn read_from_data(archive_data: &[u8], rlib_path: &Path) -> Option<RmetaLink
     let archive = ArchiveFile::parse(archive_data).ok()?;
     read(&archive, archive_data, rlib_path)
 }
+
+#[derive(Default)]
+pub struct RmetaLinkCache {
+    cache: RefCell<FxHashMap<PathBuf, Option<Rc<RmetaLink>>>>,
+}
+
+impl RmetaLinkCache {
+    pub fn get_or_insert_with(
+        &self,
+        rlib_path: &Path,
+        load: impl FnOnce() -> Option<RmetaLink>,
+    ) -> Option<Rc<RmetaLink>> {
+        if let Some(cached) = self.cache.borrow().get(rlib_path) {
+            return cached.clone();
+        }
+        let loaded = load().map(Rc::new);
+        self.cache.borrow_mut().insert(rlib_path.to_path_buf(), loaded.clone());
+        loaded
+    }
+}