Adds RmetaLinkCache a per-link cache that uses path as the key of decoded lib.rmeta-link archive members, and routes add_archive read through it so each rlib link metadata is decoded at most once per link. This is a demand that originated from the discussion in #156735 and we split it out as its own PR. It gives that PR a decode once path tp read instead of reparsing each rlib per crate once native_lib_filenames moves to a link time read.

Author: mehdiakiki

compiler/rustc_codegen_ssa/src/back/archive.rs

@@ -22,7 +22,7 @@ use rustc_target::spec::Arch;
 use tracing::trace;
 
 use super::metadata::{create_compressed_metadata_file, search_for_section};
-use super::rmeta_link;
+use super::rmeta_link::{self, RmetaLinkCache};
 use super::symbol_edit::{apply_edits, collect_internal_names};
 use crate::common;
 // Public for ArchiveBuilderBuilder::extract_bundled_libs
@@ -311,7 +311,7 @@ fn find_binutils_dlltool(sess: &Session) -> OsString {
 }
 
 pub enum AddArchiveKind<'a> {
-    Rlib(/*skip*/ &'a dyn Fn(&str, ArchiveEntryKind) -> bool),
+    Rlib(&'a mut RmetaLinkCache, /*skip*/ &'a dyn Fn(&str, ArchiveEntryKind) -> bool),
     Other,
 }
 
@@ -466,7 +466,11 @@ pub fn try_extract_macho_fat_archive(
 }
 
 impl<'a> ArchiveBuilder for ArArchiveBuilder<'a> {
-    fn add_archive(&mut self, archive_path: &Path, ar_kind: AddArchiveKind<'_>) -> io::Result<()> {
+    fn add_archive(
+        &mut self,
+        archive_path: &Path,
+        mut ar_kind: AddArchiveKind<'_>,
+    ) -> io::Result<()> {
         let mut archive_path = archive_path.to_path_buf();
         if self.sess.target.llvm_target.contains("-apple-macosx")
             && let Some(new_archive_path) = try_extract_macho_fat_archive(self.sess, &archive_path)?
@@ -481,8 +485,14 @@ impl<'a> ArchiveBuilder for ArArchiveBuilder<'a> {
         let archive_map = unsafe { Mmap::map(File::open(&archive_path)?)? };
         let archive = ArchiveFile::parse(&*archive_map)
             .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?;
-        let metadata_link = match ar_kind {
-            AddArchiveKind::Rlib(..) => rmeta_link::read(&archive, &archive_map, &archive_path),
+        let skip = match &ar_kind {
+            AddArchiveKind::Rlib(_, skip) => Some(*skip),
+            AddArchiveKind::Other => None,
+        };
+        let metadata_link = match &mut ar_kind {
+            AddArchiveKind::Rlib(cache, _) => cache.get_or_insert_with(&archive_path, || {
+                rmeta_link::read(&archive, &archive_map, &archive_path)
+            }),
             AddArchiveKind::Other => None,
         };
         let archive_index = self.src_archives.len();
@@ -512,9 +522,9 @@ impl<'a> ArchiveBuilder for ArArchiveBuilder<'a> {
             } else {
                 ArchiveEntryKind::Other
             };
-            let drop = match ar_kind {
-                AddArchiveKind::Rlib(skip) => skip(&file_name, kind),
-                AddArchiveKind::Other => false,
+            let drop = match skip {
+                Some(skip) => skip(&file_name, kind),
+                None => false,
             };
             if !drop {
                 let source = if entry.is_thin() {

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -59,6 +59,7 @@ use super::archive::{
 use super::command::Command;
 use super::linker::{self, Linker};
 use super::metadata::{MetadataPosition, create_wrapper_file};
+use super::rmeta_link::RmetaLinkCache;
 use super::rpath::{self, RPathConfig};
 use super::{apple, rmeta_link, versioned_llvm_target};
 use crate::base::needs_allocator_shim_for_linking;
@@ -86,6 +87,7 @@ pub fn link_binary(
     let _timer = sess.timer("link_binary");
     let output_metadata = sess.opts.output_types.contains_key(&OutputType::Metadata);
     let mut tempfiles_for_stdout_output: Vec<PathBuf> = Vec::new();
+    let mut rmeta_link_cache = RmetaLinkCache::default();
     for &crate_type in &crate_info.crate_types {
         // Ignore executable crates if we have -Z no-codegen, as they will error.
         if (sess.opts.unstable_opts.no_codegen || !sess.opts.output_types.should_codegen())
@@ -139,6 +141,7 @@ pub fn link_binary(
                     link_staticlib(
                         sess,
                         archive_builder_builder,
+                        &mut rmeta_link_cache,
                         &compiled_modules,
                         &crate_info,
                         &metadata,
@@ -150,6 +153,7 @@ pub fn link_binary(
                     link_natively(
                         sess,
                         archive_builder_builder,
+                        &mut rmeta_link_cache,
                         crate_type,
                         &out_filename,
                         &compiled_modules,
@@ -502,6 +506,7 @@ fn link_rlib<'a>(
 fn link_staticlib(
     sess: &Session,
     archive_builder_builder: &dyn ArchiveBuilderBuilder,
+    rmeta_link_cache: &mut RmetaLinkCache,
     compiled_modules: &CompiledModules,
     crate_info: &CrateInfo,
     metadata: &EncodedMetadata,
@@ -531,7 +536,7 @@ fn link_staticlib(
         let bundled_libs: FxIndexSet<_> = native_libs.filter_map(|lib| lib.filename).collect();
         ab.add_archive(
             path,
-            AddArchiveKind::Rlib(&|fname: &str, entry_kind| {
+            AddArchiveKind::Rlib(rmeta_link_cache, &|fname: &str, entry_kind| {
                 // Ignore metadata and rmeta-link files.
                 if fname == METADATA_FILENAME || fname == rmeta_link::FILENAME {
                     return true;
@@ -939,6 +944,7 @@ fn report_linker_output(
 fn link_natively(
     sess: &Session,
     archive_builder_builder: &dyn ArchiveBuilderBuilder,
+    rmeta_link_cache: &mut RmetaLinkCache,
     crate_type: CrateType,
     out_filename: &Path,
     compiled_modules: &CompiledModules,
@@ -965,6 +971,7 @@ fn link_natively(
         flavor,
         sess,
         archive_builder_builder,
+        rmeta_link_cache,
         crate_type,
         tmpdir,
         temp_filename,
@@ -2491,6 +2498,7 @@ fn linker_with_args(
     flavor: LinkerFlavor,
     sess: &Session,
     archive_builder_builder: &dyn ArchiveBuilderBuilder,
+    rmeta_link_cache: &mut RmetaLinkCache,
     crate_type: CrateType,
     tmpdir: &Path,
     out_filename: &Path,
@@ -2619,6 +2627,7 @@ fn linker_with_args(
         cmd,
         sess,
         archive_builder_builder,
+        rmeta_link_cache,
         crate_info,
         crate_type,
         tmpdir,
@@ -3055,6 +3064,7 @@ fn add_upstream_rust_crates(
     cmd: &mut dyn Linker,
     sess: &Session,
     archive_builder_builder: &dyn ArchiveBuilderBuilder,
+    rmeta_link_cache: &mut RmetaLinkCache,
     crate_info: &CrateInfo,
     crate_type: CrateType,
     tmpdir: &Path,
@@ -3107,6 +3117,7 @@ fn add_upstream_rust_crates(
                         cmd,
                         sess,
                         archive_builder_builder,
+                        rmeta_link_cache,
                         crate_info,
                         tmpdir,
                         cnum,
@@ -3238,6 +3249,7 @@ fn add_static_crate(
     cmd: &mut dyn Linker,
     sess: &Session,
     archive_builder_builder: &dyn ArchiveBuilderBuilder,
+    rmeta_link_cache: &mut RmetaLinkCache,
     crate_info: &CrateInfo,
     tmpdir: &Path,
     cnum: CrateNum,
@@ -3268,7 +3280,7 @@ fn add_static_crate(
         let mut archive = archive_builder_builder.new_archive_builder(sess);
         if let Err(error) = archive.add_archive(
             cratepath,
-            AddArchiveKind::Rlib(&|f, entry_kind| {
+            AddArchiveKind::Rlib(rmeta_link_cache, &|f, entry_kind| {
                 if f == METADATA_FILENAME || f == rmeta_link::FILENAME {
                     return true;
                 }

compiler/rustc_codegen_ssa/src/back/rmeta_link.rs

@@ -2,9 +2,10 @@
 //! and potentially other data collected and used when building or linking a rlib.
 //! See <https://github.com/rust-lang/rust/issues/138243>.
 
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
 use object::read::archive::ArchiveFile;
+use rustc_data_structures::fx::FxHashMap;
 use rustc_serialize::opaque::mem_encoder::MemEncoder;
 use rustc_serialize::opaque::{MAGIC_END_BYTES, MemDecoder};
 use rustc_serialize::{Decodable, Encodable};
@@ -54,3 +55,18 @@ pub fn read_from_data(archive_data: &[u8], rlib_path: &Path) -> Option<RmetaLink
     let archive = ArchiveFile::parse(archive_data).ok()?;
     read(&archive, archive_data, rlib_path)
 }
+
+#[derive(Default)]
+pub struct RmetaLinkCache {
+    cache: FxHashMap<PathBuf, Option<RmetaLink>>,
+}
+
+impl RmetaLinkCache {
+    pub fn get_or_insert_with(
+        &mut self,
+        rlib_path: &Path,
+        load: impl FnOnce() -> Option<RmetaLink>,
+    ) -> Option<&RmetaLink> {
+        self.cache.entry(rlib_path.to_path_buf()).or_insert_with(load).as_ref()
+    }
+}