Rollup merge of rust-lang#156950 - cezarbbb:staticlib-rename-internal-symbols, r=petrochenkov

Staticlib rename internal symbols

Follow-up to rust-lang#155338.

`-Zstaticlib-rename-internal-symbols` appends a crate-specific suffix (`_rs{StableCrateId}`) to non-exported symbols, resolving duplicate symbol conflicts when linking multiple Rust staticlibs into the same binary.

The implementation collects all defined `GLOBAL/WEAK` symbol names not in the exported set across all .o files, then renames them by extending the strtab and patching symbol name offsets. When combined with `-Zstaticlib-hide-internal-symbols`, the renamed symbols also receive `STV_HIDDEN` visibility.

Supported on ELF targets (Linux, BSD, etc.) and Apple targets (macOS, iOS, etc.). On unsupported targets (Windows), a warning is emitted and the flag has no effect.

r?@bjorn3 @petrochenkov

Author: JonathanBrouwer

compiler/rustc_codegen_ssa/src/back/archive.rs

@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::env;
 use std::error::Error;
 use std::ffi::OsString;
@@ -22,7 +23,7 @@ use tracing::trace;
 
 use super::metadata::{create_compressed_metadata_file, search_for_section};
 use super::rmeta_link;
-use super::symbol_edit::apply_hide;
+use super::symbol_edit::{apply_edits, collect_internal_names};
 use crate::common;
 // Public for ArchiveBuilderBuilder::extract_bundled_libs
 pub use crate::errors::ExtractBundledLibsError;
@@ -314,12 +315,18 @@ pub enum AddArchiveKind<'a> {
     Other,
 }
 
+pub struct ArchiveSymbols {
+    pub exported: FxHashSet<String>,
+    pub rename_suffix: Option<String>,
+    pub hide: bool,
+}
+
 pub trait ArchiveBuilder {
     fn add_file(&mut self, path: &Path, kind: ArchiveEntryKind);
 
     fn add_archive(&mut self, archive: &Path, kind: AddArchiveKind<'_>) -> io::Result<()>;
 
-    fn build(self: Box<Self>, output: &Path, exported_symbols: Option<FxHashSet<String>>) -> bool;
+    fn build(self: Box<Self>, output: &Path, symbols: Option<ArchiveSymbols>) -> bool;
 }
 
 fn target_archive_format_to_object_kind(format: &str) -> Option<ObjectArchiveKind> {
@@ -534,9 +541,9 @@ impl<'a> ArchiveBuilder for ArArchiveBuilder<'a> {
 
     /// Combine the provided files, rlibs, and native libraries into a single
     /// `Archive`.
-    fn build(self: Box<Self>, output: &Path, exported_symbols: Option<FxHashSet<String>>) -> bool {
+    fn build(self: Box<Self>, output: &Path, symbols: Option<ArchiveSymbols>) -> bool {
         let sess = self.sess;
-        match self.build_inner(output, exported_symbols) {
+        match self.build_inner(output, symbols) {
             Ok(any_members) => any_members,
             Err(error) => {
                 sess.dcx().emit_fatal(ArchiveBuildFailure { path: output.to_owned(), error })
@@ -546,11 +553,7 @@ impl<'a> ArchiveBuilder for ArArchiveBuilder<'a> {
 }
 
 impl<'a> ArArchiveBuilder<'a> {
-    fn build_inner(
-        self,
-        output: &Path,
-        exported_symbols: Option<FxHashSet<String>>,
-    ) -> io::Result<bool> {
+    fn build_inner(self, output: &Path, symbols: Option<ArchiveSymbols>) -> io::Result<bool> {
         let archive_kind = match &*self.sess.target.archive_format {
             "gnu" => ArchiveKind::Gnu,
             "bsd" => ArchiveKind::Bsd,
@@ -562,6 +565,39 @@ impl<'a> ArArchiveBuilder<'a> {
             }
         };
 
+        // Collect all internally-defined symbol names across every Rust object file.
+        // This set is needed because rename must also apply to *undefined* references
+        // (cross-object calls within the staticlib), but we cannot use `!exported.contains(name)`
+        // alone — that would also match external C symbols like `malloc` which must not be renamed.
+        let rename = if let Some(sym) = &symbols
+            && let Some(rename_suffix) = sym.rename_suffix.as_deref()
+        {
+            let mut names = FxHashSet::default();
+            for (_, entry) in &self.entries {
+                if entry.kind != ArchiveEntryKind::RustObj {
+                    continue;
+                }
+                match &entry.source {
+                    ArchiveEntrySource::Archive { archive_index, file_range } => {
+                        let src_archive = &self.src_archives[*archive_index];
+                        let start = file_range.0 as usize;
+                        let end = start + file_range.1 as usize;
+                        if let Some(data) = src_archive.1.get(start..end) {
+                            collect_internal_names(data, &sym.exported, &mut names);
+                        }
+                    }
+                    ArchiveEntrySource::File(file) => {
+                        if let Ok(data) = fs::read(file) {
+                            collect_internal_names(&data, &sym.exported, &mut names);
+                        }
+                    }
+                }
+            }
+            Some((names, rename_suffix))
+        } else {
+            None
+        };
+
         let mut entries = Vec::new();
 
         for (entry_name, entry) in self.entries {
@@ -588,9 +624,9 @@ impl<'a> ArArchiveBuilder<'a> {
                     };
 
                     if entry.kind == ArchiveEntryKind::RustObj
-                        && let Some(exported) = &exported_symbols
+                        && let Some(sym) = &symbols
                     {
-                        Box::new(apply_hide(data, exported))
+                        Box::new(apply_edits(data, &sym.exported, sym.hide, rename.as_ref()))
                     } else {
                         Box::new(data)
                     }
@@ -602,9 +638,13 @@ impl<'a> ArArchiveBuilder<'a> {
                     )
                     .map_err(|err| io_error_context("failed to map object file", err))?;
                     if entry.kind == ArchiveEntryKind::RustObj
-                        && let Some(exported) = &exported_symbols
+                        && let Some(sym) = &symbols
                     {
-                        Box::new(apply_hide(&mmap, exported))
+                        let edited = apply_edits(&mmap, &sym.exported, sym.hide, rename.as_ref());
+                        match edited {
+                            Cow::Borrowed(_) => Box::new(mmap) as Box<dyn AsRef<[u8]>>,
+                            Cow::Owned(v) => Box::new(v),
+                        }
                     } else {
                         Box::new(mmap) as Box<dyn AsRef<[u8]>>
                     }

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -53,7 +53,9 @@ use rustc_target::spec::{
 };
 use tracing::{debug, info, warn};
 
-use super::archive::{AddArchiveKind, ArchiveBuilder, ArchiveBuilderBuilder, ArchiveEntryKind};
+use super::archive::{
+    AddArchiveKind, ArchiveBuilder, ArchiveBuilderBuilder, ArchiveEntryKind, ArchiveSymbols,
+};
 use super::command::Command;
 use super::linker::{self, Linker};
 use super::metadata::{MetadataPosition, create_wrapper_file};
@@ -566,11 +568,21 @@ fn link_staticlib(
         sess.dcx().emit_fatal(e);
     }
 
-    let exported_symbols = if sess.opts.unstable_opts.staticlib_hide_internal_symbols {
+    let hide = sess.opts.unstable_opts.staticlib_hide_internal_symbols;
+    let rename = sess.opts.unstable_opts.staticlib_rename_internal_symbols;
+
+    let exported_symbols = if hide || rename {
         if !matches!(sess.target.binary_format, BinaryFormat::Elf | BinaryFormat::MachO) {
-            sess.dcx().emit_warn(errors::StaticlibHideInternalSymbolsUnsupported {
-                binary_format: sess.target.archive_format.to_string(),
-            });
+            if hide {
+                sess.dcx().emit_warn(errors::StaticlibHideInternalSymbolsUnsupported {
+                    binary_format: sess.target.archive_format.to_string(),
+                });
+            }
+            if rename {
+                sess.dcx().emit_warn(errors::StaticlibRenameInternalSymbolsUnsupported {
+                    binary_format: sess.target.archive_format.to_string(),
+                });
+            }
             None
         } else {
             crate_info
@@ -581,7 +593,14 @@ fn link_staticlib(
     } else {
         None
     };
-    ab.build(out_filename, exported_symbols);
+
+    let symbols = exported_symbols.map(|exported| ArchiveSymbols {
+        exported,
+        rename_suffix: rename.then(|| crate_info.symbol_rename_suffix.clone()),
+        hide,
+    });
+
+    ab.build(out_filename, symbols);
 
     let crates = crate_info.used_crates.iter();