Auto merge of #155338 - cezarbbb:staticlib-symbol-hygiene, r=<try>

Staticlib hide internal symbols


try-job: i686-*

Author: bors

compiler/rustc_codegen_ssa/src/back/archive.rs

@@ -1,17 +1,19 @@
-use std::env;
 use std::error::Error;
 use std::ffi::OsString;
 use std::fs::{self, File};
 use std::io::{self, BufWriter, Write};
 use std::path::{Path, PathBuf};
+use std::{env, mem};
 
 use ar_archive_writer::{
     ArchiveKind, COFFShortExport, MachineTypes, NewArchiveMember, write_archive_to_stream,
 };
 pub use ar_archive_writer::{DEFAULT_OBJECT_READER, ObjectReader};
 use object::read::archive::{ArchiveFile, ArchiveKind as ObjectArchiveKind};
-use object::read::macho::FatArch;
-use rustc_data_structures::fx::FxIndexSet;
+use object::read::elf::Sym as _;
+use object::read::macho::{FatArch, Nlist};
+use object::{Endianness, elf, macho};
+use rustc_data_structures::fx::{FxHashSet, FxIndexSet};
 use rustc_data_structures::memmap::Mmap;
 use rustc_fs_util::TempDirBuilder;
 use rustc_metadata::EncodedMetadata;
@@ -318,7 +320,7 @@ pub trait ArchiveBuilder {
 
     fn add_archive(&mut self, archive: &Path, kind: AddArchiveKind<'_>) -> io::Result<()>;
 
-    fn build(self: Box<Self>, output: &Path) -> bool;
+    fn build(self: Box<Self>, output: &Path, exported_symbols: Option<FxHashSet<String>>) -> bool;
 }
 
 fn target_archive_format_to_object_kind(format: &str) -> Option<ObjectArchiveKind> {
@@ -401,7 +403,6 @@ enum ArchiveEntrySource {
 #[derive(Debug)]
 struct ArchiveEntry {
     source: ArchiveEntrySource,
-    #[expect(dead_code)] // used in #155338
     kind: ArchiveEntryKind,
 }
 
@@ -534,9 +535,9 @@ impl<'a> ArchiveBuilder for ArArchiveBuilder<'a> {
 
     /// Combine the provided files, rlibs, and native libraries into a single
     /// `Archive`.
-    fn build(self: Box<Self>, output: &Path) -> bool {
+    fn build(self: Box<Self>, output: &Path, exported_symbols: Option<FxHashSet<String>>) -> bool {
         let sess = self.sess;
-        match self.build_inner(output) {
+        match self.build_inner(output, exported_symbols) {
             Ok(any_members) => any_members,
             Err(error) => {
                 sess.dcx().emit_fatal(ArchiveBuildFailure { path: output.to_owned(), error })
@@ -546,7 +547,11 @@ impl<'a> ArchiveBuilder for ArArchiveBuilder<'a> {
 }
 
 impl<'a> ArArchiveBuilder<'a> {
-    fn build_inner(self, output: &Path) -> io::Result<bool> {
+    fn build_inner(
+        self,
+        output: &Path,
+        exported_symbols: Option<FxHashSet<String>>,
+    ) -> io::Result<bool> {
         let archive_kind = match &*self.sess.target.archive_format {
             "gnu" => ArchiveKind::Gnu,
             "bsd" => ArchiveKind::Bsd,
@@ -561,40 +566,51 @@ impl<'a> ArArchiveBuilder<'a> {
         let mut entries = Vec::new();
 
         for (entry_name, entry) in self.entries {
-            let data =
-                match entry.source {
-                    ArchiveEntrySource::Archive { archive_index, file_range } => {
-                        let src_archive = &self.src_archives[archive_index];
-                        let archive_data = &src_archive.1;
-                        let start = file_range.0 as usize;
-                        let end = start + file_range.1 as usize;
-                        let Some(data) = archive_data.get(start..end) else {
-                            return Err(io_error_context(
-                                "invalid archive member",
-                                io::Error::new(
-                                    io::ErrorKind::InvalidData,
-                                    format!(
-                                        "archive member at offset {start} with size {} \
+            let data: Box<dyn AsRef<[u8]>> = match entry.source {
+                ArchiveEntrySource::Archive { archive_index, file_range } => {
+                    let src_archive = &self.src_archives[archive_index];
+                    let archive_data = &src_archive.1;
+                    let start = file_range.0 as usize;
+                    let end = start + file_range.1 as usize;
+                    let Some(data) = archive_data.get(start..end) else {
+                        return Err(io_error_context(
+                            "invalid archive member",
+                            io::Error::new(
+                                io::ErrorKind::InvalidData,
+                                format!(
+                                    "archive member at offset {start} with size {} \
                                          exceeds archive size {} in `{}`",
-                                        file_range.1,
-                                        archive_data.len(),
-                                        src_archive.0.display(),
-                                    ),
+                                    file_range.1,
+                                    archive_data.len(),
+                                    src_archive.0.display(),
                                 ),
-                            ));
-                        };
-
-                        Box::new(data) as Box<dyn AsRef<[u8]>>
+                            ),
+                        ));
+                    };
+
+                    if entry.kind == ArchiveEntryKind::RustObj
+                        && let Some(exported) = &exported_symbols
+                    {
+                        Box::new(apply_hide(data, exported))
+                    } else {
+                        Box::new(data)
                     }
-                    ArchiveEntrySource::File(file) => unsafe {
-                        Box::new(
-                            Mmap::map(File::open(file).map_err(|err| {
-                                io_error_context("failed to open object file", err)
-                            })?)
-                            .map_err(|err| io_error_context("failed to map object file", err))?,
-                        ) as Box<dyn AsRef<[u8]>>
-                    },
-                };
+                }
+                ArchiveEntrySource::File(file) => unsafe {
+                    let mmap = Mmap::map(
+                        File::open(file)
+                            .map_err(|err| io_error_context("failed to open object file", err))?,
+                    )
+                    .map_err(|err| io_error_context("failed to map object file", err))?;
+                    if entry.kind == ArchiveEntryKind::RustObj
+                        && let Some(exported) = &exported_symbols
+                    {
+                        Box::new(apply_hide(&mmap, exported))
+                    } else {
+                        Box::new(mmap) as Box<dyn AsRef<[u8]>>
+                    }
+                },
+            };
 
             entries.push(NewArchiveMember {
                 buf: data,
@@ -655,3 +671,152 @@ impl<'a> ArArchiveBuilder<'a> {
 fn io_error_context(context: &str, err: io::Error) -> io::Error {
     io::Error::new(io::ErrorKind::Other, format!("{context}: {err}"))
 }
+
+// We use the `object` crate for the read-only pass over ELF/Mach-O object files
+// because its `Sym`/`Nlist` traits provide clean access to symbol properties without
+// manual byte parsing. However, `object` does not expose mutable views into the data,
+// so we cannot use it to modify symbol fields in place. Instead, the read-only pass
+// collects byte-level patches (offset + new value), and the write pass
+// (`apply_patches`) applies them to a copy of the byte buffer without any ELF/Mach-O
+// parsing — similar to how linker relocations work.
+
+/// A byte-level patch collected in the read-only pass and applied in the write pass.
+struct Patch {
+    offset: usize,
+    value: u8,
+}
+
+/// Apply a list of byte patches to `data`, returning the (possibly modified) bytes.
+fn apply_patches(data: &[u8], patches: &[Patch]) -> Vec<u8> {
+    let mut buf = data.to_vec();
+    for p in patches {
+        buf[p.offset] = p.value;
+    }
+    buf
+}
+
+// ---------------------------------------------------------------------------
+// ELF hide – read-only pass uses `object` crate, write pass uses `Patch` list
+// ---------------------------------------------------------------------------
+
+fn elf_hide_patches_impl<'data, Elf: object::read::elf::FileHeader<Endian = Endianness>>(
+    data: &'data [u8],
+    st_other_offset: usize,
+    exported: &FxHashSet<String>,
+) -> Option<Vec<Patch>>
+where
+    u64: From<Elf::Word>,
+{
+    let header = Elf::parse(data).ok()?;
+    let endian = header.endian().ok()?;
+    let sections = header.sections(endian, data).ok()?;
+    let symtab = sections.symbols(endian, data, elf::SHT_SYMTAB).ok()?;
+
+    let data_ptr = data.as_ptr() as usize;
+    let strings = symtab.strings();
+    let mut patches = Vec::new();
+
+    for sym in symtab.iter() {
+        let binding = sym.st_bind();
+        if binding != elf::STB_GLOBAL && binding != elf::STB_WEAK {
+            continue;
+        }
+        if sym.is_undefined(endian) {
+            continue;
+        }
+        let Ok(name_bytes) = sym.name(endian, strings) else { continue };
+        let Ok(name) = str::from_utf8(name_bytes) else { continue };
+        if !exported.contains(name) {
+            let sym_addr = sym as *const Elf::Sym as usize;
+            let offset = sym_addr - data_ptr + st_other_offset;
+            let new_vis = (sym.st_other() & !0x03) | elf::STV_HIDDEN;
+            patches.push(Patch { offset, value: new_vis });
+        }
+    }
+
+    Some(patches)
+}
+
+// ---------------------------------------------------------------------------
+// Mach-O hide – same architecture: read-only pass via `object`, write via patches
+// ---------------------------------------------------------------------------
+
+fn macho_hide_patches_impl<'data, Mach: object::read::macho::MachHeader<Endian = Endianness>>(
+    data: &'data [u8],
+    n_type_offset: usize,
+    exported: &FxHashSet<String>,
+) -> Option<Vec<Patch>> {
+    let header = Mach::parse(data, 0).ok()?;
+    let endian = header.endian().ok()?;
+    let mut commands = header.load_commands(endian, data, 0).ok()?;
+
+    let symtab_cmd = loop {
+        let cmd = commands.next().ok()??;
+        if let Some(st) = cmd.symtab().ok().flatten() {
+            break st;
+        }
+    };
+    let symtab: object::read::macho::SymbolTable<'_, Mach, &_> =
+        symtab_cmd.symbols(endian, data).ok()?;
+
+    let data_ptr = data.as_ptr() as usize;
+    let strings = symtab.strings();
+    let mut patches = Vec::new();
+
+    for nlist in symtab.iter() {
+        if nlist.is_stab() {
+            continue;
+        }
+        if nlist.is_undefined() {
+            continue;
+        }
+        if nlist.n_type() & macho::N_EXT == 0 {
+            continue;
+        }
+        let Ok(name_bytes) = nlist.name(endian, strings) else { continue };
+        let Ok(name) = str::from_utf8(name_bytes) else { continue };
+        if !exported.contains(name) {
+            let nlist_addr = nlist as *const Mach::Nlist as usize;
+            let offset = nlist_addr - data_ptr + n_type_offset;
+            patches.push(Patch { offset, value: nlist.n_type() | macho::N_PEXT });
+        }
+    }
+
+    Some(patches)
+}
+
+// ---------------------------------------------------------------------------
+// Unified dispatch: top-level detection via `object::File::parse`
+// ---------------------------------------------------------------------------
+
+fn hide_patches(data: &[u8], exported: &FxHashSet<String>) -> Option<Vec<Patch>> {
+    let file = object::File::parse(data).ok()?;
+    match file {
+        object::File::Elf64(_) => elf_hide_patches_impl::<elf::FileHeader64<Endianness>>(
+            data,
+            mem::offset_of!(elf::Sym64<Endianness>, st_other),
+            exported,
+        ),
+        object::File::Elf32(_) => elf_hide_patches_impl::<elf::FileHeader32<Endianness>>(
+            data,
+            mem::offset_of!(elf::Sym32<Endianness>, st_other),
+            exported,
+        ),
+        object::File::MachO64(_) => macho_hide_patches_impl::<macho::MachHeader64<Endianness>>(
+            data,
+            mem::offset_of!(macho::Nlist64<Endianness>, n_type),
+            exported,
+        ),
+        object::File::MachO32(_) => macho_hide_patches_impl::<macho::MachHeader32<Endianness>>(
+            data,
+            mem::offset_of!(macho::Nlist32<Endianness>, n_type),
+            exported,
+        ),
+        _ => None,
+    }
+}
+
+fn apply_hide(data: &[u8], exported: &FxHashSet<String>) -> Vec<u8> {
+    let patches = hide_patches(data, exported).unwrap_or_default();
+    apply_patches(data, &patches)
+}

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -131,7 +131,7 @@ pub fn link_binary(
                         RlibFlavor::Normal,
                         &path,
                     )
-                    .build(&out_filename);
+                    .build(&out_filename, None);
                 }
                 CrateType::StaticLib => {
                     link_staticlib(
@@ -566,7 +566,22 @@ fn link_staticlib(
         sess.dcx().emit_fatal(e);
     }
 
-    ab.build(out_filename);
+    let exported_symbols = if sess.opts.unstable_opts.staticlib_hide_internal_symbols {
+        if !matches!(sess.target.binary_format, BinaryFormat::Elf | BinaryFormat::MachO) {
+            sess.dcx().emit_warn(errors::StaticlibHideInternalSymbolsUnsupported {
+                binary_format: sess.target.archive_format.to_string(),
+            });
+            None
+        } else {
+            crate_info
+                .exported_symbols
+                .get(&CrateType::StaticLib)
+                .map(|symbols| symbols.iter().map(|(s, _)| s.clone()).collect())
+        }
+    } else {
+        None
+    };
+    ab.build(out_filename, exported_symbols);
 
     let crates = crate_info.used_crates.iter();
 
@@ -1265,7 +1280,7 @@ fn link_natively(
     if should_archive {
         let mut ab = archive_builder_builder.new_archive_builder(sess);
         ab.add_file(temp_filename, ArchiveEntryKind::Other);
-        ab.build(out_filename);
+        ab.build(out_filename, None);
     }
 }
 
@@ -3265,7 +3280,7 @@ fn add_static_crate(
             sess.dcx()
                 .emit_fatal(errors::RlibArchiveBuildFailure { path: cratepath.clone(), error });
         }
-        if archive.build(&dst) {
+        if archive.build(&dst, None) {
             link_upstream(&dst);
         }
     });

compiler/rustc_codegen_ssa/src/errors.rs

@@ -693,6 +693,14 @@ pub(crate) struct IncompatibleArchiveFormat {
 #[diag("linking static libraries is not supported for BPF")]
 pub(crate) struct BpfStaticlibNotSupported;
 
+#[derive(Diagnostic)]
+#[diag(
+    "-Zstaticlib-hide-internal-symbols only supports ELF and Mach-O targets, but the target uses `{$binary_format}`"
+)]
+pub(crate) struct StaticlibHideInternalSymbolsUnsupported {
+    pub binary_format: String,
+}
+
 #[derive(Diagnostic)]
 #[diag("entry symbol `main` declared multiple times")]
 #[help(

compiler/rustc_interface/src/tests.rs

@@ -866,6 +866,7 @@ fn test_unstable_options_tracking_hash() {
     tracked!(split_lto_unit, Some(true));
     tracked!(src_hash_algorithm, Some(SourceFileHashAlgorithm::Sha1));
     tracked!(stack_protector, StackProtector::All);
+    tracked!(staticlib_hide_internal_symbols, true);
     tracked!(teach, true);
     tracked!(thinlto, Some(true));
     tracked!(tiny_const_eval_limit, true);

compiler/rustc_interface/src/util.rs

@@ -471,7 +471,7 @@ impl ArchiveBuilderBuilder for DummyArchiveBuilderBuilder {
         output_path: &Path,
     ) {
         // Build an empty static library to avoid calling an external dlltool on mingw
-        ArArchiveBuilderBuilder.new_archive_builder(sess).build(output_path);
+        ArArchiveBuilderBuilder.new_archive_builder(sess).build(output_path, None);
     }
 }
 

compiler/rustc_session/src/config.rs

@@ -2465,6 +2465,14 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M
     let mut collected_options = Default::default();
 
     let mut unstable_opts = UnstableOptions::build(early_dcx, matches, &mut collected_options);
+
+    if unstable_opts.staticlib_hide_internal_symbols && !crate_types.contains(&CrateType::StaticLib)
+    {
+        early_dcx.early_warn(
+            "-Zstaticlib-hide-internal-symbols has no effect without `--crate-type staticlib`",
+        );
+    }
+
     let (lint_opts, describe_lints, lint_cap) = get_cmd_lint_options(early_dcx, matches);
 
     if !unstable_opts.unstable_options && json_timings {