staticlib symbol trimming: hide non-exported symbols in staticlibs

Author: cezarbbb

compiler/rustc_codegen_ssa/src/back/archive.rs

@@ -11,7 +11,7 @@ use ar_archive_writer::{
 pub use ar_archive_writer::{DEFAULT_OBJECT_READER, ObjectReader};
 use object::read::archive::ArchiveFile;
 use object::read::macho::FatArch;
-use rustc_data_structures::fx::FxIndexSet;
+use rustc_data_structures::fx::{FxHashSet, FxIndexSet};
 use rustc_data_structures::memmap::Mmap;
 use rustc_fs_util::TempDirBuilder;
 use rustc_metadata::EncodedMetadata;
@@ -318,6 +318,8 @@ pub trait ArchiveBuilder {
     ) -> io::Result<()>;
 
     fn build(self: Box<Self>, output: &Path) -> bool;
+
+    fn set_keep_symbols(&mut self, keep: FxHashSet<String>);
 }
 
 pub struct ArArchiveBuilderBuilder;
@@ -337,6 +339,7 @@ pub struct ArArchiveBuilder<'a> {
     // Don't use an `HashMap` here, as the order is important. `lib.rmeta` needs
     // to be at the end of an archive in some cases for linkers to not get confused.
     entries: Vec<(Vec<u8>, ArchiveEntry)>,
+    keep_symbols: Option<FxHashSet<String>>,
 }
 
 #[derive(Debug)]
@@ -347,7 +350,17 @@ enum ArchiveEntry {
 
 impl<'a> ArArchiveBuilder<'a> {
     pub fn new(sess: &'a Session, object_reader: &'static ObjectReader) -> ArArchiveBuilder<'a> {
-        ArArchiveBuilder { sess, object_reader, src_archives: vec![], entries: vec![] }
+        ArArchiveBuilder {
+            sess,
+            object_reader,
+            src_archives: vec![],
+            entries: vec![],
+            keep_symbols: None,
+        }
+    }
+
+    pub fn set_keep_symbols(&mut self, keep: FxHashSet<String>) {
+        self.keep_symbols = Some(keep);
     }
 }
 
@@ -460,6 +473,10 @@ impl<'a> ArchiveBuilder for ArArchiveBuilder<'a> {
             }
         }
     }
+
+    fn set_keep_symbols(&mut self, keep: FxHashSet<String>) {
+        self.keep_symbols = Some(keep);
+    }
 }
 
 impl<'a> ArArchiveBuilder<'a> {
@@ -478,7 +495,7 @@ impl<'a> ArArchiveBuilder<'a> {
         let mut entries = Vec::new();
 
         for (entry_name, entry) in self.entries {
-            let data =
+            let data: Box<dyn AsRef<[u8]>> =
                 match entry {
                     ArchiveEntry::FromArchive { archive_index, file_range } => {
                         let src_archive = &self.src_archives[archive_index];
@@ -498,6 +515,16 @@ impl<'a> ArArchiveBuilder<'a> {
                     },
                 };
 
+            let data: Box<dyn AsRef<[u8]>> = if let Some(ref keep) = self.keep_symbols {
+                if let Some(filtered) = elf_filter_global_symbols(data.as_ref().as_ref(), keep) {
+                    Box::new(filtered)
+                } else {
+                    data
+                }
+            } else {
+                data
+            };
+
             entries.push(NewArchiveMember {
                 buf: data,
                 object_reader: self.object_reader,
@@ -557,3 +584,142 @@ impl<'a> ArArchiveBuilder<'a> {
 fn io_error_context(context: &str, err: io::Error) -> io::Error {
     io::Error::new(io::ErrorKind::Other, format!("{context}: {err}"))
 }
+
+/// For ELF object files, set `STV_HIDDEN` visibility on GLOBAL/WEAK symbols
+/// that are NOT in the `keep_symbols` set. Returns `Some(modified_data)` if
+/// any changes were made, `None` if the data is not an ELF object or no
+/// changes were needed.
+fn elf_filter_global_symbols(data: &[u8], keep_symbols: &FxHashSet<String>) -> Option<Vec<u8>> {
+    use object::{Endianness, elf};
+
+    if data.len() < 16 || &data[0..4] != elf::ELFMAG {
+        return None;
+    }
+
+    match data[4] {
+        elf::ELFCLASS64 => elf_filter_symbols_inner::<elf::FileHeader64<Endianness>>(
+            data,
+            keep_symbols,
+            /* sym_entry_size= */ 24,
+            /* st_info_offset= */ 4,
+            /* st_other_offset= */ 5,
+        ),
+        elf::ELFCLASS32 => elf_filter_symbols_inner::<elf::FileHeader32<Endianness>>(
+            data,
+            keep_symbols,
+            /* sym_entry_size= */ 16,
+            /* st_info_offset= */ 12,
+            /* st_other_offset= */ 13,
+        ),
+        _ => None,
+    }
+}
+
+fn elf_filter_symbols_inner<Elf: object::read::elf::FileHeader<Endian = object::Endianness>>(
+    data: &[u8],
+    keep_symbols: &FxHashSet<String>,
+    sym_entry_size: usize,
+    st_info_offset: usize,
+    st_other_offset: usize,
+) -> Option<Vec<u8>>
+where
+    u64: From<Elf::Word>,
+{
+    use object::read::elf::SectionHeader;
+    use object::{Endianness, elf};
+
+    let endian = match Elf::parse(data) {
+        Ok(h) => match h.endian() {
+            Ok(e) => e,
+            Err(_) => return None,
+        },
+        Err(_) => return None,
+    };
+
+    let header = Elf::parse(data).unwrap();
+    let sections = match header.sections(endian, data) {
+        Ok(s) => s,
+        Err(_) => return None,
+    };
+
+    let mut modified: Option<Vec<u8>> = None;
+
+    for section in sections.iter() {
+        if section.sh_type(endian) != elf::SHT_SYMTAB {
+            continue;
+        }
+
+        let strtab_index = section.sh_link(endian) as usize;
+        let strtab_section = match sections.section(object::SectionIndex(strtab_index)) {
+            Ok(s) => s,
+            Err(_) => continue,
+        };
+        let strtab_data = match strtab_section.data(endian, data) {
+            Ok(d) => d,
+            Err(_) => continue,
+        };
+
+        if sym_entry_size == 0 {
+            continue;
+        }
+
+        let sym_offset = u64::from(section.sh_offset(endian)) as usize;
+        let sym_size = u64::from(section.sh_size(endian)) as usize;
+        let sym_count = sym_size / sym_entry_size;
+
+        let buf = modified.get_or_insert_with(|| data.to_vec());
+
+        for i in 1..sym_count {
+            let off = sym_offset + i * sym_entry_size;
+            if off + sym_entry_size > buf.len() {
+                break;
+            }
+
+            let st_info = buf[off + st_info_offset];
+            let binding = st_info >> 4;
+
+            // Only process GLOBAL and WEAK symbols
+            if binding != elf::STB_GLOBAL && binding != elf::STB_WEAK {
+                continue;
+            }
+
+            let st_shndx_offset = st_other_offset + 1;
+            let st_shndx_bytes: [u8; 2] =
+                buf[off + st_shndx_offset..off + st_shndx_offset + 2].try_into().unwrap_or([0, 0]);
+            let st_shndx = match endian {
+                Endianness::Little => u16::from_le_bytes(st_shndx_bytes),
+                Endianness::Big => u16::from_be_bytes(st_shndx_bytes),
+            };
+            if st_shndx == elf::SHN_UNDEF as u16 {
+                continue;
+            }
+
+            let st_name_bytes: [u8; 4] = buf[off..off + 4].try_into().unwrap_or([0; 4]);
+            let st_name_off = match endian {
+                Endianness::Little => u32::from_le_bytes(st_name_bytes),
+                Endianness::Big => u32::from_be_bytes(st_name_bytes),
+            } as usize;
+
+            if st_name_off >= strtab_data.len() {
+                continue;
+            }
+            let name_end = strtab_data[st_name_off..]
+                .iter()
+                .position(|&b| b == 0)
+                .unwrap_or(strtab_data.len() - st_name_off);
+            let name = match std::str::from_utf8(&strtab_data[st_name_off..st_name_off + name_end])
+            {
+                Ok(s) => s,
+                Err(_) => continue,
+            };
+
+            if keep_symbols.contains(name) {
+                continue;
+            }
+
+            buf[off + st_other_offset] = elf::STV_HIDDEN;
+        }
+    }
+
+    modified
+}

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -525,6 +525,24 @@ fn link_staticlib(
         sess.dcx().emit_fatal(e);
     }
 
+    if sess.opts.unstable_opts.staticlib_hide_internal_symbols {
+        if !matches!(&*sess.target.archive_format, "gnu" | "bsd") {
+            sess.dcx().emit_warn(errors::StaticlibHideInternalSymbolsUnsupported {
+                archive_format: sess.target.archive_format.to_string(),
+            });
+        } else if let Some(symbols) =
+            codegen_results.crate_info.exported_symbols.get(&CrateType::Staticlib)
+        {
+            use rustc_data_structures::fx::FxHashSet;
+            let mut keep: FxHashSet<String> = symbols.iter().map(|(s, _)| s.clone()).collect();
+            // rust_eh_personality cannot be mangled and hidden.
+            // See discussions in https://github.com/rust-lang/rust/issues/104707
+            // Keep it visible so that .eh_frame references from other object files
+            keep.insert("rust_eh_personality".to_owned());
+            ab.set_keep_symbols(keep);
+        }
+    }
+
     ab.build(out_filename);
 
     let crates = crate_info.used_crates.iter();

compiler/rustc_codegen_ssa/src/errors.rs

@@ -675,6 +675,12 @@ pub(crate) struct UnknownArchiveKind<'a> {
 #[diag("linking static libraries is not supported for BPF")]
 pub(crate) struct BpfStaticlibNotSupported;
 
+#[derive(Diagnostic)]
+#[diag("-Zstaticlib-hide-internal-symbols only supports ELF archive formats (gnu/bsd), but the target uses `{$archive_format}`")]
+pub(crate) struct StaticlibHideInternalSymbolsUnsupported {
+    pub archive_format: String,
+}
+
 #[derive(Diagnostic)]
 #[diag("entry symbol `main` declared multiple times")]
 #[help(

compiler/rustc_interface/src/tests.rs

@@ -869,6 +869,7 @@ fn test_unstable_options_tracking_hash() {
     tracked!(split_lto_unit, Some(true));
     tracked!(src_hash_algorithm, Some(SourceFileHashAlgorithm::Sha1));
     tracked!(stack_protector, StackProtector::All);
+    tracked!(staticlib_hide_internal_symbols, true);
     tracked!(teach, true);
     tracked!(thinlto, Some(true));
     tracked!(tiny_const_eval_limit, true);

compiler/rustc_session/src/config.rs

@@ -2473,6 +2473,14 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M
     let mut collected_options = Default::default();
 
     let mut unstable_opts = UnstableOptions::build(early_dcx, matches, &mut collected_options);
+
+    if unstable_opts.staticlib_hide_internal_symbols && !crate_types.contains(&CrateType::Staticlib)
+    {
+        early_dcx.early_fatal(
+            "-Zstaticlib-hide-internal-symbols can only be used with `--crate-type staticlib`",
+        );
+    }
+
     let (lint_opts, describe_lints, lint_cap) = get_cmd_lint_options(early_dcx, matches);
 
     if !unstable_opts.unstable_options && json_timings {

compiler/rustc_session/src/options.rs

@@ -2764,6 +2764,8 @@ written to standard error output)"),
         "control stack smash protection strategy (`rustc --print stack-protector-strategies` for details)"),
     staticlib_allow_rdylib_deps: bool = (false, parse_bool, [TRACKED],
         "allow staticlibs to have rust dylib dependencies"),
+    staticlib_hide_internal_symbols: bool = (false, parse_bool, [TRACKED],
+        "hide Rust internal symbols when building staticlibs"),
     staticlib_prefer_dynamic: bool = (false, parse_bool, [TRACKED],
         "prefer dynamic linking to static linking for staticlibs (default: no)"),
     strict_init_checks: bool = (false, parse_bool, [TRACKED],

src/doc/unstable-book/src/compiler-flags/staticlib-hide-internal-symbols.md

@@ -0,0 +1,11 @@
+# `staticlib-hide-internal-symbols`
+
+When building a `staticlib`, this option hides all Rust-internal symbols
+(except `rust_eh_personality`) by setting their ELF visibility to
+`STV_HIDDEN`.
+
+This option can only be used with `--crate-type staticlib`. Using it with
+other crate types will result in a compilation error.
+
+Currently only ELF targets are supported (Linux, BSD, etc.). On non-ELF
+targets (macOS, Windows), a warning is emitted and the flag has no effect.

tests/run-make/staticlib-hide-internal-symbols/lib.rs

@@ -0,0 +1,40 @@
+#![crate_type = "staticlib"]
+
+use std::collections::HashMap;
+use std::panic::{AssertUnwindSafe, catch_unwind};
+
+#[no_mangle]
+pub extern "C" fn my_add(a: i32, b: i32) -> i32 {
+    a + b
+}
+
+#[no_mangle]
+pub extern "C" fn my_hash_lookup(key: u64) -> u64 {
+    let mut map = HashMap::new();
+    for i in 0..100u64 {
+        map.insert(i, i.wrapping_mul(2654435761));
+    }
+    *map.get(&key).unwrap_or(&0)
+}
+
+fn internal_helper() -> i32 {
+    42
+}
+
+#[no_mangle]
+pub extern "C" fn call_internal() -> i32 {
+    internal_helper()
+}
+
+#[no_mangle]
+pub extern "C" fn my_safe_div(a: i32, b: i32) -> i32 {
+    match catch_unwind(AssertUnwindSafe(|| {
+        if b == 0 {
+            panic!("division by zero!");
+        }
+        a / b
+    })) {
+        Ok(result) => result,
+        Err(_) => -1,
+    }
+}

tests/run-make/staticlib-hide-internal-symbols/main.c

@@ -0,0 +1,18 @@
+extern int my_add(int a, int b);
+extern unsigned long my_hash_lookup(unsigned long key);
+extern int call_internal(void);
+extern int my_safe_div(int a, int b);
+
+int main() {
+    if (my_add(10, 20) != 30)
+        return 1;
+    if (my_hash_lookup(5) != 5UL * 2654435761UL)
+        return 1;
+    if (call_internal() != 42)
+        return 1;
+    if (my_safe_div(100, 5) != 20)
+        return 1;
+    if (my_safe_div(100, 0) != -1)
+        return 1;
+    return 0;
+}