Rename PyInit syms to avoid clashes

Built extensions in packages often have common names
like speedups, utils, _objects, cpython, etc. which
reside inside the package namespace.
The compiled extensions each have a PyInit_<module>
which needs to be renamed to PyInit_<pkg>_<module>
to avoid clashes when combined into a static binary.

Fixes #169

Author: jayvdb

pyoxidizer/Cargo.toml

@@ -36,6 +36,7 @@ hex = "0.3"
 itertools = "0.8"
 lazy_static = "1.3"
 libc = "0.2"
+object = { version = "0.15.0", features = ["read", "std", "write"] }
 regex = "1"
 reqwest = "0.9"
 rustc_version = "0.2"

pyoxidizer/src/pyrepackager/mod.rs

@@ -6,6 +6,7 @@ pub mod bytecode;
 pub mod config;
 pub mod dist;
 pub mod fsscan;
+pub mod object;
 pub mod packaging_rule;
 pub mod repackage;
 pub mod resource;

pyoxidizer/src/pyrepackager/object.rs

@@ -0,0 +1,145 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use object::{write, Object, ObjectSection, RelocationTarget, SectionKind, SymbolKind};
+use slog::{info, warn};
+use std::collections::{HashMap};
+use std::error::Error;
+use std::fmt;
+
+#[derive(Debug, Clone)]
+pub struct NoRewriteError;
+
+impl fmt::Display for NoRewriteError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "no object rewriting was performed")
+    }
+}
+
+impl Error for NoRewriteError {
+    fn source(&self) -> Option<&(dyn Error + 'static)> {
+        // Generic error, underlying cause isn't tracked.
+        None
+    }
+}
+
+/// Rename object syn PyInit_foo to PyInit_<full_name> to avoid clashes
+pub fn rename_init (
+    logger: &slog::Logger,
+    name: &String,
+    object_data: &[u8]
+) -> Result<Vec<u8>, NoRewriteError> {
+    let mut rewritten = false;
+
+    let name_prefix = name.split('.').next().unwrap();
+
+    let in_object = match object::File::parse(object_data) {
+        Ok(object) => object,
+        Err(err) => {
+            panic!("Failed to parse compiled object for {}: {}", name, err);
+        }
+    };
+
+    let mut out_object = write::Object::new(in_object.format(), in_object.architecture());
+    out_object.mangling = write::Mangling::None;
+
+    let mut out_sections = HashMap::new();
+    for in_section in in_object.sections() {
+        if in_section.kind() == SectionKind::Metadata {
+            continue;
+        }
+        let section_id = out_object.add_section(
+            in_section.segment_name().unwrap_or("").as_bytes().to_vec(),
+            in_section.name().unwrap_or("").as_bytes().to_vec(),
+            in_section.kind(),
+        );
+        let out_section = out_object.section_mut(section_id);
+        if out_section.is_bss() {
+            out_section.append_bss(in_section.size(), in_section.align());
+        } else {
+            out_section.set_data(in_section.uncompressed_data().into(), in_section.align());
+        }
+        out_sections.insert(in_section.index(), section_id);
+    }
+
+    let mut out_symbols = HashMap::new();
+    for (symbol_index, in_symbol) in in_object.symbols() {
+        if in_symbol.kind() == SymbolKind::Null {
+            continue;
+        }
+        let (section, value) = match in_symbol.section_index() {
+            Some(index) => (
+                Some(*out_sections.get(&index).unwrap()),
+                in_symbol.address() - in_object.section_by_index(index).unwrap().address(),
+            ),
+            None => (None, in_symbol.address()),
+        };
+        let in_sym_name = in_symbol.name().unwrap_or("");
+        let sym_name = if in_sym_name.starts_with("PyInit_") && !in_sym_name.contains(name_prefix) {
+            format!("PyInit_{}", name.replace(".", "_"))
+        } else {
+			String::from(in_sym_name)
+        };
+        if sym_name != in_sym_name {
+            warn!(
+                logger,
+                "rewrote object symbol name {} to {}",
+                in_sym_name,
+                sym_name,
+            );
+
+		    rewritten = true;
+		}
+
+        let out_symbol = write::Symbol {
+            name: sym_name.as_bytes().to_vec(),
+            value,
+            size: in_symbol.size(),
+            kind: in_symbol.kind(),
+            scope: in_symbol.scope(),
+            weak: in_symbol.is_weak(),
+            section,
+        };
+        let symbol_id = out_object.add_symbol(out_symbol);
+        out_symbols.insert(symbol_index, symbol_id);
+    }
+
+
+    if !rewritten {
+        info!(
+            logger,
+            "no symbol name rewriting occurred for {}",
+            name,
+        );
+		return Err(NoRewriteError)
+	}
+
+    for in_section in in_object.sections() {
+        if in_section.kind() == SectionKind::Metadata {
+            continue;
+        }
+        let out_section = *out_sections.get(&in_section.index()).unwrap();
+        for (offset, in_relocation) in in_section.relocations() {
+            let symbol = match in_relocation.target() {
+                RelocationTarget::Symbol(symbol) => *out_symbols.get(&symbol).unwrap(),
+                RelocationTarget::Section(section) => {
+                    out_object.section_symbol(*out_sections.get(&section).unwrap())
+                }
+            };
+            let out_relocation = write::Relocation {
+                offset,
+                size: in_relocation.size(),
+                kind: in_relocation.kind(),
+                encoding: in_relocation.encoding(),
+                symbol,
+                addend: in_relocation.addend(),
+            };
+            out_object
+                .add_relocation(out_section, out_relocation)
+                .unwrap();
+        }
+    }
+
+    Ok(out_object.write().unwrap())
+}

pyoxidizer/src/pyrepackager/repackage.rs

@@ -23,6 +23,7 @@ use super::dist::{
     analyze_python_distribution_tar_zst, resolve_python_distribution_archive, ExtensionModule,
     LicenseInfo, PythonDistributionInfo,
 };
+use super::object::{rename_init};
 use super::packaging_rule::{
     packages_from_module_name, packages_from_module_names, resolve_python_packaging,
 };
@@ -1126,23 +1127,39 @@ fn make_config_c(
     }
 
     for em in built_extension_modules.values() {
-        lines.push(format!("extern PyObject* {}(void);", em.init_fn));
+        let ambiguous_line = format!("extern PyObject* {}(void);", em.init_fn);
+
+        if lines.contains(&ambiguous_line) {
+            lines.push(format!("extern PyObject* PyInit_{}(void);", em.name.replace(".", "_")));
+        }
+        else {
+            lines.push(ambiguous_line);
+        }
     }
 
     lines.push(String::from("struct _inittab _PyImport_Inittab[] = {"));
 
+    let mut ambiguous_init_fns: Vec<String> = Vec::new();
+
     for em in extension_modules.values() {
         if let Some(init_fn) = &em.init_fn {
             if init_fn == "NULL" {
                 continue;
             }
 
             lines.push(format!("{{\"{}\", {}}},", em.module, init_fn));
+            ambiguous_init_fns.push(init_fn.to_string());
         }
     }
 
     for em in built_extension_modules.values() {
-        lines.push(format!("{{\"{}\", {}}},", em.name, em.init_fn));
+        if ambiguous_init_fns.contains(&em.init_fn) {
+            lines.push(format!("{{\"{}\", PyInit_{}}},", em.name, em.name.replace(".", "_")));
+        }
+        else {
+            lines.push(format!("{{\"{}\", {}}},", em.name, em.init_fn));
+            ambiguous_init_fns.push(em.init_fn.clone());
+        }
     }
 
     lines.push(String::from("{0, 0}"));
@@ -1284,12 +1301,20 @@ pub fn link_libpython(
         // TODO handle static/dynamic libraries.
     }
 
+    let mut ambiguous_init_fns: Vec<String> = Vec::new();
+
     warn!(
         logger,
         "resolving inputs for {} extension modules...",
         extension_modules.len() + built_extension_modules.len()
     );
     for (name, em) in extension_modules {
+        if let Some(init_fn) = &em.init_fn {
+            if init_fn != "NULL" {
+                ambiguous_init_fns.push(init_fn.to_string());
+            }
+        }
+
         if em.builtin_default {
             continue;
         }
@@ -1341,7 +1366,17 @@ pub fn link_libpython(
         for (i, object_data) in em.object_file_data.iter().enumerate() {
             let out_path = temp_dir_path.join(format!("{}.{}.o", name, i));
 
-            fs::write(&out_path, object_data).expect("unable to write object file");
+            if i == em.object_file_data.len() - 1 && ambiguous_init_fns.contains(&em.init_fn) {
+            //if ambiguous_init_fns.contains(&em.init_fn) {
+                match rename_init(logger, name, object_data) {
+                    Ok(val) => fs::write(&out_path, val).expect("unable to write object file"),
+                    Err(_) => fs::write(&out_path, object_data).expect("unable to write object file")
+                };
+            }
+            else {
+                fs::write(&out_path, object_data).expect("unable to write object file");
+            }
+
             build.object(&out_path);
         }
 
@@ -1350,6 +1385,10 @@ pub fn link_libpython(
             needed_libraries_external.insert(&library);
         }
 
+        if !ambiguous_init_fns.contains(&em.init_fn) {
+            ambiguous_init_fns.push(em.init_fn.clone());
+        }
+
         // TODO do something with library_dirs.
     }