Rename PyInit syms to avoid clashes

Built extensions in packages often have common names
like speedups, utils, _objects, cpython, etc. which
reside inside the package namespace.
The compiled extensions each have a PyInit_<module>
which needs to be renamed to PyInit_<pkg>_<module>
to avoid clashes when combined into a static binary.

Fixes #169

Author: jayvdb

pyoxidizer/Cargo.toml

@@ -36,6 +36,7 @@ hex = "0.3"
 itertools = "0.8"
 lazy_static = "1.3"
 libc = "0.2"
+object = { version = "0.15.0", features = ["read", "std", "write"] }
 regex = "1"
 reqwest = "0.9"
 rustc_version = "0.2"

pyoxidizer/src/pyrepackager/repackage.rs

@@ -6,9 +6,12 @@ use byteorder::{LittleEndian, WriteBytesExt};
 use glob::glob as findglob;
 use itertools::Itertools;
 use lazy_static::lazy_static;
+use object::{write, Object, ObjectSection, RelocationTarget, SectionKind, SymbolKind};
 use slog::{info, warn};
-use std::collections::{BTreeMap, BTreeSet};
+use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::env;
+use std::error::Error;
+use std::fmt;
 use std::fs;
 use std::fs::create_dir_all;
 use std::io::{BufRead, BufReader, Cursor, Error as IOError, Read, Write};
@@ -1126,7 +1129,13 @@ fn make_config_c(
     }
 
     for em in built_extension_modules.values() {
-        lines.push(format!("extern PyObject* {}(void);", em.init_fn));
+        let name_prefix = em.name.split('.').next().unwrap();
+        if em.init_fn.contains(name_prefix) {
+            lines.push(format!("extern PyObject* {}(void);", em.init_fn));
+        }
+        else {
+            lines.push(format!("extern PyObject* PyInit_{}(void);", em.name.replace(".", "_")));
+        }
     }
 
     lines.push(String::from("struct _inittab _PyImport_Inittab[] = {"));
@@ -1142,7 +1151,13 @@ fn make_config_c(
     }
 
     for em in built_extension_modules.values() {
-        lines.push(format!("{{\"{}\", {}}},", em.name, em.init_fn));
+        let name_prefix = em.name.split('.').next().unwrap();
+        if em.init_fn.contains(name_prefix) {
+            lines.push(format!("{{\"{}\", {}}},", em.name, em.init_fn));
+        }
+        else {
+            lines.push(format!("{{\"{}\", PyInit_{}}},", em.name, em.name.replace(".", "_")));
+        }
     }
 
     lines.push(String::from("{0, 0}"));
@@ -1151,6 +1166,142 @@ fn make_config_c(
     lines.join("\n")
 }
 
+#[derive(Debug, Clone)]
+struct NoRewriteError;
+
+impl fmt::Display for NoRewriteError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "no object rewriting was performed")
+    }
+}
+
+impl Error for NoRewriteError {
+    fn source(&self) -> Option<&(dyn Error + 'static)> {
+        // Generic error, underlying cause isn't tracked.
+        None
+    }
+}
+
+/// Rename object syn PyInit_foo to PyInit_<full_name> to avoid clashes
+fn rename_init (
+    logger: &slog::Logger,
+    name: &String,
+    object_data: &[u8]
+) -> Result<Vec<u8>, NoRewriteError> {
+    let mut rewritten = false;
+
+    let name_prefix = name.split('.').next().unwrap();
+
+    let in_object = match object::File::parse(object_data) {
+        Ok(object) => object,
+        Err(err) => {
+            panic!("Failed to parse compiled object for {}: {}", name, err);
+        }
+    };
+
+    let mut out_object = write::Object::new(in_object.format(), in_object.architecture());
+    out_object.mangling = write::Mangling::None;
+
+    let mut out_sections = HashMap::new();
+    for in_section in in_object.sections() {
+        if in_section.kind() == SectionKind::Metadata {
+            continue;
+        }
+        let section_id = out_object.add_section(
+            in_section.segment_name().unwrap_or("").as_bytes().to_vec(),
+            in_section.name().unwrap_or("").as_bytes().to_vec(),
+            in_section.kind(),
+        );
+        let out_section = out_object.section_mut(section_id);
+        if out_section.is_bss() {
+            out_section.append_bss(in_section.size(), in_section.align());
+        } else {
+            out_section.set_data(in_section.uncompressed_data().into(), in_section.align());
+        }
+        out_sections.insert(in_section.index(), section_id);
+    }
+
+    let mut out_symbols = HashMap::new();
+    for (symbol_index, in_symbol) in in_object.symbols() {
+        if in_symbol.kind() == SymbolKind::Null {
+            continue;
+        }
+        let (section, value) = match in_symbol.section_index() {
+            Some(index) => (
+                Some(*out_sections.get(&index).unwrap()),
+                in_symbol.address() - in_object.section_by_index(index).unwrap().address(),
+            ),
+            None => (None, in_symbol.address()),
+        };
+        let in_sym_name = in_symbol.name().unwrap_or("");
+        let sym_name = if in_sym_name.starts_with("PyInit_") && !in_sym_name.contains(name_prefix) {
+            format!("PyInit_{}", name.replace(".", "_"))
+        } else {
+			String::from(in_sym_name)
+        };
+        if sym_name != in_sym_name {
+            warn!(
+                logger,
+                "rewrote object symbol name {} to {}",
+                in_sym_name,
+                sym_name,
+            );
+
+		    rewritten = true;
+		}
+
+        let out_symbol = write::Symbol {
+            name: sym_name.as_bytes().to_vec(),
+            value,
+            size: in_symbol.size(),
+            kind: in_symbol.kind(),
+            scope: in_symbol.scope(),
+            weak: in_symbol.is_weak(),
+            section,
+        };
+        let symbol_id = out_object.add_symbol(out_symbol);
+        out_symbols.insert(symbol_index, symbol_id);
+    }
+
+
+    if !rewritten {
+        info!(
+            logger,
+            "no symbol name rewriting occurred for {}",
+            name,
+        );
+		return Err(NoRewriteError)
+	}
+
+    for in_section in in_object.sections() {
+        if in_section.kind() == SectionKind::Metadata {
+            continue;
+        }
+        let out_section = *out_sections.get(&in_section.index()).unwrap();
+        for (offset, in_relocation) in in_section.relocations() {
+            let symbol = match in_relocation.target() {
+                RelocationTarget::Symbol(symbol) => *out_symbols.get(&symbol).unwrap(),
+                RelocationTarget::Section(section) => {
+                    out_object.section_symbol(*out_sections.get(&section).unwrap())
+                }
+            };
+            let out_relocation = write::Relocation {
+                offset,
+                size: in_relocation.size(),
+                kind: in_relocation.kind(),
+                encoding: in_relocation.encoding(),
+                symbol,
+                addend: in_relocation.addend(),
+            };
+            out_object
+                .add_relocation(out_section, out_relocation)
+                .unwrap();
+        }
+    }
+
+    Ok(out_object.write().unwrap())
+}
+
 #[derive(Debug)]
 pub struct LibpythonInfo {
     path: PathBuf,
@@ -1341,7 +1492,11 @@ pub fn link_libpython(
         for (i, object_data) in em.object_file_data.iter().enumerate() {
             let out_path = temp_dir_path.join(format!("{}.{}.o", name, i));
 
-            fs::write(&out_path, object_data).expect("unable to write object file");
+            match rename_init(logger, name, object_data) {
+                Ok(val) => fs::write(&out_path, val).expect("unable to write object file"),
+                Err(_) => fs::write(&out_path, object_data).expect("unable to write object file")
+            };
+
             build.object(&out_path);
         }