Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions build.rs
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
use std::env;

use pyo3_build_config::pyo3_build_script_impl::{cargo_env_var, errors::Result};
use pyo3_build_config::pyo3_build_script_impl::errors::Result;
use pyo3_build_config::{
add_python_framework_link_args, bail, print_feature_cfgs, InterpreterConfig,
add_python_framework_link_args, bail, print_feature_cfgs, InterpreterConfig, BUILD_CTX,
};

fn ensure_auto_initialize_ok(interpreter_config: &InterpreterConfig) -> Result<()> {
if cargo_env_var("CARGO_FEATURE_AUTO_INITIALIZE").is_some() && !interpreter_config.shared {
if *BUILD_CTX.cargo.cargo_feature_auto_initialize && !interpreter_config.shared {
bail!(
"The `auto-initialize` feature is enabled, but your python installation only supports \
embedding the Python interpreter statically. If you are attempting to run tests, or a \
Expand Down
214 changes: 166 additions & 48 deletions pyo3-build-config/src/impl_.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ use std::{
path::{Path, PathBuf},
process::{Command, Stdio},
str::{self, FromStr},
sync::LazyLock,
};

pub use target_lexicon::Triple;
Expand Down Expand Up @@ -51,26 +52,146 @@ thread_local! {
static READ_ENV_VARS: RefCell<Vec<String>> = const { RefCell::new(Vec::new()) };
}

/// Gets an environment variable owned by cargo.
///
/// Environment variables set by cargo are expected to be valid UTF8.
pub fn cargo_env_var(var: &str) -> Option<String> {
env::var_os(var).map(|os_string| os_string.to_str().unwrap().into())
pub static BUILD_CTX: LazyLock<BuildScriptContext> = LazyLock::new(BuildScriptContext::new);

pub struct BuildScriptContext {
pub ext: ExtEnv,
pub cargo: CargoEnv,
}

/// Gets an external environment variable, and registers the build script to rerun if
/// the variable changes.
pub fn env_var(var: &str) -> Option<OsString> {
if cfg!(feature = "resolve-config") {
println!("cargo:rerun-if-env-changed={var}");
pub struct ExtEnv {
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
pub struct ExtEnv {
/// External environment variables which can be used to configure PyO3.
///
/// Modifying these will generally cause PyO3 to rebuild (subject to some
/// ordering between which variables take priority for certain cases).
pub struct ExtEnv {

pub is_print_config: LazyLock<bool>,
pub use_abi13_forward_compatibility: LazyLock<bool>,
pub pyo3_config_file: LazyLock<Option<PathBuf>>,
pub pyo3_python: LazyLock<Option<OsString>>,
pub pyo3_no_python: LazyLock<bool>,
pub pyo3_build_extension_module: LazyLock<bool>,
pub pyo3_cross: LazyLock<Option<OsString>>,
pub pyo3_cross_lib_dir: LazyLock<Option<OsString>>,
pub pyo3_cross_python_version: LazyLock<Option<OsString>>,
pub pyo3_cross_python_implementation: LazyLock<Option<OsString>>,
pub python_sysconfigdata_name: LazyLock<Option<OsString>>,
pub virtual_env: LazyLock<Option<OsString>>,
pub conda_prefix: LazyLock<Option<OsString>>,
}

pub struct CargoEnv {
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
pub struct CargoEnv {
/// Environment variables set by cargo, which PyO3's build scripts consume.
pub struct CargoEnv {

pub dep_python_pyo3_config: LazyLock<Option<String>>,
pub cargo_feature_abi3: LazyLock<bool>,
pub cargo_feature_extension_module: LazyLock<bool>,

/// The minimum supported Python version from PyO3 `abi3-py*` features.
/// Must be called from a PyO3 crate build script.
pub abi3_version: LazyLock<Option<PythonVersion>>,
pub cargo_cfg_target_pointer_width:
LazyLock<Result<u32, Box<dyn std::error::Error + Send + Sync>>>,
pub cargo_cfg_target_os: LazyLock<String>,
pub cargo_feature_auto_initialize: LazyLock<bool>,
}
Comment on lines +62 to +90
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have an intuition that it'd be helpful if these were named after their environment variables, e.g. is_print_config -> pyo3_print_config etc.

That'd make it really easy to see all the environment accesses at a glance.


impl BuildScriptContext {
pub fn new() -> Self {
Self {
ext: ExtEnv::new(),
cargo: CargoEnv::new(),
}
}
#[cfg(test)]
{
READ_ENV_VARS.with(|env_vars| {
env_vars.borrow_mut().push(var.to_owned());
});
}

impl Default for BuildScriptContext {
fn default() -> Self {
Self::new()
}
}

impl ExtEnv {
pub fn new() -> Self {
Self {
is_print_config: LazyLock::new(|| {
Self::env_var("PYO3_PRINT_CONFIG").is_some_and(|os_str| os_str == "1")
}),
use_abi13_forward_compatibility: LazyLock::new(|| {
Self::env_var("PYO3_USE_ABI3_FORWARD_COMPATIBILITY")
.is_some_and(&|os_str| os_str == "1")
}),
pyo3_config_file: LazyLock::new(|| {
Self::env_var("PYO3_CONFIG_FILE").map(PathBuf::from)
}),
pyo3_python: LazyLock::new(|| Self::env_var("PYO3_PYTHON")),
pyo3_no_python: LazyLock::new(|| Self::env_var("PYO3_NO_PYTHON").is_none()),
pyo3_build_extension_module: LazyLock::new(|| {
Self::env_var("PYO3_BUILD_EXTENSION_MODULE").is_some()
}),
pyo3_cross: LazyLock::new(|| Self::env_var("PYO3_CROSS")),
pyo3_cross_lib_dir: LazyLock::new(|| Self::env_var("PYO3_CROSS_LIB_DIR")),
pyo3_cross_python_version: LazyLock::new(|| Self::env_var("PYO3_CROSS_PYTHON_VERSION")),
pyo3_cross_python_implementation: LazyLock::new(|| {
Self::env_var("PYO3_CROSS_PYTHON_IMPLEMENTATION")
}),
python_sysconfigdata_name: LazyLock::new(|| {
Self::env_var("_PYTHON_SYSCONFIGDATA_NAME")
}),
virtual_env: LazyLock::new(|| Self::env_var("VIRTUAL_ENV")),
conda_prefix: LazyLock::new(|| Self::env_var("CONDA_PREFIX")),
}
}

/// Gets an external environment variable, and registers the build script to rerun if
/// the variable changes.
pub fn env_var(var: &str) -> Option<OsString> {
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can probably make this private, and similar for cargo_env_var below.

Suggested change
pub fn env_var(var: &str) -> Option<OsString> {
fn env_var(var: &str) -> Option<OsString> {

if cfg!(feature = "resolve-config") {
println!("cargo:rerun-if-env-changed={var}");
}
#[cfg(test)]
{
READ_ENV_VARS.with(|env_vars| {
env_vars.borrow_mut().push(var.to_owned());
});
}
env::var_os(var)
}
}

impl CargoEnv {
fn new() -> Self {
Self {
dep_python_pyo3_config: LazyLock::new(|| Self::cargo_env_var("DEP_PYTHON_PYO3_CONFIG")),
cargo_feature_abi3: LazyLock::new(|| {
Self::cargo_env_var("CARGO_FEATURE_ABI3").is_some()
}),
cargo_feature_extension_module: LazyLock::new(|| {
Self::cargo_env_var("CARGO_FEATURE_EXTENSION_MODULE").is_some()
}),
abi3_version: LazyLock::new(|| {
let minor_version = (MINIMUM_SUPPORTED_VERSION.minor..=ABI3_MAX_MINOR)
.find(|i| Self::cargo_env_var(&format!("CARGO_FEATURE_ABI3_PY3{i}")).is_some());
minor_version.map(|minor| PythonVersion { major: 3, minor })
}),
cargo_cfg_target_pointer_width: LazyLock::new(|| {
Ok(
match Self::cargo_env_var("CARGO_CFG_TARGET_POINTER_WIDTH").as_deref() {
Some("64") => 64,
Some("32") => 32,
Some(x) => bail!("unexpected Rust target pointer width: {}", x),
None => bail!("CARGO_CFG_TARGET_POINTER_WIDTH is unset"),
},
)
}),
cargo_cfg_target_os: LazyLock::new(|| {
Self::cargo_env_var("CARGO_CFG_TARGET_OS").unwrap()
}),
cargo_feature_auto_initialize: LazyLock::new(|| {
Self::cargo_env_var("CARGO_FEATURE_AUTO_INITIALIZE").is_some()
}),
}
}

/// Gets an environment variable owned by cargo.
///
/// Environment variables set by cargo are expected to be valid UTF8.
pub fn cargo_env_var(var: &str) -> Option<String> {
env::var_os(var).map(|os_string| os_string.to_str().unwrap().into())
}
env::var_os(var)
}

/// Gets the compilation target triple from environment variables set by Cargo.
Expand Down Expand Up @@ -455,8 +576,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED"))
/// The `abi3` features, if set, may apply an `abi3` constraint to the Python version.
#[allow(dead_code)] // only used in build.rs
pub(super) fn from_pyo3_config_file_env() -> Option<Result<Self>> {
env_var("PYO3_CONFIG_FILE").map(|path| {
let path = Path::new(&path);
BUILD_CTX.ext.pyo3_config_file.as_ref().map(|path| {
println!("cargo:rerun-if-changed={}", path.display());
// Absolute path is necessary because this build script is run with a cwd different to the
// original `cargo build` instruction.
Expand All @@ -473,7 +593,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED"))
// TODO: abi3 is a property of the build mode, not the interpreter. Should this be
// removed from `InterpreterConfig`?
config.abi3 |= is_abi3();
config.fixup_for_abi3_version(get_abi3_version())?;
config.fixup_for_abi3_version(*BUILD_CTX.cargo.abi3_version)?;

Ok(config)
})
Expand All @@ -490,8 +610,11 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED"))

#[doc(hidden)]
pub fn from_cargo_dep_env() -> Option<Result<Self>> {
cargo_env_var("DEP_PYTHON_PYO3_CONFIG")
.map(|buf| InterpreterConfig::from_reader(&*unescape(&buf)))
BUILD_CTX
.cargo
.dep_python_pyo3_config
.as_ref()
.map(|buf| InterpreterConfig::from_reader(&*unescape(buf)))
}

#[doc(hidden)]
Expand Down Expand Up @@ -834,24 +957,14 @@ impl FromStr for PythonImplementation {
///
/// Returns `false` if `PYO3_NO_PYTHON` environment variable is set.
fn have_python_interpreter() -> bool {
env_var("PYO3_NO_PYTHON").is_none()
*BUILD_CTX.ext.pyo3_no_python
}

/// Checks if `abi3` or any of the `abi3-py3*` features is enabled for the PyO3 crate.
///
/// Must be called from a PyO3 crate build script.
fn is_abi3() -> bool {
cargo_env_var("CARGO_FEATURE_ABI3").is_some()
|| env_var("PYO3_USE_ABI3_FORWARD_COMPATIBILITY").is_some_and(|os_str| os_str == "1")
}

/// Gets the minimum supported Python version from PyO3 `abi3-py*` features.
///
/// Must be called from a PyO3 crate build script.
pub fn get_abi3_version() -> Option<PythonVersion> {
let minor_version = (MINIMUM_SUPPORTED_VERSION.minor..=ABI3_MAX_MINOR)
.find(|i| cargo_env_var(&format!("CARGO_FEATURE_ABI3_PY3{i}")).is_some());
minor_version.map(|minor| PythonVersion { major: 3, minor })
*BUILD_CTX.cargo.cargo_feature_abi3 || *BUILD_CTX.ext.use_abi13_forward_compatibility
}

/// Checks if the `extension-module` feature is enabled for the PyO3 crate.
Expand All @@ -862,8 +975,7 @@ pub fn get_abi3_version() -> Option<PythonVersion> {
///
/// Must be called from a PyO3 crate build script.
pub fn is_extension_module() -> bool {
cargo_env_var("CARGO_FEATURE_EXTENSION_MODULE").is_some()
|| env_var("PYO3_BUILD_EXTENSION_MODULE").is_some()
*BUILD_CTX.cargo.cargo_feature_extension_module || *BUILD_CTX.ext.pyo3_build_extension_module
}

/// Checks if we need to link to `libpython` for the target.
Expand Down Expand Up @@ -1002,10 +1114,13 @@ impl CrossCompileEnvVars {
/// Registers the build script to rerun if any of the variables changes.
fn from_env() -> Self {
CrossCompileEnvVars {
pyo3_cross: env_var("PYO3_CROSS"),
pyo3_cross_lib_dir: env_var("PYO3_CROSS_LIB_DIR"),
pyo3_cross_python_version: env_var("PYO3_CROSS_PYTHON_VERSION"),
pyo3_cross_python_implementation: env_var("PYO3_CROSS_PYTHON_IMPLEMENTATION"),
pyo3_cross: BUILD_CTX.ext.pyo3_cross.clone(),
pyo3_cross_lib_dir: BUILD_CTX.ext.pyo3_cross_lib_dir.clone(),
pyo3_cross_python_version: BUILD_CTX.ext.pyo3_cross_python_version.clone(),
pyo3_cross_python_implementation: BUILD_CTX
.ext
.pyo3_cross_python_implementation
.clone(),
}
}

Expand Down Expand Up @@ -1412,13 +1527,13 @@ pub fn find_all_sysconfigdata(cross: &CrossCompileConfig) -> Result<Vec<PathBuf>
return Ok(Vec::new());
};

let sysconfig_name = env_var("_PYTHON_SYSCONFIGDATA_NAME");
let sysconfig_name = BUILD_CTX.ext.python_sysconfigdata_name.as_deref();
let mut sysconfig_paths = sysconfig_paths
.iter()
.filter_map(|p| {
let canonical = fs::canonicalize(p).ok();
match &sysconfig_name {
Some(_) => canonical.filter(|p| p.file_stem() == sysconfig_name.as_deref()),
Some(_) => canonical.filter(|p| p.file_stem() == sysconfig_name),
None => canonical,
}
})
Expand Down Expand Up @@ -1554,7 +1669,7 @@ fn cross_compile_from_sysconfigdata(
fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result<InterpreterConfig> {
let version = cross_compile_config
.version
.or_else(get_abi3_version)
.or_else(|| *BUILD_CTX.cargo.abi3_version)
.ok_or_else(||
format!(
"PYO3_CROSS_PYTHON_VERSION or an abi3-py3* feature must be specified \
Expand Down Expand Up @@ -1829,11 +1944,14 @@ fn conda_env_interpreter(conda_prefix: &OsStr, windows: bool) -> PathBuf {
}

fn get_env_interpreter() -> Option<PathBuf> {
match (env_var("VIRTUAL_ENV"), env_var("CONDA_PREFIX")) {
match (
BUILD_CTX.ext.virtual_env.as_ref(),
BUILD_CTX.ext.conda_prefix.as_ref(),
) {
// Use cfg rather than CARGO_CFG_TARGET_OS because this affects where files are located on the
// build host
(Some(dir), None) => Some(venv_interpreter(&dir, cfg!(windows))),
(None, Some(dir)) => Some(conda_env_interpreter(&dir, cfg!(windows))),
(Some(dir), None) => Some(venv_interpreter(dir, cfg!(windows))),
(None, Some(dir)) => Some(conda_env_interpreter(dir, cfg!(windows))),
(Some(_), Some(_)) => {
warn!(
"Both VIRTUAL_ENV and CONDA_PREFIX are set. PyO3 will ignore both of these for \
Expand All @@ -1857,7 +1975,7 @@ pub fn find_interpreter() -> Result<PathBuf> {
// See https://github.com/PyO3/pyo3/issues/2724
println!("cargo:rerun-if-env-changed=PYO3_ENVIRONMENT_SIGNATURE");

if let Some(exe) = env_var("PYO3_PYTHON") {
if let Some(exe) = BUILD_CTX.ext.pyo3_python.as_ref() {
Ok(exe.into())
} else if let Some(env_interpreter) = get_env_interpreter() {
Ok(env_interpreter)
Expand Down Expand Up @@ -1900,7 +2018,7 @@ fn get_host_interpreter(abi3_version: Option<PythonVersion>) -> Result<Interpret
pub fn make_cross_compile_config() -> Result<Option<InterpreterConfig>> {
let interpreter_config = if let Some(cross_config) = cross_compiling_from_cargo_env()? {
let mut interpreter_config = load_cross_compile_config(cross_config)?;
interpreter_config.fixup_for_abi3_version(get_abi3_version())?;
interpreter_config.fixup_for_abi3_version(*BUILD_CTX.cargo.abi3_version)?;
Some(interpreter_config)
} else {
None
Expand All @@ -1914,7 +2032,7 @@ pub fn make_cross_compile_config() -> Result<Option<InterpreterConfig>> {
#[allow(dead_code, unused_mut)]
pub fn make_interpreter_config() -> Result<InterpreterConfig> {
let host = Triple::host();
let abi3_version = get_abi3_version();
let abi3_version = *BUILD_CTX.cargo.abi3_version;

// See if we can safely skip the Python interpreter configuration detection.
// Unix "abi3" extension modules can usually be built without any interpreter.
Expand Down
7 changes: 4 additions & 3 deletions pyo3-build-config/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ use std::{env, process::Command, str::FromStr, sync::OnceLock};

pub use impl_::{
cross_compiling_from_to, find_all_sysconfigdata, parse_sysconfigdata, BuildFlag, BuildFlags,
CrossCompileConfig, InterpreterConfig, PythonImplementation, PythonVersion, Triple,
BuildScriptContext, CrossCompileConfig, InterpreterConfig, PythonImplementation, PythonVersion,
Triple, BUILD_CTX,
Comment on lines +23 to +24
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we should make BUILD_CTX not a public export but instead part of the pyo3_build_script_impl exports below (i.e. private implementation detail).

};

use target_lexicon::OperatingSystem;
Expand Down Expand Up @@ -314,8 +315,8 @@ pub mod pyo3_build_script_impl {
pub use crate::errors::*;
}
pub use crate::impl_::{
cargo_env_var, env_var, is_linking_libpython_for_target, make_cross_compile_config,
target_triple_from_env, InterpreterConfig, PythonVersion,
is_linking_libpython_for_target, make_cross_compile_config, target_triple_from_env,
InterpreterConfig, PythonVersion,
};
pub enum BuildConfigSource {
/// Config was provided by `PYO3_CONFIG_FILE`.
Expand Down
Loading
Loading