diff --git a/guide/src/type-stub.md b/guide/src/type-stub.md index 0ac11a54e5f..3069c88434d 100644 --- a/guide/src/type-stub.md +++ b/guide/src/type-stub.md @@ -11,7 +11,7 @@ It works using: 1. PyO3 macros (`#[pyclass]`) that generate constant JSON strings that are then included in the built binaries by rustc if the `experimental-inspect` feature is enabled. 2. The `pyo3-introspection` crate that can parse the generated binaries, extract the JSON strings and build stub files from it. -3. \[Not done yet\] Build tools like `maturin` exposing `pyo3-introspection` features in their CLI API. +3. Build tools like `maturin` exposing options in their CLI API to generate the stubs file. For example, the following Rust code @@ -72,6 +72,11 @@ def list_of_int_identity(arg: "list[int]") -> "list[int]": ... The only piece of new syntax is that the `#[pyo3(signature = ...)]` attribute can contain type annotations like `#[pyo3(signature = (arg: "list[int]") -> "list[int]")]` (note the `""` around type annotations). This is useful when PyO3 is not able to derive proper type annotations by itself. +To generate stubs file with `maturin` you can use `maturin generate-stubs --output stubs` that will build the project then generate the stubs in the `stubs` directory. +You can also directly integrate the stubs in the built wheels by doing `maturin build --generate-stubs` (works also with `maturin develop`). + +PyO3 also provides the smaller `pyo3-introspection` binary that allows to generate stubs from an existing built extension using something like `pyo3-introspection my_module_binary.so my_module_name output` to introspect the `my_module_binary.so` dynamic library for the `my_module_name` Python module and generate its stubs in the `output` directory. + ## Constraints and limitations - The `experimental-inspect` feature is required to generate the introspection fragments. diff --git a/newsfragments/5904.added.md b/newsfragments/5904.added.md new file mode 100644 index 00000000000..cd5eeb1f123 --- /dev/null +++ b/newsfragments/5904.added.md @@ -0,0 +1 @@ +`pyo3-introspection`: add a small CLI to generate stubs \ No newline at end of file diff --git a/noxfile.py b/noxfile.py index b7130b30d4c..dcea53ff427 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1427,35 +1427,62 @@ def update_ui_tests(session: nox.Session): @nox.session(name="test-introspection") def test_introspection(session: nox.Session): - session.install("maturin") - session.install("ruff") - options = [] - target = os.environ.get("CARGO_BUILD_TARGET") - if target is not None: - options += ("--target", target) - profile = os.environ.get("CARGO_BUILD_PROFILE") - if profile == "release": - options.append("--release") - session.run_always( - "maturin", - "develop", - "-m", - "./pytests/Cargo.toml", - "--features", - "experimental-async,experimental-inspect", - *options, - ) - lib_file = session.run( - "python", - "-c", - "import pyo3_pytests; print(pyo3_pytests.pyo3_pytests.__file__)", - silent=True, - ).strip() - _run_cargo_test( - session, - package="pyo3-introspection", - env={"PYO3_PYTEST_LIB_PATH": lib_file}, - ) + with tempfile.TemporaryDirectory() as stub_dir: + session.install("maturin") + session.install("ruff") + options = [] + target = os.environ.get("CARGO_BUILD_TARGET") + if target is not None: + options += ("--target", target) + profile = os.environ.get("CARGO_BUILD_PROFILE") + if profile == "release": + options.append("--release") + _run( + session, + "maturin", + "develop", + "-m", + "./pytests/Cargo.toml", + "--features", + "experimental-async,experimental-inspect", + *options, + ) + lib_file = session.run( + "python", + "-c", + "import pyo3_pytests; print(pyo3_pytests.pyo3_pytests.__file__)", + silent=True, + ).strip() + _run_cargo( + session, + "run", + "-p", + "pyo3-introspection", + "--", + lib_file, + "pyo3_pytests", + stub_dir, + ) + _run(session, "ruff", "format", stub_dir) + _ensure_directory_equals(Path(stub_dir), Path("pytests/stubs")) + + +def _ensure_directory_equals(expected_dir: Path, actual_dir: Path): + # Assert all expected files are in actual and are equals + for expected_file_path in expected_dir.rglob("*"): + file_path = expected_file_path.relative_to(expected_dir) + actual_file_path = actual_dir / file_path + assert actual_file_path.exists(), f"File {file_path} does not exist" + assert expected_file_path.read_text() == actual_file_path.read_text(), ( + f"Content is different in {file_path}" + ) + # Assert all actual files are expected + for actual_file_path in actual_dir.rglob("*"): + file_path = actual_file_path.relative_to(actual_dir) + expected_file_path = expected_dir / file_path + assert expected_file_path.exists(), ( + f"File {file_path} exist even if not expected" + ) @lru_cache() diff --git a/pyo3-introspection/Cargo.toml b/pyo3-introspection/Cargo.toml index 0958555c20b..c023c69248a 100644 --- a/pyo3-introspection/Cargo.toml +++ b/pyo3-introspection/Cargo.toml @@ -14,8 +14,5 @@ goblin = ">=0.9, <0.11" serde = { version = "1", features = ["derive"] } serde_json = "1" -[dev-dependencies] -tempfile = "3.12.0" - [lints] workspace = true diff --git a/pyo3-introspection/src/main.rs b/pyo3-introspection/src/main.rs new file mode 100644 index 00000000000..f0c64031da2 --- /dev/null +++ b/pyo3-introspection/src/main.rs @@ -0,0 +1,24 @@ +//! Small CLI entry point to introspect a Python cdylib built using PyO3 and generate [type stubs](https://typing.readthedocs.io/en/latest/source/stubs.html). + +use anyhow::{anyhow, Context, Result}; +use pyo3_introspection::{introspect_cdylib, module_stub_files}; +use std::path::Path; +use std::{env, fs}; + +fn main() -> Result<()> { + let [_, binary_path, module_name, output_path] = env::args().collect::>().try_into().map_err(|_| anyhow!("pyo3-introspection takes three arguments, the path of the binary to introspect, the name of the python module to introspect and and the path of the directory to write the stub to"))?; + let module = introspect_cdylib(&binary_path, &module_name) + .with_context(|| format!("Failed to introspect module {binary_path}"))?; + let actual_stubs = module_stub_files(&module); + for (path, module) in actual_stubs { + let file_path = Path::new(&output_path).join(path); + if let Some(parent) = file_path.parent() { + fs::create_dir_all(parent).with_context(|| { + format!("Failed to create output directory {}", file_path.display()) + })?; + } + fs::write(&file_path, module) + .with_context(|| format!("Failed to write module {}", file_path.display()))?; + } + Ok(()) +} diff --git a/pyo3-introspection/tests/test.rs b/pyo3-introspection/tests/test.rs deleted file mode 100644 index cf01329e9b1..00000000000 --- a/pyo3-introspection/tests/test.rs +++ /dev/null @@ -1,106 +0,0 @@ -use anyhow::{ensure, Result}; -use pyo3_introspection::{introspect_cdylib, module_stub_files}; -use std::collections::HashMap; -use std::io::{Read, Seek, SeekFrom, Write}; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::{env, fs}; -use tempfile::NamedTempFile; - -#[test] -fn pytests_stubs() -> Result<()> { - // We run the introspection - let binary = env::var_os("PYO3_PYTEST_LIB_PATH") - .expect("The PYO3_PYTEST_LIB_PATH constant must be set and target the pyo3-pytests cdylib"); - let module = introspect_cdylib(binary, "pyo3_pytests")?; - let actual_stubs = module_stub_files(&module); - - // We read the expected stubs - let expected_subs_dir = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("pytests") - .join("stubs"); - let mut expected_subs = HashMap::new(); - add_dir_files( - &expected_subs_dir, - &expected_subs_dir.canonicalize()?, - &mut expected_subs, - )?; - - // We ensure we do not have extra generated files - for file_name in actual_stubs.keys() { - assert!( - expected_subs.contains_key(file_name), - "The generated file {} is not in the expected stubs directory pytests/stubs", - file_name.display() - ); - } - - // We ensure the expected files are generated properly - for (file_name, expected_file_content) in &expected_subs { - let actual_file_content = actual_stubs.get(file_name).unwrap_or_else(|| { - panic!( - "The expected stub file {} has not been generated", - file_name.display() - ) - }); - - let actual_file_content = format_with_ruff(actual_file_content)?; - - // We normalize line jumps for compatibility with Windows - assert_eq!( - expected_file_content.replace('\r', ""), - actual_file_content.replace('\r', ""), - "The content of file {} is different", - file_name.display() - ) - } - - Ok(()) -} - -fn add_dir_files( - dir_path: &Path, - base_dir_path: &Path, - output: &mut HashMap, -) -> Result<()> { - for entry in fs::read_dir(dir_path)? { - let entry = entry?; - if entry.file_type()?.is_dir() { - add_dir_files(&entry.path(), base_dir_path, output)?; - } else { - output.insert( - entry - .path() - .canonicalize()? - .strip_prefix(base_dir_path)? - .into(), - fs::read_to_string(entry.path())?, - ); - } - } - Ok(()) -} - -fn format_with_ruff(code: &str) -> Result { - let temp_file = NamedTempFile::with_suffix(".pyi")?; - // Write to file - { - let mut file = temp_file.as_file(); - file.write_all(code.as_bytes())?; - file.flush()?; - file.seek(SeekFrom::Start(0))?; - } - ensure!( - Command::new("ruff") - .arg("format") - .arg(temp_file.path()) - .status()? - .success(), - "Failed to run ruff" - ); - let mut content = String::new(); - temp_file.as_file().read_to_string(&mut content)?; - Ok(content) -}