diff --git a/Cargo.lock b/Cargo.lock index 25f6feb8e..aa04372cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1677,6 +1677,7 @@ dependencies = [ "pep508_rs", "platform-info", "pretty_assertions", + "pyo3-introspection", "pyproject-toml", "python-pkginfo", "reflink-copy", @@ -2220,6 +2221,18 @@ dependencies = [ "unicase", ] +[[package]] +name = "pyo3-introspection" +version = "0.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc11f40a1f5ec62a36963d4b4b0c051fac90c879c640baa975f45cd01afd3c38" +dependencies = [ + "anyhow", + "goblin", + "serde", + "serde_json", +] + [[package]] name = "pyproject-toml" version = "0.13.7" diff --git a/Cargo.toml b/Cargo.toml index 86dad8900..0c12af032 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -74,6 +74,7 @@ semver = "1.0.22" target-lexicon = "0.13.3" indexmap = "2.2.3" pyproject-toml = { version = "0.13.5", features = ["pep639-glob"] } +pyo3-introspection = "0.28" python-pkginfo = "0.6.8" textwrap = "0.16.1" ignore = "0.4.20" diff --git a/src/binding_generator/mod.rs b/src/binding_generator/mod.rs index 4ac5ad444..1a18e587c 100644 --- a/src/binding_generator/mod.rs +++ b/src/binding_generator/mod.rs @@ -310,9 +310,15 @@ where .rust_module .join(format!("{ext_name}.pyi")); if type_stub.exists() { - eprintln!("📖 Found type stub file at {ext_name}.pyi"); - writer.add_file(module.join("__init__.pyi"), type_stub, false)?; - writer.add_empty_file(module.join("py.typed"))?; + if context.artifact.generate_stubs { + eprintln!( + "⚠️ Warning: Ignoring the type stub file at {ext_name}.pyi, stubs are automatically generated instead" + ); + } else { + eprintln!("📖 Found type stub file at {ext_name}.pyi"); + writer.add_file(module.join("__init__.pyi"), type_stub, false)?; + writer.add_empty_file(module.join("py.typed"))?; + } } } diff --git a/src/binding_generator/pyo3_binding.rs b/src/binding_generator/pyo3_binding.rs index d776153fd..cf5d60cd5 100644 --- a/src/binding_generator/pyo3_binding.rs +++ b/src/binding_generator/pyo3_binding.rs @@ -9,6 +9,7 @@ use anyhow::Context; use anyhow::Result; use anyhow::anyhow; use anyhow::bail; +use pyo3_introspection::{introspect_cdylib, module_stub_files}; use tempfile::TempDir; use tracing::debug; @@ -123,6 +124,28 @@ if hasattr({ext_name}, "__all__"): executable: false, }; files.insert(module.join("__init__.py"), ArchiveSource::Generated(source)); + if context.artifact.generate_stubs { + let module_introspection = introspect_cdylib(&artifact.path, ext_name).context("Failed to introspect the built libraries to generate type stubs, have you enabled the \"experimental-inspect\" PyO3 Cargo feature?")?; + eprintln!("📖 Type stub extracted from the built binary"); + for (path, stub_content) in module_stub_files(&module_introspection) { + files.insert( + module.join(path), + ArchiveSource::Generated(GeneratedSourceData { + data: stub_content.into_bytes(), + path: None, + executable: false, + }), + ); + } + files.insert( + module.join("py.typed"), + ArchiveSource::Generated(GeneratedSourceData { + data: Vec::new(), + path: None, + executable: false, + }), + ); + } Some(files) } }; diff --git a/src/build_context/builder.rs b/src/build_context/builder.rs index 80666c6a8..52472062c 100644 --- a/src/build_context/builder.rs +++ b/src/build_context/builder.rs @@ -9,7 +9,10 @@ use crate::python_interpreter::{InterpreterResolver, PythonInterpreter}; use crate::target::{ detect_arch_from_python, detect_target_from_cross_python, is_arch_supported_by_pypi, }; -use crate::{BridgeModel, BuildContext, PyProjectToml, Target}; +use crate::{ + ArtifactContext, BridgeModel, BuildContext, ProjectContext, PyProjectToml, PythonContext, + Target, +}; use anyhow::{Result, bail}; use std::collections::HashSet; use std::env; @@ -238,7 +241,7 @@ impl BuildContextBuilder { }; Ok(BuildContext { - project: crate::build_context::ProjectContext { + project: ProjectContext { target, project_layout, pyproject_toml_path, @@ -255,7 +258,7 @@ impl BuildContextBuilder { conditional_features, compile_targets, }, - artifact: crate::build_context::ArtifactContext { + artifact: ArtifactContext { out: wheel_dir, strip, compression: build_options.compression, @@ -264,8 +267,9 @@ impl BuildContextBuilder { include_debuginfo, pgo_phase: None, pgo_command, + generate_stubs: build_options.generate_stubs, }, - python: crate::build_context::PythonContext { + python: PythonContext { auditwheel, #[cfg(feature = "zig")] zig: build_options.platform.zig, diff --git a/src/build_context/mod.rs b/src/build_context/mod.rs index b696b8630..d4a967ba5 100644 --- a/src/build_context/mod.rs +++ b/src/build_context/mod.rs @@ -99,6 +99,8 @@ pub struct ArtifactContext { pub pgo_phase: Option, /// PGO training command from pyproject.toml (only set when --pgo is passed) pub pgo_command: Option, + /// Auto generate Python type stubs by introspecting the binary. Requires PyO3 and its "experimental-inspect" feature + pub generate_stubs: bool, } /// The constraint part of the build context. @@ -125,7 +127,7 @@ pub struct PythonContext { /// /// This structure reflects the build lifecycle: /// **Input (Project) → Constraints (Python) → Output (Artifact).** -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct BuildContext { /// Project context pub project: ProjectContext, diff --git a/src/build_options.rs b/src/build_options.rs index fc61ef07d..29310c125 100644 --- a/src/build_options.rs +++ b/src/build_options.rs @@ -124,6 +124,10 @@ pub struct BuildOptions { /// Wheel compression options #[command(flatten)] pub compression: CompressionOptions, + + /// Auto generate Python type stubs by introspecting the binary. Requires PyO3 and its "experimental-inspect" feature + #[arg(long)] + pub generate_stubs: bool, } impl Deref for BuildOptions { diff --git a/src/develop/mod.rs b/src/develop/mod.rs index 43f2f765e..bf812722b 100644 --- a/src/develop/mod.rs +++ b/src/develop/mod.rs @@ -81,6 +81,10 @@ pub struct DevelopOptions { /// Wheel compression options #[command(flatten)] pub compression: CompressionOptions, + + /// Auto generate Python type stubs by introspecting the binary. Requires PyO3 and its "experimental-inspect" feature + #[arg(long)] + pub generate_stubs: bool, } #[instrument(skip_all)] @@ -283,6 +287,7 @@ pub fn develop(develop_options: DevelopOptions, venv_dir: &Path) -> Result<()> { mut cargo_options, uv, compression, + generate_stubs, } = develop_options; compression.validate(); @@ -328,6 +333,7 @@ pub fn develop(develop_options: DevelopOptions, venv_dir: &Path) -> Result<()> { ..cargo_options }, compression, + generate_stubs, }; let build_context = build_options diff --git a/test-crates/pyo3-stub-generation/Cargo.lock b/test-crates/pyo3-stub-generation/Cargo.lock new file mode 100644 index 000000000..5d4726db4 --- /dev/null +++ b/test-crates/pyo3-stub-generation/Cargo.lock @@ -0,0 +1,133 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "libc" +version = "0.2.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf85e27e86080aafd5a22eae58a162e133a589551542b3e5cee4beb27e54f8e1" +dependencies = [ + "libc", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", +] + +[[package]] +name = "pyo3-build-config" +version = "0.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf94ee265674bf76c09fa430b0e99c26e319c945d96ca0d5a8215f31bf81cf7" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "491aa5fc66d8059dd44a75f4580a2962c1862a1c2945359db36f6c2818b748dc" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5d671734e9d7a43449f8480f8b38115df67bef8d21f76837fa75ee7aaa5e52e" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22faaa1ce6c430a1f71658760497291065e6450d7b5dc2bcf254d49f66ee700a" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "pyo3-stub-generation" +version = "1.0.0" +dependencies = [ + "pyo3", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" diff --git a/test-crates/pyo3-stub-generation/Cargo.toml b/test-crates/pyo3-stub-generation/Cargo.toml new file mode 100644 index 000000000..973ced1eb --- /dev/null +++ b/test-crates/pyo3-stub-generation/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "pyo3-stub-generation" +version = "1.0.0" +edition = "2021" + +[dependencies] +pyo3 = { version = "0.28", features = ["experimental-inspect"] } + +[lib] +name = "pyo3_stub_generation" +crate-type = ["cdylib"] diff --git a/test-crates/pyo3-stub-generation/LICENSE b/test-crates/pyo3-stub-generation/LICENSE new file mode 100644 index 000000000..fa566452c --- /dev/null +++ b/test-crates/pyo3-stub-generation/LICENSE @@ -0,0 +1,25 @@ +Copyright (c) 2018-present konstin + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/test-crates/pyo3-stub-generation/check_installed/check_installed.py b/test-crates/pyo3-stub-generation/check_installed/check_installed.py new file mode 100755 index 000000000..6acfe7c06 --- /dev/null +++ b/test-crates/pyo3-stub-generation/check_installed/check_installed.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +import os + +import pyo3_stub_generation + +assert pyo3_stub_generation.func(42) == 42 + +# Check type stub +install_path = os.path.join(os.path.dirname(pyo3_stub_generation.__file__)) +assert os.path.exists(os.path.join(install_path, "__init__.pyi")) +assert os.path.exists(os.path.join(install_path, "submodule.pyi")) +assert os.path.exists(os.path.join(install_path, "py.typed")) + +print("SUCCESS") diff --git a/test-crates/pyo3-stub-generation/pyproject.toml b/test-crates/pyo3-stub-generation/pyproject.toml new file mode 100644 index 000000000..f5b8bdfa0 --- /dev/null +++ b/test-crates/pyo3-stub-generation/pyproject.toml @@ -0,0 +1,8 @@ +[build-system] +requires = ["maturin>=1.0,<2.0"] +build-backend = "maturin" + +[project] +name = "pyo3-stub-generation" +description = "Tests compilation of packages loading stubs generated by PyO3" +license = { file = "LICENSE" } diff --git a/test-crates/pyo3-stub-generation/src/lib.rs b/test-crates/pyo3-stub-generation/src/lib.rs new file mode 100644 index 000000000..b333a1c16 --- /dev/null +++ b/test-crates/pyo3-stub-generation/src/lib.rs @@ -0,0 +1,22 @@ +use pyo3::prelude::*; + +#[pymodule] +mod pyo3_stub_generation { + use super::*; + + #[pyclass] + struct Class {} + + #[pyfunction] + fn func(a: usize) -> usize { + a + } + + #[pymodule] + mod submodule { + use super::*; + + #[pyclass] + struct Class2 {} + } +} diff --git a/tests/cmd/build.stdout b/tests/cmd/build.stdout index c5008b49f..641ab4055 100644 --- a/tests/cmd/build.stdout +++ b/tests/cmd/build.stdout @@ -119,6 +119,10 @@ Options: --compression-level Zip compression level. Defaults to method default + --generate-stubs + Auto generate Python type stubs by introspecting the binary. Requires PyO3 and its + "experimental-inspect" feature + -h, --help Print help (see a summary with '-h') diff --git a/tests/cmd/develop.stdout b/tests/cmd/develop.stdout index bbf80738e..acb81246a 100644 --- a/tests/cmd/develop.stdout +++ b/tests/cmd/develop.stdout @@ -75,6 +75,10 @@ Options: --compression-level Zip compression level. Defaults to method default + --generate-stubs + Auto generate Python type stubs by introspecting the binary. Requires PyO3 and its + "experimental-inspect" feature + -h, --help Print help (see a summary with '-h') diff --git a/tests/cmd/publish.stdout b/tests/cmd/publish.stdout index a81d617d8..2c4b4f843 100644 --- a/tests/cmd/publish.stdout +++ b/tests/cmd/publish.stdout @@ -157,6 +157,10 @@ Options: --compression-level Zip compression level. Defaults to method default + --generate-stubs + Auto generate Python type stubs by introspecting the binary. Requires PyO3 and its + "experimental-inspect" feature + -h, --help Print help (see a summary with '-h') diff --git a/tests/common/develop.rs b/tests/common/develop.rs index 11009e3ab..9c1a55e99 100644 --- a/tests/common/develop.rs +++ b/tests/common/develop.rs @@ -109,6 +109,7 @@ pub fn test_develop(case: &DevelopCase<'_>) -> Result<()> { }, uv, compression: Default::default(), + generate_stubs: false, }; develop(develop_options, &venv_dir)?; diff --git a/tests/common/integration.rs b/tests/common/integration.rs index 2d18af4fd..07d5c5967 100644 --- a/tests/common/integration.rs +++ b/tests/common/integration.rs @@ -35,6 +35,8 @@ pub struct IntegrationCase<'a> { pub zig: bool, /// Optional explicit compilation target for the build. pub target: Option<&'a str>, + /// Do introspection to generate type stubs + pub generate_stubs: bool, } impl<'a> IntegrationCase<'a> { @@ -46,6 +48,7 @@ impl<'a> IntegrationCase<'a> { bindings: None, zig: false, target: None, + generate_stubs: false, } } @@ -63,6 +66,11 @@ impl<'a> IntegrationCase<'a> { self.target = Some(target); self } + + pub fn generate_stubs(mut self) -> Self { + self.generate_stubs = true; + self + } } // The zig wrappers consult CARGO_BIN_EXE_cargo-zigbuild via process env, so zig builds run in a @@ -258,6 +266,10 @@ pub fn test_integration(case: &IntegrationCase<'_>) -> Result<()> { Some(python) }; + if case.generate_stubs { + cli.push("--generate-stubs".into()); + } + if test_zig { // Zig wrappers read CARGO_BIN_EXE_cargo-zigbuild from the process environment. Run the // build in a subprocess so the override is scoped to that command instead of mutating diff --git a/tests/run/integration.rs b/tests/run/integration.rs index 129bcf971..26d8a9782 100644 --- a/tests/run/integration.rs +++ b/tests/run/integration.rs @@ -71,6 +71,14 @@ fn integration_pyo3_bin() { "integration-workspace-inverted-order", "test-crates/workspace-inverted-order/path-dep-with-root", ))] +#[case::pyo3_stub_generation(IntegrationCase::new( + "integration-pyo3-stub-generation", + "test-crates/pyo3-stub-generation", +).generate_stubs())] +#[cfg_attr(unix, case::pyo3_stub_generation_zig(IntegrationCase::new( + "integration-pyo3-stub-generation-zig", + "test-crates/pyo3-stub-generation", +).generate_stubs().zig()))] #[test] fn integration_cases(#[case] case: IntegrationCase<'_>) { handle_result(integration::test_integration(&case));