From 5667a095d61189c6bf95cd75965a077dc62c5382 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Fri, 27 Jan 2023 06:34:12 +0000 Subject: [PATCH 1/5] hygiene: fix `#[pymethods(crate = "...")]` --- newsfragments/2923.fixed.md | 1 + pyo3-macros/src/lib.rs | 15 ++++++++++++--- src/test_hygiene/pymethods.rs | 8 ++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 newsfragments/2923.fixed.md diff --git a/newsfragments/2923.fixed.md b/newsfragments/2923.fixed.md new file mode 100644 index 00000000000..d61f5b01e65 --- /dev/null +++ b/newsfragments/2923.fixed.md @@ -0,0 +1 @@ +Fix `#[pymethods(crate = "...")]` option being ignored. diff --git a/pyo3-macros/src/lib.rs b/pyo3-macros/src/lib.rs index 730aaf65050..387934310b9 100644 --- a/pyo3-macros/src/lib.rs +++ b/pyo3-macros/src/lib.rs @@ -109,13 +109,13 @@ pub fn pyclass(attr: TokenStream, input: TokenStream) -> TokenStream { /// [10]: https://pyo3.rs/latest/class.html#method-arguments /// [11]: https://pyo3.rs/latest/class.html#object-properties-using-pyo3get-set #[proc_macro_attribute] -pub fn pymethods(_: TokenStream, input: TokenStream) -> TokenStream { +pub fn pymethods(attr: TokenStream, input: TokenStream) -> TokenStream { let methods_type = if cfg!(feature = "multiple-pymethods") { PyClassMethodsType::Inventory } else { PyClassMethodsType::Specialization }; - pymethods_impl(input, methods_type) + pymethods_impl(attr, input, methods_type) } /// A proc macro used to expose Rust functions to Python. @@ -191,8 +191,17 @@ fn pyclass_enum_impl( .into() } -fn pymethods_impl(input: TokenStream, methods_type: PyClassMethodsType) -> TokenStream { +fn pymethods_impl( + attr: TokenStream, + input: TokenStream, + methods_type: PyClassMethodsType, +) -> TokenStream { let mut ast = parse_macro_input!(input as syn::ItemImpl); + // Apply all options as a #[pyo3] attribute on the ItemImpl + // e.g. #[pymethods(crate = "crate")] impl Foo { } + // -> #[pyo3(crate = "crate")] impl Foo { } + let attr: TokenStream2 = attr.into(); + ast.attrs.push(syn::parse_quote!( #[pyo3(#attr)] )); let expanded = build_py_methods(&mut ast, methods_type).unwrap_or_compile_error(); quote!( diff --git a/src/test_hygiene/pymethods.rs b/src/test_hygiene/pymethods.rs index eb0b12f73b8..ba0fcdff2b8 100644 --- a/src/test_hygiene/pymethods.rs +++ b/src/test_hygiene/pymethods.rs @@ -807,3 +807,11 @@ impl Dummy { // PyGcProtocol // Buffer protocol? } + +// Ensure that crate argument is also accepted inline + +#[crate::pyclass(crate = "crate")] +struct Dummy2; + +#[crate::pymethods(crate = "crate")] +impl Dummy2 {} From 1a4153f71831f8b7955efa93005ae2c960503e46 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Fri, 27 Jan 2023 08:20:03 +0000 Subject: [PATCH 2/5] remove unneeded into_iter calls --- benches/bench_set.rs | 2 +- pyo3-macros-backend/src/frompyobject.rs | 1 - pyo3-macros-backend/src/pymethod.rs | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/benches/bench_set.rs b/benches/bench_set.rs index d59e3e57f4c..58abc956337 100644 --- a/benches/bench_set.rs +++ b/benches/bench_set.rs @@ -9,7 +9,7 @@ fn set_new(b: &mut Bencher<'_>) { const LEN: usize = 100_000; // Create Python objects up-front, so that the benchmark doesn't need to include // the cost of allocating LEN Python integers - let elements: Vec = (0..LEN).into_iter().map(|i| i.into_py(py)).collect(); + let elements: Vec = (0..LEN).map(|i| i.into_py(py)).collect(); b.iter(|| { let pool = unsafe { py.new_pool() }; PySet::new(py, &elements).unwrap(); diff --git a/pyo3-macros-backend/src/frompyobject.rs b/pyo3-macros-backend/src/frompyobject.rs index ba365efe265..831aed253ff 100644 --- a/pyo3-macros-backend/src/frompyobject.rs +++ b/pyo3-macros-backend/src/frompyobject.rs @@ -278,7 +278,6 @@ impl<'a> Container<'a> { let self_ty = &self.path; let struct_name = &self.name(); let field_idents: Vec<_> = (0..struct_fields.len()) - .into_iter() .map(|i| format_ident!("arg{}", i)) .collect(); let fields = struct_fields.iter().zip(&field_idents).enumerate().map(|(index, (field, ident))| { diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index fa81c6d495d..3807a1fd8be 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -1109,7 +1109,6 @@ impl SlotDef { let py = syn::Ident::new("_py", Span::call_site()); let arg_types: &Vec<_> = &arguments.iter().map(|arg| arg.ffi_type()).collect(); let arg_idents: &Vec<_> = &(0..arguments.len()) - .into_iter() .map(|i| format_ident!("arg{}", i)) .collect(); let wrapper_ident = format_ident!("__pymethod_{}__", method_name); @@ -1220,7 +1219,6 @@ impl SlotFragmentDef { let py = syn::Ident::new("_py", Span::call_site()); let arg_types: &Vec<_> = &arguments.iter().map(|arg| arg.ffi_type()).collect(); let arg_idents: &Vec<_> = &(0..arguments.len()) - .into_iter() .map(|i| format_ident!("arg{}", i)) .collect(); let body = generate_method_body(cls, spec, &py, arguments, *extract_error_mode, None)?; From ee0794dbef37c11478cc63c6c7a0301af747c0ff Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Wed, 18 Jan 2023 19:49:39 +0100 Subject: [PATCH 3/5] Added support for PyErr_WriteUnraisable --- newsfragments/2889.added.md | 1 + src/err/mod.rs | 34 +++++++++++++++++++++++++ tests/test_exceptions.rs | 51 +++++++++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+) create mode 100644 newsfragments/2889.added.md diff --git a/newsfragments/2889.added.md b/newsfragments/2889.added.md new file mode 100644 index 00000000000..0d4c9265b1d --- /dev/null +++ b/newsfragments/2889.added.md @@ -0,0 +1 @@ +Added `PyErr::write_unraisable()` to report an unraisable exception to Python. diff --git a/src/err/mod.rs b/src/err/mod.rs index 0f397153286..88b03986159 100644 --- a/src/err/mod.rs +++ b/src/err/mod.rs @@ -476,6 +476,40 @@ impl PyErr { unsafe { ffi::PyErr_Restore(ptype, pvalue, ptraceback) } } + /// Reports the error as unraisable. + /// + /// This calls `sys.unraisablehook()` using the current exception and obj argument. + /// + /// This method is useful to report errors in situations where there is no good mechanism + /// to report back to the Python land. In Python this is used to indicate errors in + /// background threads or destructors which are protected. In Rust code this is commonly + /// useful when you are calling into a Python callback which might fail, but there is no + /// obvious way to handle this error other than logging it. + /// + /// Calling this method has the benefit that the error goes back into a standardized callback + /// in Python which for instance allows unittests to ensure that no unraisable error + /// actually happend by hooking `sys.unraisablehook`. + /// + /// Example: + /// ```rust + /// # use pyo3::prelude::*; + /// # use pyo3::exceptions::PyRuntimeError; + /// # fn failing_function() -> PyResult<()> { Err(PyRuntimeError::new_err("foo")) } + /// # fn main() -> PyResult<()> { + /// Python::with_gil(|py| { + /// match failing_function() { + /// Err(pyerr) => pyerr.write_unraisable(py, None), + /// Ok(..) => { /* do something here */ } + /// } + /// Ok(()) + /// }) + /// # } + #[inline] + pub fn write_unraisable(self, py: Python<'_>, obj: Option<&PyAny>) { + self.restore(py); + unsafe { ffi::PyErr_WriteUnraisable(obj.map_or(std::ptr::null_mut(), |x| x.as_ptr())) } + } + /// Issues a warning message. /// /// May return an `Err(PyErr)` if warnings-as-errors is enabled. diff --git a/tests/test_exceptions.rs b/tests/test_exceptions.rs index 98dab27bc70..d429c93e34c 100644 --- a/tests/test_exceptions.rs +++ b/tests/test_exceptions.rs @@ -1,6 +1,8 @@ #![cfg(feature = "macros")] use pyo3::prelude::*; +#[cfg(Py_3_8)] +use pyo3::types::PyDict; use pyo3::{exceptions, py_run, PyErr, PyResult}; use std::error::Error; use std::fmt; @@ -96,3 +98,52 @@ fn test_exception_nosegfault() { assert!(io_err().is_err()); assert!(parse_int().is_err()); } + +#[test] +#[cfg(Py_3_8)] +fn test_write_unraisable() { + #[pyfunction] + fn report_unraisable(py: Python<'_>) { + use pyo3::exceptions::PyRuntimeError; + let err = PyRuntimeError::new_err("foo"); + err.write_unraisable(py, None); + + let err = PyRuntimeError::new_err("bar"); + err.write_unraisable(py, Some(py.NotImplemented().as_ref(py))); + } + + Python::with_gil(|py| { + let report_unraisable = wrap_pyfunction!(report_unraisable, py).unwrap(); + let locals = PyDict::new(py); + locals + .set_item("report_unraisable", report_unraisable) + .unwrap(); + + let source = r#"if True: + import sys + + captured = [] + def report(data): + captured.append(list(data)) + + original_hook = sys.unraisablehook + try: + sys.unraisablehook = report + report_unraisable() + + assert len(captured) == 2 + + assert captured[0][0] is RuntimeError + assert str(captured[0][1]) == 'foo' + assert captured[0][4] is None + + assert captured[1][0] is RuntimeError + assert str(captured[1][1]) == 'bar' + assert captured[1][4] is NotImplemented + finally: + sys.unraisablehook = original_hook + "#; + + py.run(source, Some(locals), None).unwrap(); + }); +} From 2228f584a1ab4add68b111775e17b643a2361018 Mon Sep 17 00:00:00 2001 From: Alex Pyattaev Date: Tue, 10 Jan 2023 23:42:07 +0200 Subject: [PATCH 4/5] added a plugin example that shows how to integrate a Python plugin into a Rust app while having option to test API without the main app --- examples/README.md | 1 + examples/plugin/Cargo.toml | 12 +++++ examples/plugin/README.md | 48 +++++++++++++++++++ examples/plugin/plugin_api/Cargo.toml | 17 +++++++ examples/plugin/plugin_api/noxfile.py | 9 ++++ examples/plugin/plugin_api/pyproject.toml | 14 ++++++ .../plugin/plugin_api/requirements-dev.txt | 3 ++ examples/plugin/plugin_api/src/lib.rs | 32 +++++++++++++ .../plugin/plugin_api/tests/test_Gadget.py | 22 +++++++++ .../plugin/plugin_api/tests/test_import.py | 2 + .../python_plugin/gadget_init_plugin.py | 12 +++++ examples/plugin/python_plugin/rng.py | 3 ++ examples/plugin/src/main.rs | 44 +++++++++++++++++ 13 files changed, 219 insertions(+) create mode 100644 examples/plugin/Cargo.toml create mode 100644 examples/plugin/README.md create mode 100644 examples/plugin/plugin_api/Cargo.toml create mode 100644 examples/plugin/plugin_api/noxfile.py create mode 100644 examples/plugin/plugin_api/pyproject.toml create mode 100644 examples/plugin/plugin_api/requirements-dev.txt create mode 100644 examples/plugin/plugin_api/src/lib.rs create mode 100644 examples/plugin/plugin_api/tests/test_Gadget.py create mode 100644 examples/plugin/plugin_api/tests/test_import.py create mode 100644 examples/plugin/python_plugin/gadget_init_plugin.py create mode 100644 examples/plugin/python_plugin/rng.py create mode 100644 examples/plugin/src/main.rs diff --git a/examples/README.md b/examples/README.md index 7b1bd4d49c7..47ab5a9dc3a 100644 --- a/examples/README.md +++ b/examples/README.md @@ -10,6 +10,7 @@ Below is a brief description of each of these: | `maturin-starter` | A template project which is configured to use [`maturin`](https://github.com/PyO3/maturin) for development. | | `setuptools-rust-starter` | A template project which is configured to use [`setuptools_rust`](https://github.com/PyO3/setuptools-rust/) for development. | | `word-count` | A quick performance comparison between word counter implementations written in each of Rust and Python. | +| `plugin` | Illustrates how to use Python as a scripting language within a Rust application | ## Creating new projects from these examples diff --git a/examples/plugin/Cargo.toml b/examples/plugin/Cargo.toml new file mode 100644 index 00000000000..08127b5003f --- /dev/null +++ b/examples/plugin/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "plugin_example" +version = "0.1.0" +edition = "2021" + + +[dependencies] +pyo3={path="../../", features=["macros"]} +plugin_api={path="plugin_api"} + + +[workspace] diff --git a/examples/plugin/README.md b/examples/plugin/README.md new file mode 100644 index 00000000000..17f2cf740bd --- /dev/null +++ b/examples/plugin/README.md @@ -0,0 +1,48 @@ +# plugin + +An example of a Rust app that uses Python for a plugin. A Python extension module built using PyO3 and [`maturin`](https://github.com/PyO3/maturin) is used to provide +interface types that can be used to exchange data between Rust and Python. This also deals with how to separately test and load python modules. + +# Building and Testing +## Host application +To run the app itself, you only need to run + +```shell +cargo run +``` +It will build the app, as well as the plugin API, then run the app, load the plugin and show it working. + +## Plugin API testing + +The plugin API is in a separate crate `plugin_api`, so you can test it separately from the main app. + +To build the API only package, first install `maturin`: + +```shell +pip install maturin +``` + +When building the plugin, simply using `maturin develop` will fail to produce a viable extension module due to the features arrangement of PyO3. +Instead, one needs to enable the optional feature as follows: + +```shell +cd plugin_api +maturin build --features "extension-module" +``` + +Alternatively, install nox and run the tests inside an isolated environment: + +```shell +nox +``` + +## Copying this example + +Use [`cargo-generate`](https://crates.io/crates/cargo-generate): + +```bash +$ cargo install cargo-generate +$ cargo generate --git https://github.com/PyO3/pyo3 examples/plugin +``` + +(`cargo generate` will take a little while to clone the PyO3 repo first; be patient when waiting for the command to run.) diff --git a/examples/plugin/plugin_api/Cargo.toml b/examples/plugin/plugin_api/Cargo.toml new file mode 100644 index 00000000000..870ad76a377 --- /dev/null +++ b/examples/plugin/plugin_api/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "plugin_api" +version = "0.1.0" +description = "Plugin API example" +edition = "2021" + +[lib] +name = "plugin_api" +crate-type = ["cdylib", "rlib"] + +[dependencies] +#!!! Important - DO NOT ENABLE extension-module FEATURE HERE!!! +pyo3 = { path = "../../../" } + +[features] +# instead extension-module feature for pyo3 is enabled conditionally when we want to build a standalone extension module to test our plugins without "main" program +extension-module = ["pyo3/extension-module"] diff --git a/examples/plugin/plugin_api/noxfile.py b/examples/plugin/plugin_api/noxfile.py new file mode 100644 index 00000000000..3b53c0c3e36 --- /dev/null +++ b/examples/plugin/plugin_api/noxfile.py @@ -0,0 +1,9 @@ +import nox + + +@nox.session +def python(session): + session.install("-rrequirements-dev.txt") + session.install("maturin") + session.run_always("maturin", "develop", "--features", "extension-module") + session.run("pytest") diff --git a/examples/plugin/plugin_api/pyproject.toml b/examples/plugin/plugin_api/pyproject.toml new file mode 100644 index 00000000000..114687eddef --- /dev/null +++ b/examples/plugin/plugin_api/pyproject.toml @@ -0,0 +1,14 @@ +[build-system] +requires = ["maturin>=0.14,<0.15"] +build-backend = "maturin" + +[project] +name = "plugin_api" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] + + diff --git a/examples/plugin/plugin_api/requirements-dev.txt b/examples/plugin/plugin_api/requirements-dev.txt new file mode 100644 index 00000000000..20c7cdfbb1c --- /dev/null +++ b/examples/plugin/plugin_api/requirements-dev.txt @@ -0,0 +1,3 @@ +pytest>=3.5.0 +pip>=21.3 +maturin>=0.14 diff --git a/examples/plugin/plugin_api/src/lib.rs b/examples/plugin/plugin_api/src/lib.rs new file mode 100644 index 00000000000..59aae55699d --- /dev/null +++ b/examples/plugin/plugin_api/src/lib.rs @@ -0,0 +1,32 @@ +use pyo3::prelude::*; + +///this is our Gadget that python plugin code can create, and rust app can then access natively. +#[pyclass] +pub struct Gadget { + #[pyo3(get, set)] + pub prop: usize, + //this field will only be accessible to rust code + pub rustonly: Vec, +} + +#[pymethods] +impl Gadget { + #[new] + fn new() -> Self { + Gadget { + prop: 777, + rustonly: Vec::new(), + } + } + + fn push(&mut self, v: usize) { + self.rustonly.push(v); + } +} + +/// A Python module for plugin interface types +#[pymodule] +pub fn plugin_api(_py: Python<'_>, m: &PyModule) -> PyResult<()> { + m.add_class::()?; + Ok(()) +} diff --git a/examples/plugin/plugin_api/tests/test_Gadget.py b/examples/plugin/plugin_api/tests/test_Gadget.py new file mode 100644 index 00000000000..f1175f2789c --- /dev/null +++ b/examples/plugin/plugin_api/tests/test_Gadget.py @@ -0,0 +1,22 @@ +import pytest + + +@pytest.fixture +def gadget(): + import plugin_api as pa + + g = pa.Gadget() + return g + + +def test_creation(gadget): + pass + + +def test_property(gadget): + gadget.prop = 42 + assert gadget.prop == 42 + + +def test_push(gadget): + gadget.push(42) diff --git a/examples/plugin/plugin_api/tests/test_import.py b/examples/plugin/plugin_api/tests/test_import.py new file mode 100644 index 00000000000..ae1d6f67f6e --- /dev/null +++ b/examples/plugin/plugin_api/tests/test_import.py @@ -0,0 +1,2 @@ +def test_import(): + import plugin_api diff --git a/examples/plugin/python_plugin/gadget_init_plugin.py b/examples/plugin/python_plugin/gadget_init_plugin.py new file mode 100644 index 00000000000..2eeba6fa7b8 --- /dev/null +++ b/examples/plugin/python_plugin/gadget_init_plugin.py @@ -0,0 +1,12 @@ +import plugin_api +import rng + + +def start(): + """create an instance of Gadget, configure it and return to Rust""" + g = plugin_api.Gadget() + g.push(1) + g.push(2) + g.push(3) + g.prop = rng.get_random_number() + return g diff --git a/examples/plugin/python_plugin/rng.py b/examples/plugin/python_plugin/rng.py new file mode 100644 index 00000000000..042e5e4b7d2 --- /dev/null +++ b/examples/plugin/python_plugin/rng.py @@ -0,0 +1,3 @@ +def get_random_number(): + # verified by the roll of a fair die to be random + return 4 diff --git a/examples/plugin/src/main.rs b/examples/plugin/src/main.rs new file mode 100644 index 00000000000..b50b54548e5 --- /dev/null +++ b/examples/plugin/src/main.rs @@ -0,0 +1,44 @@ +use plugin_api::plugin_api as pylib_module; +use pyo3::prelude::*; +use pyo3::types::PyList; +use std::path::Path; + +fn main() -> Result<(), Box> { + //"export" our API module to the python runtime + pyo3::append_to_inittab!(pylib_module); + //spawn runtime + pyo3::prepare_freethreaded_python(); + //import path for python + let path = Path::new("./python_plugin/"); + //do useful work + Python::with_gil(|py| { + //add the current directory to import path of Python (do not use this in production!) + let syspath: &PyList = py.import("sys")?.getattr("path")?.extract()?; + syspath.insert(0, &path)?; + println!("Import path is: {:?}", syspath); + + // Now we can load our python_plugin/gadget_init_plugin.py file. + // It can in turn import other stuff as it deems appropriate + let plugin = PyModule::import(py, "gadget_init_plugin")?; + // and call start function there, which will return a python reference to Gadget. + // Gadget here is a "pyclass" object reference + let gadget = plugin.getattr("start")?.call0()?; + + //now we extract (i.e. mutably borrow) the rust struct from python object + { + //this scope will have mutable access to the gadget instance, which will be dropped on + //scope exit so Python can access it again. + let mut gadget_rs: PyRefMut<'_, plugin_api::Gadget> = gadget.extract()?; + // we can now modify it as if it was a native rust struct + gadget_rs.prop = 42; + //which includes access to rust-only fields that are not visible to python + println!("rust-only vec contains {:?}", gadget_rs.rustonly); + gadget_rs.rustonly.clear(); + } + + //any modifications we make to rust object are reflected on Python object as well + let res: usize = gadget.getattr("prop")?.extract()?; + println!("{res}"); + Ok(()) + }) +} From d7601a43520df5ac90d5285f6214235a5a10d241 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Fri, 27 Jan 2023 08:31:44 +0000 Subject: [PATCH 5/5] add cargo-generate template for plugin example --- examples/plugin/.DS_Store | Bin 0 -> 6148 bytes examples/plugin/.template/Cargo.toml | 9 +++++++++ .../plugin/.template/plugin_api/Cargo.toml | 17 +++++++++++++++++ examples/plugin/.template/pre-script.rhai | 4 ++++ examples/plugin/cargo-generate.toml | 5 +++++ 5 files changed, 35 insertions(+) create mode 100644 examples/plugin/.DS_Store create mode 100644 examples/plugin/.template/Cargo.toml create mode 100644 examples/plugin/.template/plugin_api/Cargo.toml create mode 100644 examples/plugin/.template/pre-script.rhai create mode 100644 examples/plugin/cargo-generate.toml diff --git a/examples/plugin/.DS_Store b/examples/plugin/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5336624e6244d43eb38f04aca66bd1af338e0754 GIT binary patch literal 6148 zcmeHKF-`+P474FdP@0sK`vN3>u!@ow