Merge #2873 #2889 #2923 #2924

2873: A new example that shows how to integrate Python plugins that use pyclasses into a Rust app r=davidhewitt a=alexpyattaev

Example showing integration of a Python plugin into a Rust app while having option to test pyclass based API without the main app.  This also illustrates some aspects related to import of Python modules into a Rust app while also having an API module available for the Python code to be able to produce Rust objects. 

CI seems to fail on my local machine for reasons unrelated to the example just added:
```
error: unused macro definition: `check_struct`
  --> pyo3-ffi-check/src/main.rs:13:18
   |
13 |     macro_rules! check_struct {
   |                  ^^^^^^^^^^^^
   |
```

2889: Added support for PyErr_WriteUnraisable r=davidhewitt a=mitsuhiko

Fixes #2884

2923: hygiene: fix `#[pymethods(crate = "...")]` r=davidhewitt a=davidhewitt

Got to the bottom of the hygiene issue in test of #2914 

Turns out that `#[pymethods] #[pyo3(crate = "...")]` works, but `#[pymethods(crate = "...")]` was ignoring the argument.

Added a tweak to fix this and a snippet in the hygiene test (which fails on `main`). 

2924: remove unneeded into_iter calls r=davidhewitt a=davidhewitt

Clippy complaining about these to me this morning locally.

Co-authored-by: Alex Pyattaev <[email protected]>
Co-authored-by: David Hewitt <[email protected]>
Co-authored-by: Armin Ronacher <[email protected]>

benches/bench_set.rs

@@ -9,7 +9,7 @@ fn set_new(b: &mut Bencher<'_>) {
         const LEN: usize = 100_000;
         // Create Python objects up-front, so that the benchmark doesn't need to include
         // the cost of allocating LEN Python integers
-        let elements: Vec<PyObject> = (0..LEN).into_iter().map(|i| i.into_py(py)).collect();
+        let elements: Vec<PyObject> = (0..LEN).map(|i| i.into_py(py)).collect();
         b.iter(|| {
             let pool = unsafe { py.new_pool() };
             PySet::new(py, &elements).unwrap();

examples/README.md

@@ -10,6 +10,7 @@ Below is a brief description of each of these:
 | `maturin-starter` | A template project which is configured to use [`maturin`](https://github.com/PyO3/maturin) for development. |
 | `setuptools-rust-starter` | A template project which is configured to use [`setuptools_rust`](https://github.com/PyO3/setuptools-rust/) for development. |
 | `word-count` | A quick performance comparison between word counter implementations written in each of Rust and Python. |
+| `plugin` | Illustrates how to use Python as a scripting language within a Rust application |
 
 ## Creating new projects from these examples
 

examples/plugin/.template/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+authors = ["{{authors}}"]
+name = "{{project-name}}"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+pyo3 = "{{PYO3_VERSION}}"
+plugin_api = { path = "plugin_api" }

examples/plugin/.template/plugin_api/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "plugin_api"
+version = "0.1.0"
+description = "Plugin API example"
+edition = "2021"
+
+[lib]
+name = "plugin_api"
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+#!!! Important - DO NOT ENABLE extension-module FEATURE HERE!!!
+pyo3 = "{{PYO3_VERSION}}"
+
+[features]
+# instead extension-module feature for pyo3 is enabled conditionally when we want to build a standalone extension module to test our plugins without "main" program
+extension-module = ["pyo3/extension-module"]

examples/plugin/.template/pre-script.rhai

@@ -0,0 +1,4 @@
+variable::set("PYO3_VERSION", "0.18.0");
+file::rename(".template/Cargo.toml", "Cargo.toml");
+file::rename(".template/plugin_api/Cargo.toml", "plugin_api/Cargo.toml");
+file::delete(".template");

examples/plugin/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "plugin_example"
+version = "0.1.0"
+edition = "2021"
+
+
+[dependencies]
+pyo3={path="../../", features=["macros"]}
+plugin_api={path="plugin_api"}
+
+
+[workspace]

examples/plugin/README.md

@@ -0,0 +1,48 @@
+# plugin
+
+An example of a Rust app that uses Python for a plugin. A Python extension module built using PyO3 and [`maturin`](https://github.com/PyO3/maturin) is used to provide
+interface types that can be used to exchange data between Rust and Python. This also deals with how to separately test and load python modules.
+
+# Building and Testing
+## Host application
+To run the app itself, you only need to run 
+
+```shell
+cargo run
+```
+It will build the app, as well as the plugin API, then run the app, load the plugin and show it working.
+
+## Plugin API testing
+
+The plugin API is in a separate crate `plugin_api`, so you can test it separately from the main app. 
+
+To build the API only package, first install `maturin`:
+
+```shell
+pip install maturin
+```
+
+When building the plugin, simply using `maturin develop` will fail to produce a viable extension module due to the features arrangement of PyO3. 
+Instead, one needs to enable the optional feature as follows:
+
+```shell
+cd plugin_api
+maturin build --features "extension-module"
+```
+
+Alternatively, install nox and run the tests inside an isolated environment:
+
+```shell
+nox
+```
+
+## Copying this example
+
+Use [`cargo-generate`](https://crates.io/crates/cargo-generate):
+
+```bash
+$ cargo install cargo-generate
+$ cargo generate --git https://github.com/PyO3/pyo3 examples/plugin
+```
+
+(`cargo generate` will take a little while to clone the PyO3 repo first; be patient when waiting for the command to run.)

examples/plugin/cargo-generate.toml

@@ -0,0 +1,5 @@
+[template]
+ignore = [".nox"]
+
+[hooks]
+pre = [".template/pre-script.rhai"]

examples/plugin/plugin_api/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "plugin_api"
+version = "0.1.0"
+description = "Plugin API example"
+edition = "2021"
+
+[lib]
+name = "plugin_api"
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+#!!! Important - DO NOT ENABLE extension-module FEATURE HERE!!!
+pyo3 = { path = "../../../" }
+
+[features]
+# instead extension-module feature for pyo3 is enabled conditionally when we want to build a standalone extension module to test our plugins without "main" program
+extension-module = ["pyo3/extension-module"]

examples/plugin/plugin_api/noxfile.py

@@ -0,0 +1,9 @@
+import nox
+
+
+@nox.session
+def python(session):
+    session.install("-rrequirements-dev.txt")
+    session.install("maturin")
+    session.run_always("maturin", "develop", "--features", "extension-module")
+    session.run("pytest")

examples/plugin/plugin_api/pyproject.toml

@@ -0,0 +1,14 @@
+[build-system]
+requires = ["maturin>=0.14,<0.15"]
+build-backend = "maturin"
+
+[project]
+name = "plugin_api"
+requires-python = ">=3.7"
+classifiers = [
+    "Programming Language :: Rust",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: Implementation :: PyPy",
+]
+
+

examples/plugin/plugin_api/requirements-dev.txt

@@ -0,0 +1,3 @@
+pytest>=3.5.0
+pip>=21.3
+maturin>=0.14

examples/plugin/plugin_api/src/lib.rs

@@ -0,0 +1,32 @@
+use pyo3::prelude::*;
+
+///this is our Gadget that python plugin code can create, and rust app can then access natively.
+#[pyclass]
+pub struct Gadget {
+    #[pyo3(get, set)]
+    pub prop: usize,
+    //this field will only be accessible to rust code
+    pub rustonly: Vec<usize>,
+}
+
+#[pymethods]
+impl Gadget {
+    #[new]
+    fn new() -> Self {
+        Gadget {
+            prop: 777,
+            rustonly: Vec::new(),
+        }
+    }
+
+    fn push(&mut self, v: usize) {
+        self.rustonly.push(v);
+    }
+}
+
+/// A Python module for plugin interface types
+#[pymodule]
+pub fn plugin_api(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
+    m.add_class::<Gadget>()?;
+    Ok(())
+}

examples/plugin/plugin_api/tests/test_Gadget.py

@@ -0,0 +1,22 @@
+import pytest
+
+
+@pytest.fixture
+def gadget():
+    import plugin_api as pa
+
+    g = pa.Gadget()
+    return g
+
+
+def test_creation(gadget):
+    pass
+
+
+def test_property(gadget):
+    gadget.prop = 42
+    assert gadget.prop == 42
+
+
+def test_push(gadget):
+    gadget.push(42)

examples/plugin/plugin_api/tests/test_import.py

@@ -0,0 +1,2 @@
+def test_import():
+    import plugin_api

examples/plugin/python_plugin/gadget_init_plugin.py

@@ -0,0 +1,12 @@
+import plugin_api
+import rng
+
+
+def start():
+    """create an instance of Gadget, configure it and return to Rust"""
+    g = plugin_api.Gadget()
+    g.push(1)
+    g.push(2)
+    g.push(3)
+    g.prop = rng.get_random_number()
+    return g

examples/plugin/python_plugin/rng.py

@@ -0,0 +1,3 @@
+def get_random_number():
+    # verified by the roll of a fair die to be random
+    return 4

examples/plugin/src/main.rs

@@ -0,0 +1,44 @@
+use plugin_api::plugin_api as pylib_module;
+use pyo3::prelude::*;
+use pyo3::types::PyList;
+use std::path::Path;
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    //"export" our API module to the python runtime
+    pyo3::append_to_inittab!(pylib_module);
+    //spawn runtime
+    pyo3::prepare_freethreaded_python();
+    //import path for python
+    let path = Path::new("./python_plugin/");
+    //do useful work
+    Python::with_gil(|py| {
+        //add the current directory to import path of Python (do not use this in production!)
+        let syspath: &PyList = py.import("sys")?.getattr("path")?.extract()?;
+        syspath.insert(0, &path)?;
+        println!("Import path is: {:?}", syspath);
+
+        // Now we can load our python_plugin/gadget_init_plugin.py file.
+        // It can in turn import other stuff as it deems appropriate
+        let plugin = PyModule::import(py, "gadget_init_plugin")?;
+        // and call start function there, which will return a python reference to Gadget.
+        // Gadget here is a "pyclass" object reference
+        let gadget = plugin.getattr("start")?.call0()?;
+
+        //now we extract (i.e. mutably borrow) the rust struct from python object
+        {
+            //this scope will have mutable access to the gadget instance, which will be dropped on
+            //scope exit so Python can access it again.
+            let mut gadget_rs: PyRefMut<'_, plugin_api::Gadget> = gadget.extract()?;
+            // we can now modify it as if it was a native rust struct
+            gadget_rs.prop = 42;
+            //which includes access to rust-only fields that are not visible to python
+            println!("rust-only vec contains {:?}", gadget_rs.rustonly);
+            gadget_rs.rustonly.clear();
+        }
+
+        //any modifications we make to rust object are reflected on Python object as well
+        let res: usize = gadget.getattr("prop")?.extract()?;
+        println!("{res}");
+        Ok(())
+    })
+}

newsfragments/2889.added.md

@@ -0,0 +1 @@
+Added `PyErr::write_unraisable()` to report an unraisable exception to Python.

newsfragments/2923.fixed.md

@@ -0,0 +1 @@
+Fix `#[pymethods(crate = "...")]` option being ignored.

pyo3-macros-backend/src/frompyobject.rs

@@ -278,7 +278,6 @@ impl<'a> Container<'a> {
         let self_ty = &self.path;
         let struct_name = &self.name();
         let field_idents: Vec<_> = (0..struct_fields.len())
-            .into_iter()
             .map(|i| format_ident!("arg{}", i))
             .collect();
         let fields = struct_fields.iter().zip(&field_idents).enumerate().map(|(index, (field, ident))| {

pyo3-macros-backend/src/pymethod.rs

@@ -1109,7 +1109,6 @@ impl SlotDef {
         let py = syn::Ident::new("_py", Span::call_site());
         let arg_types: &Vec<_> = &arguments.iter().map(|arg| arg.ffi_type()).collect();
         let arg_idents: &Vec<_> = &(0..arguments.len())
-            .into_iter()
             .map(|i| format_ident!("arg{}", i))
             .collect();
         let wrapper_ident = format_ident!("__pymethod_{}__", method_name);
@@ -1220,7 +1219,6 @@ impl SlotFragmentDef {
         let py = syn::Ident::new("_py", Span::call_site());
         let arg_types: &Vec<_> = &arguments.iter().map(|arg| arg.ffi_type()).collect();
         let arg_idents: &Vec<_> = &(0..arguments.len())
-            .into_iter()
             .map(|i| format_ident!("arg{}", i))
             .collect();
         let body = generate_method_body(cls, spec, &py, arguments, *extract_error_mode, None)?;

pyo3-macros/src/lib.rs

@@ -109,13 +109,13 @@ pub fn pyclass(attr: TokenStream, input: TokenStream) -> TokenStream {
 /// [10]: https://pyo3.rs/latest/class.html#method-arguments
 /// [11]: https://pyo3.rs/latest/class.html#object-properties-using-pyo3get-set
 #[proc_macro_attribute]
-pub fn pymethods(_: TokenStream, input: TokenStream) -> TokenStream {
+pub fn pymethods(attr: TokenStream, input: TokenStream) -> TokenStream {
     let methods_type = if cfg!(feature = "multiple-pymethods") {
         PyClassMethodsType::Inventory
     } else {
         PyClassMethodsType::Specialization
     };
-    pymethods_impl(input, methods_type)
+    pymethods_impl(attr, input, methods_type)
 }
 
 /// A proc macro used to expose Rust functions to Python.
@@ -191,8 +191,17 @@ fn pyclass_enum_impl(
     .into()
 }
 
-fn pymethods_impl(input: TokenStream, methods_type: PyClassMethodsType) -> TokenStream {
+fn pymethods_impl(
+    attr: TokenStream,
+    input: TokenStream,
+    methods_type: PyClassMethodsType,
+) -> TokenStream {
     let mut ast = parse_macro_input!(input as syn::ItemImpl);
+    // Apply all options as a #[pyo3] attribute on the ItemImpl
+    // e.g. #[pymethods(crate = "crate")] impl Foo { }
+    // -> #[pyo3(crate = "crate")] impl Foo { }
+    let attr: TokenStream2 = attr.into();
+    ast.attrs.push(syn::parse_quote!( #[pyo3(#attr)] ));
     let expanded = build_py_methods(&mut ast, methods_type).unwrap_or_compile_error();
 
     quote!(

src/err/mod.rs

@@ -476,6 +476,40 @@ impl PyErr {
         unsafe { ffi::PyErr_Restore(ptype, pvalue, ptraceback) }
     }
 
+    /// Reports the error as unraisable.
+    ///
+    /// This calls `sys.unraisablehook()` using the current exception and obj argument.
+    ///
+    /// This method is useful to report errors in situations where there is no good mechanism
+    /// to report back to the Python land.  In Python this is used to indicate errors in
+    /// background threads or destructors which are protected.  In Rust code this is commonly
+    /// useful when you are calling into a Python callback which might fail, but there is no
+    /// obvious way to handle this error other than logging it.
+    ///
+    /// Calling this method has the benefit that the error goes back into a standardized callback
+    /// in Python which for instance allows unittests to ensure that no unraisable error
+    /// actually happend by hooking `sys.unraisablehook`.
+    ///
+    /// Example:
+    /// ```rust
+    /// # use pyo3::prelude::*;
+    /// # use pyo3::exceptions::PyRuntimeError;
+    /// # fn failing_function() -> PyResult<()> { Err(PyRuntimeError::new_err("foo")) }
+    /// # fn main() -> PyResult<()> {
+    /// Python::with_gil(|py| {
+    ///     match failing_function() {
+    ///         Err(pyerr) => pyerr.write_unraisable(py, None),
+    ///         Ok(..) => { /* do something here */ }
+    ///     }
+    ///     Ok(())
+    /// })
+    /// # }
+    #[inline]
+    pub fn write_unraisable(self, py: Python<'_>, obj: Option<&PyAny>) {
+        self.restore(py);
+        unsafe { ffi::PyErr_WriteUnraisable(obj.map_or(std::ptr::null_mut(), |x| x.as_ptr())) }
+    }
+
     /// Issues a warning message.
     ///
     /// May return an `Err(PyErr)` if warnings-as-errors is enabled.