diff --git a/CHANGELOG.md b/CHANGELOG.md index a49c7e425b0..5dcc38fc8c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added +- Add unsafe API `with_embedded_python_interpreter` to initalize a Python interpreter, execute a closure, and finalize the interpreter. [#1355](https://github.com/PyO3/pyo3/pull/1355) - Add `serde` feature to support `Serialize/Deserialize` for `Py`. [#1366](https://github.com/PyO3/pyo3/pull/1366) +### Changed +- `prepare_freethreaded_python` will no longer register an `atexit` handler to call `Py_Finalize`. [#1355](https://github.com/PyO3/pyo3/pull/1355) + ## [0.13.1] - 2021-01-10 ### Added - Add support for `#[pyclass(dict)]` and `#[pyclass(weakref)]` with the `abi3` feature on Python 3.9 and up. [#1342](https://github.com/PyO3/pyo3/pull/1342) diff --git a/guide/src/features.md b/guide/src/features.md index a7b620946b0..8a6e98a4145 100644 --- a/guide/src/features.md +++ b/guide/src/features.md @@ -36,6 +36,8 @@ This feature changes [`Python::with_gil`](https://docs.rs/pyo3/latest/pyo3/struc This feature is not needed for extension modules, but for compatibility it is enabled by default until at least the PyO3 0.14 release. +If you choose not to enable this feature, you should call `pyo3::prepare_freethreaded_python()` before attempting to call any other Python APIs. + > This feature is enabled by default. To disable it, set `default-features = false` for the `pyo3` entry in your Cargo.toml. ## Advanced Features diff --git a/src/gil.rs b/src/gil.rs index aaedbb96d7c..2d69bba3b54 100644 --- a/src/gil.rs +++ b/src/gil.rs @@ -46,68 +46,136 @@ pub(crate) fn gil_is_acquired() -> bool { /// this function has no effect. /// /// # Availability -/// /// This function is only available when linking against Python distributions that contain a /// shared library. /// /// This function is not available on PyPy. /// -/// # Panic -/// If the Python interpreter is initialized but Python threading is not, -/// a panic occurs. -/// It is not possible to safely access the Python runtime unless the main -/// thread (the thread which originally initialized Python) also initializes -/// threading. +/// # Panics +/// - If the Python interpreter is initialized but Python threading is not, +/// a panic occurs. +/// It is not possible to safely access the Python runtime unless the main +/// thread (the thread which originally initialized Python) also initializes +/// threading. +/// +/// # Example +/// ```rust +/// use pyo3::prelude::*; +/// +/// # #[allow(clippy::needless_doctest_main)] +/// fn main() { +/// pyo3::prepare_freethreaded_python(); +/// Python::with_gil(|py| { +/// py.run("print('Hello World')", None, None) +/// }); +/// } +/// ``` #[cfg(all(Py_SHARED, not(PyPy)))] pub fn prepare_freethreaded_python() { - // Protect against race conditions when Python is not yet initialized - // and multiple threads concurrently call 'prepare_freethreaded_python()'. - // Note that we do not protect against concurrent initialization of the Python runtime - // by other users of the Python C API. - START.call_once(|| unsafe { + // Protect against race conditions when Python is not yet initialized and multiple threads + // concurrently call 'prepare_freethreaded_python()'. Note that we do not protect against + // concurrent initialization of the Python runtime by other users of the Python C API. + START.call_once_force(|_| unsafe { + // Use call_once_force because if initialization panics, it's okay to try again. if ffi::Py_IsInitialized() != 0 { // If Python is already initialized, we expect Python threading to also be initialized, // as we can't make the existing Python main thread acquire the GIL. assert_ne!(ffi::PyEval_ThreadsInitialized(), 0); } else { - // Initialize Python. - // We use Py_InitializeEx() with initsigs=0 to disable Python signal handling. - // Signal handling depends on the notion of a 'main thread', which doesn't exist in this case. - // Note that the 'main thread' notion in Python isn't documented properly; - // and running Python without one is not officially supported. - ffi::Py_InitializeEx(0); - // Make sure Py_Finalize will be called before exiting. - extern "C" fn finalize() { - unsafe { - if ffi::Py_IsInitialized() != 0 { - ffi::PyGILState_Ensure(); - ffi::Py_Finalize(); - } - } - } - libc::atexit(finalize); - - // > Changed in version 3.7: This function is now called by Py_Initialize(), so you don’t have - // > to call it yourself anymore. + // Changed in version 3.7: This function is now called by Py_Initialize(), so you don’t + // have to call it yourself anymore. #[cfg(not(Py_3_7))] if ffi::PyEval_ThreadsInitialized() == 0 { ffi::PyEval_InitThreads(); } - // Py_InitializeEx() will acquire the GIL, but we don't want to hold it at this point - // (it's not acquired in the other code paths) - // So immediately release the GIL: - let _thread_state = ffi::PyEval_SaveThread(); - // Note that the PyThreadState returned by PyEval_SaveThread is also held in TLS by the Python runtime, - // and will be restored by PyGILState_Ensure. + // Release the GIL. + ffi::PyEval_SaveThread(); } }); } -/// RAII type that represents the Global Interpreter Lock acquisition. To get hold of a value -/// of this type, see [`Python::acquire_gil`](struct.Python.html#method.acquire_gil). +/// Executes the provided closure with an embedded Python interpreter. +/// +/// This function intializes the Python interpreter, executes the provided closure, and then +/// finalizes the Python interpreter. +/// +/// After execution all Python resources are cleaned up, and no further Python APIs can be called. +/// Because many Python modules implemented in C do not support multiple Python interpreters in a +/// single process, it is not safe to call this function more than once. (Many such modules will not +/// initialize correctly on the second run.) +/// +/// # Availability +/// This function is only available when linking against Python distributions that contain a shared +/// library. +/// +/// This function is not available on PyPy. +/// +/// # Panics +/// - If the Python interpreter is already initalized before calling this function. +/// +/// # Safety +/// - This function should only ever be called once per process (usually as part of the `main` +/// function). It is also not thread-safe. +/// - No Python APIs can be used after this function has finished executing. +/// - The return value of the closure must not contain any Python value, _including_ `PyResult`. +/// +/// # Example +/// ```rust +/// use pyo3::prelude::*; +/// +/// # #[allow(clippy::needless_doctest_main)] +/// fn main() { +/// unsafe { +/// pyo3::with_embedded_python_interpreter(|py| { +/// py.run("print('Hello World')", None, None) +/// }); +/// } +/// } +/// ``` +#[cfg(all(Py_SHARED, not(PyPy)))] +pub unsafe fn with_embedded_python_interpreter(f: F) -> R +where + F: for<'p> FnOnce(Python<'p>) -> R, +{ + assert_eq!( + ffi::Py_IsInitialized(), + 0, + "called `with_embedded_python_interpreter` but a Python interpreter is already running." + ); + + ffi::Py_InitializeEx(0); + + // Changed in version 3.7: This function is now called by Py_Initialize(), so you don’t have to + // call it yourself anymore. + #[cfg(not(Py_3_7))] + if ffi::PyEval_ThreadsInitialized() == 0 { + ffi::PyEval_InitThreads(); + } + + // Safe: the GIL is already held because of the Py_IntializeEx call. + let pool = GILPool::new(); + + // Import the threading module - this ensures that it will associate this thread as the "main" + // thread, which is important to avoid an `AssertionError` at finalization. + pool.python().import("threading").unwrap(); + + // Execute the closure. + let result = f(pool.python()); + + // Drop the pool before finalizing. + drop(pool); + + // Finalize the Python interpreter. + ffi::Py_Finalize(); + + result +} + +/// RAII type that represents the Global Interpreter Lock acquisition. To get hold of a value of +/// this type, see [`Python::acquire_gil`](struct.Python.html#method.acquire_gil). /// /// # Example /// ``` @@ -133,13 +201,13 @@ impl GILGuard { /// PyO3 internal API for acquiring the GIL. The public API is Python::acquire_gil. /// - /// If PyO3 does not yet have a `GILPool` for tracking owned PyObject references, then this - /// new `GILGuard` will also contain a `GILPool`. + /// If PyO3 does not yet have a `GILPool` for tracking owned PyObject references, then this new + /// `GILGuard` will also contain a `GILPool`. pub(crate) fn acquire() -> GILGuard { // Maybe auto-initialize the GIL: // - If auto-initialize feature set and supported, try to initalize the interpreter. - // - If the auto-initialize feature is set but unsupported, emit hard errors only when - // the extension-module feature is not activated - extension modules don't care about + // - If the auto-initialize feature is set but unsupported, emit hard errors only when the + // extension-module feature is not activated - extension modules don't care about // auto-initialize so this avoids breaking existing builds. // - Otherwise, just check the GIL is initialized. cfg_if::cfg_if! { @@ -170,8 +238,9 @@ impl GILGuard { // extension module feature enabled and PyPy or static linking // OR auto-initialize feature not enabled START.call_once_force(|_| unsafe { - // Use call_once_force because if there is a panic because the interpreter is not - // initialized, it's fine for the user to initialize the interpreter and retry. + // Use call_once_force because if there is a panic because the interpreter is + // not initialized, it's fine for the user to initialize the interpreter and + // retry. assert_ne!( ffi::Py_IsInitialized(), 0, diff --git a/src/lib.rs b/src/lib.rs index afa599f2052..462d3249634 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -152,7 +152,7 @@ pub use crate::conversion::{ }; pub use crate::err::{PyDowncastError, PyErr, PyErrArguments, PyResult}; #[cfg(all(Py_SHARED, not(PyPy)))] -pub use crate::gil::prepare_freethreaded_python; +pub use crate::gil::{prepare_freethreaded_python, with_embedded_python_interpreter}; pub use crate::gil::{GILGuard, GILPool}; pub use crate::instance::{Py, PyNativeType, PyObject}; pub use crate::pycell::{PyCell, PyRef, PyRefMut};