Merge pull request #2293 from ravenexp/abi3-no-python

pyo3-build-config: Build "abi3" extensions without an interpreter

CHANGELOG.md

@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- Allow to compile "abi3" extensions without a working build host Python interpreter. [#2293](https://github.com/PyO3/pyo3/pull/2293)
 - Default to "m" ABI tag when choosing `libpython` link name for CPython 3.7 on Unix. [#2288](https://github.com/PyO3/pyo3/pull/2288)
 - Improved performance of failing calls to `FromPyObject::extract` which is common when functions accept multiple distinct types. [#2279](https://github.com/PyO3/pyo3/pull/2279)
 

Cargo.toml

@@ -67,7 +67,7 @@ pyproto = ["pyo3-macros/pyproto"]
 # Use this feature when building an extension module.
 # It tells the linker to keep the python symbols unresolved,
 # so that the module can also be used with statically linked python interpreters.
-extension-module = ["pyo3-ffi/extension-module"]
+extension-module = ["pyo3-build-config/extension-module", "pyo3-ffi/extension-module"]
 
 # Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more.
 abi3 = ["pyo3-build-config/abi3", "pyo3-ffi/abi3", "pyo3-macros/abi3"]

guide/src/building_and_distribution.md

@@ -157,6 +157,8 @@ PyO3 is only able to link your extension module to api3 version up to and includ
 #### Building `abi3` extensions without a Python interpreter
 
 As an advanced feature, you can build PyO3 wheel without calling Python interpreter with the environment variable `PYO3_NO_PYTHON` set.
+Also, if the build host Python interpreter is not found or is too old or otherwise unusable,
+PyO3 will still attempt to compile `abi3` extension modules after displaying a warning message.
 On Unix-like systems this works unconditionally; on Windows you must also set the `RUSTFLAGS` environment variable
 to contain `-L native=/path/to/python/libs` so that the linker can find `python3.lib`.
 

pyo3-build-config/Cargo.toml

@@ -26,6 +26,10 @@ default = []
 # script. If this feature isn't enabled, the build script no-ops.
 resolve-config = []
 
+# This feature is enabled by pyo3 when building an extension module.
+extension-module = []
+
+# These features are enabled by pyo3 when building Stable ABI extension modules.
 abi3 = []
 abi3-py37 = ["abi3-py38"]
 abi3-py38 = ["abi3-py39"]

pyo3-build-config/src/impl_.rs

@@ -233,6 +233,13 @@ print("mingw", get_platform().startswith("mingw"))
 "#;
         let output = run_python_script(interpreter.as_ref(), SCRIPT)?;
         let map: HashMap<String, String> = parse_script_output(&output);
+
+        ensure!(
+            !map.is_empty(),
+            "broken Python interpreter: {}",
+            interpreter.as_ref().display()
+        );
+
         let shared = map["shared"].as_str() == "True";
 
         let version = PythonVersion {
@@ -678,21 +685,33 @@ pub fn is_extension_module() -> bool {
 
 /// Checks if we need to link to `libpython` for the current build target.
 ///
-/// Must be called from a crate PyO3 build script.
+/// Must be called from a PyO3 crate build script.
 pub fn is_linking_libpython() -> bool {
     is_linking_libpython_for_target(&target_triple_from_env())
 }
 
 /// Checks if we need to link to `libpython` for the target.
 ///
-/// Must be called from a crate PyO3 build script.
+/// Must be called from a PyO3 crate build script.
 fn is_linking_libpython_for_target(target: &Triple) -> bool {
     target.operating_system == OperatingSystem::Windows
         || target.environment == Environment::Android
         || target.environment == Environment::Androideabi
         || !is_extension_module()
 }
 
+/// Checks if we need to discover the Python library directory
+/// to link the extension module binary.
+///
+/// Must be called from a PyO3 crate build script.
+fn require_libdir_for_target(target: &Triple) -> bool {
+    let is_generating_libpython = cfg!(feature = "python3-dll-a")
+        && target.operating_system == OperatingSystem::Windows
+        && is_abi3();
+
+    is_linking_libpython_for_target(target) && !is_generating_libpython
+}
+
 /// Configuration needed by PyO3 to cross-compile for a target platform.
 ///
 /// Usually this is collected from the environment (i.e. `PYO3_CROSS_*` and `CARGO_CFG_TARGET_*`)
@@ -1642,6 +1661,18 @@ pub fn find_interpreter() -> Result<PathBuf> {
     }
 }
 
+/// Locates and extracts the build host Python interpreter configuration.
+///
+/// Lowers the configured Python version to `abi3_version` if required.
+fn get_host_interpreter(abi3_version: Option<PythonVersion>) -> Result<InterpreterConfig> {
+    let interpreter_path = find_interpreter()?;
+
+    let mut interpreter_config = InterpreterConfig::from_interpreter(interpreter_path)?;
+    interpreter_config.fixup_for_abi3_version(abi3_version)?;
+
+    Ok(interpreter_config)
+}
+
 /// Generates an interpreter config suitable for cross-compilation.
 ///
 /// This must be called from PyO3's build script, because it relies on environment variables such as
@@ -1662,26 +1693,40 @@ pub fn make_cross_compile_config() -> Result<Option<InterpreterConfig>> {
 /// Only used by `pyo3-build-config` build script.
 #[allow(dead_code, unused_mut)]
 pub fn make_interpreter_config() -> Result<InterpreterConfig> {
+    let host = Triple::host();
     let abi3_version = get_abi3_version();
 
+    // See if we can safely skip the Python interpreter configuration detection.
+    // Unix "abi3" extension modules can usually be built without any interpreter.
+    let need_interpreter = abi3_version.is_none() || require_libdir_for_target(&host);
+
     if have_python_interpreter() {
-        let mut interpreter_config = InterpreterConfig::from_interpreter(find_interpreter()?)?;
-        interpreter_config.fixup_for_abi3_version(abi3_version)?;
-        Ok(interpreter_config)
-    } else if let Some(version) = abi3_version {
-        let host = Triple::host();
-        let mut interpreter_config = default_abi3_config(&host, version);
-
-        // Auto generate python3.dll import libraries for Windows targets.
-        #[cfg(feature = "python3-dll-a")]
-        {
-            interpreter_config.lib_dir = self::abi3_import_lib::generate_abi3_import_lib(&host)?;
+        match get_host_interpreter(abi3_version) {
+            Ok(interpreter_config) => return Ok(interpreter_config),
+            // Bail if the interpreter configuration is required to build.
+            Err(e) if need_interpreter => return Err(e),
+            _ => {
+                // Fall back to the "abi3" defaults just as if `PYO3_NO_PYTHON`
+                // environment variable was set.
+                warn!("Compiling without a working Python interpreter.");
+            }
         }
-
-        Ok(interpreter_config)
     } else {
-        bail!("An abi3-py3* feature must be specified when compiling without a Python interpreter.")
+        ensure!(
+            abi3_version.is_some(),
+            "An abi3-py3* feature must be specified when compiling without a Python interpreter."
+        );
+    };
+
+    let mut interpreter_config = default_abi3_config(&host, abi3_version.unwrap());
+
+    // Auto generate python3.dll import libraries for Windows targets.
+    #[cfg(feature = "python3-dll-a")]
+    {
+        interpreter_config.lib_dir = self::abi3_import_lib::generate_abi3_import_lib(&host)?;
     }
+
+    Ok(interpreter_config)
 }
 
 fn escape(bytes: &[u8]) -> String {