pyo3-build-config: many tidy ups

build.rs

@@ -1,9 +1,11 @@
-use std::{env, io::Cursor, path::Path, process::Command};
+use std::{env, process::Command};
 
 use pyo3_build_config::{
-    bail, cargo_env_var, ensure, env_var,
-    errors::{Context, Result},
-    make_cross_compile_config, InterpreterConfig, PythonVersion,
+    bail, ensure,
+    pyo3_build_script_impl::{
+        cargo_env_var, env_var, errors::Result, resolve_interpreter_config, InterpreterConfig,
+        PythonVersion,
+    },
 };
 
 /// Minimum Python version PyO3 supports.
@@ -20,24 +22,54 @@ fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> {
     Ok(())
 }
 
-fn ensure_target_pointer_width(pointer_width: u32) -> Result<()> {
-    // Try to check whether the target architecture matches the python library
-    let rust_target = match cargo_env_var("CARGO_CFG_TARGET_POINTER_WIDTH")
-        .unwrap()
-        .as_str()
-    {
-        "64" => 64,
-        "32" => 32,
-        x => bail!("unexpected Rust target pointer width: {}", x),
-    };
+fn ensure_target_pointer_width(interpreter_config: &InterpreterConfig) -> Result<()> {
+    if let Some(pointer_width) = interpreter_config.pointer_width {
+        // Try to check whether the target architecture matches the python library
+        let rust_target = match cargo_env_var("CARGO_CFG_TARGET_POINTER_WIDTH")
+            .unwrap()
+            .as_str()
+        {
+            "64" => 64,
+            "32" => 32,
+            x => bail!("unexpected Rust target pointer width: {}", x),
+        };
+
+        ensure!(
+            rust_target == pointer_width,
+            "your Rust target architecture ({}-bit) does not match your python interpreter ({}-bit)",
+            rust_target,
+            pointer_width
+        );
+    }
+    Ok(())
+}
 
-    ensure!(
-        rust_target == pointer_width,
-        "your Rust target architecture ({}-bit) does not match your python interpreter ({}-bit)",
-        rust_target,
-        pointer_width
-    );
+fn ensure_auto_initialize_ok(interpreter_config: &InterpreterConfig) -> Result<()> {
+    if cargo_env_var("CARGO_FEATURE_AUTO_INITIALIZE").is_some() {
+        if !interpreter_config.shared {
+            bail!(
+                "The `auto-initialize` feature is enabled, but your python installation only supports \
+                embedding the Python interpreter statically. If you are attempting to run tests, or a \
+                binary which is okay to link dynamically, install a Python distribution which ships \
+                with the Python shared library.\n\
+                \n\
+                Embedding the Python interpreter statically does not yet have first-class support in \
+                PyO3. If you are sure you intend to do this, disable the `auto-initialize` feature.\n\
+                \n\
+                For more information, see \
+                https://pyo3.rs/v{pyo3_version}/\
+                    building_and_distribution.html#embedding-python-in-rust",
+                pyo3_version = env::var("CARGO_PKG_VERSION").unwrap()
+            );
+        }
 
+        // TODO: PYO3_CI env is a hack to workaround CI with PyPy, where the `dev-dependencies`
+        // currently cause `auto-initialize` to be enabled in CI.
+        // Once MSRV is 1.51 or higher, use cargo's `resolver = "2"` instead.
+        if interpreter_config.implementation.is_pypy() && env::var_os("PYO3_CI").is_none() {
+            bail!("the `auto-initialize` feature is not supported with PyPy");
+        }
+    }
     Ok(())
 }
 
@@ -70,89 +102,36 @@ fn emit_cargo_configuration(interpreter_config: &InterpreterConfig) -> Result<()
             } else {
                 ""
             },
-            lib_name = interpreter_config
-                .lib_name
-                .as_ref()
-                .ok_or("config does not contain lib_name")?,
+            lib_name = interpreter_config.lib_name.as_ref().ok_or(
+                "attempted to link to Python shared library but config does not contain lib_name"
+            )?,
         );
         if let Some(lib_dir) = &interpreter_config.lib_dir {
             println!("cargo:rustc-link-search=native={}", lib_dir);
         }
     }
 
-    if cargo_env_var("CARGO_FEATURE_AUTO_INITIALIZE").is_some() {
-        if !interpreter_config.shared {
-            bail!(
-                "The `auto-initialize` feature is enabled, but your python installation only supports \
-                embedding the Python interpreter statically. If you are attempting to run tests, or a \
-                binary which is okay to link dynamically, install a Python distribution which ships \
-                with the Python shared library.\n\
-                \n\
-                Embedding the Python interpreter statically does not yet have first-class support in \
-                PyO3. If you are sure you intend to do this, disable the `auto-initialize` feature.\n\
-                \n\
-                For more information, see \
-                https://pyo3.rs/v{pyo3_version}/\
-                    building_and_distribution.html#embedding-python-in-rust",
-                pyo3_version = env::var("CARGO_PKG_VERSION").unwrap()
-            );
-        }
-
-        // TODO: PYO3_CI env is a hack to workaround CI with PyPy, where the `dev-dependencies`
-        // currently cause `auto-initialize` to be enabled in CI.
-        // Once MSRV is 1.51 or higher, use cargo's `resolver = "2"` instead.
-        if interpreter_config.is_pypy() && env::var_os("PYO3_CI").is_none() {
-            bail!("the `auto-initialize` feature is not supported with PyPy");
-        }
-    }
-
     Ok(())
 }
 
-/// Generates the interpreter config suitable for the host / target / cross-compilation at hand.
+/// Prepares the PyO3 crate for compilation.
+///
+/// This loads the config from pyo3-build-config and then makes some additional checks to improve UX
+/// for users.
 ///
-/// The result is written to pyo3_build_config::PATH, which downstream scripts can read from
-/// (including `pyo3-macros-backend` during macro expansion).
+/// Emits the cargo configuration based on this config as well as a few checks of the Rust compiler
+/// version to enable features which aren't supported on MSRV.
 fn configure_pyo3() -> Result<()> {
-    let interpreter_config = if let Some(path) = env_var("PYO3_CONFIG_FILE") {
-        let path = Path::new(&path);
-        // This is necessary because the compilations that access PYO3_CONFIG_FILE (build scripts,
-        // proc macros) have many different working directories, so a relative path is no good.
-        ensure!(path.is_absolute(), "PYO3_CONFIG_FILE must be an absolute path");
-        println!("cargo:rerun-if-changed={}", path.display());
-        InterpreterConfig::from_path(path)?
-    } else if let Some(interpreter_config) = make_cross_compile_config()? {
-        // This is a cross compile, need to write the config file.
-        let path = Path::new(&pyo3_build_config::DEFAULT_CROSS_COMPILE_CONFIG_PATH);
-        let parent_dir = path.parent().ok_or_else(|| {
-            format!(
-                "failed to resolve parent directory of config file {}",
-                path.display()
-            )
-        })?;
-        std::fs::create_dir_all(&parent_dir).with_context(|| {
-            format!(
-                "failed to create config file directory {}",
-                parent_dir.display()
-            )
-        })?;
-        interpreter_config
-            .to_writer(&mut std::fs::File::create(&path).with_context(|| {
-                format!("failed to create config file at {}", path.display())
-            })?)?;
-        interpreter_config
-    } else {
-        InterpreterConfig::from_reader(Cursor::new(pyo3_build_config::HOST_CONFIG))?
-    };
+    let interpreter_config = resolve_interpreter_config()?;
 
     if env_var("PYO3_PRINT_CONFIG").map_or(false, |os_str| os_str == "1") {
         print_config_and_exit(&interpreter_config);
     }
 
     ensure_python_version(&interpreter_config)?;
-    if let Some(pointer_width) = interpreter_config.pointer_width {
-        ensure_target_pointer_width(pointer_width)?;
-    }
+    ensure_target_pointer_width(&interpreter_config)?;
+    ensure_auto_initialize_ok(&interpreter_config)?;
+
     emit_cargo_configuration(&interpreter_config)?;
     interpreter_config.emit_pyo3_cfgs();
 

guide/src/building_and_distribution.md

@@ -30,23 +30,22 @@ Caused by:
   cargo:rerun-if-env-changed=PYO3_CROSS
   cargo:rerun-if-env-changed=PYO3_CROSS_LIB_DIR
   cargo:rerun-if-env-changed=PYO3_CROSS_PYTHON_VERSION
-  cargo:rerun-if-env-changed=PYO3_PYTHON
-  cargo:rerun-if-env-changed=VIRTUAL_ENV
-  cargo:rerun-if-env-changed=CONDA_PREFIX
-  cargo:rerun-if-env-changed=PATH
   cargo:rerun-if-env-changed=PYO3_PRINT_CONFIG
 
   -- PYO3_PRINT_CONFIG=1 is set, printing configuration and halting compile --
-  implementation: CPython
-  interpreter version: 3.8
-  interpreter path: Some("/usr/bin/python")
-  libdir: Some("/usr/lib")
-  shared: true
-  base prefix: Some("/usr")
-  ld_version: Some("3.8")
-  pointer width: Some(8)
+  implementation=CPython
+  version=3.8
+  shared=true
+  abi3=false
+  lib_name=python3.8
+  lib_dir=/usr/lib
+  executable=/usr/bin/python
+  pointer_width=64
+  build_flags=WITH_THREAD
 ```
 
+> Note: if you safe the output config to a file, it is possible to manually override the and feed it back into PyO3 using the `PYO3_CONFIG_FILE` env var. For now, this is an advanced feature that should not be needed for most users. The format of the config file and its contents are deliberately unstable and undocumented. If you have a production use-case for this config file, please file an issue and help us stabilize it!
+
 ## Building Python extension modules
 
 Python extension modules need to be compiled differently depending on the OS (and architecture) that they are being compiled for. As well as multiple OSes (and architectures), there are also many different Python versions which are actively supported. Packages uploaded to [PyPI](https://pypi.org/) usually want to upload prebuilt "wheels" covering many OS/arch/version combinations so that users on all these different platforms don't have to compile the package themselves. Package vendors can opt-in to the "abi3" limited Python API which allows their wheels to be used on multiple Python versions, reducing the number of wheels they need to compile, but restricts the functionality they can use.

pyo3-build-config/build.rs

@@ -11,21 +11,85 @@ mod errors;
 
 use std::{env, path::Path};
 
-use errors::{Result, Context};
+use errors::{Context, Result};
+use impl_::{
+    env_var, get_abi3_version, make_interpreter_config, BuildFlags, InterpreterConfig,
+    PythonImplementation,
+};
 
-fn generate_build_config() -> Result<()> {
-    // Create new interpreter config and write it to the default location
-    let interpreter_config = impl_::make_interpreter_config()?;
+fn configure(interpreter_config: Option<InterpreterConfig>, name: &str) -> Result<bool> {
+    let target = Path::new(&env::var_os("OUT_DIR").unwrap()).join(name);
+    if let Some(config) = interpreter_config {
+        config
+            .to_writer(&mut std::fs::File::create(&target).with_context(|| {
+                format!("failed to write config file at {}", target.display())
+            })?)?;
+        Ok(true)
+    } else {
+        std::fs::File::create(&target)
+            .with_context(|| format!("failed to create new file at {}", target.display()))?;
+        Ok(false)
+    }
+}
+
+/// If PYO3_CONFIG_FILE is set, copy it into the crate.
+fn config_file() -> Result<Option<InterpreterConfig>> {
+    if let Some(path) = env_var("PYO3_CONFIG_FILE") {
+        let path = Path::new(&path);
+        println!("cargo:rerun-if-changed={}", path.display());
+        // Absolute path is necessary because this build script is run with a cwd different to the
+        // original `cargo build` instruction.
+        ensure!(
+            path.is_absolute(),
+            "PYO3_CONFIG_FILE must be an absolute path"
+        );
+
+        let interpreter_config = InterpreterConfig::from_path(path)
+            .context("failed to parse contents of PYO3_CONFIG_FILE")?;
+        Ok(Some(interpreter_config))
+    } else {
+        Ok(None)
+    }
+}
 
-    let path = Path::new(&env::var_os("OUT_DIR").unwrap()).join("pyo3-build-config.txt");
-    interpreter_config
-        .to_writer(&mut std::fs::File::create(&path).with_context(|| {
-            format!("failed to create config file at {}", path.display())
-        })?)
+/// If PYO3_NO_PYTHON is set with abi3, use standard abi3 settings.
+pub fn abi3_config() -> Option<InterpreterConfig> {
+    if let Some(version) = get_abi3_version() {
+        if env_var("PYO3_NO_PYTHON").is_some() {
+            return Some(InterpreterConfig {
+                version,
+                // NB PyPy doesn't support abi3 yet
+                implementation: PythonImplementation::CPython,
+                abi3: true,
+                lib_name: None,
+                lib_dir: None,
+                build_flags: BuildFlags::abi3(),
+                pointer_width: None,
+                executable: None,
+                shared: true,
+            });
+        }
+    }
+    None
+}
+
+fn generate_build_configs() -> Result<()> {
+    let mut configured = false;
+    configured |= configure(config_file()?, "pyo3-build-config-file.txt")?;
+    configured |= configure(abi3_config(), "pyo3-build-config-abi3.txt")?;
+
+    if configured {
+        // Don't bother trying to find an interpreter on the host system if at least one of the
+        // config file or abi3 settings are present
+        configure(None, "pyo3-build-config.txt")?;
+    } else {
+        configure(Some(make_interpreter_config()?), "pyo3-build-config.txt")?;
+    }
+    Ok(())
 }
 
 fn main() {
-    if let Err(e) = generate_build_config() {
+    if let Err(e) = generate_build_configs() {
         eprintln!("error: {}", e.report());
         std::process::exit(1)
     }

pyo3-build-config/src/errors.rs

@@ -1,18 +1,21 @@
 /// A simple macro for returning an error. Resembles anyhow::bail.
 #[macro_export]
+#[doc(hidden)]
 macro_rules! bail {
     ($msg: expr) => { return Err($msg.into()); };
     ($fmt: literal $($args: tt)+) => { return Err(format!($fmt $($args)+).into()); };
 }
 
 /// A simple macro for checking a condition. Resembles anyhow::ensure.
 #[macro_export]
+#[doc(hidden)]
 macro_rules! ensure {
     ($condition:expr, $($args: tt)+) => { if !($condition) { bail!($($args)+) } };
 }
 
 /// Show warning. If needed, please extend this macro to support arguments.
 #[macro_export]
+#[doc(hidden)]
 macro_rules! warn {
     ($msg: literal) => {
         println!(concat!("cargo:warning=", $msg));
@@ -27,7 +30,7 @@ pub struct Error {
 }
 
 /// Error report inspired by
-/// https://blog.rust-lang.org/inside-rust/2021/07/01/What-the-error-handling-project-group-is-working-towards.html#2-error-reporter
+/// <https://blog.rust-lang.org/inside-rust/2021/07/01/What-the-error-handling-project-group-is-working-towards.html#2-error-reporter>
 pub struct ErrorReport<'a>(&'a Error);
 
 impl Error {