From 79c7149d524f86dc2503597759fb2f19d4de21de Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Wed, 19 May 2021 21:50:25 +0100 Subject: [PATCH 1/5] pyo3-build-config: new crate to re-use build.rs across crates --- Cargo.toml | 14 +- build.rs | 882 ++--------------------------- examples/pyo3-pytests/Cargo.toml | 6 +- examples/pyo3-pytests/build.rs | 27 +- pyo3-build-config/Cargo.toml | 22 + pyo3-build-config/LICENSE | 1 + pyo3-build-config/build.rs | 11 + pyo3-build-config/src/impl_.rs | 882 +++++++++++++++++++++++++++++ pyo3-build-config/src/lib.rs | 19 + pyo3-macros-backend/Cargo.toml | 3 + pyo3-macros-backend/build.rs | 3 + pyo3-macros-backend/src/pyproto.rs | 5 +- 12 files changed, 991 insertions(+), 884 deletions(-) create mode 100644 pyo3-build-config/Cargo.toml create mode 120000 pyo3-build-config/LICENSE create mode 100644 pyo3-build-config/build.rs create mode 100644 pyo3-build-config/src/impl_.rs create mode 100644 pyo3-build-config/src/lib.rs create mode 100644 pyo3-macros-backend/build.rs diff --git a/Cargo.toml b/Cargo.toml index 6dcc5d2d02d..03cbfc8ad4e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,6 @@ documentation = "https://docs.rs/crate/pyo3/" categories = ["api-bindings", "development-tools::ffi"] license = "Apache-2.0" exclude = ["/.gitignore", ".cargo/config", "/codecov.yml", "/Makefile", "/pyproject.toml", "/tox.ini"] -build = "build.rs" edition = "2018" [dependencies] @@ -40,6 +39,9 @@ proptest = { version = "0.10.1", default-features = false, features = ["std"] } pyo3 = { path = ".", default-features = false, features = ["macros", "auto-initialize"] } serde_json = "1.0.61" +[build-dependencies] +pyo3-build-config = { path = "pyo3-build-config", version = "=0.14.0-alpha.0" } + [features] default = ["macros"] @@ -55,13 +57,13 @@ multiple-pymethods = ["inventory"] extension-module = [] # Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more. -abi3 = [] +abi3 = ["pyo3-build-config/abi3"] # With abi3, we can manually set the minimum Python version. -abi3-py36 = ["abi3-py37"] -abi3-py37 = ["abi3-py38"] -abi3-py38 = ["abi3-py39"] -abi3-py39 = ["abi3"] +abi3-py36 = ["abi3-py37", "pyo3-build-config/abi3-py36"] +abi3-py37 = ["abi3-py38", "pyo3-build-config/abi3-py37"] +abi3-py38 = ["abi3-py39", "pyo3-build-config/abi3-py38"] +abi3-py39 = ["abi3", "pyo3-build-config/abi3-py39"] # Changes `Python::with_gil` and `Python::acquire_gil` to automatically initialize the # Python interpreter if needed. diff --git a/build.rs b/build.rs index 01731aed9d5..637258f30f6 100644 --- a/build.rs +++ b/build.rs @@ -1,591 +1,18 @@ -use std::{ - collections::{HashMap, HashSet}, - convert::AsRef, - env, - ffi::OsString, - fmt::Display, - fs::{self, DirEntry}, - path::{Path, PathBuf}, - process::{Command, Stdio}, - str::FromStr, -}; +use std::{env, process::Command}; -/// Minimum Python version PyO3 supports. -const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 6 }; -/// Maximum Python version that can be used as minimum required Python version with abi3. -const ABI3_MAX_MINOR: u8 = 9; +use pyo3_build_config::{InterpreterConfig, PythonImplementation}; type Result = std::result::Result>; -// A simple macro for returning an error. Resembles anyhow::bail. -macro_rules! bail { - ($msg: expr) => { return Err($msg.into()); }; - ($fmt: literal $($args: tt)+) => { return Err(format!($fmt $($args)+).into()); }; -} - -// A simple macro for checking a condition. Resembles anyhow::ensure. -macro_rules! ensure { - ($condition:expr, $($args: tt)+) => { if !($condition) { bail!($($args)+) } }; -} - -// Show warning. If needed, please extend this macro to support arguments. -macro_rules! warn { - ($msg: literal) => { - println!(concat!("cargo:warning=", $msg)); - }; -} - -/// Gets an environment variable owned by cargo. -/// -/// Environment variables set by cargo are expected to be valid UTF8. -fn cargo_env_var(var: &str) -> Option { - env::var_os(var).map(|os_string| os_string.to_str().unwrap().into()) -} - -/// Gets an external environment variable, and registers the build script to rerun if -/// the variable changes. -fn env_var(var: &str) -> Option { - println!("cargo:rerun-if-env-changed={}", var); - env::var_os(var) -} - -/// Configuration needed by PyO3 to build for the correct Python implementation. -/// -/// Usually this is queried directly from the Python interpreter. When the `PYO3_NO_PYTHON` variable -/// is set, or during cross compile situations, then alternative strategies are used to populate -/// this type. -#[derive(Debug)] -struct InterpreterConfig { - version: PythonVersion, - libdir: Option, - shared: bool, - ld_version: Option, - base_prefix: Option, - executable: Option, - calcsize_pointer: Option, - implementation: PythonImplementation, - build_flags: BuildFlags, -} - -impl InterpreterConfig { - fn is_pypy(&self) -> bool { - self.implementation == PythonImplementation::PyPy - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] -struct PythonVersion { - major: u8, - minor: u8, -} - -impl PythonVersion { - const PY37: Self = PythonVersion { major: 3, minor: 7 }; -} - -impl Display for PythonVersion { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}.{}", self.major, self.minor) - } -} - -#[derive(Debug, Copy, Clone, PartialEq)] -pub enum PythonImplementation { - CPython, - PyPy, -} - -impl FromStr for PythonImplementation { - type Err = Box; - fn from_str(s: &str) -> Result { - match s { - "CPython" => Ok(PythonImplementation::CPython), - "PyPy" => Ok(PythonImplementation::PyPy), - _ => bail!("Invalid interpreter: {}", s), - } - } -} - -fn is_abi3() -> bool { - cargo_env_var("CARGO_FEATURE_ABI3").is_some() -} - -trait GetPrimitive { - fn get_bool(&self, key: &str) -> Result; - fn get_numeric(&self, key: &str) -> Result; -} - -impl GetPrimitive for HashMap { - fn get_bool(&self, key: &str) -> Result { - match self - .get(key) - .map(|x| x.as_str()) - .ok_or(format!("{} is not defined", key))? - { - "1" | "true" | "True" => Ok(true), - "0" | "false" | "False" => Ok(false), - _ => bail!("{} must be a bool (1/true/True or 0/false/False", key), - } - } - - fn get_numeric(&self, key: &str) -> Result { - self.get(key) - .ok_or(format!("{} is not defined", key))? - .parse::() - .map_err(|_| format!("Could not parse value of {}", key).into()) - } -} - -struct CrossCompileConfig { - lib_dir: PathBuf, - version: Option, - os: String, - arch: String, -} - -fn cross_compiling() -> Result> { - let cross = env_var("PYO3_CROSS"); - let cross_lib_dir = env_var("PYO3_CROSS_LIB_DIR"); - let cross_python_version = env_var("PYO3_CROSS_PYTHON_VERSION"); - - let target_arch = cargo_env_var("CARGO_CFG_TARGET_ARCH"); - let target_vendor = cargo_env_var("CARGO_CFG_TARGET_VENDOR"); - let target_os = cargo_env_var("CARGO_CFG_TARGET_OS"); - - if cross.is_none() && cross_lib_dir.is_none() && cross_python_version.is_none() { - // No cross-compiling environment variables set; try to determine if this is a known case - // which is not cross-compilation. - - let target = cargo_env_var("TARGET").unwrap(); - let host = cargo_env_var("HOST").unwrap(); - if target == host { - // Not cross-compiling - return Ok(None); - } - - if target == "i686-pc-windows-msvc" && host == "x86_64-pc-windows-msvc" { - // Not cross-compiling to compile for 32-bit Python from windows 64-bit - return Ok(None); - } - - if target == "x86_64-apple-darwin" && host == "aarch64-apple-darwin" { - // Not cross-compiling to compile for x86-64 Python from macOS arm64 - return Ok(None); - } - - if target == "aarch64-apple-darwin" && host == "x86_64-apple-darwin" { - // Not cross-compiling to compile for arm64 Python from macOS x86_64 - return Ok(None); - } - - if let (Some(arch), Some(vendor), Some(os)) = (&target_arch, &target_vendor, &target_os) { - if host.starts_with(&format!("{}-{}-{}", arch, vendor, os)) { - // Not cross-compiling if arch-vendor-os is all the same - // e.g. x86_64-unknown-linux-musl on x86_64-unknown-linux-gnu host - return Ok(None); - } - } - } - - // At this point we assume that we are cross compiling. - - Ok(Some(CrossCompileConfig { - lib_dir: cross_lib_dir - .ok_or("The PYO3_CROSS_LIB_DIR environment variable must be set when cross-compiling")? - .into(), - os: target_os.unwrap(), - arch: target_arch.unwrap(), - version: cross_python_version - .map(|os_string| { - os_string - .to_str() - .ok_or("PYO3_CROSS_PYTHON_VERSION is not valid utf-8.") - .map(str::to_owned) - }) - .transpose()?, - })) -} - -/// A list of python interpreter compile-time preprocessor defines that -/// we will pick up and pass to rustc via `--cfg=py_sys_config={varname}`; -/// this allows using them conditional cfg attributes in the .rs files, so -/// -/// #[cfg(py_sys_config="{varname}"] -/// -/// is the equivalent of `#ifdef {varname}` in C. -/// -/// see Misc/SpecialBuilds.txt in the python source for what these mean. -#[derive(Debug)] -struct BuildFlags(HashSet<&'static str>); - -impl BuildFlags { - const ALL: [&'static str; 5] = [ - // TODO: Remove WITH_THREAD once Python 3.6 support dropped (as it's always on). - "WITH_THREAD", - "Py_DEBUG", - "Py_REF_DEBUG", - "Py_TRACE_REFS", - "COUNT_ALLOCS", - ]; - - fn from_config_map(config_map: &HashMap) -> Self { - Self( - BuildFlags::ALL - .iter() - .copied() - .filter(|flag| config_map.get(*flag).map_or(false, |value| value == "1")) - .collect(), - ) - } - - /// Examine python's compile flags to pass to cfg by launching - /// the interpreter and printing variables of interest from - /// sysconfig.get_config_vars. - fn from_interpreter(interpreter: &Path) -> Result { - if cargo_env_var("CARGO_CFG_TARGET_OS").unwrap() == "windows" { - return Ok(Self::windows_hardcoded()); - } - - let mut script = String::from("import sysconfig\n"); - script.push_str("config = sysconfig.get_config_vars()\n"); - - for k in BuildFlags::ALL.iter() { - script.push_str(&format!("print(config.get('{}', '0'))\n", k)); - } - - let stdout = run_python_script(&interpreter, &script)?; - let split_stdout: Vec<&str> = stdout.trim_end().lines().collect(); - ensure!( - split_stdout.len() == BuildFlags::ALL.len(), - "Python stdout len didn't return expected number of lines: {}", - split_stdout.len() - ); - let flags = BuildFlags::ALL - .iter() - .zip(split_stdout) - .filter(|(_, flag_value)| *flag_value == "1") - .map(|(&flag, _)| flag) - .collect(); - - Ok(Self(flags)) - } - - fn windows_hardcoded() -> Self { - // sysconfig is missing all the flags on windows, so we can't actually - // query the interpreter directly for its build flags. - let mut flags = HashSet::new(); - flags.insert("WITH_THREAD"); - - // Uncomment these manually if your python was built with these and you want - // the cfg flags to be set in rust. - // - // map.insert("Py_DEBUG", "1"); - // map.insert("Py_REF_DEBUG", "1"); - // map.insert("Py_TRACE_REFS", "1"); - // map.insert("COUNT_ALLOCS", 1"); - Self(flags) - } - - fn abi3() -> Self { - let mut flags = HashSet::new(); - flags.insert("WITH_THREAD"); - Self(flags) - } - - fn fixup(mut self, version: PythonVersion, implementation: PythonImplementation) -> Self { - if self.0.contains("Py_DEBUG") { - self.0.insert("Py_REF_DEBUG"); - if version <= PythonVersion::PY37 { - // Py_DEBUG only implies Py_TRACE_REFS until Python 3.7 - self.0.insert("Py_TRACE_REFS"); - } - } - - // WITH_THREAD is always on for Python 3.7, and for PyPy. - if implementation == PythonImplementation::PyPy || version >= PythonVersion::PY37 { - self.0.insert("WITH_THREAD"); - } - - self - } -} - -fn parse_script_output(output: &str) -> HashMap { - output - .lines() - .filter_map(|line| { - let mut i = line.splitn(2, ' '); - Some((i.next()?.into(), i.next()?.into())) - }) - .collect() -} - -/// Parse sysconfigdata file -/// -/// The sysconfigdata is simply a dictionary containing all the build time variables used for the -/// python executable and library. Here it is read and added to a script to extract only what is -/// necessary. This necessitates a python interpreter for the host machine to work. -fn parse_sysconfigdata(config_path: impl AsRef) -> Result> { - let mut script = fs::read_to_string(config_path)?; - script += r#" -print("version_major", build_time_vars["VERSION"][0]) # 3 -print("version_minor", build_time_vars["VERSION"][2]) # E.g., 8 -KEYS = [ - "WITH_THREAD", - "Py_DEBUG", - "Py_REF_DEBUG", - "Py_TRACE_REFS", - "COUNT_ALLOCS", - "Py_ENABLE_SHARED", - "LDVERSION", - "SIZEOF_VOID_P" -] -for key in KEYS: - print(key, build_time_vars.get(key, 0)) -"#; - let output = run_python_script(&find_interpreter()?, &script)?; - - Ok(parse_script_output(&output)) -} - -fn starts_with(entry: &DirEntry, pat: &str) -> bool { - let name = entry.file_name(); - name.to_string_lossy().starts_with(pat) -} -fn ends_with(entry: &DirEntry, pat: &str) -> bool { - let name = entry.file_name(); - name.to_string_lossy().ends_with(pat) -} - -/// Finds the `_sysconfigdata*.py` file in the library path. -/// -/// From the python source for `_sysconfigdata*.py` is always going to be located at -/// `build/lib.{PLATFORM}-{PY_MINOR_VERSION}` when built from source. The [exact line][1] is defined as: -/// -/// ```py -/// pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version_info[:2]) -/// ``` -/// -/// Where get_platform returns a kebab-case formated string containing the os, the architecture and -/// possibly the os' kernel version (not the case on linux). However, when installed using a package -/// manager, the `_sysconfigdata*.py` file is installed in the `${PREFIX}/lib/python3.Y/` directory. -/// The `_sysconfigdata*.py` is generally in a sub-directory of the location of `libpython3.Y.so`. -/// So we must find the file in the following possible locations: -/// -/// ```sh -/// # distribution from package manager, lib_dir should include lib/ -/// ${INSTALL_PREFIX}/lib/python3.Y/_sysconfigdata*.py -/// ${INSTALL_PREFIX}/lib/libpython3.Y.so -/// ${INSTALL_PREFIX}/lib/python3.Y/config-3.Y-${HOST_TRIPLE}/libpython3.Y.so -/// -/// # Built from source from host -/// ${CROSS_COMPILED_LOCATION}/build/lib.linux-x86_64-Y/_sysconfigdata*.py -/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so -/// -/// # if cross compiled, kernel release is only present on certain OS targets. -/// ${CROSS_COMPILED_LOCATION}/build/lib.{OS}(-{OS-KERNEL-RELEASE})?-{ARCH}-Y/_sysconfigdata*.py -/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so -/// ``` -/// -/// [1]: https://github.com/python/cpython/blob/3.5/Lib/sysconfig.py#L389 -fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result { - let sysconfig_paths = search_lib_dir(&cross.lib_dir, &cross); - let sysconfig_name = env_var("_PYTHON_SYSCONFIGDATA_NAME"); - let mut sysconfig_paths = sysconfig_paths - .iter() - .filter_map(|p| { - let canonical = fs::canonicalize(p).ok(); - match &sysconfig_name { - Some(_) => canonical.filter(|p| p.file_stem() == sysconfig_name.as_deref()), - None => canonical, - } - }) - .collect::>(); - sysconfig_paths.dedup(); - if sysconfig_paths.is_empty() { - bail!( - "Could not find either libpython.so or _sysconfigdata*.py in {}", - cross.lib_dir.display() - ); - } else if sysconfig_paths.len() > 1 { - let mut error_msg = String::from( - "Detected multiple possible Python versions. Please set either the \ - PYO3_CROSS_PYTHON_VERSION variable to the wanted version or the \ - _PYTHON_SYSCONFIGDATA_NAME variable to the wanted sysconfigdata file name.\n\n\ - sysconfigdata files found:", - ); - for path in sysconfig_paths { - error_msg += &format!("\n\t{}", path.display()); - } - bail!("{}", error_msg); - } - - Ok(sysconfig_paths.remove(0)) -} - -/// recursive search for _sysconfigdata, returns all possibilities of sysconfigdata paths -fn search_lib_dir(path: impl AsRef, cross: &CrossCompileConfig) -> Vec { - let mut sysconfig_paths = vec![]; - let version_pat = if let Some(v) = &cross.version { - format!("python{}", v) - } else { - "python3.".into() - }; - for f in fs::read_dir(path).expect("Path does not exist") { - let sysc = match &f { - Ok(f) if starts_with(f, "_sysconfigdata") && ends_with(f, "py") => vec![f.path()], - Ok(f) if starts_with(f, "build") => search_lib_dir(f.path(), cross), - Ok(f) if starts_with(f, "lib.") => { - let name = f.file_name(); - // check if right target os - if !name.to_string_lossy().contains(if cross.os == "android" { - "linux" - } else { - &cross.os - }) { - continue; - } - // Check if right arch - if !name.to_string_lossy().contains(&cross.arch) { - continue; - } - search_lib_dir(f.path(), cross) - } - Ok(f) if starts_with(f, &version_pat) => search_lib_dir(f.path(), cross), - _ => continue, - }; - sysconfig_paths.extend(sysc); - } - sysconfig_paths -} - -/// Find cross compilation information from sysconfigdata file -/// -/// first find sysconfigdata file which follows the pattern [`_sysconfigdata_{abi}_{platform}_{multiarch}`][1] -/// on python 3.6 or greater. On python 3.5 it is simply `_sysconfigdata.py`. -/// -/// [1]: https://github.com/python/cpython/blob/3.8/Lib/sysconfig.py#L348 -fn load_cross_compile_from_sysconfigdata( - cross_compile_config: CrossCompileConfig, -) -> Result { - let sysconfig_path = find_sysconfigdata(&cross_compile_config)?; - let sysconfig_data = parse_sysconfigdata(sysconfig_path)?; - - let major = sysconfig_data.get_numeric("version_major")?; - let minor = sysconfig_data.get_numeric("version_minor")?; - let ld_version = match sysconfig_data.get("LDVERSION") { - Some(s) => s.clone(), - None => format!("{}.{}", major, minor), - }; - let calcsize_pointer = sysconfig_data.get_numeric("SIZEOF_VOID_P").ok(); - - let version = PythonVersion { major, minor }; - let implementation = PythonImplementation::CPython; - - Ok(InterpreterConfig { - version, - libdir: cross_compile_config.lib_dir.to_str().map(String::from), - shared: sysconfig_data.get_bool("Py_ENABLE_SHARED")?, - ld_version: Some(ld_version), - base_prefix: None, - executable: None, - calcsize_pointer, - implementation, - build_flags: BuildFlags::from_config_map(&sysconfig_data).fixup(version, implementation), - }) -} - -fn windows_hardcoded_cross_compile( - cross_compile_config: CrossCompileConfig, -) -> Result { - let (major, minor) = if let Some(version) = cross_compile_config.version { - let mut parts = version.split('.'); - match ( - parts.next().and_then(|major| major.parse().ok()), - parts.next().and_then(|minor| minor.parse().ok()), - parts.next(), - ) { - (Some(major), Some(minor), None) => (major, minor), - _ => bail!( - "Expected major.minor version (e.g. 3.9) for PYO3_CROSS_PYTHON_VERSION, got `{}`", - version - ), - } - } else if let Some(minor_version) = get_abi3_minor_version() { - (3, minor_version) - } else { - bail!("PYO3_CROSS_PYTHON_VERSION or an abi3-py3* feature must be specified when cross-compiling for Windows.") - }; - - Ok(InterpreterConfig { - version: PythonVersion { major, minor }, - libdir: cross_compile_config.lib_dir.to_str().map(String::from), - shared: true, - ld_version: None, - base_prefix: None, - executable: None, - calcsize_pointer: None, - implementation: PythonImplementation::CPython, - build_flags: BuildFlags::windows_hardcoded(), - }) -} - -fn load_cross_compile_info(cross_compile_config: CrossCompileConfig) -> Result { - match cargo_env_var("CARGO_CFG_TARGET_FAMILY") { - // Configure for unix platforms using the sysconfigdata file - Some(os) if os == "unix" => load_cross_compile_from_sysconfigdata(cross_compile_config), - // Use hardcoded interpreter config when targeting Windows - Some(os) if os == "windows" => windows_hardcoded_cross_compile(cross_compile_config), - // sysconfigdata works fine on wasm/wasi - Some(os) if os == "wasm" => load_cross_compile_from_sysconfigdata(cross_compile_config), - // Waiting for users to tell us what they expect on their target platform - Some(os) => bail!( - "Unsupported target OS family for cross-compilation: {:?}", - os - ), - // Unknown os family - try to do something useful - None => load_cross_compile_from_sysconfigdata(cross_compile_config), - } -} - -/// Run a python script using the specified interpreter binary. -fn run_python_script(interpreter: &Path, script: &str) -> Result { - let out = Command::new(interpreter) - .env("PYTHONIOENCODING", "utf-8") - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::inherit()) - .spawn() - .and_then(|mut child| { - use std::io::Write; - child - .stdin - .as_mut() - .expect("piped stdin") - .write_all(script.as_bytes())?; - child.wait_with_output() - }); - - match out { - Err(err) => bail!( - "failed to run the Python interpreter at {}: {}", - interpreter.display(), - err - ), - Ok(ok) if !ok.status.success() => bail!("Python script failed"), - Ok(ok) => Ok(String::from_utf8(ok.stdout)?), - } -} - fn get_rustc_link_lib(config: &InterpreterConfig) -> Result { - let link_name = if cargo_env_var("CARGO_CFG_TARGET_OS").unwrap() == "windows" { - if is_abi3() { + let link_name = if env::var_os("CARGO_CFG_TARGET_OS").unwrap() == "windows" { + if config.abi3 { // Link against python3.lib for the stable ABI on Windows. // See https://www.python.org/dev/peps/pep-0384/#linkage // // This contains only the limited ABI symbols. "pythonXY:python3".to_owned() - } else if cargo_env_var("CARGO_CFG_TARGET_ENV").unwrap() == "gnu" { + } else if env::var_os("CARGO_CFG_TARGET_ENV").unwrap() == "gnu" { // https://packages.msys2.org/base/mingw-w64-python format!( "pythonXY:python{}.{}", @@ -601,7 +28,9 @@ fn get_rustc_link_lib(config: &InterpreterConfig) -> Result { match config.implementation { PythonImplementation::CPython => match &config.ld_version { Some(ld_version) => format!("python{}", ld_version), - None => bail!("failed to configure `ld_version` when compiling for unix"), + None => { + return Err("failed to configure `ld_version` when compiling for unix".into()) + } }, PythonImplementation::PyPy => format!("pypy{}-c", config.version.major), } @@ -614,140 +43,6 @@ fn get_rustc_link_lib(config: &InterpreterConfig) -> Result { )) } -fn get_venv_path() -> Option { - match (env_var("VIRTUAL_ENV"), env_var("CONDA_PREFIX")) { - (Some(dir), None) => Some(PathBuf::from(dir)), - (None, Some(dir)) => Some(PathBuf::from(dir)), - (Some(_), Some(_)) => { - warn!( - "Both VIRTUAL_ENV and CONDA_PREFIX are set. PyO3 will ignore both of these for \ - locating the Python interpreter until you unset one of them." - ); - None - } - (None, None) => None, - } -} - -/// Attempts to locate a python interpreter. Locations are checked in the order listed: -/// 1. If `PYO3_PYTHON` is set, this intepreter is used. -/// 2. If in a virtualenv, that environment's interpreter is used. -/// 3. `python`, if this is functional a Python 3.x interpreter -/// 4. `python3`, as above -fn find_interpreter() -> Result { - if let Some(exe) = env_var("PYO3_PYTHON") { - Ok(exe.into()) - } else if let Some(venv_path) = get_venv_path() { - match cargo_env_var("CARGO_CFG_TARGET_OS").unwrap().as_str() { - "windows" => Ok(venv_path.join("Scripts\\python")), - _ => Ok(venv_path.join("bin/python")), - } - } else { - println!("cargo:rerun-if-env-changed=PATH"); - ["python", "python3"] - .iter() - .find(|bin| { - if let Ok(out) = Command::new(bin).arg("--version").output() { - // begin with `Python 3.X.X :: additional info` - out.stdout.starts_with(b"Python 3") || out.stderr.starts_with(b"Python 3") - } else { - false - } - }) - .map(PathBuf::from) - .ok_or_else(|| "no Python 3.x interpreter found".into()) - } -} - -/// Extract compilation vars from the specified interpreter. -fn get_config_from_interpreter(interpreter: &Path) -> Result { - let script = r#" -# Allow the script to run on Python 2, so that nicer error can be printed later. -from __future__ import print_function - -import os.path -import platform -import struct -import sys -from sysconfig import get_config_var - -PYPY = platform.python_implementation() == "PyPy" - -# sys.base_prefix is missing on Python versions older than 3.3; this allows the script to continue -# so that the version mismatch can be reported in a nicer way later. -base_prefix = getattr(sys, "base_prefix", None) - -if base_prefix: - # Anaconda based python distributions have a static python executable, but include - # the shared library. Use the shared library for embedding to avoid rust trying to - # LTO the static library (and failing with newer gcc's, because it is old). - ANACONDA = os.path.exists(os.path.join(base_prefix, "conda-meta")) -else: - ANACONDA = False - -def print_if_set(varname, value): - if value is not None: - print(varname, value) - -libdir = get_config_var("LIBDIR") - -print("version_major", sys.version_info[0]) -print("version_minor", sys.version_info[1]) -print("implementation", platform.python_implementation()) -print_if_set("libdir", libdir) -print_if_set("ld_version", get_config_var("LDVERSION")) -print_if_set("base_prefix", base_prefix) -print("framework", bool(get_config_var("PYTHONFRAMEWORK"))) -print("shared", PYPY or ANACONDA or bool(get_config_var("Py_ENABLE_SHARED"))) -print("executable", sys.executable) -print("calcsize_pointer", struct.calcsize("P")) -"#; - let output = run_python_script(interpreter, script)?; - let map: HashMap = parse_script_output(&output); - let shared = match ( - cargo_env_var("CARGO_CFG_TARGET_OS").unwrap().as_str(), - map["framework"].as_str(), - map["shared"].as_str(), - ) { - (_, _, "True") // Py_ENABLE_SHARED is set - | ("windows", _, _) // Windows always uses shared linking - | ("macos", "True", _) // MacOS framework package uses shared linking - => true, - (_, _, "False") => false, // Any other platform, Py_ENABLE_SHARED not set - _ => bail!("Unrecognised link model combination") - }; - - let version = PythonVersion { - major: map["version_major"].parse()?, - minor: map["version_minor"].parse()?, - }; - - let implementation = map["implementation"].parse()?; - - Ok(InterpreterConfig { - version, - implementation, - libdir: map.get("libdir").cloned(), - shared, - ld_version: map.get("ld_version").cloned(), - base_prefix: map.get("base_prefix").cloned(), - executable: map.get("executable").cloned(), - calcsize_pointer: Some(map["calcsize_pointer"].parse()?), - build_flags: BuildFlags::from_interpreter(interpreter)?.fixup(version, implementation), - }) -} - -fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> { - ensure!( - interpreter_config.version >= MINIMUM_SUPPORTED_VERSION, - "the configured Python interpreter version ({}) is lower than PyO3's minimum supported version ({})", - interpreter_config.version, - MINIMUM_SUPPORTED_VERSION, - ); - - Ok(()) -} - fn rustc_minor_version() -> Option { let rustc = env::var_os("RUSTC")?; let output = Command::new(rustc).arg("--version").output().ok()?; @@ -760,8 +55,8 @@ fn rustc_minor_version() -> Option { } fn emit_cargo_configuration(interpreter_config: &InterpreterConfig) -> Result<()> { - let target_os = cargo_env_var("CARGO_CFG_TARGET_OS").unwrap(); - let is_extension_module = cargo_env_var("CARGO_FEATURE_EXTENSION_MODULE").is_some(); + let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap(); + let is_extension_module = env::var_os("CARGO_FEATURE_EXTENSION_MODULE").is_some(); match (is_extension_module, target_os.as_str()) { (_, "windows") => { // always link on windows, even with extension module @@ -802,69 +97,41 @@ fn emit_cargo_configuration(interpreter_config: &InterpreterConfig) -> Result<() } if env::var_os("CARGO_FEATURE_AUTO_INITIALIZE").is_some() { - ensure!( - interpreter_config.shared, - "The `auto-initialize` feature is enabled, but your python installation only supports \ - embedding the Python interpreter statically. If you are attempting to run tests, or a \ - binary which is okay to link dynamically, install a Python distribution which ships \ - with the Python shared library.\n\ - \n\ - Embedding the Python interpreter statically does not yet have first-class support in \ - PyO3. If you are sure you intend to do this, disable the `auto-initialize` feature.\n\ - \n\ - For more information, see \ - https://pyo3.rs/v{pyo3_version}/\ - building_and_distribution.html#embedding-python-in-rust", - pyo3_version = env::var("CARGO_PKG_VERSION").unwrap() - ); + if !interpreter_config.shared { + return Err(format!( + "The `auto-initialize` feature is enabled, but your python installation only supports \ + embedding the Python interpreter statically. If you are attempting to run tests, or a \ + binary which is okay to link dynamically, install a Python distribution which ships \ + with the Python shared library.\n\ + \n\ + Embedding the Python interpreter statically does not yet have first-class support in \ + PyO3. If you are sure you intend to do this, disable the `auto-initialize` feature.\n\ + \n\ + For more information, see \ + https://pyo3.rs/v{pyo3_version}/\ + building_and_distribution.html#embedding-python-in-rust", + pyo3_version = env::var("CARGO_PKG_VERSION").unwrap() + ) + .into()); + } // TODO: PYO3_CI env is a hack to workaround CI with PyPy, where the `dev-dependencies` // currently cause `auto-initialize` to be enabled in CI. // Once cargo's `resolver = "2"` is stable (~ MSRV Rust 1.52), remove this. - ensure!( - !interpreter_config.is_pypy() || env::var_os("PYO3_CI").is_some(), - "The `auto-initialize` feature is not supported with PyPy." - ); - } - - let is_abi3 = is_abi3(); - - if interpreter_config.is_pypy() { - println!("cargo:rustc-cfg=PyPy"); - if is_abi3 { - warn!( - "PyPy does not yet support abi3 so the build artifacts will be version-specific. \ - See https://foss.heptapod.net/pypy/pypy/-/issues/3397 for more information." - ) - } - }; - - let minor = if is_abi3 { - println!("cargo:rustc-cfg=Py_LIMITED_API"); - // Check any `abi3-py3*` feature is set. If not, use the interpreter version. - match get_abi3_minor_version() { - Some(minor) if minor > interpreter_config.version.minor => bail!( - "You cannot set a mininimum Python version 3.{} higher than the interpreter version 3.{}", - minor, - interpreter_config.version.minor - ), - Some(minor) => minor, - None => interpreter_config.version.minor + if interpreter_config.is_pypy() && env::var_os("PYO3_CI").is_none() { + return Err("The `auto-initialize` feature is not supported with PyPy.".into()); } - } else { - interpreter_config.version.minor - }; - - for i in MINIMUM_SUPPORTED_VERSION.minor..=minor { - println!("cargo:rustc-cfg=Py_3_{}", i); } - for flag in &interpreter_config.build_flags.0 { - println!("cargo:rustc-cfg=py_sys_config=\"{}\"", flag) - } + Ok(()) +} - // Enable use of const generics on Rust 1.51 and greater +fn configure_pyo3() -> Result<()> { + let cfg = pyo3_build_config::get(); + emit_cargo_configuration(&cfg)?; + cfg.emit_pyo3_cfgs(); + // Enable use of const generics on Rust 1.51 and greater if rustc_minor_version().unwrap_or(0) >= 51 { println!("cargo:rustc-cfg=min_const_generics"); } @@ -872,83 +139,6 @@ fn emit_cargo_configuration(interpreter_config: &InterpreterConfig) -> Result<() Ok(()) } -fn ensure_target_architecture(interpreter_config: &InterpreterConfig) -> Result<()> { - // Try to check whether the target architecture matches the python library - let rust_target = match cargo_env_var("CARGO_CFG_TARGET_POINTER_WIDTH") - .unwrap() - .as_str() - { - "64" => "64-bit", - "32" => "32-bit", - x => bail!("unexpected Rust target pointer width: {}", x), - }; - - // The reason we don't use platform.architecture() here is that it's not - // reliable on macOS. See https://stackoverflow.com/a/1405971/823869. - // Similarly, sys.maxsize is not reliable on Windows. See - // https://stackoverflow.com/questions/1405913/how-do-i-determine-if-my-python-shell-is-executing-in-32bit-or-64bit-mode-on-os/1405971#comment6209952_1405971 - // and https://stackoverflow.com/a/3411134/823869. - let python_target = match interpreter_config.calcsize_pointer { - Some(8) => "64-bit", - Some(4) => "32-bit", - None => { - // Unset, e.g. because we're cross-compiling. Don't check anything - // in this case. - return Ok(()); - } - Some(n) => bail!("unexpected Python calcsize_pointer value: {}", n), - }; - - ensure!( - rust_target == python_target, - "Your Rust target architecture ({}) does not match your python interpreter ({})", - rust_target, - python_target - ); - - Ok(()) -} - -fn get_abi3_minor_version() -> Option { - (MINIMUM_SUPPORTED_VERSION.minor..=ABI3_MAX_MINOR) - .find(|i| cargo_env_var(&format!("CARGO_FEATURE_ABI3_PY3{}", i)).is_some()) -} - -fn get_interpreter_config() -> Result { - // If PYO3_NO_PYTHON is set with abi3, we can build PyO3 without calling Python. - if let Some(abi3_version) = get_abi3_minor_version() { - if env_var("PYO3_NO_PYTHON").is_some() { - return Ok(InterpreterConfig { - version: PythonVersion { - major: 3, - minor: abi3_version, - }, - implementation: PythonImplementation::CPython, - libdir: None, - build_flags: BuildFlags::abi3(), - base_prefix: None, - calcsize_pointer: None, - executable: None, - ld_version: None, - shared: true, - }); - } - } - - if let Some(paths) = cross_compiling()? { - load_cross_compile_info(paths) - } else { - get_config_from_interpreter(&find_interpreter()?) - } -} - -fn configure_pyo3() -> Result<()> { - let interpreter_config = get_interpreter_config()?; - ensure_python_version(&interpreter_config)?; - ensure_target_architecture(&interpreter_config)?; - emit_cargo_configuration(&interpreter_config) -} - fn main() { // Print out error messages using display, to get nicer formatting. if let Err(e) = configure_pyo3() { diff --git a/examples/pyo3-pytests/Cargo.toml b/examples/pyo3-pytests/Cargo.toml index 41334f2c03a..8fdcda88bf2 100644 --- a/examples/pyo3-pytests/Cargo.toml +++ b/examples/pyo3-pytests/Cargo.toml @@ -6,10 +6,10 @@ description = "Python-based tests for PyO3" edition = "2018" [dependencies] +pyo3 = { path = "../../", features = ["extension-module"] } -[dependencies.pyo3] -path = "../../" -features = ["extension-module"] +[build-dependencies] +pyo3-build-config = { path = "../../pyo3-build-config" } [lib] name = "pyo3_pytests" diff --git a/examples/pyo3-pytests/build.rs b/examples/pyo3-pytests/build.rs index edd8b0ceb53..0475124bb4e 100644 --- a/examples/pyo3-pytests/build.rs +++ b/examples/pyo3-pytests/build.rs @@ -1,28 +1,3 @@ -use std::process::Command; - fn main() { - let out = Command::new("python") - .args(&["-c", "import sys; import platform; print(sys.version_info[1]); print(platform.python_implementation())"]) - .output() - .expect("python version did not print"); - - let output = String::from_utf8_lossy(&out.stdout); - let mut lines = output.trim().lines(); - - println!("{}", output); - - let version: u8 = lines - .next() - .unwrap() - .parse() - .expect("python version was not parsed"); - let implementation = lines.next().unwrap(); - - for each in 6..version { - println!("cargo:rustc-cfg=Py_3_{}", each); - } - - if implementation == "PyPy" { - println!("cargo:rustc-cfg=PyPy"); - } + pyo3_build_config::use_pyo3_cfgs(); } diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml new file mode 100644 index 00000000000..43cec7af06f --- /dev/null +++ b/pyo3-build-config/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "pyo3-build-config" +version = "0.14.0-alpha.0" +description = "Build configuration for the PyO3 ecosystem" +authors = ["PyO3 Project and Contributors "] +keywords = ["pyo3", "python", "cpython", "ffi"] +homepage = "https://github.com/pyo3/pyo3" +repository = "https://github.com/pyo3/pyo3" +categories = ["api-bindings", "development-tools::ffi"] +license = "Apache-2.0" +edition = "2018" + +[dependencies] + +[features] +default = [] + +abi3 = [] +abi3-py36 = ["abi3-py37"] +abi3-py37 = ["abi3-py38"] +abi3-py38 = ["abi3-py39"] +abi3-py39 = ["abi3"] diff --git a/pyo3-build-config/LICENSE b/pyo3-build-config/LICENSE new file mode 120000 index 00000000000..ea5b60640b0 --- /dev/null +++ b/pyo3-build-config/LICENSE @@ -0,0 +1 @@ +../LICENSE \ No newline at end of file diff --git a/pyo3-build-config/build.rs b/pyo3-build-config/build.rs new file mode 100644 index 00000000000..3904aeb5157 --- /dev/null +++ b/pyo3-build-config/build.rs @@ -0,0 +1,11 @@ +#[allow(dead_code)] +#[path = "src/impl_.rs"] +mod impl_; + +fn main() { + // Print out error messages using display, to get nicer formatting. + if let Err(e) = impl_::configure() { + eprintln!("error: {}", e); + std::process::exit(1) + } +} diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs new file mode 100644 index 00000000000..a1964614557 --- /dev/null +++ b/pyo3-build-config/src/impl_.rs @@ -0,0 +1,882 @@ +use std::{ + collections::{HashMap, HashSet}, + convert::AsRef, + env, + ffi::OsString, + fmt::Display, + fs::{self, DirEntry, File}, + io::Write, + path::{Path, PathBuf}, + process::{Command, Stdio}, + str::FromStr, +}; + +/// Minimum Python version PyO3 supports. +const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 6 }; +/// Maximum Python version that can be used as minimum required Python version with abi3. +const ABI3_MAX_MINOR: u8 = 9; + +type Result = std::result::Result>; + +// A simple macro for returning an error. Resembles anyhow::bail. +macro_rules! bail { + ($msg: expr) => { return Err($msg.into()); }; + ($fmt: literal $($args: tt)+) => { return Err(format!($fmt $($args)+).into()); }; +} + +// A simple macro for checking a condition. Resembles anyhow::ensure. +macro_rules! ensure { + ($condition:expr, $($args: tt)+) => { if !($condition) { bail!($($args)+) } }; +} + +// Show warning. If needed, please extend this macro to support arguments. +macro_rules! warn { + ($msg: literal) => { + println!(concat!("cargo:warning=", $msg)); + }; +} + +/// Gets an environment variable owned by cargo. +/// +/// Environment variables set by cargo are expected to be valid UTF8. +fn cargo_env_var(var: &str) -> Option { + env::var_os(var).map(|os_string| os_string.to_str().unwrap().into()) +} + +/// Gets an external environment variable, and registers the build script to rerun if +/// the variable changes. +fn env_var(var: &str) -> Option { + println!("cargo:rerun-if-env-changed={}", var); + env::var_os(var) +} + +/// Configuration needed by PyO3 to build for the correct Python implementation. +/// +/// Usually this is queried directly from the Python interpreter. When the `PYO3_NO_PYTHON` variable +/// is set, or during cross compile situations, then alternative strategies are used to populate +/// this type. +#[derive(Debug)] +pub struct InterpreterConfig { + pub version: PythonVersion, + pub libdir: Option, + pub shared: bool, + pub abi3: bool, + pub ld_version: Option, + pub base_prefix: Option, + pub executable: Option, + pub calcsize_pointer: Option, + pub implementation: PythonImplementation, + pub build_flags: BuildFlags, +} + +impl InterpreterConfig { + pub fn emit_pyo3_cfgs(&self) { + // This should have been checked during pyo3-build-config build time. + assert!(self.version >= MINIMUM_SUPPORTED_VERSION); + for i in MINIMUM_SUPPORTED_VERSION.minor..=self.version.minor { + println!("cargo:rustc-cfg=Py_3_{}", i); + } + + if self.abi3 { + println!("cargo:rustc-cfg=Py_LIMITED_API"); + } + + if self.is_pypy() { + println!("cargo:rustc-cfg=PyPy"); + if self.abi3 { + warn!( + "PyPy does not yet support abi3 so the build artifacts will be version-specific. \ + See https://foss.heptapod.net/pypy/pypy/-/issues/3397 for more information." + ) + } + }; + + for flag in &self.build_flags.0 { + println!("cargo:rustc-cfg=py_sys_config=\"{}\"", flag) + } + } + + pub fn is_pypy(&self) -> bool { + self.implementation == PythonImplementation::PyPy + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct PythonVersion { + pub major: u8, + pub minor: u8, +} + +impl PythonVersion { + const PY37: Self = PythonVersion { major: 3, minor: 7 }; +} + +impl Display for PythonVersion { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}.{}", self.major, self.minor) + } +} + +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum PythonImplementation { + CPython, + PyPy, +} + +impl FromStr for PythonImplementation { + type Err = Box; + fn from_str(s: &str) -> Result { + match s { + "CPython" => Ok(PythonImplementation::CPython), + "PyPy" => Ok(PythonImplementation::PyPy), + _ => bail!("Invalid interpreter: {}", s), + } + } +} + +fn is_abi3() -> bool { + cargo_env_var("CARGO_FEATURE_ABI3").is_some() +} + +trait GetPrimitive { + fn get_bool(&self, key: &str) -> Result; + fn get_numeric(&self, key: &str) -> Result; +} + +impl GetPrimitive for HashMap { + fn get_bool(&self, key: &str) -> Result { + match self + .get(key) + .map(|x| x.as_str()) + .ok_or(format!("{} is not defined", key))? + { + "1" | "true" | "True" => Ok(true), + "0" | "false" | "False" => Ok(false), + _ => bail!("{} must be a bool (1/true/True or 0/false/False", key), + } + } + + fn get_numeric(&self, key: &str) -> Result { + self.get(key) + .ok_or(format!("{} is not defined", key))? + .parse::() + .map_err(|_| format!("Could not parse value of {}", key).into()) + } +} + +struct CrossCompileConfig { + lib_dir: PathBuf, + version: Option, + os: String, + arch: String, +} + +fn cross_compiling() -> Result> { + let cross = env_var("PYO3_CROSS"); + let cross_lib_dir = env_var("PYO3_CROSS_LIB_DIR"); + let cross_python_version = env_var("PYO3_CROSS_PYTHON_VERSION"); + + let target_arch = cargo_env_var("CARGO_CFG_TARGET_ARCH"); + let target_vendor = cargo_env_var("CARGO_CFG_TARGET_VENDOR"); + let target_os = cargo_env_var("CARGO_CFG_TARGET_OS"); + + if cross.is_none() && cross_lib_dir.is_none() && cross_python_version.is_none() { + // No cross-compiling environment variables set; try to determine if this is a known case + // which is not cross-compilation. + + let target = cargo_env_var("TARGET").unwrap(); + let host = cargo_env_var("HOST").unwrap(); + if target == host { + // Not cross-compiling + return Ok(None); + } + + if target == "i686-pc-windows-msvc" && host == "x86_64-pc-windows-msvc" { + // Not cross-compiling to compile for 32-bit Python from windows 64-bit + return Ok(None); + } + + if target == "x86_64-apple-darwin" && host == "aarch64-apple-darwin" { + // Not cross-compiling to compile for x86-64 Python from macOS arm64 + return Ok(None); + } + + if target == "aarch64-apple-darwin" && host == "x86_64-apple-darwin" { + // Not cross-compiling to compile for arm64 Python from macOS x86_64 + return Ok(None); + } + + if let (Some(arch), Some(vendor), Some(os)) = (&target_arch, &target_vendor, &target_os) { + if host.starts_with(&format!("{}-{}-{}", arch, vendor, os)) { + // Not cross-compiling if arch-vendor-os is all the same + // e.g. x86_64-unknown-linux-musl on x86_64-unknown-linux-gnu host + return Ok(None); + } + } + } + + // At this point we assume that we are cross compiling. + + Ok(Some(CrossCompileConfig { + lib_dir: cross_lib_dir + .ok_or("The PYO3_CROSS_LIB_DIR environment variable must be set when cross-compiling")? + .into(), + os: target_os.unwrap(), + arch: target_arch.unwrap(), + version: cross_python_version + .map(|os_string| { + os_string + .to_str() + .ok_or("PYO3_CROSS_PYTHON_VERSION is not valid utf-8.") + .map(str::to_owned) + }) + .transpose()?, + })) +} + +/// A list of python interpreter compile-time preprocessor defines that +/// we will pick up and pass to rustc via `--cfg=py_sys_config={varname}`; +/// this allows using them conditional cfg attributes in the .rs files, so +/// +/// #[cfg(py_sys_config="{varname}"] +/// +/// is the equivalent of `#ifdef {varname}` in C. +/// +/// see Misc/SpecialBuilds.txt in the python source for what these mean. +#[derive(Debug)] +pub struct BuildFlags(pub HashSet<&'static str>); + +impl BuildFlags { + const ALL: [&'static str; 5] = [ + // TODO: Remove WITH_THREAD once Python 3.6 support dropped (as it's always on). + "WITH_THREAD", + "Py_DEBUG", + "Py_REF_DEBUG", + "Py_TRACE_REFS", + "COUNT_ALLOCS", + ]; + + fn from_config_map(config_map: &HashMap) -> Self { + Self( + BuildFlags::ALL + .iter() + .copied() + .filter(|flag| config_map.get(*flag).map_or(false, |value| value == "1")) + .collect(), + ) + } + + /// Examine python's compile flags to pass to cfg by launching + /// the interpreter and printing variables of interest from + /// sysconfig.get_config_vars. + fn from_interpreter(interpreter: &Path) -> Result { + if cargo_env_var("CARGO_CFG_TARGET_OS").unwrap() == "windows" { + return Ok(Self::windows_hardcoded()); + } + + let mut script = String::from("import sysconfig\n"); + script.push_str("config = sysconfig.get_config_vars()\n"); + + for k in BuildFlags::ALL.iter() { + script.push_str(&format!("print(config.get('{}', '0'))\n", k)); + } + + let stdout = run_python_script(&interpreter, &script)?; + let split_stdout: Vec<&str> = stdout.trim_end().lines().collect(); + ensure!( + split_stdout.len() == BuildFlags::ALL.len(), + "Python stdout len didn't return expected number of lines: {}", + split_stdout.len() + ); + let flags = BuildFlags::ALL + .iter() + .zip(split_stdout) + .filter(|(_, flag_value)| *flag_value == "1") + .map(|(&flag, _)| flag) + .collect(); + + Ok(Self(flags)) + } + + fn windows_hardcoded() -> Self { + // sysconfig is missing all the flags on windows, so we can't actually + // query the interpreter directly for its build flags. + let mut flags = HashSet::new(); + flags.insert("WITH_THREAD"); + + // Uncomment these manually if your python was built with these and you want + // the cfg flags to be set in rust. + // + // map.insert("Py_DEBUG", "1"); + // map.insert("Py_REF_DEBUG", "1"); + // map.insert("Py_TRACE_REFS", "1"); + // map.insert("COUNT_ALLOCS", 1"); + Self(flags) + } + + fn abi3() -> Self { + let mut flags = HashSet::new(); + flags.insert("WITH_THREAD"); + Self(flags) + } + + fn fixup(mut self, version: PythonVersion, implementation: PythonImplementation) -> Self { + if self.0.contains("Py_DEBUG") { + self.0.insert("Py_REF_DEBUG"); + if version <= PythonVersion::PY37 { + // Py_DEBUG only implies Py_TRACE_REFS until Python 3.7 + self.0.insert("Py_TRACE_REFS"); + } + } + + // WITH_THREAD is always on for Python 3.7, and for PyPy. + if implementation == PythonImplementation::PyPy || version >= PythonVersion::PY37 { + self.0.insert("WITH_THREAD"); + } + + self + } +} + +fn parse_script_output(output: &str) -> HashMap { + output + .lines() + .filter_map(|line| { + let mut i = line.splitn(2, ' '); + Some((i.next()?.into(), i.next()?.into())) + }) + .collect() +} + +/// Parse sysconfigdata file +/// +/// The sysconfigdata is simply a dictionary containing all the build time variables used for the +/// python executable and library. Here it is read and added to a script to extract only what is +/// necessary. This necessitates a python interpreter for the host machine to work. +fn parse_sysconfigdata(config_path: impl AsRef) -> Result> { + let mut script = fs::read_to_string(config_path)?; + script += r#" +print("version_major", build_time_vars["VERSION"][0]) # 3 +print("version_minor", build_time_vars["VERSION"][2]) # E.g., 8 +KEYS = [ + "WITH_THREAD", + "Py_DEBUG", + "Py_REF_DEBUG", + "Py_TRACE_REFS", + "COUNT_ALLOCS", + "Py_ENABLE_SHARED", + "LDVERSION", + "SIZEOF_VOID_P" +] +for key in KEYS: + print(key, build_time_vars.get(key, 0)) +"#; + let output = run_python_script(&find_interpreter()?, &script)?; + + Ok(parse_script_output(&output)) +} + +fn starts_with(entry: &DirEntry, pat: &str) -> bool { + let name = entry.file_name(); + name.to_string_lossy().starts_with(pat) +} +fn ends_with(entry: &DirEntry, pat: &str) -> bool { + let name = entry.file_name(); + name.to_string_lossy().ends_with(pat) +} + +/// Finds the `_sysconfigdata*.py` file in the library path. +/// +/// From the python source for `_sysconfigdata*.py` is always going to be located at +/// `build/lib.{PLATFORM}-{PY_MINOR_VERSION}` when built from source. The [exact line][1] is defined as: +/// +/// ```py +/// pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version_info[:2]) +/// ``` +/// +/// Where get_platform returns a kebab-case formated string containing the os, the architecture and +/// possibly the os' kernel version (not the case on linux). However, when installed using a package +/// manager, the `_sysconfigdata*.py` file is installed in the `${PREFIX}/lib/python3.Y/` directory. +/// The `_sysconfigdata*.py` is generally in a sub-directory of the location of `libpython3.Y.so`. +/// So we must find the file in the following possible locations: +/// +/// ```sh +/// # distribution from package manager, lib_dir should include lib/ +/// ${INSTALL_PREFIX}/lib/python3.Y/_sysconfigdata*.py +/// ${INSTALL_PREFIX}/lib/libpython3.Y.so +/// ${INSTALL_PREFIX}/lib/python3.Y/config-3.Y-${HOST_TRIPLE}/libpython3.Y.so +/// +/// # Built from source from host +/// ${CROSS_COMPILED_LOCATION}/build/lib.linux-x86_64-Y/_sysconfigdata*.py +/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so +/// +/// # if cross compiled, kernel release is only present on certain OS targets. +/// ${CROSS_COMPILED_LOCATION}/build/lib.{OS}(-{OS-KERNEL-RELEASE})?-{ARCH}-Y/_sysconfigdata*.py +/// ${CROSS_COMPILED_LOCATION}/libpython3.Y.so +/// ``` +/// +/// [1]: https://github.com/python/cpython/blob/3.5/Lib/sysconfig.py#L389 +fn find_sysconfigdata(cross: &CrossCompileConfig) -> Result { + let sysconfig_paths = search_lib_dir(&cross.lib_dir, &cross); + let sysconfig_name = env_var("_PYTHON_SYSCONFIGDATA_NAME"); + let mut sysconfig_paths = sysconfig_paths + .iter() + .filter_map(|p| { + let canonical = fs::canonicalize(p).ok(); + match &sysconfig_name { + Some(_) => canonical.filter(|p| p.file_stem() == sysconfig_name.as_deref()), + None => canonical, + } + }) + .collect::>(); + sysconfig_paths.dedup(); + if sysconfig_paths.is_empty() { + bail!( + "Could not find either libpython.so or _sysconfigdata*.py in {}", + cross.lib_dir.display() + ); + } else if sysconfig_paths.len() > 1 { + let mut error_msg = String::from( + "Detected multiple possible Python versions. Please set either the \ + PYO3_CROSS_PYTHON_VERSION variable to the wanted version or the \ + _PYTHON_SYSCONFIGDATA_NAME variable to the wanted sysconfigdata file name.\n\n\ + sysconfigdata files found:", + ); + for path in sysconfig_paths { + error_msg += &format!("\n\t{}", path.display()); + } + bail!("{}", error_msg); + } + + Ok(sysconfig_paths.remove(0)) +} + +/// recursive search for _sysconfigdata, returns all possibilities of sysconfigdata paths +fn search_lib_dir(path: impl AsRef, cross: &CrossCompileConfig) -> Vec { + let mut sysconfig_paths = vec![]; + let version_pat = if let Some(v) = &cross.version { + format!("python{}", v) + } else { + "python3.".into() + }; + for f in fs::read_dir(path).expect("Path does not exist") { + let sysc = match &f { + Ok(f) if starts_with(f, "_sysconfigdata") && ends_with(f, "py") => vec![f.path()], + Ok(f) if starts_with(f, "build") => search_lib_dir(f.path(), cross), + Ok(f) if starts_with(f, "lib.") => { + let name = f.file_name(); + // check if right target os + if !name.to_string_lossy().contains(if cross.os == "android" { + "linux" + } else { + &cross.os + }) { + continue; + } + // Check if right arch + if !name.to_string_lossy().contains(&cross.arch) { + continue; + } + search_lib_dir(f.path(), cross) + } + Ok(f) if starts_with(f, &version_pat) => search_lib_dir(f.path(), cross), + _ => continue, + }; + sysconfig_paths.extend(sysc); + } + sysconfig_paths +} + +/// Find cross compilation information from sysconfigdata file +/// +/// first find sysconfigdata file which follows the pattern [`_sysconfigdata_{abi}_{platform}_{multiarch}`][1] +/// on python 3.6 or greater. On python 3.5 it is simply `_sysconfigdata.py`. +/// +/// [1]: https://github.com/python/cpython/blob/3.8/Lib/sysconfig.py#L348 +fn load_cross_compile_from_sysconfigdata( + cross_compile_config: CrossCompileConfig, +) -> Result { + let sysconfig_path = find_sysconfigdata(&cross_compile_config)?; + let sysconfig_data = parse_sysconfigdata(sysconfig_path)?; + + let major = sysconfig_data.get_numeric("version_major")?; + let minor = sysconfig_data.get_numeric("version_minor")?; + let ld_version = match sysconfig_data.get("LDVERSION") { + Some(s) => s.clone(), + None => format!("{}.{}", major, minor), + }; + let calcsize_pointer = sysconfig_data.get_numeric("SIZEOF_VOID_P").ok(); + + let version = PythonVersion { major, minor }; + let implementation = PythonImplementation::CPython; + + Ok(InterpreterConfig { + version, + libdir: cross_compile_config.lib_dir.to_str().map(String::from), + shared: sysconfig_data.get_bool("Py_ENABLE_SHARED")?, + abi3: is_abi3(), + ld_version: Some(ld_version), + base_prefix: None, + executable: None, + calcsize_pointer, + implementation, + build_flags: BuildFlags::from_config_map(&sysconfig_data).fixup(version, implementation), + }) +} + +fn windows_hardcoded_cross_compile( + cross_compile_config: CrossCompileConfig, +) -> Result { + let (major, minor) = if let Some(version) = cross_compile_config.version { + let mut parts = version.split('.'); + match ( + parts.next().and_then(|major| major.parse().ok()), + parts.next().and_then(|minor| minor.parse().ok()), + parts.next(), + ) { + (Some(major), Some(minor), None) => (major, minor), + _ => bail!( + "Expected major.minor version (e.g. 3.9) for PYO3_CROSS_PYTHON_VERSION, got `{}`", + version + ), + } + } else if let Some(minor_version) = get_abi3_minor_version() { + (3, minor_version) + } else { + bail!("PYO3_CROSS_PYTHON_VERSION or an abi3-py3* feature must be specified when cross-compiling for Windows.") + }; + + Ok(InterpreterConfig { + version: PythonVersion { major, minor }, + libdir: cross_compile_config.lib_dir.to_str().map(String::from), + shared: true, + abi3: is_abi3(), + ld_version: None, + base_prefix: None, + executable: None, + calcsize_pointer: None, + implementation: PythonImplementation::CPython, + build_flags: BuildFlags::windows_hardcoded(), + }) +} + +fn load_cross_compile_info(cross_compile_config: CrossCompileConfig) -> Result { + match cargo_env_var("CARGO_CFG_TARGET_FAMILY") { + // Configure for unix platforms using the sysconfigdata file + Some(os) if os == "unix" => load_cross_compile_from_sysconfigdata(cross_compile_config), + // Use hardcoded interpreter config when targeting Windows + Some(os) if os == "windows" => windows_hardcoded_cross_compile(cross_compile_config), + // sysconfigdata works fine on wasm/wasi + Some(os) if os == "wasm" => load_cross_compile_from_sysconfigdata(cross_compile_config), + // Waiting for users to tell us what they expect on their target platform + Some(os) => bail!( + "Unsupported target OS family for cross-compilation: {:?}", + os + ), + // Unknown os family - try to do something useful + None => load_cross_compile_from_sysconfigdata(cross_compile_config), + } +} + +/// Run a python script using the specified interpreter binary. +fn run_python_script(interpreter: &Path, script: &str) -> Result { + let out = Command::new(interpreter) + .env("PYTHONIOENCODING", "utf-8") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() + .and_then(|mut child| { + child + .stdin + .as_mut() + .expect("piped stdin") + .write_all(script.as_bytes())?; + child.wait_with_output() + }); + + match out { + Err(err) => bail!( + "failed to run the Python interpreter at {}: {}", + interpreter.display(), + err + ), + Ok(ok) if !ok.status.success() => bail!("Python script failed"), + Ok(ok) => Ok(String::from_utf8(ok.stdout)?), + } +} + +fn get_venv_path() -> Option { + match (env_var("VIRTUAL_ENV"), env_var("CONDA_PREFIX")) { + (Some(dir), None) => Some(PathBuf::from(dir)), + (None, Some(dir)) => Some(PathBuf::from(dir)), + (Some(_), Some(_)) => { + warn!( + "Both VIRTUAL_ENV and CONDA_PREFIX are set. PyO3 will ignore both of these for \ + locating the Python interpreter until you unset one of them." + ); + None + } + (None, None) => None, + } +} + +/// Attempts to locate a python interpreter. Locations are checked in the order listed: +/// 1. If `PYO3_PYTHON` is set, this intepreter is used. +/// 2. If in a virtualenv, that environment's interpreter is used. +/// 3. `python`, if this is functional a Python 3.x interpreter +/// 4. `python3`, as above +fn find_interpreter() -> Result { + if let Some(exe) = env_var("PYO3_PYTHON") { + Ok(exe.into()) + } else if let Some(venv_path) = get_venv_path() { + match cargo_env_var("CARGO_CFG_TARGET_OS").unwrap().as_str() { + "windows" => Ok(venv_path.join("Scripts\\python")), + _ => Ok(venv_path.join("bin/python")), + } + } else { + println!("cargo:rerun-if-env-changed=PATH"); + ["python", "python3"] + .iter() + .find(|bin| { + if let Ok(out) = Command::new(bin).arg("--version").output() { + // begin with `Python 3.X.X :: additional info` + out.stdout.starts_with(b"Python 3") || out.stderr.starts_with(b"Python 3") + } else { + false + } + }) + .map(PathBuf::from) + .ok_or_else(|| "no Python 3.x interpreter found".into()) + } +} + +/// Extract compilation vars from the specified interpreter. +fn get_config_from_interpreter(interpreter: &Path) -> Result { + let script = r#" +# Allow the script to run on Python 2, so that nicer error can be printed later. +from __future__ import print_function + +import os.path +import platform +import struct +import sys +from sysconfig import get_config_var + +PYPY = platform.python_implementation() == "PyPy" + +# sys.base_prefix is missing on Python versions older than 3.3; this allows the script to continue +# so that the version mismatch can be reported in a nicer way later. +base_prefix = getattr(sys, "base_prefix", None) + +if base_prefix: + # Anaconda based python distributions have a static python executable, but include + # the shared library. Use the shared library for embedding to avoid rust trying to + # LTO the static library (and failing with newer gcc's, because it is old). + ANACONDA = os.path.exists(os.path.join(base_prefix, "conda-meta")) +else: + ANACONDA = False + +def print_if_set(varname, value): + if value is not None: + print(varname, value) + +libdir = get_config_var("LIBDIR") + +print("version_major", sys.version_info[0]) +print("version_minor", sys.version_info[1]) +print("implementation", platform.python_implementation()) +print_if_set("libdir", libdir) +print_if_set("ld_version", get_config_var("LDVERSION")) +print_if_set("base_prefix", base_prefix) +print("framework", bool(get_config_var("PYTHONFRAMEWORK"))) +print("shared", PYPY or ANACONDA or bool(get_config_var("Py_ENABLE_SHARED"))) +print("executable", sys.executable) +print("calcsize_pointer", struct.calcsize("P")) +"#; + let output = run_python_script(interpreter, script)?; + let map: HashMap = parse_script_output(&output); + let shared = match ( + cargo_env_var("CARGO_CFG_TARGET_OS").unwrap().as_str(), + map["framework"].as_str(), + map["shared"].as_str(), + ) { + (_, _, "True") // Py_ENABLE_SHARED is set + | ("windows", _, _) // Windows always uses shared linking + | ("macos", "True", _) // MacOS framework package uses shared linking + => true, + (_, _, "False") => false, // Any other platform, Py_ENABLE_SHARED not set + _ => bail!("Unrecognised link model combination") + }; + + let version = PythonVersion { + major: map["version_major"].parse()?, + minor: map["version_minor"].parse()?, + }; + + let implementation = map["implementation"].parse()?; + + Ok(InterpreterConfig { + version, + implementation, + libdir: map.get("libdir").cloned(), + shared, + abi3: is_abi3(), + ld_version: map.get("ld_version").cloned(), + base_prefix: map.get("base_prefix").cloned(), + executable: map.get("executable").cloned(), + calcsize_pointer: Some(map["calcsize_pointer"].parse()?), + build_flags: BuildFlags::from_interpreter(interpreter)?.fixup(version, implementation), + }) +} + +fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> { + ensure!( + interpreter_config.version >= MINIMUM_SUPPORTED_VERSION, + "the configured Python interpreter version ({}) is lower than PyO3's minimum supported version ({})", + interpreter_config.version, + MINIMUM_SUPPORTED_VERSION, + ); + + Ok(()) +} + +fn ensure_target_architecture(interpreter_config: &InterpreterConfig) -> Result<()> { + // Try to check whether the target architecture matches the python library + let rust_target = match cargo_env_var("CARGO_CFG_TARGET_POINTER_WIDTH") + .unwrap() + .as_str() + { + "64" => "64-bit", + "32" => "32-bit", + x => bail!("unexpected Rust target pointer width: {}", x), + }; + + // The reason we don't use platform.architecture() here is that it's not + // reliable on macOS. See https://stackoverflow.com/a/1405971/823869. + // Similarly, sys.maxsize is not reliable on Windows. See + // https://stackoverflow.com/questions/1405913/how-do-i-determine-if-my-python-shell-is-executing-in-32bit-or-64bit-mode-on-os/1405971#comment6209952_1405971 + // and https://stackoverflow.com/a/3411134/823869. + let python_target = match interpreter_config.calcsize_pointer { + Some(8) => "64-bit", + Some(4) => "32-bit", + None => { + // Unset, e.g. because we're cross-compiling. Don't check anything + // in this case. + return Ok(()); + } + Some(n) => bail!("unexpected Python calcsize_pointer value: {}", n), + }; + + ensure!( + rust_target == python_target, + "Your Rust target architecture ({}) does not match your python interpreter ({})", + rust_target, + python_target + ); + + Ok(()) +} + +fn get_abi3_minor_version() -> Option { + (MINIMUM_SUPPORTED_VERSION.minor..=ABI3_MAX_MINOR) + .find(|i| cargo_env_var(&format!("CARGO_FEATURE_ABI3_PY3{}", i)).is_some()) +} + +fn get_interpreter_config() -> Result { + let abi3_version = get_abi3_minor_version(); + + // If PYO3_NO_PYTHON is set with abi3, we can build PyO3 without calling Python. + if let Some(abi3_minor_version) = abi3_version { + if env_var("PYO3_NO_PYTHON").is_some() { + return Ok(InterpreterConfig { + version: PythonVersion { + major: 3, + minor: abi3_minor_version, + }, + implementation: PythonImplementation::CPython, + abi3: true, + libdir: None, + build_flags: BuildFlags::abi3(), + base_prefix: None, + calcsize_pointer: None, + executable: None, + ld_version: None, + shared: true, + }); + } + } + + let mut interpreter_config = if let Some(paths) = cross_compiling()? { + load_cross_compile_info(paths)? + } else { + get_config_from_interpreter(&find_interpreter()?)? + }; + + // Fixup minor version if abi3-pyXX feature set + if let Some(abi3_minor_version) = abi3_version { + ensure!( + abi3_minor_version <= interpreter_config.version.minor, + "You cannot set a mininimum Python version 3.{} higher than the interpreter version 3.{}", + abi3_minor_version, + interpreter_config.version.minor + ); + + interpreter_config.version.minor = abi3_minor_version; + } + + Ok(interpreter_config) +} + +pub fn configure() -> Result<()> { + let interpreter_config = get_interpreter_config()?; + ensure_python_version(&interpreter_config)?; + ensure_target_architecture(&interpreter_config)?; + write_interpreter_config(&interpreter_config) +} + +fn write_interpreter_config(interpreter_config: &InterpreterConfig) -> Result<()> { + let out_dir = env::var_os("OUT_DIR").unwrap(); + let mut out = File::create(Path::new(&out_dir).join("pyo3-build-config.rs"))?; + + writeln!(out, "{{")?; + writeln!( + out, + "let mut build_flags = std::collections::HashSet::new();" + )?; + for flag in &interpreter_config.build_flags.0 { + writeln!(out, "build_flags.insert({:?});", flag)?; + } + + writeln!( + out, + r#"crate::impl_::InterpreterConfig {{ + version: crate::impl_::PythonVersion {{ + major: {major}, + minor: {minor}, + }}, + implementation: crate::impl_::PythonImplementation::{implementation:?}, + libdir: {libdir:?}.map(|str: &str| str.to_string()), + abi3: {abi3}, + build_flags: crate::impl_::BuildFlags(build_flags), + base_prefix: {base_prefix:?}.map(|str: &str| str.to_string()), + calcsize_pointer: {calcsize_pointer:?}, + executable: {executable:?}.map(|str: &str| str.to_string()), + ld_version: {ld_version:?}.map(|str: &str| str.to_string()), + shared: {shared:?} + }}"#, + major = interpreter_config.version.major, + minor = interpreter_config.version.minor, + implementation = interpreter_config.implementation, + base_prefix = interpreter_config.base_prefix, + calcsize_pointer = interpreter_config.calcsize_pointer, + executable = interpreter_config.executable, + ld_version = interpreter_config.ld_version, + libdir = interpreter_config.libdir, + shared = interpreter_config.shared, + abi3 = interpreter_config.abi3, + )?; + writeln!(out, "}}")?; + + Ok(()) +} diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs new file mode 100644 index 00000000000..300ed4437d8 --- /dev/null +++ b/pyo3-build-config/src/lib.rs @@ -0,0 +1,19 @@ +//! Configuration used by PyO3 for conditional support of varying Python versions. +//! +//! The only public API currently exposed is [`use_pyo3_cfgs`], which is intended to be used in +//! build scripts to add a standard set of `#[cfg]` flags for handling multiple Python versions. + +#[allow(dead_code)] // TODO cover this using tests +mod impl_; + +#[doc(hidden)] +pub use crate::impl_::{InterpreterConfig, PythonImplementation}; + +#[doc(hidden)] +pub fn get() -> InterpreterConfig { + include!(concat!(env!("OUT_DIR"), "/pyo3-build-config.rs")) +} + +pub fn use_pyo3_cfgs() { + get().emit_pyo3_cfgs(); +} diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index 4c9247ca7c9..d8e0c443ca2 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -21,3 +21,6 @@ proc-macro2 = { version = "1", default-features = false } version = "1" default-features = false features = ["derive", "parsing", "printing", "clone-impls", "full", "extra-traits"] + +[build-dependencies] +pyo3-build-config = { path = "../pyo3-build-config", version = "0.14.0-alpha.0" } diff --git a/pyo3-macros-backend/build.rs b/pyo3-macros-backend/build.rs new file mode 100644 index 00000000000..0475124bb4e --- /dev/null +++ b/pyo3-macros-backend/build.rs @@ -0,0 +1,3 @@ +fn main() { + pyo3_build_config::use_pyo3_cfgs(); +} diff --git a/pyo3-macros-backend/src/pyproto.rs b/pyo3-macros-backend/src/pyproto.rs index de69f20103e..b698e1e61cf 100644 --- a/pyo3-macros-backend/src/pyproto.rs +++ b/pyo3-macros-backend/src/pyproto.rs @@ -132,10 +132,9 @@ fn impl_proto_methods( let slots_trait_slots = proto.slots_trait_slots(); let mut maybe_buffer_methods = None; + + #[cfg(not(Py_3_9))] if proto.name == "Buffer" { - // On Python 3.9 we have to use PyBufferProcs to set buffer slots. - // For now we emit this always for buffer methods, even on 3.9+. - // Maybe in the future we can access Py_3_9 here and define it. maybe_buffer_methods = Some(quote! { impl pyo3::class::impl_::PyBufferProtocolProcs<#ty> for pyo3::class::impl_::PyClassImplCollector<#ty> From 825ec086814b5b4b8f3f8e2e2531cc9db1e68606 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Fri, 21 May 2021 08:34:51 +0100 Subject: [PATCH 2/5] pyo3-build-config: docs wip --- guide/src/SUMMARY.md | 1 + guide/src/building_and_distribution.md | 7 ++- .../multiple_python_versions.md | 60 +++++++++++++++++++ guide/src/faq.md | 2 +- pyo3-build-config/src/lib.rs | 2 + pyo3-macros-backend/Cargo.toml | 2 +- 6 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 guide/src/building_and_distribution/multiple_python_versions.md diff --git a/guide/src/SUMMARY.md b/guide/src/SUMMARY.md index 28c4a6fb170..4d2df2f977f 100644 --- a/guide/src/SUMMARY.md +++ b/guide/src/SUMMARY.md @@ -19,6 +19,7 @@ - [Features Reference](features.md) - [Advanced Topics](advanced.md) - [Building and Distribution](building_and_distribution.md) + - [Supporting multiple Python versions](building_and_distribution/multiple_python_versions.md) - [PyPy support](building_and_distribution/pypy.md) - [Useful Crates](ecosystem.md) - [Logging](ecosystem/logging.md) diff --git a/guide/src/building_and_distribution.md b/guide/src/building_and_distribution.md index b1293241c2b..57a45f269c7 100644 --- a/guide/src/building_and_distribution.md +++ b/guide/src/building_and_distribution.md @@ -43,8 +43,7 @@ There are two ways to distribute your module as a Python package: [setuptools-ru By default, Python extension modules can only be used with the same Python version they were compiled against -- if you build an extension module with Python 3.5, you can't import it using Python 3.8. [PEP 384](https://www.python.org/dev/peps/pep-0384/) introduced the idea of the limited Python API, which would have a stable ABI enabling extension modules built with it to be used against multiple Python versions. This is also known as `abi3`. -Note that [maturin] >= 0.9.0 or [setuptools-rust] >= 0.11.4 support `abi3` wheels. -See the [corresponding](https://github.com/PyO3/maturin/pull/353) [PRs](https://github.com/PyO3/setuptools-rust/pull/82) for more. +The advantage of building extension module using the limited Python API is that you only need to build and distribute a single copy (for each OS / architecture), and your users can install it on all Python versions from your [minimum version](#minimum-python-version-for-abi3) and up. The downside of this is that PyO3 can't use optimizations which rely on being compiled against a known exact Python version. It's up to you to decide whether this matters for your extension module. It's also possible to design your extension module such that you can distribute `abi3` wheels but allow users compiling from source to benefit from additional optimizations - see the [support for multiple python versions](./building_and_distribution/multiple_python_versions.html) section of this guide, in particular the `#[cfg(Py_LIMITED_API)]` flag. There are three steps involved in making use of `abi3` when building Python packages as wheels: @@ -55,7 +54,8 @@ There are three steps involved in making use of `abi3` when building Python pack pyo3 = { {{#PYO3_CRATE_VERSION}}, features = ["abi3"] } ``` -2. Ensure that the built shared objects are correctly marked as `abi3`. This is accomplished by telling your build system that you're using the limited API. +2. Ensure that the built shared objects are correctly marked as `abi3`. This is accomplished by telling your build system that you're using the limited API. [maturin] >= 0.9.0 and [setuptools-rust] >= 0.11.4 support `abi3` wheels. +See the [corresponding](https://github.com/PyO3/maturin/pull/353) [PRs](https://github.com/PyO3/setuptools-rust/pull/82) for more. 3. Ensure that the `.whl` is correctly marked as `abi3`. For projects using `setuptools`, this is accomplished by passing `--py-limited-api=cp3x` (where `x` is the minimum Python version supported by the wheel, e.g. `--py-limited-api=cp35` for Python 3.5) to `setup.py bdist_wheel`. @@ -76,6 +76,7 @@ not work when compiling for `abi3`. These are: - `#[text_signature]` does not work on classes until Python 3.10 or greater. - The `dict` and `weakref` options on classes are not supported until Python 3.9 or greater. - The buffer API is not supported. +- Optimizations which rely on knowledge of the exact Python version compiled against. ## Cross Compiling diff --git a/guide/src/building_and_distribution/multiple_python_versions.md b/guide/src/building_and_distribution/multiple_python_versions.md new file mode 100644 index 00000000000..153d6e321c5 --- /dev/null +++ b/guide/src/building_and_distribution/multiple_python_versions.md @@ -0,0 +1,60 @@ +# Supporting multiple Python versions + +PyO3 supports all actively-supported Python 3 and PyPy versions. As much as possible, this is done internally to PyO3 so that your crate's code does not need to adapt to the differences between each version. However, as Python features grow and change between versions, PyO3 cannot a completely identical API for every Python version. This may require you to add conditional compilation to your crate or runtime checks for the Python version. + +This section of the guide first introduces the `pyo3-build-config` crate, which you can use as a `build-dependency` to add additional `#[cfg]` flags which allow you to support multiple Python versions at compile-time. + +Second, we'll show how to check the Python version at runtime. This can be useful when building for multiple versions with the `abi3` feature, where the Python API compiled against is not always the same as the one in use. + +## Conditional compilation for different Python versions + +The `pyo3-build-config` exposes multiple [`#[cfg]` flags](https://doc.rust-lang.org/rust-by-example/attribute/cfg.html) which can be used to conditionally compile code for a given Python version. PyO3 itself depends on this crate, so by using it you can be sure that you are configured correctly for the Python version PyO3 is building against. + +This allows us to write code like the following + +```rust,ignore +#[cfg(Py_3_7)] +fn function_only_supported_on_python_3_7_and_up() { } + +#[cfg(not(Py_3_8))] +fn function_only_supported_before_python_3_8() { } + +#[cfg(not(Py_LIMITED_API))] +fn function_incompatible_with_abi3_feature() { } +``` + +The following sections first show how to add these `#[cfg]` flags to your build process, and then cover some common patterns flags in a little more detail. + +To see a full reference of all the `#[cfg]` flags provided, see the [`pyo3-build-cfg` docs](https://docs.rs/pyo3-build-config). + +### Using `pyo3-build-config` + +You can use the `#[cfg]` flags in just two steps: + +1. Add `pyo3-build-config` it to your crate's build dependencies in `Cargo.toml`: + + ```toml + [build-dependencies] + pyo3-build-config = "{{#PYO3_CRATE_VERSION}}" + ``` + +2. Add a [`build.rs`](https://doc.rust-lang.org/cargo/reference/build-scripts.html) file to your crate with the following contents: + + ```rust,ignore + fn main() { + // If you have an existing build.rs file, just add this line to it. + pyo3_build_config::use_pyo3_cfgs(); + } + ``` + +After these steps you are ready to annotate your code! + +### Common usages of `pyo3-build-cfg` flags + +The following are some common patterns implemented using these flags: + +// TODO + +## Checking the Python version at runtime + +// TODO diff --git a/guide/src/faq.md b/guide/src/faq.md index aeb2bc0051f..38b5260a7e0 100644 --- a/guide/src/faq.md +++ b/guide/src/faq.md @@ -21,7 +21,7 @@ Currently, [#341](https://github.com/PyO3/pyo3/issues/341) causes `cargo test` t ```toml [dependencies.pyo3] -version = "{{#PYO3_VERSION}}" +version = "{{#PYO3_CRATE_VERSION}}" [features] extension-module = ["pyo3/extension-module"] diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index 300ed4437d8..13525e29f01 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -2,6 +2,8 @@ //! //! The only public API currently exposed is [`use_pyo3_cfgs`], which is intended to be used in //! build scripts to add a standard set of `#[cfg]` flags for handling multiple Python versions. +//! +//! TODO: tabulate all the flags here #[allow(dead_code)] // TODO cover this using tests mod impl_; diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index d8e0c443ca2..91979046aca 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -23,4 +23,4 @@ default-features = false features = ["derive", "parsing", "printing", "clone-impls", "full", "extra-traits"] [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "0.14.0-alpha.0" } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.14.0-alpha.0" } From 6def8fe7147796c82cc78802f4bfeba56be76fcb Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Fri, 21 May 2021 08:38:59 +0100 Subject: [PATCH 3/5] pyo3-build-config: check python version and architecture in PyO3 crate --- build.rs | 69 ++++++++++++++++++++++++++++++++-- pyo3-build-config/src/impl_.rs | 50 ------------------------ pyo3-build-config/src/lib.rs | 2 +- 3 files changed, 66 insertions(+), 55 deletions(-) diff --git a/build.rs b/build.rs index 637258f30f6..13d348968c8 100644 --- a/build.rs +++ b/build.rs @@ -1,9 +1,68 @@ use std::{env, process::Command}; -use pyo3_build_config::{InterpreterConfig, PythonImplementation}; +use pyo3_build_config::{InterpreterConfig, PythonImplementation, PythonVersion}; type Result = std::result::Result>; +/// Minimum Python version PyO3 supports. +const MINIMUM_SUPPORTED_VERSION: PythonVersion = PythonVersion { major: 3, minor: 6 }; + +// A simple macro for returning an error. Resembles anyhow::bail. +macro_rules! bail { + ($msg: expr) => { return Err($msg.into()); }; + ($fmt: literal $($args: tt)+) => { return Err(format!($fmt $($args)+).into()); }; +} + +// A simple macro for checking a condition. Resembles anyhow::ensure. +macro_rules! ensure { + ($condition:expr, $($args: tt)+) => { if !($condition) { bail!($($args)+) } }; +} + +fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> { + ensure!( + interpreter_config.version >= MINIMUM_SUPPORTED_VERSION, + "the configured Python interpreter version ({}) is lower than PyO3's minimum supported version ({})", + interpreter_config.version, + MINIMUM_SUPPORTED_VERSION, + ); + + Ok(()) +} + +fn ensure_target_architecture(interpreter_config: &InterpreterConfig) -> Result<()> { + // Try to check whether the target architecture matches the python library + let rust_target = match env::var("CARGO_CFG_TARGET_POINTER_WIDTH").unwrap().as_str() { + "64" => "64-bit", + "32" => "32-bit", + x => bail!("unexpected Rust target pointer width: {}", x), + }; + + // The reason we don't use platform.architecture() here is that it's not + // reliable on macOS. See https://stackoverflow.com/a/1405971/823869. + // Similarly, sys.maxsize is not reliable on Windows. See + // https://stackoverflow.com/questions/1405913/how-do-i-determine-if-my-python-shell-is-executing-in-32bit-or-64bit-mode-on-os/1405971#comment6209952_1405971 + // and https://stackoverflow.com/a/3411134/823869. + let python_target = match interpreter_config.calcsize_pointer { + Some(8) => "64-bit", + Some(4) => "32-bit", + None => { + // Unset, e.g. because we're cross-compiling. Don't check anything + // in this case. + return Ok(()); + } + Some(n) => bail!("unexpected Python calcsize_pointer value: {}", n), + }; + + ensure!( + rust_target == python_target, + "Your Rust target architecture ({}) does not match your python interpreter ({})", + rust_target, + python_target + ); + + Ok(()) +} + fn get_rustc_link_lib(config: &InterpreterConfig) -> Result { let link_name = if env::var_os("CARGO_CFG_TARGET_OS").unwrap() == "windows" { if config.abi3 { @@ -127,9 +186,11 @@ fn emit_cargo_configuration(interpreter_config: &InterpreterConfig) -> Result<() } fn configure_pyo3() -> Result<()> { - let cfg = pyo3_build_config::get(); - emit_cargo_configuration(&cfg)?; - cfg.emit_pyo3_cfgs(); + let interpreter_config = pyo3_build_config::get(); + ensure_python_version(&interpreter_config)?; + ensure_target_architecture(&interpreter_config)?; + emit_cargo_configuration(&interpreter_config)?; + interpreter_config.emit_pyo3_cfgs(); // Enable use of const generics on Rust 1.51 and greater if rustc_minor_version().unwrap_or(0) >= 51 { diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index a1964614557..b80d2d9fa2f 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -730,54 +730,6 @@ print("calcsize_pointer", struct.calcsize("P")) }) } -fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> { - ensure!( - interpreter_config.version >= MINIMUM_SUPPORTED_VERSION, - "the configured Python interpreter version ({}) is lower than PyO3's minimum supported version ({})", - interpreter_config.version, - MINIMUM_SUPPORTED_VERSION, - ); - - Ok(()) -} - -fn ensure_target_architecture(interpreter_config: &InterpreterConfig) -> Result<()> { - // Try to check whether the target architecture matches the python library - let rust_target = match cargo_env_var("CARGO_CFG_TARGET_POINTER_WIDTH") - .unwrap() - .as_str() - { - "64" => "64-bit", - "32" => "32-bit", - x => bail!("unexpected Rust target pointer width: {}", x), - }; - - // The reason we don't use platform.architecture() here is that it's not - // reliable on macOS. See https://stackoverflow.com/a/1405971/823869. - // Similarly, sys.maxsize is not reliable on Windows. See - // https://stackoverflow.com/questions/1405913/how-do-i-determine-if-my-python-shell-is-executing-in-32bit-or-64bit-mode-on-os/1405971#comment6209952_1405971 - // and https://stackoverflow.com/a/3411134/823869. - let python_target = match interpreter_config.calcsize_pointer { - Some(8) => "64-bit", - Some(4) => "32-bit", - None => { - // Unset, e.g. because we're cross-compiling. Don't check anything - // in this case. - return Ok(()); - } - Some(n) => bail!("unexpected Python calcsize_pointer value: {}", n), - }; - - ensure!( - rust_target == python_target, - "Your Rust target architecture ({}) does not match your python interpreter ({})", - rust_target, - python_target - ); - - Ok(()) -} - fn get_abi3_minor_version() -> Option { (MINIMUM_SUPPORTED_VERSION.minor..=ABI3_MAX_MINOR) .find(|i| cargo_env_var(&format!("CARGO_FEATURE_ABI3_PY3{}", i)).is_some()) @@ -830,8 +782,6 @@ fn get_interpreter_config() -> Result { pub fn configure() -> Result<()> { let interpreter_config = get_interpreter_config()?; - ensure_python_version(&interpreter_config)?; - ensure_target_architecture(&interpreter_config)?; write_interpreter_config(&interpreter_config) } diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index 13525e29f01..01f6e54e2b8 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -9,7 +9,7 @@ mod impl_; #[doc(hidden)] -pub use crate::impl_::{InterpreterConfig, PythonImplementation}; +pub use crate::impl_::{InterpreterConfig, PythonImplementation, PythonVersion}; #[doc(hidden)] pub fn get() -> InterpreterConfig { From 284ad1f98acec1682a68326733770533da32e5f9 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Fri, 21 May 2021 23:05:36 +0100 Subject: [PATCH 4/5] pyo3-build-config: fix build --- pyo3-macros-backend/src/pyproto.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyo3-macros-backend/src/pyproto.rs b/pyo3-macros-backend/src/pyproto.rs index b698e1e61cf..a2f62ae37bd 100644 --- a/pyo3-macros-backend/src/pyproto.rs +++ b/pyo3-macros-backend/src/pyproto.rs @@ -133,8 +133,7 @@ fn impl_proto_methods( let mut maybe_buffer_methods = None; - #[cfg(not(Py_3_9))] - if proto.name == "Buffer" { + if cfg!(not(Py_3_9)) && proto.name == "Buffer" { maybe_buffer_methods = Some(quote! { impl pyo3::class::impl_::PyBufferProtocolProcs<#ty> for pyo3::class::impl_::PyClassImplCollector<#ty> From 1e1605f0dbae53c555ce9cb510b4684fe2cc44b5 Mon Sep 17 00:00:00 2001 From: David Hewitt <1939362+davidhewitt@users.noreply.github.com> Date: Mon, 24 May 2021 08:38:21 +0100 Subject: [PATCH 5/5] pyo3-build-config: finish docs --- guide/src/building_and_distribution.md | 6 ++- .../multiple_python_versions.md | 49 +++++++++++++++++-- pyo3-build-config/src/lib.rs | 13 ++++- 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/guide/src/building_and_distribution.md b/guide/src/building_and_distribution.md index 57a45f269c7..44b44789dd0 100644 --- a/guide/src/building_and_distribution.md +++ b/guide/src/building_and_distribution.md @@ -63,11 +63,15 @@ See the [corresponding](https://github.com/PyO3/maturin/pull/353) [PRs](https:// Because a single `abi3` wheel can be used with many different Python versions, PyO3 has feature flags `abi3-py36`, `abi3-py37`, `abi-py38` etc. to set the minimum required Python version for your `abi3` wheel. For example, if you set the `abi3-py36` feature, your extension wheel can be used on all Python 3 versions from Python 3.6 and up. `maturin` and `setuptools-rust` will give the wheel a name like `my-extension-1.0-cp36-abi3-manylinux2020_x86_64.whl`. -If you set more that one of these api version feature flags the highest version always wins. For example, with both `abi3-py36` and `abi3-py38` set, PyO3 would build a wheel which supports Python 3.8 and up. + +As your extension module may be run with multiple different Python versions you may occasionally find you need to check the Python version at runtime to customize behavior. See [the relevant section of this guide](./building_and_distribution/multiple_python_versions.html#checking-the-python-version-at-runtime) on supporting multiple Python versions at runtime. + PyO3 is only able to link your extension module to api3 version up to and including your host Python version. E.g., if you set `abi3-py38` and try to compile the crate with a host of Python 3.6, the build will fail. As an advanced feature, you can build PyO3 wheel without calling Python interpreter with the environment variable `PYO3_NO_PYTHON` set. On unix systems this works unconditionally; on Windows you must also set the `RUSTFLAGS` evironment variable to contain `-L native=/path/to/python/libs` so that the linker can find `python3.lib`. +> Note: If you set more that one of these api version feature flags the highest version always wins. For example, with both `abi3-py36` and `abi3-py38` set, PyO3 would build a wheel which supports Python 3.8 and up. + ### Missing features Due to limitations in the Python API, there are a few `pyo3` features that do diff --git a/guide/src/building_and_distribution/multiple_python_versions.md b/guide/src/building_and_distribution/multiple_python_versions.md index 153d6e321c5..ff442072e75 100644 --- a/guide/src/building_and_distribution/multiple_python_versions.md +++ b/guide/src/building_and_distribution/multiple_python_versions.md @@ -51,10 +51,53 @@ After these steps you are ready to annotate your code! ### Common usages of `pyo3-build-cfg` flags -The following are some common patterns implemented using these flags: +The `#[cfg]` flags added by `pyo3-build-cfg` can be combined with all of Rust's logic in the `#[cfg]` attribute to create very precise conditional code generation. The following are some common patterns implemented using these flags: -// TODO +``` +#[cfg(Py_3_7)] +``` + +This `#[cfg]` marks code that will only be present on Python 3.7 and upwards. There are similar options `Py_3_8`, `Py_3_9`, `Py_3_10` and so on for each minor version. + +``` +#[cfg(not(Py_3_7))] +``` + +This `#[cfg]` marks code that will only be present on Python versions before (but not including) Python 3.7. + +``` +#[cfg(not(Py_LIMITED_API))] +``` + +This `#[cfg]` marks code that is only available when building for the unlimited Python API (i.e. PyO3's `abi3` feature is not enabled). This might be useful if you want to ship your extension module as an `abi3` wheel and also allow users to compile it from source to make use of optimizations only possible with the unlimited API. + +``` +#[cfg(any(Py_3_9, not(Py_LIMITED_API)))] +``` + +This `#[cfg]` marks code which is available when running Python 3.9 or newer, or when using the unlimited API with an older Python version. Patterns like this are commonly seen on Python APIs which were added to the limited Python API in a specific minor version. + +``` +#[cfg(PyPy)] +``` + +This `#[cfg]` marks code which is running on PyPy. ## Checking the Python version at runtime -// TODO +When building with PyO3's `abi3` feature, your extension module will be compiled against a specific [minimum version](../building_and_distribution.html#minimum-python-version-for-abi3) of Python, but may be running on newer Python versions. + +For example with PyO3's `abi3-py38` feature, your extension will be compiled as if it were for Python 3.8. If you were using `pyo3-build-config`, `#[cfg(Py_3_8)]` would be present. Your user could freely install and run your abi3 extension on Python 3.9. + +There's no way to detect your user doing that at compile time, so instead you need to fall back to runtime checks. + +PyO3 provides the APIs [`Python::version()`] and [`Python::version_info()`] to query the running Python version. This allows you to do the following, for example: + +```rust,ignore +if py.version_info() >= (3, 9) { + // run this code only if Python 3.9 or up +} +``` + +[`Python::version()`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Python.html#method.version +[`Python::version_info()`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Python.html#method.version_info diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index 01f6e54e2b8..ed145576ad1 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -1,9 +1,18 @@ //! Configuration used by PyO3 for conditional support of varying Python versions. //! //! The only public API currently exposed is [`use_pyo3_cfgs`], which is intended to be used in -//! build scripts to add a standard set of `#[cfg]` flags for handling multiple Python versions. +//! build scripts to add a standard set of `#[cfg]` attributes for handling multiple Python +//! versions. //! -//! TODO: tabulate all the flags here +//! The full list of attributes added are the following: +//! +//! | Flag | Description | +//! | ---- | ----------- | +//! | `#[cfg(Py_3_6)]`, `#[cfg(Py_3_7)]`, `#[cfg(Py_3_8)]`, `#[cfg(Py_3_9)]`, `#[cfg(Py_3_10)]` | These attributes mark code only for a given Python version and up. For example, `#[cfg(Py_3_6)]` marks code which can run on Python 3.6 **and newer**. | +//! | `#[cfg(Py_LIMITED_API)]` | This marks code which is run when compiling with PyO3's `abi3` feature enabled. | +//! | `#[cfg(PyPy)]` | This marks code which is run when compiling for PyPy. | +//! +//! For examples of how to use these attributes, [see PyO3's guide](https://pyo3.rs/main/building_and_distribution/multiple_python_versions.html). #[allow(dead_code)] // TODO cover this using tests mod impl_;