From cfcc8e2d1cd3d4d4f2dea1c9c0793a657d272df6 Mon Sep 17 00:00:00 2001 From: konstin Date: Tue, 11 Apr 2023 19:25:36 +0200 Subject: [PATCH 1/5] Document binary and library in a single pacakge by entrypoint workaround --- guide/src/bindings.md | 36 +++++++++++++++++++ .../check_installed/check_installed.py | 36 +++++++++++++++++++ test-crates/pyo3-mixed/pyo3_mixed/__init__.py | 2 +- test-crates/pyo3-mixed/pyproject.toml | 1 + test-crates/pyo3-mixed/src/lib.rs | 19 ++++++++++ 5 files changed, 93 insertions(+), 1 deletion(-) diff --git a/guide/src/bindings.md b/guide/src/bindings.md index 1a9dcf6e3..8a8a0e473 100644 --- a/guide/src/bindings.md +++ b/guide/src/bindings.md @@ -91,6 +91,42 @@ directory of a virtual environment) once installed. > **Note**: Maturin _does not_ automatically detect `bin` bindings. You _must_ > specify them via either command line with `-b bin` or in `pyproject.toml`. +### Both binary and library? + +Shipping both a binary and library would double the size of your wheel. Consider instead exposing a CLI function in the library and using a Python entrypoint: + +```rust +#[pyfunction] +fn print_cli_args(py: Python) -> PyResult<()> { + // This one includes python and the name of the wrapper script itself, e.g. + // `["/home/ferris/.venv/bin/python", "/home/ferris/.venv/bin/print_cli_args", "a", "b", "c"]` + println!("{:?}", env::args().collect::>()); + // This one includes only the name of the wrapper script itself, e.g. + // `["/home/ferris/.venv/bin/print_cli_args", "a", "b", "c"])` + println!( + "{:?}", + py.import("sys")? + .getattr("argv")? + .extract::>()? + ); + Ok(()) +} + +#[pymodule] +fn my_module(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_wrapped(wrap_pyfunction!(print_cli_args))?; + + Ok(()) +} +``` + +In pyproject.toml: + +```toml +[project.scripts] +print_cli_args = "my_module:print_cli_args" +``` + ## `uniffi` uniffi bindings use [uniffi-rs](https://mozilla.github.io/uniffi-rs/) to generate Python `ctypes` bindings diff --git a/test-crates/pyo3-mixed/check_installed/check_installed.py b/test-crates/pyo3-mixed/check_installed/check_installed.py index 8b53ed615..ea484ee4d 100755 --- a/test-crates/pyo3-mixed/check_installed/check_installed.py +++ b/test-crates/pyo3-mixed/check_installed/check_installed.py @@ -1,9 +1,45 @@ #!/usr/bin/env python3 +import json +import platform +import sys +from pathlib import Path +from subprocess import check_output from boltons.strutils import slugify + import pyo3_mixed assert pyo3_mixed.get_42() == 42 assert slugify("First post! Hi!!!!~1 ") == "first_post_hi_1" +script_name = "print_cli_args" +args = ["a", "b", "c"] +[rust_args, python_args] = check_output([script_name, *args], text=True).splitlines() +# The rust vec debug format is also valid json +rust_args = json.loads(rust_args) +python_args = json.loads(python_args) + +# On linux we get sys.executable, windows resolve the path and mac os gives us a third +# path ( +# {prefix}/Python.framework/Versions/3.10/Resources/Python.app/Contents/MacOS/Python +# vs +# {prefix}/Python.framework/Versions/3.10/bin/python3.10 +# on cirrus ci) +# We also skip windows because cpython resolves while pypy doesn't +if platform.system() == "Linux": + assert rust_args[0] == sys.executable, (rust_args, sys.executable) + +# Windows can't decide if it's with or without .exe, FreeBSB just doesn't work for some reason +if platform.system() in ["Darwin", "Linux"]: + # Unix venv layout (and hopefully also on more exotic platforms) + print_cli_args = str(Path(sys.prefix).joinpath("bin").joinpath(script_name)) + assert rust_args[1] == print_cli_args, (rust_args, print_cli_args) + assert python_args[0] == print_cli_args, (python_args, print_cli_args) + +# FreeBSB just doesn't work for some reason +if platform.system() in ["Darwin", "Linux", "Windows"]: + # Rust contains the python executable as first argument but python does not + assert rust_args[2:] == args, rust_args + assert python_args[1:] == args, python_args + print("SUCCESS") diff --git a/test-crates/pyo3-mixed/pyo3_mixed/__init__.py b/test-crates/pyo3-mixed/pyo3_mixed/__init__.py index c34ac83b4..a0bb1223f 100644 --- a/test-crates/pyo3-mixed/pyo3_mixed/__init__.py +++ b/test-crates/pyo3-mixed/pyo3_mixed/__init__.py @@ -1,5 +1,5 @@ +from .pyo3_mixed import get_21, print_cli_args # noqa: F401 from .python_module.double import double -from .pyo3_mixed import get_21 def get_42() -> int: diff --git a/test-crates/pyo3-mixed/pyproject.toml b/test-crates/pyo3-mixed/pyproject.toml index 27f5d59a9..921fa44f7 100644 --- a/test-crates/pyo3-mixed/pyproject.toml +++ b/test-crates/pyo3-mixed/pyproject.toml @@ -13,3 +13,4 @@ dependencies = ["boltons"] [project.scripts] get_42 = "pyo3_mixed:get_42" +print_cli_args = "pyo3_mixed:print_cli_args" diff --git a/test-crates/pyo3-mixed/src/lib.rs b/test-crates/pyo3-mixed/src/lib.rs index 625c4b6df..04934d63c 100644 --- a/test-crates/pyo3-mixed/src/lib.rs +++ b/test-crates/pyo3-mixed/src/lib.rs @@ -1,13 +1,32 @@ use pyo3::prelude::*; +use std::env; #[pyfunction] fn get_21() -> usize { 21 } +/// Prints the CLI arguments, once from Rust's point of view and once from Python's point of view. +#[pyfunction] +fn print_cli_args(py: Python) -> PyResult<()> { + // This one includes Python and the name of the wrapper script itself, e.g. + // `["/home/ferris/.venv/bin/python", "/home/ferris/.venv/bin/print_cli_args", "a", "b", "c"]` + println!("{:?}", env::args().collect::>()); + // This one includes only the name of the wrapper script itself, e.g. + // `["/home/ferris/.venv/bin/print_cli_args", "a", "b", "c"])` + println!( + "{:?}", + py.import("sys")? + .getattr("argv")? + .extract::>()? + ); + Ok(()) +} + #[pymodule] fn pyo3_mixed(_py: Python, m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(get_21))?; + m.add_wrapped(wrap_pyfunction!(print_cli_args))?; Ok(()) } From 05df456229028d760ddcc021f83076e9806140ea Mon Sep 17 00:00:00 2001 From: konstin Date: Tue, 25 Apr 2023 17:56:14 -0600 Subject: [PATCH 2/5] Use samefile and allow empty rust_args on musl --- .../pyo3-mixed/check_installed/check_installed.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test-crates/pyo3-mixed/check_installed/check_installed.py b/test-crates/pyo3-mixed/check_installed/check_installed.py index ea484ee4d..cbef480d1 100755 --- a/test-crates/pyo3-mixed/check_installed/check_installed.py +++ b/test-crates/pyo3-mixed/check_installed/check_installed.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import json +import os.path import platform import sys from pathlib import Path @@ -25,9 +26,15 @@ # vs # {prefix}/Python.framework/Versions/3.10/bin/python3.10 # on cirrus ci) -# We also skip windows because cpython resolves while pypy doesn't -if platform.system() == "Linux": - assert rust_args[0] == sys.executable, (rust_args, sys.executable) +# On windows, cpython resolves while pypy doesn't. +# On alpine/musl, rust_args is empty +if len(rust_args) > 0: + assert os.path.samefile(rust_args[0], sys.executable), ( + rust_args, + sys.executable, + os.path.realpath(rust_args[0]), + os.path.realpath(sys.executable), + ) # Windows can't decide if it's with or without .exe, FreeBSB just doesn't work for some reason if platform.system() in ["Darwin", "Linux"]: From d773f755b28c608554912b56ecd10f3cc12cc0e9 Mon Sep 17 00:00:00 2001 From: konstin Date: Tue, 25 Apr 2023 18:10:54 -0600 Subject: [PATCH 3/5] sys.interpreter is a distinct executable on windows --- test-crates/pyo3-mixed/check_installed/check_installed.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test-crates/pyo3-mixed/check_installed/check_installed.py b/test-crates/pyo3-mixed/check_installed/check_installed.py index cbef480d1..8a89de9cc 100755 --- a/test-crates/pyo3-mixed/check_installed/check_installed.py +++ b/test-crates/pyo3-mixed/check_installed/check_installed.py @@ -26,9 +26,10 @@ # vs # {prefix}/Python.framework/Versions/3.10/bin/python3.10 # on cirrus ci) -# On windows, cpython resolves while pypy doesn't. +# On windows, cpython resolves while pypy doesn't, also the script for cpython is +# actually as distinct file from the system interpreter # On alpine/musl, rust_args is empty -if len(rust_args) > 0: +if len(rust_args) > 0 and platform.system() != "Windows": assert os.path.samefile(rust_args[0], sys.executable), ( rust_args, sys.executable, From d91865a63f60a61b57eeabfc912bcb7807bf8510 Mon Sep 17 00:00:00 2001 From: konstin Date: Tue, 25 Apr 2023 18:32:52 -0600 Subject: [PATCH 4/5] Limit to sys.executbale check to linux again --- test-crates/pyo3-mixed/check_installed/check_installed.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test-crates/pyo3-mixed/check_installed/check_installed.py b/test-crates/pyo3-mixed/check_installed/check_installed.py index 8a89de9cc..7da3432bb 100755 --- a/test-crates/pyo3-mixed/check_installed/check_installed.py +++ b/test-crates/pyo3-mixed/check_installed/check_installed.py @@ -26,10 +26,11 @@ # vs # {prefix}/Python.framework/Versions/3.10/bin/python3.10 # on cirrus ci) -# On windows, cpython resolves while pypy doesn't, also the script for cpython is -# actually as distinct file from the system interpreter +# On windows, cpython resolves while pypy doesn't. +# The script for cpython is actually a distinct file from the system interpreter for +# windows and mac # On alpine/musl, rust_args is empty -if len(rust_args) > 0 and platform.system() != "Windows": +if len(rust_args) > 0 and platform.system() == "Linux": assert os.path.samefile(rust_args[0], sys.executable), ( rust_args, sys.executable, From 9fae01e63d08d6894f4ccb4dcb4e3fd3fa548fde Mon Sep 17 00:00:00 2001 From: konstin Date: Tue, 25 Apr 2023 21:02:32 -0600 Subject: [PATCH 5/5] Skip alpine/musl altogether --- .../check_installed/check_installed.py | 61 ++++++++++--------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/test-crates/pyo3-mixed/check_installed/check_installed.py b/test-crates/pyo3-mixed/check_installed/check_installed.py index 7da3432bb..428b7f811 100755 --- a/test-crates/pyo3-mixed/check_installed/check_installed.py +++ b/test-crates/pyo3-mixed/check_installed/check_installed.py @@ -20,35 +20,36 @@ rust_args = json.loads(rust_args) python_args = json.loads(python_args) -# On linux we get sys.executable, windows resolve the path and mac os gives us a third -# path ( -# {prefix}/Python.framework/Versions/3.10/Resources/Python.app/Contents/MacOS/Python -# vs -# {prefix}/Python.framework/Versions/3.10/bin/python3.10 -# on cirrus ci) -# On windows, cpython resolves while pypy doesn't. -# The script for cpython is actually a distinct file from the system interpreter for -# windows and mac -# On alpine/musl, rust_args is empty -if len(rust_args) > 0 and platform.system() == "Linux": - assert os.path.samefile(rust_args[0], sys.executable), ( - rust_args, - sys.executable, - os.path.realpath(rust_args[0]), - os.path.realpath(sys.executable), - ) - -# Windows can't decide if it's with or without .exe, FreeBSB just doesn't work for some reason -if platform.system() in ["Darwin", "Linux"]: - # Unix venv layout (and hopefully also on more exotic platforms) - print_cli_args = str(Path(sys.prefix).joinpath("bin").joinpath(script_name)) - assert rust_args[1] == print_cli_args, (rust_args, print_cli_args) - assert python_args[0] == print_cli_args, (python_args, print_cli_args) - -# FreeBSB just doesn't work for some reason -if platform.system() in ["Darwin", "Linux", "Windows"]: - # Rust contains the python executable as first argument but python does not - assert rust_args[2:] == args, rust_args - assert python_args[1:] == args, python_args +# On alpine/musl, rust_args is empty so we skip all tests on musl +if len(rust_args) > 0: + # On linux we get sys.executable, windows resolve the path and mac os gives us a third + # path ( + # {prefix}/Python.framework/Versions/3.10/Resources/Python.app/Contents/MacOS/Python + # vs + # {prefix}/Python.framework/Versions/3.10/bin/python3.10 + # on cirrus ci) + # On windows, cpython resolves while pypy doesn't. + # The script for cpython is actually a distinct file from the system interpreter for + # windows and mac + if platform.system() == "Linux": + assert os.path.samefile(rust_args[0], sys.executable), ( + rust_args, + sys.executable, + os.path.realpath(rust_args[0]), + os.path.realpath(sys.executable), + ) + + # Windows can't decide if it's with or without .exe, FreeBSB just doesn't work for some reason + if platform.system() in ["Darwin", "Linux"]: + # Unix venv layout (and hopefully also on more exotic platforms) + print_cli_args = str(Path(sys.prefix).joinpath("bin").joinpath(script_name)) + assert rust_args[1] == print_cli_args, (rust_args, print_cli_args) + assert python_args[0] == print_cli_args, (python_args, print_cli_args) + + # FreeBSB just doesn't work for some reason + if platform.system() in ["Darwin", "Linux", "Windows"]: + # Rust contains the python executable as first argument but python does not + assert rust_args[2:] == args, rust_args + assert python_args[1:] == args, python_args print("SUCCESS")