Skip to content

Commit

Permalink
mac build: force using source dist for most of our python dependencies
Browse files Browse the repository at this point in the history
We compile from tar.gz, instead of using pre-built binary wheels from PyPI.
(or if the dep is pure-python, use tar.gz instead of "source-only" wheel)

-----
Some unorganised things below for future reference.

```
$ dsymutil -dump-debug-map dist1/hid.cpython-39-darwin.so
warning: (x86_64) /private/var/folders/1n/zc14m3td0rg4nt0ftklmm7z00000gn/T/pip-install-bm88zvc1/hidapi_cd307bc31ab34252b77d11d6d7212fc5/build/temp.macosx-10.9-x86_64-3.9/hid.o unable to open object file: No such file or directory
warning: (x86_64) /private/var/folders/1n/zc14m3td0rg4nt0ftklmm7z00000gn/T/pip-install-bm88zvc1/hidapi_cd307bc31ab34252b77d11d6d7212fc5/build/temp.macosx-10.9-x86_64-3.9/hidapi/mac/hid.o unable to open object file: No such file or directory
---
triple:          'x86_64-apple-darwin'
binary-path:     'dist1/hid.cpython-39-darwin.so'
...
```

```
$ nm -pa dist1/hid.cpython-39-darwin.so
```

- https://stackoverflow.com/questions/10044697/where-how-does-apples-gcc-store-dwarf-inside-an-executable
- pypa/pip#6505
- pypa/pip#7808 (comment)
- NixOS/nixpkgs#91272
- cython/cython#1576
- https://github.com/cython/cython/blob/9d2ba1611b28999663ab71657f4938b0ba92fe07/Cython/Compiler/ModuleNode.py#L913
  • Loading branch information
SomberNight committed Sep 26, 2022
1 parent 1478d9d commit 0abecff
Show file tree
Hide file tree
Showing 2 changed files with 31 additions and 8 deletions.
12 changes: 8 additions & 4 deletions contrib/build-linux/appimage/make_appimage.sh
Original file line number Diff line number Diff line change
Expand Up @@ -101,16 +101,20 @@ info "preparing electrum-locale."


info "Installing build dependencies."
# note: re pip installing from PyPI,
# we prefer compiling C extensions ourselves, instead of using binary wheels,
# hence "--no-binary :all:" flags. However, we specifically allow
# - PyQt5, as it's harder to build from source
# - cryptography, as it's harder to build from source
# - the whole of "requirements-build-base.txt", which includes pip and friends, as it also includes "wheel",
# and I am not quite sure how to break the circular dependence there (I guess we could introduce
# "requirements-build-base-base.txt" with just wheel in it...)
"$python" -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
--cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements-build-base.txt"
"$python" -m pip install --no-build-isolation --no-dependencies --no-binary :all: --no-warn-script-location \
--cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements-build-appimage.txt"

info "installing electrum and its dependencies."
# note: we prefer compiling C extensions ourselves, instead of using binary wheels,
# hence "--no-binary :all:" flags. However, we specifically allow
# - PyQt5, as it's harder to build from source
# - cryptography, as building it would need openssl 1.1, not available on ubuntu 16.04
"$python" -m pip install --no-build-isolation --no-dependencies --no-binary :all: --no-warn-script-location \
--cache-dir "$PIP_CACHE_DIR" -r "$CONTRIB/deterministic-build/requirements.txt"
"$python" -m pip install --no-build-isolation --no-dependencies --no-binary :all: --only-binary PyQt5,PyQt5-Qt5,cryptography --no-warn-script-location \
Expand Down
27 changes: 23 additions & 4 deletions contrib/osx/make_osx
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ set -e

# Parameterize
PYTHON_VERSION=3.9.11
PY_VER_MAJOR="3.9" # as it appears in fs paths
PACKAGE=Electrum
GIT_REPO=https://github.com/spesmilo/electrum

Expand Down Expand Up @@ -93,13 +94,23 @@ source $VENV_DIR/bin/activate

# don't add debug info to compiled C files (e.g. when pip calls setuptools/wheel calls gcc)
# see https://github.com/pypa/pip/issues/6505#issuecomment-526613584
# note: this does not seem sufficient when cython is involved (although it is on linux, just not on mac... weird.)
# see additional "strip" pass on built files later in the file.
export CFLAGS="-g0"

info "Installing build dependencies"
# note: re pip installing from PyPI,
# we prefer compiling C extensions ourselves, instead of using binary wheels,
# hence "--no-binary :all:" flags. However, we specifically allow
# - PyQt5, as it's harder to build from source
# - cryptography, as it's harder to build from source
# - the whole of "requirements-build-base.txt", which includes pip and friends, as it also includes "wheel",
# and I am not quite sure how to break the circular dependence there (I guess we could introduce
# "requirements-build-base-base.txt" with just wheel in it...)
python3 -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
-Ir ./contrib/deterministic-build/requirements-build-base.txt \
|| fail "Could not install build dependencies (base)"
python3 -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
python3 -m pip install --no-build-isolation --no-dependencies --no-binary :all: --no-warn-script-location \
-Ir ./contrib/deterministic-build/requirements-build-mac.txt \
|| fail "Could not install build dependencies (mac)"

Expand Down Expand Up @@ -192,24 +203,32 @@ cp "$PROJECT_ROOT"/electrum/libusb-1.0.dylib "$CONTRIB"/osx


info "Installing requirements..."
python3 -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
python3 -m pip install --no-build-isolation --no-dependencies --no-binary :all: \
--no-warn-script-location \
-Ir ./contrib/deterministic-build/requirements.txt \
|| fail "Could not install requirements"

info "Installing hardware wallet requirements..."
python3 -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
python3 -m pip install --no-build-isolation --no-dependencies --no-binary :all: --only-binary cryptography \
--no-warn-script-location \
-Ir ./contrib/deterministic-build/requirements-hw.txt \
|| fail "Could not install hardware wallet requirements"

info "Installing dependencies specific to binaries..."
python3 -m pip install --no-build-isolation --no-dependencies --no-warn-script-location \
python3 -m pip install --no-build-isolation --no-dependencies --no-binary :all: --only-binary PyQt5,PyQt5-Qt5,cryptography \
--no-warn-script-location \
-Ir ./contrib/deterministic-build/requirements-binaries-mac.txt \
|| fail "Could not install dependencies specific to binaries"

info "Building $PACKAGE..."
python3 -m pip install --no-build-isolation --no-dependencies \
--no-warn-script-location . > /dev/null || fail "Could not build $PACKAGE"

# strip debug symbols of some compiled libs
# - hidapi (hid.cpython-39-darwin.so) in particular is not reproducible without this
find "$VENV_DIR/lib/python$PY_VER_MAJOR/site-packages/" -type f -name '*.so' -print0 \
| xargs -0 -t strip -x

info "Faking timestamps..."
find . -exec touch -t '200101220000' {} + || true

Expand Down

0 comments on commit 0abecff

Please sign in to comment.