From 9850236630847803e142c3af6a6393654953453c Mon Sep 17 00:00:00 2001 From: Robert Coup Date: Wed, 15 Sep 2021 13:31:58 +0100 Subject: [PATCH] docker: Add ability to build shared library flavours of CPython. Enable via: PY_SHARED=1 \ PLATFORM=$(uname -m) POLICY=manylinux2014 COMMIT_SHA=latest \ ./build.sh If set, builds both static & shared versions of CPython into the images. Shared versions end up in (eg) /opt/python/cp37-cp37m-shared/ alongside the existing static /opt/python/cp37-cp37m/, with a Python binary as /usr/local/bin/python3.7-shared. --- README.rst | 18 ++++++ build.sh | 8 ++- docker/Dockerfile | 2 + docker/build_scripts/build-cpython.sh | 83 +++++++++++++++++---------- docker/build_scripts/finalize.sh | 14 ++++- 5 files changed, 91 insertions(+), 34 deletions(-) diff --git a/README.rst b/README.rst index 8074ce999..27e7d8b72 100644 --- a/README.rst +++ b/README.rst @@ -178,6 +178,24 @@ current (root) directory: Please note that the Docker build is using `buildx `_. +Shared Library CPython +~~~~~~~~~~~~~~~~~~~~~~ + +You can build the Docker images with shared library CPython builds alongside the +static CPython builds. This might be useful for special packaging requirements, +but ``libpythonX.Y`` is `not a library that a manylinux extension is allowed to +link to `, so it is +not part of the default manylinux images. + +To build the Docker images with shared library CPython builds: + + $ PY_SHARED=1 PLATFORM=$(uname -m) POLICY=manylinux2014 COMMIT_SHA=latest ./build.sh + +The shared CPython interpreters are installed in +``/opt/python/--shared``. The directories are named after +the PEP 425 tags for each environment -- e.g. ``/opt/python/cp37-cp37m-shared`` +contains a shared library CPython 3.7 build. + Updating the requirements ------------------------- diff --git a/build.sh b/build.sh index b9dfca1ce..2b27209e4 100755 --- a/build.sh +++ b/build.sh @@ -66,14 +66,20 @@ else echo "Unsupported policy: '${POLICY}'" exit 1 fi + +# default is not to include shared interpreter builds +: "${PY_SHARED:=0}" + export BASEIMAGE export DEVTOOLSET_ROOTPATH export PREPEND_PATH export LD_LIBRARY_PATH_ARG +export PY_SHARED BUILD_ARGS_COMMON=" --build-arg POLICY --build-arg PLATFORM --build-arg BASEIMAGE --build-arg DEVTOOLSET_ROOTPATH --build-arg PREPEND_PATH --build-arg LD_LIBRARY_PATH_ARG + --build-arg PY_SHARED --rm -t quay.io/pypa/${POLICY}_${PLATFORM}:${COMMIT_SHA} -f docker/Dockerfile docker/ " @@ -94,7 +100,7 @@ elif [ "${MANYLINUX_BUILD_FRONTEND}" == "buildkit" ]; then --import-cache type=local,src=$(pwd)/.buildx-cache-${POLICY}_${PLATFORM} \ --export-cache type=local,dest=$(pwd)/.buildx-cache-staging-${POLICY}_${PLATFORM} \ --opt build-arg:POLICY=${POLICY} --opt build-arg:PLATFORM=${PLATFORM} --opt build-arg:BASEIMAGE=${BASEIMAGE} \ - --opt "build-arg:DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}" --opt "build-arg:PREPEND_PATH=${PREPEND_PATH}" --opt "build-arg:LD_LIBRARY_PATH_ARG=${LD_LIBRARY_PATH_ARG}" \ + --opt "build-arg:DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}" --opt "build-arg:PREPEND_PATH=${PREPEND_PATH}" --opt "build-arg:LD_LIBRARY_PATH_ARG=${LD_LIBRARY_PATH_ARG}" --opt "build-arg:PY_SHARED=${PY_SHARED}"\ --output type=docker,name=quay.io/pypa/${POLICY}_${PLATFORM}:${COMMIT_SHA} | docker load else echo "Unsupported build frontend: '${MANYLINUX_BUILD_FRONTEND}'" diff --git a/docker/Dockerfile b/docker/Dockerfile index d7388066e..2b00a9285 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -5,6 +5,7 @@ ARG PLATFORM=x86_64 ARG DEVTOOLSET_ROOTPATH= ARG LD_LIBRARY_PATH_ARG= ARG PREPEND_PATH= +ARG PY_SHARED=0 FROM $BASEIMAGE AS runtime_base ARG POLICY @@ -12,6 +13,7 @@ ARG PLATFORM ARG DEVTOOLSET_ROOTPATH ARG LD_LIBRARY_PATH_ARG ARG PREPEND_PATH +ARG PY_SHARED LABEL maintainer="The ManyLinux project" ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM} diff --git a/docker/build_scripts/build-cpython.sh b/docker/build_scripts/build-cpython.sh index f744d771c..5f48eba81 100755 --- a/docker/build_scripts/build-cpython.sh +++ b/docker/build_scripts/build-cpython.sh @@ -30,36 +30,59 @@ fetch_source Python-${CPYTHON_VERSION}.tgz.asc ${CPYTHON_DOWNLOAD_URL}/${CPYTHON gpg --import ${MY_DIR}/cpython-pubkeys.txt gpg --verify Python-${CPYTHON_VERSION}.tgz.asc tar -xzf Python-${CPYTHON_VERSION}.tgz -pushd Python-${CPYTHON_VERSION} -PREFIX="/opt/_internal/cpython-${CPYTHON_VERSION}" -mkdir -p ${PREFIX}/lib -if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then - # The _ctypes stdlib module build started to fail with 3.10.0rc1 - # No clue what changed exactly yet - # This workaround fixes the build - LIBFFI_INCLUDEDIR=$(pkg-config --cflags-only-I libffi | tr -d '[:space:]') - LIBFFI_INCLUDEDIR=${LIBFFI_INCLUDEDIR:2} - cp ${LIBFFI_INCLUDEDIR}/ffi.h ${LIBFFI_INCLUDEDIR}/ffitarget.h /usr/include/ -fi -# configure with hardening options only for the interpreter & stdlib C extensions -# do not change the default for user built extension (yet?) -./configure \ - CFLAGS_NODIST="${MANYLINUX_CFLAGS} ${MANYLINUX_CPPFLAGS}" \ - LDFLAGS_NODIST="${MANYLINUX_LDFLAGS}" \ - --prefix=${PREFIX} --disable-shared --with-ensurepip=no > /dev/null -make > /dev/null -make install > /dev/null -if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then - rm -f /usr/include/ffi.h /usr/include/ffitarget.h -fi -popd -rm -rf Python-${CPYTHON_VERSION} Python-${CPYTHON_VERSION}.tgz Python-${CPYTHON_VERSION}.tgz.asc -# we don't need libpython*.a, and they're many megabytes -find ${PREFIX} -name '*.a' -print0 | xargs -0 rm -f +function build { + IS_SHARED=$1 + pushd Python-${CPYTHON_VERSION} + PREFIX="/opt/_internal/cpython-${CPYTHON_VERSION}" + if [ ${IS_SHARED} -eq 1 ]; then + PREFIX="${PREFIX}-shared" + fi + mkdir -p ${PREFIX}/lib + if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then + # The _ctypes stdlib module build started to fail with 3.10.0rc1 + # No clue what changed exactly yet + # This workaround fixes the build + LIBFFI_INCLUDEDIR=$(pkg-config --cflags-only-I libffi | tr -d '[:space:]') + LIBFFI_INCLUDEDIR=${LIBFFI_INCLUDEDIR:2} + cp ${LIBFFI_INCLUDEDIR}/ffi.h ${LIBFFI_INCLUDEDIR}/ffitarget.h /usr/include/ + fi + # configure with hardening options only for the interpreter & stdlib C extensions + # do not change the default for user built extension (yet?) + if [ ${IS_SHARED} -eq 1 ]; then + FLAVOR="--enable-shared" + FLAVOR_LDFLAGS="-Wl,-rpath=${PREFIX}/lib" + else + FLAVOR="--disable-shared" + FLAVOR_LDFLAGS= + fi + + ./configure \ + CFLAGS_NODIST="${MANYLINUX_CFLAGS} ${MANYLINUX_CPPFLAGS}" \ + LDFLAGS_NODIST="${MANYLINUX_LDFLAGS} ${FLAVOR_LDFLAGS}" \ + --prefix=${PREFIX} ${FLAVOR} --with-ensurepip=no > /dev/null + make > /dev/null + make install > /dev/null + if [ "${AUDITWHEEL_POLICY}" == "manylinux2010" ]; then + rm -f /usr/include/ffi.h /usr/include/ffitarget.h + fi + popd + + if [ ${IS_SHARED} -eq 0 ]; then + # we don't need libpython*.a, and they're many megabytes + find ${PREFIX} -name '*.a' -print0 | xargs -0 rm -f + fi -# We do not need precompiled .pyc and .pyo files. -clean_pyc ${PREFIX} + # We do not need precompiled .pyc and .pyo files. + clean_pyc ${PREFIX} -# Strip ELF files found in ${PREFIX} -strip_ ${PREFIX} + # Strip ELF files found in ${PREFIX} + strip_ ${PREFIX} +} + +build 0 +if [ ${PY_SHARED-0} -eq 1 ]; then + build 1 +fi + +rm -rf Python-${CPYTHON_VERSION} Python-${CPYTHON_VERSION}.tgz Python-${CPYTHON_VERSION}.tgz.asc diff --git a/docker/build_scripts/finalize.sh b/docker/build_scripts/finalize.sh index e35618254..5cce8721a 100755 --- a/docker/build_scripts/finalize.sh +++ b/docker/build_scripts/finalize.sh @@ -11,6 +11,12 @@ source $MY_DIR/build_utils.sh mkdir /opt/python for PREFIX in $(find /opt/_internal/ -mindepth 1 -maxdepth 1 \( -name 'cpython*' -o -name 'pypy*' \)); do + if [[ "${PREFIX}" =~ -shared$ ]]; then + SUFFIX=-shared + else + SUFFIX= + fi + # Some python's install as bin/python3. Make them available as # bin/python. if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then @@ -24,14 +30,16 @@ for PREFIX in $(find /opt/_internal/ -mindepth 1 -maxdepth 1 \( -name 'cpython*' # Since we fall back on a canned copy of pip, we might not have # the latest pip and friends. Upgrade them to make sure. ${PREFIX}/bin/pip install -U --require-hashes -r ${MY_DIR}/requirements${PY_VER}.txt + # Create a symlink to PREFIX using the ABI_TAG in /opt/python/ + ABI_TAG=$(${PREFIX}/bin/python ${MY_DIR}/python-tag-abi-tag.py) - ln -s ${PREFIX} /opt/python/${ABI_TAG} + ln -s ${PREFIX} /opt/python/${ABI_TAG}${SUFFIX:-} # Make versioned python commands available directly in environment. if [[ "${PREFIX}" == *"/pypy"* ]]; then - ln -s ${PREFIX}/bin/python /usr/local/bin/pypy${PY_VER} + ln -s ${PREFIX}/bin/python /usr/local/bin/pypy${PY_VER}${SUFFIX} else - ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER} + ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER}${SUFFIX} fi done