diff --git a/CHANGELOG.md b/CHANGELOG.md index 51ee3d60..67e7d1e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## [Unreleased] +- Added support for Python 3.13. ([#1661](https://github.com/heroku/heroku-buildpack-python/pull/1661)) +- Removed the `idle3` and `pydoc3` scripts since they do not work with relocated Python and so have been broken for some time. Invoke them via their modules instead (e.g. `python -m pydoc`). ([#1661](https://github.com/heroku/heroku-buildpack-python/pull/1661)) ## [v259] - 2024-10-09 diff --git a/README.md b/README.md index 3c78ac63..f590addf 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ Specify a Python Runtime Supported runtime options include: +- `python-3.13.0` on all [supported stacks](https://devcenter.heroku.com/articles/stack#stack-support-details) - `python-3.12.7` on all [supported stacks](https://devcenter.heroku.com/articles/stack#stack-support-details) - `python-3.11.10` on all [supported stacks](https://devcenter.heroku.com/articles/stack#stack-support-details) - `python-3.10.15` on all [supported stacks](https://devcenter.heroku.com/articles/stack#stack-support-details) diff --git a/bin/compile b/bin/compile index 12c74f3b..bc6621c1 100755 --- a/bin/compile +++ b/bin/compile @@ -201,11 +201,11 @@ package_manager_install_start_time=$(nowms) bundled_pip_module_path="$(utils::bundled_pip_module_path "${BUILD_DIR}")" case "${package_manager}" in pip) - pip::install_pip_setuptools_wheel "${bundled_pip_module_path}" + pip::install_pip_setuptools_wheel "${bundled_pip_module_path}" "${python_major_version}" ;; pipenv) # TODO: Stop installing pip when using Pipenv. - pip::install_pip_setuptools_wheel "${bundled_pip_module_path}" + pip::install_pip_setuptools_wheel "${bundled_pip_module_path}" "${python_major_version}" pipenv::install_pipenv ;; *) @@ -217,10 +217,13 @@ meta_time "package_manager_install_duration" "${package_manager_install_start_ti # SQLite3 support. # Installs the sqlite3 dev headers and sqlite3 binary but not the # libsqlite3-0 library since that exists in the base image. -install_sqlite_start_time=$(nowms) -source "${BUILDPACK_DIR}/bin/steps/sqlite3" -buildpack_sqlite3_install -meta_time "sqlite_install_duration" "${install_sqlite_start_time}" +# We skip this step on Python 3.13, as a first step towards removing this feature. +if [[ "${python_major_version}" == +(3.8|3.9|3.10|3.11|3.12) ]]; then + install_sqlite_start_time=$(nowms) + source "${BUILDPACK_DIR}/bin/steps/sqlite3" + buildpack_sqlite3_install + meta_time "sqlite_install_duration" "${install_sqlite_start_time}" +fi # Install app dependencies. dependencies_install_start_time=$(nowms) diff --git a/builds/build_python_runtime.sh b/builds/build_python_runtime.sh index ea5c0eb2..39e66d58 100755 --- a/builds/build_python_runtime.sh +++ b/builds/build_python_runtime.sh @@ -27,6 +27,7 @@ case "${STACK:?}" in "3.10" "3.11" "3.12" + "3.13" ) ;; heroku-20) @@ -36,6 +37,7 @@ case "${STACK:?}" in "3.10" "3.11" "3.12" + "3.13" ) ;; *) @@ -49,6 +51,10 @@ fi # The release keys can be found on https://www.python.org/downloads/ -> "OpenPGP Public Keys". case "${PYTHON_MAJOR_VERSION}" in + 3.13) + # https://github.com/Yhg1s.gpg + GPG_KEY_FINGERPRINT='7169605F62C751356D054A26A821E680E5FA6305' + ;; 3.12) # https://github.com/Yhg1s.gpg GPG_KEY_FINGERPRINT='7169605F62C751356D054A26A821E680E5FA6305' @@ -108,7 +114,7 @@ CONFIGURE_OPTS=( "--with-system-expat" ) -if [[ "${PYTHON_MAJOR_VERSION}" != 3.[8-9] ]]; then +if [[ "${PYTHON_MAJOR_VERSION}" != +(3.8|3.9) ]]; then CONFIGURE_OPTS+=( # Shared builds are beneficial for a number of reasons: # - Reduces the size of the build, since it avoids the duplication between @@ -133,7 +139,7 @@ if [[ "${PYTHON_MAJOR_VERSION}" != 3.[8-9] ]]; then ) fi -if [[ "${PYTHON_MAJOR_VERSION}" == "3.11" || "${PYTHON_MAJOR_VERSION}" == "3.12" ]]; then +if [[ "${PYTHON_MAJOR_VERSION}" != +(3.8|3.9|3.10) ]]; then CONFIGURE_OPTS+=( # Skip building the test modules, since we remove them after the build anyway. # This feature was added in Python 3.10+, however it wasn't until Python 3.11 @@ -156,7 +162,7 @@ fi # - https://github.com/docker-library/python/issues/810 # We only use `dpkg-buildflags` for Python versions where we build in shared mode (Python 3.9+), # since some of the options it enables interferes with the stripping of static libraries. -if [[ "${PYTHON_MAJOR_VERSION}" == 3.[8-9] ]]; then +if [[ "${PYTHON_MAJOR_VERSION}" == +(3.8|3.9) ]]; then EXTRA_CFLAGS='' LDFLAGS='-Wl,--strip-all' else @@ -168,7 +174,7 @@ CPU_COUNT="$(nproc)" make -j "${CPU_COUNT}" "EXTRA_CFLAGS=${EXTRA_CFLAGS}" "LDFLAGS=${LDFLAGS}" make install -if [[ "${PYTHON_MAJOR_VERSION}" == 3.[8-9] ]]; then +if [[ "${PYTHON_MAJOR_VERSION}" == +(3.8|3.9) ]]; then # On older versions of Python we're still building the static library, which has to be # manually stripped since the linker stripping enabled in LDFLAGS doesn't cover them. # We're using `--strip-unneeded` since `--strip-all` would remove the `.symtab` section @@ -213,6 +219,16 @@ find "${INSTALL_DIR}" -depth -type f -name "*.pyc" -delete # https://github.com/python/cpython/blob/v3.11.3/Makefile.pre.in#L2087-L2113 LD_LIBRARY_PATH="${SRC_DIR}" "${SRC_DIR}/python" -m compileall -f --invalidation-mode unchecked-hash --workers 0 "${INSTALL_DIR}" +# Delete entrypoint scripts (and their symlinks) that don't work with relocated Python since they +# hardcode the Python install directory in their shebangs (e.g. `#!/tmp/python/bin/python3.NN`). +# These scripts are rarely used in production, and can still be accessed via their Python module +# (e.g. `python -m pydoc`) if needed. +rm "${INSTALL_DIR}"/bin/{idle,pydoc}* +# The 2to3 module and entrypoint was removed from the stdlib in Python 3.13. +if [[ "${PYTHON_MAJOR_VERSION}" == +(3.8|3.9|3.10|3.11|3.12) ]]; then + rm "${INSTALL_DIR}"/bin/2to3* +fi + # Support using Python 3 via the version-less `python` command, for parity with virtualenvs, # the Python Docker images and to also ensure buildpack Python shadows any installed system # Python, should that provide a version-less alias too. diff --git a/builds/test_python_runtime.sh b/builds/test_python_runtime.sh index 9eb41a59..190c7b62 100755 --- a/builds/test_python_runtime.sh +++ b/builds/test_python_runtime.sh @@ -10,6 +10,8 @@ function abort() { exit 1 } +set -x + # We intentionally extract the Python runtime into a different directory to the one into which it # was originally installed before being packaged, to check that relocation works (since buildpacks # depend on it). Since the Python binary was built in shared mode, `LD_LIBRARY_PATH` must be set @@ -25,10 +27,26 @@ tar --zstd --extract --verbose --file "${ARCHIVE_FILEPATH}" --directory "${INSTA "${INSTALL_DIR}/bin/python3" --version "${INSTALL_DIR}/bin/python" --version +# Check the Python config script still exists/works after the deletion of scripts with broken shebang lines. +"${INSTALL_DIR}/bin/python3-config" --help + +set +x + +# Check that the broken bin entrypoints and symlinks (such as `idle3` and `pydoc3`) were deleted. +UNEXPECTED_BIN_FILES="$(find "${INSTALL_DIR}/bin" -type 'f,l' -not -name 'python*')" +if [[ -n "${UNEXPECTED_BIN_FILES}" ]]; then + echo "${UNEXPECTED_BIN_FILES}" + abort "The above files were found in the bin/ directory but were not expected!" +else + echo "No unexpected files found in the bin/ directory." +fi + # Check that all dynamically linked libraries exist in the run image (since it has fewer packages than the build image). LDD_OUTPUT=$(find "${INSTALL_DIR}" -type f,l \( -name 'python3' -o -name '*.so*' \) -exec ldd '{}' +) if grep 'not found' <<<"${LDD_OUTPUT}" | sort --unique; then abort "The above dynamically linked libraries were not found!" +else + echo "All dynamically linked libraries were found." fi # Check that optional and/or system library dependent stdlib modules were built. @@ -47,9 +65,11 @@ optional_stdlib_modules=( xml.parsers.expat zlib ) -if ! "${INSTALL_DIR}/bin/python3" -c "import $( +if "${INSTALL_DIR}/bin/python3" -c "import $( IFS=, echo "${optional_stdlib_modules[*]}" )"; then + echo "Successful imported: ${optional_stdlib_modules[*]}" +else abort "The above optional stdlib module failed to import! Check the compile logs to see if it was skipped due to missing libraries/headers." fi diff --git a/lib/pip.sh b/lib/pip.sh index 2fa7e14b..c8f55473 100644 --- a/lib/pip.sh +++ b/lib/pip.sh @@ -8,19 +8,41 @@ function pip::install_pip_setuptools_wheel() { # We use the pip wheel bundled within Python's standard library to install our chosen # pip version, since it's faster than `ensurepip` followed by an upgrade in place. local bundled_pip_module_path="${1}" + local python_major_version="${2}" # TODO: Either make these `local` or move elsewhere as part of the cache invalidation refactoring. PIP_VERSION=$(get_requirement_version 'pip') - SETUPTOOLS_VERSION=$(get_requirement_version 'setuptools') - WHEEL_VERSION=$(get_requirement_version 'wheel') meta_set "pip_version" "${PIP_VERSION}" - meta_set "setuptools_version" "${SETUPTOOLS_VERSION}" - meta_set "wheel_version" "${WHEEL_VERSION}" - puts-step "Installing pip ${PIP_VERSION}, setuptools ${SETUPTOOLS_VERSION} and wheel ${WHEEL_VERSION}" + local packages_to_install=( + "pip==${PIP_VERSION}" + ) + local packages_display_text="pip ${PIP_VERSION}" + + # We only install setuptools and wheel on Python 3.12 and older, since: + # - If either is not installed, pip will automatically install them into an isolated build + # environment if needed when installing packages from an sdist. This means that for + # all packages that correctly declare their metadata, it's no longer necessary to have + # them installed. + # - Most of the Python ecosystem has stopped installing them for Python 3.12+ already. + # See the Python CNB's removal for more details: https://github.com/heroku/buildpacks-python/pull/243 + if [[ "${python_major_version}" == +(3.8|3.9|3.10|3.11|3.12) ]]; then + SETUPTOOLS_VERSION=$(get_requirement_version 'setuptools') + WHEEL_VERSION=$(get_requirement_version 'wheel') + meta_set "setuptools_version" "${SETUPTOOLS_VERSION}" + meta_set "wheel_version" "${WHEEL_VERSION}" + + packages_to_install+=( + "setuptools==${SETUPTOOLS_VERSION}" + "wheel==${WHEEL_VERSION}" + ) + packages_display_text+=", setuptools ${SETUPTOOLS_VERSION} and wheel ${WHEEL_VERSION}" + fi + + puts-step "Installing ${packages_display_text}" /app/.heroku/python/bin/python "${bundled_pip_module_path}" install --quiet --disable-pip-version-check --no-cache-dir \ - "pip==${PIP_VERSION}" "setuptools==${SETUPTOOLS_VERSION}" "wheel==${WHEEL_VERSION}" + "${packages_to_install[@]}" } function pip::install_dependencies() { diff --git a/lib/python_version.sh b/lib/python_version.sh index 8b88e6e3..820daf3a 100644 --- a/lib/python_version.sh +++ b/lib/python_version.sh @@ -9,6 +9,7 @@ LATEST_PYTHON_3_9="3.9.20" LATEST_PYTHON_3_10="3.10.15" LATEST_PYTHON_3_11="3.11.10" LATEST_PYTHON_3_12="3.12.7" +LATEST_PYTHON_3_13="3.13.0" DEFAULT_PYTHON_FULL_VERSION="${LATEST_PYTHON_3_12}" DEFAULT_PYTHON_MAJOR_VERSION="${DEFAULT_PYTHON_FULL_VERSION%.*}" @@ -233,7 +234,7 @@ function python_version::resolve_python_version() { return 1 fi - if (((major == 3 && minor > 12) || major >= 4)); then + if (((major == 3 && minor > 13) || major >= 4)); then if [[ "${python_version_origin}" == "cached" ]]; then display_error <<-EOF Error: The cached Python version is not recognised. @@ -281,6 +282,7 @@ function python_version::resolve_python_version() { 3.10) echo "${LATEST_PYTHON_3_10}" ;; 3.11) echo "${LATEST_PYTHON_3_11}" ;; 3.12) echo "${LATEST_PYTHON_3_12}" ;; + 3.13) echo "${LATEST_PYTHON_3_13}" ;; *) utils::abort_internal_error "Unhandled Python major version: ${requested_python_version}" ;; esac } diff --git a/spec/fixtures/pipenv_python_3.13/Pipfile b/spec/fixtures/pipenv_python_3.13/Pipfile new file mode 100644 index 00000000..73f50fc8 --- /dev/null +++ b/spec/fixtures/pipenv_python_3.13/Pipfile @@ -0,0 +1,12 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +urllib3 = "*" + +[dev-packages] + +[requires] +python_version = "3.13" diff --git a/spec/fixtures/pipenv_python_3.13/Pipfile.lock b/spec/fixtures/pipenv_python_3.13/Pipfile.lock new file mode 100644 index 00000000..dcd0b3ca --- /dev/null +++ b/spec/fixtures/pipenv_python_3.13/Pipfile.lock @@ -0,0 +1,30 @@ +{ + "_meta": { + "hash": { + "sha256": "60dc67ee4223a391c5e0ae2b4d7ea54a7b245773d76b6ff82156dda97a3e4fb2" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.13" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "urllib3": { + "hashes": [ + "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac", + "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==2.2.3" + } + }, + "develop": {} +} diff --git a/spec/fixtures/python_3.13/requirements.txt b/spec/fixtures/python_3.13/requirements.txt new file mode 100644 index 00000000..a42590be --- /dev/null +++ b/spec/fixtures/python_3.13/requirements.txt @@ -0,0 +1 @@ +urllib3 diff --git a/spec/fixtures/python_3.13/runtime.txt b/spec/fixtures/python_3.13/runtime.txt new file mode 100644 index 00000000..dae7fec9 --- /dev/null +++ b/spec/fixtures/python_3.13/runtime.txt @@ -0,0 +1 @@ +python-3.13.0 diff --git a/spec/hatchet/pipenv_spec.rb b/spec/hatchet/pipenv_spec.rb index 942bdb80..91d4fbb1 100644 --- a/spec/hatchet/pipenv_spec.rb +++ b/spec/hatchet/pipenv_spec.rb @@ -208,6 +208,24 @@ include_examples 'builds using Pipenv with the requested Python version', '3.12', LATEST_PYTHON_3_12 end + context 'with a Pipfile.lock containing python_version 3.13' do + let(:app) { Hatchet::Runner.new('spec/fixtures/pipenv_python_3.13') } + + it 'builds with latest Python 3.13' do + app.deploy do |app| + expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX)) + remote: -----> Python app detected + remote: -----> Using Python 3.13 specified in Pipfile.lock + remote: -----> Installing Python #{LATEST_PYTHON_3_13} + remote: -----> Installing pip #{PIP_VERSION} + remote: -----> Installing Pipenv #{PIPENV_VERSION} + remote: -----> Installing dependencies with Pipenv + remote: Installing dependencies from Pipfile.lock \\(.+\\)... + REGEX + end + end + end + # As well as testing `python_full_version`, this also tests: # 1. That `python_full_version` takes precedence over `python_version`. # 2. That Pipenv works on the oldest Python version supported by all stacks. diff --git a/spec/hatchet/python_version_spec.rb b/spec/hatchet/python_version_spec.rb index e0d01049..e9751a92 100644 --- a/spec/hatchet/python_version_spec.rb +++ b/spec/hatchet/python_version_spec.rb @@ -5,15 +5,26 @@ RSpec.shared_examples 'builds with the requested Python version' do |requested_version| it "builds with Python #{requested_version}" do app.deploy do |app| - expect(clean_output(app.output)).to include(<<~OUTPUT) - remote: -----> Python app detected - remote: -----> Using Python #{requested_version} specified in runtime.txt - remote: -----> Installing Python #{requested_version} - remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} - remote: -----> Installing SQLite3 - remote: -----> Installing requirements with pip - remote: Collecting urllib3 (from -r requirements.txt (line 1)) - OUTPUT + if requested_version.start_with?('3.13.') + expect(clean_output(app.output)).to include(<<~OUTPUT) + remote: -----> Python app detected + remote: -----> Using Python #{requested_version} specified in runtime.txt + remote: -----> Installing Python #{requested_version} + remote: -----> Installing pip #{PIP_VERSION} + remote: -----> Installing requirements with pip + remote: Collecting urllib3 (from -r requirements.txt (line 1)) + OUTPUT + else + expect(clean_output(app.output)).to include(<<~OUTPUT) + remote: -----> Python app detected + remote: -----> Using Python #{requested_version} specified in runtime.txt + remote: -----> Installing Python #{requested_version} + remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} + remote: -----> Installing SQLite3 + remote: -----> Installing requirements with pip + remote: Collecting urllib3 (from -r requirements.txt (line 1)) + OUTPUT + end expect(app.run('python -V')).to include("Python #{requested_version}") end end @@ -200,6 +211,12 @@ include_examples 'builds with the requested Python version', LATEST_PYTHON_3_12 end + context 'when runtime.txt contains python-3.13.0' do + let(:app) { Hatchet::Runner.new('spec/fixtures/python_3.13') } + + include_examples 'builds with the requested Python version', LATEST_PYTHON_3_13 + end + context 'when runtime.txt contains an invalid Python version string' do let(:app) { Hatchet::Runner.new('spec/fixtures/python_version_invalid', allow_failure: true) } diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 3c0632a6..4fddb376 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -14,6 +14,7 @@ LATEST_PYTHON_3_10 = '3.10.15' LATEST_PYTHON_3_11 = '3.11.10' LATEST_PYTHON_3_12 = '3.12.7' +LATEST_PYTHON_3_13 = '3.13.0' DEFAULT_PYTHON_FULL_VERSION = LATEST_PYTHON_3_12 DEFAULT_PYTHON_MAJOR_VERSION = DEFAULT_PYTHON_FULL_VERSION.gsub(/\.\d+$/, '')