From 198f9bb8e8bfabc92c90635e732c302bd1e37350 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 29 Feb 2024 11:28:57 +0100 Subject: [PATCH] Switch from --user to venv environment for PROD image This PR introduces a joint way to treat the .local (--user) folder as both - venv and `--user` package installation. This allow two things: * we can use `uv` to built the production image, which gives 40%-50% saving for image build time. * user do not have to use `pip install --user` to install new packages * python -m venv --system-site-packages continues to use the .local packages from the .local installation (and not uses them if --system-site-packages is not used * you can use uv to install packages when you extend the image as a user * this PR switches to use `uv` by default for all prod images, but it adds a check if the image still builds with `pip` * we also switch to more PEP standard way of installing packages from local filesystem (package-name @ file:///FILE) Fixes: #37785 Co-authored-by: Wei Lee --- .github/workflows/build-images.yml | 2 + .github/workflows/ci.yml | 60 ++++ Dockerfile | 278 +++++++++++------- Dockerfile.ci | 82 ++---- .../doc/images/output_prod-image_build.svg | 206 ++++++------- .../doc/images/output_prod-image_build.txt | 2 +- .../airflow_breeze/commands/common_options.py | 9 + .../commands/production_image_commands.py | 4 + .../production_image_commands_config.py | 11 +- .../params/build_prod_params.py | 2 + docs/docker-stack/build-arg-ref.rst | 2 +- docs/docker-stack/build.rst | 74 +++-- docs/docker-stack/changelog.rst | 25 +- .../add-pypi-packages-constraints/Dockerfile | 20 ++ .../Dockerfile | 20 ++ .../extending/add-pypi-packages-uv/Dockerfile | 24 ++ scripts/docker/common.sh | 40 ++- ...all_pip_version.sh => create_prod_venv.sh} | 11 +- scripts/docker/get_package_specs.py | 46 +++ .../docker/install_additional_dependencies.sh | 5 +- scripts/docker/install_airflow.sh | 9 +- ...ll_airflow_dependencies_from_branch_tip.sh | 3 +- .../install_from_docker_context_files.sh | 73 ++--- scripts/docker/install_packaging_tools.sh | 23 ++ scripts/in_container/_in_container_utils.sh | 6 +- 25 files changed, 660 insertions(+), 377 deletions(-) create mode 100644 docs/docker-stack/docker-examples/extending/add-pypi-packages-constraints/Dockerfile create mode 100644 docs/docker-stack/docker-examples/extending/add-pypi-packages-uv-constraints/Dockerfile create mode 100644 docs/docker-stack/docker-examples/extending/add-pypi-packages-uv/Dockerfile rename scripts/docker/{install_pip_version.sh => create_prod_venv.sh} (88%) create mode 100755 scripts/docker/get_package_specs.py create mode 100644 scripts/docker/install_packaging_tools.sh diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index da76fcbeffdaf1..42dc2da0acc5d6 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -180,6 +180,7 @@ jobs: RUNS_ON: "${{ needs.build-info.outputs.runs-on }}" BACKEND: sqlite VERSION_SUFFIX_FOR_PYPI: "dev0" + USE_UV: "true" steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" @@ -258,6 +259,7 @@ jobs: BACKEND: sqlite VERSION_SUFFIX_FOR_PYPI: "dev0" INCLUDE_NOT_READY_PROVIDERS: "true" + USE_UV: "true" steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cb19201ddb052..461be73498d82a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -282,6 +282,7 @@ jobs: # Force more parallelism for build even on public images PARALLELISM: 6 VERSION_SUFFIX_FOR_PYPI: "dev0" + USE_UV: "true" steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" @@ -1863,6 +1864,7 @@ jobs: BACKEND: sqlite VERSION_SUFFIX_FOR_PYPI: "dev0" DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} + USE_UV: "true" steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" @@ -1898,6 +1900,58 @@ jobs: PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} + build-prod-images-pip: + strategy: + matrix: + python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }} + timeout-minutes: 80 + name: ${{needs.build-info.outputs.build-job-description}} PROD image pip (main) ${{matrix.python-version}} + runs-on: ["ubuntu-22.04"] + needs: [build-info, build-ci-images] + env: + DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} + DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }} + RUNS_ON: "${{needs.build-info.outputs.runs-on}}" + BACKEND: sqlite + VERSION_SUFFIX_FOR_PYPI: "dev0" + DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} + USE_UV: "false" + steps: + - name: Cleanup repo + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + if: > + needs.build-info.outputs.in-workflow-build == 'true' && + needs.build-info.outputs.default-branch == 'main' + - uses: actions/checkout@v4 + with: + ref: ${{ needs.build-info.outputs.targetCommitSha }} + persist-credentials: false + if: > + needs.build-info.outputs.in-workflow-build == 'true' && + needs.build-info.outputs.default-branch == 'main' + - name: "Install Breeze" + uses: ./.github/actions/breeze + with: + python-version: ${{ env.REPRODUCIBLE_PYTHON_VERSION }} + if: > + needs.build-info.outputs.in-workflow-build == 'true' && + needs.build-info.outputs.default-branch == 'main' + - name: Build PROD Image pip ${{ matrix.python-version }}:${{env.IMAGE_TAG}} + uses: ./.github/actions/build-prod-images + if: > + needs.build-info.outputs.in-workflow-build == 'true' && + needs.build-info.outputs.default-branch == 'main' + with: + build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} + chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} + python-version: ${{ matrix.python-version }} + env: + UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} + PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} + DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} + IMAGE_TAG: "pip-${{ github.event.pull_request.head.sha || github.sha }}" + build-prod-images-bullseye: strategy: matrix: @@ -1914,6 +1968,7 @@ jobs: BACKEND: sqlite VERSION_SUFFIX_FOR_PYPI: "dev0" DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} + USE_UV: "true" steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" @@ -1970,6 +2025,7 @@ jobs: BACKEND: sqlite VERSION_SUFFIX_FOR_PYPI: "dev0" DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} + USE_UV: "true" steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" @@ -2027,6 +2083,7 @@ jobs: BACKEND: sqlite VERSION_SUFFIX_FOR_PYPI: "dev0" DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} + USE_UV: "true" steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" @@ -2078,6 +2135,7 @@ jobs: BACKEND: sqlite VERSION_SUFFIX_FOR_PYPI: "dev0" DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} + USE_UV: "true" steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" @@ -2134,6 +2192,7 @@ jobs: BACKEND: sqlite VERSION_SUFFIX_FOR_PYPI: "dev0" DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} + USE_UV: "true" steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" @@ -2530,6 +2589,7 @@ jobs: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" # Force more parallelism for build even on small instances PARALLELISM: 6 + USE_UV: "true" if: > needs.build-info.outputs.in-workflow-build == 'true' && needs.build-info.outputs.canary-run != 'true' diff --git a/Dockerfile b/Dockerfile index a6ab7af291a0ce..ef04b451e23cc4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,8 +23,9 @@ # airflow-build-image - there all airflow dependencies can be installed (and # built - for those dependencies that require # build essentials). Airflow is installed there with -# --user switch so that all the dependencies are -# installed to ${HOME}/.local +# ${HOME}/.local virtualenv which is also considered +# As --user folder by python when creating venv with +# --system-site-packages # # main - this is the actual production image that is much # smaller because it does not contain all the build @@ -50,7 +51,7 @@ ARG PYTHON_BASE_IMAGE="python:3.8-slim-bookworm" ARG AIRFLOW_PIP_VERSION=24.0 ARG AIRFLOW_UV_VERSION=0.1.12 -ARG AIRFLOW_USE_UV="false" +ARG AIRFLOW_USE_UV="true" ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" ARG AIRFLOW_IMAGE_README_URL="https://raw.githubusercontent.com/apache/airflow/main/docs/docker-stack/README.md" @@ -412,18 +413,13 @@ if [[ ${INSTALL_POSTGRES_CLIENT:="true"} == "true" ]]; then fi EOF -# The content below is automatically copied from scripts/docker/install_pip_version.sh -COPY <<"EOF" /install_pip_version.sh +# The content below is automatically copied from scripts/docker/install_packaging_tools.sh +COPY <<"EOF" /install_packaging_tools.sh #!/usr/bin/env bash . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" common::get_colors -common::get_packaging_tool -common::get_airflow_version_specification -common::override_pip_version_if_needed -common::show_packaging_tool_version_and_location - -common::install_packaging_tool +common::install_packaging_tools EOF # The content below is automatically copied from scripts/docker/install_airflow_dependencies_from_branch_tip.sh @@ -454,7 +450,7 @@ function install_airflow_dependencies_from_branch_tip() { ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "apache-airflow[${AIRFLOW_EXTRAS}] @ https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz" - common::install_packaging_tool + common::install_packaging_tools # Uninstall airflow and providers to keep only the dependencies. In the future when # planned https://github.com/pypa/pip/issues/11440 is implemented in pip we might be able to use this # flag and skip the remove step. @@ -469,7 +465,6 @@ function install_airflow_dependencies_from_branch_tip() { common::get_colors common::get_packaging_tool common::get_airflow_version_specification -common::override_pip_version_if_needed common::get_constraints_location common::show_packaging_tool_version_and_location @@ -509,7 +504,11 @@ function common::get_packaging_tool() { echo export PACKAGING_TOOL="uv" export PACKAGING_TOOL_CMD="uv pip" - export EXTRA_INSTALL_FLAGS="--python ${PYTHON_BIN}" + if [[ -z ${VIRTUAL_ENV=} ]]; then + export EXTRA_INSTALL_FLAGS="--python ${PYTHON_BIN}" + else + export EXTRA_INSTALL_FLAGS="" + fi export EXTRA_UNINSTALL_FLAGS="--python ${PYTHON_BIN}" export RESOLUTION_HIGHEST_FLAG="--resolution highest" export RESOLUTION_LOWEST_DIRECT_FLAG="--resolution lowest-direct" @@ -534,14 +533,6 @@ function common::get_airflow_version_specification() { fi } -function common::override_pip_version_if_needed() { - if [[ -n ${AIRFLOW_VERSION} ]]; then - if [[ ${AIRFLOW_VERSION} =~ ^2\.0.* || ${AIRFLOW_VERSION} =~ ^1\.* ]]; then - export AIRFLOW_PIP_VERSION=24.0 - fi - fi -} - function common::get_constraints_location() { # auto-detect Airflow-constraint reference and location if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE=}" ]]; then @@ -558,6 +549,8 @@ function common::get_constraints_location() { python_version="$(python --version 2>/dev/stdout | cut -d " " -f 2 | cut -d "." -f 1-2)" AIRFLOW_CONSTRAINTS_LOCATION="${constraints_base}/${AIRFLOW_CONSTRAINTS_MODE}-${python_version}.txt" fi + + curl -o "${HOME}/constraints.txt" "${AIRFLOW_CONSTRAINTS_LOCATION}" } function common::show_packaging_tool_version_and_location() { @@ -568,12 +561,12 @@ function common::show_packaging_tool_version_and_location() { echo "Using pip: $(pip --version)" else echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}" - echo "uv on path: $(which uv)" + echo "uv on path: $(which uv 2>/dev/null || echo "Not installed")" echo "Using uv: $(uv --version)" fi } -function common::install_packaging_tool() { +function common::install_packaging_tools() { echo echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}" echo @@ -584,17 +577,15 @@ function common::install_packaging_tool() { # shellcheck disable=SC2086 pip install --root-user-action ignore --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" fi - if [[ ${AIRFLOW_USE_UV} == "true" ]]; then - echo - echo "${COLOR_BLUE}Installing uv version ${AIRFLOW_UV_VERSION}${COLOR_RESET}" - echo - if [[ ${AIRFLOW_UV_VERSION} =~ .*https.* ]]; then - # shellcheck disable=SC2086 - pip install --root-user-action ignore --disable-pip-version-check "uv @ ${AIRFLOW_UV_VERSION}" - else - # shellcheck disable=SC2086 - pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}" - fi + echo + echo "${COLOR_BLUE}Installing uv version ${AIRFLOW_UV_VERSION}${COLOR_RESET}" + echo + if [[ ${AIRFLOW_UV_VERSION} =~ .*https.* ]]; then + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "uv @ ${AIRFLOW_UV_VERSION}" + else + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}" fi mkdir -p "${HOME}/.local/bin" } @@ -653,6 +644,7 @@ COPY <<"EOF" /install_from_docker_context_files.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" + function install_airflow_and_providers_from_docker_context_files(){ if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,} @@ -668,46 +660,36 @@ function install_airflow_and_providers_from_docker_context_files(){ exit 1 fi - # shellcheck disable=SC2206 - local packaging_flags=( - # Don't quote this -- if it is empty we don't want it to create an - # empty array element - --find-links="file:///docker-context-files" - ) + # This is needed to get package names for local context packages + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} --constraint ${HOME}/constraints.txt packaging - # Find Apache Airflow packages in docker-context files - local reinstalling_apache_airflow_package - reinstalling_apache_airflow_package=$(ls \ - /docker-context-files/apache?airflow?[0-9]*.{whl,tar.gz} 2>/dev/null || true) - # Add extras when installing airflow - if [[ -n "${reinstalling_apache_airflow_package}" ]]; then - # When a provider depends on a dev version of Airflow, we need to - # specify `apache-airflow==$VER`, otherwise pip will look for it on - # pip, and fail to find it - - # This will work as long as the wheel file is correctly named, which it - # will be if it was build by wheel tooling - local ver - ver=$(basename "$reinstalling_apache_airflow_package" | cut -d "-" -f 2) - reinstalling_apache_airflow_package="apache-airflow[${AIRFLOW_EXTRAS}]==$ver" + if [[ -n ${AIRFLOW_EXTRAS=} ]]; then + AIRFLOW_EXTRAS_TO_INSTALL="[${AIRFLOW_EXTRAS}]" + else + AIRFLOW_EXTRAS_TO_INSTALL="" fi - if [[ -z "${reinstalling_apache_airflow_package}" && ${AIRFLOW_VERSION=} != "" ]]; then + # Find Apache Airflow package in docker-context files + readarray -t install_airflow_package < <(EXTRAS="${AIRFLOW_EXTRAS_TO_INSTALL}" \ + python /scripts/docker/get_package_specs.py /docker-context-files/apache?airflow?[0-9]*.{whl,tar.gz} 2>/dev/null || true) + echo + echo "${COLOR_BLUE}Found airflow packages in docker-context-files folder: ${install_airflow_package[*]}${COLOR_RESET}" + echo + + if [[ -z "${install_airflow_package[*]}" && ${AIRFLOW_VERSION=} != "" ]]; then # When we install only provider packages from docker-context files, we need to still # install airflow from PyPI when AIRFLOW_VERSION is set. This handles the case where # pre-release dockerhub image of airflow is built, but we want to install some providers from # docker-context files - reinstalling_apache_airflow_package="apache-airflow[${AIRFLOW_EXTRAS}]==${AIRFLOW_VERSION}" - fi - # Find Apache Airflow packages in docker-context files - local reinstalling_apache_airflow_providers_packages - reinstalling_apache_airflow_providers_packages=$(ls \ - /docker-context-files/apache?airflow?providers*.{whl,tar.gz} 2>/dev/null || true) - if [[ -z "${reinstalling_apache_airflow_package}" && \ - -z "${reinstalling_apache_airflow_providers_packages}" ]]; then - return + install_airflow_package=("apache-airflow[${AIRFLOW_EXTRAS}]==${AIRFLOW_VERSION}") fi + # Find Provider packages in docker-context files + readarray -t installing_providers_packages< <(python /scripts/docker/get_package_specs.py /docker-context-files/apache?airflow?providers*.{whl,tar.gz} 2>/dev/null || true) + echo + echo "${COLOR_BLUE}Found provider packages in docker-context-files folder: ${installing_providers_packages[*]}${COLOR_RESET}" + echo + if [[ ${USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES=} == "true" ]]; then local python_version python_version=$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') @@ -719,19 +701,19 @@ function install_airflow_and_providers_from_docker_context_files(){ echo # force reinstall all airflow + provider packages with constraints found in set -x - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "${packaging_flags[@]}" --upgrade \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade \ ${ADDITIONAL_PIP_INSTALL_FLAGS} --constraint "${local_constraints_file}" \ - ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} + "${install_airflow_package[@]}" "${installing_providers_packages[@]}" set +x else echo echo "${COLOR_BLUE}Installing docker-context-files packages with constraints from GitHub${COLOR_RESET}" echo set -x - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "${packaging_flags[@]}" \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ - --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" \ - ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} + --constraint "${HOME}/constraints.txt" \ + "${install_airflow_package[@]}" "${installing_providers_packages[@]}" set +x fi else @@ -739,12 +721,12 @@ function install_airflow_and_providers_from_docker_context_files(){ echo "${COLOR_BLUE}Installing docker-context-files packages without constraints${COLOR_RESET}" echo set -x - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "${packaging_flags[@]}" \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ - ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} + "${install_airflow_package[@]}" "${installing_providers_packages[@]}" set +x fi - common::install_packaging_tool + common::install_packaging_tools pip check } @@ -760,7 +742,7 @@ function install_all_other_packages_from_docker_context_files() { set -x ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} \ --force-reinstall --no-deps --no-index ${reinstalling_other_packages} - common::install_packaging_tool + common::install_packaging_tools set +x fi } @@ -768,7 +750,6 @@ function install_all_other_packages_from_docker_context_files() { common::get_colors common::get_packaging_tool common::get_airflow_version_specification -common::override_pip_version_if_needed common::get_constraints_location common::show_packaging_tool_version_and_location @@ -777,6 +758,41 @@ install_airflow_and_providers_from_docker_context_files install_all_other_packages_from_docker_context_files EOF +# The content below is automatically copied from scripts/docker/get_package_specs.py +COPY <<"EOF" /get_package_specs.py +#!/usr/bin/env python +from __future__ import annotations + +import os +import sys +from pathlib import Path + +from packaging.utils import ( + InvalidSdistFilename, + InvalidWheelFilename, + parse_sdist_filename, + parse_wheel_filename, +) + + +def print_package_specs(extras: str = ""): + for package_path in sys.argv[1:]: + try: + package, _, _, _ = parse_wheel_filename(Path(package_path).name) + except InvalidWheelFilename: + try: + package, _ = parse_sdist_filename(Path(package_path).name) + except InvalidSdistFilename: + print(f"Could not parse package name from {package_path}", file=sys.stderr) + continue + print(f"{package}{extras} @ file://{package_path}") + + +if __name__ == "__main__": + print_package_specs(extras=os.environ.get("EXTRAS", "")) +EOF + + # The content below is automatically copied from scripts/docker/install_airflow.sh COPY <<"EOF" /install_airflow.sh #!/usr/bin/env bash @@ -819,7 +835,7 @@ function install_airflow() { "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} set +x - common::install_packaging_tool + common::install_packaging_tools echo echo "${COLOR_BLUE}Running '${PACKAGING_TOOL} check'${COLOR_RESET}" echo @@ -832,14 +848,14 @@ function install_airflow() { ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ - --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true - common::install_packaging_tool + --constraint ${HOME}/constraints.txt || true + common::install_packaging_tools # then upgrade if needed without using constraints to account for new limits in pyproject.toml ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_LOWEST_DIRECT_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" - common::install_packaging_tool + common::install_packaging_tools set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -852,7 +868,6 @@ function install_airflow() { common::get_colors common::get_packaging_tool common::get_airflow_version_specification -common::override_pip_version_if_needed common::get_constraints_location common::show_packaging_tool_version_and_location @@ -878,7 +893,7 @@ function install_additional_dependencies() { ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_HIGHEST_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} - common::install_packaging_tool + common::install_packaging_tools set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -892,7 +907,7 @@ function install_additional_dependencies() { ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "${RESOLUTION_LOWEST_DIRECT_FLAG}" \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} - common::install_packaging_tool + common::install_packaging_tools set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -904,13 +919,47 @@ function install_additional_dependencies() { common::get_colors common::get_packaging_tool common::get_airflow_version_specification -common::override_pip_version_if_needed common::get_constraints_location common::show_packaging_tool_version_and_location install_additional_dependencies EOF +# The content below is automatically copied from scripts/docker/create_prod_venv.sh +COPY <<"EOF" /create_prod_venv.sh +#!/usr/bin/env bash +. "$( dirname "${BASH_SOURCE[0]}" )/common.sh" + +function create_prod_venv() { + rm -rf ~/.local + python -m venv ~/.local +} + +common::get_colors +common::get_packaging_tool +common::show_packaging_tool_version_and_location +create_prod_venv +common::install_packaging_tools +EOF + +# The content below is automatically copied from scripts/docker/create_prod_venv.sh +COPY <<"EOF" /create_prod_venv.sh +#!/usr/bin/env bash +. "$( dirname "${BASH_SOURCE[0]}" )/common.sh" + +function create_prod_venv() { + rm -rf ~/.local + python -m venv ~/.local +} + +common::get_colors +common::get_packaging_tool +common::show_packaging_tool_version_and_location +create_prod_venv +common::install_packaging_tools +EOF + + # The content below is automatically copied from scripts/docker/entrypoint_prod.sh COPY <<"EOF" /entrypoint_prod.sh @@ -1292,9 +1341,6 @@ ARG INSTALL_MYSQL_CLIENT="true" ARG INSTALL_MYSQL_CLIENT_TYPE="mariadb" ARG INSTALL_MSSQL_CLIENT="true" ARG INSTALL_POSTGRES_CLIENT="true" -ARG AIRFLOW_PIP_VERSION -ARG AIRFLOW_UV_VERSION -ARG AIRFLOW_USE_UV ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \ INSTALL_MYSQL_CLIENT_TYPE=${INSTALL_MYSQL_CLIENT_TYPE} \ @@ -1315,9 +1361,6 @@ ENV PATH=${PATH}:/opt/mssql-tools/bin # By default we do not install from docker context files but if we decide to install from docker context # files, we should override those variables to "docker-context-files" ARG DOCKER_CONTEXT_FILES="Dockerfile" - -COPY ${DOCKER_CONTEXT_FILES} /docker-context-files - ARG AIRFLOW_HOME ARG AIRFLOW_USER_HOME_DIR ARG AIRFLOW_UID @@ -1326,6 +1369,8 @@ RUN adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-passw --quiet "airflow" --uid "${AIRFLOW_UID}" --gid "0" --home "${AIRFLOW_USER_HOME_DIR}" && \ mkdir -p ${AIRFLOW_HOME} && chown -R "airflow:0" "${AIRFLOW_USER_HOME_DIR}" ${AIRFLOW_HOME} +COPY --chown=${AIRFLOW_UID}:0 ${DOCKER_CONTEXT_FILES} /docker-context-files + USER airflow ARG AIRFLOW_REPO=apache/airflow @@ -1375,6 +1420,10 @@ RUN if [[ -f /docker-context-files/pip.conf ]]; then \ # Additional PIP flags passed to all pip install commands except reinstalling pip itself ARG ADDITIONAL_PIP_INSTALL_FLAGS="" +ARG AIRFLOW_PIP_VERSION +ARG AIRFLOW_UV_VERSION +ARG AIRFLOW_USE_UV + ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ AIRFLOW_USE_UV=${AIRFLOW_USE_UV} \ @@ -1392,21 +1441,20 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ AIRFLOW_CONSTRAINTS_REFERENCE=${AIRFLOW_CONSTRAINTS_REFERENCE} \ AIRFLOW_CONSTRAINTS_LOCATION=${AIRFLOW_CONSTRAINTS_LOCATION} \ DEFAULT_CONSTRAINTS_BRANCH=${DEFAULT_CONSTRAINTS_BRANCH} \ - PATH=${PATH}:${AIRFLOW_USER_HOME_DIR}/.local/bin \ + PATH=${AIRFLOW_USER_HOME_DIR}/.local/bin:${PATH} \ PIP_PROGRESS_BAR=${PIP_PROGRESS_BAR} \ ADDITIONAL_PIP_INSTALL_FLAGS=${ADDITIONAL_PIP_INSTALL_FLAGS} \ AIRFLOW_USER_HOME_DIR=${AIRFLOW_USER_HOME_DIR} \ AIRFLOW_HOME=${AIRFLOW_HOME} \ AIRFLOW_UID=${AIRFLOW_UID} \ AIRFLOW_INSTALL_EDITABLE_FLAG="" \ - UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} \ - # By default PIP installs everything to ~/.local - PIP_USER="true" + UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} + # Copy all scripts required for installation - changing any of those should lead to # rebuilding from here -COPY --from=scripts common.sh install_pip_version.sh \ - install_airflow_dependencies_from_branch_tip.sh /scripts/docker/ +COPY --from=scripts common.sh install_packaging_tools.sh \ + install_airflow_dependencies_from_branch_tip.sh create_prod_venv.sh /scripts/docker/ # We can set this value to true in case we want to install .whl/.tar.gz packages placed in the # docker-context-files folder. This can be done for both additional packages you want to install @@ -1425,13 +1473,19 @@ ARG USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES="false" ARG AIRFLOW_CI_BUILD_EPOCH="10" ENV AIRFLOW_CI_BUILD_EPOCH=${AIRFLOW_CI_BUILD_EPOCH} + # In case of Production build image segment we want to pre-install main version of airflow # dependencies from GitHub so that we do not have to always reinstall it from the scratch. # The Airflow and providers are uninstalled, only dependencies remain # the cache is only used when "upgrade to newer dependencies" is not set to automatically # account for removed dependencies (we do not install them in the first place) and in case # INSTALL_PACKAGES_FROM_CONTEXT is not set (because then caching it from main makes no sense). -RUN bash /scripts/docker/install_pip_version.sh; \ + +# By default PIP installs everything to ~/.local and it's also treated as VIRTUALENV +ENV VIRTUAL_ENV="${AIRFLOW_USER_HOME_DIR}/.local" + +RUN bash /scripts/docker/install_packaging_tools.sh; \ + bash /scripts/docker/create_prod_venv.sh; \ if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \ ${INSTALL_PACKAGES_FROM_CONTEXT} == "false" && \ ${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \ @@ -1454,7 +1508,7 @@ ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} \ WORKDIR ${AIRFLOW_HOME} COPY --from=scripts install_from_docker_context_files.sh install_airflow.sh \ - install_additional_dependencies.sh /scripts/docker/ + install_additional_dependencies.sh create_prod_venv.sh get_package_specs.py /scripts/docker/ # Useful for creating a cache id based on the underlying architecture, preventing the use of cached python packages from # an incorrect architecture. @@ -1464,7 +1518,7 @@ ARG PIP_CACHE_EPOCH="9" # hadolint ignore=SC2086, SC2010, DL3042 RUN --mount=type=cache,id=$PYTHON_BASE_IMAGE-$AIRFLOW_PIP_VERSION-$TARGETARCH-$PIP_CACHE_EPOCH,target=/tmp/.cache/pip,uid=${AIRFLOW_UID} \ - if [[ ${INSTALL_PACKAGES_FROM_CONTEXT} == "true" ]]; then \ + if [[ ${INSTALL_PACKAGES_FROM_CONTEXT} == "true" ]]; then \ bash /scripts/docker/install_from_docker_context_files.sh; \ fi; \ if ! airflow version 2>/dev/null >/dev/null; then \ @@ -1476,8 +1530,8 @@ RUN --mount=type=cache,id=$PYTHON_BASE_IMAGE-$AIRFLOW_PIP_VERSION-$TARGETARCH-$P find "${AIRFLOW_USER_HOME_DIR}/.local/" -name '*.pyc' -print0 | xargs -0 rm -f || true ; \ find "${AIRFLOW_USER_HOME_DIR}/.local/" -type d -name '__pycache__' -print0 | xargs -0 rm -rf || true ; \ # make sure that all directories and files in .local are also group accessible - find "${AIRFLOW_USER_HOME_DIR}/.local" -executable -print0 | xargs --null chmod g+x; \ - find "${AIRFLOW_USER_HOME_DIR}/.local" -print0 | xargs --null chmod g+rw + find "${AIRFLOW_USER_HOME_DIR}/.local" -executable ! -type l -print0 | xargs --null chmod g+x; \ + find "${AIRFLOW_USER_HOME_DIR}/.local" ! -type l -print0 | xargs --null chmod g+rw # In case there is a requirements.txt file in "docker-context-files" it will be installed # during the build additionally to whatever has been installed so far. It is recommended that @@ -1485,7 +1539,7 @@ RUN --mount=type=cache,id=$PYTHON_BASE_IMAGE-$AIRFLOW_PIP_VERSION-$TARGETARCH-$P # hadolint ignore=DL3042 RUN --mount=type=cache,id=additional-requirements-$PYTHON_BASE_IMAGE-$AIRFLOW_PIP_VERSION-$TARGETARCH-$PIP_CACHE_EPOCH,target=/tmp/.cache/pip,uid=${AIRFLOW_UID} \ if [[ -f /docker-context-files/requirements.txt ]]; then \ - pip install --user -r /docker-context-files/requirements.txt; \ + pip install -r /docker-context-files/requirements.txt; \ fi ############################################################################################## @@ -1507,9 +1561,6 @@ LABEL org.apache.airflow.distro="debian" \ org.apache.airflow.uid="${AIRFLOW_UID}" ARG PYTHON_BASE_IMAGE -ARG AIRFLOW_PIP_VERSION -ARG AIRFLOW_UV_VERSION -ARG AIRFLOW_USE_UV ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \ # Make sure noninteractive debian install is used and language variables set @@ -1550,6 +1601,7 @@ ARG AIRFLOW_HOME # By default PIP installs everything to ~/.local ENV PATH="${AIRFLOW_USER_HOME_DIR}/.local/bin:${PATH}" \ + VIRTUAL_ENV="${AIRFLOW_USER_HOME_DIR}/.local" \ AIRFLOW_UID=${AIRFLOW_UID} \ AIRFLOW_USER_HOME_DIR=${AIRFLOW_USER_HOME_DIR} \ AIRFLOW_HOME=${AIRFLOW_HOME} @@ -1575,8 +1627,12 @@ RUN bash /scripts/docker/install_mysql.sh prod \ && mkdir -pv "${AIRFLOW_HOME}/logs" \ && chown -R airflow:0 "${AIRFLOW_USER_HOME_DIR}" "${AIRFLOW_HOME}" \ && chmod -R g+rw "${AIRFLOW_USER_HOME_DIR}" "${AIRFLOW_HOME}" \ - && find "${AIRFLOW_HOME}" -executable -print0 | xargs --null chmod g+x \ - && find "${AIRFLOW_USER_HOME_DIR}" -executable -print0 | xargs --null chmod g+x + && find "${AIRFLOW_HOME}" -executable ! -type l -print0 | xargs --null chmod g+x \ + && find "${AIRFLOW_USER_HOME_DIR}" -executable ! -type l -print0 | xargs --null chmod g+x + +ARG AIRFLOW_PIP_VERSION +ARG AIRFLOW_UV_VERSION +ARG AIRFLOW_USE_UV COPY --from=airflow-build-image --chown=airflow:0 \ "${AIRFLOW_USER_HOME_DIR}/.local" "${AIRFLOW_USER_HOME_DIR}/.local" @@ -1585,9 +1641,6 @@ COPY --from=scripts clean-logs.sh /clean-logs COPY --from=scripts airflow-scheduler-autorestart.sh /airflow-scheduler-autorestart -ARG AIRFLOW_PIP_VERSION -ARG AIRFLOW_UV_VERSION -ARG AIRFLOW_USE_UV # Make /etc/passwd root-group-writeable so that user can be dynamically added by OpenShift # See https://github.com/apache/airflow/issues/9248 @@ -1604,8 +1657,8 @@ RUN sed --in-place=.bak "s/secure_path=\"/secure_path=\"\/.venv\/bin:/" /etc/sud ARG AIRFLOW_VERSION -COPY --from=scripts install_pip_version.sh /scripts/docker/ -RUN bash /scripts/docker/install_pip_version.sh +COPY --from=scripts install_packaging_tools.sh /scripts/docker/ +RUN bash /scripts/docker/install_packaging_tools.sh # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation # to learn more about the way how signals are handled by the image @@ -1614,7 +1667,6 @@ ENV DUMB_INIT_SETSID="1" \ PS1="(airflow)" \ AIRFLOW_VERSION=${AIRFLOW_VERSION} \ AIRFLOW__CORE__LOAD_EXAMPLES="false" \ - PIP_USER="true" \ PATH="/root/bin:${PATH}" \ AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ diff --git a/Dockerfile.ci b/Dockerfile.ci index 9b1aaf3aa0d61f..10071c00ed0dea 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -370,18 +370,13 @@ if [[ ${INSTALL_POSTGRES_CLIENT:="true"} == "true" ]]; then fi EOF -# The content below is automatically copied from scripts/docker/install_pip_version.sh -COPY <<"EOF" /install_pip_version.sh +# The content below is automatically copied from scripts/docker/install_packaging_tools.sh +COPY <<"EOF" /install_packaging_tools.sh #!/usr/bin/env bash . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" common::get_colors -common::get_packaging_tool -common::get_airflow_version_specification -common::override_pip_version_if_needed -common::show_packaging_tool_version_and_location - -common::install_packaging_tool +common::install_packaging_tools EOF # The content below is automatically copied from scripts/docker/install_airflow_dependencies_from_branch_tip.sh @@ -412,7 +407,7 @@ function install_airflow_dependencies_from_branch_tip() { ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "apache-airflow[${AIRFLOW_EXTRAS}] @ https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz" - common::install_packaging_tool + common::install_packaging_tools # Uninstall airflow and providers to keep only the dependencies. In the future when # planned https://github.com/pypa/pip/issues/11440 is implemented in pip we might be able to use this # flag and skip the remove step. @@ -427,7 +422,6 @@ function install_airflow_dependencies_from_branch_tip() { common::get_colors common::get_packaging_tool common::get_airflow_version_specification -common::override_pip_version_if_needed common::get_constraints_location common::show_packaging_tool_version_and_location @@ -467,7 +461,11 @@ function common::get_packaging_tool() { echo export PACKAGING_TOOL="uv" export PACKAGING_TOOL_CMD="uv pip" - export EXTRA_INSTALL_FLAGS="--python ${PYTHON_BIN}" + if [[ -z ${VIRTUAL_ENV=} ]]; then + export EXTRA_INSTALL_FLAGS="--python ${PYTHON_BIN}" + else + export EXTRA_INSTALL_FLAGS="" + fi export EXTRA_UNINSTALL_FLAGS="--python ${PYTHON_BIN}" export RESOLUTION_HIGHEST_FLAG="--resolution highest" export RESOLUTION_LOWEST_DIRECT_FLAG="--resolution lowest-direct" @@ -492,14 +490,6 @@ function common::get_airflow_version_specification() { fi } -function common::override_pip_version_if_needed() { - if [[ -n ${AIRFLOW_VERSION} ]]; then - if [[ ${AIRFLOW_VERSION} =~ ^2\.0.* || ${AIRFLOW_VERSION} =~ ^1\.* ]]; then - export AIRFLOW_PIP_VERSION=24.0 - fi - fi -} - function common::get_constraints_location() { # auto-detect Airflow-constraint reference and location if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE=}" ]]; then @@ -516,6 +506,8 @@ function common::get_constraints_location() { python_version="$(python --version 2>/dev/stdout | cut -d " " -f 2 | cut -d "." -f 1-2)" AIRFLOW_CONSTRAINTS_LOCATION="${constraints_base}/${AIRFLOW_CONSTRAINTS_MODE}-${python_version}.txt" fi + + curl -o "${HOME}/constraints.txt" "${AIRFLOW_CONSTRAINTS_LOCATION}" } function common::show_packaging_tool_version_and_location() { @@ -526,12 +518,12 @@ function common::show_packaging_tool_version_and_location() { echo "Using pip: $(pip --version)" else echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}" - echo "uv on path: $(which uv)" + echo "uv on path: $(which uv 2>/dev/null || echo "Not installed")" echo "Using uv: $(uv --version)" fi } -function common::install_packaging_tool() { +function common::install_packaging_tools() { echo echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}" echo @@ -542,17 +534,15 @@ function common::install_packaging_tool() { # shellcheck disable=SC2086 pip install --root-user-action ignore --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" fi - if [[ ${AIRFLOW_USE_UV} == "true" ]]; then - echo - echo "${COLOR_BLUE}Installing uv version ${AIRFLOW_UV_VERSION}${COLOR_RESET}" - echo - if [[ ${AIRFLOW_UV_VERSION} =~ .*https.* ]]; then - # shellcheck disable=SC2086 - pip install --root-user-action ignore --disable-pip-version-check "uv @ ${AIRFLOW_UV_VERSION}" - else - # shellcheck disable=SC2086 - pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}" - fi + echo + echo "${COLOR_BLUE}Installing uv version ${AIRFLOW_UV_VERSION}${COLOR_RESET}" + echo + if [[ ${AIRFLOW_UV_VERSION} =~ .*https.* ]]; then + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "uv @ ${AIRFLOW_UV_VERSION}" + else + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}" fi mkdir -p "${HOME}/.local/bin" } @@ -659,7 +649,7 @@ function install_airflow() { "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} set +x - common::install_packaging_tool + common::install_packaging_tools echo echo "${COLOR_BLUE}Running '${PACKAGING_TOOL} check'${COLOR_RESET}" echo @@ -672,14 +662,14 @@ function install_airflow() { ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ - --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true - common::install_packaging_tool + --constraint ${HOME}/constraints.txt || true + common::install_packaging_tools # then upgrade if needed without using constraints to account for new limits in pyproject.toml ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_LOWEST_DIRECT_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" - common::install_packaging_tool + common::install_packaging_tools set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -692,7 +682,6 @@ function install_airflow() { common::get_colors common::get_packaging_tool common::get_airflow_version_specification -common::override_pip_version_if_needed common::get_constraints_location common::show_packaging_tool_version_and_location @@ -718,7 +707,7 @@ function install_additional_dependencies() { ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_HIGHEST_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} - common::install_packaging_tool + common::install_packaging_tools set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -732,7 +721,7 @@ function install_additional_dependencies() { ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "${RESOLUTION_LOWEST_DIRECT_FLAG}" \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} - common::install_packaging_tool + common::install_packaging_tools set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -744,7 +733,6 @@ function install_additional_dependencies() { common::get_colors common::get_packaging_tool common::get_airflow_version_specification -common::override_pip_version_if_needed common::get_constraints_location common::show_packaging_tool_version_and_location @@ -1205,12 +1193,6 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\ AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ AIRFLOW_UV_VERSION=${AIRFLOW_UV_VERSION} \ AIRFLOW_USE_UV=${AIRFLOW_USE_UV} \ -# In the CI image we always: -# * install MySQL, MsSQL -# * install airflow from current sources, not from PyPI package -# * install airflow without `--user` flag -# * install airflow in editable mode -# * install always current version of airflow INSTALL_MYSQL_CLIENT="true" \ INSTALL_MYSQL_CLIENT_TYPE=${INSTALL_MYSQL_CLIENT_TYPE} \ INSTALL_MSSQL_CLIENT="true" \ @@ -1229,7 +1211,7 @@ RUN echo "Airflow version: ${AIRFLOW_VERSION}" # Copy all scripts required for installation - changing any of those should lead to # rebuilding from here -COPY --from=scripts install_pip_version.sh install_airflow_dependencies_from_branch_tip.sh \ +COPY --from=scripts install_packaging_tools.sh install_airflow_dependencies_from_branch_tip.sh \ common.sh /scripts/docker/ # We are first creating a venv where all python packages and .so binaries needed by those are @@ -1240,7 +1222,7 @@ COPY --from=scripts install_pip_version.sh install_airflow_dependencies_from_bra # The Airflow and providers are uninstalled, only dependencies remain. # the cache is only used when "upgrade to newer dependencies" is not set to automatically # account for removed dependencies (we do not install them in the first place) -RUN bash /scripts/docker/install_pip_version.sh; \ +RUN bash /scripts/docker/install_packaging_tools.sh; \ if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" ]]; then \ bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \ fi @@ -1290,12 +1272,12 @@ COPY --from=scripts entrypoint_ci.sh /entrypoint COPY --from=scripts entrypoint_exec.sh /entrypoint-exec RUN chmod a+x /entrypoint /entrypoint-exec -COPY --from=scripts install_pip_version.sh install_additional_dependencies.sh /scripts/docker/ +COPY --from=scripts install_packaging_tools.sh install_additional_dependencies.sh /scripts/docker/ # Additional python deps to install ARG ADDITIONAL_PYTHON_DEPS="" -RUN bash /scripts/docker/install_pip_version.sh; \ +RUN bash /scripts/docker/install_packaging_tools.sh; \ if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \ bash /scripts/docker/install_additional_dependencies.sh; \ fi diff --git a/dev/breeze/doc/images/output_prod-image_build.svg b/dev/breeze/doc/images/output_prod-image_build.svg index be7de879be8310..94d173ae69b2c1 100644 --- a/dev/breeze/doc/images/output_prod-image_build.svg +++ b/dev/breeze/doc/images/output_prod-image_build.svg @@ -1,4 +1,4 @@ - + - + @@ -348,9 +348,12 @@ + + + - Command: prod-image build + Command: prod-image build @@ -366,102 +369,103 @@ Build Production image. Include building multiple images for all or selected Python versions sequentially. ╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.8< | 3.9 | 3.10 | 3.11)                                 │ -│[default: 3.8]                                              │ -│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ -│--image-tagTag the image after building it.(TEXT)[default: latest]│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when │ -│you build or pull image with --image-tag.                                            │ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ -│[default: registry]                          │ -│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ -│--build-progressBuild progress.(auto | plain | tty)[default: auto]│ -│--docker-hostOptional - docker host to use when running docker commands. When set, the `--builder`│ -│option is ignored when building images.                                              │ -│(TEXT)                                                                               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of parameters.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.8 3.9 3.10 3.11]                                                  │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced build options (for power users) ───────────────────────────────────────────────────────────────────────────╮ -│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`       │ -│itself).                                                                          │ -│(TEXT)                                                                            │ -│--commit-shaCommit SHA that is used to build the images.(TEXT)│ -│--debian-versionDebian version used in Airflow image as base for building images.│ -│(bookworm | bullseye)                                            │ -│[default: bookworm]                                              │ -│--python-imageIf specified this is the base python image used to build the image. Should be     │ -│something like: python:VERSION-slim-bookworm.                                     │ -│(TEXT)                                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Selecting constraint location (for power users) ────────────────────────────────────────────────────────────────────╮ -│--airflow-constraints-locationLocation of airflow constraints to use (remote URL or local context file).(TEXT)│ -│--airflow-constraints-modeMode of constraints for Airflow for PROD image building.               │ -│(constraints | constraints-no-providers | constraints-source-providers)│ -│[default: constraints]                                                 │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Choosing dependencies and extras (for power users) ─────────────────────────────────────────────────────────────────╮ -│--airflow-extrasExtras to install by default.                                                   │ -│(TEXT)                                                                          │ -│[default:                                                                       │ -│aiobotocore,amazon,async,celery,cncf-kubernetes,common-io,docker,elasticsearch,…│ -│--additional-airflow-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ -│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ -│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ -│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced customization options (for specific customization needs) ──────────────────────────────────────────────────╮ -│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)[default: .]│ -│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ -│--install-packages-from-contextInstall wheels from local docker-context-files when building image.       │ -│Implies --disable-airflow-repo-cache.                                     │ -│--install-mysql-client-typeWhich client to choose when installing.(mariadb | mysql)│ -│--cleanup-contextClean up docker context files before running build (cannot be used        │ -│together with --install-packages-from-context).                           │ -│--use-constraints-for-context-packagesUses constraints for context packages installation - either from          │ -│constraints store in docker-context-files or from github.                 │ -│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ -│--disable-mysql-client-installationDo not install MySQL client.│ -│--disable-mssql-client-installationDo not install MsSQl client.│ -│--disable-postgres-client-installationDo not install Postgres client.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ -│[default: autodetect]                                         │ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--pushPush image after building it.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Github authentication ──────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +│--build-progressBuild progress.(auto | plain | tty)[default: auto]│ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│[default: registry]                          │ +│--docker-hostOptional - docker host to use when running docker commands. When set, the `--builder`│ +│option is ignored when building images.                                              │ +│(TEXT)                                                                               │ +│--image-tagTag the image after building it.(TEXT)[default: latest]│ +│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.8< | 3.9 | 3.10 | 3.11)                                 │ +│[default: 3.8]                                              │ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when │ +│you build or pull image with --image-tag.                                            │ +│--use-uv/--no-use-uvUse uv instead of pip as packaging tool.[default: no-use-uv]│ +│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of parameters.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.8 3.9 3.10 3.11]                                                  │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced build options (for power users) ───────────────────────────────────────────────────────────────────────────╮ +│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`       │ +│itself).                                                                          │ +│(TEXT)                                                                            │ +│--commit-shaCommit SHA that is used to build the images.(TEXT)│ +│--debian-versionDebian version used in Airflow image as base for building images.│ +│(bookworm | bullseye)                                            │ +│[default: bookworm]                                              │ +│--python-imageIf specified this is the base python image used to build the image. Should be     │ +│something like: python:VERSION-slim-bookworm.                                     │ +│(TEXT)                                                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Selecting constraint location (for power users) ────────────────────────────────────────────────────────────────────╮ +│--airflow-constraints-locationLocation of airflow constraints to use (remote URL or local context file).(TEXT)│ +│--airflow-constraints-modeMode of constraints for Airflow for PROD image building.               │ +│(constraints | constraints-no-providers | constraints-source-providers)│ +│[default: constraints]                                                 │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Choosing dependencies and extras (for power users) ─────────────────────────────────────────────────────────────────╮ +│--airflow-extrasExtras to install by default.                                                   │ +│(TEXT)                                                                          │ +│[default:                                                                       │ +│aiobotocore,amazon,async,celery,cncf-kubernetes,common-io,docker,elasticsearch,…│ +│--additional-airflow-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ +│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ +│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ +│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced customization options (for specific customization needs) ──────────────────────────────────────────────────╮ +│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)[default: .]│ +│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ +│--install-packages-from-contextInstall wheels from local docker-context-files when building image.       │ +│Implies --disable-airflow-repo-cache.                                     │ +│--install-mysql-client-typeWhich client to choose when installing.(mariadb | mysql)│ +│--cleanup-contextClean up docker context files before running build (cannot be used        │ +│together with --install-packages-from-context).                           │ +│--use-constraints-for-context-packagesUses constraints for context packages installation - either from          │ +│constraints store in docker-context-files or from github.                 │ +│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ +│--disable-mysql-client-installationDo not install MySQL client.│ +│--disable-mssql-client-installationDo not install MsSQl client.│ +│--disable-postgres-client-installationDo not install Postgres client.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│[default: autodetect]                                         │ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--pushPush image after building it.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Github authentication ──────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_prod-image_build.txt b/dev/breeze/doc/images/output_prod-image_build.txt index 86876deb7fc759..95c068f6f48167 100644 --- a/dev/breeze/doc/images/output_prod-image_build.txt +++ b/dev/breeze/doc/images/output_prod-image_build.txt @@ -1 +1 @@ -07693b2597b00fdb156949c753dae783 +5f950a14dace5a9db35d468734dfbb6b diff --git a/dev/breeze/src/airflow_breeze/commands/common_options.py b/dev/breeze/src/airflow_breeze/commands/common_options.py index af30a9d2995e8b..ed820610593390 100644 --- a/dev/breeze/src/airflow_breeze/commands/common_options.py +++ b/dev/breeze/src/airflow_breeze/commands/common_options.py @@ -341,6 +341,15 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, "--use-uv/--no-use-uv", is_flag=True, default=True, + show_default=True, + help="Use uv instead of pip as packaging tool.", + envvar="USE_UV", +) +option_use_uv_default_disabled = click.option( + "--use-uv/--no-use-uv", + is_flag=True, + default=False, + show_default=True, help="Use uv instead of pip as packaging tool.", envvar="USE_UV", ) diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index a98462c1630f5e..911e8b56a5b088 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -68,6 +68,7 @@ option_python_versions, option_run_in_parallel, option_skip_cleanup, + option_use_uv_default_disabled, option_verbose, option_version_suffix_for_pypi, ) @@ -244,6 +245,7 @@ def prod_image(): @option_runtime_apt_deps @option_skip_cleanup @option_tag_as_latest +@option_use_uv_default_disabled @option_verbose @option_version_suffix_for_pypi def build( @@ -296,6 +298,7 @@ def build( skip_cleanup: bool, tag_as_latest: bool, use_constraints_for_context_packages: bool, + use_uv: bool, version_suffix_for_pypi: str, ): """ @@ -354,6 +357,7 @@ def run_build(prod_image_params: BuildProdParams) -> None: runtime_apt_deps=runtime_apt_deps, tag_as_latest=tag_as_latest, use_constraints_for_context_packages=use_constraints_for_context_packages, + use_uv=use_uv, version_suffix_for_pypi=version_suffix_for_pypi, ) if platform: diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py index df780dbbd8c29d..3be1c160264935 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py @@ -29,14 +29,15 @@ { "name": "Basic usage", "options": [ - "--python", - "--install-airflow-version", + "--build-progress", + "--docker-cache", + "--docker-host", "--image-tag", + "--install-airflow-version", + "--python", "--tag-as-latest", - "--docker-cache", + "--use-uv", "--version-suffix-for-pypi", - "--build-progress", - "--docker-host", ], }, { diff --git a/dev/breeze/src/airflow_breeze/params/build_prod_params.py b/dev/breeze/src/airflow_breeze/params/build_prod_params.py index 342ac0c1435d14..b5cb80d2167e1e 100644 --- a/dev/breeze/src/airflow_breeze/params/build_prod_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_prod_params.py @@ -55,6 +55,7 @@ class BuildProdParams(CommonBuildParams): runtime_apt_command: str | None = None runtime_apt_deps: str | None = None use_constraints_for_context_packages: bool = False + use_uv: bool = True @property def airflow_version(self) -> str: @@ -206,6 +207,7 @@ def prepare_arguments_for_docker_build_command(self) -> list[str]: self._req_arg("AIRFLOW_IMAGE_README_URL", self.airflow_image_readme_url) self._req_arg("AIRFLOW_IMAGE_REPOSITORY", self.airflow_image_repository) self._req_arg("AIRFLOW_PRE_CACHED_PIP_PACKAGES", self.airflow_pre_cached_pip_packages) + self._opt_arg("AIRFLOW_USE_UV", self.use_uv) self._req_arg("AIRFLOW_VERSION", self.airflow_version) self._req_arg("BUILD_ID", self.build_id) self._req_arg("CONSTRAINTS_GITHUB_REPOSITORY", self.constraints_github_repository) diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst index 9a63e67213d26f..768a73c8ee93a3 100644 --- a/docs/docker-stack/build-arg-ref.rst +++ b/docs/docker-stack/build-arg-ref.rst @@ -49,7 +49,7 @@ Those are the most common arguments that you use when you want to build a custom +------------------------------------------+------------------------------------------+---------------------------------------------+ | ``AIRFLOW_UV_VERSION`` | ```` | UV version used. | +------------------------------------------+------------------------------------------+---------------------------------------------+ -| ``AIRFLOW_USE_UV`` | ``false`` | Whether to use UV. | +| ``AIRFLOW_USE_UV`` | ``false`` | Whether to use UV to build the image. | +------------------------------------------+------------------------------------------+---------------------------------------------+ | ``ADDITIONAL_PIP_INSTALL_FLAGS`` | | additional ``pip`` flags passed to the | | | | installation commands (except when | diff --git a/docs/docker-stack/build.rst b/docs/docker-stack/build.rst index b94aaff59ff87c..192ff11b27216f 100644 --- a/docs/docker-stack/build.rst +++ b/docs/docker-stack/build.rst @@ -334,7 +334,7 @@ Naming conventions for the images: Important notes for the base images ----------------------------------- -You should be aware, about a few things: +You should be aware, about a few things * The production image of airflow uses "airflow" user, so if you want to add some of the tools as ``root`` user, you need to switch to it with ``USER`` directive of the Dockerfile and switch back to @@ -342,14 +342,19 @@ You should be aware, about a few things: `best practices of Dockerfiles `_ to make sure your image is lean and small. -* The PyPI dependencies in Apache Airflow are installed in the user library, of the "airflow" user, so - PIP packages are installed to ``~/.local`` folder as if the ``--user`` flag was specified when running PIP. - Note also that using ``--no-cache-dir`` is a good idea that can help to make your image smaller. +* You can use regular ``pip install`` commands (and as of Dockerfile coming in Airflow 2.9 also + ``uv pip install`` (experimental) to install PyPI packages. Regular ``install`` commands should be used + however you should remember to add ``apache-airflow==${AIRFLOW_VERSION}`` to the command to avoid + accidentally upgrading or downgrading the version of Apache Airflow. Depends on the scenario you might + also use constraints file. As of Dockerfile available in Airflow 2.9.0, the constraints file used to + build the image is available in ``${HOME}/constraints.txt.`` -.. note:: - Only as of ``2.0.1`` image the ``--user`` flag is turned on by default by setting ``PIP_USER`` environment - variable to ``true``. This can be disabled by un-setting the variable or by setting it to ``false``. In the - 2.0.0 image you had to add the ``--user`` flag as ``pip install --user`` command. +* The PyPI dependencies in Apache Airflow are installed in the ``~/.local`` virtualenv, of the "airflow" user, + so PIP packages are installed to ``~/.local`` folder as if the ``--user`` flag was specified when running + PIP. This has the effect that when you create a virtualenv with ``--system-site-packages`` flag, the + virtualenv created will automatically have all the same packages installed as local airflow installation. + Note also that using ``--no-cache-dir`` in ``pip`` or ``--no-cache`` in ``uv`` is a good idea that can + help to make your image smaller. * If your apt, or PyPI dependencies require some of the ``build-essential`` or other packages that need to compile your python dependencies, then your best choice is to follow the "Customize the image" route, @@ -373,19 +378,6 @@ You should be aware, about a few things: ``umask 0002`` is set as default when you enter the image, so any directories you create by default in runtime, will have ``GID=0`` and will be group-writable. -.. note:: - When you build image for Airflow version < ``2.1`` (for example 2.0.2 or 1.10.15) the image is built with - PIP 20.2.4 because ``PIP21+`` is only supported for ``Airflow 2.1+`` - -.. note:: - Only as of ``2.0.2`` the default group of ``airflow`` user is ``root``. Previously it was ``airflow``, - so if you are building your images based on an earlier image, you need to manually change the default - group for airflow user: - -.. code-block:: docker - - RUN usermod -g 0 airflow - Examples of image extending --------------------------- @@ -432,6 +424,46 @@ The following example adds ``lxml`` python package from PyPI to the image. :start-after: [START Dockerfile] :end-before: [END Dockerfile] +Example of adding ``PyPI`` package with constraints +................................................... + +The following example adds ``lxml`` python package from PyPI to the image with constraints that were +used to install airflow. This allows you to use version of packages that you know were tested with the +given version of Airflow, so you can use it if you do not want to use potentially newer versions +that were released after the version of Airflow you are using. + +.. exampleinclude:: docker-examples/extending/add-pypi-packages-constraints/Dockerfile + :language: Dockerfile + :start-after: [START Dockerfile] + :end-before: [END Dockerfile] + + +Example of adding ``PyPI`` package with uv +.......................................... + +The following example adds ``lxml`` python package from PyPI to the image using ``uv``. This is an +experimental feature as ``uv`` is a very fast but also very new tool in Python ecosystem. + +.. exampleinclude:: docker-examples/extending/add-pypi-packages-uv/Dockerfile + :language: Dockerfile + :start-after: [START Dockerfile] + :end-before: [END Dockerfile] + +Example of adding ``PyPI`` package with uv and constraints +.......................................................... + +The following example adds ``lxml`` python package from PyPI to the image with constraints that were +used to install airflow. This allows you to use version of packages that you know were tested with the +given version of Airflow, so you can use it if you do not want to use potentially newer versions +that were released after the version of Airflow you are using. It's using ``uv`` to install the packages +which is an experimental feature as ``uv`` is a very fast but also very new tool in Python ecosystem. + +.. exampleinclude:: docker-examples/extending/add-pypi-packages-uv-constraints/Dockerfile + :language: Dockerfile + :start-after: [START Dockerfile] + :end-before: [END Dockerfile] + + Example of adding packages from requirements.txt ................................................ diff --git a/docs/docker-stack/changelog.rst b/docs/docker-stack/changelog.rst index 90930afdda34c2..7814c344b482cd 100644 --- a/docs/docker-stack/changelog.rst +++ b/docs/docker-stack/changelog.rst @@ -45,13 +45,24 @@ Airflow 2.9 ``apache/airflow:slim-2.9.0-python-3.8`` images respectively so while the change is potentially breaking, it is very easy to switch to the previous behaviour. -Airflow 2.9 -~~~~~~~~~~~ - -The ``gosu`` binary was removed from the image. This is a potentially breaking change for users who relied on -``gosu`` to change the user in the container. The ``gosu`` binary was removed because it was a source of -security vulnerabilities as it was linked against older go standard libraries. - + * The ``PIP_USER`` flag is removed and replaced by ``VIRTUAL_ENV`` pointing to ``~/.local`` where Airflow + is installed. This has the effect that Airflow installation is treated as regular virtual environment, + but unlike a regular virtualenv, the ``~/.local`` directory is seen as ``system level`` and when + worker creates dynamically virtualenv with ``--system-site-packages`` flag, Airflow installation and all + packages there are also present in the new virtualenv - but when you do not use the flag, they are not + copied there which is backwards-compatible behaviour with having ``PIP_USER`` set. + + * The image contains latest ``uv`` binary (latest at the moment of release) - which is a new faster + replacement for ``pip``. While the image is still using ``pip`` by default, you can use ``uv`` by default + to install packages and - experimentally - you can also build custom images with + ``--arg AIRFLOW_USE_UV=true`` which will us ``uv`` to perform the installation. + + * Constraints used to install the image are available in "${HOME}/constraints.txt" now - you can use them + to install additional packages in the image without having to find out which constraints you should use. + + * The ``gosu`` binary was removed from the image. This is a potentially breaking change for users who relied on + ``gosu`` to change the user in the container. The ``gosu`` binary was removed because it was a source of + security vulnerabilities as it was linked against older go standard libraries. Airflow 2.8 ~~~~~~~~~~~ diff --git a/docs/docker-stack/docker-examples/extending/add-pypi-packages-constraints/Dockerfile b/docs/docker-stack/docker-examples/extending/add-pypi-packages-constraints/Dockerfile new file mode 100644 index 00000000000000..cd7baf973c1646 --- /dev/null +++ b/docs/docker-stack/docker-examples/extending/add-pypi-packages-constraints/Dockerfile @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example Dockerfile. It is not intended for PRODUCTION use +# [START Dockerfile] +FROM apache/airflow:2.9.0.dev0 +RUN pip install --no-cache-dir "apache-airflow==${AIRFLOW_VERSION}" lxml --constraint "${HOME}/constraints.txt" +# [END Dockerfile] diff --git a/docs/docker-stack/docker-examples/extending/add-pypi-packages-uv-constraints/Dockerfile b/docs/docker-stack/docker-examples/extending/add-pypi-packages-uv-constraints/Dockerfile new file mode 100644 index 00000000000000..5c98c049b8dd36 --- /dev/null +++ b/docs/docker-stack/docker-examples/extending/add-pypi-packages-uv-constraints/Dockerfile @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example Dockerfile. It is not intended for PRODUCTION use +# [START Dockerfile] +FROM apache/airflow:2.9.0.dev0 +RUN uv pip install --no-cache "apache-airflow==${AIRFLOW_VERSION}" lxml --constraint "${HOME}/constraints.txt" +# [END Dockerfile] diff --git a/docs/docker-stack/docker-examples/extending/add-pypi-packages-uv/Dockerfile b/docs/docker-stack/docker-examples/extending/add-pypi-packages-uv/Dockerfile new file mode 100644 index 00000000000000..7cf60668426319 --- /dev/null +++ b/docs/docker-stack/docker-examples/extending/add-pypi-packages-uv/Dockerfile @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is an example Dockerfile. It is not intended for PRODUCTION use +# [START Dockerfile] +FROM apache/airflow:2.9.0.dev0 + +# The `uv` tools is Rust packaging tool that is much faster than `pip` and other installer +# Support for uv as installation tool is experimental + +RUN uv pip install --no-cache "apache-airflow==${AIRFLOW_VERSION}" lxml +# [END Dockerfile] diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh index 29da11bac7cc04..e642483701f420 100644 --- a/scripts/docker/common.sh +++ b/scripts/docker/common.sh @@ -46,7 +46,11 @@ function common::get_packaging_tool() { echo export PACKAGING_TOOL="uv" export PACKAGING_TOOL_CMD="uv pip" - export EXTRA_INSTALL_FLAGS="--python ${PYTHON_BIN}" + if [[ -z ${VIRTUAL_ENV=} ]]; then + export EXTRA_INSTALL_FLAGS="--python ${PYTHON_BIN}" + else + export EXTRA_INSTALL_FLAGS="" + fi export EXTRA_UNINSTALL_FLAGS="--python ${PYTHON_BIN}" export RESOLUTION_HIGHEST_FLAG="--resolution highest" export RESOLUTION_LOWEST_DIRECT_FLAG="--resolution lowest-direct" @@ -71,14 +75,6 @@ function common::get_airflow_version_specification() { fi } -function common::override_pip_version_if_needed() { - if [[ -n ${AIRFLOW_VERSION} ]]; then - if [[ ${AIRFLOW_VERSION} =~ ^2\.0.* || ${AIRFLOW_VERSION} =~ ^1\.* ]]; then - export AIRFLOW_PIP_VERSION=24.0 - fi - fi -} - function common::get_constraints_location() { # auto-detect Airflow-constraint reference and location if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE=}" ]]; then @@ -95,6 +91,8 @@ function common::get_constraints_location() { python_version="$(python --version 2>/dev/stdout | cut -d " " -f 2 | cut -d "." -f 1-2)" AIRFLOW_CONSTRAINTS_LOCATION="${constraints_base}/${AIRFLOW_CONSTRAINTS_MODE}-${python_version}.txt" fi + + curl -o "${HOME}/constraints.txt" "${AIRFLOW_CONSTRAINTS_LOCATION}" } function common::show_packaging_tool_version_and_location() { @@ -105,12 +103,12 @@ function common::show_packaging_tool_version_and_location() { echo "Using pip: $(pip --version)" else echo "${COLOR_BLUE}Using 'uv' to install Airflow${COLOR_RESET}" - echo "uv on path: $(which uv)" + echo "uv on path: $(which uv 2>/dev/null || echo "Not installed")" echo "Using uv: $(uv --version)" fi } -function common::install_packaging_tool() { +function common::install_packaging_tools() { echo echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}" echo @@ -121,17 +119,15 @@ function common::install_packaging_tool() { # shellcheck disable=SC2086 pip install --root-user-action ignore --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" fi - if [[ ${AIRFLOW_USE_UV} == "true" ]]; then - echo - echo "${COLOR_BLUE}Installing uv version ${AIRFLOW_UV_VERSION}${COLOR_RESET}" - echo - if [[ ${AIRFLOW_UV_VERSION} =~ .*https.* ]]; then - # shellcheck disable=SC2086 - pip install --root-user-action ignore --disable-pip-version-check "uv @ ${AIRFLOW_UV_VERSION}" - else - # shellcheck disable=SC2086 - pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}" - fi + echo + echo "${COLOR_BLUE}Installing uv version ${AIRFLOW_UV_VERSION}${COLOR_RESET}" + echo + if [[ ${AIRFLOW_UV_VERSION} =~ .*https.* ]]; then + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "uv @ ${AIRFLOW_UV_VERSION}" + else + # shellcheck disable=SC2086 + pip install --root-user-action ignore --disable-pip-version-check "uv==${AIRFLOW_UV_VERSION}" fi mkdir -p "${HOME}/.local/bin" } diff --git a/scripts/docker/install_pip_version.sh b/scripts/docker/create_prod_venv.sh similarity index 88% rename from scripts/docker/install_pip_version.sh rename to scripts/docker/create_prod_venv.sh index af8d25e06ef635..0a8115d626921c 100644 --- a/scripts/docker/install_pip_version.sh +++ b/scripts/docker/create_prod_venv.sh @@ -19,10 +19,13 @@ # shellcheck source=scripts/docker/common.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" +function create_prod_venv() { + rm -rf ~/.local + python -m venv ~/.local +} + common::get_colors common::get_packaging_tool -common::get_airflow_version_specification -common::override_pip_version_if_needed common::show_packaging_tool_version_and_location - -common::install_packaging_tool +create_prod_venv +common::install_packaging_tools diff --git a/scripts/docker/get_package_specs.py b/scripts/docker/get_package_specs.py new file mode 100755 index 00000000000000..bff99a2c1cbdb4 --- /dev/null +++ b/scripts/docker/get_package_specs.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os +import sys +from pathlib import Path + +from packaging.utils import ( + InvalidSdistFilename, + InvalidWheelFilename, + parse_sdist_filename, + parse_wheel_filename, +) + + +def print_package_specs(extras: str = "") -> None: + for package_path in sys.argv[1:]: + try: + package, _, _, _ = parse_wheel_filename(Path(package_path).name) + except InvalidWheelFilename: + try: + package, _ = parse_sdist_filename(Path(package_path).name) + except InvalidSdistFilename: + print(f"Could not parse package name from {package_path}", file=sys.stderr) + continue + print(f"{package}{extras} @ file://{package_path}") + + +if __name__ == "__main__": + print_package_specs(extras=os.environ.get("EXTRAS", "")) diff --git a/scripts/docker/install_additional_dependencies.sh b/scripts/docker/install_additional_dependencies.sh index 79e83eeb9e9024..952de84ed32fe7 100644 --- a/scripts/docker/install_additional_dependencies.sh +++ b/scripts/docker/install_additional_dependencies.sh @@ -34,7 +34,7 @@ function install_additional_dependencies() { ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_HIGHEST_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} - common::install_packaging_tool + common::install_packaging_tools set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -48,7 +48,7 @@ function install_additional_dependencies() { ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade "${RESOLUTION_LOWEST_DIRECT_FLAG}" \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${ADDITIONAL_PYTHON_DEPS} - common::install_packaging_tool + common::install_packaging_tools set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -60,7 +60,6 @@ function install_additional_dependencies() { common::get_colors common::get_packaging_tool common::get_airflow_version_specification -common::override_pip_version_if_needed common::get_constraints_location common::show_packaging_tool_version_and_location diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh index e8fb806ac648e0..9ada99723948e8 100644 --- a/scripts/docker/install_airflow.sh +++ b/scripts/docker/install_airflow.sh @@ -65,7 +65,7 @@ function install_airflow() { "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} set +x - common::install_packaging_tool + common::install_packaging_tools echo echo "${COLOR_BLUE}Running '${PACKAGING_TOOL} check'${COLOR_RESET}" echo @@ -78,14 +78,14 @@ function install_airflow() { ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \ - --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true - common::install_packaging_tool + --constraint ${HOME}/constraints.txt || true + common::install_packaging_tools # then upgrade if needed without using constraints to account for new limits in pyproject.toml ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade ${RESOLUTION_LOWEST_DIRECT_FLAG} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ ${AIRFLOW_INSTALL_EDITABLE_FLAG} \ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" - common::install_packaging_tool + common::install_packaging_tools set +x echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -98,7 +98,6 @@ function install_airflow() { common::get_colors common::get_packaging_tool common::get_airflow_version_specification -common::override_pip_version_if_needed common::get_constraints_location common::show_packaging_tool_version_and_location diff --git a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh index 202a40c41e940c..1510b7b1897f9e 100644 --- a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh +++ b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh @@ -52,7 +52,7 @@ function install_airflow_dependencies_from_branch_tip() { ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ "apache-airflow[${AIRFLOW_EXTRAS}] @ https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz" - common::install_packaging_tool + common::install_packaging_tools # Uninstall airflow and providers to keep only the dependencies. In the future when # planned https://github.com/pypa/pip/issues/11440 is implemented in pip we might be able to use this # flag and skip the remove step. @@ -67,7 +67,6 @@ function install_airflow_dependencies_from_branch_tip() { common::get_colors common::get_packaging_tool common::get_airflow_version_specification -common::override_pip_version_if_needed common::get_constraints_location common::show_packaging_tool_version_and_location diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh index 6e286b9d9cf686..d6fab1e8273ce4 100644 --- a/scripts/docker/install_from_docker_context_files.sh +++ b/scripts/docker/install_from_docker_context_files.sh @@ -24,6 +24,8 @@ # shellcheck source=scripts/docker/common.sh . "$( dirname "${BASH_SOURCE[0]}" )/common.sh" +# TODO: rewrite it all in Python (and all other scripts in scripts/docker) + function install_airflow_and_providers_from_docker_context_files(){ if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,} @@ -39,46 +41,36 @@ function install_airflow_and_providers_from_docker_context_files(){ exit 1 fi - # shellcheck disable=SC2206 - local packaging_flags=( - # Don't quote this -- if it is empty we don't want it to create an - # empty array element - --find-links="file:///docker-context-files" - ) - - # Find Apache Airflow packages in docker-context files - local reinstalling_apache_airflow_package - reinstalling_apache_airflow_package=$(ls \ - /docker-context-files/apache?airflow?[0-9]*.{whl,tar.gz} 2>/dev/null || true) - # Add extras when installing airflow - if [[ -n "${reinstalling_apache_airflow_package}" ]]; then - # When a provider depends on a dev version of Airflow, we need to - # specify `apache-airflow==$VER`, otherwise pip will look for it on - # pip, and fail to find it + # This is needed to get package names for local context packages + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} --constraint ${HOME}/constraints.txt packaging - # This will work as long as the wheel file is correctly named, which it - # will be if it was build by wheel tooling - local ver - ver=$(basename "$reinstalling_apache_airflow_package" | cut -d "-" -f 2) - reinstalling_apache_airflow_package="apache-airflow[${AIRFLOW_EXTRAS}]==$ver" + if [[ -n ${AIRFLOW_EXTRAS=} ]]; then + AIRFLOW_EXTRAS_TO_INSTALL="[${AIRFLOW_EXTRAS}]" + else + AIRFLOW_EXTRAS_TO_INSTALL="" fi - if [[ -z "${reinstalling_apache_airflow_package}" && ${AIRFLOW_VERSION=} != "" ]]; then + # Find Apache Airflow package in docker-context files + readarray -t install_airflow_package < <(EXTRAS="${AIRFLOW_EXTRAS_TO_INSTALL}" \ + python /scripts/docker/get_package_specs.py /docker-context-files/apache?airflow?[0-9]*.{whl,tar.gz} 2>/dev/null || true) + echo + echo "${COLOR_BLUE}Found airflow packages in docker-context-files folder: ${install_airflow_package[*]}${COLOR_RESET}" + echo + + if [[ -z "${install_airflow_package[*]}" && ${AIRFLOW_VERSION=} != "" ]]; then # When we install only provider packages from docker-context files, we need to still # install airflow from PyPI when AIRFLOW_VERSION is set. This handles the case where # pre-release dockerhub image of airflow is built, but we want to install some providers from # docker-context files - reinstalling_apache_airflow_package="apache-airflow[${AIRFLOW_EXTRAS}]==${AIRFLOW_VERSION}" - fi - # Find Apache Airflow packages in docker-context files - local reinstalling_apache_airflow_providers_packages - reinstalling_apache_airflow_providers_packages=$(ls \ - /docker-context-files/apache?airflow?providers*.{whl,tar.gz} 2>/dev/null || true) - if [[ -z "${reinstalling_apache_airflow_package}" && \ - -z "${reinstalling_apache_airflow_providers_packages}" ]]; then - return + install_airflow_package=("apache-airflow[${AIRFLOW_EXTRAS}]==${AIRFLOW_VERSION}") fi + # Find Provider packages in docker-context files + readarray -t installing_providers_packages< <(python /scripts/docker/get_package_specs.py /docker-context-files/apache?airflow?providers*.{whl,tar.gz} 2>/dev/null || true) + echo + echo "${COLOR_BLUE}Found provider packages in docker-context-files folder: ${installing_providers_packages[*]}${COLOR_RESET}" + echo + if [[ ${USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES=} == "true" ]]; then local python_version python_version=$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') @@ -90,19 +82,19 @@ function install_airflow_and_providers_from_docker_context_files(){ echo # force reinstall all airflow + provider packages with constraints found in set -x - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "${packaging_flags[@]}" --upgrade \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade \ ${ADDITIONAL_PIP_INSTALL_FLAGS} --constraint "${local_constraints_file}" \ - ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} + "${install_airflow_package[@]}" "${installing_providers_packages[@]}" set +x else echo echo "${COLOR_BLUE}Installing docker-context-files packages with constraints from GitHub${COLOR_RESET}" echo set -x - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "${packaging_flags[@]}" \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ - --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" \ - ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} + --constraint "${HOME}/constraints.txt" \ + "${install_airflow_package[@]}" "${installing_providers_packages[@]}" set +x fi else @@ -110,12 +102,12 @@ function install_airflow_and_providers_from_docker_context_files(){ echo "${COLOR_BLUE}Installing docker-context-files packages without constraints${COLOR_RESET}" echo set -x - ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} "${packaging_flags[@]}" \ + ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} \ ${ADDITIONAL_PIP_INSTALL_FLAGS} \ - ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} + "${install_airflow_package[@]}" "${installing_providers_packages[@]}" set +x fi - common::install_packaging_tool + common::install_packaging_tools pip check } @@ -135,7 +127,7 @@ function install_all_other_packages_from_docker_context_files() { set -x ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} ${ADDITIONAL_PIP_INSTALL_FLAGS} \ --force-reinstall --no-deps --no-index ${reinstalling_other_packages} - common::install_packaging_tool + common::install_packaging_tools set +x fi } @@ -143,7 +135,6 @@ function install_all_other_packages_from_docker_context_files() { common::get_colors common::get_packaging_tool common::get_airflow_version_specification -common::override_pip_version_if_needed common::get_constraints_location common::show_packaging_tool_version_and_location diff --git a/scripts/docker/install_packaging_tools.sh b/scripts/docker/install_packaging_tools.sh new file mode 100644 index 00000000000000..ca4d125777190f --- /dev/null +++ b/scripts/docker/install_packaging_tools.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# shellcheck shell=bash disable=SC2086 +# shellcheck source=scripts/docker/common.sh +. "$( dirname "${BASH_SOURCE[0]}" )/common.sh" + +common::get_colors +common::install_packaging_tools diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index 4e3cce5114bfc9..9e95af43ad3eb1 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -71,7 +71,11 @@ function in_container_get_packaging_tool() { echo export PACKAGING_TOOL="" export PACKAGING_TOOL_CMD="uv pip" - export EXTRA_INSTALL_FLAGS="--python ${PYTHON_BIN}" + if [[ -z ${VIRTUAL_ENV=} ]]; then + export EXTRA_INSTALL_FLAGS="--python ${PYTHON_BIN}" + else + export EXTRA_INSTALL_FLAGS="" + fi export EXTRA_UNINSTALL_FLAGS="--python ${PYTHON_BIN}" export RESOLUTION_HIGHEST_FLAG="--resolution highest" export RESOLUTION_LOWEST_DIRECT_FLAG="--resolution lowest-direct"