diff --git a/BREEZE.rst b/BREEZE.rst index 86e8e88a9769ad..5b23c84f1f4d83 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -542,7 +542,7 @@ or ``apache/airflow:v1-10-test-python3.6``). However in many cases you want to add your own custom version of the image - with added apt dependencies, python dependencies, additional Airflow extras. Breeze's ``build-image`` command helps to build your own, -customised variant of the image that contains everything you need. +customized variant of the image that contains everything you need. You can switch to building the production image by adding ``--production-image`` flag to the ``build_image`` command. Note, that the images can also be build using ``docker build`` command by passing appropriate @@ -566,8 +566,8 @@ This install additional pypi dependency - torchio in specified version. .. code-block:: bash - ./breeze build-image --production-image --additional-dev-deps "libasound2-dev" \ - --additional-runtime-deps "libasound2" + ./breeze build-image --production-image --additional-dev-apt-deps "libasound2-dev" \ + --additional-runtime-apt-deps "libasound2" This install additional apt dependencies - ``libasound2-dev`` in build image and ``libasound`` in the final image. Those are development dependencies that might be needed to build and use python packages added @@ -580,7 +580,7 @@ suffix and they need to also be paired with corresponding runtime dependency add .. code-block:: bash ./breeze build-image --production-image --python 3.7 --additional-dev-deps "libasound2-dev" \ - --additional-runtime-deps "libasound2" + --additional-runtime-apt-deps "libasound2" Same as above but uses python 3.7. @@ -1197,6 +1197,8 @@ This is the current syntax for `./breeze <./breeze>`_: images are pulled by default only for the first time you run the environment, later the locally build images are used as cache. + Customization options: + -E, --extras EXTRAS Extras to pass to build images The default are different for CI and production images: @@ -1210,20 +1212,49 @@ This is the current syntax for `./breeze <./breeze>`_: --image-tag TAG Additional tag in the image. + --disable-pip-cache + Disables GitHub PIP cache during the build. Useful if github is not reachable during build. + --additional-extras ADDITIONAL_EXTRAS Additional extras to pass to build images The default is no additional extras. --additional-python-deps ADDITIONAL_PYTHON_DEPS Additional python dependencies to use when building the images. - --additional-dev-deps ADDITIONAL_DEV_DEPS + --dev-apt-command DEV_APT_COMMAND + The basic command executed before dev apt deps are installed. + + --additional-dev-apt-command ADDITIONAL_DEV_APT_COMMAND + Additional command executed before dev apt deps are installed. + + --additional-dev-apt-deps ADDITIONAL_DEV_APT_DEPS Additional apt dev dependencies to use when building the images. - --additional-runtime-deps ADDITIONAL_RUNTIME_DEPS + --dev-apt-deps DEV_APT_DEPS + The basic apt dev dependencies to use when building the images. + + --additional-dev-apt-deps ADDITIONAL_DEV_DEPS + Additional apt dev dependencies to use when building the images. + + --additional-dev-apt-envs ADDITIONAL_DEV_APT_ENVS + Additional environment variables set when adding dev dependencies. + + --runtime-apt-command RUNTIME_APT_COMMAND + The basic command executed before runtime apt deps are installed. + + --additional-runtime-apt-command ADDITIONAL_RUNTIME_APT_COMMAND + Additional command executed before runtime apt deps are installed. + + --runtime-apt-deps ADDITIONAL_RUNTIME_APT_DEPS + The basic apt runtime dependencies to use when building the images. + + --additional-runtime-apt-deps ADDITIONAL_RUNTIME_DEPS Additional apt runtime dependencies to use when building the images. - --disable-pip-cache - Disables GitHub PIP cache during the build. Useful if github is not reachable during build. + --additional-runtime-apt-envs ADDITIONAL_RUNTIME_APT_DEPS + Additional environment variables set when adding runtime dependencies. + + Build options: --disable-mysql-client-installation Disables installation of the mysql client which might be problematic if you are building @@ -1654,6 +1685,8 @@ This is the current syntax for `./breeze <./breeze>`_: images are pulled by default only for the first time you run the environment, later the locally build images are used as cache. + Customization options: + -E, --extras EXTRAS Extras to pass to build images The default are different for CI and production images: @@ -1667,20 +1700,49 @@ This is the current syntax for `./breeze <./breeze>`_: --image-tag TAG Additional tag in the image. + --disable-pip-cache + Disables GitHub PIP cache during the build. Useful if github is not reachable during build. + --additional-extras ADDITIONAL_EXTRAS Additional extras to pass to build images The default is no additional extras. --additional-python-deps ADDITIONAL_PYTHON_DEPS Additional python dependencies to use when building the images. - --additional-dev-deps ADDITIONAL_DEV_DEPS + --dev-apt-command DEV_APT_COMMAND + The basic command executed before dev apt deps are installed. + + --additional-dev-apt-command ADDITIONAL_DEV_APT_COMMAND + Additional command executed before dev apt deps are installed. + + --additional-dev-apt-deps ADDITIONAL_DEV_APT_DEPS + Additional apt dev dependencies to use when building the images. + + --dev-apt-deps DEV_APT_DEPS + The basic apt dev dependencies to use when building the images. + + --additional-dev-apt-deps ADDITIONAL_DEV_DEPS Additional apt dev dependencies to use when building the images. - --additional-runtime-deps ADDITIONAL_RUNTIME_DEPS + --additional-dev-apt-envs ADDITIONAL_DEV_APT_ENVS + Additional environment variables set when adding dev dependencies. + + --runtime-apt-command RUNTIME_APT_COMMAND + The basic command executed before runtime apt deps are installed. + + --additional-runtime-apt-command ADDITIONAL_RUNTIME_APT_COMMAND + Additional command executed before runtime apt deps are installed. + + --runtime-apt-deps ADDITIONAL_RUNTIME_APT_DEPS + The basic apt runtime dependencies to use when building the images. + + --additional-runtime-apt-deps ADDITIONAL_RUNTIME_DEPS Additional apt runtime dependencies to use when building the images. - --disable-pip-cache - Disables GitHub PIP cache during the build. Useful if github is not reachable during build. + --additional-runtime-apt-envs ADDITIONAL_RUNTIME_APT_DEPS + Additional environment variables set when adding runtime dependencies. + + Build options: --disable-mysql-client-installation Disables installation of the mysql client which might be problematic if you are building @@ -1974,6 +2036,8 @@ This is the current syntax for `./breeze <./breeze>`_: images are pulled by default only for the first time you run the environment, later the locally build images are used as cache. + Customization options: + -E, --extras EXTRAS Extras to pass to build images The default are different for CI and production images: @@ -1987,20 +2051,49 @@ This is the current syntax for `./breeze <./breeze>`_: --image-tag TAG Additional tag in the image. + --disable-pip-cache + Disables GitHub PIP cache during the build. Useful if github is not reachable during build. + --additional-extras ADDITIONAL_EXTRAS Additional extras to pass to build images The default is no additional extras. --additional-python-deps ADDITIONAL_PYTHON_DEPS Additional python dependencies to use when building the images. - --additional-dev-deps ADDITIONAL_DEV_DEPS + --dev-apt-command DEV_APT_COMMAND + The basic command executed before dev apt deps are installed. + + --additional-dev-apt-command ADDITIONAL_DEV_APT_COMMAND + Additional command executed before dev apt deps are installed. + + --additional-dev-apt-deps ADDITIONAL_DEV_APT_DEPS + Additional apt dev dependencies to use when building the images. + + --dev-apt-deps DEV_APT_DEPS + The basic apt dev dependencies to use when building the images. + + --additional-dev-apt-deps ADDITIONAL_DEV_DEPS Additional apt dev dependencies to use when building the images. - --additional-runtime-deps ADDITIONAL_RUNTIME_DEPS + --additional-dev-apt-envs ADDITIONAL_DEV_APT_ENVS + Additional environment variables set when adding dev dependencies. + + --runtime-apt-command RUNTIME_APT_COMMAND + The basic command executed before runtime apt deps are installed. + + --additional-runtime-apt-command ADDITIONAL_RUNTIME_APT_COMMAND + Additional command executed before runtime apt deps are installed. + + --runtime-apt-deps ADDITIONAL_RUNTIME_APT_DEPS + The basic apt runtime dependencies to use when building the images. + + --additional-runtime-apt-deps ADDITIONAL_RUNTIME_DEPS Additional apt runtime dependencies to use when building the images. - --disable-pip-cache - Disables GitHub PIP cache during the build. Useful if github is not reachable during build. + --additional-runtime-apt-envs ADDITIONAL_RUNTIME_APT_DEPS + Additional environment variables set when adding runtime dependencies. + + Build options: --disable-mysql-client-installation Disables installation of the mysql client which might be problematic if you are building diff --git a/Dockerfile b/Dockerfile index ca7aa7ca39b2bc..cb46c69924f909 100644 --- a/Dockerfile +++ b/Dockerfile @@ -63,7 +63,7 @@ ENV PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION} ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \ LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 -# Install curl and gnupg2 - needed to download nodejs in the next step +# Install curl and gnupg2 - needed for many other installation steps RUN apt-get update \ && apt-get install -y --no-install-recommends \ curl \ @@ -72,52 +72,66 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -ARG ADDITIONAL_DEV_DEPS="" -ENV ADDITIONAL_DEV_DEPS=${ADDITIONAL_DEV_DEPS} +ARG DEV_APT_DEPS="\ + apt-transport-https \ + apt-utils \ + build-essential \ + ca-certificates \ + gnupg \ + dirmngr \ + freetds-bin \ + freetds-dev \ + gosu \ + krb5-user \ + ldap-utils \ + libffi-dev \ + libkrb5-dev \ + libpq-dev \ + libsasl2-2 \ + libsasl2-dev \ + libsasl2-modules \ + libssl-dev \ + locales \ + lsb-release \ + nodejs \ + openssh-client \ + postgresql-client \ + python-selinux \ + sasl2-bin \ + software-properties-common \ + sqlite3 \ + sudo \ + unixodbc \ + unixodbc-dev \ + yarn" +ENV DEV_APT_DEPS=${DEV_APT_DEPS} + +ARG ADDITIONAL_DEV_APT_DEPS="" +ENV ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS} + +ARG DEV_APT_COMMAND="\ + curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ + && curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > /dev/null \ + && echo 'deb https://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list" +ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} + +ARG ADDITIONAL_DEV_APT_COMMAND="echo" +ENV ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND} + +ARG ADDITIONAL_DEV_ENV_VARS="" +# Note missing man directories on debian-buster +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 # Install basic and additional apt dependencies -RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ - && curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > /dev/null \ - && echo "deb https://dl.yarnpkg.com/debian/ stable main" > /etc/apt/sources.list.d/yarn.list \ - # Note missing man directories on debian-buster - # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 - && mkdir -pv /usr/share/man/man1 \ +RUN mkdir -pv /usr/share/man/man1 \ && mkdir -pv /usr/share/man/man7 \ + && export ${ADDITIONAL_DEV_ENV_VARS?} \ + && bash -o pipefail -e -u -x -c "${DEV_APT_COMMAND}" \ + && bash -o pipefail -e -u -x -c "${ADDITIONAL_DEV_APT_COMMAND}" \ && apt-get update \ && apt-get install -y --no-install-recommends \ - apt-transport-https \ - apt-utils \ - build-essential \ - ca-certificates \ - curl \ - gnupg \ - dirmngr \ - freetds-bin \ - freetds-dev \ - gosu \ - krb5-user \ - ldap-utils \ - libffi-dev \ - libkrb5-dev \ - libpq-dev \ - libsasl2-2 \ - libsasl2-dev \ - libsasl2-modules \ - libssl-dev \ - locales \ - lsb-release \ - nodejs \ - openssh-client \ - postgresql-client \ - python-selinux \ - sasl2-bin \ - software-properties-common \ - sqlite3 \ - sudo \ - unixodbc \ - unixodbc-dev \ - yarn \ - ${ADDITIONAL_DEV_DEPS} \ + ${DEV_APT_DEPS} \ + ${ADDITIONAL_DEV_APT_DEPS} \ && apt-get autoremove -yqq --purge \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -253,45 +267,69 @@ ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} ARG AIRFLOW_VERSION ENV AIRFLOW_VERSION=${AIRFLOW_VERSION} -ARG ADDITIONAL_RUNTIME_DEPS="" -ENV ADDITIONAL_RUNTIME_DEPS=${ADDITIONAL_RUNTIME_DEPS} - # Make sure noninteractive debian install is used and language variables set ENV DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \ LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 +# Install curl and gnupg2 - needed for many other installation steps +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + curl \ + gnupg2 \ + && apt-get autoremove -yqq --purge \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +ARG RUNTIME_APT_DEPS="\ + apt-transport-https \ + apt-utils \ + ca-certificates \ + curl \ + dumb-init \ + freetds-bin \ + gnupg \ + gosu \ + krb5-user \ + ldap-utils \ + libffi6 \ + libsasl2-2 \ + libsasl2-modules \ + libssl1.1 \ + locales \ + lsb-release \ + netcat \ + openssh-client \ + postgresql-client \ + rsync \ + sasl2-bin \ + sqlite3 \ + sudo \ + unixodbc" +ENV RUNTIME_APT_DEPS=${RUNTIME_APT_DEPS} + +ARG ADDITIONAL_RUNTIME_APT_DEPS="" +ENV ADDITIONAL_RUNTIME_APT_DEPS=${ADDITIONAL_RUNTIME_APT_DEPS} + +ARG RUNTIME_APT_COMMAND="echo" +ENV RUNTIME_APT_COMMAND=${RUNTIME_APT_COMMAND} + +ARG ADDITIONAL_RUNTIME_APT_COMMAND="" +ENV ADDITIONAL_RUNTIME_APT_COMMAND=${ADDITIONAL_RUNTIME_APT_COMMAND} + +ARG ADDITIONAL_RUNTIME_ENV_VARS="" + # Note missing man directories on debian-buster # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 # Install basic and additional apt dependencies RUN mkdir -pv /usr/share/man/man1 \ && mkdir -pv /usr/share/man/man7 \ + && export ${ADDITIONAL_RUNTIME_ENV_VARS?} \ + && bash -o pipefail -e -u -x -c "${RUNTIME_APT_COMMAND}" \ + && bash -o pipefail -e -u -x -c "${ADDITIONAL_RUNTIME_APT_COMMAND}" \ && apt-get update \ && apt-get install -y --no-install-recommends \ - apt-transport-https \ - apt-utils \ - ca-certificates \ - curl \ - dumb-init \ - freetds-bin \ - gnupg \ - gosu \ - krb5-user \ - ldap-utils \ - libffi6 \ - libsasl2-2 \ - libsasl2-modules \ - libssl1.1 \ - locales \ - lsb-release \ - netcat \ - openssh-client \ - postgresql-client \ - rsync \ - sasl2-bin \ - sqlite3 \ - sudo \ - unixodbc \ - ${ADDITIONAL_RUNTIME_DEPS} \ + ${RUNTIME_APT_DEPS} \ + ${ADDITIONAL_RUNTIME_APT_DEPS} \ && apt-get autoremove -yqq --purge \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/Dockerfile.ci b/Dockerfile.ci index 8ea42d359d385f..2bbc9acdf64866 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -51,13 +51,26 @@ RUN apt-get update \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -ARG ADDITIONAL_DEV_DEPS="" -ENV ADDITIONAL_DEV_DEPS=${ADDITIONAL_DEV_DEPS} +ARG ADDITIONAL_DEV_APT_DEPS="" +ENV ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS} -# Install basic and additional apt dependencies -RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ +ARG DEV_APT_COMMAND="\ + curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ && curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > /dev/null \ - && echo "deb https://dl.yarnpkg.com/debian/ stable main" > /etc/apt/sources.list.d/yarn.list \ + && echo 'deb https://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list" +ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} + +ARG ADDITIONAL_DEV_APT_COMMAND="" +ENV ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND} + +ARG ADDITIONAL_DEV_ENV_VARS="" + +# Install basic and additional apt dependencies +RUN mkdir -pv /usr/share/man/man1 \ + && mkdir -pv /usr/share/man/man7 \ + && export ${ADDITIONAL_DEV_ENV_VARS?} \ + && bash -o pipefail -e -u -x -c "${DEV_APT_COMMAND}" \ + && bash -o pipefail -e -u -x -c "${ADDITIONAL_DEV_APT_COMMAND}" \ && apt-get update \ && apt-get install -y --no-install-recommends \ apt-utils \ @@ -86,7 +99,7 @@ RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ unixodbc \ unixodbc-dev \ yarn \ - ${ADDITIONAL_DEV_DEPS} \ + ${ADDITIONAL_DEV_APT_DEPS} \ && apt-get autoremove -yqq --purge \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -98,15 +111,9 @@ RUN adduser airflow \ && echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \ && chmod 0440 /etc/sudoers.d/airflow -ARG ADDITIONAL_RUNTIME_DEPS="" -ENV ADDITIONAL_RUNTIME_DEPS=${ADDITIONAL_RUNTIME_DEPS} - -# Note missing man directories on debian-buster -# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 -RUN mkdir -pv /usr/share/man/man1 \ - && mkdir -pv /usr/share/man/man7 \ - && apt-get update \ - && apt-get install --no-install-recommends -y \ +# The latest buster images do not have libpython 2.7 installed and it is needed +# To run virtualenv tests with python 2 +ARG RUNTIME_APT_DEPS="\ gnupg \ libgcc-8-dev \ apt-transport-https \ @@ -116,8 +123,6 @@ RUN mkdir -pv /usr/share/man/man1 \ krb5-user \ ldap-utils \ less \ - # The latest buster images do not have libpython 2.7 installed and it is needed - # To run virtualenv tests with python 2 libpython2.7-stdlib \ lsb-release \ net-tools \ @@ -127,8 +132,30 @@ RUN mkdir -pv /usr/share/man/man1 \ sqlite3 \ tmux \ unzip \ - vim \ - ${ADDITIONAL_RUNTIME_DEPS} \ + vim" + +ARG ADDITIONAL_RUNTIME_APT_DEPS="" +ENV ADDITIONAL_RUNTIME_APT_DEPS=${ADDITIONAL_RUNTIME_APT_DEPS} + +ARG RUNTIME_APT_COMMAND="" +ENV RUNTIME_APT_COMMAND=${RUNTIME_APT_COMMAND} + +ARG ADDITIONAL_RUNTIME_APT_COMMAND="" +ENV ADDITIONAL_RUNTIME_APT_COMMAND=${ADDITIONAL_RUNTIME_APT_COMMAND} + +ARG ADDITIONAL_RUNTIME_ENV_VARS="" + +# Note missing man directories on debian-buster +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 +RUN mkdir -pv /usr/share/man/man1 \ + && mkdir -pv /usr/share/man/man7 \ + && export ${ADDITIONAL_RUNTIME_ENV_VARS?} \ + && bash -o pipefail -e -u -x -c "${RUNTIME_APT_COMMAND}" \ + && bash -o pipefail -e -u -x -c "${ADDITIONAL_RUNTIME_APT_COMMAND}" \ + && apt-get update \ + && apt-get install --no-install-recommends -y \ + ${RUNTIME_APT_DEPS} \ + ${ADDITIONAL_RUNTIME_APT_DEPS} \ && apt-get autoremove -yqq --purge \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/IMAGES.rst b/IMAGES.rst index 6599d96975e19e..68467c6131f74d 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -255,15 +255,86 @@ The image is primarily optimised for size of the final image, but also for speed It first pre-installs them from the right github branch and only after that final airflow installation is done from either local sources or remote location (PIP or github repository). -Manually building the images ----------------------------- +Customizing the image +..................... +Customizing the image is an alternative way of adding your own dependencies to the image. + +The easiest way to build the image image is to use ``breeze`` script, but you can also build such customized +image by running appropriately crafted docker build in which you specify all the ``build-args`` +that you need to add to customize it. You can read about all the args and ways you can build the image +in the `<#ci-image-build-arguments>`_ chapter below. + +Here just a few examples are presented which should give you general understanding of what you can customize. + +This builds the production image in version 3.7 with additional airflow extras from 1.10.10 Pypi package and +additional apt dev and runtime dependencies. + +.. code-block:: bash + + docker build . -f Dockerfile.ci \ + --build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \ + --build-arg PYTHON_MAJOR_MINOR_VERSION=3.7 \ + --build-arg AIRFLOW_INSTALL_SOURCES="apache-airflow" \ + --build-arg AIRFLOW_INSTALL_VERSION="==1.10.12" \ + --build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-1-10" \ + --build-arg AIRFLOW_SOURCES_FROM="empty" \ + --build-arg AIRFLOW_SOURCES_TO="/empty" \ + --build-arg ADDITIONAL_AIRFLOW_EXTRAS="jdbc" + --build-arg ADDITIONAL_PYTHON_DEPS="pandas" + --build-arg ADDITIONAL_DEV_APT_DEPS="gcc g++" + --build-arg ADDITIONAL_RUNTIME_APT_DEPS="default-jre-headless" + --tag my-image + + +the same image can be built using ``breeze`` (it supports auto-completion of the options): + +.. code-block:: bash + + ./breeze build-image -f Dockerfile.ci \ + --production-image --python 3.7 --install-airflow-version=1.10.12 \ + --additional-extras=jdbc --additional-python-deps="pandas" \ + --additional-dev-apt-deps="gcc g++" --additional-runtime-apt-deps="default-jre-headless" You can build the default production image with standard ``docker build`` command but they will only build default versions of the image and will not use the dockerhub versions of images as cache. -CI images -......... +You can customize more aspects of the image - such as additional commands executed before apt dependencies +are installed, or adding extra sources to install your dependencies from. You can see all the arguments +described below but here is an example of rather complex command to customize the image +based on example in `this comment `_: + +.. code-block:: bash + + docker build . -f Dockerfile.ci \ + --build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \ + --build-arg PYTHON_MAJOR_MINOR_VERSION=3.7 \ + --build-arg AIRFLOW_INSTALL_SOURCES="apache-airflow" \ + --build-arg AIRFLOW_INSTALL_VERSION="==1.10.12" \ + --build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-1-10" \ + --build-arg AIRFLOW_SOURCES_FROM="empty" \ + --build-arg AIRFLOW_SOURCES_TO="/empty" \ + --build-arg ADDITIONAL_AIRFLOW_EXTRAS="slack" \ + --build-arg ADDITIONAL_PYTHON_DEPS="apache-airflow-backport-providers-odbc \ + azure-storage-blob \ + sshtunnel \ + google-api-python-client \ + oauth2client \ + beautifulsoup4 \ + dateparser \ + rocketchat_API \ + typeform" \ + --build-arg ADDITIONAL_DEV_APT_DEPS="msodbcsql17 unixodbc-dev g++" \ + --build-arg ADDITIONAL_DEV_APT_COMMAND="curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add --no-tty - && curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list" \ + --build-arg ADDITIONAL_DEV_ENV_VARS="ACCEPT_EULA=Y" \ + --build-arg ADDITIONAL_RUNTIME_APT_COMMAND="curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add --no-tty - && curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list" \ + --build-arg ADDITIONAL_RUNTIME_APT_DEPS="msodbcsql17 unixodbc git procps vim" \ + --build-arg ADDITIONAL_RUNTIME_ENV_VARS="ACCEPT_EULA=Y" \ + --tag my-image + + +CI image build arguments +........................ The following build arguments (``--build-arg`` in docker build command) can be used for CI images: @@ -322,11 +393,37 @@ The following build arguments (``--build-arg`` in docker build command) can be u | ``ADDITIONAL_PYTHON_DEPS`` | | additional python dependencies to | | | | install | +------------------------------------------+------------------------------------------+------------------------------------------+ -| ``ADDITIONAL_DEV_DEPS`` | | additional apt dev dependencies to | -| | | install | +| ``DEV_APT_COMMAND`` | (see Dockerfile) | Dev apt command executed before dev deps | +| | | are installed in the first part of image | +------------------------------------------+------------------------------------------+------------------------------------------+ -| ``ADDITIONAL_RUNTIME_DEPS`` | | additional apt runtime dependencies to | -| | | install | +| ``ADDITIONAL_DEV_APT_COMMAND`` | | Additional Dev apt command executed | +| | | before dev dep are installed | +| | | in the first part of the image | ++------------------------------------------+------------------------------------------+------------------------------------------+ +| ``DEV_APT_DEPS`` | (see Dockerfile) | Dev APT dependencies installed | +| | | in the first part of the image | ++------------------------------------------+------------------------------------------+------------------------------------------+ +| ``ADDITIONAL_DEV_APT_DEPS`` | | Additional apt dev dependencies | +| | | installed in the first part of the image | ++------------------------------------------+------------------------------------------+------------------------------------------+ +| ``ADDITIONAL_DEV_APT_ENV`` | | Additional env variables defined | +| | | when installing dev deps | ++------------------------------------------+------------------------------------------+------------------------------------------+ +| ``RUNTIME_APT_COMMAND`` | (see Dockerfile) | Runtime apt command executed before deps | +| | | are installed in first part of the image | ++------------------------------------------+------------------------------------------+------------------------------------------+ +| ``ADDITIONAL_RUNTIME_APT_COMMAND`` | | Additional Runtime apt command executed | +| | | before runtime dep are installed | +| | | in the second part of the image | ++------------------------------------------+------------------------------------------+------------------------------------------+ +| ``RUNTIME_APT_DEPS`` | (see Dockerfile) | Runtime APT dependencies installed | +| | | in the second part of the image | ++------------------------------------------+------------------------------------------+------------------------------------------+ +| ``ADDITIONAL_RUNTIME_APT_DEPS`` | | Additional apt runtime dependencies | +| | | installed in second part of the image | ++------------------------------------------+------------------------------------------+------------------------------------------+ +| ``ADDITIONAL_RUNTIME_APT_ENV`` | | Additional env variables defined | +| | | when installing runtime deps | +------------------------------------------+------------------------------------------+------------------------------------------+ Here are some examples of how CI images can built manually. CI is always built from local sources. @@ -366,7 +463,7 @@ This builds the CI image in version 3.6 with "gcc" and "g++" additional apt dev .. code-block:: docker build . -f Dockerfile.ci --build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \ - --build-arg PYTHON_MAJOR_MINOR_VERSION=3.6 --build-arg ADDITIONAL_DEV_DEPS="gcc g++" + --build-arg PYTHON_MAJOR_MINOR_VERSION=3.6 --build-arg ADDITIONAL_DEV_APT_DEPS="gcc g++" This builds the CI image in version 3.6 with "jdbc" extra and "default-jre-headless" additional apt runtime dependencies added. diff --git a/breeze b/breeze index af62f0a5ed1521..63471d0d71d45c 100755 --- a/breeze +++ b/breeze @@ -950,14 +950,54 @@ function breeze::parse_arguments() { echo "Additional python dependencies: ${ADDITIONAL_PYTHON_DEPS}" shift 2 ;; - --additional-dev-deps) - export ADDITIONAL_DEV_DEPS="${2}" - echo "Additional apt dev dependencies: ${ADDITIONAL_DEV_DEPS}" + --dev-apt-deps) + export DEV_APT_DEPS="${2}" + echo "Apt dev dependencies: ${DEV_APT_DEPS}" shift 2 ;; - --additional-runtime-deps) - export ADDITIONAL_RUNTIME_DEPS="${2}" - echo "Additional apt runtime dependencies: ${ADDITIONAL_RUNTIME_DEPS}" + --additional-dev-apt-deps) + export ADDITIONAL_DEV_APT_DEPS="${2}" + echo "Additional apt dev dependencies: ${ADDITIONAL_DEV_APT_DEPS}" + shift 2 + ;; + --dev-apt-commad) + export DEV_APT_COMMAND="${2}" + echo "Apt dev command: ${DEV_APT_COMMAND}" + shift 2 + ;; + --additional-dev-apt-command) + export ADDITIONAL_DEV_APT_COMMAND="${2}" + echo "Additional Apt dev command: ${ADDITIONAL_DEV_APT_COMMAND}" + shift 2 + ;; + --additional-dev-apt-env) + export ADDITIONAL_DEV_APT_ENV="${2}" + echo "Additional Apt dev environment variables: ${ADDITIONAL_DEV_APT_ENV}" + shift 2 + ;; + --runtime-apt-deps) + export RUNTIME_APT_DEPS="${2}" + echo "Apt runtime dependencies: ${RUNTIME_APT_DEPS}" + shift 2 + ;; + --additional-runtime-apt-deps) + export ADDITIONAL_RUNTIME_APT_DEPS="${2}" + echo "Additional apt runtime dependencies: ${ADDITIONAL_RUNTIME_APT_DEPS}" + shift 2 + ;; + --runtime-apt-commad) + export RUNTIME_APT_COMMAND="${2}" + echo "Apt runtime command: ${RUNTIME_APT_COMMAND}" + shift 2 + ;; + --additional-runtime-apt-command) + export ADDITIONAL_RUNTIME_APT_COMMAND="${2}" + echo "Additional Apt runtime command: ${ADDITIONAL_RUNTIME_APT_COMMAND}" + shift 2 + ;; + --additional-runtime-apt-env) + export ADDITIONAL_RUNTIME_APT_ENV="${2}" + echo "Additional Apt runtime environment variables: ${ADDITIONAL_RUNTIME_APT_ENV}" shift 2 ;; --disable-mysql-client-installation) @@ -2122,6 +2162,8 @@ function breeze::flag_build_docker_images() { images are pulled by default only for the first time you run the environment, later the locally build images are used as cache. +Customization options: + -E, --extras EXTRAS Extras to pass to build images The default are different for CI and production images: @@ -2134,20 +2176,49 @@ ${FORMATTED_DEFAULT_PROD_EXTRAS} --image-tag TAG Additional tag in the image. +--disable-pip-cache + Disables GitHub PIP cache during the build. Useful if github is not reachable during build. + --additional-extras ADDITIONAL_EXTRAS Additional extras to pass to build images The default is no additional extras. --additional-python-deps ADDITIONAL_PYTHON_DEPS Additional python dependencies to use when building the images. ---additional-dev-deps ADDITIONAL_DEV_DEPS +--dev-apt-command DEV_APT_COMMAND + The basic command executed before dev apt deps are installed. + +--additional-dev-apt-command ADDITIONAL_DEV_APT_COMMAND + Additional command executed before dev apt deps are installed. + +--additional-dev-apt-deps ADDITIONAL_DEV_APT_DEPS + Additional apt dev dependencies to use when building the images. + +--dev-apt-deps DEV_APT_DEPS + The basic apt dev dependencies to use when building the images. + +--additional-dev-apt-deps ADDITIONAL_DEV_DEPS Additional apt dev dependencies to use when building the images. ---additional-runtime-deps ADDITIONAL_RUNTIME_DEPS +--additional-dev-apt-envs ADDITIONAL_DEV_APT_ENVS + Additional environment variables set when adding dev dependencies. + +--runtime-apt-command RUNTIME_APT_COMMAND + The basic command executed before runtime apt deps are installed. + +--additional-runtime-apt-command ADDITIONAL_RUNTIME_APT_COMMAND + Additional command executed before runtime apt deps are installed. + +--runtime-apt-deps ADDITIONAL_RUNTIME_APT_DEPS + The basic apt runtime dependencies to use when building the images. + +--additional-runtime-apt-deps ADDITIONAL_RUNTIME_DEPS Additional apt runtime dependencies to use when building the images. ---disable-pip-cache - Disables GitHub PIP cache during the build. Useful if github is not reachable during build. +--additional-runtime-apt-envs ADDITIONAL_RUNTIME_APT_DEPS + Additional environment variables set when adding runtime dependencies. + +Build options: --disable-mysql-client-installation Disables installation of the mysql client which might be problematic if you are building diff --git a/breeze-complete b/breeze-complete index 0ec7babad4e825..8b1d5970eb261e 100644 --- a/breeze-complete +++ b/breeze-complete @@ -137,6 +137,9 @@ dockerhub-user: dockerhub-repo: github-registry github-repository: github-image- postgres-version: mysql-version: additional-extras: additional-python-deps: additional-dev-deps: additional-runtime-deps: image-tag: disable-mysql-client-installation +additional-extras: additional-python-deps: +dev-apt-deps: additional-dev-apt-deps: dev-apt-command: additional-dev-apt-command: additional-dev-apt-env: +runtime-apt-deps: additional-runtime-apt-deps: runtime-apt-command: additional-runtime-apt-command: additional-runtime-apt-env: load-default-connections load-example-dags " diff --git a/docs/production-deployment.rst b/docs/production-deployment.rst index 83f455d86438da..95e08ed2b337ba 100644 --- a/docs/production-deployment.rst +++ b/docs/production-deployment.rst @@ -140,8 +140,8 @@ additional apt dev and runtime dependencies. --build-arg AIRFLOW_SOURCES_TO="/empty" \ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="jdbc" --build-arg ADDITIONAL_PYTHON_DEPS="pandas" - --build-arg ADDITIONAL_DEV_DEPS="gcc g++" - --build-arg ADDITIONAL_RUNTIME_DEPS="default-jre-headless" + --build-arg ADDITIONAL_DEV_APT_DEPS="gcc g++" + --build-arg ADDITIONAL_RUNTIME_APT_DEPS="default-jre-headless" --tag my-image @@ -152,7 +152,43 @@ the same image can be built using ``breeze`` (it supports auto-completion of the ./breeze build-image \ --production-image --python 3.7 --install-airflow-version=1.10.12 \ --additional-extras=jdbc --additional-python-deps="pandas" \ - --additional-dev-deps="gcc g++" --additional-runtime-deps="default-jre-headless" + --additional-dev-apt-deps="gcc g++" --additional-runtime-apt-deps="default-jre-headless" + + +You can customize more aspects of the image - such as additional commands executed before apt dependencies +are installed, or adding extra sources to install your dependencies from. You can see all the arguments +described below but here is an example of rather complex command to customize the image +based on example in `this comment `_: + +.. code-block:: bash + + docker build . -f Dockerfile \ + --build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \ + --build-arg PYTHON_MAJOR_MINOR_VERSION=3.7 \ + --build-arg AIRFLOW_INSTALL_SOURCES="apache-airflow" \ + --build-arg AIRFLOW_INSTALL_VERSION="==1.10.12" \ + --build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-1-10" \ + --build-arg AIRFLOW_SOURCES_FROM="empty" \ + --build-arg AIRFLOW_SOURCES_TO="/empty" \ + --build-arg ADDITIONAL_AIRFLOW_EXTRAS="slack" \ + --build-arg ADDITIONAL_PYTHON_DEPS="apache-airflow-backport-providers-odbc \ + apache-airflow-backport-providers-odbc \ + azure-storage-blob \ + sshtunnel \ + google-api-python-client \ + oauth2client \ + beautifulsoup4 \ + dateparser \ + rocketchat_API \ + typeform" \ + --build-arg ADDITIONAL_DEV_APT_DEPS="msodbcsql17 unixodbc-dev g++" \ + --build-arg ADDITIONAL_DEV_APT_COMMAND="curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add --no-tty - && curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list" \ + --build-arg ADDITIONAL_DEV_ENV_VARS="ACCEPT_EULA=Y" \ + --build-arg ADDITIONAL_RUNTIME_APT_COMMAND="curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add --no-tty - && curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list" \ + --build-arg ADDITIONAL_RUNTIME_APT_DEPS="msodbcsql17 unixodbc git procps vim" \ + --build-arg ADDITIONAL_RUNTIME_ENV_VARS="ACCEPT_EULA=Y" \ + --tag my-image + Customizing & extending the image together .......................................... @@ -303,11 +339,37 @@ The following build arguments (``--build-arg`` in docker build command) can be u | ``ADDITIONAL_PYTHON_DEPS`` | | Optional python packages to extend | | | | the image with some extra dependencies | +------------------------------------------+------------------------------------------+------------------------------------------+ -| ``ADDITIONAL_DEV_DEPS`` | | additional apt dev dependencies to | -| | | install | +| ``DEV_APT_COMMAND`` | (see Dockerfile) | Dev apt command executed before dev deps | +| | | are installed in the Build image | ++------------------------------------------+------------------------------------------+------------------------------------------+ +| ``ADDITIONAL_DEV_APT_COMMAND`` | | Additional Dev apt command executed | +| | | before dev dep are installed | +| | | in the Build image. Should start with && | ++------------------------------------------+------------------------------------------+------------------------------------------+ +| ``DEV_APT_DEPS`` | (see Dockerfile) | Dev APT dependencies installed | +| | | in the Build image | ++------------------------------------------+------------------------------------------+------------------------------------------+ +| ``ADDITIONAL_DEV_APT_DEPS`` | | Additional apt dev dependencies | +| | | installed in the Build image | ++------------------------------------------+------------------------------------------+------------------------------------------+ +| ``ADDITIONAL_DEV_APT_ENV`` | | Additional env variables defined | +| | | when installing dev deps | ++------------------------------------------+------------------------------------------+------------------------------------------+ +| ``RUNTIME_APT_COMMAND`` | (see Dockerfile) | Runtime apt command executed before deps | +| | | are installed in the Main image | ++------------------------------------------+------------------------------------------+------------------------------------------+ +| ``ADDITIONAL_RUNTIME_APT_COMMAND`` | | Additional Runtime apt command executed | +| | | before runtime dep are installed | +| | | in the Main image. Should start with && | ++------------------------------------------+------------------------------------------+------------------------------------------+ +| ``RUNTIME_APT_DEPS`` | (see Dockerfile) | Runtime APT dependencies installed | +| | | in the Main image | ++------------------------------------------+------------------------------------------+------------------------------------------+ +| ``ADDITIONAL_RUNTIME_APT_DEPS`` | | Additional apt runtime dependencies | +| | | installed in the Main image | +------------------------------------------+------------------------------------------+------------------------------------------+ -| ``ADDITIONAL_RUNTIME_DEPS`` | | additional apt runtime dependencies to | -| | | install | +| ``ADDITIONAL_RUNTIME_APT_ENV`` | | Additional env variables defined | +| | | when installing runtime deps | +------------------------------------------+------------------------------------------+------------------------------------------+ | ``AIRFLOW_HOME`` | ``/opt/airflow`` | Airflow’s HOME (that’s where logs and | | | | sqlite databases are stored) | @@ -458,8 +520,8 @@ additional apt dev and runtime dependencies. --build-arg AIRFLOW_SOURCES_FROM="empty" \ --build-arg AIRFLOW_SOURCES_TO="/empty" \ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="jdbc" - --build-arg ADDITIONAL_DEV_DEPS="gcc g++" - --build-arg ADDITIONAL_RUNTIME_DEPS="default-jre-headless" + --build-arg ADDITIONAL_DEV_APT_DEPS="gcc g++" + --build-arg ADDITIONAL_RUNTIME_APT_DEPS="default-jre-headless" More details about the images diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh index 8f5755fdd2b946..bd5b03a8854fcd 100644 --- a/scripts/ci/libraries/_build_images.sh +++ b/scripts/ci/libraries/_build_images.sh @@ -549,6 +549,23 @@ Docker building ${AIRFLOW_CI_IMAGE}. " >"${DETECTED_TERMINAL}" fi set +u + + local additional_dev_args=() + if [[ ${DEV_APT_DEPS} != "" ]]; then + additional_dev_args+=("--build-arg" "DEV_APT_DEPS=\"${DEV_APT_DEPS}\"") + fi + if [[ ${DEV_APT_COMMAND} != "" ]]; then + additional_dev_args+=("--build-arg" "DEV_APT_COMMAND=\"${DEV_APT_COMMAND}\"") + fi + + local additional_runtime_args=() + if [[ ${RUNTIME_APT_DEPS} != "" ]]; then + additional_runtime_args+=("--build-arg" "RUNTIME_APT_DEPS=\"${RUNTIME_APT_DEPS}\"") + fi + if [[ ${RUNTIME_APT_COMMAND} != "" ]]; then + additional_runtime_args+=("--build-arg" "RUNTIME_APT_COMMAND=\"${RUNTIME_APT_COMMAND}\"") + fi + docker build \ "${EXTRA_DOCKER_CI_BUILD_FLAGS[@]}" \ --build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE}" \ @@ -559,11 +576,17 @@ Docker building ${AIRFLOW_CI_IMAGE}. --build-arg AIRFLOW_PRE_CACHED_PIP_PACKAGES="${AIRFLOW_PRE_CACHED_PIP_PACKAGES}" \ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \ --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \ - --build-arg ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS}" \ - --build-arg ADDITIONAL_RUNTIME_DEPS="${ADDITIONAL_RUNTIME_DEPS}" \ + --build-arg ADDITIONAL_DEV_APT_COMMAND="${ADDITIONAL_DEV_APT_COMMAND}" \ + --build-arg ADDITIONAL_DEV_APT_DEPS="${ADDITIONAL_DEV_APT_DEPS}" \ + --build-arg ADDITIONAL_DEV_APT_ENV="${ADDITIONAL_DEV_APT_ENV}" \ + --build-arg ADDITIONAL_RUNTIME_APT_COMMAND="${ADDITIONAL_RUNTIME_APT_COMMAND}" \ + --build-arg ADDITIONAL_RUNTIME_APT_DEPS="${ADDITIONAL_RUNTIME_APT_DEPS}" \ + --build-arg ADDITIONAL_RUNTIME_APT_ENV="${ADDITIONAL_RUNTIME_APT_ENV}" \ --build-arg UPGRADE_TO_LATEST_CONSTRAINTS="${UPGRADE_TO_LATEST_CONSTRAINTS}" \ --build-arg BUILD_ID="${CI_BUILD_ID}" \ --build-arg COMMIT_SHA="${COMMIT_SHA}" \ + "${additional_dev_args[@]}" \ + "${additional_runtime_args[@]}" \ "${DOCKER_CACHE_CI_DIRECTIVE[@]}" \ -t "${AIRFLOW_CI_IMAGE}" \ --target "main" \ @@ -684,6 +707,13 @@ function build_images::build_prod_images() { exit 1 fi set +u + local additional_dev_args=() + if [[ ${DEV_APT_DEPS} != "" ]]; then + additional_dev_args+=("--build-arg" "DEV_APT_DEPS=\"${DEV_APT_DEPS}\"") + fi + if [[ ${DEV_APT_COMMAND} != "" ]]; then + additional_dev_args+=("--build-arg" "DEV_APT_COMMAND=\"${DEV_APT_COMMAND}\"") + fi docker build \ "${EXTRA_DOCKER_PROD_BUILD_FLAGS[@]}" \ --build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE}" \ @@ -695,13 +725,23 @@ function build_images::build_prod_images() { --build-arg AIRFLOW_PRE_CACHED_PIP_PACKAGES="${AIRFLOW_PRE_CACHED_PIP_PACKAGES}" \ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \ --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \ - --build-arg ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS}" \ + "${additional_dev_args[@]}" \ + --build-arg ADDITIONAL_DEV_APT_COMMAND="${ADDITIONAL_DEV_APT_COMMAND}" \ + --build-arg ADDITIONAL_DEV_APT_DEPS="${ADDITIONAL_DEV_APT_DEPS}" \ + --build-arg ADDITIONAL_DEV_APT_ENV="${ADDITIONAL_DEV_APT_ENV}" \ --build-arg BUILD_ID="${CI_BUILD_ID}" \ --build-arg COMMIT_SHA="${COMMIT_SHA}" \ "${DOCKER_CACHE_PROD_BUILD_DIRECTIVE[@]}" \ -t "${AIRFLOW_PROD_BUILD_IMAGE}" \ --target "airflow-build-image" \ . -f Dockerfile + local additional_runtime_args=() + if [[ ${RUNTIME_APT_DEPS} != "" ]]; then + additional_runtime_args+=("--build-arg" "RUNTIME_APT_DEPS=\"${RUNTIME_APT_DEPS}\"") + fi + if [[ ${RUNTIME_APT_COMMAND} != "" ]]; then + additional_runtime_args+=("--build-arg" "RUNTIME_APT_COMMAND=\"${RUNTIME_APT_COMMAND}\"") + fi docker build \ "${EXTRA_DOCKER_PROD_BUILD_FLAGS[@]}" \ --build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE}" \ @@ -709,14 +749,20 @@ function build_images::build_prod_images() { --build-arg INSTALL_MYSQL_CLIENT="${INSTALL_MYSQL_CLIENT}" \ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \ --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \ - --build-arg ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS}" \ - --build-arg ADDITIONAL_RUNTIME_DEPS="${ADDITIONAL_RUNTIME_DEPS}" \ + --build-arg ADDITIONAL_DEV_APT_COMMAND="${ADDITIONAL_DEV_APT_COMMAND}" \ + --build-arg ADDITIONAL_DEV_APT_DEPS="${ADDITIONAL_DEV_APT_DEPS}" \ + --build-arg ADDITIONAL_DEV_APT_ENV="${ADDITIONAL_DEV_APT_ENV}" \ + --build-arg ADDITIONAL_RUNTIME_APT_COMMAND="${ADDITIONAL_RUNTIME_APT_COMMAND}" \ + --build-arg ADDITIONAL_RUNTIME_APT_DEPS="${ADDITIONAL_RUNTIME_APT_DEPS}" \ + --build-arg ADDITIONAL_RUNTIME_APT_ENV="${ADDITIONAL_RUNTIME_APT_ENV}" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg AIRFLOW_BRANCH="${AIRFLOW_BRANCH_FOR_PYPI_PRELOADING}" \ --build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \ --build-arg AIRFLOW_PRE_CACHED_PIP_PACKAGES="${AIRFLOW_PRE_CACHED_PIP_PACKAGES}" \ --build-arg BUILD_ID="${CI_BUILD_ID}" \ --build-arg COMMIT_SHA="${COMMIT_SHA}" \ + "${additional_dev_args[@]}" \ + "${additional_runtime_args[@]}" \ "${DOCKER_CACHE_PROD_DIRECTIVE[@]}" \ -t "${AIRFLOW_PROD_IMAGE}" \ --target "main" \ diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index eb56560f46a19e..db51773bdc4382 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -317,10 +317,27 @@ function initialization::initialize_image_build_variables() { export ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS:=""}" # Additional python dependencies on top of the default ones export ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS:=""}" + # Use default DEV_APT_COMMAND + export DEV_APT_COMMAND="" + # Use default DEV_APT_DEPS + export DEV_APT_DEPS="" + # Use empty ADDITIONAL_DEV_APT_COMMAND + export ADDITIONAL_DEV_APT_COMMAND="" # additional development apt dependencies on top of the default ones - export ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS:=""}" + export ADDITIONAL_DEV_APT_DEPS="${ADDITIONAL_DEV_APT_DEPS:=""}" + # Use empty ADDITIONAL_DEV_APT_ENV + export ADDITIONAL_DEV_APT_ENV="${ADDITIONAL_DEV_APT_ENV:=""}" + # Use default RUNTIME_APT_COMMAND + export RUNTIME_APT_COMMAND="" + # Use default RUNTIME_APT_DEVS + export RUNTIME_APT_DEVS="" + # Use empty ADDITIONAL_RUNTIME_APT_COMMAND + export ADDITIONAL_RUNTIME_APT_COMMAND="" # additional runtime apt dependencies on top of the default ones export ADDITIONAL_RUNTIME_DEPS="${ADDITIONAL_RUNTIME_DEPS:=""}" + export ADDITIONAL_RUNTIME_APT_DEPS="${ADDITIONAL_RUNTIME_APT_DEPS:=""}" + # Use empty ADDITIONAL_RUNTIME_APT_ENV + export ADDITIONAL_RUNTIME_APT_ENV="${ADDITIONAL_RUNTIME_APT_ENV:=""}" # whether pre cached pip packages are used during build export AIRFLOW_PRE_CACHED_PIP_PACKAGES="${AIRFLOW_PRE_CACHED_PIP_PACKAGES:="true"}" # by default install mysql client @@ -640,13 +657,28 @@ function initialization::make_constants_read_only() { readonly CI_BUILD_ID readonly CI_JOB_ID + readonly IMAGE_TAG + + readonly AIRFLOW_PRE_CACHED_PIP_PACKAGES # AIRFLOW_EXTRAS are made readonly by the time the image is built (either PROD or CI) readonly ADDITIONAL_AIRFLOW_EXTRAS readonly ADDITIONAL_PYTHON_DEPS - readonly ADDITIONAL_DEV_DEPS - readonly ADDITIONAL_RUNTIME_DEPS + readonly AIRFLOW_PRE_CACHED_PIP_PACKAGES - readonly IMAGE_TAG + + readonly DEV_APT_COMMAND + readonly DEV_APT_DEPS + + readonly ADDITIONAL_DEV_APT_COMMAND + readonly ADDITIONAL_DEV_APT_DEPS + readonly ADDITIONAL_DEV_APT_ENV + + readonly RUNTIME_APT_COMMAND + readonly RUNTIME_APT_DEPS + + readonly ADDITIONAL_RUNTIME_APT_COMMAND + readonly ADDITIONAL_RUNTIME_APT_DEPS + readonly ADDITIONAL_RUNTIME_APT_ENV readonly DOCKERHUB_USER readonly DOCKERHUB_REPO