[AIRFLOW-6462] Limit exported variables in Dockerfile/Breeze #7057

mik-laj · 2020-01-05T03:46:50Z

We have a lot of variables after typing the command export in a clean environment. I would like to limit their number. to make the result of the export command more readable.

After:

declare -x ADDITIONAL_PATH="~/.local/bin"
declare -x AIRFLOW_BRANCH="master"
declare -x AIRFLOW_CI_BUILD_EPOCH="1"
declare -x AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD="true"
declare -x AIRFLOW_CONTAINER_DOCKER_IMAGE="apache/airflow:master-python3.6-ci"
declare -x AIRFLOW_EXTRAS="devel_ci"
declare -x AIRFLOW_HOME="/root/airflow"
declare -x AIRFLOW_REPO="apache/airflow"
declare -x AIRFLOW_SOURCES="/opt/airflow"
declare -x AIRFLOW_VERSION="2.0.0.dev0"
declare -x AIRFLOW__CORE__DAGS_FOLDER="/opt/airflow/tests/dags"
declare -x AIRFLOW__CORE__EXECUTOR="SequentialExecutor"
declare -x AIRFLOW__CORE__SQL_ALCHEMY_CONN="sqlite:////root/airflow/airflow.db"
declare -x AIRFLOW__CORE__UNIT_TEST_MODE="True"
declare -x AWS_DEFAULT_REGION="us-east-1"
declare -x BACKEND="sqlite"
declare -x BREEZE="true"
declare -x CASS_DRIVER_BUILD_CONCURRENCY="8"
declare -x CASS_DRIVER_NO_CYTHON="1"
declare -x CELERY_BROKER_URLS="amqp://guest:guest@rabbitmq:5672,redis://redis:6379/0"
declare -x DEBIAN_FRONTEND="noninteractive"
declare -x DEPENDENCIES_EPOCH_NUMBER="2"
declare -x DISABLE_CHECKS_FOR_TESTS="missing-docstring,no-self-use,too-many-public-methods,protected-access,do-not-use-asserts"
declare -x DOCKER_HOST="tcp://docker:2375"
declare -x ENV="docker"
declare -x GPG_KEY="0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D"
declare -x HADOOP_DISTRO="cdh"
declare -x HADOOP_HOME="/opt/hadoop-cdh"
declare -x HADOOP_OPTS="-D/opt/krb5.conf"
declare -x HIVE_HOME="/opt/hive"
declare -x HOME="/root"
declare -x HOSTNAME="e484de340f32"
declare -x JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
declare -x KRB5_CONFIG="/etc/krb5.conf"
declare -x KRB5_KTNAME="/etc/airflow.keytab"
declare -x KUBERNETES_MODE="git_mode"
declare -x KUBERNETES_VERSION="v1.13.0"
declare -x LANG="C.UTF-8"
declare -x LANGUAGE="C.UTF-8"
declare -x LC_ALL="C.UTF-8"
declare -x LC_CTYPE="C.UTF-8"
declare -x LC_MESSAGES="C.UTF-8"
declare -x MINICLUSTER_HOME="/opt/minicluster"
declare -x OLDPWD="/opt/airflow"
declare -x PATH="/root:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hive/bin:/opt/airflow"
declare -x PIP_DEPENDENCIES_EPOCH_NUMBER="2"
declare -x PIP_NO_CACHE_DIR="true"
declare -x PIP_VERSION="19.0.2"
declare -x PWD="/opt/airflow"
declare -x PYTHONDONTWRITEBYTECODE="true"
declare -x PYTHON_BASE_IMAGE="python:3.6-slim-stretch"
declare -x PYTHON_GET_PIP_SHA256="b86f36cc4345ae87bfd4f10ef6b2dbfa7a872fbff70608a1e43944d283fd0eee"
declare -x PYTHON_GET_PIP_URL="https://github.com/pypa/get-pip/raw/ffe826207a010164265d9cc807978e3604d18ca0/get-pip.py"
declare -x PYTHON_PIP_VERSION="19.3.1"
declare -x PYTHON_VERSION="3.6.10"
declare -x RUN_TESTS="false"
declare -x SHLVL="2"
declare -x SOURCE_BRANCH="master"
declare -x TERM="xterm"
declare -x USER="root"
declare -x XUNIT_FILE="/root/airflow/logs/all_tests.xml"

Before:

declare -x ADDITIONAL_PATH="~/.local/bin"
declare -x AIRFLOW_BRANCH="master"
declare -x AIRFLOW_CI_BUILD_EPOCH="1"
declare -x AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD="true"
declare -x AIRFLOW_CONTAINER_DOCKER_IMAGE="apache/airflow:master-python3.6-ci"
declare -x AIRFLOW_EXTRAS="devel_ci"
declare -x AIRFLOW_GITHUB_DOWNLOAD="https://raw.githubusercontent.com/apache/airflow/master"
declare -x AIRFLOW_HOME="/root/airflow"
declare -x AIRFLOW_REPO="apache/airflow"
declare -x AIRFLOW_SOURCES="/opt/airflow"
declare -x AIRFLOW_VERSION="2.0.0.dev0"
declare -x AIRFLOW__CORE__DAGS_FOLDER="/opt/airflow/tests/dags"
declare -x AIRFLOW__CORE__EXECUTOR="SequentialExecutor"
declare -x AIRFLOW__CORE__SQL_ALCHEMY_CONN="sqlite:////root/airflow/airflow.db"
declare -x AIRFLOW__CORE__UNIT_TEST_MODE="True"
declare -x AWS_DEFAULT_REGION="us-east-1"
declare -x BACKEND="sqlite"
declare -x BREEZE="true"
declare -x CASS_DRIVER_BUILD_CONCURRENCY="8"
declare -x CASS_DRIVER_NO_CYTHON="1"
declare -x CELERY_BROKER_URLS="amqp://guest:guest@rabbitmq:5672,redis://redis:6379/0"
declare -x DEBIAN_FRONTEND="noninteractive"
declare -x DEPENDENCIES_EPOCH_NUMBER="2"
declare -x DISABLE_CHECKS_FOR_TESTS="missing-docstring,no-self-use,too-many-public-methods,protected-access,do-not-use-asserts"
declare -x DOCKER_HOST="tcp://docker:2375"
declare -x ENV="docker"
declare -x GPG_KEY="0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D"
declare -x HADOOP_DISTRO="cdh"
declare -x HADOOP_DISTRO_VERSION="5.11.0"
declare -x HADOOP_DOWNLOAD_URL="https://archive.cloudera.com/cdh5/cdh/5/hadoop-2.6.0-cdh5.11.0.tar.gz"
declare -x HADOOP_HOME="/opt/hadoop-cdh"
declare -x HADOOP_MAJOR="5"
declare -x HADOOP_OPTS="-D/opt/krb5.conf"
declare -x HADOOP_TMP_FILE="/tmp/hadoop.tar.gz"
declare -x HADOOP_URL="https://archive.cloudera.com/cdh5/cdh/5/"
declare -x HADOOP_VERSION="2.6.0"
declare -x HIVE_HOME="/opt/hive"
declare -x HIVE_TMP_FILE="/tmp/hive.tar.gz"
declare -x HIVE_URL="https://archive.cloudera.com/cdh5/cdh/5/hive-1.1.0-cdh5.11.0.tar.gz"
declare -x HIVE_VERSION="1.1.0"
declare -x HOME="/root"
declare -x HOSTNAME="10d460be0a01"
declare -x JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
declare -x KIND_VERSION="v0.5.0"
declare -x KRB5_CONFIG="/etc/krb5.conf"
declare -x KRB5_KTNAME="/etc/airflow.keytab"
declare -x KUBECTL_VERSION="v1.15.0"
declare -x KUBERNETES_MODE="git_mode"
declare -x KUBERNETES_VERSION="v1.13.0"
declare -x LANG="C.UTF-8"
declare -x LANGUAGE="C.UTF-8"
declare -x LC_ALL="C.UTF-8"
declare -x LC_CTYPE="C.UTF-8"
declare -x LC_MESSAGES="C.UTF-8"
declare -x MINICLUSTER_BASE="https://github.com/bolkedebruin/minicluster/releases/download/"
declare -x MINICLUSTER_HOME="/opt/minicluster"
declare -x MINICLUSTER_TMP_FILE="/tmp/minicluster.zip"
declare -x MINICLUSTER_URL="https://github.com/bolkedebruin/minicluster/releases/download/1.1/minicluster-1.1-SNAPSHOT-bin.zip"
declare -x MINICLUSTER_VER="1.1"
declare -x OLDPWD="/opt/airflow"
declare -x PATH="/root:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/hive/bin:/opt/airflow"
declare -x PIP_DEPENDENCIES_EPOCH_NUMBER="2"
declare -x PIP_NO_CACHE_DIR="true"
declare -x PIP_VERSION="19.0.2"
declare -x PWD="/opt/airflow"
declare -x PYTHONDONTWRITEBYTECODE="true"
declare -x PYTHON_BASE_IMAGE="python:3.6-slim-stretch"
declare -x PYTHON_GET_PIP_SHA256="b86f36cc4345ae87bfd4f10ef6b2dbfa7a872fbff70608a1e43944d283fd0eee"
declare -x PYTHON_GET_PIP_URL="https://github.com/pypa/get-pip/raw/ffe826207a010164265d9cc807978e3604d18ca0/get-pip.py"
declare -x PYTHON_PIP_VERSION="19.3.1"
declare -x PYTHON_VERSION="3.6.10"
declare -x RAT_JAR="/opt/apache-rat-0.13.jar"
declare -x RAT_JAR_MD5="/opt/apache-rat-0.13.jar.md5"
declare -x RAT_URL="https://repo1.maven.org/maven2/org/apache/rat/apache-rat/0.13/apache-rat-0.13.jar"
declare -x RAT_URL_MD5="https://repo1.maven.org/maven2/org/apache/rat/apache-rat/0.13/apache-rat-0.13.jar.md5"
declare -x RAT_VERSION="0.13"
declare -x RUN_TESTS="false"
declare -x SHLVL="2"
declare -x SOURCE_BRANCH="master"
declare -x TERM="xterm"
declare -x USER="root"
declare -x XUNIT_FILE="/root/airflow/logs/all_tests.xml"

Link to JIRA issue: https://issues.apache.org/jira/browse/AIRFLOW-6462

Description above provides context of the change
Commit message starts with [AIRFLOW-NNNN], where AIRFLOW-NNNN = JIRA ID*
Unit tests coverage for changes (not needed for documentation changes)
Commits follow "How to write a good git commit message"
Relevant documentation is updated including usage instructions.
I will engage committers as explained in Contribution Workflow Example.

(*) For document-only changes, no JIRA issue is needed. Commit message starts [AIRFLOW-XXXX].

In case of fundamental code change, Airflow Improvement Proposal (AIP) is needed.
In case of a new dependency, check compliance with the ASF 3rd Party License Policy.
In case of backwards incompatible changes please leave a note in UPDATING.md.
Read the Pull Request Guidelines for more information.

mik-laj · 2020-01-05T03:48:05Z

Dockerfile

@@ -266,8 +271,6 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}
 ARG AIRFLOW_BRANCH=master
 ENV AIRFLOW_BRANCH=${AIRFLOW_BRANCH}

-ENV AIRFLOW_GITHUB_DOWNLOAD=https://raw.githubusercontent.com/${AIRFLOW_REPO}/${AIRFLOW_BRANCH}


It is an unused variable.

potiuk · 2020-01-05T09:20:29Z

Needs rebase :)

mik-laj · 2020-01-05T15:08:35Z

I rebased and added a small fixup related to Apache RAT and Hadoop/Hive.

(cherry picked from commit 5ae2f96)

…7057)

…7057) (cherry picked from commit 5ae2f96) (cherry picked from commit a02c6e6)

mik-laj requested review from potiuk and kaxil January 5, 2020 03:46

mik-laj commented Jan 5, 2020

View reviewed changes

kaxil approved these changes Jan 5, 2020

View reviewed changes

mik-laj force-pushed the AIRFLOW-6462 branch from dad8c2a to 11fadba Compare January 5, 2020 06:05

potiuk approved these changes Jan 5, 2020

View reviewed changes

[AIRFLOW-6462] Limit exported variables in Dockerfile/Breeze

2136e98

mik-laj force-pushed the AIRFLOW-6462 branch from 11fadba to 2136e98 Compare January 5, 2020 14:27

boring-cyborg bot added the area:dev-tools label Jan 5, 2020

mik-laj changed the title ~~[AIRFLOW-6462][depends on AIRFLOW-6461] Limit exported variables in Dockerfile/Breeze~~ [AIRFLOW-6462] Limit exported variables in Dockerfile/Breeze Jan 5, 2020

fixup! [AIRFLOW-6462] Limit exported variables in Dockerfile/Breeze

7d9b556

mik-laj force-pushed the AIRFLOW-6462 branch from be35583 to 7d9b556 Compare January 5, 2020 15:07

potiuk approved these changes Jan 5, 2020

View reviewed changes

potiuk merged commit 5ae2f96 into apache:master Jan 5, 2020

potiuk pushed a commit that referenced this pull request Jan 21, 2020

[AIRFLOW-6462] Limit exported variables in Dockerfile/Breeze (#7057)

1c7bd29

(cherry picked from commit 5ae2f96)

kaxil pushed a commit that referenced this pull request Jan 22, 2020

[AIRFLOW-6462] Limit exported variables in Dockerfile/Breeze (#7057)

0289281

(cherry picked from commit 5ae2f96)

kaxil pushed a commit that referenced this pull request Jan 23, 2020

[AIRFLOW-6462] Limit exported variables in Dockerfile/Breeze (#7057)

11ffdd1

(cherry picked from commit 5ae2f96)

potiuk pushed a commit that referenced this pull request Jan 26, 2020

[AIRFLOW-6462] Limit exported variables in Dockerfile/Breeze (#7057)

edbeb2d

(cherry picked from commit 5ae2f96)

kaxil pushed a commit that referenced this pull request Jan 26, 2020

[AIRFLOW-6462] Limit exported variables in Dockerfile/Breeze (#7057)

a02c6e6

(cherry picked from commit 5ae2f96)

galuszkak pushed a commit to FlyrInc/apache-airflow that referenced this pull request Mar 5, 2020

[AIRFLOW-6462] Limit exported variables in Dockerfile/Breeze (apache#…

d7412c0

…7057)

kaxil pushed a commit to astronomer/airflow that referenced this pull request Mar 30, 2020

[AIRFLOW-6462] Limit exported variables in Dockerfile/Breeze (apache#…

5a29928

…7057) (cherry picked from commit 5ae2f96) (cherry picked from commit a02c6e6)

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[AIRFLOW-6462] Limit exported variables in Dockerfile/Breeze #7057

[AIRFLOW-6462] Limit exported variables in Dockerfile/Breeze #7057

mik-laj commented Jan 5, 2020

mik-laj Jan 5, 2020

potiuk commented Jan 5, 2020

mik-laj commented Jan 5, 2020

[AIRFLOW-6462] Limit exported variables in Dockerfile/Breeze #7057

[AIRFLOW-6462] Limit exported variables in Dockerfile/Breeze #7057

Conversation

mik-laj commented Jan 5, 2020

mik-laj Jan 5, 2020

Choose a reason for hiding this comment

potiuk commented Jan 5, 2020

mik-laj commented Jan 5, 2020