Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allow to specify pip version via github URL in Docker/CI workflow #28697

Merged
merged 1 commit into from
Jan 5, 2023
Merged

Allow to specify pip version via github URL in Docker/CI workflow #28697

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 22 additions & 26 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,20 +325,12 @@ COPY <<"EOF" /install_pip_version.sh

: "${AIRFLOW_PIP_VERSION:?Should be set}"

function install_pip_version() {
echo
echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
echo
pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" &&
mkdir -p ${HOME}/.local/bin
}

common::get_colors
common::get_airflow_version_specification
common::override_pip_version_if_needed
common::show_pip_version_and_location

install_pip_version
common::install_pip_version
EOF

# The content below is automatically copied from scripts/docker/install_airflow_dependencies_from_branch_tip.sh
Expand Down Expand Up @@ -369,8 +361,7 @@ function install_airflow_dependencies_from_branch_tip() {
${ADDITIONAL_PIP_INSTALL_FLAGS} \
"https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
common::install_pip_version
pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true
set +x
echo
Expand Down Expand Up @@ -445,6 +436,18 @@ function common::show_pip_version_and_location() {
echo "pip on path: $(which pip)"
echo "Using pip: $(pip --version)"
}

function common::install_pip_version() {
echo
echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
echo
if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then
pip install --disable-pip-version-check --no-cache-dir "pip @ ${AIRFLOW_PIP_VERSION}"
else
pip install --disable-pip-version-check --no-cache-dir "pip==${AIRFLOW_PIP_VERSION}"
fi
mkdir -p "${HOME}/.local/bin"
}
EOF

# The content below is automatically copied from scripts/docker/pip
Expand Down Expand Up @@ -531,8 +534,7 @@ function install_airflow_and_providers_from_docker_context_files(){
${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
set +x

# make sure correct PIP version is left installed
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
common::install_pip_version
pip check
}

Expand All @@ -549,9 +551,8 @@ function install_all_other_packages_from_docker_context_files() {
set -x
pip install ${ADDITIONAL_PIP_INSTALL_FLAGS} \
--root-user-action ignore --force-reinstall --no-deps --no-index ${reinstalling_other_packages}
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
set -x
common::install_pip_version
set +x
fi
}

Expand Down Expand Up @@ -613,8 +614,7 @@ function install_airflow() {
set +x
fi

# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
common::install_pip_version
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
Expand All @@ -628,15 +628,13 @@ function install_airflow() {
${ADDITIONAL_PIP_INSTALL_FLAGS} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
common::install_pip_version
# then upgrade if needed without using constraints to account for new limits in setup.py
pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
${AIRFLOW_INSTALL_EDITABLE_FLAG} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
common::install_pip_version
set +x
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
Expand Down Expand Up @@ -675,8 +673,7 @@ function install_additional_dependencies() {
pip install --root-user-action ignore --upgrade --upgrade-strategy eager \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
common::install_pip_version
set +x
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
Expand All @@ -690,8 +687,7 @@ function install_additional_dependencies() {
pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
${ADDITIONAL_PYTHON_DEPS}
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
common::install_pip_version
set +x
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
Expand Down
40 changes: 19 additions & 21 deletions Dockerfile.ci
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -285,20 +285,12 @@ COPY <<"EOF" /install_pip_version.sh

: "${AIRFLOW_PIP_VERSION:?Should be set}"

function install_pip_version() {
echo
echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
echo
pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" &&
mkdir -p ${HOME}/.local/bin
}

common::get_colors
common::get_airflow_version_specification
common::override_pip_version_if_needed
common::show_pip_version_and_location

install_pip_version
common::install_pip_version
EOF

# The content below is automatically copied from scripts/docker/install_airflow_dependencies_from_branch_tip.sh
Expand Down Expand Up @@ -329,8 +321,7 @@ function install_airflow_dependencies_from_branch_tip() {
${ADDITIONAL_PIP_INSTALL_FLAGS} \
"https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
common::install_pip_version
pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true
set +x
echo
Expand Down Expand Up @@ -405,6 +396,18 @@ function common::show_pip_version_and_location() {
echo "pip on path: $(which pip)"
echo "Using pip: $(pip --version)"
}

function common::install_pip_version() {
echo
echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
echo
if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then
pip install --disable-pip-version-check --no-cache-dir "pip @ ${AIRFLOW_PIP_VERSION}"
else
pip install --disable-pip-version-check --no-cache-dir "pip==${AIRFLOW_PIP_VERSION}"
fi
mkdir -p "${HOME}/.local/bin"
}
EOF

# The content below is automatically copied from scripts/docker/install_pipx_tools.sh
Expand Down Expand Up @@ -480,8 +483,7 @@ function install_airflow() {
set +x
fi

# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
common::install_pip_version
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
Expand All @@ -495,15 +497,13 @@ function install_airflow() {
${ADDITIONAL_PIP_INSTALL_FLAGS} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
common::install_pip_version
# then upgrade if needed without using constraints to account for new limits in setup.py
pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
${AIRFLOW_INSTALL_EDITABLE_FLAG} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
common::install_pip_version
set +x
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
Expand Down Expand Up @@ -542,8 +542,7 @@ function install_additional_dependencies() {
pip install --root-user-action ignore --upgrade --upgrade-strategy eager \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
common::install_pip_version
set +x
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
Expand All @@ -557,8 +556,7 @@ function install_additional_dependencies() {
pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
${ADDITIONAL_PYTHON_DEPS}
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
common::install_pip_version
set +x
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
Expand Down
12 changes: 12 additions & 0 deletions scripts/docker/common.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,3 +70,15 @@ function common::show_pip_version_and_location() {
echo "pip on path: $(which pip)"
echo "Using pip: $(pip --version)"
}

function common::install_pip_version() {
echo
echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
echo
if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then
pip install --disable-pip-version-check --no-cache-dir "pip @ ${AIRFLOW_PIP_VERSION}"
else
pip install --disable-pip-version-check --no-cache-dir "pip==${AIRFLOW_PIP_VERSION}"
fi
mkdir -p "${HOME}/.local/bin"
}
6 changes: 2 additions & 4 deletions scripts/docker/install_additional_dependencies.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,7 @@ function install_additional_dependencies() {
pip install --root-user-action ignore --upgrade --upgrade-strategy eager \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
common::install_pip_version
set +x
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
Expand All @@ -50,8 +49,7 @@ function install_additional_dependencies() {
pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
${ADDITIONAL_PYTHON_DEPS}
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
common::install_pip_version
set +x
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
Expand Down
9 changes: 3 additions & 6 deletions scripts/docker/install_airflow.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,7 @@ function install_airflow() {
set +x
fi

# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
common::install_pip_version
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
echo
Expand All @@ -84,15 +83,13 @@ function install_airflow() {
${ADDITIONAL_PIP_INSTALL_FLAGS} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
common::install_pip_version
# then upgrade if needed without using constraints to account for new limits in setup.py
pip install --root-user-action ignore --upgrade --upgrade-strategy only-if-needed \
${ADDITIONAL_PIP_INSTALL_FLAGS} \
${AIRFLOW_INSTALL_EDITABLE_FLAG} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
common::install_pip_version
set +x
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
Expand Down
3 changes: 1 addition & 2 deletions scripts/docker/install_airflow_dependencies_from_branch_tip.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,7 @@ function install_airflow_dependencies_from_branch_tip() {
${ADDITIONAL_PIP_INSTALL_FLAGS} \
"https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
common::install_pip_version
pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true
set +x
echo
Expand Down
8 changes: 3 additions & 5 deletions scripts/docker/install_from_docker_context_files.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,7 @@ function install_airflow_and_providers_from_docker_context_files(){
${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
set +x

# make sure correct PIP version is left installed
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
common::install_pip_version
pip check
}

Expand All @@ -107,9 +106,8 @@ function install_all_other_packages_from_docker_context_files() {
set -x
pip install ${ADDITIONAL_PIP_INSTALL_FLAGS} \
--root-user-action ignore --force-reinstall --no-deps --no-index ${reinstalling_other_packages}
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" 2>/dev/null
set -x
common::install_pip_version
set +x
fi
}

Expand Down
10 changes: 1 addition & 9 deletions scripts/docker/install_pip_version.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,9 @@

: "${AIRFLOW_PIP_VERSION:?Should be set}"

function install_pip_version() {
echo
echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
echo
pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" &&
mkdir -p ${HOME}/.local/bin
}

common::get_colors
common::get_airflow_version_specification
common::override_pip_version_if_needed
common::show_pip_version_and_location

install_pip_version
common::install_pip_version
7 changes: 6 additions & 1 deletion scripts/in_container/_in_container_utils.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -353,7 +353,12 @@ function setup_provider_packages() {


function install_supported_pip_version() {
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
if [[ ${AIRFLOW_PIP_VERSION} =~ .*https.* ]]; then
pip install --disable-pip-version-check "pip @ ${AIRFLOW_PIP_VERSION}"
else
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
fi

}

function filename_to_python_module() {
Expand Down
3 changes: 2 additions & 1 deletion scripts/in_container/run_prepare_airflow_packages.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ function prepare_airflow_packages() {
rm -rf -- *egg-info*
rm -rf -- build

pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}"
install_supported_pip_version
pip install "wheel==${WHEEL_VERSION}"

local packages=()

Expand Down