Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Refactor dev/dev-env-setup.sh to separate mandatory and optional setup #12253

Merged
merged 8 commits into from
Jun 7, 2024
Merged

Refactor dev/dev-env-setup.sh to separate mandatory and optional setup #12253

Changes from 5 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
3a992b3
[ML-36509]Make dev/dev-env-setup.sh easy to resume
xq-yin Jun 5, 2024
806155e
refactor
xq-yin Jun 5, 2024
a3a9d3e
amend
xq-yin Jun 5, 2024
bd8551d
revert unrelated formatting
xq-yin Jun 5, 2024
0443d25
clean
xq-yin Jun 5, 2024
e89fffb
more refactoring
xq-yin Jun 6, 2024
29025c2
fix mlflow typo
xq-yin Jun 6, 2024
aa2d246
resolve comments
xq-yin Jun 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
331 changes: 180 additions & 151 deletions dev/dev-env-setup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ EOF

directory="$(pwd)/.venvs/mlflow-dev"
MLFLOW_HOME="$(pwd)"
REPO_ROOT=$(git rev-parse --show-toplevel)
rd="$REPO_ROOT/requirements"
VENV_DIR="$directory/bin/activate"
while :
do
case "$1" in
Expand Down Expand Up @@ -145,189 +148,215 @@ version_gt() {
return $(( ${#VER1[@]} <= ${#VER2[@]} ))
}


# Check if pyenv is installed and offer to install it if not present
pyenv_exist=$(command -v pyenv)
check_and_install_pyenv() {
pyenv_exist=$(command -v pyenv)

if [ -z "$pyenv_exist" ]; then
if [ -z "$GITHUB_ACTIONS" ]; then
read -p "pyenv is required to be installed to manage python versions. Would you like to install it? $(tput bold)(y/n)$(tput sgr0): " -n 1 -r
echo
fi
if [[ $REPLY =~ ^[Yy]$ || -n "$GITHUB_ACTIONS" ]]; then
if [[ "$machine" == mac ]]; then
check_and_install_brew "pyenv"
echo "Installing pyenv..."
echo "Note: this will probably take a considerable amount of time."
brew install pyenv
brew install openssl readline sqlite3 xz zlib libomp
elif [[ "$machine" == linux ]]; then
sudo apt-get update -y
sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \
libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \
libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev
# Install pyenv from source
git clone --depth 1 https://github.com/pyenv/pyenv.git "$HOME/.pyenv"
PYENV_ROOT="$HOME/.pyenv"
PYENV_BIN="$PYENV_ROOT/bin"
PATH="$PYENV_BIN:$PATH"
if [ -n "$GITHUB_ACTIONS" ]; then
echo "$PYENV_BIN" >> "$GITHUB_PATH"
echo "PYENV_ROOT=$PYENV_ROOT" >> "$GITHUB_ENV"
if [ -z "$pyenv_exist" ]; then
if [ -z "$GITHUB_ACTIONS" ]; then
read -p "pyenv is required to be installed to manage python versions. Would you like to install it? $(tput bold)(y/n)$(tput sgr0): " -n 1 -r
echo
fi
if [[ $REPLY =~ ^[Yy]$ || -n "$GITHUB_ACTIONS" ]]; then
if [[ "$machine" == mac ]]; then
check_and_install_brew "pyenv"
echo "Installing pyenv..."
echo "Note: this will probably take a considerable amount of time."
brew install pyenv
brew install openssl readline sqlite3 xz zlib libomp
elif [[ "$machine" == linux ]]; then
sudo apt-get update -y
sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \
libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \
libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev
# Install pyenv from source
git clone --depth 1 https://github.com/pyenv/pyenv.git "$HOME/.pyenv"
PYENV_ROOT="$HOME/.pyenv"
PYENV_BIN="$PYENV_ROOT/bin"
PATH="$PYENV_BIN:$PATH"
if [ -n "$GITHUB_ACTIONS" ]; then
echo "$PYENV_BIN" >>"$GITHUB_PATH"
echo "PYENV_ROOT=$PYENV_ROOT" >>"$GITHUB_ENV"
fi
else
echo "Unknown operating system environment: $machine exiting."
exit 1
xq-yin marked this conversation as resolved.
Show resolved Hide resolved
fi
else
echo "Unknown operating system environment: $machine exiting."
PYENV_README=https://github.com/pyenv/pyenv/blob/master/README.md
echo "pyenv is required to use this environment setup script. Please install by following instructions here: $PYENV_README"
exit 1
fi
else
PYENV_README=https://github.com/pyenv/pyenv/blob/master/README.md
echo "pyenv is required to use this environment setup script. Please install by following instructions here: $PYENV_README"
exit 1
fi
fi

REPO_ROOT=$(git rev-parse --show-toplevel)
rd="$REPO_ROOT/requirements"

# Get the minimum supported version for development purposes
min_py_version="3.8"

echo "The minimum version of Python to ensure backwards compatibility for MLflow development is: $(tput bold; tput setaf 3)$min_py_version$(tput sgr0)"
}

if [[ -n "$override_py_ver" ]]; then
version_levels=$(grep -o '\.' <<< "$override_py_ver" | wc -l)
if [[ $version_levels -eq 1 ]]; then
PY_INSTALL_VERSION=$(minor_to_micro $override_py_ver)
elif [[ $version_levels -eq 2 ]]; then
PY_INSTALL_VERSION=$override_py_ver
check_and_install_min_py_version() {
# Get the minimum supported version for development purposes
min_py_version="3.8"

echo "The minimum version of Python to ensure backwards compatibility for MLflow development is: $(
tput bold
tput setaf 3
)$min_py_version$(tput sgr0)"

if [[ -n "$override_py_ver" ]]; then
version_levels=$(grep -o '\.' <<<"$override_py_ver" | wc -l)
if [[ $version_levels -eq 1 ]]; then
PY_INSTALL_VERSION=$(minor_to_micro $override_py_ver)
elif [[ $version_levels -eq 2 ]]; then
PY_INSTALL_VERSION=$override_py_ver
else
echo "You must supply a python override version with either minor (e.g., '3.9') or micro (e.g., '3.9.5'). '$override_py_ver' is invalid."
exit 1
fi
else
echo "You must supply a python override version with either minor (e.g., '3.9') or micro (e.g., '3.9.5'). '$override_py_ver' is invalid."
exit 1
PY_INSTALL_VERSION=$(minor_to_micro $min_py_version)
fi
else
PY_INSTALL_VERSION=$(minor_to_micro $min_py_version)
fi

echo "The top-level dependencies that will be installed are: "

if [[ -n "$full" ]]; then
files=("$rd/extra-ml-requirements.txt" "$rd/test-requirements.txt" "$rd/lint-requirements.txt" "$rd/doc-requirements.txt")
echo "Files:"
echo "MLflow test plugin: $MLFLOW_HOME/tests/resources/mlflow-test-plugin"
echo "The local development branch of MLflow installed in editable mode with 'extras' requirements"
echo "The following packages: "
else
files=("$rd/test-requirements.txt" "$rd/lint-requirements.txt" "$rd/doc-requirements.txt")
fi
tail -n +1 "${files[@]}" | grep "^[^#= ]" | sort | cat
echo "The top-level dependencies that will be installed are: "

echo "$(tput setaf 2) Installing Python version $(tput bold)$PY_INSTALL_VERSION$(tput sgr0)"
if [[ -n "$full" ]]; then
files=("$rd/extra-ml-requirements.txt" "$rd/test-requirements.txt" "$rd/lint-requirements.txt" "$rd/doc-requirements.txt")
echo "Files:"
echo "MLflow test plugin: $MLFLOW_HOME/tests/resources/mlflow-test-plugin"
echo "The local development branch of MLflow installed in editable mode with 'extras' requirements"
echo "The following packages: "
else
files=("$rd/test-requirements.txt" "$rd/lint-requirements.txt" "$rd/doc-requirements.txt")
fi
tail -n +1 "${files[@]}" | grep "^[^#= ]" | sort | cat

# Install the Python version if it cannot be found
pyenv install -s "$PY_INSTALL_VERSION"
pyenv local "$PY_INSTALL_VERSION"
pyenv exec pip install $(quiet_command) --upgrade pip
pyenv exec pip install $(quiet_command) virtualenv
pyenv exec pip install $(quiet_command) pre-commit
echo "$(tput setaf 2) Installing Python version $(tput bold)$PY_INSTALL_VERSION$(tput sgr0)"

VENV_DIR="$directory/bin/activate"
# Install the Python version if it cannot be found
pyenv install -s "$PY_INSTALL_VERSION"
pyenv local "$PY_INSTALL_VERSION"
pyenv exec pip install $(quiet_command) --upgrade pip
pyenv exec pip install $(quiet_command) virtualenv
pyenv exec pip install $(quiet_command) pre-commit
}

# Check if the virtualenv already exists at the specified path
if [[ -d "$directory" ]]; then
if [ -z "$GITHUB_ACTIONS" ]; then
read -p "A virtual environment is already located at $directory. Do you wish to replace it? $(tput bold; tput setaf 2)(y/n) $(tput sgr0)" -n 1 -r
echo
fi
if [[ $REPLY =~ ^[Yy]$ || -n "$GITHUB_ACTIONS" ]]; then
echo "Replacing Virtual environment in '$directory'. Installing new instance."
pyenv exec virtualenv --clear "$directory"
create_virtualenv() {
if [[ -d "$directory" ]]; then
if [ -z "$GITHUB_ACTIONS" ]; then
read -p "A virtual environment is already located at $directory. Do you wish to replace it? $(
tput bold
tput setaf 2
)(y/n) $(tput sgr0)" -n 1 -r
echo
fi
if [[ $REPLY =~ ^[Yy]$ || -n "$GITHUB_ACTIONS" ]]; then
echo "Replacing Virtual environment in '$directory'. Installing new instance."
pyenv exec virtualenv --clear "$directory"
fi
else
# Create a virtual environment with the specified Python version
pyenv exec virtualenv --python "$PY_INSTALL_VERSION" "$directory"
fi
else
# Create a virtual environment with the specified Python version
pyenv exec virtualenv --python "$PY_INSTALL_VERSION" "$directory"
fi

# Activate the virtual environment
# shellcheck disable=SC1090
source "$VENV_DIR"
# Activate the virtual environment
# shellcheck disable=SC1090
source "$VENV_DIR"

echo "$(tput setaf 2)Current Python version: $(tput bold)$(python --version)$(tput sgr0)"
echo "$(tput setaf 3)Activated environment is located: $(tput bold) $directory/bin/activate$(tput sgr0)"

echo "Installing pip dependencies for development environment."
echo "$(tput setaf 2)Current Python version: $(tput bold)$(python --version)$(tput sgr0)"
echo "$(tput setaf 3)Activated environment is located: $(tput bold) $directory/bin/activate$(tput sgr0)"
}

# Install current checked out version of MLflow (local)
pip install $(quiet_command) -e .[extras]
pip install pre-commit

if [[ -n "$full" ]]; then
# Install dev requirements and test plugin
pip install $(quiet_command) -r "$rd/dev-requirements.txt"
# Install test plugin
pip install $(quiet_command) -e "$MLFLOW_HOME/tests/resources//mlflow-test-plugin"
echo "Finished installing pip dependencies."
else
files=("$rd/test-requirements.txt" "$rd/lint-requirements.txt" "$rd/doc-requirements.txt")
for r in "${files[@]}";
do
pip install $(quiet_command) -r "$r"
done
fi

echo "$(tput setaf 2; tput smul)Python packages that have been installed:$(tput rmul)"
echo "$(pip freeze)$(tput sgr0)"
install_mlflow() {
echo "Installing pip dependencies for development environment."

pip install $(quiet_command) -e .[extras]
pip install pre-commit
xq-yin marked this conversation as resolved.
Show resolved Hide resolved

if [[ -n "$full" ]]; then
# Install dev requirements and test plugin
pip install $(quiet_command) -r "$rd/dev-requirements.txt"
xq-yin marked this conversation as resolved.
Show resolved Hide resolved
# Install test plugin
pip install $(quiet_command) -e "$MLFLOW_HOME/tests/resources//mlflow-test-plugin"
echo "Finished installing pip dependencies."
else
files=("$rd/test-requirements.txt" "$rd/lint-requirements.txt" "$rd/doc-requirements.txt")
for r in "${files[@]}"; do
pip install $(quiet_command) -r "$r"
done
fi

command -v docker >/dev/null 2>&1 || echo "$(tput bold; tput setaf 1)A docker installation cannot be found. Please ensure that docker is installed to run all tests locally.$(tput sgr0)"
echo "$(
tput setaf 2
tput smul
)Python packages that have been installed:$(tput rmul)"
echo "$(pip freeze)$(tput sgr0)"

command -v docker >/dev/null 2>&1 || echo "$(
tput bold
tput setaf 1
)A docker installation cannot be found. Please ensure that docker is installed to run all tests locally.$(tput sgr0)"
xq-yin marked this conversation as resolved.
Show resolved Hide resolved
}

# check if pandoc with required version is installed and offer to install it if not present
pandoc_version=$(pandoc --version | grep "pandoc" | awk '{print $2}')
if [[ -z "$pandoc_version" ]] || ! version_gt "$pandoc_version" "2.2.1"; then
if [ -z "$GITHUB_ACTIONS" ]; then
read -p "Pandoc version 2.2.1 or above is required to generate documentation. Would you like to install it? $(tput bold)(y/n)$(tput sgr0): " -n 1 -r
echo
fi
# Check if pandoc with required version is installed and offer to install it if not present
check_and_install_pandoc() {
pandoc_version=$(pandoc --version | grep "pandoc" | awk '{print $2}')
if [[ -z "$pandoc_version" ]] || ! version_gt "$pandoc_version" "2.2.1"; then
if [ -z "$GITHUB_ACTIONS" ]; then
read -p "Pandoc version 2.2.1 or above is required to generate documentation. Would you like to install it? $(tput bold)(y/n)$(tput sgr0): " -n 1 -r
xq-yin marked this conversation as resolved.
Show resolved Hide resolved
echo
fi

if [[ $REPLY =~ ^[Yy]$ || -n "$GITHUB_ACTIONS" ]]; then
echo "Installing Pandoc..."
if [[ "$machine" == mac ]]; then
check_and_install_brew "pandoc"
brew install pandoc
elif [[ "$machine" == linux ]]; then
# install pandoc via deb package as `apt-get` gives too old version
TEMP_DEB=$(mktemp) && \
wget --directory-prefix $TEMP_DEB https://github.com/jgm/pandoc/releases/download/2.16.2/pandoc-2.16.2-1-amd64.deb && \
sudo dpkg --install $(find $TEMP_DEB -name '*.deb') && \
rm -rf $TEMP_DEB
else
echo "Unknown operating system environment: $machine exiting."
exit 1
if [[ $REPLY =~ ^[Yy]$ || -n "$GITHUB_ACTIONS" ]]; then
echo "Installing Pandoc..."
if [[ "$machine" == mac ]]; then
check_and_install_brew "pandoc"
brew install pandoc
elif [[ "$machine" == linux ]]; then
# Install pandoc via deb package as `apt-get` gives too old version
TEMP_DEB=$(mktemp) &&
wget --directory-prefix $TEMP_DEB https://github.com/jgm/pandoc/releases/download/2.16.2/pandoc-2.16.2-1-amd64.deb &&
sudo dpkg --install $(find $TEMP_DEB -name '*.deb') &&
rm -rf $TEMP_DEB
else
echo "Unknown operating system environment: $machine exiting."
exit 1
fi
fi
fi
fi

}

# Setup git environment configuration for proper signing of commits
git_user=$(git config user.name)
git_email=$(git config user.email)
# Set up git environment configuration for proper signing of commits
set_up_git_signoff() {
git_user=$(git config user.name)
git_email=$(git config user.email)

if [[ -z "$git_email" || -z "$git_user" ]]; then
read -p "Your git environment is not setup to automatically sign your commits. Would you like to configure it? $(tput bold; tput setaf 2)(y/n): $(tput sgr0)" -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
read -p "Enter the user name you would like to have associated with your commit signature: " -r git_user_name
if [[ -z "$git_email" || -z "$git_user" ]]; then
read -p "Your git environment is not setup to automatically sign your commits. Would you like to configure it? $(
tput bold
tput setaf 2
)(y/n): $(tput sgr0)" -n 1 -r
echo
git config --global user.name "$git_user_name"
echo "Git user name set as: $(git config user.name)"
read -p "Enter your email address for your commit signature: " -r git_user_email
git config --global user.email "$git_user_email"
echo "Git user email set as: $(git config user.email)"
else
echo "Failing to set git 'user.name' and 'user.email' will result in unsigned commits. Ensure that you sign commits manually for CI checks to pass."
if [[ $REPLY =~ ^[Yy]$ ]]; then
read -p "Enter the user name you would like to have associated with your commit signature: " -r git_user_name
echo
git config --global user.name "$git_user_name"
echo "Git user name set as: $(git config user.name)"
read -p "Enter your email address for your commit signature: " -r git_user_email
git config --global user.email "$git_user_email"
echo "Git user email set as: $(git config user.email)"
else
echo "Failing to set git 'user.name' and 'user.email' will result in unsigned commits. Ensure that you sign commits manually for CI checks to pass."
fi
fi
fi
}

# Main env setup starts
check_and_install_pyenv
check_and_install_min_py_version
create_virtualenv
install_mlflow
check_and_install_pandoc
set_up_git_signoff

# setup pre-commit hooks
# Set up pre-commit hooks
pre-commit install -t pre-commit -t prepare-commit-msg

echo "$(tput setaf 2)Your MLflow development environment can be activated by running: $(tput bold)source $VENV_DIR$(tput sgr0)"