diff --git a/.gitignore b/.gitignore index b77214960..0e0e5a5ef 100644 --- a/.gitignore +++ b/.gitignore @@ -1,23 +1,44 @@ +# Python venv __pycache__ -cudnn_windows -.vscode *.egg-info build wd14_tagger_model + +# IDE and Editor specific +.vscode + +# CUDNN for Windows +cudnn_windows + +# Cache and temporary files +.cache .DS_Store + +# Scripts and executables locon gui-user.bat gui-user.ps1 -.vscode + +# Version control +SmilingWolf wandb + +# Setup and logs setup.log logs -SmilingWolf + +# Miscellaneous uninstall.txt + +# Test files test/output test/logs test/*.json test/ft + +# Temporary requirements requirements_tmp_for_setup.txt -0.13.3 + +# Version specific +0.13.3 \ No newline at end of file diff --git a/config_files/accelerate/runpod.yaml b/config_files/accelerate/runpod.yaml new file mode 100644 index 000000000..1198fe6d2 --- /dev/null +++ b/config_files/accelerate/runpod.yaml @@ -0,0 +1,22 @@ +command_file: null +commands: null +compute_environment: LOCAL_MACHINE +deepspeed_config: {} +distributed_type: 'NO' +downcast_bf16: 'no' +dynamo_backend: 'NO' +fsdp_config: {} +gpu_ids: all +machine_rank: 0 +main_process_ip: null +main_process_port: null +main_training_function: main +megatron_lm_config: {} +mixed_precision: 'no' +num_machines: 1 +num_processes: 1 +rdzv_backend: static +same_network: true +tpu_name: null +tpu_zone: null +use_cpu: false \ No newline at end of file diff --git a/gui.sh b/gui.sh index 9fa29e21f..7f66a3eff 100755 --- a/gui.sh +++ b/gui.sh @@ -29,9 +29,10 @@ SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" && pwd) # Step into GUI local directory cd "$SCRIPT_DIR" || exit 1 -if [ "$RUNPOD" = false ]; then - # Activate the virtual environment +if [ -d "$SCRIPT_DIR/venv" ]; then source "$SCRIPT_DIR/venv/bin/activate" || exit 1 +else + echo "venv folder does not exist. Not activating..." fi # Check if LD_LIBRARY_PATH environment variable exists @@ -45,6 +46,8 @@ if [[ -z "${LD_LIBRARY_PATH}" ]]; then echo -e "${YELLOW}Certain functionalities may not work correctly.${RESET}" echo -e "${YELLOW}Please ensure that the required libraries are properly configured.${RESET}" echo -e " " + echo -e "${YELLOW}If you use WSL2 you may want to: export LD_LIBRARY_PATH=/usr/lib/wsl/lib/${RESET}" + echo -e " " fi # Determine the requirements file based on the system diff --git a/requirements_runpod.txt b/requirements_runpod.txt index ded3ea27c..71ef35ea4 100644 --- a/requirements_runpod.txt +++ b/requirements_runpod.txt @@ -1,4 +1,4 @@ -torch==2.0.1+cu117 torchvision==0.15.2+cu117 --extra-index-url https://download.pytorch.org/whl/cu117 # no_verify leave this to specify not checking this a verification stage +torch==2.0.1+cu118 torchvision==0.15.2+cu118 --extra-index-url https://download.pytorch.org/whl/cu118 # no_verify leave this to specify not checking this a verification stage xformers==0.0.20 bitsandbytes==0.39.1 accelerate==0.15.0 tensorboard==2.12.1 tensorflow==2.12.0 tensorrt diff --git a/setup-runpod.sh b/setup-runpod.sh new file mode 100755 index 000000000..16ebc8b9c --- /dev/null +++ b/setup-runpod.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# This gets the directory the script is run from so pathing can work relative to the script where needed. +SCRIPT_DIR="$(cd -- "$(dirname -- "$0")" && pwd)" + +# Install tk and python3.10-venv +echo "Installing tk and python3.10-venv..." +apt update -y && apt install -y python3-tk python3.10-venv + +# Check if the venv folder doesn't exist +if [ ! -d "$SCRIPT_DIR/venv" ]; then + echo "Creating venv..." + python3 -m venv "$SCRIPT_DIR/venv" +fi + +# Activate the virtual environment +echo "Activating venv..." +source "$SCRIPT_DIR/venv/bin/activate" || exit 1 + +# Run setup_linux.py script with platform requirements +echo "Running setup_linux.py..." +python "$SCRIPT_DIR/setup/setup_linux.py" --platform-requirements-file=requirements_runpod.txt + +# Configure accelerate +echo "Configuring accelerate..." +mkdir -p "/root/.cache/huggingface/accelerate" +cp "$SCRIPT_DIR/config_files/accelerate/runpod.yaml" "/root/.cache/huggingface/accelerate/default_config.yaml" +# echo "To manually configure accelerate run: $SCRIPT_DIR/venv/bin/activate" diff --git a/setup.sh b/setup.sh index e8bd61117..3f521f37c 100755 --- a/setup.sh +++ b/setup.sh @@ -1,11 +1,6 @@ #!/usr/bin/env bash -# This file will be the host environment setup file for all operating systems other than base Windows. - -# Set the required package versions here. -TENSORFLOW_MACOS_VERSION="2.12.0" -TENSORFLOW_METAL_VERSION="0.8.0" - +# Function to display help information display_help() { cat <&5 + line=${line##*=} + echo "$line" + return 0 + elif command -v python >/dev/null; then + line="$(python -mplatform)" + echo "$line" + return 0 + elif command -v python3 >/dev/null; then + line="$(python3 -mplatform)" + echo "$line" + return 0 + else + line="None" + echo "$line" + return 1 fi - - case $opt in - b | branch) BRANCH="$OPTARG" ;; - d | dir) DIR="$OPTARG" ;; - g | git-repo) GIT_REPO="$OPTARG" ;; - i | interactive) INTERACTIVE=true ;; - n | no-git-update) SKIP_GIT_UPDATE=true ;; - p | public) PUBLIC=true ;; - r | runpod) RUNPOD=true ;; - s | skip-space-check) SKIP_SPACE_CHECK=true ;; - u | no-gui) SKIP_GUI=true ;; - v) ((VERBOSITY = VERBOSITY + 1)) ;; - h) display_help && exit 0 ;; - *) display_help && exit 0 ;; - esac -done -shift $((OPTIND - 1)) - -# Just in case someone puts in a relative path into $DIR, -# we're going to get the absolute path of that. -if [[ "$DIR" != /* ]] && [[ "$DIR" != ~* ]]; then - DIR="$( - cd "$(dirname "$DIR")" || exit 1 - pwd - )/$(basename "$DIR")" -fi - -for v in $( #Start counting from 3 since 1 and 2 are standards (stdout/stderr). - seq 3 $VERBOSITY -); do - (("$v" <= "$MAXVERBOSITY")) && eval exec "$v>&2" #Don't change anything higher than the maximum verbosity allowed. -done - -for v in $( #From the verbosity level one higher than requested, through the maximum; - seq $((VERBOSITY + 1)) $MAXVERBOSITY -); do - (("$v" > "2")) && eval exec "$v>/dev/null" #Redirect these to bitbucket, provided that they don't match stdout and stderr. -done - -# Example of how to use the verbosity levels. -# printf "%s\n" "This message is seen at verbosity level 1 and above." >&3 -# printf "%s\n" "This message is seen at verbosity level 2 and above." >&4 -# printf "%s\n" "This message is seen at verbosity level 3 and above." >&5 - -# Debug variable dump at max verbosity -echo "BRANCH: $BRANCH -DIR: $DIR -GIT_REPO: $GIT_REPO -INTERACTIVE: $INTERACTIVE -PUBLIC: $PUBLIC -RUNPOD: $RUNPOD -SKIP_SPACE_CHECK: $SKIP_SPACE_CHECK -VERBOSITY: $VERBOSITY -Script directory is ${SCRIPT_DIR}." >&5 - -# This must be set after the getopts loop to account for $DIR changes. -PARENT_DIR="$(dirname "${DIR}")" -VENV_DIR="$DIR/venv" - -if [ -w "$PARENT_DIR" ] && [ ! -d "$DIR" ]; then - echo "Creating install folder ${DIR}." - mkdir "$DIR" -fi - -if [ ! -w "$DIR" ]; then - echo "We cannot write to ${DIR}." - echo "Please ensure the install directory is accurate and you have the correct permissions." - exit 1 -fi +} -# Shared functions -# This checks for free space on the installation drive and returns that in Gb. -size_available() { - local folder - if [ -d "$DIR" ]; then - folder="$DIR" - elif [ -d "$PARENT_DIR" ]; then - folder="$PARENT_DIR" - elif [ -d "$(echo "$DIR" | cut -d "/" -f2)" ]; then - folder="$(echo "$DIR" | cut -d "/" -f2)" +# Function to get the distro family +get_distro_family() { + local line + if [ -f /etc/os-release ]; then + if grep -Eiq '^ID_LIKE=' /etc/os-release >/dev/null; then + line="$(grep -Ei '^ID_LIKE=' /etc/os-release)" + echo "Raw detected os-release distro family line: $line" >&5 + line=${line##*=} + echo "$line" + return 0 + else + line="None" + echo "$line" + return 1 + fi else - echo "We are assuming a root drive install for space-checking purposes." - folder='/' + line="None" + echo "$line" + return 1 fi +} - local FREESPACEINKB - FREESPACEINKB="$(df -Pk "$folder" | sed 1d | grep -v used | awk '{ print $4 "\t" }')" - echo "Detected available space in Kb: $FREESPACEINKB" >&5 - local FREESPACEINGB - FREESPACEINGB=$((FREESPACEINKB / 1024 / 1024)) - echo "$FREESPACEINGB" +# Function to check available storage space +check_storage_space() { + if [ "$SKIP_SPACE_CHECK" = false ]; then + if [ "$(size_available)" -lt 10 ]; then + echo "You have less than 10Gb of free space. This installation may fail." + MSGTIMEOUT=10 # In seconds + MESSAGE="Continuing in..." + echo "Press control-c to cancel the installation." + for ((i = MSGTIMEOUT; i >= 0; i--)); do + printf "\r${MESSAGE} %ss. " "${i}" + sleep 1 + done + fi + fi } -# The expected usage is create_symlinks symlink target_file +# Function to create symlinks create_symlinks() { local symlink="$1" local target_file="$2" @@ -217,7 +178,7 @@ create_symlinks() { fi } - +# Function to install Python dependencies install_python_dependencies() { local TEMP_REQUIREMENTS_FILE @@ -238,13 +199,6 @@ install_python_dependencies() { source "$DIR/venv/bin/activate" fi - # Updating pip if there is one - # echo "Checking for pip updates before Python operations." - # pip install --upgrade pip - - # echo "Installing python dependencies. This could take a few minutes as it downloads files." - # echo "If this operation ever runs too long, you can rerun this script in verbose mode to check." - case "$OSTYPE" in "lin"*) if [ "$RUNPOD" = true ]; then @@ -262,29 +216,6 @@ install_python_dependencies() { ;; esac - # DEBUG ONLY (Update this version number to whatever PyCharm recommends) - # pip install pydevd-pycharm~=223.8836.43 - - # Create a temporary requirements file - # TEMP_REQUIREMENTS_FILE=$(mktemp) - - # if [[ "$OSTYPE" == "darwin"* ]]; then - # echo "Copying $DIR/requirements_macos.txt to $TEMP_REQUIREMENTS_FILE" >&3 - # echo "Replacing the . for lib to our DIR variable in $TEMP_REQUIREMENTS_FILE." >&3 - # awk -v dir="$DIR" '/#.*kohya_ss.*library/{print; getline; sub(/^\.$/, dir)}1' "$DIR/requirements_macos.txt" >"$TEMP_REQUIREMENTS_FILE" - # else - # echo "Copying $DIR/requirements_linux.txt to $TEMP_REQUIREMENTS_FILE" >&3 - # echo "Replacing the . for lib to our DIR variable in $TEMP_REQUIREMENTS_FILE." >&3 - # awk -v dir="$DIR" '/#.*kohya_ss.*library/{print; getline; sub(/^\.$/, dir)}1' "$DIR/requirements_linux.txt" >"$TEMP_REQUIREMENTS_FILE" - # fi - - # # Install the Python dependencies from the temporary requirements file - # if [ $VERBOSITY == 2 ]; then - # python -m pip install --quiet --upgrade -r "$TEMP_REQUIREMENTS_FILE" - # else - # python -m pip install --upgrade -r "$TEMP_REQUIREMENTS_FILE" - # fi - if [ -n "$VIRTUAL_ENV" ] && ! inDocker; then if command -v deactivate >/dev/null; then echo "Exiting Python virtual environment." @@ -295,10 +226,7 @@ install_python_dependencies() { fi } - -# Attempt to non-interactively install a default accelerate config file unless specified otherwise. -# Documentation for order of precedence locations for configuration file for automated installation: -# https://huggingface.co/docs/accelerate/basic_tutorials/launch#custom-configurations +# Function to configure accelerate configure_accelerate() { echo "Source accelerate config location: $DIR/config_files/accelerate/default_config.yaml" >&3 if [ "$INTERACTIVE" = true ]; then @@ -333,45 +261,7 @@ configure_accelerate() { fi } -# Offer a warning and opportunity to cancel the installation if < 10Gb of Free Space detected -check_storage_space() { - if [ "$SKIP_SPACE_CHECK" = false ]; then - if [ "$(size_available)" -lt 10 ]; then - echo "You have less than 10Gb of free space. This installation may fail." - MSGTIMEOUT=10 # In seconds - MESSAGE="Continuing in..." - echo "Press control-c to cancel the installation." - for ((i = MSGTIMEOUT; i >= 0; i--)); do - printf "\r${MESSAGE} %ss. " "${i}" - sleep 1 - done - fi - fi -} - -isContainerOrPod() { - local cgroup=/proc/1/cgroup - test -f $cgroup && (grep -qE ':cpuset:/(docker|kubepods)' $cgroup || grep -q ':/docker/' $cgroup) -} - -isDockerBuildkit() { - local cgroup=/proc/1/cgroup - test -f $cgroup && grep -q ':cpuset:/docker/buildkit' $cgroup -} - -isDockerContainer() { - [ -e /.dockerenv ] -} - -inDocker() { - if isContainerOrPod || isDockerBuildkit || isDockerContainer; then - return 0 - else - return 1 - fi -} - -# These are the git operations that will run to update or clone the repo +# Function to update Kohya_SS repo update_kohya_ss() { if [ "$SKIP_GIT_UPDATE" = false ]; then if command -v git >/dev/null; then @@ -407,6 +297,130 @@ update_kohya_ss() { fi } +# Section: Command-line options parsing + +while getopts ":vb:d:g:inprus-:" opt; do + # support long options: https://stackoverflow.com/a/28466267/519360 + if [ "$opt" = "-" ]; then # long option: reformulate OPT and OPTARG + opt="${OPTARG%%=*}" # extract long option name + OPTARG="${OPTARG#$opt}" # extract long option argument (may be empty) + OPTARG="${OPTARG#=}" # if long option argument, remove assigning `=` + fi + + case $opt in + b | branch) BRANCH="$OPTARG" ;; + d | dir) DIR="$OPTARG" ;; + g | git-repo) GIT_REPO="$OPTARG" ;; + i | interactive) INTERACTIVE=true ;; + n | no-git-update) SKIP_GIT_UPDATE=true ;; + p | public) PUBLIC=true ;; + r | runpod) RUNPOD=true ;; + s | skip-space-check) SKIP_SPACE_CHECK=true ;; + u | no-gui) SKIP_GUI=true ;; + v) ((VERBOSITY = VERBOSITY + 1)) ;; + h) display_help && exit 0 ;; + *) display_help && exit 0 ;; + esac +done +shift $((OPTIND - 1)) + +# Just in case someone puts in a relative path into $DIR, +# we're going to get the absolute path of that. +if [[ "$DIR" != /* ]] && [[ "$DIR" != ~* ]]; then + DIR="$( + cd "$(dirname "$DIR")" || exit 1 + pwd + )/$(basename "$DIR")" +fi + +for v in $( #Start counting from 3 since 1 and 2 are standards (stdout/stderr). + seq 3 $VERBOSITY +); do + (("$v" <= "$MAXVERBOSITY")) && eval exec "$v>&2" #Don't change anything higher than the maximum verbosity allowed. +done + +for v in $( #From the verbosity level one higher than requested, through the maximum; + seq $((VERBOSITY + 1)) $MAXVERBOSITY +); do + (("$v" > "2")) && eval exec "$v>/dev/null" #Redirect these to bitbucket, provided that they don't match stdout and stderr. +done + +# Example of how to use the verbosity levels. +# printf "%s\n" "This message is seen at verbosity level 1 and above." >&3 +# printf "%s\n" "This message is seen at verbosity level 2 and above." >&4 +# printf "%s\n" "This message is seen at verbosity level 3 and above." >&5 + +# Debug variable dump at max verbosity +echo "BRANCH: $BRANCH +DIR: $DIR +GIT_REPO: $GIT_REPO +INTERACTIVE: $INTERACTIVE +PUBLIC: $PUBLIC +RUNPOD: $RUNPOD +SKIP_SPACE_CHECK: $SKIP_SPACE_CHECK +VERBOSITY: $VERBOSITY +Script directory is ${SCRIPT_DIR}." >&5 + +# This must be set after the getopts loop to account for $DIR changes. +PARENT_DIR="$(dirname "${DIR}")" +VENV_DIR="$DIR/venv" + +if [ -w "$PARENT_DIR" ] && [ ! -d "$DIR" ]; then + echo "Creating install folder ${DIR}." + mkdir "$DIR" +fi + +if [ ! -w "$DIR" ]; then + echo "We cannot write to ${DIR}." + echo "Please ensure the install directory is accurate and you have the correct permissions." + exit 1 +fi + +# Shared functions +# This checks for free space on the installation drive and returns that in Gb. +size_available() { + local folder + if [ -d "$DIR" ]; then + folder="$DIR" + elif [ -d "$PARENT_DIR" ]; then + folder="$PARENT_DIR" + elif [ -d "$(echo "$DIR" | cut -d "/" -f2)" ]; then + folder="$(echo "$DIR" | cut -d "/" -f2)" + else + echo "We are assuming a root drive install for space-checking purposes." + folder='/' + fi + + local FREESPACEINKB + FREESPACEINKB="$(df -Pk "$folder" | sed 1d | grep -v used | awk '{ print $4 "\t" }')" + echo "Detected available space in Kb: $FREESPACEINKB" >&5 + local FREESPACEINGB + FREESPACEINGB=$((FREESPACEINKB / 1024 / 1024)) + echo "$FREESPACEINGB" +} + +isContainerOrPod() { + local cgroup=/proc/1/cgroup + test -f $cgroup && (grep -qE ':cpuset:/(docker|kubepods)' $cgroup || grep -q ':/docker/' $cgroup) +} + +isDockerBuildkit() { + local cgroup=/proc/1/cgroup + test -f $cgroup && grep -q ':cpuset:/docker/buildkit' $cgroup +} + +isDockerContainer() { + [ -e /.dockerenv ] +} + +inDocker() { + if isContainerOrPod || isDockerBuildkit || isDockerContainer; then + return 0 + else + return 1 + fi +} + # Start OS-specific detection and work if [[ "$OSTYPE" == "lin"* ]]; then # Check if root or sudo @@ -419,55 +433,6 @@ if [[ "$OSTYPE" == "lin"* ]]; then root=true fi - get_distro_name() { - local line - if [ -f /etc/os-release ]; then - # We search for the line starting with ID= - # Then we remove the ID= prefix to get the name itself - line="$(grep -Ei '^ID=' /etc/os-release)" - echo "Raw detected os-release distro line: $line" >&5 - line=${line##*=} - echo "$line" - return 0 - elif command -v python >/dev/null; then - line="$(python -mplatform)" - echo "$line" - return 0 - elif command -v python3 >/dev/null; then - line="$(python3 -mplatform)" - echo "$line" - return 0 - else - line="None" - echo "$line" - return 1 - fi - } - - # We search for the line starting with ID_LIKE= - # Then we remove the ID_LIKE= prefix to get the name itself - # This is the "type" of distro. For example, Ubuntu returns "debian". - get_distro_family() { - local line - if [ -f /etc/os-release ]; then - if grep -Eiq '^ID_LIKE=' /etc/os-release >/dev/null; then - line="$(grep -Ei '^ID_LIKE=' /etc/os-release)" - echo "Raw detected os-release distro family line: $line" >&5 - line=${line##*=} - echo "$line" - return 0 - else - line="None" - echo "$line" - return 1 - fi - else - line="None" - echo "$line" - return 1 - fi - } - check_storage_space update_kohya_ss