From 38bdcea3c5cef7e9ac0a3cd421d9188efb057bd9 Mon Sep 17 00:00:00 2001 From: devdn Date: Fri, 3 Mar 2023 18:17:29 -0500 Subject: [PATCH 1/4] Add setup script for ubuntu users --- README-ja.md | 12 ++++++++++++ README.md | 13 +++++++++++++ ubuntu_setup.sh | 29 +++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100755 ubuntu_setup.sh diff --git a/README-ja.md b/README-ja.md index 064464c0e..319efe915 100644 --- a/README-ja.md +++ b/README-ja.md @@ -38,6 +38,18 @@ PowerShellを使う場合、venvを使えるようにするためには以下の - 「Set-ExecutionPolicy Unrestricted」と入力し、Yと答えます。 - 管理者のPowerShellを閉じます。 +## Ubuntu環境でのインストール + +``` +git clone https://github.com/kohya-ss/sd-scripts.git +cd sd-scripts +bash ubuntu_setup.sh +``` + +をコマンドプロンプトで実行し、tkをインストールし、accelerateの質問をWindowsと同じように答えます。 + +`./gui.sh`でGUIを実行します。 + ## Windows環境でのインストール 以下の例ではPyTorchは1.12.1／CUDA 11.6版をインストールします。CUDA 11.3版やPyTorch 1.13を使う場合は適宜書き換えください。 diff --git a/README.md b/README.md index 7c5dc8fe7..2dd649bab 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,19 @@ If you run on Linux and would like to use the GUI, there is now a port of it as ## Installation +### Ubuntu +In the terminal, run + +``` +git clone https://github.com/bmaltais/kohya_ss.git +cd kohya_ss +bash ubuntu_setup.sh +``` + +then configure accelerate with the same answers as in the Windows instructions when prompted. + +### Windows + Give unrestricted script access to powershell so venv can work: - Run PowerShell as an administrator diff --git a/ubuntu_setup.sh b/ubuntu_setup.sh new file mode 100755 index 000000000..5bc862b64 --- /dev/null +++ b/ubuntu_setup.sh @@ -0,0 +1,29 @@ +#!/bin/bash +echo installing tk +sudo apt install python3-tk +python3 -m venv venv +source venv/bin/activate +pip install torch==1.12.1+cu116 torchvision==0.13.1+cu116 --extra-index-url https://download.pytorch.org/whl/cu116 +pip install --use-pep517 --upgrade -r requirements.txt +pip install -U -I --no-deps https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/linux/xformers-0.0.14.dev0-cp310-cp310-linux_x86_64.whl + +accelerate config + +rm *.bat +rm *.ps1 + +set +H +echo "#!/bin/bash" > gui.sh +echo "source venv/bin/activate" >> gui.sh +echo "python kohya_gui.py $@" >> gui.sh +chmod +x gui.sh +sources=$(find $(ls | grep -v venv) | egrep "\.py$") + +for source in $sources +do + #fix accelerate commands + sed -i -r 's/(accelerate launch .*?)"(.+?\.py)"/\1\2/g' $source + sed -i -r 's/subprocess.run\(run_cmd\)/subprocess.run(run_cmd, shell=True)/g' $source +done + +echo -e "setup finished! run \e[0;92m./gui.sh\e[0m to start" From bc45bd7e70e6e748dd86ef4a442d6c74dc667fb4 Mon Sep 17 00:00:00 2001 From: devdn Date: Sat, 4 Mar 2023 00:11:23 -0500 Subject: [PATCH 2/4] fix bugs that make some processes not work on linux --- dreambooth_gui.py | 2 +- finetune_gui.py | 11 ++++++----- gui.sh | 3 +++ library/basic_caption_gui.py | 4 ++-- library/blip_caption_gui.py | 6 +++--- library/convert_model_gui.py | 6 +++--- library/extract_lora_gui.py | 6 +++--- library/git_caption_gui.py | 4 ++-- library/merge_lora_gui.py | 6 +++--- library/resize_lora_gui.py | 6 +++--- library/tensorboard_gui.py | 4 ++-- library/verify_lora_gui.py | 12 +++++++----- library/wd14_caption_gui.py | 3 ++- lora_gui.py | 2 +- textual_inversion_gui.py | 2 +- ubuntu_setup.sh | 17 ----------------- 16 files changed, 42 insertions(+), 52 deletions(-) create mode 100755 gui.sh diff --git a/dreambooth_gui.py b/dreambooth_gui.py index 017e749ff..a2f0beb10 100644 --- a/dreambooth_gui.py +++ b/dreambooth_gui.py @@ -456,7 +456,7 @@ def train_model( print(run_cmd) # Run the command - subprocess.run(run_cmd) + os.system(run_cmd) # check if output_dir/last is a folder... therefore it is a diffuser model last_dir = pathlib.Path(f'{output_dir}/{output_name}') diff --git a/finetune_gui.py b/finetune_gui.py index 762db05e7..00eb086f2 100644 --- a/finetune_gui.py +++ b/finetune_gui.py @@ -33,6 +33,7 @@ save_style_symbol = '\U0001f4be' # 💾 document_symbol = '\U0001F4C4' # 📄 +PYTHON = "python3" if os.name == 'posix' else './venv/Scripts/python.exe' def save_configuration( save_as, @@ -296,7 +297,7 @@ def train_model( os.mkdir(train_dir) run_cmd = ( - f'./venv/Scripts/python.exe finetune/merge_captions_to_metadata.py' + f'{PYTHON} finetune/merge_captions_to_metadata.py' ) if caption_extension == '': run_cmd += f' --caption_extension=".caption"' @@ -310,12 +311,12 @@ def train_model( print(run_cmd) # Run the command - subprocess.run(run_cmd) + os.system(run_cmd) # create images buckets if generate_image_buckets: run_cmd = ( - f'./venv/Scripts/python.exe finetune/prepare_buckets_latents.py' + f'{PYTHON} finetune/prepare_buckets_latents.py' ) run_cmd += f' "{image_folder}"' run_cmd += f' "{train_dir}/{caption_metadata_filename}"' @@ -334,7 +335,7 @@ def train_model( print(run_cmd) # Run the command - subprocess.run(run_cmd) + os.system(run_cmd) image_num = len( [ @@ -444,7 +445,7 @@ def train_model( print(run_cmd) # Run the command - subprocess.run(run_cmd) + os.system(run_cmd) # check if output_dir/last is a folder... therefore it is a diffuser model last_dir = pathlib.Path(f'{output_dir}/{output_name}') diff --git a/gui.sh b/gui.sh new file mode 100755 index 000000000..90b26dbf5 --- /dev/null +++ b/gui.sh @@ -0,0 +1,3 @@ +#!/bin/bash +source venv/bin/activate +python kohya_gui.py diff --git a/library/basic_caption_gui.py b/library/basic_caption_gui.py index 2412dfbde..23c94fd82 100644 --- a/library/basic_caption_gui.py +++ b/library/basic_caption_gui.py @@ -2,7 +2,7 @@ from easygui import msgbox import subprocess from .common_gui import get_folder_path, add_pre_postfix, find_replace - +import os def caption_images( caption_text_input, @@ -38,7 +38,7 @@ def caption_images( print(run_cmd) # Run the command - subprocess.run(run_cmd) + os.system(run_cmd) if overwrite_input: if not prefix == '' or not postfix == '': diff --git a/library/blip_caption_gui.py b/library/blip_caption_gui.py index 61acd75ce..4f7525e66 100644 --- a/library/blip_caption_gui.py +++ b/library/blip_caption_gui.py @@ -3,7 +3,7 @@ import subprocess import os from .common_gui import get_folder_path, add_pre_postfix - +PYTHON = "python3" if os.name == 'posix' else './venv/Scripts/python.exe' def caption_images( train_data_dir, @@ -32,7 +32,7 @@ def caption_images( return print(f'Captioning files in {train_data_dir}...') - run_cmd = f'.\\venv\\Scripts\\python.exe "finetune/make_captions.py"' + run_cmd = f'{PYTHON} "finetune/make_captions.py"' run_cmd += f' --batch_size="{int(batch_size)}"' run_cmd += f' --num_beams="{int(num_beams)}"' run_cmd += f' --top_p="{top_p}"' @@ -48,7 +48,7 @@ def caption_images( print(run_cmd) # Run the command - subprocess.run(run_cmd) + os.system(run_cmd) # Add prefix and postfix add_pre_postfix( diff --git a/library/convert_model_gui.py b/library/convert_model_gui.py index fca75fc9f..b99011a20 100644 --- a/library/convert_model_gui.py +++ b/library/convert_model_gui.py @@ -9,7 +9,7 @@ refresh_symbol = '\U0001f504' # 🔄 save_style_symbol = '\U0001f4be' # 💾 document_symbol = '\U0001F4C4' # 📄 - +PYTHON = "python3" if os.name == 'posix' else './venv/Scripts/python.exe' def convert_model( source_model_input, @@ -40,7 +40,7 @@ def convert_model( msgbox('The provided target folder does not exist') return - run_cmd = f'.\\venv\Scripts\python.exe "tools/convert_diffusers20_original_sd.py"' + run_cmd = f'{PYTHON} "tools/convert_diffusers20_original_sd.py"' v1_models = [ 'runwayml/stable-diffusion-v1-5', @@ -87,7 +87,7 @@ def convert_model( print(run_cmd) # Run the command - subprocess.run(run_cmd) + os.system(run_cmd) if ( not target_model_type == 'diffuser' diff --git a/library/extract_lora_gui.py b/library/extract_lora_gui.py index 074e3ed12..d3864f1a1 100644 --- a/library/extract_lora_gui.py +++ b/library/extract_lora_gui.py @@ -12,7 +12,7 @@ refresh_symbol = '\U0001f504' # 🔄 save_style_symbol = '\U0001f4be' # 💾 document_symbol = '\U0001F4C4' # 📄 - +PYTHON = "python3" if os.name == 'posix' else './venv/Scripts/python.exe' def extract_lora( model_tuned, @@ -41,7 +41,7 @@ def extract_lora( return run_cmd = ( - f'.\\venv\Scripts\python.exe "networks\extract_lora_from_models.py"' + f'{PYTHON} "{os.path.join("networks","extract_lora_from_models.py")}"' ) run_cmd += f' --save_precision {save_precision}' run_cmd += f' --save_to "{save_to}"' @@ -54,7 +54,7 @@ def extract_lora( print(run_cmd) # Run the command - subprocess.run(run_cmd) + os.system(run_cmd) ### diff --git a/library/git_caption_gui.py b/library/git_caption_gui.py index c65fb3c58..965d43073 100644 --- a/library/git_caption_gui.py +++ b/library/git_caption_gui.py @@ -3,7 +3,7 @@ import subprocess import os from .common_gui import get_folder_path, add_pre_postfix - +PYTHON = "python3" if os.name == 'posix' else './venv/Scripts/python.exe' def caption_images( train_data_dir, @@ -25,7 +25,7 @@ def caption_images( return print(f'GIT captioning files in {train_data_dir}...') - run_cmd = f'.\\venv\\Scripts\\python.exe "finetune/make_captions.py"' + run_cmd = f'{PYTHON} "finetune/make_captions.py"' if not model_id == '': run_cmd += f' --model_id="{model_id}"' run_cmd += f' --batch_size="{int(batch_size)}"' diff --git a/library/merge_lora_gui.py b/library/merge_lora_gui.py index 03a1257a0..434d1a6c8 100644 --- a/library/merge_lora_gui.py +++ b/library/merge_lora_gui.py @@ -12,7 +12,7 @@ refresh_symbol = '\U0001f504' # 🔄 save_style_symbol = '\U0001f4be' # 💾 document_symbol = '\U0001F4C4' # 📄 - +PYTHON = "python3" if os.name == 'posix' else './venv/Scripts/python.exe' def merge_lora( lora_a_model, @@ -43,7 +43,7 @@ def merge_lora( ratio_a = ratio ratio_b = 1 - ratio - run_cmd = f'.\\venv\Scripts\python.exe "networks\merge_lora.py"' + run_cmd = f'{PYTHON} "{os.path.join("networks","merge_lora.py")}"' run_cmd += f' --save_precision {save_precision}' run_cmd += f' --precision {precision}' run_cmd += f' --save_to "{save_to}"' @@ -53,7 +53,7 @@ def merge_lora( print(run_cmd) # Run the command - subprocess.run(run_cmd) + os.system(run_cmd) ### diff --git a/library/resize_lora_gui.py b/library/resize_lora_gui.py index b9dec29af..c7fff1967 100644 --- a/library/resize_lora_gui.py +++ b/library/resize_lora_gui.py @@ -3,7 +3,7 @@ import subprocess import os from .common_gui import get_saveasfilename_path, get_file_path - +PYTHON = "python3" if os.name == 'posix' else './venv/Scripts/python.exe' folder_symbol = '\U0001f4c2' # 📂 refresh_symbol = '\U0001f504' # 🔄 save_style_symbol = '\U0001f4be' # 💾 @@ -30,7 +30,7 @@ def resize_lora( if device == '': device = 'cuda' - run_cmd = f'.\\venv\Scripts\python.exe "networks\\resize_lora.py"' + run_cmd = f'{PYTHON} "{os.path.join("networks","resize_lora.py")}"' run_cmd += f' --save_precision {save_precision}' run_cmd += f' --save_to {save_to}' run_cmd += f' --model {model}' @@ -40,7 +40,7 @@ def resize_lora( print(run_cmd) # Run the command - subprocess.run(run_cmd) + os.system(run_cmd) ### diff --git a/library/tensorboard_gui.py b/library/tensorboard_gui.py index 11827736c..8852e7261 100644 --- a/library/tensorboard_gui.py +++ b/library/tensorboard_gui.py @@ -5,7 +5,7 @@ import time tensorboard_proc = None # I know... bad but heh - +TENSORBOARD = "tensorboard" if os.name == 'posix' else 'tensorboard.exe' def start_tensorboard(logging_dir): global tensorboard_proc @@ -15,7 +15,7 @@ def start_tensorboard(logging_dir): msgbox(msg='Error: log folder is empty') return - run_cmd = f'tensorboard.exe --logdir "{logging_dir}"' + run_cmd = [f'{TENSORBOARD}', '--logdir', f'{logging_dir}'] print(run_cmd) if tensorboard_proc is not None: diff --git a/library/verify_lora_gui.py b/library/verify_lora_gui.py index 51fa51099..6ba9c336d 100644 --- a/library/verify_lora_gui.py +++ b/library/verify_lora_gui.py @@ -7,7 +7,7 @@ get_any_file_path, get_file_path, ) - +PYTHON = "python3" if os.name == 'posix' else './venv/Scripts/python.exe' folder_symbol = '\U0001f4c2' # 📂 refresh_symbol = '\U0001f504' # 🔄 save_style_symbol = '\U0001f4be' # 💾 @@ -27,13 +27,15 @@ def verify_lora( msgbox('The provided model A is not a file') return - run_cmd = f'.\\venv\Scripts\python.exe "networks\check_lora_weights.py"' - run_cmd += f' {lora_model}' + run_cmd = [ + PYTHON, + os.path.join("networks","check_lora_weights.py"), + f'{lora_model}' + ] - print(run_cmd) + print(" ".join(run_cmd)) # Run the command - subprocess.run(run_cmd) process = subprocess.Popen( run_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) diff --git a/library/wd14_caption_gui.py b/library/wd14_caption_gui.py index 3d97ebf68..afbbfef4c 100644 --- a/library/wd14_caption_gui.py +++ b/library/wd14_caption_gui.py @@ -2,6 +2,7 @@ from easygui import msgbox import subprocess from .common_gui import get_folder_path +import os def caption_images(train_data_dir, caption_extension, batch_size, thresh): @@ -30,7 +31,7 @@ def caption_images(train_data_dir, caption_extension, batch_size, thresh): print(run_cmd) # Run the command - subprocess.run(run_cmd) + os.system(run_cmd) print('...captioning done') diff --git a/lora_gui.py b/lora_gui.py index bf5c8e671..ec60ec7de 100644 --- a/lora_gui.py +++ b/lora_gui.py @@ -544,7 +544,7 @@ def train_model( print(run_cmd) # Run the command - subprocess.run(run_cmd) + os.system(run_cmd) # check if output_dir/last is a folder... therefore it is a diffuser model last_dir = pathlib.Path(f'{output_dir}/{output_name}') diff --git a/textual_inversion_gui.py b/textual_inversion_gui.py index 364d2a53d..1af572802 100644 --- a/textual_inversion_gui.py +++ b/textual_inversion_gui.py @@ -498,7 +498,7 @@ def train_model( print(run_cmd) # Run the command - subprocess.run(run_cmd) + os.system(run_cmd) # check if output_dir/last is a folder... therefore it is a diffuser model last_dir = pathlib.Path(f'{output_dir}/{output_name}') diff --git a/ubuntu_setup.sh b/ubuntu_setup.sh index 5bc862b64..143115574 100755 --- a/ubuntu_setup.sh +++ b/ubuntu_setup.sh @@ -9,21 +9,4 @@ pip install -U -I --no-deps https://github.com/C43H66N12O12S2/stable-diffusion-w accelerate config -rm *.bat -rm *.ps1 - -set +H -echo "#!/bin/bash" > gui.sh -echo "source venv/bin/activate" >> gui.sh -echo "python kohya_gui.py $@" >> gui.sh -chmod +x gui.sh -sources=$(find $(ls | grep -v venv) | egrep "\.py$") - -for source in $sources -do - #fix accelerate commands - sed -i -r 's/(accelerate launch .*?)"(.+?\.py)"/\1\2/g' $source - sed -i -r 's/subprocess.run\(run_cmd\)/subprocess.run(run_cmd, shell=True)/g' $source -done - echo -e "setup finished! run \e[0;92m./gui.sh\e[0m to start" From 7af9e48d304b2396d5d6d0da41a7973c183f8ad8 Mon Sep 17 00:00:00 2001 From: devdn Date: Sat, 4 Mar 2023 00:11:23 -0500 Subject: [PATCH 3/4] fix paths with spaces --- library/extract_lora_gui.py | 2 +- library/merge_lora_gui.py | 2 +- library/resize_lora_gui.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/library/extract_lora_gui.py b/library/extract_lora_gui.py index d3864f1a1..c5e8b7de9 100644 --- a/library/extract_lora_gui.py +++ b/library/extract_lora_gui.py @@ -41,7 +41,7 @@ def extract_lora( return run_cmd = ( - f'{PYTHON} "{os.path.join("networks","extract_lora_from_models.py")}"' + f'{PYTHON} {os.path.join("networks","extract_lora_from_models.py")}' ) run_cmd += f' --save_precision {save_precision}' run_cmd += f' --save_to "{save_to}"' diff --git a/library/merge_lora_gui.py b/library/merge_lora_gui.py index 434d1a6c8..33c366f03 100644 --- a/library/merge_lora_gui.py +++ b/library/merge_lora_gui.py @@ -43,7 +43,7 @@ def merge_lora( ratio_a = ratio ratio_b = 1 - ratio - run_cmd = f'{PYTHON} "{os.path.join("networks","merge_lora.py")}"' + run_cmd = f'{PYTHON} {os.path.join("networks","merge_lora.py")}' run_cmd += f' --save_precision {save_precision}' run_cmd += f' --precision {precision}' run_cmd += f' --save_to "{save_to}"' diff --git a/library/resize_lora_gui.py b/library/resize_lora_gui.py index c7fff1967..d092d4a55 100644 --- a/library/resize_lora_gui.py +++ b/library/resize_lora_gui.py @@ -30,7 +30,7 @@ def resize_lora( if device == '': device = 'cuda' - run_cmd = f'{PYTHON} "{os.path.join("networks","resize_lora.py")}"' + run_cmd = f'{PYTHON} {os.path.join("networks","resize_lora.py")}' run_cmd += f' --save_precision {save_precision}' run_cmd += f' --save_to {save_to}' run_cmd += f' --model {model}' From 129cd5f97d3ed7e8e638d55ae2eae314f00cf9ef Mon Sep 17 00:00:00 2001 From: devdn Date: Sat, 4 Mar 2023 17:13:02 -0500 Subject: [PATCH 4/4] Revert "fix paths with spaces" This reverts commit 7af9e48d304b2396d5d6d0da41a7973c183f8ad8. --- library/extract_lora_gui.py | 2 +- library/merge_lora_gui.py | 2 +- library/resize_lora_gui.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/library/extract_lora_gui.py b/library/extract_lora_gui.py index c5e8b7de9..d3864f1a1 100644 --- a/library/extract_lora_gui.py +++ b/library/extract_lora_gui.py @@ -41,7 +41,7 @@ def extract_lora( return run_cmd = ( - f'{PYTHON} {os.path.join("networks","extract_lora_from_models.py")}' + f'{PYTHON} "{os.path.join("networks","extract_lora_from_models.py")}"' ) run_cmd += f' --save_precision {save_precision}' run_cmd += f' --save_to "{save_to}"' diff --git a/library/merge_lora_gui.py b/library/merge_lora_gui.py index 33c366f03..434d1a6c8 100644 --- a/library/merge_lora_gui.py +++ b/library/merge_lora_gui.py @@ -43,7 +43,7 @@ def merge_lora( ratio_a = ratio ratio_b = 1 - ratio - run_cmd = f'{PYTHON} {os.path.join("networks","merge_lora.py")}' + run_cmd = f'{PYTHON} "{os.path.join("networks","merge_lora.py")}"' run_cmd += f' --save_precision {save_precision}' run_cmd += f' --precision {precision}' run_cmd += f' --save_to "{save_to}"' diff --git a/library/resize_lora_gui.py b/library/resize_lora_gui.py index d092d4a55..c7fff1967 100644 --- a/library/resize_lora_gui.py +++ b/library/resize_lora_gui.py @@ -30,7 +30,7 @@ def resize_lora( if device == '': device = 'cuda' - run_cmd = f'{PYTHON} {os.path.join("networks","resize_lora.py")}' + run_cmd = f'{PYTHON} "{os.path.join("networks","resize_lora.py")}"' run_cmd += f' --save_precision {save_precision}' run_cmd += f' --save_to {save_to}' run_cmd += f' --model {model}'