From 5badc7ac0ec8432ac8f7d289135f66c069710bb6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 05:25:24 +0000 Subject: [PATCH 01/13] Bump crate-ci/typos from 1.16.26 to 1.17.2 Bumps [crate-ci/typos](https://github.com/crate-ci/typos) from 1.16.26 to 1.17.2. - [Release notes](https://github.com/crate-ci/typos/releases) - [Changelog](https://github.com/crate-ci/typos/blob/master/CHANGELOG.md) - [Commits](https://github.com/crate-ci/typos/compare/v1.16.26...v1.17.2) --- updated-dependencies: - dependency-name: crate-ci/typos dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/typos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/typos.yaml b/.github/workflows/typos.yaml index bd4ef334e..c9edf2650 100644 --- a/.github/workflows/typos.yaml +++ b/.github/workflows/typos.yaml @@ -18,4 +18,4 @@ jobs: - uses: actions/checkout@v4 - name: typos-action - uses: crate-ci/typos@v1.16.26 + uses: crate-ci/typos@v1.17.2 From 22c7942a2e9c11aa5e6d08f23526e0c0191fdb86 Mon Sep 17 00:00:00 2001 From: bmaltais Date: Fri, 2 Feb 2024 21:17:33 -0500 Subject: [PATCH 02/13] Add support for multi-gpu parameters --- .release | 2 +- README.md | 3 + dreambooth_gui.py | 28 +- finetune_gui.py | 26 +- library/class_advanced_training.py | 22 + library/common_gui.py | 641 ++++++++++++++--------------- lora_gui.py | 28 +- textual_inversion_gui.py | 26 +- 8 files changed, 427 insertions(+), 349 deletions(-) diff --git a/.release b/.release index 535e1cca7..ef7f77232 100644 --- a/.release +++ b/.release @@ -1 +1 @@ -v22.6.0 +v22.6.1 diff --git a/README.md b/README.md index c95a930a7..a9666f99c 100644 --- a/README.md +++ b/README.md @@ -503,6 +503,9 @@ masterpiece, best quality, 1boy, in business suit, standing at street, looking b ## Change History +* 2024/01/27 (v22.6.1) +- Add support for multi-gpu parameters in the GUI under the "Parameters > Advanced" tab. + * 2024/01/27 (v22.6.0) - Merge sd-scripts v0.8.3 code update - Fixed a bug that the training crashes when `--fp8_base` is specified with `--save_state`. PR [#1079](https://github.com/kohya-ss/sd-scripts/pull/1079) Thanks to feffy380! diff --git a/dreambooth_gui.py b/dreambooth_gui.py index 6426e4a9a..4ef60a445 100644 --- a/dreambooth_gui.py +++ b/dreambooth_gui.py @@ -101,6 +101,10 @@ def save_configuration( flip_aug, clip_skip, vae, + num_processes, + num_machines, + multi_gpu, + gpu_ids, output_name, max_token_length, max_train_epochs, @@ -225,6 +229,10 @@ def open_configuration( flip_aug, clip_skip, vae, + num_processes, + num_machines, + multi_gpu, + gpu_ids, output_name, max_token_length, max_train_epochs, @@ -344,6 +352,10 @@ def train_model( flip_aug, clip_skip, vae, + num_processes, + num_machines, + multi_gpu, + gpu_ids, output_name, max_token_length, max_train_epochs, @@ -539,9 +551,15 @@ def train_model( log.info(f"lr_warmup_steps = {lr_warmup_steps}") # run_cmd = f'accelerate launch --num_cpu_threads_per_process={num_cpu_threads_per_process} "train_db.py"' - run_cmd = ( - f"accelerate launch --num_cpu_threads_per_process={num_cpu_threads_per_process}" - ) + run_cmd = "accelerate launch" + + run_cmd += run_cmd_advanced_training( + num_processes=num_processes, + num_machines=num_machines, + multi_gpu=multi_gpu, + gpu_ids=gpu_ids, + num_cpu_threads_per_process=num_cpu_threads_per_process) + if sdxl: run_cmd += f' "./sdxl_train.py"' else: @@ -827,6 +845,10 @@ def dreambooth_tab( advanced_training.flip_aug, advanced_training.clip_skip, advanced_training.vae, + advanced_training.num_processes, + advanced_training.num_machines, + advanced_training.multi_gpu, + advanced_training.gpu_ids, folders.output_name, advanced_training.max_token_length, basic_training.max_train_epochs, diff --git a/finetune_gui.py b/finetune_gui.py index 7a9e77cd3..797afea01 100644 --- a/finetune_gui.py +++ b/finetune_gui.py @@ -94,6 +94,10 @@ def save_configuration( # use_8bit_adam, xformers, clip_skip, + num_processes, + num_machines, + multi_gpu, + gpu_ids, save_state, resume, gradient_checkpointing, @@ -224,6 +228,10 @@ def open_configuration( # use_8bit_adam, xformers, clip_skip, + num_processes, + num_machines, + multi_gpu, + gpu_ids, save_state, resume, gradient_checkpointing, @@ -363,6 +371,10 @@ def train_model( # use_8bit_adam, xformers, clip_skip, + num_processes, + num_machines, + multi_gpu, + gpu_ids, save_state, resume, gradient_checkpointing, @@ -533,7 +545,15 @@ def train_model( lr_warmup_steps = round(float(int(lr_warmup) * int(max_train_steps) / 100)) log.info(f'lr_warmup_steps = {lr_warmup_steps}') - run_cmd = f'accelerate launch --num_cpu_threads_per_process={num_cpu_threads_per_process}' + run_cmd = "accelerate launch" + + run_cmd += run_cmd_advanced_training( + num_processes=num_processes, + num_machines=num_machines, + multi_gpu=multi_gpu, + gpu_ids=gpu_ids, + num_cpu_threads_per_process=num_cpu_threads_per_process) + if sdxl_checkbox: run_cmd += f' "./sdxl_train.py"' else: @@ -967,6 +987,10 @@ def list_presets(path): basic_training.caption_extension, advanced_training.xformers, advanced_training.clip_skip, + advanced_training.num_processes, + advanced_training.num_machines, + advanced_training.multi_gpu, + advanced_training.gpu_ids, advanced_training.save_state, advanced_training.resume, advanced_training.gradient_checkpointing, diff --git a/library/class_advanced_training.py b/library/class_advanced_training.py index df432209a..c46b40cb0 100644 --- a/library/class_advanced_training.py +++ b/library/class_advanced_training.py @@ -298,6 +298,28 @@ def full_options_update(full_fp16, full_bf16): placeholder='(Optional) Override number of epoch. Default: 8', value='0', ) + with gr.Row(): + self.num_processes = gr.Number( + label='Number of processes', + value=1, + precision=0, + minimum=1 + ) + self.num_machines = gr.Number( + label='Number of machines', + value=1, + precision=0, + minimum=1 + ) + self.multi_gpu = gr.Checkbox( + label='Multi GPU', + value=False + ) + self.gpu_ids = gr.Textbox( + label='GPU IDs', + value="", + placeholder="example: 0,1" + ) with gr.Row(): self.wandb_api_key = gr.Textbox( label='WANDB API Key', diff --git a/library/common_gui.py b/library/common_gui.py index f39cf102c..450119f8a 100644 --- a/library/common_gui.py +++ b/library/common_gui.py @@ -14,71 +14,63 @@ # Set up logging log = setup_logging() -folder_symbol = '\U0001f4c2' # 📂 -refresh_symbol = '\U0001f504' # 🔄 -save_style_symbol = '\U0001f4be' # 💾 -document_symbol = '\U0001F4C4' # 📄 +folder_symbol = "\U0001f4c2" # 📂 +refresh_symbol = "\U0001f504" # 🔄 +save_style_symbol = "\U0001f4be" # 💾 +document_symbol = "\U0001F4C4" # 📄 # define a list of substrings to search for v2 base models V2_BASE_MODELS = [ - 'stabilityai/stable-diffusion-2-1-base/blob/main/v2-1_512-ema-pruned', - 'stabilityai/stable-diffusion-2-1-base', - 'stabilityai/stable-diffusion-2-base', + "stabilityai/stable-diffusion-2-1-base/blob/main/v2-1_512-ema-pruned", + "stabilityai/stable-diffusion-2-1-base", + "stabilityai/stable-diffusion-2-base", ] # define a list of substrings to search for v_parameterization models V_PARAMETERIZATION_MODELS = [ - 'stabilityai/stable-diffusion-2-1/blob/main/v2-1_768-ema-pruned', - 'stabilityai/stable-diffusion-2-1', - 'stabilityai/stable-diffusion-2', + "stabilityai/stable-diffusion-2-1/blob/main/v2-1_768-ema-pruned", + "stabilityai/stable-diffusion-2-1", + "stabilityai/stable-diffusion-2", ] # define a list of substrings to v1.x models V1_MODELS = [ - 'CompVis/stable-diffusion-v1-4', - 'runwayml/stable-diffusion-v1-5', + "CompVis/stable-diffusion-v1-4", + "runwayml/stable-diffusion-v1-5", ] # define a list of substrings to search for SDXL base models SDXL_MODELS = [ - 'stabilityai/stable-diffusion-xl-base-1.0', - 'stabilityai/stable-diffusion-xl-refiner-1.0', + "stabilityai/stable-diffusion-xl-base-1.0", + "stabilityai/stable-diffusion-xl-refiner-1.0", ] # define a list of substrings to search for -ALL_PRESET_MODELS = ( - V2_BASE_MODELS + V_PARAMETERIZATION_MODELS + V1_MODELS + SDXL_MODELS -) +ALL_PRESET_MODELS = V2_BASE_MODELS + V_PARAMETERIZATION_MODELS + V1_MODELS + SDXL_MODELS -ENV_EXCLUSION = ['COLAB_GPU', 'RUNPOD_POD_ID'] +ENV_EXCLUSION = ["COLAB_GPU", "RUNPOD_POD_ID"] -def check_if_model_exist( - output_name, output_dir, save_model_as, headless=False -): +def check_if_model_exist(output_name, output_dir, save_model_as, headless=False): if headless: log.info( - 'Headless mode, skipping verification if model already exist... if model already exist it will be overwritten...' + "Headless mode, skipping verification if model already exist... if model already exist it will be overwritten..." ) return False - if save_model_as in ['diffusers', 'diffusers_safetendors']: + if save_model_as in ["diffusers", "diffusers_safetendors"]: ckpt_folder = os.path.join(output_dir, output_name) if os.path.isdir(ckpt_folder): - msg = f'A diffuser model with the same name {ckpt_folder} already exists. Do you want to overwrite it?' - if not easygui.ynbox(msg, 'Overwrite Existing Model?'): - log.info( - 'Aborting training due to existing model with same name...' - ) + msg = f"A diffuser model with the same name {ckpt_folder} already exists. Do you want to overwrite it?" + if not easygui.ynbox(msg, "Overwrite Existing Model?"): + log.info("Aborting training due to existing model with same name...") return True - elif save_model_as in ['ckpt', 'safetensors']: - ckpt_file = os.path.join(output_dir, output_name + '.' + save_model_as) + elif save_model_as in ["ckpt", "safetensors"]: + ckpt_file = os.path.join(output_dir, output_name + "." + save_model_as) if os.path.isfile(ckpt_file): - msg = f'A model with the same file name {ckpt_file} already exists. Do you want to overwrite it?' - if not easygui.ynbox(msg, 'Overwrite Existing Model?'): - log.info( - 'Aborting training due to existing model with same name...' - ) + msg = f"A model with the same file name {ckpt_file} already exists. Do you want to overwrite it?" + if not easygui.ynbox(msg, "Overwrite Existing Model?"): + log.info("Aborting training due to existing model with same name...") return True else: log.info( @@ -89,7 +81,7 @@ def check_if_model_exist( return False -def output_message(msg='', title='', headless=False): +def output_message(msg="", title="", headless=False): if headless: log.info(msg) else: @@ -98,22 +90,17 @@ def output_message(msg='', title='', headless=False): def update_my_data(my_data): # Update the optimizer based on the use_8bit_adam flag - use_8bit_adam = my_data.get('use_8bit_adam', False) - my_data.setdefault('optimizer', 'AdamW8bit' if use_8bit_adam else 'AdamW') + use_8bit_adam = my_data.get("use_8bit_adam", False) + my_data.setdefault("optimizer", "AdamW8bit" if use_8bit_adam else "AdamW") # Update model_list to custom if empty or pretrained_model_name_or_path is not a preset model - model_list = my_data.get('model_list', []) - pretrained_model_name_or_path = my_data.get( - 'pretrained_model_name_or_path', '' - ) - if ( - not model_list - or pretrained_model_name_or_path not in ALL_PRESET_MODELS - ): - my_data['model_list'] = 'custom' + model_list = my_data.get("model_list", []) + pretrained_model_name_or_path = my_data.get("pretrained_model_name_or_path", "") + if not model_list or pretrained_model_name_or_path not in ALL_PRESET_MODELS: + my_data["model_list"] = "custom" # Convert values to int if they are strings - for key in ['epoch', 'save_every_n_epochs', 'lr_warmup']: + for key in ["epoch", "save_every_n_epochs", "lr_warmup"]: value = my_data.get(key, 0) if isinstance(value, str) and value.strip().isdigit(): my_data[key] = int(value) @@ -121,7 +108,7 @@ def update_my_data(my_data): my_data[key] = 0 # Convert values to float if they are strings - for key in ['noise_offset', 'learning_rate', 'text_encoder_lr', 'unet_lr']: + for key in ["noise_offset", "learning_rate", "text_encoder_lr", "unet_lr"]: value = my_data.get(key, 0) if isinstance(value, str) and value.strip().isdigit(): my_data[key] = float(value) @@ -129,28 +116,28 @@ def update_my_data(my_data): my_data[key] = 0 # Update LoRA_type if it is set to LoCon - if my_data.get('LoRA_type', 'Standard') == 'LoCon': - my_data['LoRA_type'] = 'LyCORIS/LoCon' + if my_data.get("LoRA_type", "Standard") == "LoCon": + my_data["LoRA_type"] = "LyCORIS/LoCon" # Update model save choices due to changes for LoRA and TI training - if 'save_model_as' in my_data: + if "save_model_as" in my_data: if ( - my_data.get('LoRA_type') or my_data.get('num_vectors_per_token') - ) and my_data.get('save_model_as') not in ['safetensors', 'ckpt']: - message = 'Updating save_model_as to safetensors because the current value in the config file is no longer applicable to {}' - if my_data.get('LoRA_type'): - log.info(message.format('LoRA')) - if my_data.get('num_vectors_per_token'): - log.info(message.format('TI')) - my_data['save_model_as'] = 'safetensors' + my_data.get("LoRA_type") or my_data.get("num_vectors_per_token") + ) and my_data.get("save_model_as") not in ["safetensors", "ckpt"]: + message = "Updating save_model_as to safetensors because the current value in the config file is no longer applicable to {}" + if my_data.get("LoRA_type"): + log.info(message.format("LoRA")) + if my_data.get("num_vectors_per_token"): + log.info(message.format("TI")) + my_data["save_model_as"] = "safetensors" # Update xformers if it is set to True and is a boolean - xformers_value = my_data.get('xformers', None) + xformers_value = my_data.get("xformers", None) if isinstance(xformers_value, bool): if xformers_value: - my_data['xformers'] = 'xformers' + my_data["xformers"] = "xformers" else: - my_data['xformers'] = 'none' + my_data["xformers"] = "none" return my_data @@ -161,12 +148,9 @@ def get_dir_and_file(file_path): def get_file_path( - file_path='', default_extension='.json', extension_name='Config files' + file_path="", default_extension=".json", extension_name="Config files" ): - if ( - not any(var in os.environ for var in ENV_EXCLUSION) - and sys.platform != 'darwin' - ): + if not any(var in os.environ for var in ENV_EXCLUSION) and sys.platform != "darwin": current_file_path = file_path # log.info(f'current file path: {current_file_path}') @@ -174,14 +158,14 @@ def get_file_path( # Create a hidden Tkinter root window root = Tk() - root.wm_attributes('-topmost', 1) + root.wm_attributes("-topmost", 1) root.withdraw() # Show the open file dialog and get the selected file path file_path = filedialog.askopenfilename( filetypes=( - (extension_name, f'*{default_extension}'), - ('All files', '*.*'), + (extension_name, f"*{default_extension}"), + ("All files", "*.*"), ), defaultextension=default_extension, initialfile=initial_file, @@ -200,18 +184,15 @@ def get_file_path( return file_path -def get_any_file_path(file_path=''): - if ( - not any(var in os.environ for var in ENV_EXCLUSION) - and sys.platform != 'darwin' - ): +def get_any_file_path(file_path=""): + if not any(var in os.environ for var in ENV_EXCLUSION) and sys.platform != "darwin": current_file_path = file_path # log.info(f'current file path: {current_file_path}') initial_dir, initial_file = get_dir_and_file(file_path) root = Tk() - root.wm_attributes('-topmost', 1) + root.wm_attributes("-topmost", 1) root.withdraw() file_path = filedialog.askopenfilename( initialdir=initial_dir, @@ -219,7 +200,7 @@ def get_any_file_path(file_path=''): ) root.destroy() - if file_path == '': + if file_path == "": file_path = current_file_path return file_path @@ -227,51 +208,45 @@ def get_any_file_path(file_path=''): def remove_doublequote(file_path): if file_path != None: - file_path = file_path.replace('"', '') + file_path = file_path.replace('"', "") return file_path -def get_folder_path(folder_path=''): - if ( - not any(var in os.environ for var in ENV_EXCLUSION) - and sys.platform != 'darwin' - ): +def get_folder_path(folder_path=""): + if not any(var in os.environ for var in ENV_EXCLUSION) and sys.platform != "darwin": current_folder_path = folder_path initial_dir, initial_file = get_dir_and_file(folder_path) root = Tk() - root.wm_attributes('-topmost', 1) + root.wm_attributes("-topmost", 1) root.withdraw() folder_path = filedialog.askdirectory(initialdir=initial_dir) root.destroy() - if folder_path == '': + if folder_path == "": folder_path = current_folder_path return folder_path def get_saveasfile_path( - file_path='', defaultextension='.json', extension_name='Config files' + file_path="", defaultextension=".json", extension_name="Config files" ): - if ( - not any(var in os.environ for var in ENV_EXCLUSION) - and sys.platform != 'darwin' - ): + if not any(var in os.environ for var in ENV_EXCLUSION) and sys.platform != "darwin": current_file_path = file_path # log.info(f'current file path: {current_file_path}') initial_dir, initial_file = get_dir_and_file(file_path) root = Tk() - root.wm_attributes('-topmost', 1) + root.wm_attributes("-topmost", 1) root.withdraw() save_file_path = filedialog.asksaveasfile( filetypes=( - (f'{extension_name}', f'{defaultextension}'), - ('All files', '*'), + (f"{extension_name}", f"{defaultextension}"), + ("All files", "*"), ), defaultextension=defaultextension, initialdir=initial_dir, @@ -293,24 +268,21 @@ def get_saveasfile_path( def get_saveasfilename_path( - file_path='', extensions='*', extension_name='Config files' + file_path="", extensions="*", extension_name="Config files" ): - if ( - not any(var in os.environ for var in ENV_EXCLUSION) - and sys.platform != 'darwin' - ): + if not any(var in os.environ for var in ENV_EXCLUSION) and sys.platform != "darwin": current_file_path = file_path # log.info(f'current file path: {current_file_path}') initial_dir, initial_file = get_dir_and_file(file_path) root = Tk() - root.wm_attributes('-topmost', 1) + root.wm_attributes("-topmost", 1) root.withdraw() save_file_path = filedialog.asksaveasfilename( filetypes=( - (f'{extension_name}', f'{extensions}'), - ('All files', '*'), + (f"{extension_name}", f"{extensions}"), + ("All files", "*"), ), defaultextension=extensions, initialdir=initial_dir, @@ -318,7 +290,7 @@ def get_saveasfilename_path( ) root.destroy() - if save_file_path == '': + if save_file_path == "": file_path = current_file_path else: # log.info(save_file_path) @@ -328,10 +300,10 @@ def get_saveasfilename_path( def add_pre_postfix( - folder: str = '', - prefix: str = '', - postfix: str = '', - caption_file_ext: str = '.caption', + folder: str = "", + prefix: str = "", + postfix: str = "", + caption_file_ext: str = ".caption", ) -> None: """ Add prefix and/or postfix to the content of caption files within a folder. @@ -344,10 +316,10 @@ def add_pre_postfix( caption_file_ext (str, optional): Extension of the caption files. """ - if prefix == '' and postfix == '': + if prefix == "" and postfix == "": return - image_extensions = ('.jpg', '.jpeg', '.png', '.webp') + image_extensions = (".jpg", ".jpeg", ".png", ".webp") image_files = [ f for f in os.listdir(folder) if f.lower().endswith(image_extensions) ] @@ -357,19 +329,19 @@ def add_pre_postfix( caption_file_path = os.path.join(folder, caption_file_name) if not os.path.exists(caption_file_path): - with open(caption_file_path, 'w', encoding='utf8') as f: - separator = ' ' if prefix and postfix else '' - f.write(f'{prefix}{separator}{postfix}') + with open(caption_file_path, "w", encoding="utf8") as f: + separator = " " if prefix and postfix else "" + f.write(f"{prefix}{separator}{postfix}") else: - with open(caption_file_path, 'r+', encoding='utf8') as f: + with open(caption_file_path, "r+", encoding="utf8") as f: content = f.read() content = content.rstrip() f.seek(0, 0) - prefix_separator = ' ' if prefix else '' - postfix_separator = ' ' if postfix else '' + prefix_separator = " " if prefix else "" + postfix_separator = " " if postfix else "" f.write( - f'{prefix}{prefix_separator}{content}{postfix_separator}{postfix}' + f"{prefix}{prefix_separator}{content}{postfix_separator}{postfix}" ) @@ -391,10 +363,10 @@ def has_ext_files(folder_path: str, file_extension: str) -> bool: def find_replace( - folder_path: str = '', - caption_file_ext: str = '.caption', - search_text: str = '', - replace_text: str = '', + folder_path: str = "", + caption_file_ext: str = ".caption", + search_text: str = "", + replace_text: str = "", ) -> None: """ Find and replace text in caption files within a folder. @@ -405,30 +377,26 @@ def find_replace( search_text (str, optional): Text to search for in the caption files. replace_text (str, optional): Text to replace the search text with. """ - log.info('Running caption find/replace') + log.info("Running caption find/replace") if not has_ext_files(folder_path, caption_file_ext): msgbox( - f'No files with extension {caption_file_ext} were found in {folder_path}...' + f"No files with extension {caption_file_ext} were found in {folder_path}..." ) return - if search_text == '': + if search_text == "": return - caption_files = [ - f for f in os.listdir(folder_path) if f.endswith(caption_file_ext) - ] + caption_files = [f for f in os.listdir(folder_path) if f.endswith(caption_file_ext)] for caption_file in caption_files: - with open( - os.path.join(folder_path, caption_file), 'r', errors='ignore' - ) as f: + with open(os.path.join(folder_path, caption_file), "r", errors="ignore") as f: content = f.read() content = content.replace(search_text, replace_text) - with open(os.path.join(folder_path, caption_file), 'w') as f: + with open(os.path.join(folder_path, caption_file), "w") as f: f.write(content) @@ -458,19 +426,19 @@ def save_inference_file(output_dir, v2, v_parameterization, output_name): # Copy the v2-inference-v.yaml file to the current file, with a .yaml extension if v2 and v_parameterization: log.info( - f'Saving v2-inference-v.yaml as {output_dir}/{file_name}.yaml' + f"Saving v2-inference-v.yaml as {output_dir}/{file_name}.yaml" ) shutil.copy( - f'./v2_inference/v2-inference-v.yaml', - f'{output_dir}/{file_name}.yaml', + f"./v2_inference/v2-inference-v.yaml", + f"{output_dir}/{file_name}.yaml", ) elif v2: log.info( - f'Saving v2-inference.yaml as {output_dir}/{file_name}.yaml' + f"Saving v2-inference.yaml as {output_dir}/{file_name}.yaml" ) shutil.copy( - f'./v2_inference/v2-inference.yaml', - f'{output_dir}/{file_name}.yaml', + f"./v2_inference/v2-inference.yaml", + f"{output_dir}/{file_name}.yaml", ) @@ -485,7 +453,7 @@ def set_pretrained_model_name_or_path_input( ): # Check if the given model_list is in the list of SDXL models if str(model_list) in SDXL_MODELS: - log.info('SDXL model selected. Setting sdxl parameters') + log.info("SDXL model selected. Setting sdxl parameters") v2 = gr.Checkbox.update(value=False, visible=False) v_parameterization = gr.Checkbox.update(value=False, visible=False) sdxl = gr.Checkbox.update(value=True, visible=False) @@ -506,7 +474,7 @@ def set_pretrained_model_name_or_path_input( # Check if the given model_list is in the list of V2 base models if str(model_list) in V2_BASE_MODELS: - log.info('SD v2 base model selected. Setting --v2 parameter') + log.info("SD v2 base model selected. Setting --v2 parameter") v2 = gr.Checkbox.update(value=True, visible=False) v_parameterization = gr.Checkbox.update(value=False, visible=False) sdxl = gr.Checkbox.update(value=False, visible=False) @@ -528,7 +496,7 @@ def set_pretrained_model_name_or_path_input( # Check if the given model_list is in the list of V parameterization models if str(model_list) in V_PARAMETERIZATION_MODELS: log.info( - 'SD v2 model selected. Setting --v2 and --v_parameterization parameters' + "SD v2 model selected. Setting --v2 and --v_parameterization parameters" ) v2 = gr.Checkbox.update(value=True, visible=False) v_parameterization = gr.Checkbox.update(value=True, visible=False) @@ -550,7 +518,7 @@ def set_pretrained_model_name_or_path_input( # Check if the given model_list is in the list of V1 models if str(model_list) in V1_MODELS: - log.info(f'{model_list} model selected.') + log.info(f"{model_list} model selected.") v2 = gr.Checkbox.update(value=False, visible=False) v_parameterization = gr.Checkbox.update(value=False, visible=False) sdxl = gr.Checkbox.update(value=False, visible=False) @@ -570,7 +538,7 @@ def set_pretrained_model_name_or_path_input( ) # Check if the model_list is set to 'custom' - if model_list == 'custom': + if model_list == "custom": v2 = gr.Checkbox.update(visible=True) v_parameterization = gr.Checkbox.update(visible=True) sdxl = gr.Checkbox.update(visible=True) @@ -593,12 +561,8 @@ def set_pretrained_model_name_or_path_input( ### -def get_pretrained_model_name_or_path_file( - model_list, pretrained_model_name_or_path -): - pretrained_model_name_or_path = get_any_file_path( - pretrained_model_name_or_path - ) +def get_pretrained_model_name_or_path_file(model_list, pretrained_model_name_or_path): + pretrained_model_name_or_path = get_any_file_path(pretrained_model_name_or_path) # set_model_list(model_list, pretrained_model_name_or_path) @@ -612,7 +576,7 @@ def get_int_or_default(kwargs, key, default_value=0): return int(value) else: log.info( - f'{key} is not an int, float or a string, setting value to {default_value}' + f"{key} is not an int, float or a string, setting value to {default_value}" ) return default_value @@ -627,12 +591,12 @@ def get_float_or_default(kwargs, key, default_value=0.0): return float(value) else: log.info( - f'{key} is not an int, float or a string, setting value to {default_value}' + f"{key} is not an int, float or a string, setting value to {default_value}" ) return default_value -def get_str_or_default(kwargs, key, default_value=''): +def get_str_or_default(kwargs, key, default_value=""): value = kwargs.get(key, default_value) if isinstance(value, str): return value @@ -645,263 +609,264 @@ def get_str_or_default(kwargs, key, default_value=''): def run_cmd_training(**kwargs): - run_cmd = '' + run_cmd = "" - learning_rate = kwargs.get('learning_rate', '') + learning_rate = kwargs.get("learning_rate", "") if learning_rate: run_cmd += f' --learning_rate="{learning_rate}"' - lr_scheduler = kwargs.get('lr_scheduler', '') + lr_scheduler = kwargs.get("lr_scheduler", "") if lr_scheduler: run_cmd += f' --lr_scheduler="{lr_scheduler}"' - lr_warmup_steps = kwargs.get('lr_warmup_steps', '') + lr_warmup_steps = kwargs.get("lr_warmup_steps", "") if lr_warmup_steps: - if lr_scheduler == 'constant': - log.info( - "Can't use LR warmup with LR Scheduler constant... ignoring..." - ) + if lr_scheduler == "constant": + log.info("Can't use LR warmup with LR Scheduler constant... ignoring...") else: run_cmd += f' --lr_warmup_steps="{lr_warmup_steps}"' - train_batch_size = kwargs.get('train_batch_size', '') + train_batch_size = kwargs.get("train_batch_size", "") if train_batch_size: run_cmd += f' --train_batch_size="{train_batch_size}"' - max_train_steps = kwargs.get('max_train_steps', '') + max_train_steps = kwargs.get("max_train_steps", "") if max_train_steps: run_cmd += f' --max_train_steps="{max_train_steps}"' - save_every_n_epochs = kwargs.get('save_every_n_epochs') + save_every_n_epochs = kwargs.get("save_every_n_epochs") if save_every_n_epochs: run_cmd += f' --save_every_n_epochs="{int(save_every_n_epochs)}"' - mixed_precision = kwargs.get('mixed_precision', '') + mixed_precision = kwargs.get("mixed_precision", "") if mixed_precision: run_cmd += f' --mixed_precision="{mixed_precision}"' - save_precision = kwargs.get('save_precision', '') + save_precision = kwargs.get("save_precision", "") if save_precision: run_cmd += f' --save_precision="{save_precision}"' - seed = kwargs.get('seed', '') - if seed != '': + seed = kwargs.get("seed", "") + if seed != "": run_cmd += f' --seed="{seed}"' - caption_extension = kwargs.get('caption_extension', '') + caption_extension = kwargs.get("caption_extension", "") if caption_extension: run_cmd += f' --caption_extension="{caption_extension}"' - cache_latents = kwargs.get('cache_latents') + cache_latents = kwargs.get("cache_latents") if cache_latents: - run_cmd += ' --cache_latents' + run_cmd += " --cache_latents" - cache_latents_to_disk = kwargs.get('cache_latents_to_disk') + cache_latents_to_disk = kwargs.get("cache_latents_to_disk") if cache_latents_to_disk: - run_cmd += ' --cache_latents_to_disk' + run_cmd += " --cache_latents_to_disk" - optimizer_type = kwargs.get('optimizer', 'AdamW') + optimizer_type = kwargs.get("optimizer", "AdamW") run_cmd += f' --optimizer_type="{optimizer_type}"' - optimizer_args = kwargs.get('optimizer_args', '') - if optimizer_args != '': - run_cmd += f' --optimizer_args {optimizer_args}' + optimizer_args = kwargs.get("optimizer_args", "") + if optimizer_args != "": + run_cmd += f" --optimizer_args {optimizer_args}" - lr_scheduler_args = kwargs.get('lr_scheduler_args', '') - if lr_scheduler_args != '': - run_cmd += f' --lr_scheduler_args {lr_scheduler_args}' - - max_grad_norm = kwargs.get('max_grad_norm', '') - if max_grad_norm != '': + lr_scheduler_args = kwargs.get("lr_scheduler_args", "") + if lr_scheduler_args != "": + run_cmd += f" --lr_scheduler_args {lr_scheduler_args}" + + max_grad_norm = kwargs.get("max_grad_norm", "") + if max_grad_norm != "": run_cmd += f' --max_grad_norm="{max_grad_norm}"' - + return run_cmd def run_cmd_advanced_training(**kwargs): - run_cmd = '' + run_cmd = "" - max_train_epochs = kwargs.get('max_train_epochs', '') - if max_train_epochs: - run_cmd += f' --max_train_epochs={max_train_epochs}' + additional_parameters = kwargs.get("additional_parameters") + if additional_parameters: + run_cmd += f" {additional_parameters}" - max_data_loader_n_workers = kwargs.get('max_data_loader_n_workers', '') - if max_data_loader_n_workers: + bucket_no_upscale = kwargs.get("bucket_no_upscale") + if bucket_no_upscale: + run_cmd += " --bucket_no_upscale" + + bucket_reso_steps = kwargs.get("bucket_reso_steps") + if bucket_reso_steps: + run_cmd += f" --bucket_reso_steps={int(bucket_reso_steps)}" + + caption_dropout_every_n_epochs = kwargs.get("caption_dropout_every_n_epochs") + if caption_dropout_every_n_epochs and int(caption_dropout_every_n_epochs) > 0: run_cmd += ( - f' --max_data_loader_n_workers="{max_data_loader_n_workers}"' + f' --caption_dropout_every_n_epochs="{int(caption_dropout_every_n_epochs)}"' ) - max_token_length = int(kwargs.get('max_token_length', 75)) - if max_token_length > 75: - run_cmd += f' --max_token_length={max_token_length}' - - clip_skip = int(kwargs.get('clip_skip', 1)) - if clip_skip > 1: - run_cmd += f' --clip_skip={clip_skip}' + caption_dropout_rate = kwargs.get("caption_dropout_rate") + if caption_dropout_rate and float(caption_dropout_rate) > 0: + run_cmd += f' --caption_dropout_rate="{caption_dropout_rate}"' - resume = kwargs.get('resume', '') - if resume: - run_cmd += f' --resume="{resume}"' + clip_skip = kwargs.get("clip_skip") + if clip_skip and int(clip_skip) > 1: + run_cmd += f" --clip_skip={int(clip_skip)}" - keep_tokens = int(kwargs.get('keep_tokens', 0)) - if keep_tokens > 0: - run_cmd += f' --keep_tokens="{keep_tokens}"' + color_aug = kwargs.get("color_aug") + if color_aug: + run_cmd += " --color_aug" - caption_dropout_every_n_epochs = int( - kwargs.get('caption_dropout_every_n_epochs', 0) - ) - if caption_dropout_every_n_epochs > 0: - run_cmd += f' --caption_dropout_every_n_epochs="{caption_dropout_every_n_epochs}"' + flip_aug = kwargs.get("flip_aug") + if flip_aug: + run_cmd += " --flip_aug" - caption_dropout_rate = float(kwargs.get('caption_dropout_rate', 0)) - if caption_dropout_rate > 0: - run_cmd += f' --caption_dropout_rate="{caption_dropout_rate}"' + fp8_base = kwargs.get("fp8_base") + if fp8_base: + run_cmd += " --fp8_base" - vae_batch_size = int(kwargs.get('vae_batch_size', 0)) - if vae_batch_size > 0: - run_cmd += f' --vae_batch_size="{vae_batch_size}"' + full_fp16 = kwargs.get("full_fp16") + if full_fp16: + run_cmd += " --full_fp16" - bucket_reso_steps = int(kwargs.get('bucket_reso_steps', 64)) - run_cmd += f' --bucket_reso_steps={bucket_reso_steps}' + gradient_checkpointing = kwargs.get("gradient_checkpointing") + if gradient_checkpointing: + run_cmd += " --gradient_checkpointing" - v_pred_like_loss = float(kwargs.get('v_pred_like_loss', 0)) - if v_pred_like_loss > 0: - run_cmd += f' --v_pred_like_loss="{v_pred_like_loss}"' + keep_tokens = kwargs.get("keep_tokens") + if keep_tokens and int(keep_tokens) > 0: + run_cmd += f' --keep_tokens="{int(keep_tokens)}"' - save_every_n_steps = int(kwargs.get('save_every_n_steps', 0)) - if save_every_n_steps > 0: - run_cmd += f' --save_every_n_steps="{save_every_n_steps}"' - save_last_n_steps = int(kwargs.get('save_last_n_steps', 0)) - if save_last_n_steps > 0: - run_cmd += f' --save_last_n_steps="{save_last_n_steps}"' + gpu_ids = kwargs.get("gpu_ids") + if gpu_ids: + run_cmd += f' --gpu_ids="{gpu_ids}"' - save_last_n_steps_state = int(kwargs.get('save_last_n_steps_state', 0)) - if save_last_n_steps_state > 0: - run_cmd += f' --save_last_n_steps_state="{save_last_n_steps_state}"' + max_data_loader_n_workers = kwargs.get("max_data_loader_n_workers") + if max_data_loader_n_workers: + run_cmd += f' --max_data_loader_n_workers="{max_data_loader_n_workers}"' - min_snr_gamma = int(kwargs.get('min_snr_gamma', 0)) - if min_snr_gamma >= 1: - run_cmd += f' --min_snr_gamma={min_snr_gamma}' + max_token_length = kwargs.get("max_token_length") + if max_token_length and int(max_token_length) > 75: + run_cmd += f" --max_token_length={int(max_token_length)}" - min_timestep = int(kwargs.get('min_timestep', 0)) - if min_timestep > 0: - run_cmd += f' --min_timestep={min_timestep}' + max_train_epochs = kwargs.get("max_train_epochs") + if max_train_epochs: + run_cmd += f" --max_train_epochs={max_train_epochs}" - max_timestep = int(kwargs.get('max_timestep', 1000)) - if max_timestep < 1000: - run_cmd += f' --max_timestep={max_timestep}' + min_snr_gamma = kwargs.get("min_snr_gamma") + if min_snr_gamma and int(min_snr_gamma) >= 1: + run_cmd += f" --min_snr_gamma={int(min_snr_gamma)}" - save_state = kwargs.get('save_state') - if save_state: - run_cmd += ' --save_state' + max_timestep = kwargs.get("max_timestep") + if max_timestep and int(max_timestep) < 1000: + run_cmd += f" --max_timestep={int(max_timestep)}" - mem_eff_attn = kwargs.get('mem_eff_attn') + mem_eff_attn = kwargs.get("mem_eff_attn") if mem_eff_attn: - run_cmd += ' --mem_eff_attn' - - color_aug = kwargs.get('color_aug') - if color_aug: - run_cmd += ' --color_aug' + run_cmd += " --mem_eff_attn" + + min_timestep = kwargs.get("min_timestep") + if min_timestep and int(min_timestep) > 0: + run_cmd += f" --min_timestep={int(min_timestep)}" + + multi_gpu = kwargs.get("multi_gpu") + if multi_gpu: + run_cmd += " --multi_gpu" + + noise_offset_type = kwargs.get("noise_offset_type") + if noise_offset_type and noise_offset_type == "Original": + noise_offset = kwargs.get("noise_offset") + if noise_offset and float(noise_offset) > 0: + run_cmd += f" --noise_offset={float(noise_offset)}" + + adaptive_noise_scale = kwargs.get("adaptive_noise_scale") + if adaptive_noise_scale and float(adaptive_noise_scale) != 0 and float(noise_offset) > 0: + run_cmd += f" --adaptive_noise_scale={float(adaptive_noise_scale)}" + elif noise_offset_type and noise_offset_type == "Multires": + multires_noise_iterations = kwargs.get("multires_noise_iterations") + if int(multires_noise_iterations) > 0: + run_cmd += f' --multires_noise_iterations="{int(multires_noise_iterations)}"' + + multires_noise_discount = kwargs.get("multires_noise_discount") + if multires_noise_discount and float(multires_noise_discount) > 0: + run_cmd += f' --multires_noise_discount="{float(multires_noise_discount)}"' + + num_machines = kwargs.get("num_machines") + if num_machines and int(num_machines) > 1: + run_cmd += f" --num_machines={int(num_machines)}" - flip_aug = kwargs.get('flip_aug') - if flip_aug: - run_cmd += ' --flip_aug' + num_processes = kwargs.get("num_processes") + if num_processes and int(num_processes) > 1: + run_cmd += f" --num_processes={int(num_processes)}" - shuffle_caption = kwargs.get('shuffle_caption') - if shuffle_caption: - run_cmd += ' --shuffle_caption' + num_cpu_threads_per_process = kwargs.get("num_cpu_threads_per_process") + if num_cpu_threads_per_process and int(num_cpu_threads_per_process) > 1: + run_cmd += f" --num_cpu_threads_per_process={int(num_cpu_threads_per_process)}" - gradient_checkpointing = kwargs.get('gradient_checkpointing') - if gradient_checkpointing: - run_cmd += ' --gradient_checkpointing' - - fp8_base = kwargs.get('fp8_base') - if fp8_base: - run_cmd += ' --fp8_base' + persistent_data_loader_workers = kwargs.get("persistent_data_loader_workers") + if persistent_data_loader_workers: + run_cmd += " --persistent_data_loader_workers" - full_fp16 = kwargs.get('full_fp16') - if full_fp16: - run_cmd += ' --full_fp16' + random_crop = kwargs.get("random_crop") + if random_crop: + run_cmd += " --random_crop" - xformers = kwargs.get('xformers') - if xformers == 'xformers': - run_cmd += ' --xformers' - elif xformers == 'sdpa': - run_cmd += ' --sdpa' + resume = kwargs.get("resume") + if resume: + run_cmd += f' --resume="{resume}"' - # sdpa = kwargs.get('sdpa') - # if sdpa: - # run_cmd += ' --sdpa' + save_every_n_steps = kwargs.get("save_every_n_steps") + if save_every_n_steps and int(save_every_n_steps) > 0: + run_cmd += f' --save_every_n_steps="{int(save_every_n_steps)}"' - persistent_data_loader_workers = kwargs.get( - 'persistent_data_loader_workers' - ) - if persistent_data_loader_workers: - run_cmd += ' --persistent_data_loader_workers' + save_last_n_steps = kwargs.get("save_last_n_steps") + if save_last_n_steps and int(save_last_n_steps) > 0: + run_cmd += f' --save_last_n_steps="{int(save_last_n_steps)}"' - bucket_no_upscale = kwargs.get('bucket_no_upscale') - if bucket_no_upscale: - run_cmd += ' --bucket_no_upscale' + save_last_n_steps_state = kwargs.get("save_last_n_steps_state") + if save_last_n_steps_state and int(save_last_n_steps_state) > 0: + run_cmd += f' --save_last_n_steps_state="{int(save_last_n_steps_state)}"' - random_crop = kwargs.get('random_crop') - if random_crop: - run_cmd += ' --random_crop' + save_state = kwargs.get("save_state") + if save_state: + run_cmd += " --save_state" - scale_v_pred_loss_like_noise_pred = kwargs.get( - 'scale_v_pred_loss_like_noise_pred' - ) + scale_v_pred_loss_like_noise_pred = kwargs.get("scale_v_pred_loss_like_noise_pred") if scale_v_pred_loss_like_noise_pred: - run_cmd += ' --scale_v_pred_loss_like_noise_pred' + run_cmd += " --scale_v_pred_loss_like_noise_pred" - noise_offset_type = kwargs.get('noise_offset_type', 'Original') - if noise_offset_type == 'Original': - noise_offset = float(kwargs.get('noise_offset', 0)) - run_cmd += f' --noise_offset={noise_offset}' + shuffle_caption = kwargs.get("shuffle_caption") + if shuffle_caption: + run_cmd += " --shuffle_caption" - adaptive_noise_scale = float(kwargs.get('adaptive_noise_scale', 0)) - if adaptive_noise_scale != 0 and noise_offset > 0: - run_cmd += f' --adaptive_noise_scale={adaptive_noise_scale}' - else: - multires_noise_iterations = int( - kwargs.get('multires_noise_iterations', 0) - ) - if multires_noise_iterations > 0: - run_cmd += ( - f' --multires_noise_iterations="{multires_noise_iterations}"' - ) + use_wandb = kwargs.get("use_wandb") + if use_wandb: + run_cmd += " --log_with wandb" - multires_noise_discount = float( - kwargs.get('multires_noise_discount', 0) - ) - if multires_noise_discount > 0: - run_cmd += ( - f' --multires_noise_discount="{multires_noise_discount}"' - ) + v_pred_like_loss = kwargs.get("v_pred_like_loss") + if v_pred_like_loss and float(v_pred_like_loss) > 0: + run_cmd += f' --v_pred_like_loss="{float(v_pred_like_loss)}"' - additional_parameters = kwargs.get('additional_parameters', '') - if additional_parameters: - run_cmd += f' {additional_parameters}' + vae = kwargs.get("vae") + if vae: + run_cmd += f' --vae="{vae}"' - use_wandb = kwargs.get('use_wandb') - if use_wandb: - run_cmd += ' --log_with wandb' + vae_batch_size = kwargs.get("vae_batch_size") + if vae_batch_size and int(vae_batch_size) > 0: + run_cmd += f' --vae_batch_size="{int(vae_batch_size)}"' - wandb_api_key = kwargs.get('wandb_api_key', '') + wandb_api_key = kwargs.get("wandb_api_key") if wandb_api_key: run_cmd += f' --wandb_api_key="{wandb_api_key}"' - - vae = kwargs.get( - 'vae' - ) - if vae: - run_cmd += f' --vae="{vae}"' + + xformers = kwargs.get("xformers") + if xformers and xformers == "xformers": + run_cmd += " --xformers" + elif xformers and xformers == "sdpa": + run_cmd += " --sdpa" return run_cmd def verify_image_folder_pattern(folder_path): - false_response = True # temporarily set to true to prevent stopping training in case of false positive + false_response = True # temporarily set to true to prevent stopping training in case of false positive true_response = True # Check if the folder exists @@ -915,7 +880,7 @@ def verify_image_folder_pattern(folder_path): # The pattern should start with one or more digits (\d+) followed by an underscore (_) # After the underscore, it should match one or more word characters (\w+), which can be letters, numbers, or underscores # Example of a valid pattern matching name: 123_example_folder - pattern = r'^\d+_\w+' + pattern = r"^\d+_\w+" # Get the list of sub-folders in the directory subfolders = [ @@ -938,25 +903,25 @@ def verify_image_folder_pattern(folder_path): f"The following folders do not match the required pattern _: {', '.join(non_matching_subfolders)}" ) log.error( - f'Please follow the folder structure documentation found at docs\image_folder_structure.md ...' + f"Please follow the folder structure documentation found at docs\image_folder_structure.md ..." ) return false_response # Check if no sub-folders exist if not matching_subfolders: log.error( - f'No image folders found in {folder_path}. Please follow the folder structure documentation found at docs\image_folder_structure.md ...' + f"No image folders found in {folder_path}. Please follow the folder structure documentation found at docs\image_folder_structure.md ..." ) return false_response - log.info(f'Valid image folder names found in: {folder_path}') + log.info(f"Valid image folder names found in: {folder_path}") return true_response def SaveConfigFile( parameters, file_path: str, - exclusion=['file_path', 'save_as', 'headless', 'print_only'], + exclusion=["file_path", "save_as", "headless", "print_only"], ): # Return the values of the variables as a dictionary variables = { @@ -966,33 +931,31 @@ def SaveConfigFile( } # Save the data to the selected file - with open(file_path, 'w') as file: + with open(file_path, "w") as file: json.dump(variables, file, indent=2) def save_to_file(content): - logs_directory = 'logs' - file_path = os.path.join(logs_directory, 'print_command.txt') + logs_directory = "logs" + file_path = os.path.join(logs_directory, "print_command.txt") try: # Create the 'logs' directory if it does not exist if not os.path.exists(logs_directory): os.makedirs(logs_directory) - with open(file_path, 'a') as file: - file.write(content + '\n') + with open(file_path, "a") as file: + file.write(content + "\n") except IOError as e: - print(f'Error: Could not write to file - {e}') + print(f"Error: Could not write to file - {e}") except OSError as e: print(f"Error: Could not create 'logs' directory - {e}") def check_duplicate_filenames( - folder_path, image_extension=['.gif', '.png', '.jpg', '.jpeg', '.webp'] + folder_path, image_extension=[".gif", ".png", ".jpg", ".jpeg", ".webp"] ): - log.info( - 'Checking for duplicate image filenames in training data directory...' - ) + log.info("Checking for duplicate image filenames in training data directory...") for root, dirs, files in os.walk(folder_path): filenames = {} for file in files: @@ -1005,8 +968,8 @@ def check_duplicate_filenames( print( f"Warning: Same filename '{filename}' with different image extension found. This will cause training issues. Rename one of the file." ) - print(f'Existing file: {existing_path}') - print(f'Current file: {full_path}') + print(f"Existing file: {existing_path}") + print(f"Current file: {full_path}") else: filenames[filename] = full_path @@ -1017,11 +980,9 @@ def is_file_writable(file_path): return True try: - log.warning( - f"File '{file_path}' already exist... it will be overwritten..." - ) + log.warning(f"File '{file_path}' already exist... it will be overwritten...") # Check if the file can be opened in write mode (which implies it's not open by another process) - with open(file_path, 'a'): + with open(file_path, "a"): pass return True except IOError: diff --git a/lora_gui.py b/lora_gui.py index 413d92174..c00c2970a 100644 --- a/lora_gui.py +++ b/lora_gui.py @@ -104,6 +104,10 @@ def save_configuration( color_aug, flip_aug, clip_skip, + num_processes, + num_machines, + multi_gpu, + gpu_ids, gradient_accumulation_steps, mem_eff_attn, output_name, @@ -267,6 +271,10 @@ def open_configuration( color_aug, flip_aug, clip_skip, + num_processes, + num_machines, + multi_gpu, + gpu_ids, gradient_accumulation_steps, mem_eff_attn, output_name, @@ -461,6 +469,10 @@ def train_model( color_aug, flip_aug, clip_skip, + num_processes, + num_machines, + multi_gpu, + gpu_ids, gradient_accumulation_steps, mem_eff_attn, output_name, @@ -720,9 +732,15 @@ def train_model( lr_warmup_steps = round(float(int(lr_warmup) * int(max_train_steps) / 100)) log.info(f"lr_warmup_steps = {lr_warmup_steps}") - run_cmd = ( - f"accelerate launch --num_cpu_threads_per_process={num_cpu_threads_per_process}" - ) + run_cmd = "accelerate launch" + + run_cmd += run_cmd_advanced_training( + num_processes=num_processes, + num_machines=num_machines, + multi_gpu=multi_gpu, + gpu_ids=gpu_ids, + num_cpu_threads_per_process=num_cpu_threads_per_process) + if sdxl: run_cmd += f' "./sdxl_train_network.py"' else: @@ -1933,6 +1951,10 @@ def update_LoRA_settings( advanced_training.color_aug, advanced_training.flip_aug, advanced_training.clip_skip, + advanced_training.num_processes, + advanced_training.num_machines, + advanced_training.multi_gpu, + advanced_training.gpu_ids, advanced_training.gradient_accumulation_steps, advanced_training.mem_eff_attn, folders.output_name, diff --git a/textual_inversion_gui.py b/textual_inversion_gui.py index 7e9d7c7b9..05810df6a 100644 --- a/textual_inversion_gui.py +++ b/textual_inversion_gui.py @@ -96,6 +96,10 @@ def save_configuration( color_aug, flip_aug, clip_skip, + num_processes, + num_machines, + multi_gpu, + gpu_ids, vae, output_name, max_token_length, @@ -223,6 +227,10 @@ def open_configuration( color_aug, flip_aug, clip_skip, + num_processes, + num_machines, + multi_gpu, + gpu_ids, vae, output_name, max_token_length, @@ -343,6 +351,10 @@ def train_model( color_aug, flip_aug, clip_skip, + num_processes, + num_machines, + multi_gpu, + gpu_ids, vae, output_name, max_token_length, @@ -546,7 +558,15 @@ def train_model( lr_warmup_steps = round(float(int(lr_warmup) * int(max_train_steps) / 100)) log.info(f'lr_warmup_steps = {lr_warmup_steps}') - run_cmd = f'accelerate launch --num_cpu_threads_per_process={num_cpu_threads_per_process}' + run_cmd = "accelerate launch" + + run_cmd += run_cmd_advanced_training( + num_processes=num_processes, + num_machines=num_machines, + multi_gpu=multi_gpu, + gpu_ids=gpu_ids, + num_cpu_threads_per_process=num_cpu_threads_per_process) + if sdxl: run_cmd += f' "./sdxl_train_textual_inversion.py"' else: @@ -888,6 +908,10 @@ def ti_tab( advanced_training.color_aug, advanced_training.flip_aug, advanced_training.clip_skip, + advanced_training.num_processes, + advanced_training.num_machines, + advanced_training.multi_gpu, + advanced_training.gpu_ids, advanced_training.vae, folders.output_name, advanced_training.max_token_length, From 42ceeb8c923265c901b06a7327bb0c21957301dc Mon Sep 17 00:00:00 2001 From: bmaltais Date: Fri, 2 Feb 2024 23:51:31 -0500 Subject: [PATCH 03/13] Retrofit run commands structure --- dreambooth_gui.py | 85 ++++------- finetune_gui.py | 88 ++++------- library/common_gui.py | 313 ++++++++++++++++++++++++++++++--------- lora_gui.py | 67 +++------ textual_inversion_gui.py | 81 +++------- 5 files changed, 350 insertions(+), 284 deletions(-) diff --git a/dreambooth_gui.py b/dreambooth_gui.py index 4ef60a445..a2a31a092 100644 --- a/dreambooth_gui.py +++ b/dreambooth_gui.py @@ -17,7 +17,6 @@ color_aug_changed, save_inference_file, run_cmd_advanced_training, - run_cmd_training, update_my_data, check_if_model_exist, output_message, @@ -565,60 +564,7 @@ def train_model( else: run_cmd += f' "./train_db.py"' - if v2: - run_cmd += " --v2" - if v_parameterization: - run_cmd += " --v_parameterization" - if enable_bucket: - run_cmd += f" --enable_bucket --min_bucket_reso={min_bucket_reso} --max_bucket_reso={max_bucket_reso}" - if no_token_padding: - run_cmd += " --no_token_padding" - if weighted_captions: - run_cmd += " --weighted_captions" - run_cmd += f' --pretrained_model_name_or_path="{pretrained_model_name_or_path}"' - run_cmd += f' --train_data_dir="{train_data_dir}"' - if len(reg_data_dir): - run_cmd += f' --reg_data_dir="{reg_data_dir}"' - run_cmd += f' --resolution="{max_resolution}"' - run_cmd += f' --output_dir="{output_dir}"' - if not logging_dir == "": - run_cmd += f' --logging_dir="{logging_dir}"' - if not stop_text_encoder_training == 0: - run_cmd += f" --stop_text_encoder_training={stop_text_encoder_training}" - if not save_model_as == "same as source model": - run_cmd += f" --save_model_as={save_model_as}" - # if not resume == '': - # run_cmd += f' --resume={resume}' - if not float(prior_loss_weight) == 1.0: - run_cmd += f" --prior_loss_weight={prior_loss_weight}" - if full_bf16: - run_cmd += " --full_bf16" - if not vae == "": - run_cmd += f' --vae="{vae}"' - if not output_name == "": - run_cmd += f' --output_name="{output_name}"' - if not lr_scheduler_num_cycles == "": - run_cmd += f' --lr_scheduler_num_cycles="{lr_scheduler_num_cycles}"' - else: - run_cmd += f' --lr_scheduler_num_cycles="{epoch}"' - if not lr_scheduler_power == "": - run_cmd += f' --lr_scheduler_power="{lr_scheduler_power}"' - if int(max_token_length) > 75: - run_cmd += f" --max_token_length={max_token_length}" - if not max_train_epochs == "": - run_cmd += f' --max_train_epochs="{max_train_epochs}"' - if not max_data_loader_n_workers == "": - run_cmd += f' --max_data_loader_n_workers="{max_data_loader_n_workers}"' - if int(gradient_accumulation_steps) > 1: - run_cmd += f" --gradient_accumulation_steps={int(gradient_accumulation_steps)}" - - if sdxl: - run_cmd += f' --learning_rate_te1="{learning_rate_te1}"' - run_cmd += f' --learning_rate_te2="{learning_rate_te2}"' - else: - run_cmd += f' --learning_rate_te="{learning_rate_te}"' - - run_cmd += run_cmd_training( + run_cmd += run_cmd_advanced_training( learning_rate=learning_rate, lr_scheduler=lr_scheduler, lr_warmup_steps=lr_warmup_steps, @@ -634,9 +580,6 @@ def train_model( optimizer=optimizer, optimizer_args=optimizer_args, lr_scheduler_args=lr_scheduler_args, - ) - - run_cmd += run_cmd_advanced_training( max_train_epochs=max_train_epochs, max_data_loader_n_workers=max_data_loader_n_workers, max_token_length=max_token_length, @@ -674,6 +617,32 @@ def train_model( scale_v_pred_loss_like_noise_pred=scale_v_pred_loss_like_noise_pred, min_timestep=min_timestep, max_timestep=max_timestep, + v2=v2, + v_parameterization=v_parameterization, + enable_bucket=enable_bucket, + min_bucket_reso=min_bucket_reso, + max_bucket_reso=max_bucket_reso, + max_resolution=max_resolution, + no_token_padding=no_token_padding, + weighted_captions=weighted_captions, + pretrained_model_name_or_path=pretrained_model_name_or_path, + train_data_dir=train_data_dir, + reg_data_dir=reg_data_dir, + output_dir=output_dir, + logging_dir=logging_dir, + stop_text_encoder_training=stop_text_encoder_training, + save_model_as=save_model_as, + prior_loss_weight=prior_loss_weight, + full_bf16=full_bf16, + vae=vae, + output_name=output_name, + lr_scheduler_num_cycles=lr_scheduler_num_cycles, + epoch=epoch, + lr_scheduler_power=lr_scheduler_power, + gradient_accumulation_steps=gradient_accumulation_steps, + learning_rate_te1=learning_rate_te1 if sdxl else None, + learning_rate_te2=learning_rate_te2 if sdxl else None, + learning_rate_te=learning_rate_te if not sdxl else None, ) run_cmd += run_cmd_sample( diff --git a/finetune_gui.py b/finetune_gui.py index 797afea01..10b84fdb7 100644 --- a/finetune_gui.py +++ b/finetune_gui.py @@ -13,7 +13,6 @@ save_inference_file, run_cmd_advanced_training, color_aug_changed, - run_cmd_training, update_my_data, check_if_model_exist, SaveConfigFile, @@ -558,60 +557,12 @@ def train_model( run_cmd += f' "./sdxl_train.py"' else: run_cmd += f' "./fine_tune.py"' + + in_json = f'{train_dir}/{latent_metadata_filename}' if use_latent_files == 'Yes' else f'{train_dir}/{caption_metadata_filename}' + cache_text_encoder_outputs = sdxl_checkbox and sdxl_cache_text_encoder_outputs + no_half_vae = sdxl_checkbox and sdxl_no_half_vae - if v2: - run_cmd += ' --v2' - if v_parameterization: - run_cmd += ' --v_parameterization' - if train_text_encoder: - run_cmd += ' --train_text_encoder' - if sdxl_checkbox: - run_cmd += f' --learning_rate_te1="{learning_rate_te1}"' - run_cmd += f' --learning_rate_te2="{learning_rate_te2}"' - else: - run_cmd += f' --learning_rate_te="{learning_rate_te}"' - if full_bf16: - run_cmd += ' --full_bf16' - if weighted_captions: - run_cmd += ' --weighted_captions' - run_cmd += ( - f' --pretrained_model_name_or_path="{pretrained_model_name_or_path}"' - ) - if use_latent_files == 'Yes': - run_cmd += f' --in_json="{train_dir}/{latent_metadata_filename}"' - else: - run_cmd += f' --in_json="{train_dir}/{caption_metadata_filename}"' - run_cmd += f' --train_data_dir="{image_folder}"' - run_cmd += f' --output_dir="{output_dir}"' - if not logging_dir == '': - run_cmd += f' --logging_dir="{logging_dir}"' - run_cmd += f' --dataset_repeats={dataset_repeats}' - - run_cmd += ' --enable_bucket' - run_cmd += f' --resolution="{max_resolution}"' - run_cmd += f' --min_bucket_reso={min_bucket_reso}' - run_cmd += f' --max_bucket_reso={max_bucket_reso}' - - if not save_model_as == 'same as source model': - run_cmd += f' --save_model_as={save_model_as}' - if int(gradient_accumulation_steps) > 1: - run_cmd += f' --gradient_accumulation_steps={int(gradient_accumulation_steps)}' - if not block_lr == '': - run_cmd += f' --block_lr="{block_lr}"' - - if not output_name == '': - run_cmd += f' --output_name="{output_name}"' - if int(max_token_length) > 75: - run_cmd += f' --max_token_length={max_token_length}' - - if sdxl_checkbox: - if sdxl_cache_text_encoder_outputs: - run_cmd += f' --cache_text_encoder_outputs' - - if sdxl_no_half_vae: - run_cmd += f' --no_half_vae' - - run_cmd += run_cmd_training( + run_cmd += run_cmd_advanced_training( learning_rate=learning_rate, lr_scheduler=lr_scheduler, lr_warmup_steps=lr_warmup_steps, @@ -627,9 +578,6 @@ def train_model( optimizer=optimizer, optimizer_args=optimizer_args, lr_scheduler_args=lr_scheduler_args, - ) - - run_cmd += run_cmd_advanced_training( max_train_epochs=max_train_epochs, max_data_loader_n_workers=max_data_loader_n_workers, max_token_length=max_token_length, @@ -643,7 +591,6 @@ def train_model( gradient_checkpointing=gradient_checkpointing, full_fp16=full_fp16, xformers=xformers, - # use_8bit_adam=use_8bit_adam, keep_tokens=keep_tokens, persistent_data_loader_workers=persistent_data_loader_workers, bucket_no_upscale=bucket_no_upscale, @@ -668,6 +615,31 @@ def train_model( scale_v_pred_loss_like_noise_pred=scale_v_pred_loss_like_noise_pred, min_timestep=min_timestep, max_timestep=max_timestep, + v2=v2, + v_parameterization=v_parameterization, + enable_bucket=True, + min_bucket_reso=min_bucket_reso, + max_bucket_reso=max_bucket_reso, + max_resolution=max_resolution, + weighted_captions=weighted_captions, + pretrained_model_name_or_path=pretrained_model_name_or_path, + train_data_dir=image_folder, + output_dir=output_dir, + logging_dir=logging_dir, + save_model_as=save_model_as, + full_bf16=full_bf16, + output_name=output_name, + gradient_accumulation_steps=gradient_accumulation_steps, + learning_rate_te1=learning_rate_te1 if sdxl_checkbox else None, + learning_rate_te2=learning_rate_te2 if sdxl_checkbox else None, + learning_rate_te=learning_rate_te if not sdxl_checkbox else None, + train_text_encoder=train_text_encoder, + in_json=in_json, + dataset_repeats=dataset_repeats, + block_lr=block_lr, + no_half_vae=no_half_vae if sdxl_checkbox else None, + cache_text_encoder_outputs=cache_text_encoder_outputs if sdxl_checkbox else None, + ) run_cmd += run_cmd_sample( diff --git a/library/common_gui.py b/library/common_gui.py index 450119f8a..aa8dabaa1 100644 --- a/library/common_gui.py +++ b/library/common_gui.py @@ -608,84 +608,84 @@ def get_str_or_default(kwargs, key, default_value=""): return default_value -def run_cmd_training(**kwargs): - run_cmd = "" +# def run_cmd_training(**kwargs): +# run_cmd = "" - learning_rate = kwargs.get("learning_rate", "") - if learning_rate: - run_cmd += f' --learning_rate="{learning_rate}"' +# lr_scheduler = kwargs.get("lr_scheduler", "") +# if lr_scheduler: +# run_cmd += f' --lr_scheduler="{lr_scheduler}"' - lr_scheduler = kwargs.get("lr_scheduler", "") - if lr_scheduler: - run_cmd += f' --lr_scheduler="{lr_scheduler}"' +# lr_warmup_steps = kwargs.get("lr_warmup_steps", "") +# if lr_warmup_steps: +# if lr_scheduler == "constant": +# log.info("Can't use LR warmup with LR Scheduler constant... ignoring...") +# else: +# run_cmd += f' --lr_warmup_steps="{lr_warmup_steps}"' - lr_warmup_steps = kwargs.get("lr_warmup_steps", "") - if lr_warmup_steps: - if lr_scheduler == "constant": - log.info("Can't use LR warmup with LR Scheduler constant... ignoring...") - else: - run_cmd += f' --lr_warmup_steps="{lr_warmup_steps}"' +# train_batch_size = kwargs.get("train_batch_size", "") +# if train_batch_size: +# run_cmd += f' --train_batch_size="{train_batch_size}"' - train_batch_size = kwargs.get("train_batch_size", "") - if train_batch_size: - run_cmd += f' --train_batch_size="{train_batch_size}"' +# max_train_steps = kwargs.get("max_train_steps", "") +# if max_train_steps: +# run_cmd += f' --max_train_steps="{max_train_steps}"' - max_train_steps = kwargs.get("max_train_steps", "") - if max_train_steps: - run_cmd += f' --max_train_steps="{max_train_steps}"' +# save_every_n_epochs = kwargs.get("save_every_n_epochs") +# if save_every_n_epochs: +# run_cmd += f' --save_every_n_epochs="{int(save_every_n_epochs)}"' - save_every_n_epochs = kwargs.get("save_every_n_epochs") - if save_every_n_epochs: - run_cmd += f' --save_every_n_epochs="{int(save_every_n_epochs)}"' +# mixed_precision = kwargs.get("mixed_precision", "") +# if mixed_precision: +# run_cmd += f' --mixed_precision="{mixed_precision}"' - mixed_precision = kwargs.get("mixed_precision", "") - if mixed_precision: - run_cmd += f' --mixed_precision="{mixed_precision}"' +# save_precision = kwargs.get("save_precision", "") +# if save_precision: +# run_cmd += f' --save_precision="{save_precision}"' - save_precision = kwargs.get("save_precision", "") - if save_precision: - run_cmd += f' --save_precision="{save_precision}"' +# seed = kwargs.get("seed", "") +# if seed != "": +# run_cmd += f' --seed="{seed}"' - seed = kwargs.get("seed", "") - if seed != "": - run_cmd += f' --seed="{seed}"' - - caption_extension = kwargs.get("caption_extension", "") - if caption_extension: - run_cmd += f' --caption_extension="{caption_extension}"' +# caption_extension = kwargs.get("caption_extension", "") +# if caption_extension: +# run_cmd += f' --caption_extension="{caption_extension}"' - cache_latents = kwargs.get("cache_latents") - if cache_latents: - run_cmd += " --cache_latents" +# cache_latents = kwargs.get("cache_latents") +# if cache_latents: +# run_cmd += " --cache_latents" - cache_latents_to_disk = kwargs.get("cache_latents_to_disk") - if cache_latents_to_disk: - run_cmd += " --cache_latents_to_disk" +# cache_latents_to_disk = kwargs.get("cache_latents_to_disk") +# if cache_latents_to_disk: +# run_cmd += " --cache_latents_to_disk" - optimizer_type = kwargs.get("optimizer", "AdamW") - run_cmd += f' --optimizer_type="{optimizer_type}"' +# optimizer_type = kwargs.get("optimizer", "AdamW") +# run_cmd += f' --optimizer_type="{optimizer_type}"' - optimizer_args = kwargs.get("optimizer_args", "") - if optimizer_args != "": - run_cmd += f" --optimizer_args {optimizer_args}" +# optimizer_args = kwargs.get("optimizer_args", "") +# if optimizer_args != "": +# run_cmd += f" --optimizer_args {optimizer_args}" - lr_scheduler_args = kwargs.get("lr_scheduler_args", "") - if lr_scheduler_args != "": - run_cmd += f" --lr_scheduler_args {lr_scheduler_args}" +# lr_scheduler_args = kwargs.get("lr_scheduler_args", "") +# if lr_scheduler_args != "": +# run_cmd += f" --lr_scheduler_args {lr_scheduler_args}" - max_grad_norm = kwargs.get("max_grad_norm", "") - if max_grad_norm != "": - run_cmd += f' --max_grad_norm="{max_grad_norm}"' +# max_grad_norm = kwargs.get("max_grad_norm", "") +# if max_grad_norm != "": +# run_cmd += f' --max_grad_norm="{max_grad_norm}"' - return run_cmd +# return run_cmd def run_cmd_advanced_training(**kwargs): run_cmd = "" - + additional_parameters = kwargs.get("additional_parameters") if additional_parameters: run_cmd += f" {additional_parameters}" + + block_lr = kwargs.get("block_lr") + if block_lr: + run_cmd += f' --block_lr="(block_lr)"' bucket_no_upscale = kwargs.get("bucket_no_upscale") if bucket_no_upscale: @@ -695,6 +695,18 @@ def run_cmd_advanced_training(**kwargs): if bucket_reso_steps: run_cmd += f" --bucket_reso_steps={int(bucket_reso_steps)}" + cache_latents = kwargs.get("cache_latents") + if cache_latents: + run_cmd += " --cache_latents" + + cache_latents_to_disk = kwargs.get("cache_latents_to_disk") + if cache_latents_to_disk: + run_cmd += " --cache_latents_to_disk" + + cache_text_encoder_outputs = kwargs.get("cache_text_encoder_outputs") + if cache_text_encoder_outputs: + run_cmd += " --cache_text_encoder_outputs" + caption_dropout_every_n_epochs = kwargs.get("caption_dropout_every_n_epochs") if caption_dropout_every_n_epochs and int(caption_dropout_every_n_epochs) > 0: run_cmd += ( @@ -705,6 +717,10 @@ def run_cmd_advanced_training(**kwargs): if caption_dropout_rate and float(caption_dropout_rate) > 0: run_cmd += f' --caption_dropout_rate="{caption_dropout_rate}"' + caption_extension = kwargs.get("caption_extension") + if caption_extension: + run_cmd += f' --caption_extension="{caption_extension}"' + clip_skip = kwargs.get("clip_skip") if clip_skip and int(clip_skip) > 1: run_cmd += f" --clip_skip={int(clip_skip)}" @@ -712,6 +728,21 @@ def run_cmd_advanced_training(**kwargs): color_aug = kwargs.get("color_aug") if color_aug: run_cmd += " --color_aug" + + dataset_repeats = kwargs.get("dataset_repeats") + if dataset_repeats: + run_cmd += f' --dataset_repeats="{dataset_repeats}"' + + enable_bucket = kwargs.get("enable_bucket") + if enable_bucket: + min_bucket_reso = kwargs.get("min_bucket_reso") + max_bucket_reso = kwargs.get("max_bucket_reso") + if min_bucket_reso and max_bucket_reso: + run_cmd += f" --enable_bucket --min_bucket_reso={min_bucket_reso} --max_bucket_reso={max_bucket_reso}" + + in_json = kwargs.get("in_json") + if in_json: + run_cmd += f' --in_json="{in_json}"' flip_aug = kwargs.get("flip_aug") if flip_aug: @@ -720,10 +751,18 @@ def run_cmd_advanced_training(**kwargs): fp8_base = kwargs.get("fp8_base") if fp8_base: run_cmd += " --fp8_base" + + full_bf16 = kwargs.get("full_bf16") + if full_bf16: + run_cmd += " --full_bf16" full_fp16 = kwargs.get("full_fp16") if full_fp16: run_cmd += " --full_fp16" + + gradient_accumulation_steps = kwargs.get("gradient_accumulation_steps") + if gradient_accumulation_steps and int(gradient_accumulation_steps) > 1: + run_cmd += f" --gradient_accumulation_steps={int(gradient_accumulation_steps)}" gradient_checkpointing = kwargs.get("gradient_checkpointing") if gradient_checkpointing: @@ -733,43 +772,113 @@ def run_cmd_advanced_training(**kwargs): if keep_tokens and int(keep_tokens) > 0: run_cmd += f' --keep_tokens="{int(keep_tokens)}"' + learning_rate = kwargs.get("learning_rate") + if learning_rate: + run_cmd += f' --learning_rate="{learning_rate}"' + + learning_rate_te = kwargs.get("learning_rate_te") + if learning_rate_te: + run_cmd += f' --learning_rate_te="{learning_rate_te}"' + + learning_rate_te1 = kwargs.get("learning_rate_te1") + if learning_rate_te1: + run_cmd += f' --learning_rate_te1="{learning_rate_te1}"' + + learning_rate_te2 = kwargs.get("learning_rate_te2") + if learning_rate_te2: + run_cmd += f' --learning_rate_te2="{learning_rate_te2}"' + + logging_dir = kwargs.get("logging_dir") + if logging_dir: + run_cmd += f' --logging_dir="{logging_dir}"' + + lr_scheduler = kwargs.get("lr_scheduler") + if lr_scheduler: + run_cmd += f' --lr_scheduler="{lr_scheduler}"' + + lr_scheduler_args = kwargs.get("lr_scheduler_args") + if lr_scheduler_args and lr_scheduler_args != "": + run_cmd += f" --lr_scheduler_args {lr_scheduler_args}" + + lr_scheduler_num_cycles = kwargs.get("lr_scheduler_num_cycles") + if lr_scheduler_num_cycles and not lr_scheduler_num_cycles == "": + run_cmd += f' --lr_scheduler_num_cycles="{lr_scheduler_num_cycles}"' + else: + epoch = kwargs.get("epoch") + if epoch: + run_cmd += f' --lr_scheduler_num_cycles="{epoch}"' + + lr_scheduler_power = kwargs.get("lr_scheduler_power") + if lr_scheduler_power and not lr_scheduler_power == "": + run_cmd += f' --lr_scheduler_power="{lr_scheduler_power}"' + + lr_warmup_steps = kwargs.get("lr_warmup_steps") + if lr_warmup_steps: + if lr_scheduler == "constant": + log.info("Can't use LR warmup with LR Scheduler constant... ignoring...") + else: + run_cmd += f' --lr_warmup_steps="{lr_warmup_steps}"' gpu_ids = kwargs.get("gpu_ids") if gpu_ids: run_cmd += f' --gpu_ids="{gpu_ids}"' max_data_loader_n_workers = kwargs.get("max_data_loader_n_workers") - if max_data_loader_n_workers: + if max_data_loader_n_workers and not max_data_loader_n_workers == "": run_cmd += f' --max_data_loader_n_workers="{max_data_loader_n_workers}"' + max_grad_norm = kwargs.get("max_grad_norm") + if max_grad_norm and max_grad_norm != "": + run_cmd += f' --max_grad_norm="{max_grad_norm}"' + + max_resolution = kwargs.get("max_resolution") + if max_resolution: + run_cmd += f' --resolution="{max_resolution}"' + + max_timestep = kwargs.get("max_timestep") + if max_timestep and int(max_timestep) < 1000: + run_cmd += f" --max_timestep={int(max_timestep)}" + max_token_length = kwargs.get("max_token_length") if max_token_length and int(max_token_length) > 75: run_cmd += f" --max_token_length={int(max_token_length)}" max_train_epochs = kwargs.get("max_train_epochs") - if max_train_epochs: + if max_train_epochs and not max_train_epochs == '': run_cmd += f" --max_train_epochs={max_train_epochs}" - min_snr_gamma = kwargs.get("min_snr_gamma") - if min_snr_gamma and int(min_snr_gamma) >= 1: - run_cmd += f" --min_snr_gamma={int(min_snr_gamma)}" - - max_timestep = kwargs.get("max_timestep") - if max_timestep and int(max_timestep) < 1000: - run_cmd += f" --max_timestep={int(max_timestep)}" + max_train_steps = kwargs.get("max_train_steps") + if max_train_steps: + run_cmd += f' --max_train_steps="{max_train_steps}"' mem_eff_attn = kwargs.get("mem_eff_attn") if mem_eff_attn: run_cmd += " --mem_eff_attn" + min_snr_gamma = kwargs.get("min_snr_gamma") + if min_snr_gamma and int(min_snr_gamma) >= 1: + run_cmd += f" --min_snr_gamma={int(min_snr_gamma)}" + min_timestep = kwargs.get("min_timestep") if min_timestep and int(min_timestep) > 0: run_cmd += f" --min_timestep={int(min_timestep)}" + mixed_precision = kwargs.get("mixed_precision") + if mixed_precision: + run_cmd += f' --mixed_precision="{mixed_precision}"' + multi_gpu = kwargs.get("multi_gpu") if multi_gpu: run_cmd += " --multi_gpu" - + + no_half_vae = kwargs.get("no_half_vae") + if no_half_vae: + run_cmd += " --no_half_vae" + + no_token_padding = kwargs.get("no_token_padding") + if no_token_padding: + run_cmd += " --no_token_padding" + noise_offset_type = kwargs.get("noise_offset_type") if noise_offset_type and noise_offset_type == "Original": noise_offset = kwargs.get("noise_offset") @@ -800,18 +909,50 @@ def run_cmd_advanced_training(**kwargs): if num_cpu_threads_per_process and int(num_cpu_threads_per_process) > 1: run_cmd += f" --num_cpu_threads_per_process={int(num_cpu_threads_per_process)}" + optimizer_args = kwargs.get("optimizer_args") + if optimizer_args and optimizer_args != "": + run_cmd += f" --optimizer_args {optimizer_args}" + + optimizer_type = kwargs.get("optimizer") + if optimizer_type: + run_cmd += f' --optimizer_type="{optimizer_type}"' + + output_dir = kwargs.get("output_dir") + if output_dir: + run_cmd += f' --output_dir="{output_dir}"' + + output_name = kwargs.get("output_name") + if output_name and not output_name == "": + run_cmd += f' --output_name="{output_name}"' + persistent_data_loader_workers = kwargs.get("persistent_data_loader_workers") if persistent_data_loader_workers: run_cmd += " --persistent_data_loader_workers" + pretrained_model_name_or_path = kwargs.get("pretrained_model_name_or_path") + if pretrained_model_name_or_path: + run_cmd += f' --pretrained_model_name_or_path="{pretrained_model_name_or_path}"' + + prior_loss_weight = kwargs.get("prior_loss_weight") + if prior_loss_weight and not float(prior_loss_weight) == 1.0: + run_cmd += f" --prior_loss_weight={prior_loss_weight}" + random_crop = kwargs.get("random_crop") if random_crop: run_cmd += " --random_crop" + reg_data_dir = kwargs.get("reg_data_dir") + if reg_data_dir and len(reg_data_dir): + run_cmd += f' --reg_data_dir="{reg_data_dir}"' + resume = kwargs.get("resume") if resume: run_cmd += f' --resume="{resume}"' + save_every_n_epochs = kwargs.get("save_every_n_epochs") + if save_every_n_epochs: + run_cmd += f' --save_every_n_epochs="{int(save_every_n_epochs)}"' + save_every_n_steps = kwargs.get("save_every_n_steps") if save_every_n_steps and int(save_every_n_steps) > 0: run_cmd += f' --save_every_n_steps="{int(save_every_n_steps)}"' @@ -823,6 +964,14 @@ def run_cmd_advanced_training(**kwargs): save_last_n_steps_state = kwargs.get("save_last_n_steps_state") if save_last_n_steps_state and int(save_last_n_steps_state) > 0: run_cmd += f' --save_last_n_steps_state="{int(save_last_n_steps_state)}"' + + save_model_as = kwargs.get("save_model_as") + if save_model_as and not save_model_as == "same as source model": + run_cmd += f" --save_model_as={save_model_as}" + + save_precision = kwargs.get("save_precision") + if save_precision: + run_cmd += f' --save_precision="{save_precision}"' save_state = kwargs.get("save_state") if save_state: @@ -832,20 +981,48 @@ def run_cmd_advanced_training(**kwargs): if scale_v_pred_loss_like_noise_pred: run_cmd += " --scale_v_pred_loss_like_noise_pred" + seed = kwargs.get("seed") + if seed and seed != "": + run_cmd += f' --seed="{seed}"' + shuffle_caption = kwargs.get("shuffle_caption") if shuffle_caption: run_cmd += " --shuffle_caption" + + stop_text_encoder_training = kwargs.get("stop_text_encoder_training") + if stop_text_encoder_training and stop_text_encoder_training > 0: + run_cmd += f' --stop_text_encoder_training="{stop_text_encoder_training}"' + + train_batch_size = kwargs.get("train_batch_size") + if train_batch_size: + run_cmd += f' --train_batch_size="{train_batch_size}"' + + train_data_dir = kwargs.get("train_data_dir") + if train_data_dir: + run_cmd += f' --train_data_dir="{train_data_dir}"' + + train_text_encoder = kwargs.get("train_text_encoder") + if train_text_encoder: + run_cmd += " --train_text_encoder" use_wandb = kwargs.get("use_wandb") if use_wandb: run_cmd += " --log_with wandb" + + v_parameterization = kwargs.get("v_parameterization") + if v_parameterization: + run_cmd += " --v_parameterization" v_pred_like_loss = kwargs.get("v_pred_like_loss") if v_pred_like_loss and float(v_pred_like_loss) > 0: run_cmd += f' --v_pred_like_loss="{float(v_pred_like_loss)}"' + + v2 = kwargs.get("v2") + if v2: + run_cmd += " --v2" vae = kwargs.get("vae") - if vae: + if vae and not vae == "": run_cmd += f' --vae="{vae}"' vae_batch_size = kwargs.get("vae_batch_size") @@ -855,6 +1032,10 @@ def run_cmd_advanced_training(**kwargs): wandb_api_key = kwargs.get("wandb_api_key") if wandb_api_key: run_cmd += f' --wandb_api_key="{wandb_api_key}"' + + weighted_captions = kwargs.get("weighted_captions") + if weighted_captions: + run_cmd += " --weighted_captions" xformers = kwargs.get("xformers") if xformers and xformers == "xformers": diff --git a/lora_gui.py b/lora_gui.py index c00c2970a..b4fe287a3 100644 --- a/lora_gui.py +++ b/lora_gui.py @@ -10,7 +10,6 @@ get_saveasfile_path, color_aug_changed, run_cmd_advanced_training, - run_cmd_training, update_my_data, check_if_model_exist, output_message, @@ -746,33 +745,9 @@ def train_model( else: run_cmd += f' "./train_network.py"' - if v2: - run_cmd += " --v2" - if v_parameterization: - run_cmd += " --v_parameterization" - if enable_bucket: - run_cmd += f" --enable_bucket --min_bucket_reso={min_bucket_reso} --max_bucket_reso={max_bucket_reso}" - if no_token_padding: - run_cmd += " --no_token_padding" - if weighted_captions: - run_cmd += " --weighted_captions" - run_cmd += f' --pretrained_model_name_or_path="{pretrained_model_name_or_path}"' - run_cmd += f' --train_data_dir="{train_data_dir}"' - if len(reg_data_dir): - run_cmd += f' --reg_data_dir="{reg_data_dir}"' - run_cmd += f' --resolution="{max_resolution}"' - run_cmd += f' --output_dir="{output_dir}"' - if not logging_dir == "": - run_cmd += f' --logging_dir="{logging_dir}"' run_cmd += f' --network_alpha="{network_alpha}"' if not training_comment == "": run_cmd += f' --training_comment="{training_comment}"' - if not stop_text_encoder_training == 0: - run_cmd += f" --stop_text_encoder_training={stop_text_encoder_training}" - if not save_model_as == "same as source model": - run_cmd += f" --save_model_as={save_model_as}" - if not float(prior_loss_weight) == 1.0: - run_cmd += f" --prior_loss_weight={prior_loss_weight}" if LoRA_type == "LyCORIS/Diag-OFT": try: @@ -1008,17 +983,6 @@ def train_model( if dim_from_weights: run_cmd += f" --dim_from_weights" - if int(gradient_accumulation_steps) > 1: - run_cmd += f" --gradient_accumulation_steps={int(gradient_accumulation_steps)}" - if not output_name == "": - run_cmd += f' --output_name="{output_name}"' - if not lr_scheduler_num_cycles == "": - run_cmd += f' --lr_scheduler_num_cycles="{lr_scheduler_num_cycles}"' - else: - run_cmd += f' --lr_scheduler_num_cycles="{epoch}"' - if not lr_scheduler_power == "": - run_cmd += f' --lr_scheduler_power="{lr_scheduler_power}"' - if scale_weight_norms > 0.0: run_cmd += f' --scale_weight_norms="{scale_weight_norms}"' @@ -1032,13 +996,10 @@ def train_model( if sdxl_no_half_vae: run_cmd += f" --no_half_vae" - if full_bf16: - run_cmd += f" --full_bf16" - if debiased_estimation_loss: run_cmd += " --debiased_estimation_loss" - run_cmd += run_cmd_training( + run_cmd += run_cmd_advanced_training( learning_rate=learning_rate, lr_scheduler=lr_scheduler, lr_warmup_steps=lr_warmup_steps, @@ -1055,9 +1016,6 @@ def train_model( optimizer_args=optimizer_args, lr_scheduler_args=lr_scheduler_args, max_grad_norm=max_grad_norm, - ) - - run_cmd += run_cmd_advanced_training( max_train_epochs=max_train_epochs, max_data_loader_n_workers=max_data_loader_n_workers, max_token_length=max_token_length, @@ -1072,7 +1030,6 @@ def train_model( fp8_base=fp8_base, full_fp16=full_fp16, xformers=xformers, - # use_8bit_adam=use_8bit_adam, keep_tokens=keep_tokens, persistent_data_loader_workers=persistent_data_loader_workers, bucket_no_upscale=bucket_no_upscale, @@ -1098,6 +1055,28 @@ def train_model( min_timestep=min_timestep, max_timestep=max_timestep, vae=vae, + v2=v2, + v_parameterization=v_parameterization, + enable_bucket=enable_bucket, + min_bucket_reso=min_bucket_reso, + max_bucket_reso=max_bucket_reso, + max_resolution=max_resolution, + no_token_padding=no_token_padding, + weighted_captions=weighted_captions, + pretrained_model_name_or_path=pretrained_model_name_or_path, + train_data_dir=train_data_dir, + reg_data_dir=reg_data_dir, + output_dir=output_dir, + logging_dir=logging_dir, + stop_text_encoder_training=stop_text_encoder_training, + save_model_as=save_model_as, + prior_loss_weight=prior_loss_weight, + full_bf16=full_bf16, + output_name=output_name, + lr_scheduler_num_cycles=lr_scheduler_num_cycles, + epoch=epoch, + lr_scheduler_power=lr_scheduler_power, + gradient_accumulation_steps=gradient_accumulation_steps, ) run_cmd += run_cmd_sample( diff --git a/textual_inversion_gui.py b/textual_inversion_gui.py index 05810df6a..b4296128d 100644 --- a/textual_inversion_gui.py +++ b/textual_inversion_gui.py @@ -17,7 +17,6 @@ color_aug_changed, save_inference_file, run_cmd_advanced_training, - run_cmd_training, update_my_data, check_if_model_exist, output_message, @@ -572,59 +571,7 @@ def train_model( else: run_cmd += f' "./train_textual_inversion.py"' - if v2: - run_cmd += ' --v2' - if v_parameterization: - run_cmd += ' --v_parameterization' - if enable_bucket: - run_cmd += f' --enable_bucket --min_bucket_reso={min_bucket_reso} --max_bucket_reso={max_bucket_reso}' - if no_token_padding: - run_cmd += ' --no_token_padding' - run_cmd += ( - f' --pretrained_model_name_or_path="{pretrained_model_name_or_path}"' - ) - run_cmd += f' --train_data_dir="{train_data_dir}"' - if len(reg_data_dir): - run_cmd += f' --reg_data_dir="{reg_data_dir}"' - run_cmd += f' --resolution="{max_resolution}"' - run_cmd += f' --output_dir="{output_dir}"' - if not logging_dir == '': - run_cmd += f' --logging_dir="{logging_dir}"' - if not stop_text_encoder_training == 0: - run_cmd += ( - f' --stop_text_encoder_training={stop_text_encoder_training}' - ) - if not save_model_as == 'same as source model': - run_cmd += f' --save_model_as={save_model_as}' - # if not resume == '': - # run_cmd += f' --resume={resume}' - if not float(prior_loss_weight) == 1.0: - run_cmd += f' --prior_loss_weight={prior_loss_weight}' - if not vae == '': - run_cmd += f' --vae="{vae}"' - if not output_name == '': - run_cmd += f' --output_name="{output_name}"' - if not lr_scheduler_num_cycles == '': - run_cmd += f' --lr_scheduler_num_cycles="{lr_scheduler_num_cycles}"' - else: - run_cmd += f' --lr_scheduler_num_cycles="{epoch}"' - if not lr_scheduler_power == '': - run_cmd += f' --lr_scheduler_power="{lr_scheduler_power}"' - if int(max_token_length) > 75: - run_cmd += f' --max_token_length={max_token_length}' - if not max_train_epochs == '': - run_cmd += f' --max_train_epochs="{max_train_epochs}"' - if not max_data_loader_n_workers == '': - run_cmd += ( - f' --max_data_loader_n_workers="{max_data_loader_n_workers}"' - ) - if int(gradient_accumulation_steps) > 1: - run_cmd += f' --gradient_accumulation_steps={int(gradient_accumulation_steps)}' - - if sdxl and sdxl_no_half_vae: - run_cmd += f' --no_half_vae' - - run_cmd += run_cmd_training( + run_cmd += run_cmd_advanced_training( learning_rate=learning_rate, lr_scheduler=lr_scheduler, lr_warmup_steps=lr_warmup_steps, @@ -640,9 +587,6 @@ def train_model( optimizer=optimizer, optimizer_args=optimizer_args, lr_scheduler_args=lr_scheduler_args, - ) - - run_cmd += run_cmd_advanced_training( max_train_epochs=max_train_epochs, max_data_loader_n_workers=max_data_loader_n_workers, max_token_length=max_token_length, @@ -656,7 +600,6 @@ def train_model( gradient_checkpointing=gradient_checkpointing, full_fp16=full_fp16, xformers=xformers, - # use_8bit_adam=use_8bit_adam, keep_tokens=keep_tokens, persistent_data_loader_workers=persistent_data_loader_workers, bucket_no_upscale=bucket_no_upscale, @@ -681,6 +624,28 @@ def train_model( scale_v_pred_loss_like_noise_pred=scale_v_pred_loss_like_noise_pred, min_timestep=min_timestep, max_timestep=max_timestep, + v2=v2, + v_parameterization=v_parameterization, + enable_bucket=enable_bucket, + min_bucket_reso=min_bucket_reso, + max_bucket_reso=max_bucket_reso, + max_resolution=max_resolution, + no_token_padding=no_token_padding, + pretrained_model_name_or_path=pretrained_model_name_or_path, + train_data_dir=train_data_dir, + reg_data_dir=reg_data_dir, + output_dir=output_dir, + logging_dir=logging_dir, + stop_text_encoder_training=stop_text_encoder_training, + save_model_as=save_model_as, + prior_loss_weight=prior_loss_weight, + vae=vae, + output_name=output_name, + lr_scheduler_num_cycles=lr_scheduler_num_cycles, + epoch=epoch, + lr_scheduler_power=lr_scheduler_power, + gradient_accumulation_steps=gradient_accumulation_steps, + no_half_vae=True if sdxl and sdxl_no_half_vae else None, ) run_cmd += f' --token_string="{token_string}"' run_cmd += f' --init_word="{init_word}"' From 08ce96f33b208b4e02341eff19cf764f0be58a52 Mon Sep 17 00:00:00 2001 From: bmaltais Date: Sat, 3 Feb 2024 00:01:34 -0500 Subject: [PATCH 04/13] Sort run_cmd parameters --- dreambooth_gui.py | 124 +++++++++++++++++++-------------------- finetune_gui.py | 121 +++++++++++++++++++------------------- lora_gui.py | 122 +++++++++++++++++++------------------- textual_inversion_gui.py | 116 ++++++++++++++++++------------------ 4 files changed, 241 insertions(+), 242 deletions(-) diff --git a/dreambooth_gui.py b/dreambooth_gui.py index a2a31a092..02661b51d 100644 --- a/dreambooth_gui.py +++ b/dreambooth_gui.py @@ -565,84 +565,84 @@ def train_model( run_cmd += f' "./train_db.py"' run_cmd += run_cmd_advanced_training( + adaptive_noise_scale=adaptive_noise_scale, + additional_parameters=additional_parameters, + bucket_no_upscale=bucket_no_upscale, + bucket_reso_steps=bucket_reso_steps, + cache_latents=cache_latents, + cache_latents_to_disk=cache_latents_to_disk, + caption_dropout_every_n_epochs=caption_dropout_every_n_epochs, + caption_dropout_rate=caption_dropout_rate, + caption_extension=caption_extension, + clip_skip=clip_skip, + color_aug=color_aug, + enable_bucket=enable_bucket, + epoch=epoch, + flip_aug=flip_aug, + full_bf16=full_bf16, + full_fp16=full_fp16, + gradient_accumulation_steps=gradient_accumulation_steps, + gradient_checkpointing=gradient_checkpointing, + keep_tokens=keep_tokens, learning_rate=learning_rate, + learning_rate_te1=learning_rate_te1 if sdxl else None, + learning_rate_te2=learning_rate_te2 if sdxl else None, + learning_rate_te=learning_rate_te if not sdxl else None, + logging_dir=logging_dir, lr_scheduler=lr_scheduler, + lr_scheduler_args=lr_scheduler_args, + lr_scheduler_num_cycles=lr_scheduler_num_cycles, + lr_scheduler_power=lr_scheduler_power, lr_warmup_steps=lr_warmup_steps, - train_batch_size=train_batch_size, + max_bucket_reso=max_bucket_reso, + max_data_loader_n_workers=max_data_loader_n_workers, + max_resolution=max_resolution, + max_timestep=max_timestep, + max_token_length=max_token_length, + max_train_epochs=max_train_epochs, max_train_steps=max_train_steps, - save_every_n_epochs=save_every_n_epochs, + mem_eff_attn=mem_eff_attn, + min_bucket_reso=min_bucket_reso, + min_snr_gamma=min_snr_gamma, + min_timestep=min_timestep, mixed_precision=mixed_precision, - save_precision=save_precision, - seed=seed, - caption_extension=caption_extension, - cache_latents=cache_latents, - cache_latents_to_disk=cache_latents_to_disk, + multires_noise_discount=multires_noise_discount, + multires_noise_iterations=multires_noise_iterations, + no_token_padding=no_token_padding, + noise_offset=noise_offset, + noise_offset_type=noise_offset_type, optimizer=optimizer, optimizer_args=optimizer_args, - lr_scheduler_args=lr_scheduler_args, - max_train_epochs=max_train_epochs, - max_data_loader_n_workers=max_data_loader_n_workers, - max_token_length=max_token_length, - resume=resume, - save_state=save_state, - mem_eff_attn=mem_eff_attn, - clip_skip=clip_skip, - flip_aug=flip_aug, - color_aug=color_aug, - shuffle_caption=shuffle_caption, - gradient_checkpointing=gradient_checkpointing, - full_fp16=full_fp16, - xformers=xformers, - keep_tokens=keep_tokens, + output_dir=output_dir, + output_name=output_name, persistent_data_loader_workers=persistent_data_loader_workers, - bucket_no_upscale=bucket_no_upscale, + pretrained_model_name_or_path=pretrained_model_name_or_path, + prior_loss_weight=prior_loss_weight, random_crop=random_crop, - bucket_reso_steps=bucket_reso_steps, - v_pred_like_loss=v_pred_like_loss, - caption_dropout_every_n_epochs=caption_dropout_every_n_epochs, - caption_dropout_rate=caption_dropout_rate, - noise_offset_type=noise_offset_type, - noise_offset=noise_offset, - adaptive_noise_scale=adaptive_noise_scale, - multires_noise_iterations=multires_noise_iterations, - multires_noise_discount=multires_noise_discount, - additional_parameters=additional_parameters, - vae_batch_size=vae_batch_size, - min_snr_gamma=min_snr_gamma, + reg_data_dir=reg_data_dir, + resume=resume, + save_every_n_epochs=save_every_n_epochs, save_every_n_steps=save_every_n_steps, save_last_n_steps=save_last_n_steps, save_last_n_steps_state=save_last_n_steps_state, - use_wandb=use_wandb, - wandb_api_key=wandb_api_key, + save_model_as=save_model_as, + save_precision=save_precision, + save_state=save_state, scale_v_pred_loss_like_noise_pred=scale_v_pred_loss_like_noise_pred, - min_timestep=min_timestep, - max_timestep=max_timestep, + seed=seed, + shuffle_caption=shuffle_caption, + stop_text_encoder_training=stop_text_encoder_training, + train_batch_size=train_batch_size, + train_data_dir=train_data_dir, + use_wandb=use_wandb, v2=v2, v_parameterization=v_parameterization, - enable_bucket=enable_bucket, - min_bucket_reso=min_bucket_reso, - max_bucket_reso=max_bucket_reso, - max_resolution=max_resolution, - no_token_padding=no_token_padding, - weighted_captions=weighted_captions, - pretrained_model_name_or_path=pretrained_model_name_or_path, - train_data_dir=train_data_dir, - reg_data_dir=reg_data_dir, - output_dir=output_dir, - logging_dir=logging_dir, - stop_text_encoder_training=stop_text_encoder_training, - save_model_as=save_model_as, - prior_loss_weight=prior_loss_weight, - full_bf16=full_bf16, + v_pred_like_loss=v_pred_like_loss, vae=vae, - output_name=output_name, - lr_scheduler_num_cycles=lr_scheduler_num_cycles, - epoch=epoch, - lr_scheduler_power=lr_scheduler_power, - gradient_accumulation_steps=gradient_accumulation_steps, - learning_rate_te1=learning_rate_te1 if sdxl else None, - learning_rate_te2=learning_rate_te2 if sdxl else None, - learning_rate_te=learning_rate_te if not sdxl else None, + vae_batch_size=vae_batch_size, + wandb_api_key=wandb_api_key, + weighted_captions=weighted_captions, + xformers=xformers, ) run_cmd += run_cmd_sample( diff --git a/finetune_gui.py b/finetune_gui.py index 10b84fdb7..9ccd8539f 100644 --- a/finetune_gui.py +++ b/finetune_gui.py @@ -563,83 +563,82 @@ def train_model( no_half_vae = sdxl_checkbox and sdxl_no_half_vae run_cmd += run_cmd_advanced_training( + adaptive_noise_scale=adaptive_noise_scale, + additional_parameters=additional_parameters, + block_lr=block_lr, + bucket_no_upscale=bucket_no_upscale, + bucket_reso_steps=bucket_reso_steps, + cache_latents=cache_latents, + cache_latents_to_disk=cache_latents_to_disk, + cache_text_encoder_outputs=cache_text_encoder_outputs if sdxl_checkbox else None, + caption_dropout_every_n_epochs=caption_dropout_every_n_epochs, + caption_dropout_rate=caption_dropout_rate, + caption_extension=caption_extension, + clip_skip=clip_skip, + color_aug=color_aug, + dataset_repeats=dataset_repeats, + enable_bucket=True, + flip_aug=flip_aug, + full_bf16=full_bf16, + full_fp16=full_fp16, + gradient_accumulation_steps=gradient_accumulation_steps, + gradient_checkpointing=gradient_checkpointing, + in_json=in_json, + keep_tokens=keep_tokens, learning_rate=learning_rate, + learning_rate_te1=learning_rate_te1 if sdxl_checkbox else None, + learning_rate_te2=learning_rate_te2 if sdxl_checkbox else None, + learning_rate_te=learning_rate_te if not sdxl_checkbox else None, + logging_dir=logging_dir, lr_scheduler=lr_scheduler, + lr_scheduler_args=lr_scheduler_args, lr_warmup_steps=lr_warmup_steps, - train_batch_size=train_batch_size, + max_bucket_reso=max_bucket_reso, + max_data_loader_n_workers=max_data_loader_n_workers, + max_resolution=max_resolution, + max_timestep=max_timestep, + max_token_length=max_token_length, + max_train_epochs=max_train_epochs, max_train_steps=max_train_steps, - save_every_n_epochs=save_every_n_epochs, + mem_eff_attn=mem_eff_attn, + min_bucket_reso=min_bucket_reso, + min_snr_gamma=min_snr_gamma, + min_timestep=min_timestep, mixed_precision=mixed_precision, - save_precision=save_precision, - seed=seed, - caption_extension=caption_extension, - cache_latents=cache_latents, - cache_latents_to_disk=cache_latents_to_disk, + multires_noise_discount=multires_noise_discount, + multires_noise_iterations=multires_noise_iterations, + no_half_vae=no_half_vae if sdxl_checkbox else None, + noise_offset=noise_offset, + noise_offset_type=noise_offset_type, optimizer=optimizer, optimizer_args=optimizer_args, - lr_scheduler_args=lr_scheduler_args, - max_train_epochs=max_train_epochs, - max_data_loader_n_workers=max_data_loader_n_workers, - max_token_length=max_token_length, - resume=resume, - save_state=save_state, - mem_eff_attn=mem_eff_attn, - clip_skip=clip_skip, - flip_aug=flip_aug, - color_aug=color_aug, - shuffle_caption=shuffle_caption, - gradient_checkpointing=gradient_checkpointing, - full_fp16=full_fp16, - xformers=xformers, - keep_tokens=keep_tokens, + output_dir=output_dir, + output_name=output_name, persistent_data_loader_workers=persistent_data_loader_workers, - bucket_no_upscale=bucket_no_upscale, + pretrained_model_name_or_path=pretrained_model_name_or_path, random_crop=random_crop, - bucket_reso_steps=bucket_reso_steps, - v_pred_like_loss=v_pred_like_loss, - caption_dropout_every_n_epochs=caption_dropout_every_n_epochs, - caption_dropout_rate=caption_dropout_rate, - noise_offset_type=noise_offset_type, - noise_offset=noise_offset, - adaptive_noise_scale=adaptive_noise_scale, - multires_noise_iterations=multires_noise_iterations, - multires_noise_discount=multires_noise_discount, - additional_parameters=additional_parameters, - vae_batch_size=vae_batch_size, - min_snr_gamma=min_snr_gamma, + resume=resume, + save_every_n_epochs=save_every_n_epochs, save_every_n_steps=save_every_n_steps, save_last_n_steps=save_last_n_steps, save_last_n_steps_state=save_last_n_steps_state, - use_wandb=use_wandb, - wandb_api_key=wandb_api_key, + save_model_as=save_model_as, + save_precision=save_precision, + save_state=save_state, scale_v_pred_loss_like_noise_pred=scale_v_pred_loss_like_noise_pred, - min_timestep=min_timestep, - max_timestep=max_timestep, + seed=seed, + shuffle_caption=shuffle_caption, + train_batch_size=train_batch_size, + train_data_dir=image_folder, + train_text_encoder=train_text_encoder, + use_wandb=use_wandb, v2=v2, v_parameterization=v_parameterization, - enable_bucket=True, - min_bucket_reso=min_bucket_reso, - max_bucket_reso=max_bucket_reso, - max_resolution=max_resolution, + v_pred_like_loss=v_pred_like_loss, + vae_batch_size=vae_batch_size, + wandb_api_key=wandb_api_key, weighted_captions=weighted_captions, - pretrained_model_name_or_path=pretrained_model_name_or_path, - train_data_dir=image_folder, - output_dir=output_dir, - logging_dir=logging_dir, - save_model_as=save_model_as, - full_bf16=full_bf16, - output_name=output_name, - gradient_accumulation_steps=gradient_accumulation_steps, - learning_rate_te1=learning_rate_te1 if sdxl_checkbox else None, - learning_rate_te2=learning_rate_te2 if sdxl_checkbox else None, - learning_rate_te=learning_rate_te if not sdxl_checkbox else None, - train_text_encoder=train_text_encoder, - in_json=in_json, - dataset_repeats=dataset_repeats, - block_lr=block_lr, - no_half_vae=no_half_vae if sdxl_checkbox else None, - cache_text_encoder_outputs=cache_text_encoder_outputs if sdxl_checkbox else None, - + xformers=xformers, ) run_cmd += run_cmd_sample( diff --git a/lora_gui.py b/lora_gui.py index b4fe287a3..d5913d480 100644 --- a/lora_gui.py +++ b/lora_gui.py @@ -1000,83 +1000,83 @@ def train_model( run_cmd += " --debiased_estimation_loss" run_cmd += run_cmd_advanced_training( - learning_rate=learning_rate, - lr_scheduler=lr_scheduler, - lr_warmup_steps=lr_warmup_steps, - train_batch_size=train_batch_size, - max_train_steps=max_train_steps, - save_every_n_epochs=save_every_n_epochs, - mixed_precision=mixed_precision, - save_precision=save_precision, - seed=seed, - caption_extension=caption_extension, + adaptive_noise_scale=adaptive_noise_scale, + additional_parameters=additional_parameters, + bucket_no_upscale=bucket_no_upscale, + bucket_reso_steps=bucket_reso_steps, cache_latents=cache_latents, cache_latents_to_disk=cache_latents_to_disk, - optimizer=optimizer, - optimizer_args=optimizer_args, - lr_scheduler_args=lr_scheduler_args, - max_grad_norm=max_grad_norm, - max_train_epochs=max_train_epochs, - max_data_loader_n_workers=max_data_loader_n_workers, - max_token_length=max_token_length, - resume=resume, - save_state=save_state, - mem_eff_attn=mem_eff_attn, + caption_dropout_every_n_epochs=caption_dropout_every_n_epochs, + caption_dropout_rate=caption_dropout_rate, + caption_extension=caption_extension, clip_skip=clip_skip, - flip_aug=flip_aug, color_aug=color_aug, - shuffle_caption=shuffle_caption, - gradient_checkpointing=gradient_checkpointing, + enable_bucket=enable_bucket, + epoch=epoch, + flip_aug=flip_aug, fp8_base=fp8_base, + full_bf16=full_bf16, full_fp16=full_fp16, - xformers=xformers, + gradient_accumulation_steps=gradient_accumulation_steps, + gradient_checkpointing=gradient_checkpointing, keep_tokens=keep_tokens, + learning_rate=learning_rate, + logging_dir=logging_dir, + lr_scheduler=lr_scheduler, + lr_scheduler_args=lr_scheduler_args, + lr_scheduler_num_cycles=lr_scheduler_num_cycles, + lr_scheduler_power=lr_scheduler_power, + lr_warmup_steps=lr_warmup_steps, + max_bucket_reso=max_bucket_reso, + max_data_loader_n_workers=max_data_loader_n_workers, + max_grad_norm=max_grad_norm, + max_resolution=max_resolution, + max_timestep=max_timestep, + max_token_length=max_token_length, + max_train_epochs=max_train_epochs, + max_train_steps=max_train_steps, + mem_eff_attn=mem_eff_attn, + min_bucket_reso=min_bucket_reso, + min_snr_gamma=min_snr_gamma, + min_timestep=min_timestep, + mixed_precision=mixed_precision, + multires_noise_discount=multires_noise_discount, + multires_noise_iterations=multires_noise_iterations, + no_token_padding=no_token_padding, + noise_offset=noise_offset, + noise_offset_type=noise_offset_type, + optimizer=optimizer, + optimizer_args=optimizer_args, + output_dir=output_dir, + output_name=output_name, persistent_data_loader_workers=persistent_data_loader_workers, - bucket_no_upscale=bucket_no_upscale, + pretrained_model_name_or_path=pretrained_model_name_or_path, + prior_loss_weight=prior_loss_weight, random_crop=random_crop, - bucket_reso_steps=bucket_reso_steps, - v_pred_like_loss=v_pred_like_loss, - caption_dropout_every_n_epochs=caption_dropout_every_n_epochs, - caption_dropout_rate=caption_dropout_rate, - noise_offset_type=noise_offset_type, - noise_offset=noise_offset, - adaptive_noise_scale=adaptive_noise_scale, - multires_noise_iterations=multires_noise_iterations, - multires_noise_discount=multires_noise_discount, - additional_parameters=additional_parameters, - vae_batch_size=vae_batch_size, - min_snr_gamma=min_snr_gamma, + reg_data_dir=reg_data_dir, + resume=resume, + save_every_n_epochs=save_every_n_epochs, save_every_n_steps=save_every_n_steps, save_last_n_steps=save_last_n_steps, save_last_n_steps_state=save_last_n_steps_state, - use_wandb=use_wandb, - wandb_api_key=wandb_api_key, + save_model_as=save_model_as, + save_precision=save_precision, + save_state=save_state, scale_v_pred_loss_like_noise_pred=scale_v_pred_loss_like_noise_pred, - min_timestep=min_timestep, - max_timestep=max_timestep, - vae=vae, + seed=seed, + shuffle_caption=shuffle_caption, + stop_text_encoder_training=stop_text_encoder_training, + train_batch_size=train_batch_size, + train_data_dir=train_data_dir, + use_wandb=use_wandb, v2=v2, v_parameterization=v_parameterization, - enable_bucket=enable_bucket, - min_bucket_reso=min_bucket_reso, - max_bucket_reso=max_bucket_reso, - max_resolution=max_resolution, - no_token_padding=no_token_padding, + v_pred_like_loss=v_pred_like_loss, + vae=vae, + vae_batch_size=vae_batch_size, + wandb_api_key=wandb_api_key, weighted_captions=weighted_captions, - pretrained_model_name_or_path=pretrained_model_name_or_path, - train_data_dir=train_data_dir, - reg_data_dir=reg_data_dir, - output_dir=output_dir, - logging_dir=logging_dir, - stop_text_encoder_training=stop_text_encoder_training, - save_model_as=save_model_as, - prior_loss_weight=prior_loss_weight, - full_bf16=full_bf16, - output_name=output_name, - lr_scheduler_num_cycles=lr_scheduler_num_cycles, - epoch=epoch, - lr_scheduler_power=lr_scheduler_power, - gradient_accumulation_steps=gradient_accumulation_steps, + xformers=xformers, ) run_cmd += run_cmd_sample( diff --git a/textual_inversion_gui.py b/textual_inversion_gui.py index b4296128d..d54adb85b 100644 --- a/textual_inversion_gui.py +++ b/textual_inversion_gui.py @@ -572,80 +572,80 @@ def train_model( run_cmd += f' "./train_textual_inversion.py"' run_cmd += run_cmd_advanced_training( + adaptive_noise_scale=adaptive_noise_scale, + additional_parameters=additional_parameters, + bucket_no_upscale=bucket_no_upscale, + bucket_reso_steps=bucket_reso_steps, + cache_latents=cache_latents, + cache_latents_to_disk=cache_latents_to_disk, + caption_dropout_every_n_epochs=caption_dropout_every_n_epochs, + caption_dropout_rate=caption_dropout_rate, + caption_extension=caption_extension, + clip_skip=clip_skip, + color_aug=color_aug, + enable_bucket=enable_bucket, + epoch=epoch, + flip_aug=flip_aug, + full_fp16=full_fp16, + gradient_accumulation_steps=gradient_accumulation_steps, + gradient_checkpointing=gradient_checkpointing, + keep_tokens=keep_tokens, learning_rate=learning_rate, + logging_dir=logging_dir, lr_scheduler=lr_scheduler, + lr_scheduler_args=lr_scheduler_args, + lr_scheduler_num_cycles=lr_scheduler_num_cycles, + lr_scheduler_power=lr_scheduler_power, lr_warmup_steps=lr_warmup_steps, - train_batch_size=train_batch_size, + max_bucket_reso=max_bucket_reso, + max_data_loader_n_workers=max_data_loader_n_workers, + max_resolution=max_resolution, + max_timestep=max_timestep, + max_token_length=max_token_length, + max_train_epochs=max_train_epochs, max_train_steps=max_train_steps, - save_every_n_epochs=save_every_n_epochs, + mem_eff_attn=mem_eff_attn, + min_bucket_reso=min_bucket_reso, + min_snr_gamma=min_snr_gamma, + min_timestep=min_timestep, mixed_precision=mixed_precision, - save_precision=save_precision, - seed=seed, - caption_extension=caption_extension, - cache_latents=cache_latents, - cache_latents_to_disk=cache_latents_to_disk, + multires_noise_discount=multires_noise_discount, + multires_noise_iterations=multires_noise_iterations, + no_half_vae=True if sdxl and sdxl_no_half_vae else None, + no_token_padding=no_token_padding, + noise_offset=noise_offset, + noise_offset_type=noise_offset_type, optimizer=optimizer, optimizer_args=optimizer_args, - lr_scheduler_args=lr_scheduler_args, - max_train_epochs=max_train_epochs, - max_data_loader_n_workers=max_data_loader_n_workers, - max_token_length=max_token_length, - resume=resume, - save_state=save_state, - mem_eff_attn=mem_eff_attn, - clip_skip=clip_skip, - flip_aug=flip_aug, - color_aug=color_aug, - shuffle_caption=shuffle_caption, - gradient_checkpointing=gradient_checkpointing, - full_fp16=full_fp16, - xformers=xformers, - keep_tokens=keep_tokens, + output_dir=output_dir, + output_name=output_name, persistent_data_loader_workers=persistent_data_loader_workers, - bucket_no_upscale=bucket_no_upscale, + pretrained_model_name_or_path=pretrained_model_name_or_path, + prior_loss_weight=prior_loss_weight, random_crop=random_crop, - bucket_reso_steps=bucket_reso_steps, - v_pred_like_loss=v_pred_like_loss, - caption_dropout_every_n_epochs=caption_dropout_every_n_epochs, - caption_dropout_rate=caption_dropout_rate, - noise_offset_type=noise_offset_type, - noise_offset=noise_offset, - adaptive_noise_scale=adaptive_noise_scale, - multires_noise_iterations=multires_noise_iterations, - multires_noise_discount=multires_noise_discount, - additional_parameters=additional_parameters, - vae_batch_size=vae_batch_size, - min_snr_gamma=min_snr_gamma, + reg_data_dir=reg_data_dir, + resume=resume, + save_every_n_epochs=save_every_n_epochs, save_every_n_steps=save_every_n_steps, save_last_n_steps=save_last_n_steps, save_last_n_steps_state=save_last_n_steps_state, - use_wandb=use_wandb, - wandb_api_key=wandb_api_key, + save_model_as=save_model_as, + save_precision=save_precision, + save_state=save_state, scale_v_pred_loss_like_noise_pred=scale_v_pred_loss_like_noise_pred, - min_timestep=min_timestep, - max_timestep=max_timestep, + seed=seed, + shuffle_caption=shuffle_caption, + stop_text_encoder_training=stop_text_encoder_training, + train_batch_size=train_batch_size, + train_data_dir=train_data_dir, + use_wandb=use_wandb, v2=v2, v_parameterization=v_parameterization, - enable_bucket=enable_bucket, - min_bucket_reso=min_bucket_reso, - max_bucket_reso=max_bucket_reso, - max_resolution=max_resolution, - no_token_padding=no_token_padding, - pretrained_model_name_or_path=pretrained_model_name_or_path, - train_data_dir=train_data_dir, - reg_data_dir=reg_data_dir, - output_dir=output_dir, - logging_dir=logging_dir, - stop_text_encoder_training=stop_text_encoder_training, - save_model_as=save_model_as, - prior_loss_weight=prior_loss_weight, + v_pred_like_loss=v_pred_like_loss, vae=vae, - output_name=output_name, - lr_scheduler_num_cycles=lr_scheduler_num_cycles, - epoch=epoch, - lr_scheduler_power=lr_scheduler_power, - gradient_accumulation_steps=gradient_accumulation_steps, - no_half_vae=True if sdxl and sdxl_no_half_vae else None, + vae_batch_size=vae_batch_size, + wandb_api_key=wandb_api_key, + xformers=xformers, ) run_cmd += f' --token_string="{token_string}"' run_cmd += f' --init_word="{init_word}"' From 0b217a4cf8d92aefaf56b6aa71d4af71a127fb61 Mon Sep 17 00:00:00 2001 From: bmaltais Date: Sat, 3 Feb 2024 00:04:00 -0500 Subject: [PATCH 05/13] Format code --- dreambooth_gui.py | 7 +- finetune_gui.py | 372 ++++++++++++++++++--------------------- library/common_gui.py | 72 ++++---- lora_gui.py | 11 +- textual_inversion_gui.py | 261 +++++++++++++-------------- 5 files changed, 346 insertions(+), 377 deletions(-) diff --git a/dreambooth_gui.py b/dreambooth_gui.py index 02661b51d..6d5664dec 100644 --- a/dreambooth_gui.py +++ b/dreambooth_gui.py @@ -551,14 +551,15 @@ def train_model( # run_cmd = f'accelerate launch --num_cpu_threads_per_process={num_cpu_threads_per_process} "train_db.py"' run_cmd = "accelerate launch" - + run_cmd += run_cmd_advanced_training( num_processes=num_processes, num_machines=num_machines, multi_gpu=multi_gpu, gpu_ids=gpu_ids, - num_cpu_threads_per_process=num_cpu_threads_per_process) - + num_cpu_threads_per_process=num_cpu_threads_per_process, + ) + if sdxl: run_cmd += f' "./sdxl_train.py"' else: diff --git a/finetune_gui.py b/finetune_gui.py index 9ccd8539f..73e4b2353 100644 --- a/finetune_gui.py +++ b/finetune_gui.py @@ -43,12 +43,12 @@ # from easygui import msgbox -folder_symbol = '\U0001f4c2' # 📂 -refresh_symbol = '\U0001f504' # 🔄 -save_style_symbol = '\U0001f4be' # 💾 -document_symbol = '\U0001F4C4' # 📄 +folder_symbol = "\U0001f4c2" # 📂 +refresh_symbol = "\U0001f504" # 🔄 +save_style_symbol = "\U0001f4be" # 💾 +document_symbol = "\U0001F4C4" # 📄 -PYTHON = 'python3' if os.name == 'posix' else './venv/Scripts/python.exe' +PYTHON = "python3" if os.name == "posix" else "./venv/Scripts/python.exe" def save_configuration( @@ -100,7 +100,8 @@ def save_configuration( save_state, resume, gradient_checkpointing, - gradient_accumulation_steps,block_lr, + gradient_accumulation_steps, + block_lr, mem_eff_attn, shuffle_caption, output_name, @@ -153,19 +154,19 @@ def save_configuration( original_file_path = file_path - save_as_bool = True if save_as.get('label') == 'True' else False + save_as_bool = True if save_as.get("label") == "True" else False if save_as_bool: - log.info('Save as...') + log.info("Save as...") file_path = get_saveasfile_path(file_path) else: - log.info('Save...') - if file_path == None or file_path == '': + log.info("Save...") + if file_path == None or file_path == "": file_path = get_saveasfile_path(file_path) # log.info(file_path) - if file_path == None or file_path == '': + if file_path == None or file_path == "": return original_file_path # In case a file_path was provided and the user decide to cancel the open action # Extract the destination directory from the file path @@ -178,7 +179,7 @@ def save_configuration( SaveConfigFile( parameters=parameters, file_path=file_path, - exclusion=['file_path', 'save_as'], + exclusion=["file_path", "save_as"], ) return file_path @@ -234,7 +235,8 @@ def open_configuration( save_state, resume, gradient_checkpointing, - gradient_accumulation_steps,block_lr, + gradient_accumulation_steps, + block_lr, mem_eff_attn, shuffle_caption, output_name, @@ -286,33 +288,31 @@ def open_configuration( # Get list of function parameters and values parameters = list(locals().items()) - ask_for_file = True if ask_for_file.get('label') == 'True' else False - apply_preset = True if apply_preset.get('label') == 'True' else False + ask_for_file = True if ask_for_file.get("label") == "True" else False + apply_preset = True if apply_preset.get("label") == "True" else False # Check if we are "applying" a preset or a config if apply_preset: - log.info(f'Applying preset {training_preset}...') - file_path = f'./presets/finetune/{training_preset}.json' + log.info(f"Applying preset {training_preset}...") + file_path = f"./presets/finetune/{training_preset}.json" else: # If not applying a preset, set the `training_preset` field to an empty string # Find the index of the `training_preset` parameter using the `index()` method - training_preset_index = parameters.index( - ('training_preset', training_preset) - ) + training_preset_index = parameters.index(("training_preset", training_preset)) # Update the value of `training_preset` by directly assigning an empty string value - parameters[training_preset_index] = ('training_preset', '') + parameters[training_preset_index] = ("training_preset", "") original_file_path = file_path if ask_for_file: file_path = get_file_path(file_path) - if not file_path == '' and not file_path == None: + if not file_path == "" and not file_path == None: # load variables from JSON file - with open(file_path, 'r') as f: + with open(file_path, "r") as f: my_data = json.load(f) - log.info('Loading config...') + log.info("Loading config...") # Update values to fix deprecated use_8bit_adam checkbox and set appropriate optimizer if it is set to True my_data = update_my_data(my_data) else: @@ -323,7 +323,7 @@ def open_configuration( for key, value in parameters: json_value = my_data.get(key) # Set the value in the dictionary to the corresponding value in `my_data`, or the default value if not found - if not key in ['ask_for_file', 'apply_preset', 'file_path']: + if not key in ["ask_for_file", "apply_preset", "file_path"]: values.append(json_value if json_value is not None else value) return tuple(values) @@ -377,7 +377,8 @@ def train_model( save_state, resume, gradient_checkpointing, - gradient_accumulation_steps,block_lr, + gradient_accumulation_steps, + block_lr, mem_eff_attn, shuffle_caption, output_name, @@ -428,14 +429,12 @@ def train_model( # Get list of function parameters and values parameters = list(locals().items()) - print_only_bool = True if print_only.get('label') == 'True' else False - log.info(f'Start Finetuning...') + print_only_bool = True if print_only.get("label") == "True" else False + log.info(f"Start Finetuning...") - headless_bool = True if headless.get('label') == 'True' else False + headless_bool = True if headless.get("label") == "True" else False - if check_if_model_exist( - output_name, output_dir, save_model_as, headless_bool - ): + if check_if_model_exist(output_name, output_dir, save_model_as, headless_bool): return # if float(noise_offset) > 0 and ( @@ -461,52 +460,50 @@ def train_model( if not os.path.exists(train_dir): os.mkdir(train_dir) - run_cmd = f'{PYTHON} finetune/merge_captions_to_metadata.py' - if caption_extension == '': + run_cmd = f"{PYTHON} finetune/merge_captions_to_metadata.py" + if caption_extension == "": run_cmd += f' --caption_extension=".caption"' else: - run_cmd += f' --caption_extension={caption_extension}' + run_cmd += f" --caption_extension={caption_extension}" run_cmd += f' "{image_folder}"' run_cmd += f' "{train_dir}/{caption_metadata_filename}"' if full_path: - run_cmd += f' --full_path' + run_cmd += f" --full_path" log.info(run_cmd) if not print_only_bool: # Run the command - if os.name == 'posix': + if os.name == "posix": os.system(run_cmd) else: subprocess.run(run_cmd) # create images buckets if generate_image_buckets: - run_cmd = f'{PYTHON} finetune/prepare_buckets_latents.py' + run_cmd = f"{PYTHON} finetune/prepare_buckets_latents.py" run_cmd += f' "{image_folder}"' run_cmd += f' "{train_dir}/{caption_metadata_filename}"' run_cmd += f' "{train_dir}/{latent_metadata_filename}"' run_cmd += f' "{pretrained_model_name_or_path}"' - run_cmd += f' --batch_size={batch_size}' - run_cmd += f' --max_resolution={max_resolution}' - run_cmd += f' --min_bucket_reso={min_bucket_reso}' - run_cmd += f' --max_bucket_reso={max_bucket_reso}' - run_cmd += f' --mixed_precision={mixed_precision}' + run_cmd += f" --batch_size={batch_size}" + run_cmd += f" --max_resolution={max_resolution}" + run_cmd += f" --min_bucket_reso={min_bucket_reso}" + run_cmd += f" --max_bucket_reso={max_bucket_reso}" + run_cmd += f" --mixed_precision={mixed_precision}" # if flip_aug: # run_cmd += f' --flip_aug' if full_path: - run_cmd += f' --full_path' + run_cmd += f" --full_path" if sdxl_checkbox and sdxl_no_half_vae: - log.info( - 'Using mixed_precision = no because no half vae is selected...' - ) + log.info("Using mixed_precision = no because no half vae is selected...") run_cmd += f' --mixed_precision="no"' log.info(run_cmd) if not print_only_bool: # Run the command - if os.name == 'posix': + if os.name == "posix": os.system(run_cmd) else: subprocess.run(run_cmd) @@ -517,13 +514,13 @@ def train_model( for f, lower_f in ( (file, file.lower()) for file in os.listdir(image_folder) ) - if lower_f.endswith(('.jpg', '.jpeg', '.png', '.webp')) + if lower_f.endswith((".jpg", ".jpeg", ".png", ".webp")) ] ) - log.info(f'image_num = {image_num}') + log.info(f"image_num = {image_num}") repeats = int(image_num) * int(dataset_repeats) - log.info(f'repeats = {str(repeats)}') + log.info(f"repeats = {str(repeats)}") # calculate max_train_steps max_train_steps = int( @@ -539,26 +536,31 @@ def train_model( if flip_aug: max_train_steps = int(math.ceil(float(max_train_steps) / 2)) - log.info(f'max_train_steps = {max_train_steps}') + log.info(f"max_train_steps = {max_train_steps}") lr_warmup_steps = round(float(int(lr_warmup) * int(max_train_steps) / 100)) - log.info(f'lr_warmup_steps = {lr_warmup_steps}') + log.info(f"lr_warmup_steps = {lr_warmup_steps}") run_cmd = "accelerate launch" - + run_cmd += run_cmd_advanced_training( num_processes=num_processes, num_machines=num_machines, multi_gpu=multi_gpu, gpu_ids=gpu_ids, - num_cpu_threads_per_process=num_cpu_threads_per_process) - + num_cpu_threads_per_process=num_cpu_threads_per_process, + ) + if sdxl_checkbox: run_cmd += f' "./sdxl_train.py"' else: run_cmd += f' "./fine_tune.py"' - - in_json = f'{train_dir}/{latent_metadata_filename}' if use_latent_files == 'Yes' else f'{train_dir}/{caption_metadata_filename}' + + in_json = ( + f"{train_dir}/{latent_metadata_filename}" + if use_latent_files == "Yes" + else f"{train_dir}/{caption_metadata_filename}" + ) cache_text_encoder_outputs = sdxl_checkbox and sdxl_cache_text_encoder_outputs no_half_vae = sdxl_checkbox and sdxl_no_half_vae @@ -570,7 +572,9 @@ def train_model( bucket_reso_steps=bucket_reso_steps, cache_latents=cache_latents, cache_latents_to_disk=cache_latents_to_disk, - cache_text_encoder_outputs=cache_text_encoder_outputs if sdxl_checkbox else None, + cache_text_encoder_outputs=cache_text_encoder_outputs + if sdxl_checkbox + else None, caption_dropout_every_n_epochs=caption_dropout_every_n_epochs, caption_dropout_rate=caption_dropout_rate, caption_extension=caption_extension, @@ -651,7 +655,7 @@ def train_model( if print_only_bool: log.warning( - 'Here is the trainer command as a reference. It will not be executed:\n' + "Here is the trainer command as a reference. It will not be executed:\n" ) print(run_cmd) @@ -659,17 +663,15 @@ def train_model( else: # Saving config file for model current_datetime = datetime.now() - formatted_datetime = current_datetime.strftime('%Y%m%d-%H%M%S') - file_path = os.path.join( - output_dir, f'{output_name}_{formatted_datetime}.json' - ) + formatted_datetime = current_datetime.strftime("%Y%m%d-%H%M%S") + file_path = os.path.join(output_dir, f"{output_name}_{formatted_datetime}.json") - log.info(f'Saving training config to {file_path}...') + log.info(f"Saving training config to {file_path}...") SaveConfigFile( parameters=parameters, file_path=file_path, - exclusion=['file_path', 'save_as', 'headless', 'print_only'], + exclusion=["file_path", "save_as", "headless", "print_only"], ) log.info(run_cmd) @@ -678,18 +680,16 @@ def train_model( executor.execute_command(run_cmd=run_cmd) # check if output_dir/last is a folder... therefore it is a diffuser model - last_dir = pathlib.Path(f'{output_dir}/{output_name}') + last_dir = pathlib.Path(f"{output_dir}/{output_name}") if not last_dir.is_dir(): # Copy inference model for v2 if required - save_inference_file( - output_dir, v2, v_parameterization, output_name - ) + save_inference_file(output_dir, v2, v_parameterization, output_name) def remove_doublequote(file_path): if file_path != None: - file_path = file_path.replace('"', '') + file_path = file_path.replace('"', "") return file_path @@ -698,23 +698,23 @@ def finetune_tab(headless=False): dummy_db_true = gr.Label(value=True, visible=False) dummy_db_false = gr.Label(value=False, visible=False) dummy_headless = gr.Label(value=headless, visible=False) - with gr.Tab('Training'): - gr.Markdown('Train a custom model using kohya finetune python code...') + with gr.Tab("Training"): + gr.Markdown("Train a custom model using kohya finetune python code...") # Setup Configuration Files Gradio config = ConfigurationFile(headless) source_model = SourceModel(headless=headless) - with gr.Tab('Folders'): + with gr.Tab("Folders"): with gr.Row(): train_dir = gr.Textbox( - label='Training config folder', - placeholder='folder where the training configuration files will be saved', + label="Training config folder", + placeholder="folder where the training configuration files will be saved", ) train_dir_folder = gr.Button( folder_symbol, - elem_id='open_folder_small', + elem_id="open_folder_small", visible=(not headless), ) train_dir_folder.click( @@ -724,12 +724,12 @@ def finetune_tab(headless=False): ) image_folder = gr.Textbox( - label='Training Image folder', - placeholder='folder where the training images are located', + label="Training Image folder", + placeholder="folder where the training images are located", ) image_folder_input_folder = gr.Button( folder_symbol, - elem_id='open_folder_small', + elem_id="open_folder_small", visible=(not headless), ) image_folder_input_folder.click( @@ -739,12 +739,12 @@ def finetune_tab(headless=False): ) with gr.Row(): output_dir = gr.Textbox( - label='Model output folder', - placeholder='folder where the model will be saved', + label="Model output folder", + placeholder="folder where the model will be saved", ) output_dir_input_folder = gr.Button( folder_symbol, - elem_id='open_folder_small', + elem_id="open_folder_small", visible=(not headless), ) output_dir_input_folder.click( @@ -754,12 +754,12 @@ def finetune_tab(headless=False): ) logging_dir = gr.Textbox( - label='Logging folder', - placeholder='Optional: enable logging and output TensorBoard log to this folder', + label="Logging folder", + placeholder="Optional: enable logging and output TensorBoard log to this folder", ) logging_dir_input_folder = gr.Button( folder_symbol, - elem_id='open_folder_small', + elem_id="open_folder_small", visible=(not headless), ) logging_dir_input_folder.click( @@ -769,9 +769,9 @@ def finetune_tab(headless=False): ) with gr.Row(): output_name = gr.Textbox( - label='Model output name', - placeholder='Name of the model to output', - value='last', + label="Model output name", + placeholder="Name of the model to output", + value="last", interactive=True, ) train_dir.change( @@ -789,102 +789,96 @@ def finetune_tab(headless=False): inputs=[output_dir], outputs=[output_dir], ) - with gr.Tab('Dataset preparation'): + with gr.Tab("Dataset preparation"): with gr.Row(): max_resolution = gr.Textbox( - label='Resolution (width,height)', value='512,512' - ) - min_bucket_reso = gr.Textbox( - label='Min bucket resolution', value='256' + label="Resolution (width,height)", value="512,512" ) + min_bucket_reso = gr.Textbox(label="Min bucket resolution", value="256") max_bucket_reso = gr.Textbox( - label='Max bucket resolution', value='1024' + label="Max bucket resolution", value="1024" ) - batch_size = gr.Textbox(label='Batch size', value='1') + batch_size = gr.Textbox(label="Batch size", value="1") with gr.Row(): create_caption = gr.Checkbox( - label='Generate caption metadata', value=True + label="Generate caption metadata", value=True ) create_buckets = gr.Checkbox( - label='Generate image buckets metadata', value=True + label="Generate image buckets metadata", value=True ) use_latent_files = gr.Dropdown( - label='Use latent files', + label="Use latent files", choices=[ - 'No', - 'Yes', + "No", + "Yes", ], - value='Yes', + value="Yes", ) - with gr.Accordion('Advanced parameters', open=False): + with gr.Accordion("Advanced parameters", open=False): with gr.Row(): caption_metadata_filename = gr.Textbox( - label='Caption metadata filename', - value='meta_cap.json', + label="Caption metadata filename", + value="meta_cap.json", ) latent_metadata_filename = gr.Textbox( - label='Latent metadata filename', value='meta_lat.json' + label="Latent metadata filename", value="meta_lat.json" ) with gr.Row(): - full_path = gr.Checkbox(label='Use full path', value=True) + full_path = gr.Checkbox(label="Use full path", value=True) weighted_captions = gr.Checkbox( - label='Weighted captions', value=False + label="Weighted captions", value=False ) - with gr.Tab('Parameters'): - + with gr.Tab("Parameters"): + def list_presets(path): json_files = [] for file in os.listdir(path): - if file.endswith('.json'): + if file.endswith(".json"): json_files.append(os.path.splitext(file)[0]) - user_presets_path = os.path.join(path, 'user_presets') + user_presets_path = os.path.join(path, "user_presets") if os.path.isdir(user_presets_path): for file in os.listdir(user_presets_path): - if file.endswith('.json'): + if file.endswith(".json"): preset_name = os.path.splitext(file)[0] - json_files.append( - os.path.join('user_presets', preset_name) - ) + json_files.append(os.path.join("user_presets", preset_name)) return json_files training_preset = gr.Dropdown( - label='Presets', - choices=list_presets('./presets/finetune'), - elem_id='myDropdown', + label="Presets", + choices=list_presets("./presets/finetune"), + elem_id="myDropdown", ) - - with gr.Tab('Basic', elem_id='basic_tab'): + + with gr.Tab("Basic", elem_id="basic_tab"): basic_training = BasicTraining( - learning_rate_value='1e-5', finetuning=True, sdxl_checkbox=source_model.sdxl_checkbox, + learning_rate_value="1e-5", + finetuning=True, + sdxl_checkbox=source_model.sdxl_checkbox, ) # Add SDXL Parameters sdxl_params = SDXLParameters(source_model.sdxl_checkbox) with gr.Row(): - dataset_repeats = gr.Textbox( - label='Dataset repeats', value=40 - ) + dataset_repeats = gr.Textbox(label="Dataset repeats", value=40) train_text_encoder = gr.Checkbox( - label='Train text encoder', value=True + label="Train text encoder", value=True ) - with gr.Tab('Advanced', elem_id='advanced_tab'): + with gr.Tab("Advanced", elem_id="advanced_tab"): with gr.Row(): gradient_accumulation_steps = gr.Number( - label='Gradient accumulate steps', value='1' + label="Gradient accumulate steps", value="1" ) block_lr = gr.Textbox( - label='Block LR', - placeholder='(Optional)', - info='Specify the different learning rates for each U-Net block. Specify 23 values separated by commas like 1e-3,1e-3 ... 1e-3', + label="Block LR", + placeholder="(Optional)", + info="Specify the different learning rates for each U-Net block. Specify 23 values separated by commas like 1e-3,1e-3 ... 1e-3", ) - advanced_training = AdvancedTraining( - headless=headless, finetuning=True - ) + advanced_training = AdvancedTraining(headless=headless, finetuning=True) advanced_training.color_aug.change( color_aug_changed, inputs=[advanced_training.color_aug], @@ -893,15 +887,15 @@ def list_presets(path): ], # Not applicable to fine_tune.py ) - with gr.Tab('Samples', elem_id='samples_tab'): + with gr.Tab("Samples", elem_id="samples_tab"): sample = SampleImages() with gr.Row(): - button_run = gr.Button('Start training', variant='primary') + button_run = gr.Button("Start training", variant="primary") - button_stop_training = gr.Button('Stop training') + button_stop_training = gr.Button("Stop training") - button_print = gr.Button('Print training command') + button_print = gr.Button("Print training command") # Setup gradio tensorboard buttons ( @@ -1020,9 +1014,7 @@ def list_presets(path): inputs=[dummy_db_true, dummy_db_false, config.config_file_name] + settings_list + [training_preset], - outputs=[config.config_file_name] - + settings_list - + [training_preset], + outputs=[config.config_file_name] + settings_list + [training_preset], show_progress=False, ) @@ -1038,9 +1030,7 @@ def list_presets(path): inputs=[dummy_db_false, dummy_db_false, config.config_file_name] + settings_list + [training_preset], - outputs=[config.config_file_name] - + settings_list - + [training_preset], + outputs=[config.config_file_name] + settings_list + [training_preset], show_progress=False, ) @@ -1056,9 +1046,7 @@ def list_presets(path): inputs=[dummy_db_false, dummy_db_true, config.config_file_name] + settings_list + [training_preset], - outputs=[gr.Textbox()] - + settings_list - + [training_preset], + outputs=[gr.Textbox()] + settings_list + [training_preset], show_progress=False, ) @@ -1090,94 +1078,84 @@ def list_presets(path): show_progress=False, ) - with gr.Tab('Guides'): - gr.Markdown( - 'This section provide Various Finetuning guides and information...' - ) - top_level_path = './docs/Finetuning/top_level.md' + with gr.Tab("Guides"): + gr.Markdown("This section provide Various Finetuning guides and information...") + top_level_path = "./docs/Finetuning/top_level.md" if os.path.exists(top_level_path): - with open( - os.path.join(top_level_path), 'r', encoding='utf8' - ) as file: - guides_top_level = file.read() + '\n' + with open(os.path.join(top_level_path), "r", encoding="utf8") as file: + guides_top_level = file.read() + "\n" gr.Markdown(guides_top_level) def UI(**kwargs): - add_javascript(kwargs.get('language')) - css = '' + add_javascript(kwargs.get("language")) + css = "" - headless = kwargs.get('headless', False) - log.info(f'headless: {headless}') + headless = kwargs.get("headless", False) + log.info(f"headless: {headless}") - if os.path.exists('./style.css'): - with open(os.path.join('./style.css'), 'r', encoding='utf8') as file: - log.info('Load CSS...') - css += file.read() + '\n' + if os.path.exists("./style.css"): + with open(os.path.join("./style.css"), "r", encoding="utf8") as file: + log.info("Load CSS...") + css += file.read() + "\n" - interface = gr.Blocks( - css=css, title='Kohya_ss GUI', theme=gr.themes.Default() - ) + interface = gr.Blocks(css=css, title="Kohya_ss GUI", theme=gr.themes.Default()) with interface: - with gr.Tab('Finetune'): + with gr.Tab("Finetune"): finetune_tab(headless=headless) - with gr.Tab('Utilities'): + with gr.Tab("Utilities"): utilities_tab(enable_dreambooth_tab=False, headless=headless) # Show the interface launch_kwargs = {} - username = kwargs.get('username') - password = kwargs.get('password') - server_port = kwargs.get('server_port', 0) - inbrowser = kwargs.get('inbrowser', False) - share = kwargs.get('share', False) - server_name = kwargs.get('listen') - - launch_kwargs['server_name'] = server_name + username = kwargs.get("username") + password = kwargs.get("password") + server_port = kwargs.get("server_port", 0) + inbrowser = kwargs.get("inbrowser", False) + share = kwargs.get("share", False) + server_name = kwargs.get("listen") + + launch_kwargs["server_name"] = server_name if username and password: - launch_kwargs['auth'] = (username, password) + launch_kwargs["auth"] = (username, password) if server_port > 0: - launch_kwargs['server_port'] = server_port + launch_kwargs["server_port"] = server_port if inbrowser: - launch_kwargs['inbrowser'] = inbrowser + launch_kwargs["inbrowser"] = inbrowser if share: - launch_kwargs['share'] = share + launch_kwargs["share"] = share interface.launch(**launch_kwargs) -if __name__ == '__main__': +if __name__ == "__main__": # torch.cuda.set_per_process_memory_fraction(0.48) parser = argparse.ArgumentParser() parser.add_argument( - '--listen', + "--listen", type=str, - default='127.0.0.1', - help='IP to listen on for connections to Gradio', + default="127.0.0.1", + help="IP to listen on for connections to Gradio", ) parser.add_argument( - '--username', type=str, default='', help='Username for authentication' + "--username", type=str, default="", help="Username for authentication" ) parser.add_argument( - '--password', type=str, default='', help='Password for authentication' + "--password", type=str, default="", help="Password for authentication" ) parser.add_argument( - '--server_port', + "--server_port", type=int, default=0, - help='Port to run the server listener on', - ) - parser.add_argument( - '--inbrowser', action='store_true', help='Open in browser' - ) - parser.add_argument( - '--share', action='store_true', help='Share the gradio UI' + help="Port to run the server listener on", ) + parser.add_argument("--inbrowser", action="store_true", help="Open in browser") + parser.add_argument("--share", action="store_true", help="Share the gradio UI") parser.add_argument( - '--headless', action='store_true', help='Is the server headless' + "--headless", action="store_true", help="Is the server headless" ) parser.add_argument( - '--language', type=str, default=None, help='Set custom language' + "--language", type=str, default=None, help="Set custom language" ) args = parser.parse_args() diff --git a/library/common_gui.py b/library/common_gui.py index aa8dabaa1..a0a0c5767 100644 --- a/library/common_gui.py +++ b/library/common_gui.py @@ -678,11 +678,11 @@ def get_str_or_default(kwargs, key, default_value=""): def run_cmd_advanced_training(**kwargs): run_cmd = "" - + additional_parameters = kwargs.get("additional_parameters") if additional_parameters: run_cmd += f" {additional_parameters}" - + block_lr = kwargs.get("block_lr") if block_lr: run_cmd += f' --block_lr="(block_lr)"' @@ -702,7 +702,7 @@ def run_cmd_advanced_training(**kwargs): cache_latents_to_disk = kwargs.get("cache_latents_to_disk") if cache_latents_to_disk: run_cmd += " --cache_latents_to_disk" - + cache_text_encoder_outputs = kwargs.get("cache_text_encoder_outputs") if cache_text_encoder_outputs: run_cmd += " --cache_text_encoder_outputs" @@ -728,18 +728,18 @@ def run_cmd_advanced_training(**kwargs): color_aug = kwargs.get("color_aug") if color_aug: run_cmd += " --color_aug" - + dataset_repeats = kwargs.get("dataset_repeats") if dataset_repeats: run_cmd += f' --dataset_repeats="{dataset_repeats}"' - + enable_bucket = kwargs.get("enable_bucket") if enable_bucket: min_bucket_reso = kwargs.get("min_bucket_reso") max_bucket_reso = kwargs.get("max_bucket_reso") if min_bucket_reso and max_bucket_reso: run_cmd += f" --enable_bucket --min_bucket_reso={min_bucket_reso} --max_bucket_reso={max_bucket_reso}" - + in_json = kwargs.get("in_json") if in_json: run_cmd += f' --in_json="{in_json}"' @@ -751,7 +751,7 @@ def run_cmd_advanced_training(**kwargs): fp8_base = kwargs.get("fp8_base") if fp8_base: run_cmd += " --fp8_base" - + full_bf16 = kwargs.get("full_bf16") if full_bf16: run_cmd += " --full_bf16" @@ -759,7 +759,7 @@ def run_cmd_advanced_training(**kwargs): full_fp16 = kwargs.get("full_fp16") if full_fp16: run_cmd += " --full_fp16" - + gradient_accumulation_steps = kwargs.get("gradient_accumulation_steps") if gradient_accumulation_steps and int(gradient_accumulation_steps) > 1: run_cmd += f" --gradient_accumulation_steps={int(gradient_accumulation_steps)}" @@ -775,19 +775,19 @@ def run_cmd_advanced_training(**kwargs): learning_rate = kwargs.get("learning_rate") if learning_rate: run_cmd += f' --learning_rate="{learning_rate}"' - + learning_rate_te = kwargs.get("learning_rate_te") if learning_rate_te: run_cmd += f' --learning_rate_te="{learning_rate_te}"' - + learning_rate_te1 = kwargs.get("learning_rate_te1") if learning_rate_te1: run_cmd += f' --learning_rate_te1="{learning_rate_te1}"' - + learning_rate_te2 = kwargs.get("learning_rate_te2") if learning_rate_te2: run_cmd += f' --learning_rate_te2="{learning_rate_te2}"' - + logging_dir = kwargs.get("logging_dir") if logging_dir: run_cmd += f' --logging_dir="{logging_dir}"' @@ -799,7 +799,7 @@ def run_cmd_advanced_training(**kwargs): lr_scheduler_args = kwargs.get("lr_scheduler_args") if lr_scheduler_args and lr_scheduler_args != "": run_cmd += f" --lr_scheduler_args {lr_scheduler_args}" - + lr_scheduler_num_cycles = kwargs.get("lr_scheduler_num_cycles") if lr_scheduler_num_cycles and not lr_scheduler_num_cycles == "": run_cmd += f' --lr_scheduler_num_cycles="{lr_scheduler_num_cycles}"' @@ -807,7 +807,7 @@ def run_cmd_advanced_training(**kwargs): epoch = kwargs.get("epoch") if epoch: run_cmd += f' --lr_scheduler_num_cycles="{epoch}"' - + lr_scheduler_power = kwargs.get("lr_scheduler_power") if lr_scheduler_power and not lr_scheduler_power == "": run_cmd += f' --lr_scheduler_power="{lr_scheduler_power}"' @@ -830,7 +830,7 @@ def run_cmd_advanced_training(**kwargs): max_grad_norm = kwargs.get("max_grad_norm") if max_grad_norm and max_grad_norm != "": run_cmd += f' --max_grad_norm="{max_grad_norm}"' - + max_resolution = kwargs.get("max_resolution") if max_resolution: run_cmd += f' --resolution="{max_resolution}"' @@ -844,7 +844,7 @@ def run_cmd_advanced_training(**kwargs): run_cmd += f" --max_token_length={int(max_token_length)}" max_train_epochs = kwargs.get("max_train_epochs") - if max_train_epochs and not max_train_epochs == '': + if max_train_epochs and not max_train_epochs == "": run_cmd += f" --max_train_epochs={max_train_epochs}" max_train_steps = kwargs.get("max_train_steps") @@ -870,15 +870,15 @@ def run_cmd_advanced_training(**kwargs): multi_gpu = kwargs.get("multi_gpu") if multi_gpu: run_cmd += " --multi_gpu" - + no_half_vae = kwargs.get("no_half_vae") if no_half_vae: run_cmd += " --no_half_vae" - + no_token_padding = kwargs.get("no_token_padding") if no_token_padding: run_cmd += " --no_token_padding" - + noise_offset_type = kwargs.get("noise_offset_type") if noise_offset_type and noise_offset_type == "Original": noise_offset = kwargs.get("noise_offset") @@ -886,17 +886,23 @@ def run_cmd_advanced_training(**kwargs): run_cmd += f" --noise_offset={float(noise_offset)}" adaptive_noise_scale = kwargs.get("adaptive_noise_scale") - if adaptive_noise_scale and float(adaptive_noise_scale) != 0 and float(noise_offset) > 0: + if ( + adaptive_noise_scale + and float(adaptive_noise_scale) != 0 + and float(noise_offset) > 0 + ): run_cmd += f" --adaptive_noise_scale={float(adaptive_noise_scale)}" elif noise_offset_type and noise_offset_type == "Multires": multires_noise_iterations = kwargs.get("multires_noise_iterations") if int(multires_noise_iterations) > 0: - run_cmd += f' --multires_noise_iterations="{int(multires_noise_iterations)}"' + run_cmd += ( + f' --multires_noise_iterations="{int(multires_noise_iterations)}"' + ) multires_noise_discount = kwargs.get("multires_noise_discount") if multires_noise_discount and float(multires_noise_discount) > 0: run_cmd += f' --multires_noise_discount="{float(multires_noise_discount)}"' - + num_machines = kwargs.get("num_machines") if num_machines and int(num_machines) > 1: run_cmd += f" --num_machines={int(num_machines)}" @@ -916,12 +922,12 @@ def run_cmd_advanced_training(**kwargs): optimizer_type = kwargs.get("optimizer") if optimizer_type: run_cmd += f' --optimizer_type="{optimizer_type}"' - + output_dir = kwargs.get("output_dir") if output_dir: run_cmd += f' --output_dir="{output_dir}"' - - output_name = kwargs.get("output_name") + + output_name = kwargs.get("output_name") if output_name and not output_name == "": run_cmd += f' --output_name="{output_name}"' @@ -932,7 +938,7 @@ def run_cmd_advanced_training(**kwargs): pretrained_model_name_or_path = kwargs.get("pretrained_model_name_or_path") if pretrained_model_name_or_path: run_cmd += f' --pretrained_model_name_or_path="{pretrained_model_name_or_path}"' - + prior_loss_weight = kwargs.get("prior_loss_weight") if prior_loss_weight and not float(prior_loss_weight) == 1.0: run_cmd += f" --prior_loss_weight={prior_loss_weight}" @@ -964,7 +970,7 @@ def run_cmd_advanced_training(**kwargs): save_last_n_steps_state = kwargs.get("save_last_n_steps_state") if save_last_n_steps_state and int(save_last_n_steps_state) > 0: run_cmd += f' --save_last_n_steps_state="{int(save_last_n_steps_state)}"' - + save_model_as = kwargs.get("save_model_as") if save_model_as and not save_model_as == "same as source model": run_cmd += f" --save_model_as={save_model_as}" @@ -988,7 +994,7 @@ def run_cmd_advanced_training(**kwargs): shuffle_caption = kwargs.get("shuffle_caption") if shuffle_caption: run_cmd += " --shuffle_caption" - + stop_text_encoder_training = kwargs.get("stop_text_encoder_training") if stop_text_encoder_training and stop_text_encoder_training > 0: run_cmd += f' --stop_text_encoder_training="{stop_text_encoder_training}"' @@ -996,11 +1002,11 @@ def run_cmd_advanced_training(**kwargs): train_batch_size = kwargs.get("train_batch_size") if train_batch_size: run_cmd += f' --train_batch_size="{train_batch_size}"' - + train_data_dir = kwargs.get("train_data_dir") if train_data_dir: run_cmd += f' --train_data_dir="{train_data_dir}"' - + train_text_encoder = kwargs.get("train_text_encoder") if train_text_encoder: run_cmd += " --train_text_encoder" @@ -1008,7 +1014,7 @@ def run_cmd_advanced_training(**kwargs): use_wandb = kwargs.get("use_wandb") if use_wandb: run_cmd += " --log_with wandb" - + v_parameterization = kwargs.get("v_parameterization") if v_parameterization: run_cmd += " --v_parameterization" @@ -1016,7 +1022,7 @@ def run_cmd_advanced_training(**kwargs): v_pred_like_loss = kwargs.get("v_pred_like_loss") if v_pred_like_loss and float(v_pred_like_loss) > 0: run_cmd += f' --v_pred_like_loss="{float(v_pred_like_loss)}"' - + v2 = kwargs.get("v2") if v2: run_cmd += " --v2" @@ -1032,7 +1038,7 @@ def run_cmd_advanced_training(**kwargs): wandb_api_key = kwargs.get("wandb_api_key") if wandb_api_key: run_cmd += f' --wandb_api_key="{wandb_api_key}"' - + weighted_captions = kwargs.get("weighted_captions") if weighted_captions: run_cmd += " --weighted_captions" diff --git a/lora_gui.py b/lora_gui.py index d5913d480..4e1ef57bc 100644 --- a/lora_gui.py +++ b/lora_gui.py @@ -732,14 +732,15 @@ def train_model( log.info(f"lr_warmup_steps = {lr_warmup_steps}") run_cmd = "accelerate launch" - + run_cmd += run_cmd_advanced_training( num_processes=num_processes, num_machines=num_machines, multi_gpu=multi_gpu, gpu_ids=gpu_ids, - num_cpu_threads_per_process=num_cpu_threads_per_process) - + num_cpu_threads_per_process=num_cpu_threads_per_process, + ) + if sdxl: run_cmd += f' "./sdxl_train_network.py"' else: @@ -1803,7 +1804,9 @@ def update_LoRA_settings( placeholder="(Optional) eg: 2,2,2,2,4,4,4,4,6,6,6,6,8,6,6,6,6,4,4,4,4,2,2,2,2", info="Specify the alpha of each block when expanding LoRA to Conv2d 3x3. Specify 25 numbers. If omitted, the value of conv_alpha is used.", ) - advanced_training = AdvancedTraining(headless=headless, training_type="lora") + advanced_training = AdvancedTraining( + headless=headless, training_type="lora" + ) advanced_training.color_aug.change( color_aug_changed, inputs=[advanced_training.color_aug], diff --git a/textual_inversion_gui.py b/textual_inversion_gui.py index d54adb85b..b609b9171 100644 --- a/textual_inversion_gui.py +++ b/textual_inversion_gui.py @@ -153,19 +153,19 @@ def save_configuration( original_file_path = file_path - save_as_bool = True if save_as.get('label') == 'True' else False + save_as_bool = True if save_as.get("label") == "True" else False if save_as_bool: - log.info('Save as...') + log.info("Save as...") file_path = get_saveasfile_path(file_path) else: - log.info('Save...') - if file_path == None or file_path == '': + log.info("Save...") + if file_path == None or file_path == "": file_path = get_saveasfile_path(file_path) # log.info(file_path) - if file_path == None or file_path == '': + if file_path == None or file_path == "": return original_file_path # In case a file_path was provided and the user decide to cancel the open action # Extract the destination directory from the file path @@ -178,7 +178,7 @@ def save_configuration( SaveConfigFile( parameters=parameters, file_path=file_path, - exclusion=['file_path', 'save_as'], + exclusion=["file_path", "save_as"], ) return file_path @@ -282,18 +282,18 @@ def open_configuration( # Get list of function parameters and values parameters = list(locals().items()) - ask_for_file = True if ask_for_file.get('label') == 'True' else False + ask_for_file = True if ask_for_file.get("label") == "True" else False original_file_path = file_path if ask_for_file: file_path = get_file_path(file_path) - if not file_path == '' and not file_path == None: + if not file_path == "" and not file_path == None: # load variables from JSON file - with open(file_path, 'r') as f: + with open(file_path, "r") as f: my_data = json.load(f) - log.info('Loading config...') + log.info("Loading config...") # Update values to fix deprecated use_8bit_adam checkbox and set appropriate optimizer if it is set to True my_data = update_my_data(my_data) else: @@ -303,7 +303,7 @@ def open_configuration( values = [file_path] for key, value in parameters: # Set the value in the dictionary to the corresponding value in `my_data`, or the default value if not found - if not key in ['ask_for_file', 'file_path']: + if not key in ["ask_for_file", "file_path"]: values.append(my_data.get(key, value)) return tuple(values) @@ -406,36 +406,32 @@ def train_model( # Get list of function parameters and values parameters = list(locals().items()) - print_only_bool = True if print_only.get('label') == 'True' else False - log.info(f'Start training TI...') + print_only_bool = True if print_only.get("label") == "True" else False + log.info(f"Start training TI...") - headless_bool = True if headless.get('label') == 'True' else False + headless_bool = True if headless.get("label") == "True" else False - if pretrained_model_name_or_path == '': + if pretrained_model_name_or_path == "": output_message( - msg='Source model information is missing', headless=headless_bool + msg="Source model information is missing", headless=headless_bool ) return - if train_data_dir == '': - output_message( - msg='Image folder path is missing', headless=headless_bool - ) + if train_data_dir == "": + output_message(msg="Image folder path is missing", headless=headless_bool) return if not os.path.exists(train_data_dir): - output_message( - msg='Image folder does not exist', headless=headless_bool - ) + output_message(msg="Image folder does not exist", headless=headless_bool) return if not verify_image_folder_pattern(train_data_dir): return - if reg_data_dir != '': + if reg_data_dir != "": if not os.path.exists(reg_data_dir): output_message( - msg='Regularisation folder does not exist', + msg="Regularisation folder does not exist", headless=headless_bool, ) return @@ -443,26 +439,22 @@ def train_model( if not verify_image_folder_pattern(reg_data_dir): return - if output_dir == '': - output_message( - msg='Output folder path is missing', headless=headless_bool - ) + if output_dir == "": + output_message(msg="Output folder path is missing", headless=headless_bool) return - if token_string == '': - output_message(msg='Token string is missing', headless=headless_bool) + if token_string == "": + output_message(msg="Token string is missing", headless=headless_bool) return - if init_word == '': - output_message(msg='Init word is missing', headless=headless_bool) + if init_word == "": + output_message(msg="Init word is missing", headless=headless_bool) return if not os.path.exists(output_dir): os.makedirs(output_dir) - if check_if_model_exist( - output_name, output_dir, save_model_as, headless_bool - ): + if check_if_model_exist(output_name, output_dir, save_model_as, headless_bool): return # if float(noise_offset) > 0 and ( @@ -495,7 +487,7 @@ def train_model( # Loop through each subfolder and extract the number of repeats for folder in subfolders: # Extract the number of repeats from the folder name - repeats = int(folder.split('_')[0]) + repeats = int(folder.split("_")[0]) # Count the number of images in the folder num_images = len( @@ -503,11 +495,9 @@ def train_model( f for f, lower_f in ( (file, file.lower()) - for file in os.listdir( - os.path.join(train_data_dir, folder) - ) + for file in os.listdir(os.path.join(train_data_dir, folder)) ) - if lower_f.endswith(('.jpg', '.jpeg', '.png', '.webp')) + if lower_f.endswith((".jpg", ".jpeg", ".png", ".webp")) ] ) @@ -516,21 +506,21 @@ def train_model( total_steps += steps # Print the result - log.info(f'Folder {folder}: {steps} steps') + log.info(f"Folder {folder}: {steps} steps") # Print the result # log.info(f"{total_steps} total steps") - if reg_data_dir == '': + if reg_data_dir == "": reg_factor = 1 else: log.info( - 'Regularisation images are used... Will double the number of steps required...' + "Regularisation images are used... Will double the number of steps required..." ) reg_factor = 2 # calculate max_train_steps - if max_train_steps == '' or max_train_steps == '0': + if max_train_steps == "" or max_train_steps == "0": max_train_steps = int( math.ceil( float(total_steps) @@ -543,7 +533,7 @@ def train_model( else: max_train_steps = int(max_train_steps) - log.info(f'max_train_steps = {max_train_steps}') + log.info(f"max_train_steps = {max_train_steps}") # calculate stop encoder training if stop_text_encoder_training_pct == None: @@ -552,20 +542,21 @@ def train_model( stop_text_encoder_training = math.ceil( float(max_train_steps) / 100 * int(stop_text_encoder_training_pct) ) - log.info(f'stop_text_encoder_training = {stop_text_encoder_training}') + log.info(f"stop_text_encoder_training = {stop_text_encoder_training}") lr_warmup_steps = round(float(int(lr_warmup) * int(max_train_steps) / 100)) - log.info(f'lr_warmup_steps = {lr_warmup_steps}') + log.info(f"lr_warmup_steps = {lr_warmup_steps}") run_cmd = "accelerate launch" - + run_cmd += run_cmd_advanced_training( num_processes=num_processes, num_machines=num_machines, multi_gpu=multi_gpu, gpu_ids=gpu_ids, - num_cpu_threads_per_process=num_cpu_threads_per_process) - + num_cpu_threads_per_process=num_cpu_threads_per_process, + ) + if sdxl: run_cmd += f' "./sdxl_train_textual_inversion.py"' else: @@ -649,13 +640,13 @@ def train_model( ) run_cmd += f' --token_string="{token_string}"' run_cmd += f' --init_word="{init_word}"' - run_cmd += f' --num_vectors_per_token={num_vectors_per_token}' - if not weights == '': + run_cmd += f" --num_vectors_per_token={num_vectors_per_token}" + if not weights == "": run_cmd += f' --weights="{weights}"' - if template == 'object template': - run_cmd += f' --use_object_template' - elif template == 'style template': - run_cmd += f' --use_style_template' + if template == "object template": + run_cmd += f" --use_object_template" + elif template == "style template": + run_cmd += f" --use_style_template" run_cmd += run_cmd_sample( sample_every_n_steps, @@ -667,7 +658,7 @@ def train_model( if print_only_bool: log.warning( - 'Here is the trainer command as a reference. It will not be executed:\n' + "Here is the trainer command as a reference. It will not be executed:\n" ) print(run_cmd) @@ -675,17 +666,15 @@ def train_model( else: # Saving config file for model current_datetime = datetime.now() - formatted_datetime = current_datetime.strftime('%Y%m%d-%H%M%S') - file_path = os.path.join( - output_dir, f'{output_name}_{formatted_datetime}.json' - ) + formatted_datetime = current_datetime.strftime("%Y%m%d-%H%M%S") + file_path = os.path.join(output_dir, f"{output_name}_{formatted_datetime}.json") - log.info(f'Saving training config to {file_path}...') + log.info(f"Saving training config to {file_path}...") SaveConfigFile( parameters=parameters, file_path=file_path, - exclusion=['file_path', 'save_as', 'headless', 'print_only'], + exclusion=["file_path", "save_as", "headless", "print_only"], ) log.info(run_cmd) @@ -695,13 +684,11 @@ def train_model( executor.execute_command(run_cmd=run_cmd) # check if output_dir/last is a folder... therefore it is a diffuser model - last_dir = pathlib.Path(f'{output_dir}/{output_name}') + last_dir = pathlib.Path(f"{output_dir}/{output_name}") if not last_dir.is_dir(): # Copy inference model for v2 if required - save_inference_file( - output_dir, v2, v_parameterization, output_name - ) + save_inference_file(output_dir, v2, v_parameterization, output_name) def ti_tab( @@ -711,32 +698,32 @@ def ti_tab( dummy_db_false = gr.Label(value=False, visible=False) dummy_headless = gr.Label(value=headless, visible=False) - with gr.Tab('Training'): - gr.Markdown('Train a TI using kohya textual inversion python code...') + with gr.Tab("Training"): + gr.Markdown("Train a TI using kohya textual inversion python code...") # Setup Configuration Files Gradio config = ConfigurationFile(headless) source_model = SourceModel( save_model_as_choices=[ - 'ckpt', - 'safetensors', + "ckpt", + "safetensors", ], headless=headless, ) - with gr.Tab('Folders'): + with gr.Tab("Folders"): folders = Folders(headless=headless) - with gr.Tab('Parameters'): - with gr.Tab('Basic', elem_id='basic_tab'): + with gr.Tab("Parameters"): + with gr.Tab("Basic", elem_id="basic_tab"): with gr.Row(): weights = gr.Textbox( - label='Resume TI training', - placeholder='(Optional) Path to existing TI embeding file to keep training', + label="Resume TI training", + placeholder="(Optional) Path to existing TI embeding file to keep training", ) weights_file_input = gr.Button( - '📂', - elem_id='open_folder_small', + "", + elem_id="open_folder_small", visible=(not headless), ) weights_file_input.click( @@ -746,37 +733,37 @@ def ti_tab( ) with gr.Row(): token_string = gr.Textbox( - label='Token string', - placeholder='eg: cat', + label="Token string", + placeholder="eg: cat", ) init_word = gr.Textbox( - label='Init word', - value='*', + label="Init word", + value="*", ) num_vectors_per_token = gr.Slider( minimum=1, maximum=75, value=1, step=1, - label='Vectors', + label="Vectors", ) # max_train_steps = gr.Textbox( # label='Max train steps', # placeholder='(Optional) Maximum number of steps', # ) template = gr.Dropdown( - label='Template', + label="Template", choices=[ - 'caption', - 'object template', - 'style template', + "caption", + "object template", + "style template", ], - value='caption', + value="caption", ) basic_training = BasicTraining( - learning_rate_value='1e-5', - lr_scheduler_value='cosine', - lr_warmup_value='10', + learning_rate_value="1e-5", + lr_scheduler_value="cosine", + lr_warmup_value="10", sdxl_checkbox=source_model.sdxl_checkbox, ) @@ -786,7 +773,7 @@ def ti_tab( show_sdxl_cache_text_encoder_outputs=False, ) - with gr.Tab('Advanced', elem_id='advanced_tab'): + with gr.Tab("Advanced", elem_id="advanced_tab"): advanced_training = AdvancedTraining(headless=headless) advanced_training.color_aug.change( color_aug_changed, @@ -794,12 +781,12 @@ def ti_tab( outputs=[basic_training.cache_latents], ) - with gr.Tab('Samples', elem_id='samples_tab'): + with gr.Tab("Samples", elem_id="samples_tab"): sample = SampleImages() - with gr.Tab('Dataset Preparation'): + with gr.Tab("Dataset Preparation"): gr.Markdown( - 'This section provide Dreambooth tools to help setup your dataset...' + "This section provide Dreambooth tools to help setup your dataset..." ) gradio_dreambooth_folder_creation_tab( train_data_dir_input=folders.train_data_dir, @@ -811,11 +798,11 @@ def ti_tab( gradio_dataset_balancing_tab(headless=headless) with gr.Row(): - button_run = gr.Button('Start training', variant='primary') + button_run = gr.Button("Start training", variant="primary") - button_stop_training = gr.Button('Stop training') + button_stop_training = gr.Button("Stop training") - button_print = gr.Button('Print training command') + button_print = gr.Button("Print training command") # Setup gradio tensorboard buttons ( @@ -978,30 +965,28 @@ def ti_tab( def UI(**kwargs): - add_javascript(kwargs.get('language')) - css = '' + add_javascript(kwargs.get("language")) + css = "" - headless = kwargs.get('headless', False) - log.info(f'headless: {headless}') + headless = kwargs.get("headless", False) + log.info(f"headless: {headless}") - if os.path.exists('./style.css'): - with open(os.path.join('./style.css'), 'r', encoding='utf8') as file: - log.info('Load CSS...') - css += file.read() + '\n' + if os.path.exists("./style.css"): + with open(os.path.join("./style.css"), "r", encoding="utf8") as file: + log.info("Load CSS...") + css += file.read() + "\n" - interface = gr.Blocks( - css=css, title='Kohya_ss GUI', theme=gr.themes.Default() - ) + interface = gr.Blocks(css=css, title="Kohya_ss GUI", theme=gr.themes.Default()) with interface: - with gr.Tab('Dreambooth TI'): + with gr.Tab("Dreambooth TI"): ( train_data_dir_input, reg_data_dir_input, output_dir_input, logging_dir_input, ) = ti_tab(headless=headless) - with gr.Tab('Utilities'): + with gr.Tab("Utilities"): utilities_tab( train_data_dir_input=train_data_dir_input, reg_data_dir_input=reg_data_dir_input, @@ -1013,57 +998,53 @@ def UI(**kwargs): # Show the interface launch_kwargs = {} - username = kwargs.get('username') - password = kwargs.get('password') - server_port = kwargs.get('server_port', 0) - inbrowser = kwargs.get('inbrowser', False) - share = kwargs.get('share', False) - server_name = kwargs.get('listen') - - launch_kwargs['server_name'] = server_name + username = kwargs.get("username") + password = kwargs.get("password") + server_port = kwargs.get("server_port", 0) + inbrowser = kwargs.get("inbrowser", False) + share = kwargs.get("share", False) + server_name = kwargs.get("listen") + + launch_kwargs["server_name"] = server_name if username and password: - launch_kwargs['auth'] = (username, password) + launch_kwargs["auth"] = (username, password) if server_port > 0: - launch_kwargs['server_port'] = server_port + launch_kwargs["server_port"] = server_port if inbrowser: - launch_kwargs['inbrowser'] = inbrowser + launch_kwargs["inbrowser"] = inbrowser if share: - launch_kwargs['share'] = share + launch_kwargs["share"] = share interface.launch(**launch_kwargs) -if __name__ == '__main__': +if __name__ == "__main__": # torch.cuda.set_per_process_memory_fraction(0.48) parser = argparse.ArgumentParser() parser.add_argument( - '--listen', + "--listen", type=str, - default='127.0.0.1', - help='IP to listen on for connections to Gradio', + default="127.0.0.1", + help="IP to listen on for connections to Gradio", ) parser.add_argument( - '--username', type=str, default='', help='Username for authentication' + "--username", type=str, default="", help="Username for authentication" ) parser.add_argument( - '--password', type=str, default='', help='Password for authentication' + "--password", type=str, default="", help="Password for authentication" ) parser.add_argument( - '--server_port', + "--server_port", type=int, default=0, - help='Port to run the server listener on', - ) - parser.add_argument( - '--inbrowser', action='store_true', help='Open in browser' - ) - parser.add_argument( - '--share', action='store_true', help='Share the gradio UI' + help="Port to run the server listener on", ) + parser.add_argument("--inbrowser", action="store_true", help="Open in browser") + parser.add_argument("--share", action="store_true", help="Share the gradio UI") parser.add_argument( - '--headless', action='store_true', help='Is the server headless' + "--headless", action="store_true", help="Is the server headless" ) parser.add_argument( - '--language', type=str, default=None, help='Set custom language' + "--language", type=str, default=None, help="Set custom language" ) args = parser.parse_args() From 40598ba4428bcd61f8f3b11f8bc30901c8113083 Mon Sep 17 00:00:00 2001 From: bmaltais Date: Sat, 3 Feb 2024 11:16:07 -0500 Subject: [PATCH 06/13] More code refactoring --- README.md | 1 + library/common_gui.py | 58 +++++++++++ lora_gui.py | 218 ++++++++++++++---------------------------- 3 files changed, 129 insertions(+), 148 deletions(-) diff --git a/README.md b/README.md index a9666f99c..a61e35a17 100644 --- a/README.md +++ b/README.md @@ -505,6 +505,7 @@ masterpiece, best quality, 1boy, in business suit, standing at street, looking b ## Change History * 2024/01/27 (v22.6.1) - Add support for multi-gpu parameters in the GUI under the "Parameters > Advanced" tab. +- Significant rewrite of how parameters are created in the code. I hope I did not break anything in the process... Will make the code easier to update. * 2024/01/27 (v22.6.0) - Merge sd-scripts v0.8.3 code update diff --git a/library/common_gui.py b/library/common_gui.py index a0a0c5767..ff290ad29 100644 --- a/library/common_gui.py +++ b/library/common_gui.py @@ -733,6 +733,16 @@ def run_cmd_advanced_training(**kwargs): if dataset_repeats: run_cmd += f' --dataset_repeats="{dataset_repeats}"' + debiased_estimation_loss = kwargs.get("debiased_estimation_loss") + if debiased_estimation_loss: + run_cmd += " --debiased_estimation_loss" + + dim_from_weights = kwargs.get("dim_from_weights") + if dim_from_weights and kwargs.get( + "lora_network_weights" + ): # Only if lora_network_weights is true + run_cmd += f" --dim_from_weights" + enable_bucket = kwargs.get("enable_bucket") if enable_bucket: min_bucket_reso = kwargs.get("min_bucket_reso") @@ -792,6 +802,10 @@ def run_cmd_advanced_training(**kwargs): if logging_dir: run_cmd += f' --logging_dir="{logging_dir}"' + lora_network_weights = kwargs.get("lora_network_weights") + if lora_network_weights: + run_cmd += f' --lora_network_weights="{lora_network_weights}"' + lr_scheduler = kwargs.get("lr_scheduler") if lr_scheduler: run_cmd += f' --lr_scheduler="{lr_scheduler}"' @@ -871,6 +885,34 @@ def run_cmd_advanced_training(**kwargs): if multi_gpu: run_cmd += " --multi_gpu" + network_alpha = kwargs.get("network_alpha") + if network_alpha: + run_cmd += f' --network_alpha="{network_alpha}"' + + network_args = kwargs.get("network_args") + if network_args and len(network_args): + run_cmd += f" --network_args{network_args}" + + network_dim = kwargs.get("network_dim") + if network_dim: + run_cmd += f" --network_dim={network_dim}" + + network_dropout = kwargs.get("network_dropout") + if network_dropout and network_dropout > 0.0: + run_cmd += f" --network_dropout={network_dropout}" + + network_module = kwargs.get("network_module") + if network_module: + run_cmd += f" --network_module={network_module}" + + network_train_text_encoder_only = kwargs.get("network_train_text_encoder_only") + if network_train_text_encoder_only: + run_cmd += " --network_train_text_encoder_only" + + network_train_unet_only = kwargs.get("network_train_unet_only") + if network_train_unet_only: + run_cmd += " --network_train_unet_only" + no_half_vae = kwargs.get("no_half_vae") if no_half_vae: run_cmd += " --no_half_vae" @@ -987,6 +1029,10 @@ def run_cmd_advanced_training(**kwargs): if scale_v_pred_loss_like_noise_pred: run_cmd += " --scale_v_pred_loss_like_noise_pred" + scale_weight_norms = kwargs.get("scale_weight_norms") + if scale_weight_norms and scale_weight_norms > 0.0: + run_cmd += f' --scale_weight_norms="{scale_weight_norms}"' + seed = kwargs.get("seed") if seed and seed != "": run_cmd += f' --seed="{seed}"' @@ -999,10 +1045,18 @@ def run_cmd_advanced_training(**kwargs): if stop_text_encoder_training and stop_text_encoder_training > 0: run_cmd += f' --stop_text_encoder_training="{stop_text_encoder_training}"' + text_encoder_lr = kwargs.get("text_encoder_lr") + if text_encoder_lr and (float(text_encoder_lr) > 0): + run_cmd += f" --text_encoder_lr={text_encoder_lr}" + train_batch_size = kwargs.get("train_batch_size") if train_batch_size: run_cmd += f' --train_batch_size="{train_batch_size}"' + training_comment = kwargs.get("training_comment") + if training_comment and len(training_comment): + run_cmd += f' --training_comment="{training_comment}"' + train_data_dir = kwargs.get("train_data_dir") if train_data_dir: run_cmd += f' --train_data_dir="{train_data_dir}"' @@ -1011,6 +1065,10 @@ def run_cmd_advanced_training(**kwargs): if train_text_encoder: run_cmd += " --train_text_encoder" + unet_lr = kwargs.get("unet_lr") + if unet_lr and (float(unet_lr) > 0): + run_cmd += f" --unet_lr={unet_lr}" + use_wandb = kwargs.get("use_wandb") if use_wandb: run_cmd += " --log_with wandb" diff --git a/lora_gui.py b/lora_gui.py index 4e1ef57bc..697fd3612 100644 --- a/lora_gui.py +++ b/lora_gui.py @@ -3,6 +3,7 @@ import math import os import argparse +import lycoris from datetime import datetime from library.common_gui import ( get_file_path, @@ -746,118 +747,37 @@ def train_model( else: run_cmd += f' "./train_network.py"' - run_cmd += f' --network_alpha="{network_alpha}"' - if not training_comment == "": - run_cmd += f' --training_comment="{training_comment}"' - if LoRA_type == "LyCORIS/Diag-OFT": - try: - import lycoris - except ModuleNotFoundError: - log.info( - "\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program." - ) - return - run_cmd += f" --network_module=lycoris.kohya" - run_cmd += f' --network_args "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "module_dropout={module_dropout}" "use_tucker={use_tucker}" "use_scalar={use_scalar}" "rank_dropout_scale={rank_dropout_scale}" "constrain={constrain}" "rescaled={rescaled}" "algo=diag-oft" ' - # This is a hack to fix a train_network LoHA logic issue - if not network_dropout > 0.0: - run_cmd += f' --network_dropout="{network_dropout}"' + network_module = "lycoris.kohya" + network_args = f' "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "module_dropout={module_dropout}" "use_tucker={use_tucker}" "use_scalar={use_scalar}" "rank_dropout_scale={rank_dropout_scale}" "constrain={constrain}" "rescaled={rescaled}" "algo=diag-oft" ' if LoRA_type == "LyCORIS/DyLoRA": - try: - import lycoris - except ModuleNotFoundError: - log.info( - "\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program." - ) - return - run_cmd += f" --network_module=lycoris.kohya" - run_cmd += f' --network_args "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "use_tucker={use_tucker}" "block_size={unit}" "rank_dropout={rank_dropout}" "module_dropout={module_dropout}" "algo=dylora" "train_norm={train_norm}"' - # This is a hack to fix a train_network LoHA logic issue - if not network_dropout > 0.0: - run_cmd += f' --network_dropout="{network_dropout}"' + network_module = "lycoris.kohya" + network_args = f' "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "use_tucker={use_tucker}" "block_size={unit}" "rank_dropout={rank_dropout}" "module_dropout={module_dropout}" "algo=dylora" "train_norm={train_norm}"' if LoRA_type == "LyCORIS/GLoRA": - try: - import lycoris - except ModuleNotFoundError: - log.info( - "\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program." - ) - return - run_cmd += f" --network_module=lycoris.kohya" - run_cmd += f' --network_args "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "rank_dropout={rank_dropout}" "module_dropout={module_dropout}" "rank_dropout_scale={rank_dropout_scale}" "algo=glora" "train_norm={train_norm}"' + network_module = "lycoris.kohya" + network_args = f' "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "rank_dropout={rank_dropout}" "module_dropout={module_dropout}" "rank_dropout_scale={rank_dropout_scale}" "algo=glora" "train_norm={train_norm}"' if LoRA_type == "LyCORIS/iA3": - try: - import lycoris - except ModuleNotFoundError: - log.info( - "\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program." - ) - return - run_cmd += f" --network_module=lycoris.kohya" - run_cmd += f' --network_args "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "train_on_input={train_on_input}" "algo=ia3"' - # This is a hack to fix a train_network LoHA logic issue - if not network_dropout > 0.0: - run_cmd += f' --network_dropout="{network_dropout}"' + network_module = "lycoris.kohya" + network_args = f' "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "train_on_input={train_on_input}" "algo=ia3"' if LoRA_type == "LoCon" or LoRA_type == "LyCORIS/LoCon": - try: - import lycoris - except ModuleNotFoundError: - log.info( - "\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program." - ) - return - run_cmd += f" --network_module=lycoris.kohya" - run_cmd += f' --network_args "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "rank_dropout={rank_dropout}" "module_dropout={module_dropout}" "use_tucker={use_tucker}" "use_scalar={use_scalar}" "rank_dropout_scale={rank_dropout_scale}" "algo=locon" "train_norm={train_norm}"' - # This is a hack to fix a train_network LoHA logic issue - if not network_dropout > 0.0: - run_cmd += f' --network_dropout="{network_dropout}"' + network_module = "lycoris.kohya" + network_args = f' "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "rank_dropout={rank_dropout}" "module_dropout={module_dropout}" "use_tucker={use_tucker}" "use_scalar={use_scalar}" "rank_dropout_scale={rank_dropout_scale}" "algo=locon" "train_norm={train_norm}"' if LoRA_type == "LyCORIS/LoHa": - try: - import lycoris - except ModuleNotFoundError: - log.info( - "\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program." - ) - return - run_cmd += f" --network_module=lycoris.kohya" - run_cmd += f' --network_args "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "rank_dropout={rank_dropout}" "module_dropout={module_dropout}" "use_tucker={use_tucker}" "use_scalar={use_scalar}" "rank_dropout_scale={rank_dropout_scale}" "algo=loha" "train_norm={train_norm}"' - # This is a hack to fix a train_network LoHA logic issue - if not network_dropout > 0.0: - run_cmd += f' --network_dropout="{network_dropout}"' + network_module = "lycoris.kohya" + network_args = f' "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "rank_dropout={rank_dropout}" "module_dropout={module_dropout}" "use_tucker={use_tucker}" "use_scalar={use_scalar}" "rank_dropout_scale={rank_dropout_scale}" "algo=loha" "train_norm={train_norm}"' if LoRA_type == "LyCORIS/LoKr": - try: - import lycoris - except ModuleNotFoundError: - log.info( - "\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program." - ) - return - run_cmd += f" --network_module=lycoris.kohya" - run_cmd += f' --network_args "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "rank_dropout={rank_dropout}" "module_dropout={module_dropout}" "factor={factor}" "use_cp={use_cp}" "use_scalar={use_scalar}" "decompose_both={decompose_both}" "rank_dropout_scale={rank_dropout_scale}" "algo=lokr" "train_norm={train_norm}"' - # This is a hack to fix a train_network LoHA logic issue - if not network_dropout > 0.0: - run_cmd += f' --network_dropout="{network_dropout}"' + network_module = "lycoris.kohya" + network_args = f' "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "rank_dropout={rank_dropout}" "module_dropout={module_dropout}" "factor={factor}" "use_cp={use_cp}" "use_scalar={use_scalar}" "decompose_both={decompose_both}" "rank_dropout_scale={rank_dropout_scale}" "algo=lokr" "train_norm={train_norm}"' if LoRA_type == "LyCORIS/Native Fine-Tuning": - try: - import lycoris - except ModuleNotFoundError: - log.info( - "\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program." - ) - return - run_cmd += f" --network_module=lycoris.kohya" - run_cmd += f' --network_args "preset={LyCORIS_preset}" "rank_dropout={rank_dropout}" "module_dropout={module_dropout}" "use_tucker={use_tucker}" "use_scalar={use_scalar}" "rank_dropout_scale={rank_dropout_scale}" "algo=full" "train_norm={train_norm}"' - # This is a hack to fix a train_network LoHA logic issue - if not network_dropout > 0.0: - run_cmd += f' --network_dropout="{network_dropout}"' + network_module = "lycoris.kohya" + network_args = f' "preset={LyCORIS_preset}" "rank_dropout={rank_dropout}" "module_dropout={module_dropout}" "use_tucker={use_tucker}" "use_scalar={use_scalar}" "rank_dropout_scale={rank_dropout_scale}" "algo=full" "train_norm={train_norm}"' if LoRA_type in ["Kohya LoCon", "Standard"]: kohya_lora_var_list = [ @@ -873,7 +793,7 @@ def train_model( "module_dropout", ] - run_cmd += f" --network_module=networks.lora" + network_module = "networks.lora" kohya_lora_vars = { key: value for key, value in vars().items() @@ -888,9 +808,6 @@ def train_model( if value: network_args += f' {key}="{value}"' - if network_args: - run_cmd += f" --network_args{network_args}" - if LoRA_type in [ "LoRA-FA", ]: @@ -907,7 +824,7 @@ def train_model( "module_dropout", ] - run_cmd += f" --network_module=networks.lora_fa" + network_module = "networks.lora_fa" kohya_lora_vars = { key: value for key, value in vars().items() @@ -922,9 +839,6 @@ def train_model( if value: network_args += f' {key}="{value}"' - if network_args: - run_cmd += f" --network_args{network_args}" - if LoRA_type in ["Kohya DyLoRA"]: kohya_lora_var_list = [ "conv_dim", @@ -942,7 +856,7 @@ def train_model( "unit", ] - run_cmd += f" --network_module=networks.dylora" + network_module = "networks.dylora" kohya_lora_vars = { key: value for key, value in vars().items() @@ -955,50 +869,37 @@ def train_model( if value: network_args += f' {key}="{value}"' - if network_args: - run_cmd += f" --network_args{network_args}" - - if not (float(text_encoder_lr) == 0) or not (float(unet_lr) == 0): - if not (float(text_encoder_lr) == 0) and not (float(unet_lr) == 0): - run_cmd += f" --text_encoder_lr={text_encoder_lr}" - run_cmd += f" --unet_lr={unet_lr}" - elif not (float(text_encoder_lr) == 0): - run_cmd += f" --text_encoder_lr={text_encoder_lr}" - run_cmd += f" --network_train_text_encoder_only" - else: - run_cmd += f" --unet_lr={unet_lr}" - run_cmd += f" --network_train_unet_only" - else: + network_train_text_encoder_only = False + network_train_unet_only = False + + # Convert learning rates to float once and store the result for re-use + if text_encoder_lr is None: + output_message( + msg="Please input valid Text Encoder learning rate (between 0 and 1)", headless=headless_bool + ) + return + if unet_lr is None: + output_message( + msg="Please input valid Unet learning rate (between 0 and 1)", headless=headless_bool + ) + return + text_encoder_lr_float = float(text_encoder_lr) + unet_lr_float = float(unet_lr) + + + + # Determine the training configuration based on learning rate values + if text_encoder_lr_float == 0 and unet_lr_float == 0: if float(learning_rate) == 0: output_message( - msg="Please input learning rate values.", - headless=headless_bool, + msg="Please input learning rate values.", headless=headless_bool ) return - - run_cmd += f" --network_dim={network_dim}" - - # if LoRA_type not in ['LyCORIS/LoCon']: - if not lora_network_weights == "": - run_cmd += f' --network_weights="{lora_network_weights}"' - if dim_from_weights: - run_cmd += f" --dim_from_weights" - - if scale_weight_norms > 0.0: - run_cmd += f' --scale_weight_norms="{scale_weight_norms}"' - - if network_dropout > 0.0: - run_cmd += f' --network_dropout="{network_dropout}"' - - if sdxl: - if sdxl_cache_text_encoder_outputs: - run_cmd += f" --cache_text_encoder_outputs" - - if sdxl_no_half_vae: - run_cmd += f" --no_half_vae" - - if debiased_estimation_loss: - run_cmd += " --debiased_estimation_loss" + elif text_encoder_lr_float != 0 and unet_lr_float == 0: + network_train_text_encoder_only = True + elif text_encoder_lr_float == 0 and unet_lr_float != 0: + network_train_unet_only = True + # If both learning rates are non-zero, no specific flags need to be set run_cmd += run_cmd_advanced_training( adaptive_noise_scale=adaptive_noise_scale, @@ -1007,11 +908,14 @@ def train_model( bucket_reso_steps=bucket_reso_steps, cache_latents=cache_latents, cache_latents_to_disk=cache_latents_to_disk, + cache_text_encoder_outputs=True if sdxl and sdxl_cache_text_encoder_outputs else None, caption_dropout_every_n_epochs=caption_dropout_every_n_epochs, caption_dropout_rate=caption_dropout_rate, caption_extension=caption_extension, clip_skip=clip_skip, color_aug=color_aug, + debiased_estimation_loss=debiased_estimation_loss, + dim_from_weights=dim_from_weights, enable_bucket=enable_bucket, epoch=epoch, flip_aug=flip_aug, @@ -1023,6 +927,7 @@ def train_model( keep_tokens=keep_tokens, learning_rate=learning_rate, logging_dir=logging_dir, + lora_network_weights=lora_network_weights, lr_scheduler=lr_scheduler, lr_scheduler_args=lr_scheduler_args, lr_scheduler_num_cycles=lr_scheduler_num_cycles, @@ -1043,6 +948,14 @@ def train_model( mixed_precision=mixed_precision, multires_noise_discount=multires_noise_discount, multires_noise_iterations=multires_noise_iterations, + network_alpha=network_alpha, + network_args=network_args, + network_dim=network_dim, + network_dropout=network_dropout, + network_module=network_module, + network_train_unet_only=network_train_unet_only, + network_train_text_encoder_only=network_train_text_encoder_only, + no_half_vae=True if sdxl and sdxl_no_half_vae else None, no_token_padding=no_token_padding, noise_offset=noise_offset, noise_offset_type=noise_offset_type, @@ -1064,11 +977,15 @@ def train_model( save_precision=save_precision, save_state=save_state, scale_v_pred_loss_like_noise_pred=scale_v_pred_loss_like_noise_pred, + scale_weight_norms=scale_weight_norms, seed=seed, shuffle_caption=shuffle_caption, stop_text_encoder_training=stop_text_encoder_training, + text_encoder_lr=text_encoder_lr, train_batch_size=train_batch_size, train_data_dir=train_data_dir, + training_comment=training_comment, + unet_lr=unet_lr, use_wandb=use_wandb, v2=v2, v_parameterization=v_parameterization, @@ -1245,13 +1162,18 @@ def list_presets(path): with gr.Row(): text_encoder_lr = gr.Number( label="Text Encoder learning rate", - value="5e-5", - info="Optional. Se", + value="0.0001", + info="Optional", + minimum=0, + maximum=1, ) + unet_lr = gr.Number( label="Unet learning rate", value="0.0001", info="Optional", + minimum=0, + maximum=1, ) # Add SDXL Parameters From 2dec489343be7f804ee4a6a365201a70ea67fade Mon Sep 17 00:00:00 2001 From: bmaltais Date: Sun, 4 Feb 2024 19:56:45 -0500 Subject: [PATCH 07/13] Fix issue with no_token_padding in LoRA --- library/class_advanced_training.py | 7 ++++--- lora_gui.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/library/class_advanced_training.py b/library/class_advanced_training.py index c46b40cb0..810ac9d5f 100644 --- a/library/class_advanced_training.py +++ b/library/class_advanced_training.py @@ -21,9 +21,10 @@ def noise_offset_type_change(noise_offset_type): ) with gr.Row(visible=not finetuning): - self.no_token_padding = gr.Checkbox( - label='No token padding', value=False - ) + if training_type != "lora": # Not avaible for LoRA + self.no_token_padding = gr.Checkbox( + label='No token padding', value=False + ) self.gradient_accumulation_steps = gr.Slider( label='Gradient accumulate steps', info='Number of updates steps to accumulate before performing a backward/update pass', diff --git a/lora_gui.py b/lora_gui.py index 697fd3612..4cd9397d1 100644 --- a/lora_gui.py +++ b/lora_gui.py @@ -956,7 +956,7 @@ def train_model( network_train_unet_only=network_train_unet_only, network_train_text_encoder_only=network_train_text_encoder_only, no_half_vae=True if sdxl and sdxl_no_half_vae else None, - no_token_padding=no_token_padding, + # no_token_padding=no_token_padding, noise_offset=noise_offset, noise_offset_type=noise_offset_type, optimizer=optimizer, From 76f7fe6cbed422ab0197761d73b9c03f2a4a4a91 Mon Sep 17 00:00:00 2001 From: bmaltais Date: Mon, 5 Feb 2024 18:03:08 -0500 Subject: [PATCH 08/13] Fix issue created by last commit --- lora_gui.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lora_gui.py b/lora_gui.py index 4cd9397d1..8c4674e48 100644 --- a/lora_gui.py +++ b/lora_gui.py @@ -85,7 +85,7 @@ def save_configuration( gradient_checkpointing, fp8_base, full_fp16, - no_token_padding, + # no_token_padding, stop_text_encoder_training, min_bucket_reso, max_bucket_reso, @@ -252,7 +252,7 @@ def open_configuration( gradient_checkpointing, fp8_base, full_fp16, - no_token_padding, + # no_token_padding, stop_text_encoder_training, min_bucket_reso, max_bucket_reso, @@ -450,7 +450,7 @@ def train_model( gradient_checkpointing, fp8_base, full_fp16, - no_token_padding, + # no_token_padding, stop_text_encoder_training_pct, min_bucket_reso, max_bucket_reso, @@ -1837,7 +1837,7 @@ def update_LoRA_settings( advanced_training.gradient_checkpointing, advanced_training.fp8_base, advanced_training.full_fp16, - advanced_training.no_token_padding, + # advanced_training.no_token_padding, basic_training.stop_text_encoder_training, basic_training.min_bucket_reso, basic_training.max_bucket_reso, From 1a0d8ec4b2aeca96138ec683fd296a8293f6017f Mon Sep 17 00:00:00 2001 From: bmaltais Date: Mon, 5 Feb 2024 18:41:12 -0500 Subject: [PATCH 09/13] Add new options for extract lora --- library/extract_lora_gui.py | 46 ++++++++++++++++++++++++++-- train_network_appl_weights_README.md | 39 +++++++++++++++++++++++ 2 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 train_network_appl_weights_README.md diff --git a/library/extract_lora_gui.py b/library/extract_lora_gui.py index dbb9f03eb..9075cf08d 100644 --- a/library/extract_lora_gui.py +++ b/library/extract_lora_gui.py @@ -32,6 +32,9 @@ def extract_lora( clamp_quantile, min_diff, device, + load_original_model_to, + load_tuned_model_to, + load_precision, ): # Check for caption_text_input if model_tuned == '': @@ -57,6 +60,7 @@ def extract_lora( run_cmd = ( f'{PYTHON} "{os.path.join("networks","extract_lora_from_models.py")}"' ) + run_cmd += f' --load_precision {load_precision}' run_cmd += f' --save_precision {save_precision}' run_cmd += f' --save_to "{save_to}"' run_cmd += f' --model_org "{model_org}"' @@ -71,6 +75,9 @@ def extract_lora( run_cmd += f' --sdxl' run_cmd += f' --clamp_quantile {clamp_quantile}' run_cmd += f' --min_diff {min_diff}' + if sdxl: + run_cmd += f' --load_original_model_to {load_original_model_to}' + run_cmd += f' --load_tuned_model_to {load_tuned_model_to}' log.info(run_cmd) @@ -87,6 +94,11 @@ def extract_lora( def gradio_extract_lora_tab(headless=False): + def change_sdxl(sdxl): + return gr.update(visible=sdxl), gr.update(visible=sdxl) + + + with gr.Tab('Extract LoRA'): gr.Markdown( 'This utility can extract a LoRA network from a finetuned model.' @@ -113,7 +125,15 @@ def gradio_extract_lora_tab(headless=False): outputs=model_tuned, show_progress=False, ) - + load_tuned_model_to = gr.Dropdown( + label='Load finetuned model to', + choices=['cpu', 'cuda', 'cuda:0'], + value='cpu', + interactive=True, scale=1, + info="only for SDXL", + visible=False, + ) + with gr.Row(): model_org = gr.Textbox( label='Stable Diffusion base model', placeholder='Stable Diffusion original model: ckpt or safetensors file', @@ -130,11 +150,20 @@ def gradio_extract_lora_tab(headless=False): outputs=model_org, show_progress=False, ) + load_original_model_to = gr.Dropdown( + label='Load Stable Diffusion base model to', + choices=['cpu', 'cuda', 'cuda:0'], + value='cpu', + interactive=True, scale=1, + info="only for SDXL", + visible=False, + ) with gr.Row(): save_to = gr.Textbox( label='Save to', placeholder='path where to save the extracted LoRA model...', interactive=True, + scale=2, ) button_save_to = gr.Button( folder_symbol, @@ -150,8 +179,14 @@ def gradio_extract_lora_tab(headless=False): save_precision = gr.Dropdown( label='Save precision', choices=['fp16', 'bf16', 'float'], - value='float', - interactive=True, + value='fp16', + interactive=True, scale=1, + ) + load_precision = gr.Dropdown( + label='Load precision', + choices=['fp16', 'bf16', 'float'], + value='fp16', + interactive=True, scale=1, ) with gr.Row(): dim = gr.Slider( @@ -192,6 +227,8 @@ def gradio_extract_lora_tab(headless=False): value='cuda', interactive=True, ) + + sdxl.change(change_sdxl, inputs=sdxl, outputs=[load_tuned_model_to, load_original_model_to]) extract_button = gr.Button('Extract LoRA model') @@ -209,6 +246,9 @@ def gradio_extract_lora_tab(headless=False): clamp_quantile, min_diff, device, + load_original_model_to, + load_tuned_model_to, + load_precision, ], show_progress=False, ) diff --git a/train_network_appl_weights_README.md b/train_network_appl_weights_README.md new file mode 100644 index 000000000..29996cc96 --- /dev/null +++ b/train_network_appl_weights_README.md @@ -0,0 +1,39 @@ +# Exploring Layer-Specific Application Rates for LoRA +## Introduction +Added a tool, train_network_appl_weights.py, for exploring layer-specific application rates. Currently, it supports SDXL only. + +## Concept +The process involves running the standard training process with varying layer-specific application rates on trained networks like LoRA. The goal is to explore which rates produce images closest to the training data. + +## Penalty for Total Application Rates +It's possible to use the total of the layer-specific application rates as a penalty, aiming to reproduce images while minimizing the impact of less significant layers. + +## Multi-Network Exploration +The exploration can be conducted on multiple networks and requires at least one piece of training data. + +Note: The effectiveness with a specific number of images has not been confirmed, but it has been tested with approximately 50 images. The training data does not necessarily have to be from LoRA's training phase, although this has not been confirmed. + +## Command Line Options +The command line options are almost identical to those for `sdxl_train_network.py`, with the following additions and extensions: + +- `--application_loss_weight`: Weight of the layer-specific application rate when added to the loss. Default is 0.0001. Increasing this value trains the model to minimize the application rates. Setting it to 0 allows free exploration of the application rates that yield the highest fidelity. +- `--network_module`: Allows specifying multiple modules for exploration, e.g., `--network_module networks.lora networks.lora`. +- `--network_weights`: Allows specifying weights for multiple networks to be explored, e.g., `--network_weights model1.safetensors model2.safetensors`. + +## Parameters +The number of parameters for layer-specific application rates is 20, including BASE, IN00-08, MID, OUT00-08. BASE is applied to the Text Encoder (Note: LoRA's operation on the Text Encoder has not been confirmed). + +Although the parameters are saved to a file, it's recommended to copy and save the values displayed on the screen. + +## Remarks +Confirmed to work with the AdamW optimizer and a learning rate of 1e-1. The learning rate can be set quite high. With this setting, reasonable results can be obtained in about 1/20 to 1/10 the epochs used during LoRA training. +Increasing `application_loss_weight` above 0.0001 significantly reduces the total application rate, meaning LoRA is applied less. Adjust as needed. +Using negative values for the application rate can lead to minimizing the total by excessively reducing less influential layers' application rates. Negative values are weighted ten times (e.g., -0.01 is almost the same penalty as 0.1). Modify the source code to change the weighting. + +## Potential Uses +Beyond reducing unnecessary layers' application rates, potential uses include: + +- Searching for LoRA application rates to maintain a character while changing their pose based on a reference image. +- Exploring application rates for LoRA to maintain a character's style while altering the artistic style of the image. +- Exploring necessary layers to reproduce a character's attributes using an image in a different style as training data. +- Applying numerous LoRAs to an ideal image as training data and searching for the application rates that achieve the highest fidelity (though more LoRAs will slow down the training). \ No newline at end of file From 7167b33d0392094f940536bcc136227fe7acd903 Mon Sep 17 00:00:00 2001 From: bmaltais Date: Sat, 10 Feb 2024 09:58:30 -0500 Subject: [PATCH 10/13] Update code to last version of gradio 3.x --- kohya_gui.py | 1 + library/class_advanced_training.py | 12 +++--- library/class_basic_training.py | 6 +-- library/class_sdxl_parameters.py | 2 +- library/common_gui.py | 64 ++++++++++++++-------------- library/extract_lora_gui.py | 2 +- library/extract_lycoris_locon_gui.py | 10 ++--- library/manual_caption_gui.py | 10 ++--- lora_gui.py | 19 ++++++--- requirements.txt | 2 +- 10 files changed, 67 insertions(+), 61 deletions(-) diff --git a/kohya_gui.py b/kohya_gui.py index 276930933..8e58ec8ac 100644 --- a/kohya_gui.py +++ b/kohya_gui.py @@ -96,6 +96,7 @@ def UI(**kwargs): launch_kwargs["inbrowser"] = inbrowser if share: launch_kwargs["share"] = share + launch_kwargs["debug"] = True interface.launch(**launch_kwargs) diff --git a/library/class_advanced_training.py b/library/class_advanced_training.py index 810ac9d5f..748b22aab 100644 --- a/library/class_advanced_training.py +++ b/library/class_advanced_training.py @@ -11,13 +11,13 @@ def __init__(self, headless=False, finetuning: bool = False, training_type: str def noise_offset_type_change(noise_offset_type): if noise_offset_type == 'Original': return ( - gr.Group.update(visible=True), - gr.Group.update(visible=False), + gr.Group(visible=True), + gr.Group(visible=False), ) else: return ( - gr.Group.update(visible=False), - gr.Group.update(visible=True), + gr.Group(visible=False), + gr.Group(visible=True), ) with gr.Row(visible=not finetuning): @@ -88,9 +88,9 @@ def full_options_update(full_fp16, full_bf16): full_bf16_active = False if full_bf16: full_fp16_active = False - return gr.Checkbox.update( + return gr.Checkbox( interactive=full_fp16_active, - ), gr.Checkbox.update(interactive=full_bf16_active) + ), gr.Checkbox(interactive=full_bf16_active) self.keep_tokens = gr.Slider( label='Keep n tokens', value='0', minimum=0, maximum=32, step=1 diff --git a/library/class_basic_training.py b/library/class_basic_training.py index 5aaa52d81..35d3e691c 100644 --- a/library/class_basic_training.py +++ b/library/class_basic_training.py @@ -222,9 +222,9 @@ def __init__( def update_learning_rate_te(sdxl_checkbox, finetuning, dreambooth): return ( - gr.Number.update(visible=(not sdxl_checkbox and (finetuning or dreambooth))), - gr.Number.update(visible=(sdxl_checkbox and (finetuning or dreambooth))), - gr.Number.update(visible=(sdxl_checkbox and (finetuning or dreambooth))), + gr.Number(visible=(not sdxl_checkbox and (finetuning or dreambooth))), + gr.Number(visible=(sdxl_checkbox and (finetuning or dreambooth))), + gr.Number(visible=(sdxl_checkbox and (finetuning or dreambooth))), ) self.sdxl_checkbox.change( diff --git a/library/class_sdxl_parameters.py b/library/class_sdxl_parameters.py index ff51e3fff..f4956a1ca 100644 --- a/library/class_sdxl_parameters.py +++ b/library/class_sdxl_parameters.py @@ -27,7 +27,7 @@ def __init__( ) self.sdxl_checkbox.change( - lambda sdxl_checkbox: gr.Accordion.update(visible=sdxl_checkbox), + lambda sdxl_checkbox: gr.Accordion(visible=sdxl_checkbox), inputs=[self.sdxl_checkbox], outputs=[self.sdxl_row], ) diff --git a/library/common_gui.py b/library/common_gui.py index ff290ad29..10ea4cb7e 100644 --- a/library/common_gui.py +++ b/library/common_gui.py @@ -405,9 +405,9 @@ def color_aug_changed(color_aug): msgbox( 'Disabling "Cache latent" because "Color augmentation" has been selected...' ) - return gr.Checkbox.update(value=False, interactive=False) + return gr.Checkbox(value=False, interactive=False) else: - return gr.Checkbox.update(value=True, interactive=True) + return gr.Checkbox(value=True, interactive=True) def save_inference_file(output_dir, v2, v_parameterization, output_name): @@ -454,14 +454,14 @@ def set_pretrained_model_name_or_path_input( # Check if the given model_list is in the list of SDXL models if str(model_list) in SDXL_MODELS: log.info("SDXL model selected. Setting sdxl parameters") - v2 = gr.Checkbox.update(value=False, visible=False) - v_parameterization = gr.Checkbox.update(value=False, visible=False) - sdxl = gr.Checkbox.update(value=True, visible=False) - pretrained_model_name_or_path = gr.Textbox.update( + v2 = gr.Checkbox(value=False, visible=False) + v_parameterization = gr.Checkbox(value=False, visible=False) + sdxl = gr.Checkbox(value=True, visible=False) + pretrained_model_name_or_path = gr.Textbox( value=str(model_list), visible=False ) - pretrained_model_name_or_path_file = gr.Button.update(visible=False) - pretrained_model_name_or_path_folder = gr.Button.update(visible=False) + pretrained_model_name_or_path_file = gr.Button(visible=False) + pretrained_model_name_or_path_folder = gr.Button(visible=False) return ( model_list, pretrained_model_name_or_path, @@ -475,14 +475,14 @@ def set_pretrained_model_name_or_path_input( # Check if the given model_list is in the list of V2 base models if str(model_list) in V2_BASE_MODELS: log.info("SD v2 base model selected. Setting --v2 parameter") - v2 = gr.Checkbox.update(value=True, visible=False) - v_parameterization = gr.Checkbox.update(value=False, visible=False) - sdxl = gr.Checkbox.update(value=False, visible=False) - pretrained_model_name_or_path = gr.Textbox.update( + v2 = gr.Checkbox(value=True, visible=False) + v_parameterization = gr.Checkbox(value=False, visible=False) + sdxl = gr.Checkbox(value=False, visible=False) + pretrained_model_name_or_path = gr.Textbox( value=str(model_list), visible=False ) - pretrained_model_name_or_path_file = gr.Button.update(visible=False) - pretrained_model_name_or_path_folder = gr.Button.update(visible=False) + pretrained_model_name_or_path_file = gr.Button(visible=False) + pretrained_model_name_or_path_folder = gr.Button(visible=False) return ( model_list, pretrained_model_name_or_path, @@ -498,14 +498,14 @@ def set_pretrained_model_name_or_path_input( log.info( "SD v2 model selected. Setting --v2 and --v_parameterization parameters" ) - v2 = gr.Checkbox.update(value=True, visible=False) - v_parameterization = gr.Checkbox.update(value=True, visible=False) - sdxl = gr.Checkbox.update(value=False, visible=False) - pretrained_model_name_or_path = gr.Textbox.update( + v2 = gr.Checkbox(value=True, visible=False) + v_parameterization = gr.Checkbox(value=True, visible=False) + sdxl = gr.Checkbox(value=False, visible=False) + pretrained_model_name_or_path = gr.Textbox( value=str(model_list), visible=False ) - pretrained_model_name_or_path_file = gr.Button.update(visible=False) - pretrained_model_name_or_path_folder = gr.Button.update(visible=False) + pretrained_model_name_or_path_file = gr.Button(visible=False) + pretrained_model_name_or_path_folder = gr.Button(visible=False) return ( model_list, pretrained_model_name_or_path, @@ -519,14 +519,14 @@ def set_pretrained_model_name_or_path_input( # Check if the given model_list is in the list of V1 models if str(model_list) in V1_MODELS: log.info(f"{model_list} model selected.") - v2 = gr.Checkbox.update(value=False, visible=False) - v_parameterization = gr.Checkbox.update(value=False, visible=False) - sdxl = gr.Checkbox.update(value=False, visible=False) - pretrained_model_name_or_path = gr.Textbox.update( + v2 = gr.Checkbox(value=False, visible=False) + v_parameterization = gr.Checkbox(value=False, visible=False) + sdxl = gr.Checkbox(value=False, visible=False) + pretrained_model_name_or_path = gr.Textbox( value=str(model_list), visible=False ) - pretrained_model_name_or_path_file = gr.Button.update(visible=False) - pretrained_model_name_or_path_folder = gr.Button.update(visible=False) + pretrained_model_name_or_path_file = gr.Button(visible=False) + pretrained_model_name_or_path_folder = gr.Button(visible=False) return ( model_list, pretrained_model_name_or_path, @@ -539,12 +539,12 @@ def set_pretrained_model_name_or_path_input( # Check if the model_list is set to 'custom' if model_list == "custom": - v2 = gr.Checkbox.update(visible=True) - v_parameterization = gr.Checkbox.update(visible=True) - sdxl = gr.Checkbox.update(visible=True) - pretrained_model_name_or_path = gr.Textbox.update(visible=True) - pretrained_model_name_or_path_file = gr.Button.update(visible=True) - pretrained_model_name_or_path_folder = gr.Button.update(visible=True) + v2 = gr.Checkbox(visible=True) + v_parameterization = gr.Checkbox(visible=True) + sdxl = gr.Checkbox(visible=True) + pretrained_model_name_or_path = gr.Textbox(visible=True) + pretrained_model_name_or_path_file = gr.Button(visible=True) + pretrained_model_name_or_path_folder = gr.Button(visible=True) return ( model_list, pretrained_model_name_or_path, diff --git a/library/extract_lora_gui.py b/library/extract_lora_gui.py index 9075cf08d..9aaad0764 100644 --- a/library/extract_lora_gui.py +++ b/library/extract_lora_gui.py @@ -95,7 +95,7 @@ def extract_lora( def gradio_extract_lora_tab(headless=False): def change_sdxl(sdxl): - return gr.update(visible=sdxl), gr.update(visible=sdxl) + return gr(visible=sdxl), gr(visible=sdxl) diff --git a/library/extract_lycoris_locon_gui.py b/library/extract_lycoris_locon_gui.py index aec4cda05..a655371ae 100644 --- a/library/extract_lycoris_locon_gui.py +++ b/library/extract_lycoris_locon_gui.py @@ -104,13 +104,13 @@ def extract_lycoris_locon( # def update_mode(mode): # # 'fixed', 'threshold','ratio','quantile' # if mode == 'fixed': -# return gr.Row.update(visible=True), gr.Row.update(visible=False), gr.Row.update(visible=False), gr.Row.update(visible=False) +# return gr.Row(visible=True), gr.Row(visible=False), gr.Row(visible=False), gr.Row(visible=False) # if mode == 'threshold': -# return gr.Row.update(visible=False), gr.Row.update(visible=True), gr.Row.update(visible=False), gr.Row.update(visible=False) +# return gr.Row(visible=False), gr.Row(visible=True), gr.Row(visible=False), gr.Row(visible=False) # if mode == 'ratio': -# return gr.Row.update(visible=False), gr.Row.update(visible=False), gr.Row.update(visible=True), gr.Row.update(visible=False) +# return gr.Row(visible=False), gr.Row(visible=False), gr.Row(visible=True), gr.Row(visible=False) # if mode == 'threshold': -# return gr.Row.update(visible=False), gr.Row.update(visible=False), gr.Row.update(visible=False), gr.Row.update(visible=True) +# return gr.Row(visible=False), gr.Row(visible=False), gr.Row(visible=False), gr.Row(visible=True) def update_mode(mode): @@ -123,7 +123,7 @@ def update_mode(mode): # Iterate through the possible modes for m in modes: # Add a visibility update for each mode, setting it to True if the input mode matches the current mode in the loop - updates.append(gr.Row.update(visible=(mode == m))) + updates.append(gr.Row(visible=(mode == m))) # Return the visibility updates as a tuple return tuple(updates) diff --git a/library/manual_caption_gui.py b/library/manual_caption_gui.py index cb9dd8136..0f7d5b008 100644 --- a/library/manual_caption_gui.py +++ b/library/manual_caption_gui.py @@ -43,7 +43,7 @@ def _get_tag_checkbox_updates(caption, quick_tags, quick_tags_set): t for t in caption_tags_have if t not in quick_tags_set ] caption_tags_all = quick_tags + caption_tags_unique - return gr.CheckboxGroup.update( + return gr.CheckboxGroup( choices=caption_tags_all, value=caption_tags_have ) @@ -118,7 +118,7 @@ def import_tags_from_captions( """ def empty_return(): - return gr.Text.update() + return gr.Text() # Check for images_dir if not images_dir: @@ -247,7 +247,7 @@ def update_images( tag_checkboxes = _get_tag_checkbox_updates( caption, quick_tags, quick_tags_set ) - rows.append(gr.Row.update(visible=show_row)) + rows.append(gr.Row(visible=show_row)) image_paths.append(image_path) captions.append(caption) tag_checkbox_groups.append(tag_checkboxes) @@ -258,7 +258,7 @@ def update_images( + image_paths + captions + tag_checkbox_groups - + [gr.Row.update(visible=True), gr.Row.update(visible=True)] + + [gr.Row(visible=True), gr.Row(visible=True)] ) @@ -473,7 +473,7 @@ def render_pagination(): # Save buttons visibility # (on auto-save on/off) auto_save.change( - lambda auto_save: [gr.Button.update(visible=not auto_save)] + lambda auto_save: [gr.Button(visible=not auto_save)] * IMAGES_TO_SHOW, inputs=auto_save, outputs=save_buttons, diff --git a/lora_gui.py b/lora_gui.py index 8c4674e48..e0320b28e 100644 --- a/lora_gui.py +++ b/lora_gui.py @@ -362,15 +362,16 @@ def open_configuration( # Check if we are "applying" a preset or a config if apply_preset: - log.info(f"Applying preset {training_preset}...") - file_path = f"./presets/lora/{training_preset}.json" + if training_preset != "none": + log.info(f"Applying preset {training_preset}...") + file_path = f"./presets/lora/{training_preset}.json" else: # If not applying a preset, set the `training_preset` field to an empty string # Find the index of the `training_preset` parameter using the `index()` method training_preset_index = parameters.index(("training_preset", training_preset)) # Update the value of `training_preset` by directly assigning an empty string value - parameters[training_preset_index] = ("training_preset", "") + parameters[training_preset_index] = ("training_preset", "none") original_file_path = file_path @@ -414,9 +415,9 @@ def open_configuration( "LyCORIS/LoCon", "LyCORIS/GLoRA", }: - values.append(gr.Row.update(visible=True)) + values.append(gr.Row(visible=True)) else: - values.append(gr.Row.update(visible=False)) + values.append(gr.Row(visible=False)) return tuple(values) @@ -1074,6 +1075,9 @@ def lora_tab( def list_presets(path): json_files = [] + + # Insert an empty string at the beginning + json_files.insert(0, "none") for file in os.listdir(path): if file.endswith(".json"): @@ -1092,6 +1096,7 @@ def list_presets(path): label="Presets", choices=list_presets("./presets/lora"), elem_id="myDropdown", + value="none" ) with gr.Tab("Basic", elem_id="basic_tab"): @@ -1129,7 +1134,7 @@ def list_presets(path): interactive=True # info="https://github.com/KohakuBlueleaf/LyCORIS/blob/0006e2ffa05a48d8818112d9f70da74c0cd30b99/docs/Preset.md" ) - with gr.Box(): + with gr.Group(): with gr.Row(): lora_network_weights = gr.Textbox( label="LoRA network weights", @@ -1673,7 +1678,7 @@ def update_LoRA_settings( for attr, settings in lora_settings_config.items(): update_params = settings["update_params"] - results.append(settings["gr_type"].update(**update_params)) + results.append(settings["gr_type"](**update_params)) return tuple(results) diff --git a/requirements.txt b/requirements.txt index 0cf9d11f3..2d16f1cde 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ easygui==0.98.3 einops==0.6.1 fairscale==0.4.13 ftfy==6.1.1 -gradio==3.36.1 +gradio==3.50.2 huggingface-hub==0.20.1 # for loading Diffusers' SDXL invisible-watermark==0.2.0 From aa518098d4b85705c4fab58067b4c6dad9fa3ee0 Mon Sep 17 00:00:00 2001 From: Hina Chen Date: Thu, 15 Feb 2024 10:22:46 +0800 Subject: [PATCH 11/13] Update localization for zh-TW --- localizations/zh-TW.json | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/localizations/zh-TW.json b/localizations/zh-TW.json index 964655ecb..6ecaa6cbb 100644 --- a/localizations/zh-TW.json +++ b/localizations/zh-TW.json @@ -24,6 +24,8 @@ "network dim for conv layer in fixed mode": "固定模式下卷積層的網路維度", "Sparsity for sparse bias": "稀疏偏差的稀疏度", "path for the file to save...": "儲存檔案的路徑...", + "Verify LoRA": "驗證 LoRA", + "Verify": "驗證", "Verification output": "驗證輸出", "Verification error": "驗證錯誤", "New Rank": "新維度 (Network Rank)", @@ -137,7 +139,7 @@ "(Optional) eg: 0.5": " (選填) 例如：0.5", "(Optional) eg: 0.1": " (選填) 例如：0.1", "Specify the learning rate weight of the down blocks of U-Net.": "指定 U-Net 下區塊的學習率權重。", - "Specify the learning rate weight of the mid blocks of U-Net.": "指定 U-Net 中區塊的學習率權重。", + "Specify the learning rate weight of the mid block of U-Net.": "指定 U-Net 中區塊的學習率權重。", "Specify the learning rate weight of the up blocks of U-Net. The same as down_lr_weight.": "指定 U-Net 上區塊的學習率權重。與 down_lr_weight 相同。", "If the weight is not more than this value, the LoRA module is not created. The default is 0.": "如果權重不超過此值，則不會創建 LoRA 模組。預設為 0。", "Blocks": "區塊", @@ -145,6 +147,9 @@ "(Optional) eg: 2,2,2,2,4,4,4,4,6,6,6,6,8,6,6,6,6,4,4,4,4,2,2,2,2": " (選填) 例如：2,2,2,2,4,4,4,4,6,6,6,6,8,6,6,6,6,4,4,4,4,2,2,2,2", "Specify the dim (rank) of each block. Specify 25 numbers.": "指定每個區塊的維度 (Rank)。指定 25 個數字。", "Specify the alpha of each block. Specify 25 numbers as with block_dims. If omitted, the value of network_alpha is used.": "指定每個區塊的 Alpha。與區塊維度一樣，指定 25 個數字。如果省略，則使用網路 Alpha 的值。", + "Conv": "卷積", + "Conv dims": "卷積維度 (dims)", + "Conv alphas": "卷積 Alphas", "Extend LoRA to Conv2d 3x3 and specify the dim (rank) of each block. Specify 25 numbers.": "將 LoRA 擴展到 Conv2d 3x3，並指定每個區塊的維度 (Rank)。指定 25 個數字。", "Specify the alpha of each block when expanding LoRA to Conv2d 3x3. Specify 25 numbers. If omitted, the value of conv_alpha is used.": "將 LoRA 擴展到 Conv2d 3x3 時，指定每個區塊的 Alpha。指定 25 個數字。如果省略，則使用卷積 Alpha 的值。", "Weighted captions": "加權標記文字", @@ -203,8 +208,8 @@ "Dreambooth/LoRA Folder preparation": "Dreambooth/LoRA 準備資料夾", "Dropout caption every n epochs": "在每 N 個週期 (Epoch) 丟棄標記", "DyLoRA model": "DyLoRA 模型", - "Dynamic method": "動態方法", - "Dynamic parameter": "動態參數", + "Dynamic method": "壓縮演算法", + "Dynamic parameter": "壓縮參數", "e.g., \"by some artist\". Leave empty if you only want to add a prefix or postfix.": "例如，\"由某個藝術家創作\"。如果你只想加入前綴或後綴，請留空白。", "e.g., \"by some artist\". Leave empty if you want to replace with nothing.": "例如，\"由某個藝術家創作\"。如果你想用空值取代，請留空白。", "Enable buckets": "啟用資料桶", @@ -227,6 +232,8 @@ "Flip augmentation": "翻轉增強", "float16": "float16", "Folders": "資料夾", + "U-Net and Text Encoder can be trained with fp8 (experimental)": "U-Net 與 Text Encoder 可以使用 fp8 訓練 (實驗性功能)", + "fp8 base training (experimental)": "使用 fp8 基礎訓練 (實驗性功能)", "Full bf16 training (experimental)": "完整使用 bf16 訓練 (實驗性功能)", "Full fp16 training (experimental)": "完整使用 fp16 訓練 (實驗性功能)", "Generate caption files for the grouped images based on their folder name": "根據圖片的資料夾名稱生成標記文字檔案", @@ -498,4 +505,4 @@ "Training comment": "訓練註解", "Train a TI using kohya textual inversion python code…": "使用 kohya textual inversion Python 程式訓練 TI 模型", "Train a custom model using kohya finetune python code…": "使用 kohya finetune Python 程式訓練自定義模型" -} \ No newline at end of file +} From 40d7b605da10cf089a5864a48e583174bd4db4a6 Mon Sep 17 00:00:00 2001 From: Hina Chen Date: Thu, 15 Feb 2024 10:29:00 +0800 Subject: [PATCH 12/13] Update some typos --- README.md | 22 +++++++++---------- examples/caption.ps1 | 2 +- examples/caption_subfolders.ps1 | 6 ++--- ..._train_db_fixed_with-reg_SDv2 512 base.ps1 | 2 +- library/wd14_caption_gui.py | 6 ++--- localizations/zh-TW.json | 4 ++-- setup/debug_info.py | 2 +- textual_inversion_gui.py | 2 +- 8 files changed, 23 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index c95a930a7..21b4c0c2f 100644 --- a/README.md +++ b/README.md @@ -509,7 +509,7 @@ masterpiece, best quality, 1boy, in business suit, standing at street, looking b - `safetensors` is updated. Please see [Upgrade](#upgrade) and update the library. - Fixed a bug that the training crashes when `network_multiplier` is specified with multi-GPU training. PR [#1084](https://github.com/kohya-ss/sd-scripts/pull/1084) Thanks to fireicewolf! - Fixed a bug that the training crashes when training ControlNet-LLLite. - + - Merge sd-scripts v0.8.2 code update - [Experimental] The `--fp8_base` option is added to the training scripts for LoRA etc. The base model (U-Net, and Text Encoder when training modules for Text Encoder) can be trained with fp8. PR [#1057](https://github.com/kohya-ss/sd-scripts/pull/1057) Thanks to KohakuBlueleaf! - Please specify `--fp8_base` in `train_network.py` or `sdxl_train_network.py`. @@ -522,15 +522,15 @@ masterpiece, best quality, 1boy, in business suit, standing at street, looking b - For example, if you train with state A as `1.0` and state B as `-1.0`, you may be able to generate by switching between state A and B depending on the LoRA application rate. - Also, if you prepare five states and train them as `0.2`, `0.4`, `0.6`, `0.8`, and `1.0`, you may be able to generate by switching the states smoothly depending on the application rate. - Please specify `network_multiplier` in `[[datasets]]` in `.toml` file. - + - Some options are added to `networks/extract_lora_from_models.py` to reduce the memory usage. - `--load_precision` option can be used to specify the precision when loading the model. If the model is saved in fp16, you can reduce the memory usage by specifying `--load_precision fp16` without losing precision. - `--load_original_model_to` option can be used to specify the device to load the original model. `--load_tuned_model_to` option can be used to specify the device to load the derived model. The default is `cpu` for both options, but you can specify `cuda` etc. You can reduce the memory usage by loading one of them to GPU. This option is available only for SDXL. - The gradient synchronization in LoRA training with multi-GPU is improved. PR [#1064](https://github.com/kohya-ss/sd-scripts/pull/1064) Thanks to KohakuBlueleaf! - + - The code for Intel IPEX support is improved. PR [#1060](https://github.com/kohya-ss/sd-scripts/pull/1060) Thanks to akx! - + - Fixed a bug in multi-GPU Textual Inversion training. - `.toml` example for network multiplier @@ -556,7 +556,7 @@ masterpiece, best quality, 1boy, in business suit, standing at street, looking b - Fixed a bug that the VRAM usage without Text Encoder training is larger than before in training scripts for LoRA etc (`train_network.py`, `sdxl_train_network.py`). - Text Encoders were not moved to CPU. - + - Fixed typos. Thanks to akx! [PR #1053](https://github.com/kohya-ss/sd-scripts/pull/1053) * 2024/01/15 (v22.5.0) @@ -574,10 +574,10 @@ masterpiece, best quality, 1boy, in business suit, standing at street, looking b - IPEX library is updated. PR [#1030](https://github.com/kohya-ss/sd-scripts/pull/1030) Thanks to Disty0! - Fixed a bug that Diffusers format model cannot be saved. - Fix LoRA config display after load that would sometime hide some of the feilds - + * 2024/01/02 (v22.4.1) - Minor bug fixed and enhancements. - + * 2023/12/28 (v22.4.0) - Fixed to work `tools/convert_diffusers20_original_sd.py`. Thanks to Disty0! PR [#1016](https://github.com/kohya-ss/sd-scripts/pull/1016) - The issues in multi-GPU training are fixed. Thanks to Isotr0py! PR [#989](https://github.com/kohya-ss/sd-scripts/pull/989) and [#1000](https://github.com/kohya-ss/sd-scripts/pull/1000) @@ -592,13 +592,13 @@ masterpiece, best quality, 1boy, in business suit, standing at street, looking b - The optimizer `PagedAdamW` is added. Thanks to xzuyn! PR [#955](https://github.com/kohya-ss/sd-scripts/pull/955) - NaN replacement in SDXL VAE is sped up. Thanks to liubo0902! PR [#1009](https://github.com/kohya-ss/sd-scripts/pull/1009) - Fixed the path error in `finetune/make_captions.py`. Thanks to CjangCjengh! PR [#986](https://github.com/kohya-ss/sd-scripts/pull/986) - + * 2023/12/20 (v22.3.1) - Add goto button to manual caption utility -- Add missing options for various LyCORIS training algorythms +- Add missing options for various LyCORIS training algorithms - Refactor how feilds are shown or hidden - Made max value for network and convolution rank 512 except for LyCORIS/LoKr. - + * 2023/12/06 (v22.3.0) - Merge sd-scripts updates: - `finetune\tag_images_by_wd14_tagger.py` now supports the separator other than `,` with `--caption_separator` option. Thanks to KohakuBlueleaf! PR [#913](https://github.com/kohya-ss/sd-scripts/pull/913) @@ -612,4 +612,4 @@ masterpiece, best quality, 1boy, in business suit, standing at street, looking b - `--ds_ratio` option denotes the ratio of the Deep Shrink. `0.5` means the half of the original latent size for the Deep Shrink. - `--dst1`, `--dst2`, `--dsd1`, `--dsd2` and `--dsr` prompt options are also available. - Add GLoRA support -- \ No newline at end of file +- \ No newline at end of file diff --git a/examples/caption.ps1 b/examples/caption.ps1 index e61f9b310..073315594 100644 --- a/examples/caption.ps1 +++ b/examples/caption.ps1 @@ -1,6 +1,6 @@ # This powershell script will create a text file for each files in the folder # -# Usefull to create base caption that will be augmented on a per image basis +# Useful to create base caption that will be augmented on a per image basis $folder = "D:\some\folder\location\" $file_pattern="*.*" diff --git a/examples/caption_subfolders.ps1 b/examples/caption_subfolders.ps1 index 0bfba6f01..347195e8b 100644 --- a/examples/caption_subfolders.ps1 +++ b/examples/caption_subfolders.ps1 @@ -1,19 +1,19 @@ # This powershell script will create a text file for each files in the folder # -# Usefull to create base caption that will be augmented on a per image basis +# Useful to create base caption that will be augmented on a per image basis $folder = "D:\test\t2\" $file_pattern="*.*" $text_fir_file="bigeyes style" -foreach ($file in Get-ChildItem $folder\$file_pattern -File) +foreach ($file in Get-ChildItem $folder\$file_pattern -File) { New-Item -ItemType file -Path $folder -Name "$($file.BaseName).txt" -Value $text_fir_file } foreach($directory in Get-ChildItem -path $folder -Directory) { - foreach ($file in Get-ChildItem $folder\$directory\$file_pattern) + foreach ($file in Get-ChildItem $folder\$directory\$file_pattern) { New-Item -ItemType file -Path $folder\$directory -Name "$($file.BaseName).txt" -Value $text_fir_file } diff --git a/examples/kohya_train_db_fixed_with-reg_SDv2 512 base.ps1 b/examples/kohya_train_db_fixed_with-reg_SDv2 512 base.ps1 index 28aa1e70a..98fb8711c 100644 --- a/examples/kohya_train_db_fixed_with-reg_SDv2 512 base.ps1 +++ b/examples/kohya_train_db_fixed_with-reg_SDv2 512 base.ps1 @@ -61,4 +61,4 @@ accelerate launch --num_cpu_threads_per_process $num_cpu_threads_per_process tra --seed=494481440 ` --lr_scheduler=$lr_scheduler -# Add the inference yaml file along with the model for proper loading. Need to have the same name as model... Most likelly "last.yaml" in our case. +# Add the inference yaml file along with the model for proper loading. Need to have the same name as model... Most likely "last.yaml" in our case. diff --git a/library/wd14_caption_gui.py b/library/wd14_caption_gui.py index f58edb2e7..ae171a8ec 100644 --- a/library/wd14_caption_gui.py +++ b/library/wd14_caption_gui.py @@ -123,7 +123,7 @@ def gradio_wd14_caption_gui_tab(headless=False): value='.txt', interactive=True, ) - + caption_separator = gr.Textbox( label='Caption Separator', value=',', @@ -199,11 +199,11 @@ def gradio_wd14_caption_gui_tab(headless=False): ], value='SmilingWolf/wd-v1-4-convnextv2-tagger-v2', ) - + force_download = gr.Checkbox( label='Force model re-download', value=False, - info='Usefull to force model re download when switching to onnx', + info='Useful to force model re download when switching to onnx', ) general_threshold = gr.Slider( diff --git a/localizations/zh-TW.json b/localizations/zh-TW.json index 6ecaa6cbb..df172cb9f 100644 --- a/localizations/zh-TW.json +++ b/localizations/zh-TW.json @@ -51,7 +51,7 @@ "Show frequency of tags for images.": "顯示圖片的標籤頻率。", "Show tags frequency": "顯示標籤頻率", "Model": "模型", - "Usefull to force model re download when switching to onnx": "切換到 onnx 時，強制重新下載模型", + "Useful to force model re download when switching to onnx": "切換到 onnx 時，強制重新下載模型", "Force model re-download": "強制重新下載模型", "General threshold": "一般閾值", "Adjust `general_threshold` for pruning tags (less tags, less flexible)": "調整 `general_threshold` 以修剪標籤 (標籤越少，彈性越小)", @@ -101,7 +101,7 @@ "folder where the model will be saved": "模型將會被儲存的資料夾路徑", "Model type": "模型類型", "Extract LCM": "提取 LCM", - "Verfiy LoRA": "驗證 LoRA", + "Verify LoRA": "驗證 LoRA", "Path to an existing LoRA network weights to resume training from": "要從中繼續訓練的現有 LoRA 網路權重的路徑", "Seed": "種子", "(Optional) eg:1234": " (選填) 例如：1234", diff --git a/setup/debug_info.py b/setup/debug_info.py index a4c26c4a5..e46bd72ca 100644 --- a/setup/debug_info.py +++ b/setup/debug_info.py @@ -51,6 +51,6 @@ # Print VRAM warning if necessary if gpu_vram_warning: - print('\033[33mWarning: GPU VRAM is less than 8GB and will likelly result in proper operations.\033[0m') + print('\033[33mWarning: GPU VRAM is less than 8GB and will likely result in proper operations.\033[0m') print(' ') diff --git a/textual_inversion_gui.py b/textual_inversion_gui.py index 7e9d7c7b9..3362a750d 100644 --- a/textual_inversion_gui.py +++ b/textual_inversion_gui.py @@ -747,7 +747,7 @@ def ti_tab( with gr.Row(): weights = gr.Textbox( label='Resume TI training', - placeholder='(Optional) Path to existing TI embeding file to keep training', + placeholder='(Optional) Path to existing TI embedding file to keep training', ) weights_file_input = gr.Button( '📂', From 9b0165f2ff35a7f2f7bda6b1803b954656bf1163 Mon Sep 17 00:00:00 2001 From: bmaltais Date: Thu, 15 Feb 2024 07:18:09 -0500 Subject: [PATCH 13/13] Update README --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a61e35a17..743ee7906 100644 --- a/README.md +++ b/README.md @@ -503,9 +503,11 @@ masterpiece, best quality, 1boy, in business suit, standing at street, looking b ## Change History -* 2024/01/27 (v22.6.1) +* 2024/02/15 (v22.6.1) - Add support for multi-gpu parameters in the GUI under the "Parameters > Advanced" tab. - Significant rewrite of how parameters are created in the code. I hope I did not break anything in the process... Will make the code easier to update. +- Update TW locallisation +- Update gradio module version to latest 3.x * 2024/01/27 (v22.6.0) - Merge sd-scripts v0.8.3 code update