Merge pull request #2266 from bmaltais/dev

v22.1.5

.release

@@ -1 +1 @@
-v23.1.4
+v23.1.5

README.md

@@ -42,19 +42,13 @@ The GUI allows you to set the training parameters and generate and run the requi
   - [SDXL training](#sdxl-training)
   - [Masked loss](#masked-loss)
   - [Change History](#change-history)
+    - [2024/04/10 (v23.1.5)](#20240410-v2315)
+      - [Security Improvements](#security-improvements)
     - [2024/04/08 (v23.1.4)](#20240408-v2314)
     - [2024/04/08 (v23.1.3)](#20240408-v2313)
     - [2024/04/08 (v23.1.2)](#20240408-v2312)
     - [2024/04/07 (v23.1.1)](#20240407-v2311)
     - [2024/04/07 (v23.1.0)](#20240407-v2310)
-    - [2024/03/21 (v23.0.15)](#20240321-v23015)
-    - [2024/03/19 (v23.0.14)](#20240319-v23014)
-    - [2024/03/19 (v23.0.13)](#20240319-v23013)
-    - [2024/03/16 (v23.0.12)](#20240316-v23012)
-      - [New Features \& Improvements](#new-features--improvements)
-      - [Software Updates](#software-updates)
-      - [Recommendations for Users](#recommendations-for-users)
-    - [2024/03/13 (v23.0.11)](#20240313-v23011)
 
 ## 🦒 Colab
 
@@ -407,6 +401,18 @@ ControlNet dataset is used to specify the mask. The mask images should be the RG
 
 ## Change History
 
+### 2024/04/10 (v23.1.5)
+
+- Fix issue with Textual Inversion configuration file selection.
+- Upgrade to gradio 4.19.2 to fix several high security risks associated to earlier versions. This is a major upgrade, moving from 3.x to 4.x. Hoping this will not introduce undorseen issues.
+- Upgrade transformers to 4.38.0 to fix a low severity security issue.
+
+#### Security Improvements
+
+- Add explicit --do_not_share parameter to kohya_gui.py to avoid sharing the GUI on platforms like Kaggle.
+- Remove shell=True from subprocess calls to avoid security issues when using the GUI.
+- Limit caption extensions to a fixed set of extensions to limit the risk of finding and replacing text content in unexpected files.
+
 ### 2024/04/08 (v23.1.4)
 
 - Relocate config accordion to the top of the GUI.
@@ -510,53 +516,3 @@ ControlNet dataset is used to specify the mask. The mask images should be the RG
 - Added support for "Dataset Preparation" defaults via the config.toml file.
 - Added a field to allow for the input of extra accelerate launch arguments.
 - Added new caption tool from https://github.com/kainatquaderee
-
-### 2024/03/21 (v23.0.15)
-
-- Add support for toml dataset configuration fole to all trainers
-- Add new setup menu option to install Triton 2.1.0 for Windows
-- Add support for LyCORIS BOFT and DoRA and QLyCORIS options for LoHA, LoKr and LoCon
-- Fix issue with vae path validation
-- Other fixes
-
-### 2024/03/19 (v23.0.14)
-
-- Fix blip caption issue
-
-### 2024/03/19 (v23.0.13)
-
-- Fix issue with image samples.
-
-### 2024/03/16 (v23.0.12)
-
-#### New Features & Improvements
-
-- **Enhanced Logging and Tracking Capabilities**
-  - Added support for configuring advanced logging and tracking:
-    - `wandb_run_name`: Set a custom name for your Weights & Biases runs to easily identify and organize your experiments.
-    - `log_tracker_name` and `log_tracker_config`: Integrate custom logging trackers with your projects. Specify the tracker name and provide its configuration to enable detailed monitoring and logging of your runs.
-
-- **Custom Path Defaults**
-  - You can now specify custom paths more easily:
-    - Simply copy the `config example.toml` file located in the root directory of the repository to `config.toml`.
-    - Edit the `config.toml` file to adjust paths and settings according to your preferences.
-
-#### Software Updates
-
-- **sd-scripts updated to v0.8.5**
-  - **Bug Fixes:**
-    - Corrected an issue where the value of timestep embedding was incorrect during SDXL training. This fix ensures accurate training progress and results.
-    - Addressed a related inference issue with the generation script, improving the reliability of SDXL model outputs.
-  - **Note:** The exact impact of this bug is currently unknown, but it's recommended to update to v0.8.5 for anyone engaged in SDXL training to ensure optimal performance and results.
-
-- **Upgrade of `lycoris_lora` Python Module**
-  - Updated the `lycoris_lora` module to version 2.2.0.post3. This update may include bug fixes, performance improvements, and new features.
-
-#### Recommendations for Users
-
-- To benefit from the latest features and improvements, users are encouraged to update their installations and configurations accordingly.
-
-### 2024/03/13 (v23.0.11)
-
-- Increase icon size.
-- More setup fixes.

_typos.toml

@@ -27,6 +27,7 @@ rik="rik"
 koo="koo"
 yos="yos"
 wn="wn"
+parm = "parm"
 
 
 [files]

kohya_gui.py

@@ -88,6 +88,7 @@ def UI(**kwargs):
     server_port = kwargs.get("server_port", 0)
     inbrowser = kwargs.get("inbrowser", False)
     share = kwargs.get("share", False)
+    do_not_share = kwargs.get("do_not_share", False)
     server_name = kwargs.get("listen")
 
     launch_kwargs["server_name"] = server_name
@@ -97,12 +98,14 @@ def UI(**kwargs):
         launch_kwargs["server_port"] = server_port
     if inbrowser:
         launch_kwargs["inbrowser"] = inbrowser
-    if share:
-        launch_kwargs["share"] = share
+    if do_not_share:
+        launch_kwargs["share"] = False
+    else:
+        if share:
+            launch_kwargs["share"] = share
     launch_kwargs["debug"] = True
     interface.launch(**launch_kwargs)
 
-
 if __name__ == "__main__":
     # torch.cuda.set_per_process_memory_fraction(0.48)
     parser = argparse.ArgumentParser()
@@ -141,17 +144,9 @@ def UI(**kwargs):
 
     parser.add_argument("--use-ipex", action="store_true", help="Use IPEX environment")
     parser.add_argument("--use-rocm", action="store_true", help="Use ROCm environment")
+
+    parser.add_argument("--do_not_share", action="store_true", help="Do not share the gradio UI")
 
     args = parser.parse_args()
 
-    UI(
-        config_file_path=args.config,
-        username=args.username,
-        password=args.password,
-        inbrowser=args.inbrowser,
-        server_port=args.server_port,
-        share=args.share,
-        listen=args.listen,
-        headless=args.headless,
-        language=args.language,
-    )
+    UI(**vars(args))

kohya_gui/basic_caption_gui.py

@@ -83,7 +83,7 @@ def caption_images(
         )
 
         # Run the command based on the operating system
-        subprocess.run(run_cmd, shell=True, env=env)
+        subprocess.run(run_cmd, env=env)
 
     # Check if overwrite option is enabled
     if overwrite:
@@ -190,9 +190,9 @@ def list_images_dirs(path):
                 show_progress=False,
             )
             # Textbox for caption file extension
-            caption_ext = gr.Textbox(
+            caption_ext = gr.Dropdown(
                 label="Caption file extension",
-                placeholder="Extension for caption file (e.g., .caption, .txt)",
+                choices=[".cap", ".caption", ".txt"],
                 value=".txt",
                 interactive=True,
             )

kohya_gui/blip2_caption_gui.py

@@ -332,7 +332,7 @@ def list_train_dirs(path):
                     )
 
                     top_p = gr.Slider(
-                        minimum=-0,
+                        minimum=0,
                         maximum=1,
                         value=0.9,
                         step=0.1,

kohya_gui/blip_caption_gui.py

@@ -79,7 +79,7 @@ def caption_images(
     )
 
     # Run the command in the sd-scripts folder context
-    subprocess.run(run_cmd, shell=True, env=env, cwd=f"{scriptdir}/sd-scripts")
+    subprocess.run(run_cmd, env=env, cwd=f"{scriptdir}/sd-scripts")
 
     # Add prefix and postfix
     add_pre_postfix(
@@ -142,9 +142,9 @@ def list_train_dirs(path):
                 show_progress=False,
             )
         with gr.Row():
-            caption_file_ext = gr.Textbox(
+            caption_file_ext = gr.Dropdown(
                 label="Caption file extension",
-                placeholder="Extension for caption file (e.g., .caption, .txt)",
+                choices=[".cap", ".caption", ".txt"],
                 value=".txt",
                 interactive=True,
             )

kohya_gui/class_basic_training.py

@@ -19,9 +19,9 @@ class BasicTraining:
     def __init__(
         self,
         sdxl_checkbox: gr.Checkbox,
-        learning_rate_value: str = "1e-6",
+        learning_rate_value: float = "1e-6",
         lr_scheduler_value: str = "constant",
-        lr_warmup_value: str = "0",
+        lr_warmup_value: float = "0",
         finetuning: bool = False,
         dreambooth: bool = False,
         config: dict = {},
@@ -98,10 +98,11 @@ def init_training_controls(self) -> None:
                 label="Save every N epochs", value=self.config.get("basic.save_every_n_epochs", 1), precision=0
             )
             # Initialize the caption extension input
-            self.caption_extension = gr.Textbox(
-                label="Caption Extension",
-                placeholder="(Optional) default: .caption",
-                value=self.config.get("basic.caption_extension", ""),
+            self.caption_extension = gr.Dropdown(
+                label="Caption file extension",
+                choices=[".cap", ".caption", ".txt"],
+                value=".txt",
+                interactive=True,
             )
 
     def init_precision_and_resources_controls(self) -> None:

kohya_gui/class_command_executor.py

@@ -28,7 +28,7 @@ def execute_command(self, run_cmd: str, **kwargs):
         if self.process and self.process.poll() is None:
             log.info("The command is already running. Please wait for it to finish.")
         else:
-            self.process = subprocess.Popen(run_cmd, shell=True, **kwargs)
+            self.process = subprocess.Popen(run_cmd, **kwargs)
 
     def kill_command(self):
         """

kohya_gui/class_source_model.py

@@ -257,7 +257,7 @@ def list_dataset_config_dirs(path: str) -> list:
                                 min_width=60,
                             )
                     with gr.Column():
-                        gr.Box(visible=False)
+                        gr.Group(visible=False)
 
                 with gr.Row():
                     self.training_comment = gr.Textbox(

kohya_gui/common_gui.py

@@ -646,6 +646,10 @@ def add_pre_postfix(
         postfix (str, optional): Postfix to add to the content of the caption files.
         caption_file_ext (str, optional): Extension of the caption files.
     """
+    # Enforce that the provided extension is one of .caption, .cap, .txt
+    if caption_file_ext not in (".caption", ".cap", ".txt"):
+        log.error("Invalid caption file extension. Must be on of .caption, .cap, .txt")
+        return
 
     # If neither prefix nor postfix is provided, return early
     if prefix == "" and postfix == "":
@@ -715,7 +719,6 @@ def has_ext_files(folder_path: str, file_extension: str) -> bool:
     # If no file with the specified extension is found, return False
     return False
 
-
 def find_replace(
     folder_path: str = "",
     caption_file_ext: str = ".caption",
@@ -747,21 +750,41 @@ def find_replace(
         )
         # Exit the function early
         return
+
+    # Check if the caption file extension is one of the supported extensions
+    if caption_file_ext not in [".caption", ".txt", ".txt2", ".cap"]:
+        log.error(
+            f"Unsupported file extension {caption_file_ext} for caption files. Please use .caption, .txt, .txt2, or .cap."
+        )
+        # Exit the function early
+        return
+
+    # Check if the folder path exists
+    if not os.path.exists(folder_path):
+        log.error(f"The provided path '{folder_path}' is not a valid folder.")
+        return
 
     # List all caption files in the folder
-    caption_files = [f for f in os.listdir(folder_path) if f.endswith(caption_file_ext)]
+    try:
+        caption_files = [f for f in os.listdir(folder_path) if f.endswith(caption_file_ext)]
+    except Exception as e:
+        log.error(f"Error accessing folder {folder_path}: {e}")
+        return
 
     # Iterate over the list of caption files
     for caption_file in caption_files:
         # Construct the full path for each caption file
         file_path = os.path.join(folder_path, caption_file)
         # Read and replace text
-        with open(file_path, "r", errors="ignore") as f:
-            content = f.read().replace(search_text, replace_text)
-
-        # Write the updated content back to the file
-        with open(file_path, "w") as f:
-            f.write(content)
+        try:
+            with open(file_path, "r", errors="ignore") as f:
+                content = f.read().replace(search_text, replace_text)
+
+            # Write the updated content back to the file
+            with open(file_path, "w") as f:
+                f.write(content)
+        except Exception as e:
+            log.error(f"Error processing file {file_path}: {e}")
 
 
 def color_aug_changed(color_aug):

kohya_gui/convert_lcm_gui.py

@@ -59,7 +59,7 @@ def convert_lcm(name, model_path, lora_scale, model_type):
     )
 
     # Run the command
-    subprocess.run(run_cmd, shell=True, env=env)
+    subprocess.run(run_cmd, env=env)
 
     # Return a success message
     log.info("Done extracting...")

kohya_gui/convert_model_gui.py

@@ -104,7 +104,7 @@ def convert_model(
     )
 
     # Run the command
-    subprocess.run(run_cmd, shell=True, env=env)
+    subprocess.run(run_cmd, env=env)
 
 
 ###