From 5ada070d88f0527568d2c8c3ac77d3e12d77997b Mon Sep 17 00:00:00 2001 From: delta_lt_0 Date: Sat, 6 Apr 2024 21:25:19 +0800 Subject: [PATCH 01/42] feat: support download of huggingface files from a mirror website (#2637) * fix: load image number from preset (#2611) * fix: add default_image_number to preset handling * fix: use minimum image number of preset and config to prevent UI overflow * fix: use correct base dimensions for outpaint mask padding (#2612) * fix: add Civitai compatibility for LoRAs in a1111 metadata scheme by switching schema (#2615) * feat: update sha256 generation functions https://github.com/lllyasviel/stable-diffusion-webui-forge/blob/29be1da7cf2b5dccfc70fbdd33eb35c56a31ffb7/modules/hashes.py * feat: add compatibility for LoRAs in a1111 metadata scheme * feat: add backwards compatibility * refactor: extract remove_special_loras * fix: correctly apply LoRA weight for legacy schema * docs: bump version number to 2.3.1, add changelog (#2616) * feat:support download huggingface files from a mirror site --------- Co-authored-by: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> --- docker.md | 1 + fooocus_version.py | 2 +- launch.py | 4 ++ ldm_patched/modules/args_parser.py | 1 + modules/async_worker.py | 8 ++-- modules/config.py | 2 + modules/meta_parser.py | 61 ++++++++++++++++++++++-------- modules/model_loader.py | 2 + modules/util.py | 38 ++++++++++++++++--- readme.md | 1 + update_log.md | 7 ++++ 11 files changed, 101 insertions(+), 26 deletions(-) diff --git a/docker.md b/docker.md index 36cfa632a..1939d6fca 100644 --- a/docker.md +++ b/docker.md @@ -54,6 +54,7 @@ Docker specified environments are there. They are used by 'entrypoint.sh' |CMDARGS|Arguments for [entry_with_update.py](entry_with_update.py) which is called by [entrypoint.sh](entrypoint.sh)| |config_path|'config.txt' location| |config_example_path|'config_modification_tutorial.txt' location| +|HF_MIRROR| huggingface mirror site domain| You can also use the same json key names and values explained in the 'config_modification_tutorial.txt' as the environments. See examples in the [docker-compose.yml](docker-compose.yml) diff --git a/fooocus_version.py b/fooocus_version.py index a4b8895b3..b20501966 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.3.0' +version = '2.3.1' diff --git a/launch.py b/launch.py index afa667058..5c865e6d7 100644 --- a/launch.py +++ b/launch.py @@ -80,6 +80,10 @@ def ini_args(): os.environ['CUDA_VISIBLE_DEVICES'] = str(args.gpu_device_id) print("Set device to:", args.gpu_device_id) +if args.hf_mirror is not None : + os.environ['HF_MIRROR'] = str(args.hf_mirror) + print("Set hf_mirror to:", args.hf_mirror) + from modules import config os.environ['GRADIO_TEMP_DIR'] = config.temp_path diff --git a/ldm_patched/modules/args_parser.py b/ldm_patched/modules/args_parser.py index 0c6165a7b..bf8737835 100644 --- a/ldm_patched/modules/args_parser.py +++ b/ldm_patched/modules/args_parser.py @@ -37,6 +37,7 @@ def __call__(self, parser, namespace, values, option_string=None): parser.add_argument("--port", type=int, default=8188) parser.add_argument("--disable-header-check", type=str, default=None, metavar="ORIGIN", nargs="?", const="*") parser.add_argument("--web-upload-size", type=float, default=100) +parser.add_argument("--hf-mirror", type=str, default=None) parser.add_argument("--external-working-path", type=str, default=None, metavar="PATH", nargs='+', action='append') parser.add_argument("--output-path", type=str, default=None) diff --git a/modules/async_worker.py b/modules/async_worker.py index fa9593618..d8a1e072d 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -614,12 +614,12 @@ def handler(async_task): H, W, C = inpaint_image.shape if 'left' in outpaint_selections: - inpaint_image = np.pad(inpaint_image, [[0, 0], [int(H * 0.3), 0], [0, 0]], mode='edge') - inpaint_mask = np.pad(inpaint_mask, [[0, 0], [int(H * 0.3), 0]], mode='constant', + inpaint_image = np.pad(inpaint_image, [[0, 0], [int(W * 0.3), 0], [0, 0]], mode='edge') + inpaint_mask = np.pad(inpaint_mask, [[0, 0], [int(W * 0.3), 0]], mode='constant', constant_values=255) if 'right' in outpaint_selections: - inpaint_image = np.pad(inpaint_image, [[0, 0], [0, int(H * 0.3)], [0, 0]], mode='edge') - inpaint_mask = np.pad(inpaint_mask, [[0, 0], [0, int(H * 0.3)]], mode='constant', + inpaint_image = np.pad(inpaint_image, [[0, 0], [0, int(W * 0.3)], [0, 0]], mode='edge') + inpaint_mask = np.pad(inpaint_mask, [[0, 0], [0, int(W * 0.3)]], mode='constant', constant_values=255) inpaint_image = np.ascontiguousarray(inpaint_image.copy()) diff --git a/modules/config.py b/modules/config.py index 76ffd3488..b81e218a0 100644 --- a/modules/config.py +++ b/modules/config.py @@ -485,6 +485,7 @@ def init_temp_path(path: str | None, default_path: str) -> str: "default_scheduler": "scheduler", "default_overwrite_step": "steps", "default_performance": "performance", + "default_image_number": "image_number", "default_prompt": "prompt", "default_prompt_negative": "negative_prompt", "default_styles": "styles", @@ -538,6 +539,7 @@ def add_ratio(x): sdxl_lcm_lora = 'sdxl_lcm_lora.safetensors' sdxl_lightning_lora = 'sdxl_lightning_4step_lora.safetensors' +loras_metadata_remove = [sdxl_lcm_lora, sdxl_lightning_lora] def get_model_filenames(folder_paths, extensions=None, name_filter=None): diff --git a/modules/meta_parser.py b/modules/meta_parser.py index 10bc68967..70ab8860c 100644 --- a/modules/meta_parser.py +++ b/modules/meta_parser.py @@ -1,5 +1,4 @@ import json -import os import re from abc import ABC, abstractmethod from pathlib import Path @@ -12,7 +11,7 @@ import modules.sdxl_styles from modules.flags import MetadataScheme, Performance, Steps from modules.flags import SAMPLERS, CIVITAI_NO_KARRAS -from modules.util import quote, unquote, extract_styles_from_prompt, is_json, get_file_from_folder_list, calculate_sha256 +from modules.util import quote, unquote, extract_styles_from_prompt, is_json, get_file_from_folder_list, sha256 re_param_code = r'\s*(\w[\w \-/]+):\s*("(?:\\.|[^\\"])+"|[^,]*)(?:,|$)' re_param = re.compile(re_param_code) @@ -27,8 +26,9 @@ def load_parameter_button_click(raw_metadata: dict | str, is_generating: bool): loaded_parameter_dict = json.loads(raw_metadata) assert isinstance(loaded_parameter_dict, dict) - results = [len(loaded_parameter_dict) > 0, 1] + results = [len(loaded_parameter_dict) > 0] + get_image_number('image_number', 'Image Number', loaded_parameter_dict, results) get_str('prompt', 'Prompt', loaded_parameter_dict, results) get_str('negative_prompt', 'Negative Prompt', loaded_parameter_dict, results) get_list('styles', 'Styles', loaded_parameter_dict, results) @@ -92,13 +92,25 @@ def get_float(key: str, fallback: str | None, source_dict: dict, results: list, results.append(gr.update()) +def get_image_number(key: str, fallback: str | None, source_dict: dict, results: list, default=None): + try: + h = source_dict.get(key, source_dict.get(fallback, default)) + assert h is not None + h = int(h) + h = min(h, modules.config.default_max_image_number) + results.append(h) + except: + results.append(1) + + def get_steps(key: str, fallback: str | None, source_dict: dict, results: list, default=None): try: h = source_dict.get(key, source_dict.get(fallback, default)) assert h is not None h = int(h) # if not in steps or in steps and performance is not the same - if h not in iter(Steps) or Steps(h).name.casefold() != source_dict.get('performance', '').replace(' ', '_').casefold(): + if h not in iter(Steps) or Steps(h).name.casefold() != source_dict.get('performance', '').replace(' ', + '_').casefold(): results.append(h) return results.append(-1) @@ -192,7 +204,8 @@ def get_lora(key: str, fallback: str | None, source_dict: dict, results: list): def get_sha256(filepath): global hash_cache if filepath not in hash_cache: - hash_cache[filepath] = calculate_sha256(filepath) + # is_safetensors = os.path.splitext(filepath)[1].lower() == '.safetensors' + hash_cache[filepath] = sha256(filepath) return hash_cache[filepath] @@ -219,8 +232,9 @@ def parse_meta_from_preset(preset_content): height = height[:height.index(" ")] preset_prepared[meta_key] = (width, height) else: - preset_prepared[meta_key] = items[settings_key] if settings_key in items and items[settings_key] is not None else getattr(modules.config, settings_key) - + preset_prepared[meta_key] = items[settings_key] if settings_key in items and items[ + settings_key] is not None else getattr(modules.config, settings_key) + if settings_key == "default_styles" or settings_key == "default_aspect_ratio": preset_prepared[meta_key] = str(preset_prepared[meta_key]) @@ -276,6 +290,12 @@ def set_data(self, raw_prompt, full_prompt, raw_negative_prompt, full_negative_p lora_hash = get_sha256(lora_path) self.loras.append((Path(lora_name).stem, lora_weight, lora_hash)) + @staticmethod + def remove_special_loras(lora_filenames): + for lora_to_remove in modules.config.loras_metadata_remove: + if lora_to_remove in lora_filenames: + lora_filenames.remove(lora_to_remove) + class A1111MetadataParser(MetadataParser): def get_scheme(self) -> MetadataScheme: @@ -385,12 +405,19 @@ def parse_json(self, metadata: str) -> dict: data[key] = filename break - if 'lora_hashes' in data and data['lora_hashes'] != '': + lora_data = '' + if 'lora_weights' in data and data['lora_weights'] != '': + lora_data = data['lora_weights'] + elif 'lora_hashes' in data and data['lora_hashes'] != '' and data['lora_hashes'].split(', ')[0].count(':') == 2: + lora_data = data['lora_hashes'] + + if lora_data != '': lora_filenames = modules.config.lora_filenames.copy() - if modules.config.sdxl_lcm_lora in lora_filenames: - lora_filenames.remove(modules.config.sdxl_lcm_lora) - for li, lora in enumerate(data['lora_hashes'].split(', ')): - lora_name, lora_hash, lora_weight = lora.split(': ') + self.remove_special_loras(lora_filenames) + for li, lora in enumerate(lora_data.split(', ')): + lora_split = lora.split(': ') + lora_name = lora_split[0] + lora_weight = lora_split[2] if len(lora_split) == 3 else lora_split[1] for filename in lora_filenames: path = Path(filename) if lora_name == path.stem: @@ -441,11 +468,15 @@ def parse_string(self, metadata: dict) -> str: if len(self.loras) > 0: lora_hashes = [] + lora_weights = [] for index, (lora_name, lora_weight, lora_hash) in enumerate(self.loras): # workaround for Fooocus not knowing LoRA name in LoRA metadata - lora_hashes.append(f'{lora_name}: {lora_hash}: {lora_weight}') + lora_hashes.append(f'{lora_name}: {lora_hash}') + lora_weights.append(f'{lora_name}: {lora_weight}') lora_hashes_string = ', '.join(lora_hashes) + lora_weights_string = ', '.join(lora_weights) generation_params[self.fooocus_to_a1111['lora_hashes']] = lora_hashes_string + generation_params[self.fooocus_to_a1111['lora_weights']] = lora_weights_string generation_params[self.fooocus_to_a1111['version']] = data['version'] @@ -468,9 +499,7 @@ def get_scheme(self) -> MetadataScheme: def parse_json(self, metadata: dict) -> dict: model_filenames = modules.config.model_filenames.copy() lora_filenames = modules.config.lora_filenames.copy() - if modules.config.sdxl_lcm_lora in lora_filenames: - lora_filenames.remove(modules.config.sdxl_lcm_lora) - + self.remove_special_loras(lora_filenames) for key, value in metadata.items(): if value in ['', 'None']: continue diff --git a/modules/model_loader.py b/modules/model_loader.py index 8ba336a91..1143f75e2 100644 --- a/modules/model_loader.py +++ b/modules/model_loader.py @@ -14,6 +14,8 @@ def load_file_from_url( Returns the path to the downloaded file. """ + domain = os.environ.get("HF_MIRROR", "https://huggingface.co").rstrip('/') + url = str.replace(url, "https://huggingface.co", domain, 1) os.makedirs(model_dir, exist_ok=True) if not file_name: parts = urlparse(url) diff --git a/modules/util.py b/modules/util.py index 7c46d946c..9e0fb294b 100644 --- a/modules/util.py +++ b/modules/util.py @@ -7,9 +7,9 @@ import os import cv2 import json +import hashlib from PIL import Image -from hashlib import sha256 import modules.sdxl_styles @@ -182,16 +182,44 @@ def get_files_from_folder(folder_path, extensions=None, name_filter=None): return filenames -def calculate_sha256(filename, length=HASH_SHA256_LENGTH) -> str: - hash_sha256 = sha256() +def sha256(filename, use_addnet_hash=False, length=HASH_SHA256_LENGTH): + print(f"Calculating sha256 for {filename}: ", end='') + if use_addnet_hash: + with open(filename, "rb") as file: + sha256_value = addnet_hash_safetensors(file) + else: + sha256_value = calculate_sha256(filename) + print(f"{sha256_value}") + + return sha256_value[:length] if length is not None else sha256_value + + +def addnet_hash_safetensors(b): + """kohya-ss hash for safetensors from https://github.com/kohya-ss/sd-scripts/blob/main/library/train_util.py""" + hash_sha256 = hashlib.sha256() + blksize = 1024 * 1024 + + b.seek(0) + header = b.read(8) + n = int.from_bytes(header, "little") + + offset = n + 8 + b.seek(offset) + for chunk in iter(lambda: b.read(blksize), b""): + hash_sha256.update(chunk) + + return hash_sha256.hexdigest() + + +def calculate_sha256(filename) -> str: + hash_sha256 = hashlib.sha256() blksize = 1024 * 1024 with open(filename, "rb") as f: for chunk in iter(lambda: f.read(blksize), b""): hash_sha256.update(chunk) - res = hash_sha256.hexdigest() - return res[:length] if length else res + return hash_sha256.hexdigest() def quote(text): diff --git a/readme.md b/readme.md index 5f66e02aa..0ec06f198 100644 --- a/readme.md +++ b/readme.md @@ -368,6 +368,7 @@ A safer way is just to try "run_anime.bat" or "run_realistic.bat" - they should entry_with_update.py [-h] [--listen [IP]] [--port PORT] [--disable-header-check [ORIGIN]] [--web-upload-size WEB_UPLOAD_SIZE] + [--hf-mirror HF_MIRROR] [--external-working-path PATH [PATH ...]] [--output-path OUTPUT_PATH] [--temp-path TEMP_PATH] [--cache-path CACHE_PATH] [--in-browser] diff --git a/update_log.md b/update_log.md index 4e22db0a4..62c4882bc 100644 --- a/update_log.md +++ b/update_log.md @@ -1,3 +1,10 @@ +# [2.3.1](https://github.com/lllyasviel/Fooocus/releases/tag/2.3.1) + +* Remove positive prompt from anime prefix to not reset prompt after switching presets +* Fix image number being reset to 1 when switching preset, now doesn't reset anymore +* Fix outpainting dimension calculation when extending left/right +* Fix LoRA compatibility for LoRAs in a1111 metadata scheme + # [2.3.0](https://github.com/lllyasviel/Fooocus/releases/tag/2.3.0) * Add performance "lightning" (based on [SDXL-Lightning 4 step LoRA](https://huggingface.co/ByteDance/SDXL-Lightning/blob/main/sdxl_lightning_4step_lora.safetensors)) From 1dff430d4c089fb3bee6287f9371d0926352fb54 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Sat, 6 Apr 2024 15:27:35 +0200 Subject: [PATCH 02/42] feat: update interposer from v3.1 to v4.0 (#2717) * fix: load image number from preset (#2611) * fix: add default_image_number to preset handling * fix: use minimum image number of preset and config to prevent UI overflow * fix: use correct base dimensions for outpaint mask padding (#2612) * fix: add Civitai compatibility for LoRAs in a1111 metadata scheme by switching schema (#2615) * feat: update sha256 generation functions https://github.com/lllyasviel/stable-diffusion-webui-forge/blob/29be1da7cf2b5dccfc70fbdd33eb35c56a31ffb7/modules/hashes.py * feat: add compatibility for LoRAs in a1111 metadata scheme * feat: add backwards compatibility * refactor: extract remove_special_loras * fix: correctly apply LoRA weight for legacy schema * docs: bump version number to 2.3.1, add changelog (#2616) * feat: update interposer vrom v3.1 to v4.0 --- extras/vae_interpose.py | 92 ++++++++++++++++++++++++----------------- launch.py | 4 +- 2 files changed, 56 insertions(+), 40 deletions(-) diff --git a/extras/vae_interpose.py b/extras/vae_interpose.py index 72fb09a41..d407ca831 100644 --- a/extras/vae_interpose.py +++ b/extras/vae_interpose.py @@ -1,69 +1,85 @@ # https://github.com/city96/SD-Latent-Interposer/blob/main/interposer.py import os -import torch + import safetensors.torch as sf +import torch import torch.nn as nn -import ldm_patched.modules.model_management +import ldm_patched.modules.model_management from ldm_patched.modules.model_patcher import ModelPatcher from modules.config import path_vae_approx -class Block(nn.Module): - def __init__(self, size): +class ResBlock(nn.Module): + """Block with residuals""" + + def __init__(self, ch): super().__init__() self.join = nn.ReLU() + self.norm = nn.BatchNorm2d(ch) self.long = nn.Sequential( - nn.Conv2d(size, size, kernel_size=3, stride=1, padding=1), - nn.LeakyReLU(0.1), - nn.Conv2d(size, size, kernel_size=3, stride=1, padding=1), - nn.LeakyReLU(0.1), - nn.Conv2d(size, size, kernel_size=3, stride=1, padding=1), + nn.Conv2d(ch, ch, kernel_size=3, stride=1, padding=1), + nn.SiLU(), + nn.Conv2d(ch, ch, kernel_size=3, stride=1, padding=1), + nn.SiLU(), + nn.Conv2d(ch, ch, kernel_size=3, stride=1, padding=1), + nn.Dropout(0.1) ) def forward(self, x): - y = self.long(x) - z = self.join(y + x) - return z + x = self.norm(x) + return self.join(self.long(x) + x) + +class ExtractBlock(nn.Module): + """Increase no. of channels by [out/in]""" -class Interposer(nn.Module): - def __init__(self): + def __init__(self, ch_in, ch_out): super().__init__() - self.chan = 4 - self.hid = 128 - - self.head_join = nn.ReLU() - self.head_short = nn.Conv2d(self.chan, self.hid, kernel_size=3, stride=1, padding=1) - self.head_long = nn.Sequential( - nn.Conv2d(self.chan, self.hid, kernel_size=3, stride=1, padding=1), - nn.LeakyReLU(0.1), - nn.Conv2d(self.hid, self.hid, kernel_size=3, stride=1, padding=1), - nn.LeakyReLU(0.1), - nn.Conv2d(self.hid, self.hid, kernel_size=3, stride=1, padding=1), + self.join = nn.ReLU() + self.short = nn.Conv2d(ch_in, ch_out, kernel_size=3, stride=1, padding=1) + self.long = nn.Sequential( + nn.Conv2d(ch_in, ch_out, kernel_size=3, stride=1, padding=1), + nn.SiLU(), + nn.Conv2d(ch_out, ch_out, kernel_size=3, stride=1, padding=1), + nn.SiLU(), + nn.Conv2d(ch_out, ch_out, kernel_size=3, stride=1, padding=1), + nn.Dropout(0.1) ) + + def forward(self, x): + return self.join(self.long(x) + self.short(x)) + + +class InterposerModel(nn.Module): + """Main neural network""" + + def __init__(self, ch_in=4, ch_out=4, ch_mid=64, scale=1.0, blocks=12): + super().__init__() + self.ch_in = ch_in + self.ch_out = ch_out + self.ch_mid = ch_mid + self.blocks = blocks + self.scale = scale + + self.head = ExtractBlock(self.ch_in, self.ch_mid) self.core = nn.Sequential( - Block(self.hid), - Block(self.hid), - Block(self.hid), - ) - self.tail = nn.Sequential( - nn.ReLU(), - nn.Conv2d(self.hid, self.chan, kernel_size=3, stride=1, padding=1) + nn.Upsample(scale_factor=self.scale, mode="nearest"), + *[ResBlock(self.ch_mid) for _ in range(blocks)], + nn.BatchNorm2d(self.ch_mid), + nn.SiLU(), ) + self.tail = nn.Conv2d(self.ch_mid, self.ch_out, kernel_size=3, stride=1, padding=1) def forward(self, x): - y = self.head_join( - self.head_long(x) + - self.head_short(x) - ) + y = self.head(x) z = self.core(y) return self.tail(z) vae_approx_model = None -vae_approx_filename = os.path.join(path_vae_approx, 'xl-to-v1_interposer-v3.1.safetensors') +vae_approx_filename = os.path.join(path_vae_approx, 'xl-to-v1_interposer-v4.0.safetensors') def parse(x): @@ -72,7 +88,7 @@ def parse(x): x_origin = x.clone() if vae_approx_model is None: - model = Interposer() + model = InterposerModel() model.eval() sd = sf.load_file(vae_approx_filename) model.load_state_dict(sd) diff --git a/launch.py b/launch.py index 5c865e6d7..5d40cc5b0 100644 --- a/launch.py +++ b/launch.py @@ -62,8 +62,8 @@ def prepare_environment(): vae_approx_filenames = [ ('xlvaeapp.pth', 'https://huggingface.co/lllyasviel/misc/resolve/main/xlvaeapp.pth'), ('vaeapp_sd15.pth', 'https://huggingface.co/lllyasviel/misc/resolve/main/vaeapp_sd15.pt'), - ('xl-to-v1_interposer-v3.1.safetensors', - 'https://huggingface.co/lllyasviel/misc/resolve/main/xl-to-v1_interposer-v3.1.safetensors') + ('xl-to-v1_interposer-v4.0.safetensors', + 'https://huggingface.co/mashb1t/misc/resolve/main/xl-to-v1_interposer-v4.0.safetensors') ] From dbf49d323eca159499f23b2c055244144ca8fade Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Wed, 17 Apr 2024 22:23:18 +0200 Subject: [PATCH 03/42] feat: add button to reconnect UI without having to reload the page (#2727) * feat: add button to reconnect UI without having to reload the page * qa: add missing semicolon --- javascript/script.js | 37 +++++++++++++++++++++++++++++++++++++ language/en.json | 1 + webui.py | 11 ++++++++++- 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/javascript/script.js b/javascript/script.js index 9aa0b5c16..d379a783f 100644 --- a/javascript/script.js +++ b/javascript/script.js @@ -122,6 +122,43 @@ document.addEventListener("DOMContentLoaded", function() { initStylePreviewOverlay(); }); +var onAppend = function(elem, f) { + var observer = new MutationObserver(function(mutations) { + mutations.forEach(function(m) { + if (m.addedNodes.length) { + f(m.addedNodes); + } + }); + }); + observer.observe(elem, {childList: true}); +} + +function addObserverIfDesiredNodeAvailable(querySelector, callback) { + var elem = document.querySelector(querySelector); + if (!elem) { + window.setTimeout(() => addObserverIfDesiredNodeAvailable(querySelector, callback), 1000); + return; + } + + onAppend(elem, callback); +} + +/** + * Show reset button on toast "Connection errored out." + */ +addObserverIfDesiredNodeAvailable(".toast-wrap", function(added) { + added.forEach(function(element) { + if (element.innerText.includes("Connection errored out.")) { + window.setTimeout(function() { + document.getElementById("reset_button").classList.remove("hidden"); + document.getElementById("generate_button").classList.add("hidden"); + document.getElementById("skip_button").classList.add("hidden"); + document.getElementById("stop_button").classList.add("hidden"); + }); + } + }); +}); + /** * Add a ctrl+enter as a shortcut to start a generation */ diff --git a/language/en.json b/language/en.json index fefc79c47..d10c29dcf 100644 --- a/language/en.json +++ b/language/en.json @@ -4,6 +4,7 @@ "Generate": "Generate", "Skip": "Skip", "Stop": "Stop", + "Reconnect and Reset UI": "Reconnect and Reset UI", "Input Image": "Input Image", "Advanced": "Advanced", "Upscale or Variation": "Upscale or Variation", diff --git a/webui.py b/webui.py index 98780bff7..ababb8b0e 100644 --- a/webui.py +++ b/webui.py @@ -123,8 +123,9 @@ def generate_clicked(task: worker.AsyncTask): with gr.Column(scale=3, min_width=0): generate_button = gr.Button(label="Generate", value="Generate", elem_classes='type_row', elem_id='generate_button', visible=True) + reset_button = gr.Button(label="Reconnect and Reset UI", value="Reconnect and Reset UI", elem_classes='type_row', elem_id='reset_button', visible=False) load_parameter_button = gr.Button(label="Load Parameters", value="Load Parameters", elem_classes='type_row', elem_id='load_parameter_button', visible=False) - skip_button = gr.Button(label="Skip", value="Skip", elem_classes='type_row_half', visible=False) + skip_button = gr.Button(label="Skip", value="Skip", elem_classes='type_row_half', elem_id='skip_button', visible=False) stop_button = gr.Button(label="Stop", value="Stop", elem_classes='type_row_half', elem_id='stop_button', visible=False) def stop_clicked(currentTask): @@ -688,6 +689,14 @@ def trigger_metadata_import(filepath, state_is_generating): .then(fn=update_history_link, outputs=history_link) \ .then(fn=lambda: None, _js='playNotification').then(fn=lambda: None, _js='refresh_grid_delayed') + reset_button.click(lambda: [worker.AsyncTask(args=[]), False, gr.update(visible=True, interactive=True)] + + [gr.update(visible=False)] * 6 + + [gr.update(visible=True, value=[])], + outputs=[currentTask, state_is_generating, generate_button, + reset_button, stop_button, skip_button, + progress_html, progress_window, progress_gallery, gallery], + queue=False) + for notification_file in ['notification.ogg', 'notification.mp3']: if os.path.exists(notification_file): gr.Audio(interactive=False, value=notification_file, elem_id='audio_notification', visible=False) From c36e951781b17b36657369854a10664b5c09b118 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Sat, 4 May 2024 14:37:40 +0200 Subject: [PATCH 04/42] Revert "fix: use LF as line breaks for Docker entrypoint.sh (#2843)" (#2865) False alarm, worked as intended before. Sorry for the fuzz. This reverts commit d16a54edd69f82158ae7ffe5669618db33a01ac7. --- entrypoint.sh | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 57b06c6be..d0dba09c2 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1 +1,33 @@ -#!/bin/bash ORIGINALDIR=/content/app # Use predefined DATADIR if it is defined [[ x"${DATADIR}" == "x" ]] && DATADIR=/content/data # Make persistent dir from original dir function mklink () { mkdir -p $DATADIR/$1 ln -s $DATADIR/$1 $ORIGINALDIR } # Copy old files from import dir function import () { (test -d /import/$1 && cd /import/$1 && cp -Rpn . $DATADIR/$1/) } cd $ORIGINALDIR # models mklink models # Copy original files (cd $ORIGINALDIR/models.org && cp -Rpn . $ORIGINALDIR/models/) # Import old files import models # outputs mklink outputs # Import old files import outputs # Start application python launch.py $* \ No newline at end of file +#!/bin/bash + +ORIGINALDIR=/content/app +# Use predefined DATADIR if it is defined +[[ x"${DATADIR}" == "x" ]] && DATADIR=/content/data + +# Make persistent dir from original dir +function mklink () { + mkdir -p $DATADIR/$1 + ln -s $DATADIR/$1 $ORIGINALDIR +} + +# Copy old files from import dir +function import () { + (test -d /import/$1 && cd /import/$1 && cp -Rpn . $DATADIR/$1/) +} + +cd $ORIGINALDIR + +# models +mklink models +# Copy original files +(cd $ORIGINALDIR/models.org && cp -Rpn . $ORIGINALDIR/models/) +# Import old files +import models + +# outputs +mklink outputs +# Import old files +import outputs + +# Start application +python launch.py $* From c32bc5e199f7a0a45736f10c248cd1955433a609 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Thu, 9 May 2024 18:59:35 +0200 Subject: [PATCH 05/42] feat: add optional model VAE select (#2867) * Revert "fix: use LF as line breaks for Docker entrypoint.sh (#2843)" (#2865) False alarm, worked as intended before. Sorry for the fuzz. This reverts commit d16a54edd69f82158ae7ffe5669618db33a01ac7. * feat: add VAE select * feat: use different default label, add translation * fix: do not reload model when VAE stays the same * refactor: code cleanup * feat: add metadata handling --- language/en.json | 2 ++ ldm_patched/modules/sd.py | 13 +++++++++---- modules/async_worker.py | 6 ++++-- modules/config.py | 14 +++++++++++++- modules/core.py | 10 ++++++---- modules/default_pipeline.py | 22 ++++++++++++++-------- modules/flags.py | 2 ++ modules/meta_parser.py | 31 ++++++++++++++++++++++++------- modules/util.py | 3 +++ webui.py | 11 +++++++---- 10 files changed, 84 insertions(+), 30 deletions(-) diff --git a/language/en.json b/language/en.json index d10c29dcf..1fe78662b 100644 --- a/language/en.json +++ b/language/en.json @@ -340,6 +340,8 @@ "sgm_uniform": "sgm_uniform", "simple": "simple", "ddim_uniform": "ddim_uniform", + "VAE": "VAE", + "Default (model)": "Default (model)", "Forced Overwrite of Sampling Step": "Forced Overwrite of Sampling Step", "Set as -1 to disable. For developer debugging.": "Set as -1 to disable. For developer debugging.", "Forced Overwrite of Refiner Switch Step": "Forced Overwrite of Refiner Switch Step", diff --git a/ldm_patched/modules/sd.py b/ldm_patched/modules/sd.py index e197c39ca..282f2559a 100644 --- a/ldm_patched/modules/sd.py +++ b/ldm_patched/modules/sd.py @@ -427,12 +427,13 @@ class EmptyClass: return (ldm_patched.modules.model_patcher.ModelPatcher(model, load_device=model_management.get_torch_device(), offload_device=offload_device), clip, vae) -def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, output_clipvision=False, embedding_directory=None, output_model=True): +def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, output_clipvision=False, embedding_directory=None, output_model=True, vae_filename_param=None): sd = ldm_patched.modules.utils.load_torch_file(ckpt_path) sd_keys = sd.keys() clip = None clipvision = None vae = None + vae_filename = None model = None model_patcher = None clip_target = None @@ -462,8 +463,12 @@ class WeightsLoader(torch.nn.Module): model.load_model_weights(sd, "model.diffusion_model.") if output_vae: - vae_sd = ldm_patched.modules.utils.state_dict_prefix_replace(sd, {"first_stage_model.": ""}, filter_keys=True) - vae_sd = model_config.process_vae_state_dict(vae_sd) + if vae_filename_param is None: + vae_sd = ldm_patched.modules.utils.state_dict_prefix_replace(sd, {"first_stage_model.": ""}, filter_keys=True) + vae_sd = model_config.process_vae_state_dict(vae_sd) + else: + vae_sd = ldm_patched.modules.utils.load_torch_file(vae_filename_param) + vae_filename = vae_filename_param vae = VAE(sd=vae_sd) if output_clip: @@ -485,7 +490,7 @@ class WeightsLoader(torch.nn.Module): print("loaded straight to GPU") model_management.load_model_gpu(model_patcher) - return (model_patcher, clip, vae, clipvision) + return model_patcher, clip, vae, vae_filename, clipvision def load_unet_state_dict(sd): #load unet in diffusers format diff --git a/modules/async_worker.py b/modules/async_worker.py index d8a1e072d..3576c4ec8 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -166,6 +166,7 @@ def handler(async_task): adaptive_cfg = args.pop() sampler_name = args.pop() scheduler_name = args.pop() + vae_name = args.pop() overwrite_step = args.pop() overwrite_switch = args.pop() overwrite_width = args.pop() @@ -428,7 +429,7 @@ def handler(async_task): progressbar(async_task, 3, 'Loading models ...') pipeline.refresh_everything(refiner_model_name=refiner_model_name, base_model_name=base_model_name, loras=loras, base_model_additional_loras=base_model_additional_loras, - use_synthetic_refiner=use_synthetic_refiner) + use_synthetic_refiner=use_synthetic_refiner, vae_name=vae_name) progressbar(async_task, 3, 'Processing prompts ...') tasks = [] @@ -869,6 +870,7 @@ def callback(step, x0, x, total_steps, y): d.append(('Sampler', 'sampler', sampler_name)) d.append(('Scheduler', 'scheduler', scheduler_name)) + d.append(('VAE', 'vae', vae_name)) d.append(('Seed', 'seed', str(task['task_seed']))) if freeu_enabled: @@ -883,7 +885,7 @@ def callback(step, x0, x, total_steps, y): metadata_parser = modules.meta_parser.get_metadata_parser(metadata_scheme) metadata_parser.set_data(task['log_positive_prompt'], task['positive'], task['log_negative_prompt'], task['negative'], - steps, base_model_name, refiner_model_name, loras) + steps, base_model_name, refiner_model_name, loras, vae_name) d.append(('Metadata Scheme', 'metadata_scheme', metadata_scheme.value if save_metadata_to_images else save_metadata_to_images)) d.append(('Version', 'version', 'Fooocus v' + fooocus_version.version)) img_paths.append(log(x, d, metadata_parser, output_format)) diff --git a/modules/config.py b/modules/config.py index b81e218a0..f11460c8d 100644 --- a/modules/config.py +++ b/modules/config.py @@ -189,6 +189,7 @@ def get_dir_or_set_default(key, default_value, as_array=False, make_directory=Fa paths_loras = get_dir_or_set_default('path_loras', ['../models/loras/'], True) path_embeddings = get_dir_or_set_default('path_embeddings', '../models/embeddings/') path_vae_approx = get_dir_or_set_default('path_vae_approx', '../models/vae_approx/') +path_vae = get_dir_or_set_default('path_vae', '../models/vae/') path_upscale_models = get_dir_or_set_default('path_upscale_models', '../models/upscale_models/') path_inpaint = get_dir_or_set_default('path_inpaint', '../models/inpaint/') path_controlnet = get_dir_or_set_default('path_controlnet', '../models/controlnet/') @@ -346,6 +347,11 @@ def init_temp_path(path: str | None, default_path: str) -> str: default_value='karras', validator=lambda x: x in modules.flags.scheduler_list ) +default_vae = get_config_item_or_set_default( + key='default_vae', + default_value=modules.flags.default_vae, + validator=lambda x: isinstance(x, str) +) default_styles = get_config_item_or_set_default( key='default_styles', default_value=[ @@ -535,6 +541,7 @@ def add_ratio(x): model_filenames = [] lora_filenames = [] +vae_filenames = [] wildcard_filenames = [] sdxl_lcm_lora = 'sdxl_lcm_lora.safetensors' @@ -546,15 +553,20 @@ def get_model_filenames(folder_paths, extensions=None, name_filter=None): if extensions is None: extensions = ['.pth', '.ckpt', '.bin', '.safetensors', '.fooocus.patch'] files = [] + + if not isinstance(folder_paths, list): + folder_paths = [folder_paths] for folder in folder_paths: files += get_files_from_folder(folder, extensions, name_filter) + return files def update_files(): - global model_filenames, lora_filenames, wildcard_filenames, available_presets + global model_filenames, lora_filenames, vae_filenames, wildcard_filenames, available_presets model_filenames = get_model_filenames(paths_checkpoints) lora_filenames = get_model_filenames(paths_loras) + vae_filenames = get_model_filenames(path_vae) wildcard_filenames = get_files_from_folder(path_wildcards, ['.txt']) available_presets = get_presets() return diff --git a/modules/core.py b/modules/core.py index 38ee8e8dc..3ca4cc5b8 100644 --- a/modules/core.py +++ b/modules/core.py @@ -35,12 +35,13 @@ class StableDiffusionModel: - def __init__(self, unet=None, vae=None, clip=None, clip_vision=None, filename=None): + def __init__(self, unet=None, vae=None, clip=None, clip_vision=None, filename=None, vae_filename=None): self.unet = unet self.vae = vae self.clip = clip self.clip_vision = clip_vision self.filename = filename + self.vae_filename = vae_filename self.unet_with_lora = unet self.clip_with_lora = clip self.visited_loras = '' @@ -142,9 +143,10 @@ def apply_controlnet(positive, negative, control_net, image, strength, start_per @torch.no_grad() @torch.inference_mode() -def load_model(ckpt_filename): - unet, clip, vae, clip_vision = load_checkpoint_guess_config(ckpt_filename, embedding_directory=path_embeddings) - return StableDiffusionModel(unet=unet, clip=clip, vae=vae, clip_vision=clip_vision, filename=ckpt_filename) +def load_model(ckpt_filename, vae_filename=None): + unet, clip, vae, vae_filename, clip_vision = load_checkpoint_guess_config(ckpt_filename, embedding_directory=path_embeddings, + vae_filename_param=vae_filename) + return StableDiffusionModel(unet=unet, clip=clip, vae=vae, clip_vision=clip_vision, filename=ckpt_filename, vae_filename=vae_filename) @torch.no_grad() diff --git a/modules/default_pipeline.py b/modules/default_pipeline.py index 190601ecf..38f914c57 100644 --- a/modules/default_pipeline.py +++ b/modules/default_pipeline.py @@ -3,6 +3,7 @@ import torch import modules.patch import modules.config +import modules.flags import ldm_patched.modules.model_management import ldm_patched.modules.latent_formats import modules.inpaint_worker @@ -58,17 +59,21 @@ def assert_model_integrity(): @torch.no_grad() @torch.inference_mode() -def refresh_base_model(name): +def refresh_base_model(name, vae_name=None): global model_base filename = get_file_from_folder_list(name, modules.config.paths_checkpoints) - if model_base.filename == filename: + vae_filename = None + if vae_name is not None and vae_name != modules.flags.default_vae: + vae_filename = get_file_from_folder_list(vae_name, modules.config.path_vae) + + if model_base.filename == filename and model_base.vae_filename == vae_filename: return - model_base = core.StableDiffusionModel() - model_base = core.load_model(filename) + model_base = core.load_model(filename, vae_filename) print(f'Base model loaded: {model_base.filename}') + print(f'VAE loaded: {model_base.vae_filename}') return @@ -216,7 +221,7 @@ def prepare_text_encoder(async_call=True): @torch.no_grad() @torch.inference_mode() def refresh_everything(refiner_model_name, base_model_name, loras, - base_model_additional_loras=None, use_synthetic_refiner=False): + base_model_additional_loras=None, use_synthetic_refiner=False, vae_name=None): global final_unet, final_clip, final_vae, final_refiner_unet, final_refiner_vae, final_expansion final_unet = None @@ -227,11 +232,11 @@ def refresh_everything(refiner_model_name, base_model_name, loras, if use_synthetic_refiner and refiner_model_name == 'None': print('Synthetic Refiner Activated') - refresh_base_model(base_model_name) + refresh_base_model(base_model_name, vae_name) synthesize_refiner_model() else: refresh_refiner_model(refiner_model_name) - refresh_base_model(base_model_name) + refresh_base_model(base_model_name, vae_name) refresh_loras(loras, base_model_additional_loras=base_model_additional_loras) assert_model_integrity() @@ -254,7 +259,8 @@ def refresh_everything(refiner_model_name, base_model_name, loras, refresh_everything( refiner_model_name=modules.config.default_refiner_model_name, base_model_name=modules.config.default_base_model_name, - loras=get_enabled_loras(modules.config.default_loras) + loras=get_enabled_loras(modules.config.default_loras), + vae_name=modules.config.default_vae, ) diff --git a/modules/flags.py b/modules/flags.py index c9d13fd81..9f2aefb3b 100644 --- a/modules/flags.py +++ b/modules/flags.py @@ -53,6 +53,8 @@ sampler_list = SAMPLER_NAMES scheduler_list = SCHEDULER_NAMES +default_vae = 'Default (model)' + refiner_swap_method = 'joint' cn_ip = "ImagePrompt" diff --git a/modules/meta_parser.py b/modules/meta_parser.py index 70ab8860c..84032e829 100644 --- a/modules/meta_parser.py +++ b/modules/meta_parser.py @@ -46,6 +46,7 @@ def load_parameter_button_click(raw_metadata: dict | str, is_generating: bool): get_float('refiner_switch', 'Refiner Switch', loaded_parameter_dict, results) get_str('sampler', 'Sampler', loaded_parameter_dict, results) get_str('scheduler', 'Scheduler', loaded_parameter_dict, results) + get_str('vae', 'VAE', loaded_parameter_dict, results) get_seed('seed', 'Seed', loaded_parameter_dict, results) if is_generating: @@ -253,6 +254,7 @@ def __init__(self): self.refiner_model_name: str = '' self.refiner_model_hash: str = '' self.loras: list = [] + self.vae_name: str = '' @abstractmethod def get_scheme(self) -> MetadataScheme: @@ -267,7 +269,7 @@ def parse_string(self, metadata: dict) -> str: raise NotImplementedError def set_data(self, raw_prompt, full_prompt, raw_negative_prompt, full_negative_prompt, steps, base_model_name, - refiner_model_name, loras): + refiner_model_name, loras, vae_name): self.raw_prompt = raw_prompt self.full_prompt = full_prompt self.raw_negative_prompt = raw_negative_prompt @@ -289,6 +291,7 @@ def set_data(self, raw_prompt, full_prompt, raw_negative_prompt, full_negative_p lora_path = get_file_from_folder_list(lora_name, modules.config.paths_loras) lora_hash = get_sha256(lora_path) self.loras.append((Path(lora_name).stem, lora_weight, lora_hash)) + self.vae_name = Path(vae_name).stem @staticmethod def remove_special_loras(lora_filenames): @@ -310,6 +313,7 @@ def get_scheme(self) -> MetadataScheme: 'steps': 'Steps', 'sampler': 'Sampler', 'scheduler': 'Scheduler', + 'vae': 'VAE', 'guidance_scale': 'CFG scale', 'seed': 'Seed', 'resolution': 'Size', @@ -397,13 +401,12 @@ def parse_json(self, metadata: str) -> dict: data['sampler'] = k break - for key in ['base_model', 'refiner_model']: + for key in ['base_model', 'refiner_model', 'vae']: if key in data: - for filename in modules.config.model_filenames: - path = Path(filename) - if data[key] == path.stem: - data[key] = filename - break + if key == 'vae': + self.add_extension_to_filename(data, modules.config.vae_filenames, 'vae') + else: + self.add_extension_to_filename(data, modules.config.model_filenames, key) lora_data = '' if 'lora_weights' in data and data['lora_weights'] != '': @@ -433,6 +436,7 @@ def parse_string(self, metadata: dict) -> str: sampler = data['sampler'] scheduler = data['scheduler'] + if sampler in SAMPLERS and SAMPLERS[sampler] != '': sampler = SAMPLERS[sampler] if sampler not in CIVITAI_NO_KARRAS and scheduler == 'karras': @@ -451,6 +455,7 @@ def parse_string(self, metadata: dict) -> str: self.fooocus_to_a1111['performance']: data['performance'], self.fooocus_to_a1111['scheduler']: scheduler, + self.fooocus_to_a1111['vae']: Path(data['vae']).stem, # workaround for multiline prompts self.fooocus_to_a1111['raw_prompt']: self.raw_prompt, self.fooocus_to_a1111['raw_negative_prompt']: self.raw_negative_prompt, @@ -491,6 +496,14 @@ def parse_string(self, metadata: dict) -> str: negative_prompt_text = f"\nNegative prompt: {negative_prompt_resolved}" if negative_prompt_resolved else "" return f"{positive_prompt_resolved}{negative_prompt_text}\n{generation_params_text}".strip() + @staticmethod + def add_extension_to_filename(data, filenames, key): + for filename in filenames: + path = Path(filename) + if data[key] == path.stem: + data[key] = filename + break + class FooocusMetadataParser(MetadataParser): def get_scheme(self) -> MetadataScheme: @@ -499,6 +512,7 @@ def get_scheme(self) -> MetadataScheme: def parse_json(self, metadata: dict) -> dict: model_filenames = modules.config.model_filenames.copy() lora_filenames = modules.config.lora_filenames.copy() + vae_filenames = modules.config.vae_filenames.copy() self.remove_special_loras(lora_filenames) for key, value in metadata.items(): if value in ['', 'None']: @@ -507,6 +521,8 @@ def parse_json(self, metadata: dict) -> dict: metadata[key] = self.replace_value_with_filename(key, value, model_filenames) elif key.startswith('lora_combined_'): metadata[key] = self.replace_value_with_filename(key, value, lora_filenames) + elif key == 'vae': + metadata[key] = self.replace_value_with_filename(key, value, vae_filenames) else: continue @@ -533,6 +549,7 @@ def parse_string(self, metadata: list) -> str: res['refiner_model'] = self.refiner_model_name res['refiner_model_hash'] = self.refiner_model_hash + res['vae'] = self.vae_name res['loras'] = self.loras if modules.config.metadata_created_by != '': diff --git a/modules/util.py b/modules/util.py index 9e0fb294b..d2feecb64 100644 --- a/modules/util.py +++ b/modules/util.py @@ -371,6 +371,9 @@ def is_json(data: str) -> bool: def get_file_from_folder_list(name, folders): + if not isinstance(folders, list): + folders = [folders] + for folder in folders: filename = os.path.abspath(os.path.realpath(os.path.join(folder, name))) if os.path.isfile(filename): diff --git a/webui.py b/webui.py index ababb8b0e..eec6054a7 100644 --- a/webui.py +++ b/webui.py @@ -407,6 +407,8 @@ def update_history_link(): value=modules.config.default_sampler) scheduler_name = gr.Dropdown(label='Scheduler', choices=flags.scheduler_list, value=modules.config.default_scheduler) + vae_name = gr.Dropdown(label='VAE', choices=[modules.flags.default_vae] + modules.config.vae_filenames, + value=modules.config.default_vae, show_label=True) generate_image_grid = gr.Checkbox(label='Generate Image Grid for Each Batch', info='(Experimental) This may cause performance problems on some computers and certain internet conditions.', @@ -529,6 +531,7 @@ def refresh_files_clicked(): modules.config.update_files() results = [gr.update(choices=modules.config.model_filenames)] results += [gr.update(choices=['None'] + modules.config.model_filenames)] + results += [gr.update(choices=['None'] + modules.config.vae_filenames)] if not args_manager.args.disable_preset_selection: results += [gr.update(choices=modules.config.available_presets)] for i in range(modules.config.default_max_lora_number): @@ -536,7 +539,7 @@ def refresh_files_clicked(): gr.update(choices=['None'] + modules.config.lora_filenames), gr.update()] return results - refresh_files_output = [base_model, refiner_model] + refresh_files_output = [base_model, refiner_model, vae_name] if not args_manager.args.disable_preset_selection: refresh_files_output += [preset_selection] refresh_files.click(refresh_files_clicked, [], refresh_files_output + lora_ctrls, @@ -548,8 +551,8 @@ def refresh_files_clicked(): performance_selection, overwrite_step, overwrite_switch, aspect_ratios_selection, overwrite_width, overwrite_height, guidance_scale, sharpness, adm_scaler_positive, adm_scaler_negative, adm_scaler_end, refiner_swap_method, adaptive_cfg, base_model, - refiner_model, refiner_switch, sampler_name, scheduler_name, seed_random, image_seed, - generate_button, load_parameter_button] + freeu_ctrls + lora_ctrls + refiner_model, refiner_switch, sampler_name, scheduler_name, vae_name, seed_random, + image_seed, generate_button, load_parameter_button] + freeu_ctrls + lora_ctrls if not args_manager.args.disable_preset_selection: def preset_selection_change(preset, is_generating): @@ -635,7 +638,7 @@ def inpaint_mode_change(mode): ctrls += [outpaint_selections, inpaint_input_image, inpaint_additional_prompt, inpaint_mask_image] ctrls += [disable_preview, disable_intermediate_results, disable_seed_increment] ctrls += [adm_scaler_positive, adm_scaler_negative, adm_scaler_end, adaptive_cfg] - ctrls += [sampler_name, scheduler_name] + ctrls += [sampler_name, scheduler_name, vae_name] ctrls += [overwrite_step, overwrite_switch, overwrite_width, overwrite_height, overwrite_vary_strength] ctrls += [overwrite_upscale_strength, mixing_image_prompt_and_vary_upscale, mixing_image_prompt_and_inpaint] ctrls += [debugging_cn_preprocessor, skipping_cn_preprocessor, canny_low_threshold, canny_high_threshold] From f54364fe4ebd737349611c1d040703b0ac7ace68 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Thu, 9 May 2024 19:02:04 +0200 Subject: [PATCH 06/42] feat: add random style checkbox to styles selection (#2855) * feat: add random style * feat: rename random to random style, add translation * feat: add preview image for random style --- language/en.json | 1 + modules/async_worker.py | 11 ++++++++--- modules/sdxl_styles.py | 10 ++++++++-- sdxl_styles/samples/random_style.jpg | Bin 0 -> 1454 bytes 4 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 sdxl_styles/samples/random_style.jpg diff --git a/language/en.json b/language/en.json index 1fe78662b..20189b28a 100644 --- a/language/en.json +++ b/language/en.json @@ -58,6 +58,7 @@ "\ud83d\udcda History Log": "\uD83D\uDCDA History Log", "Image Style": "Image Style", "Fooocus V2": "Fooocus V2", + "Random Style": "Random Style", "Default (Slightly Cinematic)": "Default (Slightly Cinematic)", "Fooocus Masterpiece": "Fooocus Masterpiece", "Fooocus Photograph": "Fooocus Photograph", diff --git a/modules/async_worker.py b/modules/async_worker.py index 3576c4ec8..432bfe9bc 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -43,7 +43,7 @@ def worker(): import fooocus_version import args_manager - from modules.sdxl_styles import apply_style, apply_wildcards, fooocus_expansion, apply_arrays + from modules.sdxl_styles import apply_style, get_random_style, apply_wildcards, fooocus_expansion, apply_arrays, random_style_name from modules.private_logger import log from extras.expansion import safe_str from modules.util import remove_empty_str, HWC3, resize_image, get_image_shape_ceil, set_image_shape_ceil, \ @@ -450,8 +450,12 @@ def handler(async_task): positive_basic_workloads = [] negative_basic_workloads = [] + task_styles = style_selections.copy() if use_style: - for s in style_selections: + for i, s in enumerate(task_styles): + if s == random_style_name: + s = get_random_style(task_rng) + task_styles[i] = s p, n = apply_style(s, positive=task_prompt) positive_basic_workloads = positive_basic_workloads + p negative_basic_workloads = negative_basic_workloads + n @@ -479,6 +483,7 @@ def handler(async_task): negative_top_k=len(negative_basic_workloads), log_positive_prompt='\n'.join([task_prompt] + task_extra_positive_prompts), log_negative_prompt='\n'.join([task_negative_prompt] + task_extra_negative_prompts), + styles=task_styles )) if use_expansion: @@ -843,7 +848,7 @@ def callback(step, x0, x, total_steps, y): d = [('Prompt', 'prompt', task['log_positive_prompt']), ('Negative Prompt', 'negative_prompt', task['log_negative_prompt']), ('Fooocus V2 Expansion', 'prompt_expansion', task['expansion']), - ('Styles', 'styles', str(raw_style_selections)), + ('Styles', 'styles', str(task['styles'] if not use_expansion else [fooocus_expansion] + task['styles'])), ('Performance', 'performance', performance_selection.value)] if performance_selection.steps() != steps: diff --git a/modules/sdxl_styles.py b/modules/sdxl_styles.py index 77ad6b574..5b6afb590 100644 --- a/modules/sdxl_styles.py +++ b/modules/sdxl_styles.py @@ -5,6 +5,7 @@ import modules.config from modules.util import get_files_from_folder +from random import Random # cannot use modules.config - validators causing circular imports styles_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../sdxl_styles/')) @@ -50,8 +51,13 @@ def normalize_key(k): print(f'Failed to load style file {styles_file}') style_keys = list(styles.keys()) -fooocus_expansion = "Fooocus V2" -legal_style_names = [fooocus_expansion] + style_keys +fooocus_expansion = 'Fooocus V2' +random_style_name = 'Random Style' +legal_style_names = [fooocus_expansion, random_style_name] + style_keys + + +def get_random_style(rng: Random) -> str: + return rng.choice(list(styles.items()))[0] def apply_style(style, positive): diff --git a/sdxl_styles/samples/random_style.jpg b/sdxl_styles/samples/random_style.jpg new file mode 100644 index 0000000000000000000000000000000000000000..9f685108fdcf78409e488d79cc2c245fec3ad06e GIT binary patch literal 1454 zcmex=ma3|jiIItm zOAI4iKMQ#V{6EAX$idLS(7?>7#K0uT$SlbC{|JLD$RAA1j3D1Y0V4+|6B|1#Gt2*5 z3>*;g^Do3AxHTA*=9V*ky#3T}oy+sk*U|b%zddvNE zjs=&Z%SwbAm^4hL-Cdy6?3sIuNrOSPVbg*AKu3xoxen?`u-ljfU~UG|c+@eYsWTH` zU}R=uW@TexVTYK&%*e#T%ErJh$RQx45GbsuWN1_<;^5fmRI=%^sH#D5V)7PfXfWC{ zJi9LSIQG`#nC#y2^(XXI?^Z;Avz7RIyRSOgMZUH3nbY2yHnV@~>-68Qc%R*KyfZ7) zap|iSSJV1uxP7e@zxd`%$m*@TPR3;IO)7b0SsH$7g0p4a`_TKo>#n^xnz>T6^Mc3K ztMX!wOizAq3>M?NQoeOZg3GQi>w~_!6zzBOk*?a%`TAR(&gnz<_Gv6wcf0tunx=T` z%O?(dZH~FsiK}b9pKm3r#&c_~!&@GY2E}vXzak8`#ytJ4lx+H0Z{^JEF0-~w-&1nH z^3bxwq3;;%3;*TE)vP$PYgxoy!BrRMIR*1Q>srbyzT?5)W0e~V{pRwmWD@n8S2jy# z1J46(TYX9CBiGe;=_|dzwdlF7xSq3n*Vn}C;+vBH8K!e-#_!fK`c;=2yJ78)lz`?| ze{t=l z^L=YTQN$6Mh5BKF!Xd}fpMLqXb$8nBVvc3(TyipVP2N5hzVyE2V&Ls<;k$ezPEQVe zvNtYE<@+)Dr)TD_wiK_snYC6fhEx2&>ah5q-*a*p*7O{n@l9!p(=tx+#azom|H|x6 zsZsxE&EM?b{Uh`5`ndBQHPYKm-dgh<`FCAb+rO57ul>^U``=1iIVS5>-B8}U=21#W zk)3Ci!jXBZ$1g_pebA?)`tFHA<+OT!! z`3ZKb7?n6@U*NcJ!mve9i6MdG=ACOixo%rmzUka+u;+JJTHvgTpV{tK z8+;Rk|$3d_P;TUH576 z^`5UqXIFY^hcCGLswaPQ^rE~Idp0gh{kqg|PDW4Z;ovSy;U#-*lAV@IG47Q*SNi<( z`#T-IE-t=;!Y+O(KUpB~{xXweGcK3SxGbe_56aGhh(ZaHO%ov5o)MS{L4_AC1;FAA Hq~In1cXT Date: Thu, 9 May 2024 19:03:30 +0200 Subject: [PATCH 07/42] feat: update anime from animaPencilXL_v100 to animaPencilXL_v310 (#2454) * feat: update anime from animaPencilXL_v100 to animaPencilXL_v200 * feat: update animaPencilXL from 2.0.0 to 2.6.0 * feat: update animaPencilXL from 2.6.0 to 3.1.0 * feat: reduce cfg as suggested by vendor from 3.0.0 https://civitai.com/models/261336?modelVersionId=435001 "recommend to decrease CFG scale." + all examples are in CFG 6 --- presets/anime.json | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/presets/anime.json b/presets/anime.json index 2610677cc..78607edbf 100644 --- a/presets/anime.json +++ b/presets/anime.json @@ -1,5 +1,5 @@ { - "default_model": "animaPencilXL_v100.safetensors", + "default_model": "animaPencilXL_v310.safetensors", "default_refiner": "None", "default_refiner_switch": 0.5, "default_loras": [ @@ -29,7 +29,7 @@ 1.0 ] ], - "default_cfg_scale": 7.0, + "default_cfg_scale": 6.0, "default_sample_sharpness": 2.0, "default_sampler": "dpmpp_2m_sde_gpu", "default_scheduler": "karras", @@ -43,9 +43,15 @@ ], "default_aspect_ratio": "896*1152", "checkpoint_downloads": { - "animaPencilXL_v100.safetensors": "https://huggingface.co/lllyasviel/fav_models/resolve/main/fav/animaPencilXL_v100.safetensors" + "animaPencilXL_v310.safetensors": "https://huggingface.co/mashb1t/fav_models/resolve/main/fav/animaPencilXL_v310.safetensors" }, "embeddings_downloads": {}, "lora_downloads": {}, - "previous_default_models": [] + "previous_default_models": [ + "animaPencilXL_v300.safetensors", + "animaPencilXL_v260.safetensors", + "animaPencilXL_v210.safetensors", + "animaPencilXL_v200.safetensors", + "animaPencilXL_v100.safetensors" + ] } \ No newline at end of file From 052393bb9bfa6fe66d1f8d3fdf8da38605998eff Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Thu, 9 May 2024 19:13:59 +0200 Subject: [PATCH 08/42] refactor: rename label for reconnect button (#2893) * feat: add button to reconnect UI without having to reload the page * qa: add missing semicolon * refactor: rename button label to "Reconnect" --- language/en.json | 2 +- webui.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/language/en.json b/language/en.json index 20189b28a..e9cd6b737 100644 --- a/language/en.json +++ b/language/en.json @@ -4,7 +4,7 @@ "Generate": "Generate", "Skip": "Skip", "Stop": "Stop", - "Reconnect and Reset UI": "Reconnect and Reset UI", + "Reconnect": "Reconnect", "Input Image": "Input Image", "Advanced": "Advanced", "Upscale or Variation": "Upscale or Variation", diff --git a/webui.py b/webui.py index eec6054a7..85b2c0df3 100644 --- a/webui.py +++ b/webui.py @@ -123,7 +123,7 @@ def generate_clicked(task: worker.AsyncTask): with gr.Column(scale=3, min_width=0): generate_button = gr.Button(label="Generate", value="Generate", elem_classes='type_row', elem_id='generate_button', visible=True) - reset_button = gr.Button(label="Reconnect and Reset UI", value="Reconnect and Reset UI", elem_classes='type_row', elem_id='reset_button', visible=False) + reset_button = gr.Button(label="Reconnect", value="Reconnect", elem_classes='type_row', elem_id='reset_button', visible=False) load_parameter_button = gr.Button(label="Load Parameters", value="Load Parameters", elem_classes='type_row', elem_id='load_parameter_button', visible=False) skip_button = gr.Button(label="Skip", value="Skip", elem_classes='type_row_half', elem_id='skip_button', visible=False) stop_button = gr.Button(label="Stop", value="Stop", elem_classes='type_row_half', elem_id='stop_button', visible=False) From bdd6b1a9b0b182ce62c20642e4c6bd8acec0e4c3 Mon Sep 17 00:00:00 2001 From: docppp <29142757+docppp@users.noreply.github.com> Date: Thu, 9 May 2024 20:25:43 +0200 Subject: [PATCH 09/42] feat: add full raw prompt to history log (#1920) * Update async_worker.py * Update private_logger.py * refactor: only show full prompt details in logs, exclude from image metadata --------- Co-authored-by: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Co-authored-by: Manuel Schmid --- modules/async_worker.py | 2 +- modules/private_logger.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/modules/async_worker.py b/modules/async_worker.py index 432bfe9bc..cde99bdc0 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -893,7 +893,7 @@ def callback(step, x0, x, total_steps, y): steps, base_model_name, refiner_model_name, loras, vae_name) d.append(('Metadata Scheme', 'metadata_scheme', metadata_scheme.value if save_metadata_to_images else save_metadata_to_images)) d.append(('Version', 'version', 'Fooocus v' + fooocus_version.version)) - img_paths.append(log(x, d, metadata_parser, output_format)) + img_paths.append(log(x, d, metadata_parser, output_format, task)) yield_result(async_task, img_paths, do_not_show_finished_images=len(tasks) == 1 or disable_intermediate_results) except ldm_patched.modules.model_management.InterruptProcessingException as e: diff --git a/modules/private_logger.py b/modules/private_logger.py index edd9457d2..eb8f0cc5a 100644 --- a/modules/private_logger.py +++ b/modules/private_logger.py @@ -21,7 +21,7 @@ def get_current_html_path(output_format=None): return html_name -def log(img, metadata, metadata_parser: MetadataParser | None = None, output_format=None) -> str: +def log(img, metadata, metadata_parser: MetadataParser | None = None, output_format=None, task=None) -> str: path_outputs = modules.config.temp_path if args_manager.args.disable_image_log else modules.config.path_outputs output_format = output_format if output_format else modules.config.default_output_format date_string, local_temp_filename, only_name = generate_temp_filename(folder=path_outputs, extension=output_format) @@ -111,9 +111,15 @@ def log(img, metadata, metadata_parser: MetadataParser | None = None, output_for for label, key, value in metadata: value_txt = str(value).replace('\n', '
') item += f"{label}{value_txt}\n" + + if task is not None and 'positive' in task and 'negative' in task: + full_prompt_details = f"""
Positive{', '.join(task['positive'])}
+
Negative{', '.join(task['negative'])}
""" + item += f"Full raw prompt{full_prompt_details}\n" + item += "" - js_txt = urllib.parse.quote(json.dumps({k: v for _, k, v in metadata}, indent=0), safe='') + js_txt = urllib.parse.quote(json.dumps({k: v for _, k, v, in metadata}, indent=0), safe='') item += f"
" item += "" From 96bf89f782376544f4f7f20492c5ae0d6a82001f Mon Sep 17 00:00:00 2001 From: Vishvesh Khanvilkar <158825962+khanvilkarvishvesh@users.noreply.github.com> Date: Fri, 17 May 2024 20:48:45 +0530 Subject: [PATCH 10/42] fix: use correct border radius css property (#2845) --- css/style.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/css/style.css b/css/style.css index c702a7257..b9e6e2ce1 100644 --- a/css/style.css +++ b/css/style.css @@ -391,6 +391,6 @@ progress::after { background-color: #fff8; font-family: monospace; text-align: center; - border-radius-top: 5px; + border-radius: 5px 5px 0px 0px; display: none; /* remove this to enable tooltip in preview image */ } \ No newline at end of file From 5e594685e1f86ffaf4b10d6ca7f11742daca4a84 Mon Sep 17 00:00:00 2001 From: e52fa787 <31095594+e52fa787@users.noreply.github.com> Date: Fri, 17 May 2024 23:25:56 +0800 Subject: [PATCH 11/42] fix: do not close meta tag in HTML header (#2740) * fixed typo in HTML (extra tag) * refactor: remove closing slash for meta tag as of specification in https://html.com/tags/meta/, meta tagas are null elements: This element must not contain any content, and does not need a closing tag. --------- Co-authored-by: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> --- modules/ui_gradio_extensions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ui_gradio_extensions.py b/modules/ui_gradio_extensions.py index bebf9f8ca..409c7e332 100644 --- a/modules/ui_gradio_extensions.py +++ b/modules/ui_gradio_extensions.py @@ -39,7 +39,7 @@ def javascript_html(): head += f'\n' head += f'\n' head += f'\n' - head += f'\n' + head += f'\n' if args_manager.args.theme: head += f'\n' From 33fa175bd438041fe4ae715adc9a06d025a940b3 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Fri, 17 May 2024 18:25:08 +0200 Subject: [PATCH 12/42] feat: automatically describe image on uov image upload (#1938) * feat: automatically describe image on uov image upload if prompt is empty * feat: add argument to disable automatic uov image description * feat: rename argument, disable by default this prevents computers with low hardware specifications from being unnecessary blocked --- args_manager.py | 3 +++ webui.py | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/args_manager.py b/args_manager.py index 6a3ae9dc3..e023da276 100644 --- a/args_manager.py +++ b/args_manager.py @@ -31,6 +31,9 @@ args_parser.parser.add_argument("--disable-preset-download", action='store_true', help="Disables downloading models for presets", default=False) +args_parser.parser.add_argument("--enable-describe-uov-image", action='store_true', + help="Disables automatic description of uov images when prompt is empty", default=False) + args_parser.parser.add_argument("--always-download-new-model", action='store_true', help="Always download newer models ", default=False) diff --git a/webui.py b/webui.py index 85b2c0df3..f99ab1591 100644 --- a/webui.py +++ b/webui.py @@ -717,6 +717,15 @@ def trigger_describe(mode, img): desc_btn.click(trigger_describe, inputs=[desc_method, desc_input_image], outputs=[prompt, style_selections], show_progress=True, queue=True) + if args_manager.args.enable_describe_uov_image: + def trigger_uov_describe(mode, img, prompt): + # keep prompt if not empty + if prompt == '': + return trigger_describe(mode, img) + return gr.update(), gr.update() + + uov_input_image.upload(trigger_uov_describe, inputs=[desc_method, uov_input_image, prompt], + outputs=[prompt, style_selections], show_progress=True, queue=True) def dump_default_english_config(): from modules.localization import dump_english_config From 00d3d1b4b31b2effa32f6eb96f8e5caf6368f8e3 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Sat, 18 May 2024 15:50:28 +0200 Subject: [PATCH 13/42] feat: add nsfw image censoring via config and checkbox (#958) * add nsfw image censoring activatable via config, uses CompVis/stable-diffusion-safety-checker * fix progressbar call for nsfw output * use config to set cache dir for safety checker * add checkbox black_out_nsfw makes both enabling via config and checkbox possible, where config overrides the checkbox value * fix: add missing diffusers package * feat: extract safety checker, remove dependency to diffusers * feat: make code compatible again after merge with main * feat: move censor to extras, optimize safety checker file handling * refactor: rename folder safety_checker_models to safety_checker --- extras/censor.py | 56 ++++++ extras/safety_checker/configs/config.json | 171 ++++++++++++++++++ .../configs/preprocessor_config.json | 20 ++ .../safety_checker/models/safety_checker.py | 126 +++++++++++++ language/en.json | 2 + .../put_safety_checker_models_here | 0 modules/async_worker.py | 32 +++- modules/config.py | 14 ++ webui.py | 14 +- 9 files changed, 424 insertions(+), 11 deletions(-) create mode 100644 extras/censor.py create mode 100644 extras/safety_checker/configs/config.json create mode 100644 extras/safety_checker/configs/preprocessor_config.json create mode 100644 extras/safety_checker/models/safety_checker.py create mode 100644 models/safety_checker/put_safety_checker_models_here diff --git a/extras/censor.py b/extras/censor.py new file mode 100644 index 000000000..2047db246 --- /dev/null +++ b/extras/censor.py @@ -0,0 +1,56 @@ +# modified version of https://github.com/AUTOMATIC1111/stable-diffusion-webui-nsfw-censor/blob/master/scripts/censor.py +import numpy as np +import os + +from extras.safety_checker.models.safety_checker import StableDiffusionSafetyChecker +from transformers import CLIPFeatureExtractor, CLIPConfig +from PIL import Image +import modules.config + +safety_checker_repo_root = os.path.join(os.path.dirname(__file__), 'safety_checker') +config_path = os.path.join(safety_checker_repo_root, "configs", "config.json") +preprocessor_config_path = os.path.join(safety_checker_repo_root, "configs", "preprocessor_config.json") + +safety_feature_extractor = None +safety_checker = None + + +def numpy_to_pil(image): + image = (image * 255).round().astype("uint8") + pil_image = Image.fromarray(image) + + return pil_image + + +# check and replace nsfw content +def check_safety(x_image): + global safety_feature_extractor, safety_checker + + if safety_feature_extractor is None or safety_checker is None: + safety_checker_model = modules.config.downloading_safety_checker_model() + safety_feature_extractor = CLIPFeatureExtractor.from_json_file(preprocessor_config_path) + clip_config = CLIPConfig.from_json_file(config_path) + safety_checker = StableDiffusionSafetyChecker.from_pretrained(safety_checker_model, config=clip_config) + + safety_checker_input = safety_feature_extractor(numpy_to_pil(x_image), return_tensors="pt") + x_checked_image, has_nsfw_concept = safety_checker(images=x_image, clip_input=safety_checker_input.pixel_values) + + return x_checked_image, has_nsfw_concept + + +def censor_single(x): + x_checked_image, has_nsfw_concept = check_safety(x) + + # replace image with black pixels, keep dimensions + # workaround due to different numpy / pytorch image matrix format + if has_nsfw_concept[0]: + imageshape = x_checked_image.shape + x_checked_image = np.zeros((imageshape[0], imageshape[1], 3), dtype = np.uint8) + + return x_checked_image + + +def censor_batch(images): + images = [censor_single(image) for image in images] + + return images \ No newline at end of file diff --git a/extras/safety_checker/configs/config.json b/extras/safety_checker/configs/config.json new file mode 100644 index 000000000..aa454d222 --- /dev/null +++ b/extras/safety_checker/configs/config.json @@ -0,0 +1,171 @@ +{ + "_name_or_path": "clip-vit-large-patch14/", + "architectures": [ + "SafetyChecker" + ], + "initializer_factor": 1.0, + "logit_scale_init_value": 2.6592, + "model_type": "clip", + "projection_dim": 768, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_dropout": 0.0, + "bad_words_ids": null, + "bos_token_id": 0, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dropout": 0.0, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": 2, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "quick_gelu", + "hidden_size": 768, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_factor": 1.0, + "initializer_range": 0.02, + "intermediate_size": 3072, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_norm_eps": 1e-05, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 77, + "min_length": 0, + "model_type": "clip_text_model", + "no_repeat_ngram_size": 0, + "num_attention_heads": 12, + "num_beam_groups": 1, + "num_beams": 1, + "num_hidden_layers": 12, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": 1, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "task_specific_params": null, + "temperature": 1.0, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torch_dtype": null, + "torchscript": false, + "transformers_version": "4.21.0.dev0", + "typical_p": 1.0, + "use_bfloat16": false, + "vocab_size": 49408 + }, + "text_config_dict": { + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 12, + "num_hidden_layers": 12 + }, + "torch_dtype": "float32", + "transformers_version": null, + "vision_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_dropout": 0.0, + "bad_words_ids": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dropout": 0.0, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "quick_gelu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "image_size": 224, + "initializer_factor": 1.0, + "initializer_range": 0.02, + "intermediate_size": 4096, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_norm_eps": 1e-05, + "length_penalty": 1.0, + "max_length": 20, + "min_length": 0, + "model_type": "clip_vision_model", + "no_repeat_ngram_size": 0, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_hidden_layers": 24, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "patch_size": 14, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "task_specific_params": null, + "temperature": 1.0, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torch_dtype": null, + "torchscript": false, + "transformers_version": "4.21.0.dev0", + "typical_p": 1.0, + "use_bfloat16": false + }, + "vision_config_dict": { + "hidden_size": 1024, + "intermediate_size": 4096, + "num_attention_heads": 16, + "num_hidden_layers": 24, + "patch_size": 14 + } +} diff --git a/extras/safety_checker/configs/preprocessor_config.json b/extras/safety_checker/configs/preprocessor_config.json new file mode 100644 index 000000000..5294955ff --- /dev/null +++ b/extras/safety_checker/configs/preprocessor_config.json @@ -0,0 +1,20 @@ +{ + "crop_size": 224, + "do_center_crop": true, + "do_convert_rgb": true, + "do_normalize": true, + "do_resize": true, + "feature_extractor_type": "CLIPFeatureExtractor", + "image_mean": [ + 0.48145466, + 0.4578275, + 0.40821073 + ], + "image_std": [ + 0.26862954, + 0.26130258, + 0.27577711 + ], + "resample": 3, + "size": 224 +} diff --git a/extras/safety_checker/models/safety_checker.py b/extras/safety_checker/models/safety_checker.py new file mode 100644 index 000000000..ea38bf038 --- /dev/null +++ b/extras/safety_checker/models/safety_checker.py @@ -0,0 +1,126 @@ +# from https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion/safety_checker.py + +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import torch +import torch.nn as nn +from transformers import CLIPConfig, CLIPVisionModel, PreTrainedModel +from transformers.utils import logging + +logger = logging.get_logger(__name__) + + +def cosine_distance(image_embeds, text_embeds): + normalized_image_embeds = nn.functional.normalize(image_embeds) + normalized_text_embeds = nn.functional.normalize(text_embeds) + return torch.mm(normalized_image_embeds, normalized_text_embeds.t()) + + +class StableDiffusionSafetyChecker(PreTrainedModel): + config_class = CLIPConfig + main_input_name = "clip_input" + + _no_split_modules = ["CLIPEncoderLayer"] + + def __init__(self, config: CLIPConfig): + super().__init__(config) + + self.vision_model = CLIPVisionModel(config.vision_config) + self.visual_projection = nn.Linear(config.vision_config.hidden_size, config.projection_dim, bias=False) + + self.concept_embeds = nn.Parameter(torch.ones(17, config.projection_dim), requires_grad=False) + self.special_care_embeds = nn.Parameter(torch.ones(3, config.projection_dim), requires_grad=False) + + self.concept_embeds_weights = nn.Parameter(torch.ones(17), requires_grad=False) + self.special_care_embeds_weights = nn.Parameter(torch.ones(3), requires_grad=False) + + @torch.no_grad() + def forward(self, clip_input, images): + pooled_output = self.vision_model(clip_input)[1] # pooled_output + image_embeds = self.visual_projection(pooled_output) + + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + special_cos_dist = cosine_distance(image_embeds, self.special_care_embeds).cpu().float().numpy() + cos_dist = cosine_distance(image_embeds, self.concept_embeds).cpu().float().numpy() + + result = [] + batch_size = image_embeds.shape[0] + for i in range(batch_size): + result_img = {"special_scores": {}, "special_care": [], "concept_scores": {}, "bad_concepts": []} + + # increase this value to create a stronger `nfsw` filter + # at the cost of increasing the possibility of filtering benign images + adjustment = 0.0 + + for concept_idx in range(len(special_cos_dist[0])): + concept_cos = special_cos_dist[i][concept_idx] + concept_threshold = self.special_care_embeds_weights[concept_idx].item() + result_img["special_scores"][concept_idx] = round(concept_cos - concept_threshold + adjustment, 3) + if result_img["special_scores"][concept_idx] > 0: + result_img["special_care"].append({concept_idx, result_img["special_scores"][concept_idx]}) + adjustment = 0.01 + + for concept_idx in range(len(cos_dist[0])): + concept_cos = cos_dist[i][concept_idx] + concept_threshold = self.concept_embeds_weights[concept_idx].item() + result_img["concept_scores"][concept_idx] = round(concept_cos - concept_threshold + adjustment, 3) + if result_img["concept_scores"][concept_idx] > 0: + result_img["bad_concepts"].append(concept_idx) + + result.append(result_img) + + has_nsfw_concepts = [len(res["bad_concepts"]) > 0 for res in result] + + for idx, has_nsfw_concept in enumerate(has_nsfw_concepts): + if has_nsfw_concept: + if torch.is_tensor(images) or torch.is_tensor(images[0]): + images[idx] = torch.zeros_like(images[idx]) # black image + else: + images[idx] = np.zeros(images[idx].shape) # black image + + if any(has_nsfw_concepts): + logger.warning( + "Potential NSFW content was detected in one or more images. A black image will be returned instead." + " Try again with a different prompt and/or seed." + ) + + return images, has_nsfw_concepts + + @torch.no_grad() + def forward_onnx(self, clip_input: torch.Tensor, images: torch.Tensor): + pooled_output = self.vision_model(clip_input)[1] # pooled_output + image_embeds = self.visual_projection(pooled_output) + + special_cos_dist = cosine_distance(image_embeds, self.special_care_embeds) + cos_dist = cosine_distance(image_embeds, self.concept_embeds) + + # increase this value to create a stronger `nsfw` filter + # at the cost of increasing the possibility of filtering benign images + adjustment = 0.0 + + special_scores = special_cos_dist - self.special_care_embeds_weights + adjustment + # special_scores = special_scores.round(decimals=3) + special_care = torch.any(special_scores > 0, dim=1) + special_adjustment = special_care * 0.01 + special_adjustment = special_adjustment.unsqueeze(1).expand(-1, cos_dist.shape[1]) + + concept_scores = (cos_dist - self.concept_embeds_weights) + special_adjustment + # concept_scores = concept_scores.round(decimals=3) + has_nsfw_concepts = torch.any(concept_scores > 0, dim=1) + + images[has_nsfw_concepts] = 0.0 # black image + + return images, has_nsfw_concepts diff --git a/language/en.json b/language/en.json index e9cd6b737..3eb5d5e25 100644 --- a/language/en.json +++ b/language/en.json @@ -55,6 +55,8 @@ "Disable seed increment": "Disable seed increment", "Disable automatic seed increment when image number is > 1.": "Disable automatic seed increment when image number is > 1.", "Read wildcards in order": "Read wildcards in order", + "Black Out NSFW": "Black Out NSFW", + "Use black image if NSFW is detected.": "Use black image if NSFW is detected.", "\ud83d\udcda History Log": "\uD83D\uDCDA History Log", "Image Style": "Image Style", "Fooocus V2": "Fooocus V2", diff --git a/models/safety_checker/put_safety_checker_models_here b/models/safety_checker/put_safety_checker_models_here new file mode 100644 index 000000000..e69de29bb diff --git a/modules/async_worker.py b/modules/async_worker.py index cde99bdc0..6f0b30a98 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -43,6 +43,7 @@ def worker(): import fooocus_version import args_manager + from extras.censor import censor_batch, censor_single from modules.sdxl_styles import apply_style, get_random_style, apply_wildcards, fooocus_expansion, apply_arrays, random_style_name from modules.private_logger import log from extras.expansion import safe_str @@ -68,10 +69,14 @@ def progressbar(async_task, number, text): print(f'[Fooocus] {text}') async_task.yields.append(['preview', (number, text, None)]) - def yield_result(async_task, imgs, do_not_show_finished_images=False): + def yield_result(async_task, imgs, black_out_nsfw, censor=True, do_not_show_finished_images=False, progressbar_index=13): if not isinstance(imgs, list): imgs = [imgs] + if censor and (modules.config.default_black_out_nsfw or black_out_nsfw): + progressbar(async_task, progressbar_index, 'Checking for NSFW content ...') + imgs = censor_batch(imgs) + async_task.results = async_task.results + imgs if do_not_show_finished_images: @@ -160,6 +165,7 @@ def handler(async_task): disable_preview = args.pop() disable_intermediate_results = args.pop() disable_seed_increment = args.pop() + black_out_nsfw = args.pop() adm_scaler_positive = args.pop() adm_scaler_negative = args.pop() adm_scaler_end = args.pop() @@ -578,8 +584,11 @@ def handler(async_task): if direct_return: d = [('Upscale (Fast)', 'upscale_fast', '2x')] + if modules.config.default_black_out_nsfw or black_out_nsfw: + progressbar(async_task, 100, 'Checking for NSFW content ...') + uov_input_image = censor_single(uov_input_image) uov_input_image_path = log(uov_input_image, d, output_format=output_format) - yield_result(async_task, uov_input_image_path, do_not_show_finished_images=True) + yield_result(async_task, uov_input_image_path, black_out_nsfw, False, do_not_show_finished_images=True) return tiled = True @@ -643,8 +652,7 @@ def handler(async_task): ) if debugging_inpaint_preprocessor: - yield_result(async_task, inpaint_worker.current_task.visualize_mask_processing(), - do_not_show_finished_images=True) + yield_result(async_task, inpaint_worker.current_task.visualize_mask_processing(), black_out_nsfw, do_not_show_finished_images=True) return progressbar(async_task, 13, 'VAE Inpaint encoding ...') @@ -707,7 +715,7 @@ def handler(async_task): cn_img = HWC3(cn_img) task[0] = core.numpy_to_pytorch(cn_img) if debugging_cn_preprocessor: - yield_result(async_task, cn_img, do_not_show_finished_images=True) + yield_result(async_task, cn_img, black_out_nsfw, do_not_show_finished_images=True) return for task in cn_tasks[flags.cn_cpds]: cn_img, cn_stop, cn_weight = task @@ -719,7 +727,7 @@ def handler(async_task): cn_img = HWC3(cn_img) task[0] = core.numpy_to_pytorch(cn_img) if debugging_cn_preprocessor: - yield_result(async_task, cn_img, do_not_show_finished_images=True) + yield_result(async_task, cn_img, black_out_nsfw, do_not_show_finished_images=True) return for task in cn_tasks[flags.cn_ip]: cn_img, cn_stop, cn_weight = task @@ -730,7 +738,7 @@ def handler(async_task): task[0] = ip_adapter.preprocess(cn_img, ip_adapter_path=ip_adapter_path) if debugging_cn_preprocessor: - yield_result(async_task, cn_img, do_not_show_finished_images=True) + yield_result(async_task, cn_img, black_out_nsfw, do_not_show_finished_images=True) return for task in cn_tasks[flags.cn_ip_face]: cn_img, cn_stop, cn_weight = task @@ -744,7 +752,7 @@ def handler(async_task): task[0] = ip_adapter.preprocess(cn_img, ip_adapter_path=ip_adapter_face_path) if debugging_cn_preprocessor: - yield_result(async_task, cn_img, do_not_show_finished_images=True) + yield_result(async_task, cn_img, black_out_nsfw, do_not_show_finished_images=True) return all_ip_tasks = cn_tasks[flags.cn_ip] + cn_tasks[flags.cn_ip_face] @@ -844,6 +852,12 @@ def callback(step, x0, x, total_steps, y): imgs = [inpaint_worker.current_task.post_process(x) for x in imgs] img_paths = [] + + if modules.config.default_black_out_nsfw or black_out_nsfw: + progressbar(async_task, int(15.0 + 85.0 * float((current_task_id + 1) * steps) / float(all_steps)), + 'Checking for NSFW content ...') + imgs = censor_batch(imgs) + for x in imgs: d = [('Prompt', 'prompt', task['log_positive_prompt']), ('Negative Prompt', 'negative_prompt', task['log_negative_prompt']), @@ -895,7 +909,7 @@ def callback(step, x0, x, total_steps, y): d.append(('Version', 'version', 'Fooocus v' + fooocus_version.version)) img_paths.append(log(x, d, metadata_parser, output_format, task)) - yield_result(async_task, img_paths, do_not_show_finished_images=len(tasks) == 1 or disable_intermediate_results) + yield_result(async_task, img_paths, black_out_nsfw, False, do_not_show_finished_images=len(tasks) == 1 or disable_intermediate_results) except ldm_patched.modules.model_management.InterruptProcessingException as e: if async_task.last_stop == 'skip': print('User skipped') diff --git a/modules/config.py b/modules/config.py index f11460c8d..ffb74a23d 100644 --- a/modules/config.py +++ b/modules/config.py @@ -196,6 +196,7 @@ def get_dir_or_set_default(key, default_value, as_array=False, make_directory=Fa path_clip_vision = get_dir_or_set_default('path_clip_vision', '../models/clip_vision/') path_fooocus_expansion = get_dir_or_set_default('path_fooocus_expansion', '../models/prompt_expansion/fooocus_expansion') path_wildcards = get_dir_or_set_default('path_wildcards', '../wildcards/') +path_safety_checker = get_dir_or_set_default('path_safety_checker', '../models/safety_checker/') path_outputs = get_path_output() @@ -456,6 +457,11 @@ def init_temp_path(path: str | None, default_path: str) -> str: ], validator=lambda x: isinstance(x, list) and all(isinstance(v, str) for v in x) ) +default_black_out_nsfw = get_config_item_or_set_default( + key='default_black_out_nsfw', + default_value=False, + validator=lambda x: isinstance(x, bool) +) default_save_metadata_to_images = get_config_item_or_set_default( key='default_save_metadata_to_images', default_value=False, @@ -691,5 +697,13 @@ def downloading_upscale_model(): ) return os.path.join(path_upscale_models, 'fooocus_upscaler_s409985e5.bin') +def downloading_safety_checker_model(): + load_file_from_url( + url='https://huggingface.co/mashb1t/misc/resolve/main/stable-diffusion-safety-checker.bin', + model_dir=path_safety_checker, + file_name='stable-diffusion-safety-checker.bin' + ) + return os.path.join(path_safety_checker, 'stable-diffusion-safety-checker.bin') + update_files() diff --git a/webui.py b/webui.py index f99ab1591..55f3102c2 100644 --- a/webui.py +++ b/webui.py @@ -436,7 +436,8 @@ def update_history_link(): overwrite_upscale_strength = gr.Slider(label='Forced Overwrite of Denoising Strength of "Upscale"', minimum=-1, maximum=1.0, step=0.001, value=-1, info='Set as negative number to disable. For developer debugging.') - disable_preview = gr.Checkbox(label='Disable Preview', value=False, + disable_preview = gr.Checkbox(label='Disable Preview', value=modules.config.default_black_out_nsfw, + interactive=not modules.config.default_black_out_nsfw, info='Disable preview during generation.') disable_intermediate_results = gr.Checkbox(label='Disable Intermediate Results', value=modules.config.default_performance == flags.Performance.EXTREME_SPEED.value, @@ -447,6 +448,15 @@ def update_history_link(): value=False) read_wildcards_in_order = gr.Checkbox(label="Read wildcards in order", value=False) + black_out_nsfw = gr.Checkbox(label='Black Out NSFW', + value=modules.config.default_black_out_nsfw, + interactive=not modules.config.default_black_out_nsfw, + info='Use black image if NSFW is detected.') + + black_out_nsfw.change(lambda x: gr.update(value=x, interactive=not x), + inputs=black_out_nsfw, outputs=disable_preview, queue=False, + show_progress=False) + if not args_manager.args.disable_metadata: save_metadata_to_images = gr.Checkbox(label='Save Metadata to Images', value=modules.config.default_save_metadata_to_images, info='Adds parameters to generated images allowing manual regeneration.') @@ -636,7 +646,7 @@ def inpaint_mode_change(mode): ctrls += [input_image_checkbox, current_tab] ctrls += [uov_method, uov_input_image] ctrls += [outpaint_selections, inpaint_input_image, inpaint_additional_prompt, inpaint_mask_image] - ctrls += [disable_preview, disable_intermediate_results, disable_seed_increment] + ctrls += [disable_preview, disable_intermediate_results, disable_seed_increment, black_out_nsfw] ctrls += [adm_scaler_positive, adm_scaler_negative, adm_scaler_end, adaptive_cfg] ctrls += [sampler_name, scheduler_name, vae_name] ctrls += [overwrite_step, overwrite_switch, overwrite_width, overwrite_height, overwrite_vary_strength] From 3a55e7e3910b8ae58f82a5a0e4c11d7d4fa3143f Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Sat, 18 May 2024 15:53:34 +0200 Subject: [PATCH 14/42] feat: add AlignYourStepsScheduler (#2905) --- .../contrib/external_align_your_steps.py | 55 +++++++++++++++++++ modules/flags.py | 2 +- modules/sample_hijack.py | 4 ++ 3 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 ldm_patched/contrib/external_align_your_steps.py diff --git a/ldm_patched/contrib/external_align_your_steps.py b/ldm_patched/contrib/external_align_your_steps.py new file mode 100644 index 000000000..624bbce2a --- /dev/null +++ b/ldm_patched/contrib/external_align_your_steps.py @@ -0,0 +1,55 @@ +# https://github.com/comfyanonymous/ComfyUI/blob/master/nodes.py + +#from: https://research.nvidia.com/labs/toronto-ai/AlignYourSteps/howto.html +import numpy as np +import torch + +def loglinear_interp(t_steps, num_steps): + """ + Performs log-linear interpolation of a given array of decreasing numbers. + """ + xs = np.linspace(0, 1, len(t_steps)) + ys = np.log(t_steps[::-1]) + + new_xs = np.linspace(0, 1, num_steps) + new_ys = np.interp(new_xs, xs, ys) + + interped_ys = np.exp(new_ys)[::-1].copy() + return interped_ys + +NOISE_LEVELS = {"SD1": [14.6146412293, 6.4745760956, 3.8636745985, 2.6946151520, 1.8841921177, 1.3943805092, 0.9642583904, 0.6523686016, 0.3977456272, 0.1515232662, 0.0291671582], + "SDXL":[14.6146412293, 6.3184485287, 3.7681790315, 2.1811480769, 1.3405244945, 0.8620721141, 0.5550693289, 0.3798540708, 0.2332364134, 0.1114188177, 0.0291671582], + "SVD": [700.00, 54.5, 15.886, 7.977, 4.248, 1.789, 0.981, 0.403, 0.173, 0.034, 0.002]} + +class AlignYourStepsScheduler: + @classmethod + def INPUT_TYPES(s): + return {"required": + {"model_type": (["SD1", "SDXL", "SVD"], ), + "steps": ("INT", {"default": 10, "min": 10, "max": 10000}), + "denoise": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}), + } + } + RETURN_TYPES = ("SIGMAS",) + CATEGORY = "sampling/custom_sampling/schedulers" + + FUNCTION = "get_sigmas" + + def get_sigmas(self, model_type, steps, denoise): + total_steps = steps + if denoise < 1.0: + if denoise <= 0.0: + return (torch.FloatTensor([]),) + total_steps = round(steps * denoise) + + sigmas = NOISE_LEVELS[model_type][:] + if (steps + 1) != len(sigmas): + sigmas = loglinear_interp(sigmas, steps + 1) + + sigmas = sigmas[-(total_steps + 1):] + sigmas[-1] = 0 + return (torch.FloatTensor(sigmas), ) + +NODE_CLASS_MAPPINGS = { + "AlignYourStepsScheduler": AlignYourStepsScheduler, +} \ No newline at end of file diff --git a/modules/flags.py b/modules/flags.py index 9f2aefb3b..0c6054394 100644 --- a/modules/flags.py +++ b/modules/flags.py @@ -47,7 +47,7 @@ KSAMPLER_NAMES = list(KSAMPLER.keys()) -SCHEDULER_NAMES = ["normal", "karras", "exponential", "sgm_uniform", "simple", "ddim_uniform", "lcm", "turbo"] +SCHEDULER_NAMES = ["normal", "karras", "exponential", "sgm_uniform", "simple", "ddim_uniform", "lcm", "turbo", "align_your_steps"] SAMPLER_NAMES = KSAMPLER_NAMES + list(SAMPLER_EXTRA.keys()) sampler_list = SAMPLER_NAMES diff --git a/modules/sample_hijack.py b/modules/sample_hijack.py index 5936a096d..4ab3cbbde 100644 --- a/modules/sample_hijack.py +++ b/modules/sample_hijack.py @@ -3,6 +3,7 @@ import ldm_patched.modules.model_management from collections import namedtuple +from ldm_patched.contrib.external_align_your_steps import AlignYourStepsScheduler from ldm_patched.contrib.external_custom_sampler import SDTurboScheduler from ldm_patched.k_diffusion import sampling as k_diffusion_sampling from ldm_patched.modules.samplers import normal_scheduler, simple_scheduler, ddim_scheduler @@ -175,6 +176,9 @@ def calculate_sigmas_scheduler_hacked(model, scheduler_name, steps): sigmas = normal_scheduler(model, steps, sgm=True) elif scheduler_name == "turbo": sigmas = SDTurboScheduler().get_sigmas(namedtuple('Patcher', ['model'])(model=model), steps=steps, denoise=1.0)[0] + elif scheduler_name == "align_your_steps": + model_type = 'SDXL' if isinstance(model.latent_format, ldm_patched.modules.latent_formats.SDXL) else 'SD1' + sigmas = AlignYourStepsScheduler().get_sigmas(model_type=model_type, steps=steps, denoise=1.0)[0] else: raise TypeError("error invalid scheduler") return sigmas From 3bae73e23ecc85430e532ce57a25b04437c7cf67 Mon Sep 17 00:00:00 2001 From: cantor-set <32692347+cantor-set@users.noreply.github.com> Date: Sat, 18 May 2024 11:19:46 -0400 Subject: [PATCH 15/42] feat: add support for lora inline prompt references (#2323) * Adding support to inline prompt references * Added unittests * Added an initial documentation for development guidelines * Added a negative number * renamed parameter * removed wrongly committed file * Code fixes * Fixed circular reference * Fixed typo. Added TODO * Fixed merge * Code cleanup * Added missing refernce function * Removed function from util.py... again... * Update modules/async_worker.py Implemented suggested change Co-authored-by: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> * Removed another circular reference * Renamed module * Addressed PR comments * Added return type to function * refactor: move apply_wildcards to module util * refactor: code cleanup, unify usage of tuples in lora list * docs: add instructions for running unittests on embedded python, code cleanup * refactor: code cleanup, move makedirs_with_log back to util --------- Co-authored-by: cantor-set Co-authored-by: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Co-authored-by: Manuel Schmid --- development.md | 11 ++++++ modules/__init__.py | 0 modules/async_worker.py | 46 ++++++++++++++++-------- modules/config.py | 5 +-- modules/extra_utils.py | 20 +++++++++++ modules/sdxl_styles.py | 36 ++----------------- modules/util.py | 79 +++++++++++++++++++++++++++++------------ tests/__init__.py | 4 +++ tests/test_utils.py | 48 +++++++++++++++++++++++++ 9 files changed, 176 insertions(+), 73 deletions(-) create mode 100644 development.md create mode 100644 modules/__init__.py create mode 100644 modules/extra_utils.py create mode 100644 tests/__init__.py create mode 100644 tests/test_utils.py diff --git a/development.md b/development.md new file mode 100644 index 000000000..bbb3def92 --- /dev/null +++ b/development.md @@ -0,0 +1,11 @@ +## Running unit tests + +Native python: +``` +python -m unittest tests/ +``` + +Embedded python (Windows zip file installation method): +``` +..\python_embeded\python.exe -m unittest +``` diff --git a/modules/__init__.py b/modules/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/modules/async_worker.py b/modules/async_worker.py index 6f0b30a98..7f0a46e3e 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -4,6 +4,7 @@ patch_all() + class AsyncTask: def __init__(self, args): self.args = args @@ -44,11 +45,12 @@ def worker(): import args_manager from extras.censor import censor_batch, censor_single - from modules.sdxl_styles import apply_style, get_random_style, apply_wildcards, fooocus_expansion, apply_arrays, random_style_name + from modules.sdxl_styles import apply_style, get_random_style, fooocus_expansion, apply_arrays, random_style_name from modules.private_logger import log from extras.expansion import safe_str - from modules.util import remove_empty_str, HWC3, resize_image, get_image_shape_ceil, set_image_shape_ceil, \ - get_shape_ceil, resample_image, erode_or_dilate, ordinal_suffix, get_enabled_loras + from modules.util import (remove_empty_str, HWC3, resize_image, get_image_shape_ceil, set_image_shape_ceil, + get_shape_ceil, resample_image, erode_or_dilate, ordinal_suffix, get_enabled_loras, + parse_lora_references_from_prompt, apply_wildcards) from modules.upscaler import perform_upscale from modules.flags import Performance from modules.meta_parser import get_metadata_parser, MetadataScheme @@ -69,7 +71,8 @@ def progressbar(async_task, number, text): print(f'[Fooocus] {text}') async_task.yields.append(['preview', (number, text, None)]) - def yield_result(async_task, imgs, black_out_nsfw, censor=True, do_not_show_finished_images=False, progressbar_index=13): + def yield_result(async_task, imgs, black_out_nsfw, censor=True, do_not_show_finished_images=False, + progressbar_index=13): if not isinstance(imgs, list): imgs = [imgs] @@ -152,7 +155,8 @@ def handler(async_task): base_model_name = args.pop() refiner_model_name = args.pop() refiner_switch = args.pop() - loras = get_enabled_loras([[bool(args.pop()), str(args.pop()), float(args.pop())] for _ in range(modules.config.default_max_lora_number)]) + loras = get_enabled_loras([(bool(args.pop()), str(args.pop()), float(args.pop())) for _ in + range(modules.config.default_max_lora_number)]) input_image_checkbox = args.pop() current_tab = args.pop() uov_method = args.pop() @@ -202,7 +206,8 @@ def handler(async_task): inpaint_erode_or_dilate = args.pop() save_metadata_to_images = args.pop() if not args_manager.args.disable_metadata else False - metadata_scheme = MetadataScheme(args.pop()) if not args_manager.args.disable_metadata else MetadataScheme.FOOOCUS + metadata_scheme = MetadataScheme( + args.pop()) if not args_manager.args.disable_metadata else MetadataScheme.FOOOCUS cn_tasks = {x: [] for x in flags.ip_list} for _ in range(flags.controlnet_image_count): @@ -433,13 +438,16 @@ def handler(async_task): extra_negative_prompts = negative_prompts[1:] if len(negative_prompts) > 1 else [] progressbar(async_task, 3, 'Loading models ...') + + loras = parse_lora_references_from_prompt(prompt, loras, modules.config.default_max_lora_number) + pipeline.refresh_everything(refiner_model_name=refiner_model_name, base_model_name=base_model_name, loras=loras, base_model_additional_loras=base_model_additional_loras, use_synthetic_refiner=use_synthetic_refiner, vae_name=vae_name) progressbar(async_task, 3, 'Processing prompts ...') tasks = [] - + for i in range(image_number): if disable_seed_increment: task_seed = seed % (constants.MAX_SEED + 1) @@ -450,8 +458,10 @@ def handler(async_task): task_prompt = apply_wildcards(prompt, task_rng, i, read_wildcards_in_order) task_prompt = apply_arrays(task_prompt, i) task_negative_prompt = apply_wildcards(negative_prompt, task_rng, i, read_wildcards_in_order) - task_extra_positive_prompts = [apply_wildcards(pmt, task_rng, i, read_wildcards_in_order) for pmt in extra_positive_prompts] - task_extra_negative_prompts = [apply_wildcards(pmt, task_rng, i, read_wildcards_in_order) for pmt in extra_negative_prompts] + task_extra_positive_prompts = [apply_wildcards(pmt, task_rng, i, read_wildcards_in_order) for pmt in + extra_positive_prompts] + task_extra_negative_prompts = [apply_wildcards(pmt, task_rng, i, read_wildcards_in_order) for pmt in + extra_negative_prompts] positive_basic_workloads = [] negative_basic_workloads = [] @@ -652,7 +662,8 @@ def handler(async_task): ) if debugging_inpaint_preprocessor: - yield_result(async_task, inpaint_worker.current_task.visualize_mask_processing(), black_out_nsfw, do_not_show_finished_images=True) + yield_result(async_task, inpaint_worker.current_task.visualize_mask_processing(), black_out_nsfw, + do_not_show_finished_images=True) return progressbar(async_task, 13, 'VAE Inpaint encoding ...') @@ -807,7 +818,8 @@ def callback(step, x0, x, total_steps, y): done_steps = current_task_id * steps + step async_task.yields.append(['preview', ( int(15.0 + 85.0 * float(done_steps) / float(all_steps)), - f'Step {step}/{total_steps} in the {current_task_id + 1}{ordinal_suffix(current_task_id + 1)} Sampling', y)]) + f'Step {step}/{total_steps} in the {current_task_id + 1}{ordinal_suffix(current_task_id + 1)} Sampling', + y)]) for current_task_id, task in enumerate(tasks): execution_start_time = time.perf_counter() @@ -862,7 +874,8 @@ def callback(step, x0, x, total_steps, y): d = [('Prompt', 'prompt', task['log_positive_prompt']), ('Negative Prompt', 'negative_prompt', task['log_negative_prompt']), ('Fooocus V2 Expansion', 'prompt_expansion', task['expansion']), - ('Styles', 'styles', str(task['styles'] if not use_expansion else [fooocus_expansion] + task['styles'])), + ('Styles', 'styles', + str(task['styles'] if not use_expansion else [fooocus_expansion] + task['styles'])), ('Performance', 'performance', performance_selection.value)] if performance_selection.steps() != steps: @@ -885,7 +898,8 @@ def callback(step, x0, x, total_steps, y): if refiner_swap_method != flags.refiner_swap_method: d.append(('Refiner Swap Method', 'refiner_swap_method', refiner_swap_method)) if modules.patch.patch_settings[pid].adaptive_cfg != modules.config.default_cfg_tsnr: - d.append(('CFG Mimicking from TSNR', 'adaptive_cfg', modules.patch.patch_settings[pid].adaptive_cfg)) + d.append( + ('CFG Mimicking from TSNR', 'adaptive_cfg', modules.patch.patch_settings[pid].adaptive_cfg)) d.append(('Sampler', 'sampler', sampler_name)) d.append(('Scheduler', 'scheduler', scheduler_name)) @@ -905,11 +919,13 @@ def callback(step, x0, x, total_steps, y): metadata_parser.set_data(task['log_positive_prompt'], task['positive'], task['log_negative_prompt'], task['negative'], steps, base_model_name, refiner_model_name, loras, vae_name) - d.append(('Metadata Scheme', 'metadata_scheme', metadata_scheme.value if save_metadata_to_images else save_metadata_to_images)) + d.append(('Metadata Scheme', 'metadata_scheme', + metadata_scheme.value if save_metadata_to_images else save_metadata_to_images)) d.append(('Version', 'version', 'Fooocus v' + fooocus_version.version)) img_paths.append(log(x, d, metadata_parser, output_format, task)) - yield_result(async_task, img_paths, black_out_nsfw, False, do_not_show_finished_images=len(tasks) == 1 or disable_intermediate_results) + yield_result(async_task, img_paths, black_out_nsfw, False, + do_not_show_finished_images=len(tasks) == 1 or disable_intermediate_results) except ldm_patched.modules.model_management.InterruptProcessingException as e: if async_task.last_stop == 'skip': print('User skipped') diff --git a/modules/config.py b/modules/config.py index ffb74a23d..11fe31818 100644 --- a/modules/config.py +++ b/modules/config.py @@ -8,7 +8,8 @@ import modules.sdxl_styles from modules.model_loader import load_file_from_url -from modules.util import get_files_from_folder, makedirs_with_log +from modules.util import makedirs_with_log +from modules.extra_utils import get_files_from_folder from modules.flags import OutputFormat, Performance, MetadataScheme @@ -20,7 +21,7 @@ def get_config_path(key, default_value): else: return os.path.abspath(default_value) - +wildcards_max_bfs_depth = 64 config_path = get_config_path('config_path', "./config.txt") config_example_path = get_config_path('config_example_path', "config_modification_tutorial.txt") config_dict = {} diff --git a/modules/extra_utils.py b/modules/extra_utils.py new file mode 100644 index 000000000..3e95e8b56 --- /dev/null +++ b/modules/extra_utils.py @@ -0,0 +1,20 @@ +import os + + +def get_files_from_folder(folder_path, extensions=None, name_filter=None): + if not os.path.isdir(folder_path): + raise ValueError("Folder path is not a valid directory.") + + filenames = [] + + for root, _, files in os.walk(folder_path, topdown=False): + relative_path = os.path.relpath(root, folder_path) + if relative_path == ".": + relative_path = "" + for filename in sorted(files, key=lambda s: s.casefold()): + _, file_extension = os.path.splitext(filename) + if (extensions is None or file_extension.lower() in extensions) and (name_filter is None or name_filter in _): + path = os.path.join(relative_path, filename) + filenames.append(path) + + return filenames diff --git a/modules/sdxl_styles.py b/modules/sdxl_styles.py index 5b6afb590..12ab6c5ca 100644 --- a/modules/sdxl_styles.py +++ b/modules/sdxl_styles.py @@ -2,14 +2,12 @@ import re import json import math -import modules.config -from modules.util import get_files_from_folder +from modules.extra_utils import get_files_from_folder from random import Random # cannot use modules.config - validators causing circular imports styles_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../sdxl_styles/')) -wildcards_max_bfs_depth = 64 def normalize_key(k): @@ -25,7 +23,6 @@ def normalize_key(k): styles = {} - styles_files = get_files_from_folder(styles_path, ['.json']) for x in ['sdxl_styles_fooocus.json', @@ -65,34 +62,7 @@ def apply_style(style, positive): return p.replace('{prompt}', positive).splitlines(), n.splitlines() -def apply_wildcards(wildcard_text, rng, i, read_wildcards_in_order): - for _ in range(wildcards_max_bfs_depth): - placeholders = re.findall(r'__([\w-]+)__', wildcard_text) - if len(placeholders) == 0: - return wildcard_text - - print(f'[Wildcards] processing: {wildcard_text}') - for placeholder in placeholders: - try: - matches = [x for x in modules.config.wildcard_filenames if os.path.splitext(os.path.basename(x))[0] == placeholder] - words = open(os.path.join(modules.config.path_wildcards, matches[0]), encoding='utf-8').read().splitlines() - words = [x for x in words if x != ''] - assert len(words) > 0 - if read_wildcards_in_order: - wildcard_text = wildcard_text.replace(f'__{placeholder}__', words[i % len(words)], 1) - else: - wildcard_text = wildcard_text.replace(f'__{placeholder}__', rng.choice(words), 1) - except: - print(f'[Wildcards] Warning: {placeholder}.txt missing or empty. ' - f'Using "{placeholder}" as a normal word.') - wildcard_text = wildcard_text.replace(f'__{placeholder}__', placeholder) - print(f'[Wildcards] {wildcard_text}') - - print(f'[Wildcards] BFS stack overflow. Current text: {wildcard_text}') - return wildcard_text - - -def get_words(arrays, totalMult, index): +def get_words(arrays, total_mult, index): if len(arrays) == 1: return [arrays[0].split(',')[index]] else: @@ -101,7 +71,7 @@ def get_words(arrays, totalMult, index): index -= index % len(words) index /= len(words) index = math.floor(index) - return [word] + get_words(arrays[1:], math.floor(totalMult/len(words)), index) + return [word] + get_words(arrays[1:], math.floor(total_mult / len(words)), index) def apply_arrays(text, index): diff --git a/modules/util.py b/modules/util.py index d2feecb64..734302306 100644 --- a/modules/util.py +++ b/modules/util.py @@ -1,11 +1,12 @@ -import typing - import numpy as np import datetime import random import math import os import cv2 +import re +from typing import List, Tuple, AnyStr, NamedTuple + import json import hashlib @@ -14,8 +15,16 @@ import modules.sdxl_styles LANCZOS = (Image.Resampling.LANCZOS if hasattr(Image, 'Resampling') else Image.LANCZOS) + + +# Regexp compiled once. Matches entries with the following pattern: +# +# +LORAS_PROMPT_PATTERN = re.compile(r".* .*", re.X) + HASH_SHA256_LENGTH = 10 + def erode_or_dilate(x, k): k = int(k) if k > 0: @@ -163,25 +172,6 @@ def generate_temp_filename(folder='./outputs/', extension='png'): return date_string, os.path.abspath(result), filename -def get_files_from_folder(folder_path, extensions=None, name_filter=None): - if not os.path.isdir(folder_path): - raise ValueError("Folder path is not a valid directory.") - - filenames = [] - - for root, dirs, files in os.walk(folder_path, topdown=False): - relative_path = os.path.relpath(root, folder_path) - if relative_path == ".": - relative_path = "" - for filename in sorted(files, key=lambda s: s.casefold()): - _, file_extension = os.path.splitext(filename) - if (extensions is None or file_extension.lower() in extensions) and (name_filter is None or name_filter in _): - path = os.path.join(relative_path, filename) - filenames.append(path) - - return filenames - - def sha256(filename, use_addnet_hash=False, length=HASH_SHA256_LENGTH): print(f"Calculating sha256 for {filename}: ", end='') if use_addnet_hash: @@ -355,7 +345,7 @@ def extract_styles_from_prompt(prompt, negative_prompt): return list(reversed(extracted)), real_prompt, negative_prompt -class PromptStyle(typing.NamedTuple): +class PromptStyle(NamedTuple): name: str prompt: str negative_prompt: str @@ -394,4 +384,47 @@ def makedirs_with_log(path): def get_enabled_loras(loras: list) -> list: - return [[lora[1], lora[2]] for lora in loras if lora[0]] + return [(lora[1], lora[2]) for lora in loras if lora[0]] + + +def parse_lora_references_from_prompt(prompt: str, loras: List[Tuple[AnyStr, float]], loras_limit: int = 5) -> List[Tuple[AnyStr, float]]: + new_loras = [] + updated_loras = [] + for token in prompt.split(","): + m = LORAS_PROMPT_PATTERN.match(token) + + if m: + new_loras.append((f"{m.group(1)}.safetensors", float(m.group(2)))) + + for lora in loras + new_loras: + if lora[0] != "None": + updated_loras.append(lora) + + return updated_loras[:loras_limit] + + +def apply_wildcards(wildcard_text, rng, i, read_wildcards_in_order) -> str: + for _ in range(modules.config.wildcards_max_bfs_depth): + placeholders = re.findall(r'__([\w-]+)__', wildcard_text) + if len(placeholders) == 0: + return wildcard_text + + print(f'[Wildcards] processing: {wildcard_text}') + for placeholder in placeholders: + try: + matches = [x for x in modules.config.wildcard_filenames if os.path.splitext(os.path.basename(x))[0] == placeholder] + words = open(os.path.join(modules.config.path_wildcards, matches[0]), encoding='utf-8').read().splitlines() + words = [x for x in words if x != ''] + assert len(words) > 0 + if read_wildcards_in_order: + wildcard_text = wildcard_text.replace(f'__{placeholder}__', words[i % len(words)], 1) + else: + wildcard_text = wildcard_text.replace(f'__{placeholder}__', rng.choice(words), 1) + except: + print(f'[Wildcards] Warning: {placeholder}.txt missing or empty. ' + f'Using "{placeholder}" as a normal word.') + wildcard_text = wildcard_text.replace(f'__{placeholder}__', placeholder) + print(f'[Wildcards] {wildcard_text}') + + print(f'[Wildcards] BFS stack overflow. Current text: {wildcard_text}') + return wildcard_text diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..c424468fd --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,4 @@ +import sys +import pathlib + +sys.path.append(pathlib.Path(f'{__file__}/../modules').parent.resolve()) diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 000000000..0698dcc8e --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,48 @@ +import unittest + +from modules import util + + +class TestUtils(unittest.TestCase): + def test_can_parse_tokens_with_lora(self): + test_cases = [ + { + "input": ("some prompt, very cool, , cool ", [], 5), + "output": [("hey-lora.safetensors", 0.4), ("you-lora.safetensors", 0.2)], + }, + # Test can not exceed limit + { + "input": ("some prompt, very cool, , cool ", [], 1), + "output": [("hey-lora.safetensors", 0.4)], + }, + # test Loras from UI take precedence over prompt + { + "input": ( + "some prompt, very cool, , , , , , ", + [("hey-lora.safetensors", 0.4)], + 5, + ), + "output": [ + ("hey-lora.safetensors", 0.4), + ("l1.safetensors", 0.4), + ("l2.safetensors", -0.2), + ("l3.safetensors", 0.3), + ("l4.safetensors", 0.5), + ], + }, + # Test lora specification not separated by comma are ignored, only latest specified is used + { + "input": ("some prompt, very cool, ", [], 3), + "output": [("you-lora.safetensors", 0.2)], + }, + { + "input": (", , and ", [], 6), + "output": [] + } + ] + + for test in test_cases: + prompt, loras, loras_limit = test["input"] + expected = test["output"] + actual = util.parse_lora_references_from_prompt(prompt, loras, loras_limit) + self.assertEqual(expected, actual) From 2e2e8f851a501e1ae7870112b5e3144e241c015a Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Sun, 19 May 2024 13:08:33 +0200 Subject: [PATCH 16/42] feat: add tcd sampler and discrete distilled tcd scheduler based on sgm_uniform (same as lcm) (#2907) --- .../contrib/external_custom_sampler.py | 20 +++++++++++++ .../contrib/external_model_advanced.py | 5 +++- ldm_patched/k_diffusion/sampling.py | 28 ++++++++++++++++++- ldm_patched/modules/model_sampling.py | 8 +++--- ldm_patched/modules/samplers.py | 2 +- modules/async_worker.py | 8 +++--- modules/flags.py | 5 ++-- modules/patch_precision.py | 2 ++ 8 files changed, 65 insertions(+), 13 deletions(-) diff --git a/ldm_patched/contrib/external_custom_sampler.py b/ldm_patched/contrib/external_custom_sampler.py index 8f92e841f..985b03a0a 100644 --- a/ldm_patched/contrib/external_custom_sampler.py +++ b/ldm_patched/contrib/external_custom_sampler.py @@ -230,6 +230,25 @@ def get_sampler(self, eta, s_noise, r, noise_device): sampler = ldm_patched.modules.samplers.ksampler(sampler_name, {"eta": eta, "s_noise": s_noise, "r": r}) return (sampler, ) + +class SamplerTCD: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "eta": ("FLOAT", {"default": 0.3, "min": 0.0, "max": 1.0, "step": 0.01}), + } + } + RETURN_TYPES = ("SAMPLER",) + CATEGORY = "sampling/custom_sampling/samplers" + + FUNCTION = "get_sampler" + + def get_sampler(self, eta=0.3): + sampler = ldm_patched.modules.samplers.ksampler("tcd", {"eta": eta}) + return (sampler, ) + + class SamplerCustom: @classmethod def INPUT_TYPES(s): @@ -292,6 +311,7 @@ def sample(self, model, add_noise, noise_seed, cfg, positive, negative, sampler, "KSamplerSelect": KSamplerSelect, "SamplerDPMPP_2M_SDE": SamplerDPMPP_2M_SDE, "SamplerDPMPP_SDE": SamplerDPMPP_SDE, + "SamplerTCD": SamplerTCD, "SplitSigmas": SplitSigmas, "FlipSigmas": FlipSigmas, } diff --git a/ldm_patched/contrib/external_model_advanced.py b/ldm_patched/contrib/external_model_advanced.py index 03a2f0454..9b52c36b5 100644 --- a/ldm_patched/contrib/external_model_advanced.py +++ b/ldm_patched/contrib/external_model_advanced.py @@ -70,7 +70,7 @@ class ModelSamplingDiscrete: @classmethod def INPUT_TYPES(s): return {"required": { "model": ("MODEL",), - "sampling": (["eps", "v_prediction", "lcm"],), + "sampling": (["eps", "v_prediction", "lcm", "tcd"]), "zsnr": ("BOOLEAN", {"default": False}), }} @@ -90,6 +90,9 @@ def patch(self, model, sampling, zsnr): elif sampling == "lcm": sampling_type = LCM sampling_base = ModelSamplingDiscreteDistilled + elif sampling == "tcd": + sampling_type = ldm_patched.modules.model_sampling.EPS + sampling_base = ModelSamplingDiscreteDistilled class ModelSamplingAdvanced(sampling_base, sampling_type): pass diff --git a/ldm_patched/k_diffusion/sampling.py b/ldm_patched/k_diffusion/sampling.py index 761c2e0ef..d1bc1e4b2 100644 --- a/ldm_patched/k_diffusion/sampling.py +++ b/ldm_patched/k_diffusion/sampling.py @@ -752,7 +752,6 @@ def sample_lcm(model, x, sigmas, extra_args=None, callback=None, disable=None, n return x - @torch.no_grad() def sample_heunpp2(model, x, sigmas, extra_args=None, callback=None, disable=None, s_churn=0., s_tmin=0., s_tmax=float('inf'), s_noise=1.): # From MIT licensed: https://github.com/Carzit/sd-webui-samplers-scheduler/ @@ -808,3 +807,30 @@ def sample_heunpp2(model, x, sigmas, extra_args=None, callback=None, disable=Non d_prime = w1 * d + w2 * d_2 + w3 * d_3 x = x + d_prime * dt return x + + +@torch.no_grad() +def sample_tcd(model, x, sigmas, extra_args=None, callback=None, disable=None, noise_sampler=None, eta=0.3): + extra_args = {} if extra_args is None else extra_args + noise_sampler = default_noise_sampler(x) if noise_sampler is None else noise_sampler + s_in = x.new_ones([x.shape[0]]) + + model_sampling = model.inner_model.inner_model.model_sampling + timesteps_s = torch.floor((1 - eta) * model_sampling.timestep(sigmas)).to(dtype=torch.long).detach().cpu() + timesteps_s[-1] = 0 + alpha_prod_s = model_sampling.alphas_cumprod[timesteps_s] + beta_prod_s = 1 - alpha_prod_s + for i in trange(len(sigmas) - 1, disable=disable): + denoised = model(x, sigmas[i] * s_in, **extra_args) # predicted_original_sample + eps = (x - denoised) / sigmas[i] + denoised = alpha_prod_s[i + 1].sqrt() * denoised + beta_prod_s[i + 1].sqrt() * eps + + if callback is not None: + callback({"x": x, "i": i, "sigma": sigmas[i], "sigma_hat": sigmas[i], "denoised": denoised}) + + x = denoised + if eta > 0 and sigmas[i + 1] > 0: + noise = noise_sampler(sigmas[i], sigmas[i + 1]) + x = x / alpha_prod_s[i+1].sqrt() + noise * (sigmas[i+1]**2 + 1 - 1/alpha_prod_s[i+1]).sqrt() + + return x \ No newline at end of file diff --git a/ldm_patched/modules/model_sampling.py b/ldm_patched/modules/model_sampling.py index f39e275d3..57f51a000 100644 --- a/ldm_patched/modules/model_sampling.py +++ b/ldm_patched/modules/model_sampling.py @@ -50,17 +50,17 @@ def _register_schedule(self, given_betas=None, beta_schedule="linear", timesteps self.linear_start = linear_start self.linear_end = linear_end - # self.register_buffer('betas', torch.tensor(betas, dtype=torch.float32)) - # self.register_buffer('alphas_cumprod', torch.tensor(alphas_cumprod, dtype=torch.float32)) - # self.register_buffer('alphas_cumprod_prev', torch.tensor(alphas_cumprod_prev, dtype=torch.float32)) - sigmas = ((1 - alphas_cumprod) / alphas_cumprod) ** 0.5 self.set_sigmas(sigmas) + self.set_alphas_cumprod(alphas_cumprod.float()) def set_sigmas(self, sigmas): self.register_buffer('sigmas', sigmas) self.register_buffer('log_sigmas', sigmas.log()) + def set_alphas_cumprod(self, alphas_cumprod): + self.register_buffer("alphas_cumprod", alphas_cumprod.float()) + @property def sigma_min(self): return self.sigmas[0] diff --git a/ldm_patched/modules/samplers.py b/ldm_patched/modules/samplers.py index 1f69d2b10..35cb3d738 100644 --- a/ldm_patched/modules/samplers.py +++ b/ldm_patched/modules/samplers.py @@ -523,7 +523,7 @@ def sample(self, model_wrap, sigmas, extra_args, callback, noise, latent_image=N KSAMPLER_NAMES = ["euler", "euler_ancestral", "heun", "heunpp2","dpm_2", "dpm_2_ancestral", "lms", "dpm_fast", "dpm_adaptive", "dpmpp_2s_ancestral", "dpmpp_sde", "dpmpp_sde_gpu", - "dpmpp_2m", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddpm", "lcm"] + "dpmpp_2m", "dpmpp_2m_sde", "dpmpp_2m_sde_gpu", "dpmpp_3m_sde", "dpmpp_3m_sde_gpu", "ddpm", "lcm", "tcd"] class KSAMPLER(Sampler): def __init__(self, sampler_function, extra_options={}, inpaint_options={}): diff --git a/modules/async_worker.py b/modules/async_worker.py index 7f0a46e3e..1dabf89ca 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -798,19 +798,19 @@ def handler(async_task): final_sampler_name = sampler_name final_scheduler_name = scheduler_name - if scheduler_name == 'lcm': + if scheduler_name in ['lcm', 'tcd']: final_scheduler_name = 'sgm_uniform' if pipeline.final_unet is not None: pipeline.final_unet = core.opModelSamplingDiscrete.patch( pipeline.final_unet, - sampling='lcm', + sampling=scheduler_name, zsnr=False)[0] if pipeline.final_refiner_unet is not None: pipeline.final_refiner_unet = core.opModelSamplingDiscrete.patch( pipeline.final_refiner_unet, - sampling='lcm', + sampling=scheduler_name, zsnr=False)[0] - print('Using lcm scheduler.') + print(f'Using {scheduler_name} scheduler.') async_task.yields.append(['preview', (13, 'Moving model to GPU ...', None)]) diff --git a/modules/flags.py b/modules/flags.py index 0c6054394..cb4c3ec9d 100644 --- a/modules/flags.py +++ b/modules/flags.py @@ -34,7 +34,8 @@ "dpmpp_3m_sde": "", "dpmpp_3m_sde_gpu": "", "ddpm": "", - "lcm": "LCM" + "lcm": "LCM", + "tcd": "TCD" } SAMPLER_EXTRA = { @@ -47,7 +48,7 @@ KSAMPLER_NAMES = list(KSAMPLER.keys()) -SCHEDULER_NAMES = ["normal", "karras", "exponential", "sgm_uniform", "simple", "ddim_uniform", "lcm", "turbo", "align_your_steps"] +SCHEDULER_NAMES = ["normal", "karras", "exponential", "sgm_uniform", "simple", "ddim_uniform", "lcm", "turbo", "align_your_steps", "tcd"] SAMPLER_NAMES = KSAMPLER_NAMES + list(SAMPLER_EXTRA.keys()) sampler_list = SAMPLER_NAMES diff --git a/modules/patch_precision.py b/modules/patch_precision.py index 83569bdd1..22ffda0ad 100644 --- a/modules/patch_precision.py +++ b/modules/patch_precision.py @@ -51,6 +51,8 @@ def patched_register_schedule(self, given_betas=None, beta_schedule="linear", ti self.linear_end = linear_end sigmas = torch.tensor(((1 - alphas_cumprod) / alphas_cumprod) ** 0.5, dtype=torch.float32) self.set_sigmas(sigmas) + alphas_cumprod = torch.tensor(alphas_cumprod, dtype=torch.float32) + self.set_alphas_cumprod(alphas_cumprod) return From 13599edb9b5066649c3ac31bb5a7b15403fd6297 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Sun, 19 May 2024 13:23:08 +0200 Subject: [PATCH 17/42] feat: add performance hyper-sd based on 4step LoRA (#2812) * feat: add performance hyper-sd based on 4step LoRA * feat: use LoRA weight 0.8, sampler dpmpp_sde_gpu and scheduler_name karras suggested in https://github.com/lllyasviel/Fooocus/discussions/2813#discussioncomment-9245251 results see https://github.com/lllyasviel/Fooocus/discussions/2813#discussioncomment-9275251 * feat: change ByteDance huggingface profile with mashb1t * wip: add hyper-sd 8 step cfg lora with negative prompt support * feat: remove hyper-sd8 performance still waiting for the release of hyper-sd 4step CFG LoRA, not yet satisfied with any of the CFG LoRAs compared to non-cfg ones. see https://huggingface.co/ByteDance/Hyper-SD --- modules/async_worker.py | 27 +++++++++++++++++++++++++++ modules/config.py | 14 ++++++++++++-- modules/flags.py | 5 ++++- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/modules/async_worker.py b/modules/async_worker.py index 1dabf89ca..cf1eda30d 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -275,6 +275,33 @@ def handler(async_task): adm_scaler_negative = 1.0 adm_scaler_end = 0.0 + elif performance_selection == Performance.HYPER_SD: + print('Enter Hyper-SD mode.') + progressbar(async_task, 1, 'Downloading Hyper-SD components ...') + loras += [(modules.config.downloading_sdxl_hyper_sd_lora(), 0.8)] + + if refiner_model_name != 'None': + print(f'Refiner disabled in Hyper-SD mode.') + + refiner_model_name = 'None' + sampler_name = 'dpmpp_sde_gpu' + scheduler_name = 'karras' + sharpness = 0.0 + guidance_scale = 1.0 + adaptive_cfg = 1.0 + refiner_switch = 1.0 + adm_scaler_positive = 1.0 + adm_scaler_negative = 1.0 + adm_scaler_end = 0.0 + + elif performance_selection == Performance.HYPER_SD8: + print('Enter Hyper-SD8 mode.') + progressbar(async_task, 1, 'Downloading Hyper-SD components ...') + loras += [(modules.config.downloading_sdxl_hyper_sd_cfg_lora(), 0.3)] + + sampler_name = 'dpmpp_sde_gpu' + scheduler_name = 'normal' + print(f'[Parameters] Adaptive CFG = {adaptive_cfg}') print(f'[Parameters] Sharpness = {sharpness}') print(f'[Parameters] ControlNet Softness = {controlnet_softness}') diff --git a/modules/config.py b/modules/config.py index 11fe31818..db7036c53 100644 --- a/modules/config.py +++ b/modules/config.py @@ -553,7 +553,8 @@ def add_ratio(x): sdxl_lcm_lora = 'sdxl_lcm_lora.safetensors' sdxl_lightning_lora = 'sdxl_lightning_4step_lora.safetensors' -loras_metadata_remove = [sdxl_lcm_lora, sdxl_lightning_lora] +sdxl_hyper_sd_lora = 'sdxl_hyper_sd_4step_lora.safetensors' +loras_metadata_remove = [sdxl_lcm_lora, sdxl_lightning_lora, sdxl_hyper_sd_lora] def get_model_filenames(folder_paths, extensions=None, name_filter=None): @@ -627,13 +628,22 @@ def downloading_sdxl_lcm_lora(): def downloading_sdxl_lightning_lora(): load_file_from_url( - url='https://huggingface.co/ByteDance/SDXL-Lightning/resolve/main/sdxl_lightning_4step_lora.safetensors', + url='https://huggingface.co/mashb1t/misc/resolve/main/sdxl_lightning_4step_lora.safetensors', model_dir=paths_loras[0], file_name=sdxl_lightning_lora ) return sdxl_lightning_lora +def downloading_sdxl_hyper_sd_lora(): + load_file_from_url( + url='https://huggingface.co/mashb1t/misc/resolve/main/sdxl_hyper_sd_4step_lora.safetensors', + model_dir=paths_loras[0], + file_name=sdxl_hyper_sd_lora + ) + return sdxl_hyper_sd_lora + + def downloading_controlnet_canny(): load_file_from_url( url='https://huggingface.co/lllyasviel/misc/resolve/main/control-lora-canny-rank128.safetensors', diff --git a/modules/flags.py b/modules/flags.py index cb4c3ec9d..77ad012a2 100644 --- a/modules/flags.py +++ b/modules/flags.py @@ -110,6 +110,7 @@ class Steps(IntEnum): SPEED = 30 EXTREME_SPEED = 8 LIGHTNING = 4 + HYPER_SD = 4 class StepsUOV(IntEnum): @@ -117,6 +118,7 @@ class StepsUOV(IntEnum): SPEED = 18 EXTREME_SPEED = 8 LIGHTNING = 4 + HYPER_SD = 4 class Performance(Enum): @@ -124,6 +126,7 @@ class Performance(Enum): SPEED = 'Speed' EXTREME_SPEED = 'Extreme Speed' LIGHTNING = 'Lightning' + HYPER_SD = 'Hyper-SD' @classmethod def list(cls) -> list: @@ -133,7 +136,7 @@ def list(cls) -> list: def has_restricted_features(cls, x) -> bool: if isinstance(x, Performance): x = x.value - return x in [cls.EXTREME_SPEED.value, cls.LIGHTNING.value] + return x in [cls.EXTREME_SPEED.value, cls.LIGHTNING.value, cls.HYPER_SD.value] def steps(self) -> int | None: return Steps[self.name].value if Steps[self.name] else None From 0466ff944c96632e100a4cc988f7d02288ce7d3b Mon Sep 17 00:00:00 2001 From: Manuel Schmid Date: Sun, 19 May 2024 14:29:10 +0200 Subject: [PATCH 18/42] release: bump version number to 2.4.0-rc1 --- fooocus_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fooocus_version.py b/fooocus_version.py index b20501966..def34a203 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.3.1' +version = '2.4.0-rc1' From dad228907e5ae290f441510aafb3b07bd75a98ea Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Sun, 19 May 2024 17:42:46 +0200 Subject: [PATCH 19/42] fix: remove leftover code from hyper-sd8 testing (#2959) --- modules/async_worker.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/modules/async_worker.py b/modules/async_worker.py index cf1eda30d..892f99a7e 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -294,14 +294,6 @@ def handler(async_task): adm_scaler_negative = 1.0 adm_scaler_end = 0.0 - elif performance_selection == Performance.HYPER_SD8: - print('Enter Hyper-SD8 mode.') - progressbar(async_task, 1, 'Downloading Hyper-SD components ...') - loras += [(modules.config.downloading_sdxl_hyper_sd_cfg_lora(), 0.3)] - - sampler_name = 'dpmpp_sde_gpu' - scheduler_name = 'normal' - print(f'[Parameters] Adaptive CFG = {adaptive_cfg}') print(f'[Parameters] Sharpness = {sharpness}') print(f'[Parameters] ControlNet Softness = {controlnet_softness}') From 35b74dfa64b2efe8dd7652577eec443f45c56939 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Sun, 19 May 2024 18:02:24 +0200 Subject: [PATCH 20/42] feat: optimize model management of image censoring (#2960) now follows general Fooocus model management principles + includes code optimisations for reusability --- extras/censor.py | 76 ++++++++++++++++++++++------------------- modules/async_worker.py | 15 ++++---- 2 files changed, 48 insertions(+), 43 deletions(-) diff --git a/extras/censor.py b/extras/censor.py index 2047db246..45617fd85 100644 --- a/extras/censor.py +++ b/extras/censor.py @@ -1,56 +1,60 @@ -# modified version of https://github.com/AUTOMATIC1111/stable-diffusion-webui-nsfw-censor/blob/master/scripts/censor.py -import numpy as np import os -from extras.safety_checker.models.safety_checker import StableDiffusionSafetyChecker -from transformers import CLIPFeatureExtractor, CLIPConfig -from PIL import Image +import numpy as np +import torch +from transformers import CLIPConfig, CLIPImageProcessor + +import ldm_patched.modules.model_management as model_management import modules.config +from extras.safety_checker.models.safety_checker import StableDiffusionSafetyChecker +from ldm_patched.modules.model_patcher import ModelPatcher safety_checker_repo_root = os.path.join(os.path.dirname(__file__), 'safety_checker') config_path = os.path.join(safety_checker_repo_root, "configs", "config.json") preprocessor_config_path = os.path.join(safety_checker_repo_root, "configs", "preprocessor_config.json") -safety_feature_extractor = None -safety_checker = None - - -def numpy_to_pil(image): - image = (image * 255).round().astype("uint8") - pil_image = Image.fromarray(image) - - return pil_image +class Censor: + def __init__(self): + self.safety_checker_model: ModelPatcher | None = None + self.clip_image_processor: CLIPImageProcessor | None = None + self.load_device = torch.device('cpu') + self.offload_device = torch.device('cpu') -# check and replace nsfw content -def check_safety(x_image): - global safety_feature_extractor, safety_checker + def init(self): + if self.safety_checker_model is None and self.clip_image_processor is None: + safety_checker_model = modules.config.downloading_safety_checker_model() + self.clip_image_processor = CLIPImageProcessor.from_json_file(preprocessor_config_path) + clip_config = CLIPConfig.from_json_file(config_path) + model = StableDiffusionSafetyChecker.from_pretrained(safety_checker_model, config=clip_config) + model.eval() - if safety_feature_extractor is None or safety_checker is None: - safety_checker_model = modules.config.downloading_safety_checker_model() - safety_feature_extractor = CLIPFeatureExtractor.from_json_file(preprocessor_config_path) - clip_config = CLIPConfig.from_json_file(config_path) - safety_checker = StableDiffusionSafetyChecker.from_pretrained(safety_checker_model, config=clip_config) + self.load_device = model_management.text_encoder_device() + self.offload_device = model_management.text_encoder_offload_device() - safety_checker_input = safety_feature_extractor(numpy_to_pil(x_image), return_tensors="pt") - x_checked_image, has_nsfw_concept = safety_checker(images=x_image, clip_input=safety_checker_input.pixel_values) + model.to(self.offload_device) - return x_checked_image, has_nsfw_concept + self.safety_checker_model = ModelPatcher(model, load_device=self.load_device, offload_device=self.offload_device) + def censor(self, images: list | np.ndarray) -> list | np.ndarray: + self.init() + model_management.load_model_gpu(self.safety_checker_model) -def censor_single(x): - x_checked_image, has_nsfw_concept = check_safety(x) + single = False + if not isinstance(images, list) or isinstance(images, np.ndarray): + images = [images] + single = True - # replace image with black pixels, keep dimensions - # workaround due to different numpy / pytorch image matrix format - if has_nsfw_concept[0]: - imageshape = x_checked_image.shape - x_checked_image = np.zeros((imageshape[0], imageshape[1], 3), dtype = np.uint8) + safety_checker_input = self.clip_image_processor(images, return_tensors="pt") + safety_checker_input.to(device=self.load_device) + checked_images, has_nsfw_concept = self.safety_checker_model.model(images=images, + clip_input=safety_checker_input.pixel_values) + checked_images = [image.astype(np.uint8) for image in checked_images] - return x_checked_image + if single: + checked_images = checked_images[0] + return checked_images -def censor_batch(images): - images = [censor_single(image) for image in images] - return images \ No newline at end of file +default_censor = Censor().censor diff --git a/modules/async_worker.py b/modules/async_worker.py index 892f99a7e..302db84c1 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -44,7 +44,7 @@ def worker(): import fooocus_version import args_manager - from extras.censor import censor_batch, censor_single + from extras.censor import default_censor from modules.sdxl_styles import apply_style, get_random_style, fooocus_expansion, apply_arrays, random_style_name from modules.private_logger import log from extras.expansion import safe_str @@ -78,7 +78,7 @@ def yield_result(async_task, imgs, black_out_nsfw, censor=True, do_not_show_fini if censor and (modules.config.default_black_out_nsfw or black_out_nsfw): progressbar(async_task, progressbar_index, 'Checking for NSFW content ...') - imgs = censor_batch(imgs) + imgs = default_censor(imgs) async_task.results = async_task.results + imgs @@ -615,7 +615,8 @@ def handler(async_task): d = [('Upscale (Fast)', 'upscale_fast', '2x')] if modules.config.default_black_out_nsfw or black_out_nsfw: progressbar(async_task, 100, 'Checking for NSFW content ...') - uov_input_image = censor_single(uov_input_image) + uov_input_image = default_censor(uov_input_image) + progressbar(async_task, 100, 'Saving image to system ...') uov_input_image_path = log(uov_input_image, d, output_format=output_format) yield_result(async_task, uov_input_image_path, black_out_nsfw, False, do_not_show_finished_images=True) return @@ -883,12 +884,12 @@ def callback(step, x0, x, total_steps, y): imgs = [inpaint_worker.current_task.post_process(x) for x in imgs] img_paths = [] - + current_progress = int(15.0 + 85.0 * float((current_task_id + 1) * steps) / float(all_steps)) if modules.config.default_black_out_nsfw or black_out_nsfw: - progressbar(async_task, int(15.0 + 85.0 * float((current_task_id + 1) * steps) / float(all_steps)), - 'Checking for NSFW content ...') - imgs = censor_batch(imgs) + progressbar(async_task, current_progress, 'Checking for NSFW content ...') + imgs = default_censor(imgs) + progressbar(async_task, current_progress, 'Saving image to system ...') for x in imgs: d = [('Prompt', 'prompt', task['log_positive_prompt']), ('Negative Prompt', 'negative_prompt', task['log_negative_prompt']), From e94b97604f245c137bced3c5a941b45667f1483b Mon Sep 17 00:00:00 2001 From: Manuel Schmid Date: Sun, 19 May 2024 18:37:18 +0200 Subject: [PATCH 21/42] release: bump version number to 2.4.0-rc2 --- fooocus_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fooocus_version.py b/fooocus_version.py index def34a203..41556f902 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.4.0-rc1' +version = '2.4.0-rc2' From c9955117050bd32a4fbc7e4af694b51e79c64ad1 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Sun, 19 May 2024 20:43:11 +0200 Subject: [PATCH 22/42] feat: progress bar improvements (#2962) * feat: align progress bar vertically * feat: use fixed width for status text, remove ordinals * refactor: align progress to actions --- css/style.css | 6 ++++++ modules/async_worker.py | 37 +++++++++++++++++++------------------ modules/flags.py | 1 + modules/util.py | 4 ---- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/css/style.css b/css/style.css index b9e6e2ce1..b5f7a4488 100644 --- a/css/style.css +++ b/css/style.css @@ -27,6 +27,7 @@ progress { border-radius: 5px; /* Round the corners of the progress bar */ background-color: #f3f3f3; /* Light grey background */ width: 100%; + vertical-align: middle !important; } /* Style the progress bar container */ @@ -69,6 +70,11 @@ progress::after { height: 30px !important; } +.progress-bar span { + text-align: right; + width: 200px; +} + .type_row{ height: 80px !important; } diff --git a/modules/async_worker.py b/modules/async_worker.py index 302db84c1..8e4d6d957 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -49,7 +49,7 @@ def worker(): from modules.private_logger import log from extras.expansion import safe_str from modules.util import (remove_empty_str, HWC3, resize_image, get_image_shape_ceil, set_image_shape_ceil, - get_shape_ceil, resample_image, erode_or_dilate, ordinal_suffix, get_enabled_loras, + get_shape_ceil, resample_image, erode_or_dilate, get_enabled_loras, parse_lora_references_from_prompt, apply_wildcards) from modules.upscaler import perform_upscale from modules.flags import Performance @@ -72,7 +72,7 @@ def progressbar(async_task, number, text): async_task.yields.append(['preview', (number, text, None)]) def yield_result(async_task, imgs, black_out_nsfw, censor=True, do_not_show_finished_images=False, - progressbar_index=13): + progressbar_index=flags.preparation_step_count): if not isinstance(imgs, list): imgs = [imgs] @@ -456,7 +456,7 @@ def handler(async_task): extra_positive_prompts = prompts[1:] if len(prompts) > 1 else [] extra_negative_prompts = negative_prompts[1:] if len(negative_prompts) > 1 else [] - progressbar(async_task, 3, 'Loading models ...') + progressbar(async_task, 2, 'Loading models ...') loras = parse_lora_references_from_prompt(prompt, loras, modules.config.default_max_lora_number) @@ -523,25 +523,25 @@ def handler(async_task): if use_expansion: for i, t in enumerate(tasks): - progressbar(async_task, 5, f'Preparing Fooocus text #{i + 1} ...') + progressbar(async_task, 4, f'Preparing Fooocus text #{i + 1} ...') expansion = pipeline.final_expansion(t['task_prompt'], t['task_seed']) print(f'[Prompt Expansion] {expansion}') t['expansion'] = expansion t['positive'] = copy.deepcopy(t['positive']) + [expansion] # Deep copy. for i, t in enumerate(tasks): - progressbar(async_task, 7, f'Encoding positive #{i + 1} ...') + progressbar(async_task, 5, f'Encoding positive #{i + 1} ...') t['c'] = pipeline.clip_encode(texts=t['positive'], pool_top_k=t['positive_top_k']) for i, t in enumerate(tasks): if abs(float(cfg_scale) - 1.0) < 1e-4: t['uc'] = pipeline.clone_cond(t['c']) else: - progressbar(async_task, 10, f'Encoding negative #{i + 1} ...') + progressbar(async_task, 6, f'Encoding negative #{i + 1} ...') t['uc'] = pipeline.clip_encode(texts=t['negative'], pool_top_k=t['negative_top_k']) if len(goals) > 0: - progressbar(async_task, 13, 'Image processing ...') + progressbar(async_task, 7, 'Image processing ...') if 'vary' in goals: if 'subtle' in uov_method: @@ -562,7 +562,7 @@ def handler(async_task): uov_input_image = set_image_shape_ceil(uov_input_image, shape_ceil) initial_pixels = core.numpy_to_pytorch(uov_input_image) - progressbar(async_task, 13, 'VAE encoding ...') + progressbar(async_task, 8, 'VAE encoding ...') candidate_vae, _ = pipeline.get_candidate_vae( steps=steps, @@ -579,7 +579,7 @@ def handler(async_task): if 'upscale' in goals: H, W, C = uov_input_image.shape - progressbar(async_task, 13, f'Upscaling image from {str((H, W))} ...') + progressbar(async_task, 9, f'Upscaling image from {str((H, W))} ...') uov_input_image = perform_upscale(uov_input_image) print(f'Image upscaled.') @@ -628,7 +628,7 @@ def handler(async_task): denoising_strength = overwrite_upscale_strength initial_pixels = core.numpy_to_pytorch(uov_input_image) - progressbar(async_task, 13, 'VAE encoding ...') + progressbar(async_task, 10, 'VAE encoding ...') candidate_vae, _ = pipeline.get_candidate_vae( steps=steps, @@ -686,7 +686,7 @@ def handler(async_task): do_not_show_finished_images=True) return - progressbar(async_task, 13, 'VAE Inpaint encoding ...') + progressbar(async_task, 11, 'VAE Inpaint encoding ...') inpaint_pixel_fill = core.numpy_to_pytorch(inpaint_worker.current_task.interested_fill) inpaint_pixel_image = core.numpy_to_pytorch(inpaint_worker.current_task.interested_image) @@ -706,7 +706,7 @@ def handler(async_task): latent_swap = None if candidate_vae_swap is not None: - progressbar(async_task, 13, 'VAE SD15 encoding ...') + progressbar(async_task, 12, 'VAE SD15 encoding ...') latent_swap = core.encode_vae( vae=candidate_vae_swap, pixels=inpaint_pixel_fill)['samples'] @@ -832,16 +832,17 @@ def handler(async_task): zsnr=False)[0] print(f'Using {scheduler_name} scheduler.') - async_task.yields.append(['preview', (13, 'Moving model to GPU ...', None)]) + async_task.yields.append(['preview', (flags.preparation_step_count, 'Moving model to GPU ...', None)]) def callback(step, x0, x, total_steps, y): done_steps = current_task_id * steps + step async_task.yields.append(['preview', ( - int(15.0 + 85.0 * float(done_steps) / float(all_steps)), - f'Step {step}/{total_steps} in the {current_task_id + 1}{ordinal_suffix(current_task_id + 1)} Sampling', - y)]) + int(flags.preparation_step_count + (100 - flags.preparation_step_count) * float(done_steps) / float(all_steps)), + f'Sampling step {step + 1}/{total_steps}, image {current_task_id + 1}/{image_number} ...', y)]) for current_task_id, task in enumerate(tasks): + current_progress = int(flags.preparation_step_count + (100 - flags.preparation_step_count) * float(current_task_id * steps) / float(all_steps)) + progressbar(async_task, current_progress, f'Preparing task {current_task_id + 1}/{image_number} ...') execution_start_time = time.perf_counter() try: @@ -884,12 +885,12 @@ def callback(step, x0, x, total_steps, y): imgs = [inpaint_worker.current_task.post_process(x) for x in imgs] img_paths = [] - current_progress = int(15.0 + 85.0 * float((current_task_id + 1) * steps) / float(all_steps)) + current_progress = int(flags.preparation_step_count + (100 - flags.preparation_step_count) * float((current_task_id + 1) * steps) / float(all_steps)) if modules.config.default_black_out_nsfw or black_out_nsfw: progressbar(async_task, current_progress, 'Checking for NSFW content ...') imgs = default_censor(imgs) - progressbar(async_task, current_progress, 'Saving image to system ...') + progressbar(async_task, current_progress, f'Saving image {current_task_id + 1}/{image_number} to system ...') for x in imgs: d = [('Prompt', 'prompt', task['log_positive_prompt']), ('Negative Prompt', 'negative_prompt', task['log_negative_prompt']), diff --git a/modules/flags.py b/modules/flags.py index 77ad012a2..7b3ac3933 100644 --- a/modules/flags.py +++ b/modules/flags.py @@ -93,6 +93,7 @@ class MetadataScheme(Enum): ] controlnet_image_count = 4 +preparation_step_count = 13 class OutputFormat(Enum): diff --git a/modules/util.py b/modules/util.py index 734302306..8e85ffbe9 100644 --- a/modules/util.py +++ b/modules/util.py @@ -372,10 +372,6 @@ def get_file_from_folder_list(name, folders): return os.path.abspath(os.path.realpath(os.path.join(folders[0], name))) -def ordinal_suffix(number: int) -> str: - return 'th' if 10 <= number % 100 <= 20 else {1: 'st', 2: 'nd', 3: 'rd'}.get(number % 10, 'th') - - def makedirs_with_log(path): try: os.makedirs(path, exist_ok=True) From 65a8b25129c52ccb6f9fe5933202712b533c977d Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Mon, 20 May 2024 17:31:51 +0200 Subject: [PATCH 23/42] feat: inline lora optimisations (#2967) * feat: add performance loras to the end of the loras array * fix: resolve circular dependency for unit tests * feat: allow multiple matches for each token, optimize and extract method cleanup_prompt * fix: update unit tests * feat: ignore custom wildcards --- modules/async_worker.py | 12 +++++---- modules/config.py | 3 +-- modules/extra_utils.py | 6 +++++ modules/util.py | 58 +++++++++++++++++++++++++++++------------ tests/test_utils.py | 41 ++++++++++++++++++++--------- wildcards/.gitignore | 8 ++++++ 6 files changed, 92 insertions(+), 36 deletions(-) create mode 100644 wildcards/.gitignore diff --git a/modules/async_worker.py b/modules/async_worker.py index 8e4d6d957..594886d28 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -237,10 +237,12 @@ def handler(async_task): steps = performance_selection.steps() + performance_loras = [] + if performance_selection == Performance.EXTREME_SPEED: print('Enter LCM mode.') progressbar(async_task, 1, 'Downloading LCM components ...') - loras += [(modules.config.downloading_sdxl_lcm_lora(), 1.0)] + performance_loras += [(modules.config.downloading_sdxl_lcm_lora(), 1.0)] if refiner_model_name != 'None': print(f'Refiner disabled in LCM mode.') @@ -259,7 +261,7 @@ def handler(async_task): elif performance_selection == Performance.LIGHTNING: print('Enter Lightning mode.') progressbar(async_task, 1, 'Downloading Lightning components ...') - loras += [(modules.config.downloading_sdxl_lightning_lora(), 1.0)] + performance_loras += [(modules.config.downloading_sdxl_lightning_lora(), 1.0)] if refiner_model_name != 'None': print(f'Refiner disabled in Lightning mode.') @@ -278,7 +280,7 @@ def handler(async_task): elif performance_selection == Performance.HYPER_SD: print('Enter Hyper-SD mode.') progressbar(async_task, 1, 'Downloading Hyper-SD components ...') - loras += [(modules.config.downloading_sdxl_hyper_sd_lora(), 0.8)] + performance_loras += [(modules.config.downloading_sdxl_hyper_sd_lora(), 0.8)] if refiner_model_name != 'None': print(f'Refiner disabled in Hyper-SD mode.') @@ -458,8 +460,8 @@ def handler(async_task): progressbar(async_task, 2, 'Loading models ...') - loras = parse_lora_references_from_prompt(prompt, loras, modules.config.default_max_lora_number) - + loras, prompt = parse_lora_references_from_prompt(prompt, loras, modules.config.default_max_lora_number) + loras += performance_loras pipeline.refresh_everything(refiner_model_name=refiner_model_name, base_model_name=base_model_name, loras=loras, base_model_additional_loras=base_model_additional_loras, use_synthetic_refiner=use_synthetic_refiner, vae_name=vae_name) diff --git a/modules/config.py b/modules/config.py index db7036c53..913fb281a 100644 --- a/modules/config.py +++ b/modules/config.py @@ -8,8 +8,7 @@ import modules.sdxl_styles from modules.model_loader import load_file_from_url -from modules.util import makedirs_with_log -from modules.extra_utils import get_files_from_folder +from modules.extra_utils import makedirs_with_log, get_files_from_folder from modules.flags import OutputFormat, Performance, MetadataScheme diff --git a/modules/extra_utils.py b/modules/extra_utils.py index 3e95e8b56..9906c8202 100644 --- a/modules/extra_utils.py +++ b/modules/extra_utils.py @@ -1,5 +1,11 @@ import os +def makedirs_with_log(path): + try: + os.makedirs(path, exist_ok=True) + except OSError as error: + print(f'Directory {path} could not be created, reason: {error}') + def get_files_from_folder(folder_path, extensions=None, name_filter=None): if not os.path.isdir(folder_path): diff --git a/modules/util.py b/modules/util.py index 8e85ffbe9..52bc490aa 100644 --- a/modules/util.py +++ b/modules/util.py @@ -12,15 +12,15 @@ from PIL import Image +import modules.config import modules.sdxl_styles LANCZOS = (Image.Resampling.LANCZOS if hasattr(Image, 'Resampling') else Image.LANCZOS) - # Regexp compiled once. Matches entries with the following pattern: # # -LORAS_PROMPT_PATTERN = re.compile(r".* .*", re.X) +LORAS_PROMPT_PATTERN = re.compile(r"()", re.X) HASH_SHA256_LENGTH = 10 @@ -372,31 +372,57 @@ def get_file_from_folder_list(name, folders): return os.path.abspath(os.path.realpath(os.path.join(folders[0], name))) -def makedirs_with_log(path): - try: - os.makedirs(path, exist_ok=True) - except OSError as error: - print(f'Directory {path} could not be created, reason: {error}') +def get_enabled_loras(loras: list, remove_none=True) -> list: + return [(lora[1], lora[2]) for lora in loras if lora[0] and (lora[1] != 'None' if remove_none else True)] -def get_enabled_loras(loras: list) -> list: - return [(lora[1], lora[2]) for lora in loras if lora[0]] +def parse_lora_references_from_prompt(prompt: str, loras: List[Tuple[AnyStr, float]], loras_limit: int = 5, + prompt_cleanup=True, deduplicate_loras=True) -> tuple[List[Tuple[AnyStr, float]], str]: + found_loras = [] + prompt_without_loras = "" + for token in prompt.split(" "): + matches = LORAS_PROMPT_PATTERN.findall(token) + + if matches: + for match in matches: + found_loras.append((f"{match[1]}.safetensors", float(match[2]))) + prompt_without_loras += token.replace(match[0], '') + else: + prompt_without_loras += token + prompt_without_loras += ' ' + cleaned_prompt = prompt_without_loras[:-1] + if prompt_cleanup: + cleaned_prompt = cleanup_prompt(prompt_without_loras) -def parse_lora_references_from_prompt(prompt: str, loras: List[Tuple[AnyStr, float]], loras_limit: int = 5) -> List[Tuple[AnyStr, float]]: new_loras = [] - updated_loras = [] - for token in prompt.split(","): - m = LORAS_PROMPT_PATTERN.match(token) + lora_names = [lora[0] for lora in loras] + for found_lora in found_loras: + if deduplicate_loras and found_lora[0] in lora_names: + continue + new_loras.append(found_lora) - if m: - new_loras.append((f"{m.group(1)}.safetensors", float(m.group(2)))) + if len(new_loras) == 0: + return loras, cleaned_prompt + updated_loras = [] for lora in loras + new_loras: if lora[0] != "None": updated_loras.append(lora) - return updated_loras[:loras_limit] + return updated_loras[:loras_limit], cleaned_prompt + + +def cleanup_prompt(prompt): + prompt = re.sub(' +', ' ', prompt) + prompt = re.sub(',+', ',', prompt) + cleaned_prompt = '' + for token in prompt.split(','): + token = token.strip() + if token == '': + continue + cleaned_prompt += token + ', ' + return cleaned_prompt[:-2] def apply_wildcards(wildcard_text, rng, i, read_wildcards_in_order) -> str: diff --git a/tests/test_utils.py b/tests/test_utils.py index 0698dcc8e..9f81005b0 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -8,12 +8,16 @@ def test_can_parse_tokens_with_lora(self): test_cases = [ { "input": ("some prompt, very cool, , cool ", [], 5), - "output": [("hey-lora.safetensors", 0.4), ("you-lora.safetensors", 0.2)], + "output": ( + [('hey-lora.safetensors', 0.4), ('you-lora.safetensors', 0.2)], 'some prompt, very cool, cool'), }, # Test can not exceed limit { "input": ("some prompt, very cool, , cool ", [], 1), - "output": [("hey-lora.safetensors", 0.4)], + "output": ( + [('hey-lora.safetensors', 0.4)], + 'some prompt, very cool, cool' + ), }, # test Loras from UI take precedence over prompt { @@ -22,22 +26,33 @@ def test_can_parse_tokens_with_lora(self): [("hey-lora.safetensors", 0.4)], 5, ), - "output": [ - ("hey-lora.safetensors", 0.4), - ("l1.safetensors", 0.4), - ("l2.safetensors", -0.2), - ("l3.safetensors", 0.3), - ("l4.safetensors", 0.5), - ], + "output": ( + [ + ('hey-lora.safetensors', 0.4), + ('l1.safetensors', 0.4), + ('l2.safetensors', -0.2), + ('l3.safetensors', 0.3), + ('l4.safetensors', 0.5) + ], + 'some prompt, very cool' + ) }, - # Test lora specification not separated by comma are ignored, only latest specified is used { "input": ("some prompt, very cool, ", [], 3), - "output": [("you-lora.safetensors", 0.2)], + "output": ( + [ + ('hey-lora.safetensors', 0.4), + ('you-lora.safetensors', 0.2) + ], + 'some prompt, very cool, ' + ), }, { - "input": (", , and ", [], 6), - "output": [] + "input": (", , , and ", [], 6), + "output": ( + [], + ', , , and ' + ) } ] diff --git a/wildcards/.gitignore b/wildcards/.gitignore new file mode 100644 index 000000000..7e4ac188a --- /dev/null +++ b/wildcards/.gitignore @@ -0,0 +1,8 @@ +*.txt +!animal.txt +!artist.txt +!color.txt +!color_flower.txt +!extended-color.txt +!flower.txt +!nationality.txt \ No newline at end of file From ac14d9d03ce731c0f57961ab1fde9c4e276bad99 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Mon, 20 May 2024 17:33:12 +0200 Subject: [PATCH 24/42] feat: change code owner from @lllyasviel to @mashb1t (#2948) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 585eb87aa..f9876685f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @lllyasviel +* @mashb1t From 7537612bcc43cf76a5caf9901e6fcf37099d554e Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Mon, 20 May 2024 19:21:41 +0200 Subject: [PATCH 25/42] feat: only use valid inline loras, add subfolder support (#2968) --- modules/config.py | 14 +++++++++++++- modules/meta_parser.py | 21 ++++----------------- modules/util.py | 41 +++++++++++++++++++++++++++++------------ tests/test_utils.py | 32 +++++++++++++++++++++++++------- 4 files changed, 71 insertions(+), 37 deletions(-) diff --git a/modules/config.py b/modules/config.py index 913fb281a..94046661f 100644 --- a/modules/config.py +++ b/modules/config.py @@ -547,6 +547,7 @@ def add_ratio(x): model_filenames = [] lora_filenames = [] +lora_filenames_no_special = [] vae_filenames = [] wildcard_filenames = [] @@ -556,6 +557,16 @@ def add_ratio(x): loras_metadata_remove = [sdxl_lcm_lora, sdxl_lightning_lora, sdxl_hyper_sd_lora] +def remove_special_loras(lora_filenames): + global loras_metadata_remove + + loras_no_special = lora_filenames.copy() + for lora_to_remove in loras_metadata_remove: + if lora_to_remove in loras_no_special: + loras_no_special.remove(lora_to_remove) + return loras_no_special + + def get_model_filenames(folder_paths, extensions=None, name_filter=None): if extensions is None: extensions = ['.pth', '.ckpt', '.bin', '.safetensors', '.fooocus.patch'] @@ -570,9 +581,10 @@ def get_model_filenames(folder_paths, extensions=None, name_filter=None): def update_files(): - global model_filenames, lora_filenames, vae_filenames, wildcard_filenames, available_presets + global model_filenames, lora_filenames, lora_filenames_no_special, vae_filenames, wildcard_filenames, available_presets model_filenames = get_model_filenames(paths_checkpoints) lora_filenames = get_model_filenames(paths_loras) + lora_filenames_no_special = remove_special_loras(lora_filenames) vae_filenames = get_model_filenames(path_vae) wildcard_filenames = get_files_from_folder(path_wildcards, ['.txt']) available_presets = get_presets() diff --git a/modules/meta_parser.py b/modules/meta_parser.py index 84032e829..2469da5f9 100644 --- a/modules/meta_parser.py +++ b/modules/meta_parser.py @@ -205,7 +205,6 @@ def get_lora(key: str, fallback: str | None, source_dict: dict, results: list): def get_sha256(filepath): global hash_cache if filepath not in hash_cache: - # is_safetensors = os.path.splitext(filepath)[1].lower() == '.safetensors' hash_cache[filepath] = sha256(filepath) return hash_cache[filepath] @@ -293,12 +292,6 @@ def set_data(self, raw_prompt, full_prompt, raw_negative_prompt, full_negative_p self.loras.append((Path(lora_name).stem, lora_weight, lora_hash)) self.vae_name = Path(vae_name).stem - @staticmethod - def remove_special_loras(lora_filenames): - for lora_to_remove in modules.config.loras_metadata_remove: - if lora_to_remove in lora_filenames: - lora_filenames.remove(lora_to_remove) - class A1111MetadataParser(MetadataParser): def get_scheme(self) -> MetadataScheme: @@ -415,13 +408,11 @@ def parse_json(self, metadata: str) -> dict: lora_data = data['lora_hashes'] if lora_data != '': - lora_filenames = modules.config.lora_filenames.copy() - self.remove_special_loras(lora_filenames) for li, lora in enumerate(lora_data.split(', ')): lora_split = lora.split(': ') lora_name = lora_split[0] lora_weight = lora_split[2] if len(lora_split) == 3 else lora_split[1] - for filename in lora_filenames: + for filename in modules.config.lora_filenames_no_special: path = Path(filename) if lora_name == path.stem: data[f'lora_combined_{li + 1}'] = f'{filename} : {lora_weight}' @@ -510,19 +501,15 @@ def get_scheme(self) -> MetadataScheme: return MetadataScheme.FOOOCUS def parse_json(self, metadata: dict) -> dict: - model_filenames = modules.config.model_filenames.copy() - lora_filenames = modules.config.lora_filenames.copy() - vae_filenames = modules.config.vae_filenames.copy() - self.remove_special_loras(lora_filenames) for key, value in metadata.items(): if value in ['', 'None']: continue if key in ['base_model', 'refiner_model']: - metadata[key] = self.replace_value_with_filename(key, value, model_filenames) + metadata[key] = self.replace_value_with_filename(key, value, modules.config.model_filenames) elif key.startswith('lora_combined_'): - metadata[key] = self.replace_value_with_filename(key, value, lora_filenames) + metadata[key] = self.replace_value_with_filename(key, value, modules.config.lora_filenames_no_special) elif key == 'vae': - metadata[key] = self.replace_value_with_filename(key, value, vae_filenames) + metadata[key] = self.replace_value_with_filename(key, value, modules.config.vae_filenames) else: continue diff --git a/modules/util.py b/modules/util.py index 52bc490aa..cb5580fbb 100644 --- a/modules/util.py +++ b/modules/util.py @@ -1,3 +1,5 @@ +from pathlib import Path + import numpy as np import datetime import random @@ -360,6 +362,14 @@ def is_json(data: str) -> bool: return True +def get_filname_by_stem(lora_name, filenames: List[str]) -> str | None: + for filename in filenames: + path = Path(filename) + if lora_name == path.stem: + return filename + return None + + def get_file_from_folder_list(name, folders): if not isinstance(folders, list): folders = [folders] @@ -377,28 +387,35 @@ def get_enabled_loras(loras: list, remove_none=True) -> list: def parse_lora_references_from_prompt(prompt: str, loras: List[Tuple[AnyStr, float]], loras_limit: int = 5, - prompt_cleanup=True, deduplicate_loras=True) -> tuple[List[Tuple[AnyStr, float]], str]: + skip_file_check=False, prompt_cleanup=True, deduplicate_loras=True) -> tuple[List[Tuple[AnyStr, float]], str]: found_loras = [] - prompt_without_loras = "" - for token in prompt.split(" "): + prompt_without_loras = '' + cleaned_prompt = '' + for token in prompt.split(','): matches = LORAS_PROMPT_PATTERN.findall(token) - if matches: - for match in matches: - found_loras.append((f"{match[1]}.safetensors", float(match[2]))) - prompt_without_loras += token.replace(match[0], '') - else: - prompt_without_loras += token - prompt_without_loras += ' ' + if len(matches) == 0: + prompt_without_loras += token + ', ' + continue + for match in matches: + lora_name = match[1] + '.safetensors' + if not skip_file_check: + lora_name = get_filname_by_stem(match[1], modules.config.lora_filenames_no_special) + if lora_name is not None: + found_loras.append((lora_name, float(match[2]))) + token = token.replace(match[0], '') + prompt_without_loras += token + ', ' + + if prompt_without_loras != '': + cleaned_prompt = prompt_without_loras[:-2] - cleaned_prompt = prompt_without_loras[:-1] if prompt_cleanup: cleaned_prompt = cleanup_prompt(prompt_without_loras) new_loras = [] lora_names = [lora[0] for lora in loras] for found_lora in found_loras: - if deduplicate_loras and found_lora[0] in lora_names: + if deduplicate_loras and (found_lora[0] in lora_names or found_lora in new_loras): continue new_loras.append(found_lora) diff --git a/tests/test_utils.py b/tests/test_utils.py index 9f81005b0..6fd550db3 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -7,13 +7,13 @@ class TestUtils(unittest.TestCase): def test_can_parse_tokens_with_lora(self): test_cases = [ { - "input": ("some prompt, very cool, , cool ", [], 5), + "input": ("some prompt, very cool, , cool ", [], 5, True), "output": ( [('hey-lora.safetensors', 0.4), ('you-lora.safetensors', 0.2)], 'some prompt, very cool, cool'), }, # Test can not exceed limit { - "input": ("some prompt, very cool, , cool ", [], 1), + "input": ("some prompt, very cool, , cool ", [], 1, True), "output": ( [('hey-lora.safetensors', 0.4)], 'some prompt, very cool, cool' @@ -25,6 +25,7 @@ def test_can_parse_tokens_with_lora(self): "some prompt, very cool, , , , , , ", [("hey-lora.safetensors", 0.4)], 5, + True ), "output": ( [ @@ -37,18 +38,35 @@ def test_can_parse_tokens_with_lora(self): 'some prompt, very cool' ) }, + # test correct matching even if there is no space separating loras in the same token { - "input": ("some prompt, very cool, ", [], 3), + "input": ("some prompt, very cool, ", [], 3, True), "output": ( [ ('hey-lora.safetensors', 0.4), ('you-lora.safetensors', 0.2) ], - 'some prompt, very cool, ' + 'some prompt, very cool' + ), + }, + # test deduplication, also selected loras are never overridden with loras in prompt + { + "input": ( + "some prompt, very cool, ", + [('you-lora.safetensors', 0.3)], + 3, + True + ), + "output": ( + [ + ('you-lora.safetensors', 0.3), + ('hey-lora.safetensors', 0.4) + ], + 'some prompt, very cool' ), }, { - "input": (", , , and ", [], 6), + "input": (", , , and ", [], 6, True), "output": ( [], ', , , and ' @@ -57,7 +75,7 @@ def test_can_parse_tokens_with_lora(self): ] for test in test_cases: - prompt, loras, loras_limit = test["input"] + prompt, loras, loras_limit, skip_file_check = test["input"] expected = test["output"] - actual = util.parse_lora_references_from_prompt(prompt, loras, loras_limit) + actual = util.parse_lora_references_from_prompt(prompt, loras, loras_limit=loras_limit, skip_file_check=skip_file_check) self.assertEqual(expected, actual) From 302bfdf855ca8271d5ddc87a6db89504fb519718 Mon Sep 17 00:00:00 2001 From: xhoxye <129571231+xhoxye@users.noreply.github.com> Date: Thu, 23 May 2024 02:47:44 +0800 Subject: [PATCH 26/42] feat: read size and ratio of an image and provide the recommended size (#2971) * Add the information about the size and ratio of the read image * feat: use available aspect ratios from config, move function to util, change default visibility of label * refactor: extract sdxl aspect ratios to flags, use in describe as discussed in https://github.com/lllyasviel/Fooocus/pull/2971#discussion_r1608493765 https://github.com/lllyasviel/Fooocus/pull/2971#issuecomment-2123620595 --------- Co-authored-by: Manuel Schmid Co-authored-by: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> --- modules/config.py | 10 ++-------- modules/flags.py | 7 +++++++ modules/meta_parser.py | 2 +- modules/util.py | 32 ++++++++++++++++++++++++++++++++ webui.py | 11 ++++++++++- 5 files changed, 52 insertions(+), 10 deletions(-) diff --git a/modules/config.py b/modules/config.py index 94046661f..64b0b86ff 100644 --- a/modules/config.py +++ b/modules/config.py @@ -416,13 +416,7 @@ def init_temp_path(path: str | None, default_path: str) -> str: ) available_aspect_ratios = get_config_item_or_set_default( key='available_aspect_ratios', - default_value=[ - '704*1408', '704*1344', '768*1344', '768*1280', '832*1216', '832*1152', - '896*1152', '896*1088', '960*1088', '960*1024', '1024*1024', '1024*960', - '1088*960', '1088*896', '1152*896', '1152*832', '1216*832', '1280*768', - '1344*768', '1344*704', '1408*704', '1472*704', '1536*640', '1600*640', - '1664*576', '1728*576' - ], + default_value=modules.flags.sdxl_aspect_ratios, validator=lambda x: isinstance(x, list) and all('*' in v for v in x) and len(x) > 1 ) default_aspect_ratio = get_config_item_or_set_default( @@ -526,7 +520,7 @@ def add_ratio(x): default_aspect_ratio = add_ratio(default_aspect_ratio) -available_aspect_ratios = [add_ratio(x) for x in available_aspect_ratios] +available_aspect_ratios_labels = [add_ratio(x) for x in available_aspect_ratios] # Only write config in the first launch. diff --git a/modules/flags.py b/modules/flags.py index 7b3ac3933..89e1ea0f2 100644 --- a/modules/flags.py +++ b/modules/flags.py @@ -81,6 +81,13 @@ desc_type_photo = 'Photograph' desc_type_anime = 'Art/Anime' +sdxl_aspect_ratios = [ + '704*1408', '704*1344', '768*1344', '768*1280', '832*1216', '832*1152', + '896*1152', '896*1088', '960*1088', '960*1024', '1024*1024', '1024*960', + '1088*960', '1088*896', '1152*896', '1152*832', '1216*832', '1280*768', + '1344*768', '1344*704', '1408*704', '1472*704', '1536*640', '1600*640', + '1664*576', '1728*576' +] class MetadataScheme(Enum): FOOOCUS = 'fooocus' diff --git a/modules/meta_parser.py b/modules/meta_parser.py index 2469da5f9..4ce12435c 100644 --- a/modules/meta_parser.py +++ b/modules/meta_parser.py @@ -124,7 +124,7 @@ def get_resolution(key: str, fallback: str | None, source_dict: dict, results: l h = source_dict.get(key, source_dict.get(fallback, default)) width, height = eval(h) formatted = modules.config.add_ratio(f'{width}*{height}') - if formatted in modules.config.available_aspect_ratios: + if formatted in modules.config.available_aspect_ratios_labels: results.append(formatted) results.append(-1) results.append(-1) diff --git a/modules/util.py b/modules/util.py index cb5580fbb..4f975bf5c 100644 --- a/modules/util.py +++ b/modules/util.py @@ -381,6 +381,16 @@ def get_file_from_folder_list(name, folders): return os.path.abspath(os.path.realpath(os.path.join(folders[0], name))) +def ordinal_suffix(number: int) -> str: + return 'th' if 10 <= number % 100 <= 20 else {1: 'st', 2: 'nd', 3: 'rd'}.get(number % 10, 'th') + + +def makedirs_with_log(path): + try: + os.makedirs(path, exist_ok=True) + except OSError as error: + print(f'Directory {path} could not be created, reason: {error}') + def get_enabled_loras(loras: list, remove_none=True) -> list: return [(lora[1], lora[2]) for lora in loras if lora[0] and (lora[1] != 'None' if remove_none else True)] @@ -467,3 +477,25 @@ def apply_wildcards(wildcard_text, rng, i, read_wildcards_in_order) -> str: print(f'[Wildcards] BFS stack overflow. Current text: {wildcard_text}') return wildcard_text + + +def get_image_size_info(image: np.ndarray, aspect_ratios: list) -> str: + try: + image = Image.fromarray(np.uint8(image)) + width, height = image.size + ratio = round(width / height, 2) + gcd = math.gcd(width, height) + lcm_ratio = f'{width // gcd}:{height // gcd}' + size_info = f'Image Size: {width} x {height}, Ratio: {ratio}, {lcm_ratio}' + + closest_ratio = min(aspect_ratios, key=lambda x: abs(ratio - float(x.split('*')[0]) / float(x.split('*')[1]))) + recommended_width, recommended_height = map(int, closest_ratio.split('*')) + recommended_ratio = round(recommended_width / recommended_height, 2) + recommended_gcd = math.gcd(recommended_width, recommended_height) + recommended_lcm_ratio = f'{recommended_width // recommended_gcd}:{recommended_height // recommended_gcd}' + + size_info += f'\nRecommended Size: {recommended_width} x {recommended_height}, Ratio: {recommended_ratio}, {recommended_lcm_ratio}' + + return size_info + except Exception as e: + return f'Error reading image: {e}' diff --git a/webui.py b/webui.py index 55f3102c2..7606e0103 100644 --- a/webui.py +++ b/webui.py @@ -221,7 +221,16 @@ def ip_advance_checked(x): choices=[flags.desc_type_photo, flags.desc_type_anime], value=flags.desc_type_photo) desc_btn = gr.Button(value='Describe this Image into Prompt') + desc_image_size = gr.Markdown(label='Image Size', elem_id='desc_image_size', visible=False) gr.HTML('\U0001F4D4 Document') + + def trigger_show_image_properties(image): + value = modules.util.get_image_size_info(image, modules.flags.sdxl_aspect_ratios) + return gr.update(value=value, visible=True) + + desc_input_image.upload(trigger_show_image_properties, inputs=desc_input_image, + outputs=desc_image_size, show_progress=False, queue=False) + with gr.TabItem(label='Metadata') as load_tab: with gr.Column(): metadata_input_image = grh.Image(label='Drag any image generated by Fooocus here', source='upload', type='filepath') @@ -266,7 +275,7 @@ def trigger_metadata_preview(filepath): performance_selection = gr.Radio(label='Performance', choices=flags.Performance.list(), value=modules.config.default_performance) - aspect_ratios_selection = gr.Radio(label='Aspect Ratios', choices=modules.config.available_aspect_ratios, + aspect_ratios_selection = gr.Radio(label='Aspect Ratios', choices=modules.config.available_aspect_ratios_labels, value=modules.config.default_aspect_ratio, info='width × height', elem_classes='aspect_ratios') image_number = gr.Slider(label='Image Number', minimum=1, maximum=modules.config.default_max_image_number, step=1, value=modules.config.default_image_number) From 4da5a68c1015496c23c59d23d41e81e443ce1603 Mon Sep 17 00:00:00 2001 From: xyny <60004820+xynydev@users.noreply.github.com> Date: Wed, 22 May 2024 22:19:54 +0000 Subject: [PATCH 27/42] feat: build and push container image for ghcr.io, update docker.md, and other related fixes (#2805) * chore: update cuda version in container * fix: use symlink to fix error libcuda.so: cannot open shared object file: * fix: update docker entrypoint to use entry_with_update.py * feat: add container build & push workflow * fix: container action run conditions * fix: container action versions * fix: container action versions v2 * fix: docker action registry login and metadata * docs: adjust docker documentation based on latest changes, add docs for podman and docker * chore: replace image name env var with github.event.repository.name Co-authored-by: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> * chore: replace image name env var with github.event.repository.name (pt2) Co-authored-by: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> * fix: switch to semver versioning Co-authored-by: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> * fix: build only on versioned tags Co-authored-by: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> * fix: don't update in entrypoint Co-authored-by: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> * fix: remove dash in "docker-compose" Co-authored-by: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> * feat: sync pytorch for docker with version used in prepare_environment * feat: update cuda to 12.4.1 * fix: correctly clone checked out version in builds, not always main * refactor: remove irrelevant version in docker-compose.yml --------- Co-authored-by: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Co-authored-by: Manuel Schmid --- .dockerignore | 55 +++++++++++++++++- .github/dependabot.yml | 6 ++ .github/workflows/build_container.yml | 44 ++++++++++++++ Dockerfile | 4 +- docker-compose.yml | 4 +- docker.md | 82 ++++++++++++++++++++++++--- requirements_docker.txt | 7 +-- 7 files changed, 182 insertions(+), 20 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/build_container.yml diff --git a/.dockerignore b/.dockerignore index 485dee64b..d1eab8076 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,54 @@ -.idea +__pycache__ +*.ckpt +*.safetensors +*.pth +*.pt +*.bin +*.patch +*.backup +*.corrupted +*.partial +*.onnx +sorted_styles.json +/input +/cache +/language/default.json +/test_imgs +config.txt +config_modification_tutorial.txt +user_path_config.txt +user_path_config-deprecated.txt +/modules/*.png +/repositories +/fooocus_env +/venv +/tmp +/ui-config.json +/outputs +/config.json +/log +/webui.settings.bat +/embeddings +/styles.csv +/params.txt +/styles.csv.bak +/webui-user.bat +/webui-user.sh +/interrogate +/user.css +/.idea +/notification.ogg +/notification.mp3 +/SwinIR +/textual_inversion +.vscode +/extensions +/test/stdout.txt +/test/stderr.txt +/cache.json* +/config_states/ +/node_modules +/package-lock.json +/.coverage* +/auth.json +.DS_Store \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..adee0ed14 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" \ No newline at end of file diff --git a/.github/workflows/build_container.yml b/.github/workflows/build_container.yml new file mode 100644 index 000000000..1e118a1ff --- /dev/null +++ b/.github/workflows/build_container.yml @@ -0,0 +1,44 @@ +name: Create and publish a container image + +on: + push: + tags: + - 'v*' + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index b969cd0e5..1172c795a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM nvidia/cuda:12.3.1-base-ubuntu22.04 +FROM nvidia/cuda:12.4.1-base-ubuntu22.04 ENV DEBIAN_FRONTEND noninteractive ENV CMDARGS --listen @@ -23,7 +23,7 @@ RUN chown -R user:user /content WORKDIR /content USER user -RUN git clone https://github.com/lllyasviel/Fooocus /content/app +COPY . /content/app RUN mv /content/app/models /content/app/models.org CMD [ "sh", "-c", "/content/entrypoint.sh ${CMDARGS}" ] diff --git a/docker-compose.yml b/docker-compose.yml index dee7b3e7c..f724964d2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,12 +1,10 @@ -version: '3.9' - volumes: fooocus-data: services: app: build: . - image: fooocus + image: ghcr.io/lllyasviel/fooocus ports: - "7865:7865" environment: diff --git a/docker.md b/docker.md index 1939d6fca..cd75d9f5a 100644 --- a/docker.md +++ b/docker.md @@ -1,35 +1,99 @@ # Fooocus on Docker -The docker image is based on NVIDIA CUDA 12.3 and PyTorch 2.0, see [Dockerfile](Dockerfile) and [requirements_docker.txt](requirements_docker.txt) for details. +The docker image is based on NVIDIA CUDA 12.4 and PyTorch 2.1, see [Dockerfile](Dockerfile) and [requirements_docker.txt](requirements_docker.txt) for details. + +## Requirements + +- A computer with specs good enough to run Fooocus, and proprietary Nvidia drivers +- Docker, Docker Compose, or Podman ## Quick start -**This is just an easy way for testing. Please find more information in the [notes](#notes).** +**More information in the [notes](#notes).** + +### Running with Docker Compose 1. Clone this repository -2. Build the image with `docker compose build` -3. Run the docker container with `docker compose up`. Building the image takes some time. +2. Run the docker container with `docker compose up`. + +### Running with Docker + +```sh +docker run -p 7865:7865 -v fooocus-data:/content/data -it \ +--gpus all \ +-e CMDARGS=--listen \ +-e DATADIR=/content/data \ +-e config_path=/content/data/config.txt \ +-e config_example_path=/content/data/config_modification_tutorial.txt \ +-e path_checkpoints=/content/data/models/checkpoints/ \ +-e path_loras=/content/data/models/loras/ \ +-e path_embeddings=/content/data/models/embeddings/ \ +-e path_vae_approx=/content/data/models/vae_approx/ \ +-e path_upscale_models=/content/data/models/upscale_models/ \ +-e path_inpaint=/content/data/models/inpaint/ \ +-e path_controlnet=/content/data/models/controlnet/ \ +-e path_clip_vision=/content/data/models/clip_vision/ \ +-e path_fooocus_expansion=/content/data/models/prompt_expansion/fooocus_expansion/ \ +-e path_outputs=/content/app/outputs/ \ +ghcr.io/lllyasviel/fooocus +``` +### Running with Podman + +```sh +podman run -p 7865:7865 -v fooocus-data:/content/data -it \ +--security-opt=no-new-privileges --cap-drop=ALL --security-opt label=type:nvidia_container_t --device=nvidia.com/gpu=all \ +-e CMDARGS=--listen \ +-e DATADIR=/content/data \ +-e config_path=/content/data/config.txt \ +-e config_example_path=/content/data/config_modification_tutorial.txt \ +-e path_checkpoints=/content/data/models/checkpoints/ \ +-e path_loras=/content/data/models/loras/ \ +-e path_embeddings=/content/data/models/embeddings/ \ +-e path_vae_approx=/content/data/models/vae_approx/ \ +-e path_upscale_models=/content/data/models/upscale_models/ \ +-e path_inpaint=/content/data/models/inpaint/ \ +-e path_controlnet=/content/data/models/controlnet/ \ +-e path_clip_vision=/content/data/models/clip_vision/ \ +-e path_fooocus_expansion=/content/data/models/prompt_expansion/fooocus_expansion/ \ +-e path_outputs=/content/app/outputs/ \ +ghcr.io/lllyasviel/fooocus +``` When you see the message `Use the app with http://0.0.0.0:7865/` in the console, you can access the URL in your browser. -Your models and outputs are stored in the `fooocus-data` volume, which, depending on OS, is stored in `/var/lib/docker/volumes`. +Your models and outputs are stored in the `fooocus-data` volume, which, depending on OS, is stored in `/var/lib/docker/volumes/` (or `~/.local/share/containers/storage/volumes/` when using `podman`). + +## Building the container locally + +Clone the repository first, and open a terminal in the folder. + +Build with `docker`: +```sh +docker build . -t fooocus +``` + +Build with `podman`: +```sh +podman build . -t fooocus +``` ## Details -### Update the container manually +### Update the container manually (`docker compose`) When you are using `docker compose up` continuously, the container is not updated to the latest version of Fooocus automatically. Run `git pull` before executing `docker compose build --no-cache` to build an image with the latest Fooocus version. You can then start it with `docker compose up` ### Import models, outputs -If you want to import files from models or the outputs folder, you can uncomment the following settings in the [docker-compose.yml](docker-compose.yml): + +If you want to import files from models or the outputs folder, you can add the following bind mounts in the [docker-compose.yml](docker-compose.yml) or your preferred method of running the container: ``` #- ./models:/import/models # Once you import files, you don't need to mount again. #- ./outputs:/import/outputs # Once you import files, you don't need to mount again. ``` -After running `docker compose up`, your files will be copied into `/content/data/models` and `/content/data/outputs` -Since `/content/data` is a persistent volume folder, your files will be persisted even when you re-run `docker compose up --build` without above volume settings. +After running the container, your files will be copied into `/content/data/models` and `/content/data/outputs` +Since `/content/data` is a persistent volume folder, your files will be persisted even when you re-run the container without the above mounts. ### Paths inside the container diff --git a/requirements_docker.txt b/requirements_docker.txt index 3cf4aa89d..21883adfd 100644 --- a/requirements_docker.txt +++ b/requirements_docker.txt @@ -1,5 +1,2 @@ -torch==2.0.1 -torchvision==0.15.2 -torchaudio==2.0.2 -torchtext==0.15.2 -torchdata==0.6.1 +torch==2.1.0 +torchvision==0.16.0 From 7b70d270325320e8107837f37739d403d4415915 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Fri, 24 May 2024 21:36:07 +0200 Subject: [PATCH 28/42] feat: configure line ending format LF for *.sh files (#2991) --- .gitattributes | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..ce213ceb0 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Ensure that shell scripts always use lf line endings, e.g. entrypoint.sh for docker +* text=auto +*.sh text eol=lf \ No newline at end of file From 04f64ab0bcddb70e6f60a3a463bbbb59bd320216 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Fri, 24 May 2024 21:58:17 +0200 Subject: [PATCH 29/42] feat: add translation for image size describe (#2992) --- language/en.json | 14 ++++++++++++-- modules/util.py | 3 ++- webui.py | 2 +- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/language/en.json b/language/en.json index 3eb5d5e25..33a70b7ba 100644 --- a/language/en.json +++ b/language/en.json @@ -9,9 +9,19 @@ "Advanced": "Advanced", "Upscale or Variation": "Upscale or Variation", "Image Prompt": "Image Prompt", - "Inpaint or Outpaint (beta)": "Inpaint or Outpaint (beta)", - "Drag above image to here": "Drag above image to here", + "Inpaint or Outpaint": "Inpaint or Outpaint", + "Drag inpaint or outpaint image to here": "Drag inpaint or outpaint image to here", + "Outpaint Direction": "Outpaint Direction", + "Method": "Method", + "Describe": "Describe", + "Drag any image to here": "Drag any image to here", + "Content Type": "Content Type", + "Photograph": "Photograph", + "Art/Anime": "Art/Anime", + "Describe this Image into Prompt": "Describe this Image into Prompt", + "Image Size and Recommended Size": "Image Size and Recommended Size", "Upscale or Variation:": "Upscale or Variation:", + "Drag above image to here": "Drag above image to here", "Disabled": "Disabled", "Vary (Subtle)": "Vary (Subtle)", "Vary (Strong)": "Vary (Strong)", diff --git a/modules/util.py b/modules/util.py index 4f975bf5c..8317dd504 100644 --- a/modules/util.py +++ b/modules/util.py @@ -494,7 +494,8 @@ def get_image_size_info(image: np.ndarray, aspect_ratios: list) -> str: recommended_gcd = math.gcd(recommended_width, recommended_height) recommended_lcm_ratio = f'{recommended_width // recommended_gcd}:{recommended_height // recommended_gcd}' - size_info += f'\nRecommended Size: {recommended_width} x {recommended_height}, Ratio: {recommended_ratio}, {recommended_lcm_ratio}' + size_info = f'{width} x {height}, {ratio}, {lcm_ratio}' + size_info += f'\n{recommended_width} x {recommended_height}, {recommended_ratio}, {recommended_lcm_ratio}' return size_info except Exception as e: diff --git a/webui.py b/webui.py index 7606e0103..25e57222a 100644 --- a/webui.py +++ b/webui.py @@ -221,7 +221,7 @@ def ip_advance_checked(x): choices=[flags.desc_type_photo, flags.desc_type_anime], value=flags.desc_type_photo) desc_btn = gr.Button(value='Describe this Image into Prompt') - desc_image_size = gr.Markdown(label='Image Size', elem_id='desc_image_size', visible=False) + desc_image_size = gr.Textbox(label='Image Size and Recommended Size', elem_id='desc_image_size', visible=False) gr.HTML('\U0001F4D4 Document') def trigger_show_image_properties(image): From d850bca09fc8a1bac6635980035862ded4dd4b18 Mon Sep 17 00:00:00 2001 From: Alexdnk <83111151+Alexdnk@users.noreply.github.com> Date: Sat, 25 May 2024 03:05:28 +0700 Subject: [PATCH 30/42] feat: read value 'CFG Mimicking from TSNR' (adaptive_cfg) from presets (#2990) --- modules/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/config.py b/modules/config.py index 64b0b86ff..08ed99d7f 100644 --- a/modules/config.py +++ b/modules/config.py @@ -487,6 +487,7 @@ def init_temp_path(path: str | None, default_path: str) -> str: "default_loras": "", "default_cfg_scale": "guidance_scale", "default_sample_sharpness": "sharpness", + "default_cfg_tsnr": "adaptive_cfg", "default_sampler": "sampler", "default_scheduler": "scheduler", "default_overwrite_step": "steps", From 1d1a4a3ebd2ea06fa396be670272ee9659e5f66c Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Sun, 26 May 2024 11:40:15 +0200 Subject: [PATCH 31/42] feat: add inpaint color picker (#2997) Workaround as tool color-sketch applies changes directly to the image canvas and not the mask canvas. Color picker is not correctly implemented in Gradio 3.41.2 => does always get displayed as separate containers and not merged with other elements --- css/style.css | 6 +++++- webui.py | 11 +++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/css/style.css b/css/style.css index b5f7a4488..b82cf930e 100644 --- a/css/style.css +++ b/css/style.css @@ -72,7 +72,7 @@ progress::after { .progress-bar span { text-align: right; - width: 200px; + width: 215px; } .type_row{ @@ -399,4 +399,8 @@ progress::after { text-align: center; border-radius: 5px 5px 0px 0px; display: none; /* remove this to enable tooltip in preview image */ +} + +#inpaint_brush_color input[type=color]{ + background: none; } \ No newline at end of file diff --git a/webui.py b/webui.py index 25e57222a..ae0bc89f7 100644 --- a/webui.py +++ b/webui.py @@ -524,13 +524,20 @@ def update_history_link(): inpaint_mask_upload_checkbox = gr.Checkbox(label='Enable Mask Upload', value=False) invert_mask_checkbox = gr.Checkbox(label='Invert Mask', value=False) + inpaint_mask_color = gr.ColorPicker(label='Inpaint brush color', value='#FFFFFF', elem_id='inpaint_brush_color') + inpaint_ctrls = [debugging_inpaint_preprocessor, inpaint_disable_initial_latent, inpaint_engine, inpaint_strength, inpaint_respective_field, inpaint_mask_upload_checkbox, invert_mask_checkbox, inpaint_erode_or_dilate] inpaint_mask_upload_checkbox.change(lambda x: gr.update(visible=x), - inputs=inpaint_mask_upload_checkbox, - outputs=inpaint_mask_image, queue=False, show_progress=False) + inputs=inpaint_mask_upload_checkbox, + outputs=inpaint_mask_image, queue=False, + show_progress=False) + + inpaint_mask_color.change(lambda x: gr.update(brush_color=x), inputs=inpaint_mask_color, + outputs=inpaint_input_image, + queue=False, show_progress=False) with gr.Tab(label='FreeU'): freeu_enabled = gr.Checkbox(label='Enabled', value=False) From 4e5509351f3882431e6088cdc7ec3632534df2e4 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Sun, 26 May 2024 11:47:33 +0200 Subject: [PATCH 32/42] feat: remove labels from most of the image input fields (#2998) --- css/style.css | 4 ++++ language/en.json | 5 +---- webui.py | 8 ++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/css/style.css b/css/style.css index b82cf930e..649f77c5d 100644 --- a/css/style.css +++ b/css/style.css @@ -401,6 +401,10 @@ progress::after { display: none; /* remove this to enable tooltip in preview image */ } +#inpaint_canvas .canvas-tooltip-info { + top: 2px; +} + #inpaint_brush_color input[type=color]{ background: none; } \ No newline at end of file diff --git a/language/en.json b/language/en.json index 33a70b7ba..90eaf2eee 100644 --- a/language/en.json +++ b/language/en.json @@ -10,18 +10,15 @@ "Upscale or Variation": "Upscale or Variation", "Image Prompt": "Image Prompt", "Inpaint or Outpaint": "Inpaint or Outpaint", - "Drag inpaint or outpaint image to here": "Drag inpaint or outpaint image to here", "Outpaint Direction": "Outpaint Direction", "Method": "Method", "Describe": "Describe", - "Drag any image to here": "Drag any image to here", "Content Type": "Content Type", "Photograph": "Photograph", "Art/Anime": "Art/Anime", "Describe this Image into Prompt": "Describe this Image into Prompt", "Image Size and Recommended Size": "Image Size and Recommended Size", "Upscale or Variation:": "Upscale or Variation:", - "Drag above image to here": "Drag above image to here", "Disabled": "Disabled", "Vary (Subtle)": "Vary (Subtle)", "Vary (Strong)": "Vary (Strong)", @@ -394,7 +391,7 @@ "Fooocus Enhance": "Fooocus Enhance", "Fooocus Cinematic": "Fooocus Cinematic", "Fooocus Sharp": "Fooocus Sharp", - "Drag any image generated by Fooocus here": "Drag any image generated by Fooocus here", + "For images created by Fooocus": "For images created by Fooocus", "Metadata": "Metadata", "Apply Metadata": "Apply Metadata", "Metadata Scheme": "Metadata Scheme", diff --git a/webui.py b/webui.py index ae0bc89f7..b475cd90c 100644 --- a/webui.py +++ b/webui.py @@ -152,7 +152,7 @@ def skip_clicked(currentTask): with gr.TabItem(label='Upscale or Variation') as uov_tab: with gr.Row(): with gr.Column(): - uov_input_image = grh.Image(label='Drag above image to here', source='upload', type='numpy') + uov_input_image = grh.Image(label='Image', source='upload', type='numpy', show_label=False) with gr.Column(): uov_method = gr.Radio(label='Upscale or Variation:', choices=flags.uov_list, value=flags.disabled) gr.HTML('\U0001F4D4 Document') @@ -201,7 +201,7 @@ def ip_advance_checked(x): queue=False, show_progress=False) with gr.TabItem(label='Inpaint or Outpaint') as inpaint_tab: with gr.Row(): - inpaint_input_image = grh.Image(label='Drag inpaint or outpaint image to here', source='upload', type='numpy', tool='sketch', height=500, brush_color="#FFFFFF", elem_id='inpaint_canvas') + inpaint_input_image = grh.Image(label='Image', source='upload', type='numpy', tool='sketch', height=500, brush_color="#FFFFFF", elem_id='inpaint_canvas', show_label=False) inpaint_mask_image = grh.Image(label='Mask Upload', source='upload', type='numpy', height=500, visible=False) with gr.Row(): @@ -214,7 +214,7 @@ def ip_advance_checked(x): with gr.TabItem(label='Describe') as desc_tab: with gr.Row(): with gr.Column(): - desc_input_image = grh.Image(label='Drag any image to here', source='upload', type='numpy') + desc_input_image = grh.Image(label='Image', source='upload', type='numpy', show_label=False) with gr.Column(): desc_method = gr.Radio( label='Content Type', @@ -233,7 +233,7 @@ def trigger_show_image_properties(image): with gr.TabItem(label='Metadata') as load_tab: with gr.Column(): - metadata_input_image = grh.Image(label='Drag any image generated by Fooocus here', source='upload', type='filepath') + metadata_input_image = grh.Image(label='For images created by Fooocus', source='upload', type='filepath') metadata_json = gr.JSON(label='Metadata') metadata_import_button = gr.Button(value='Apply Metadata') From cc58fe52706a5a9ec75ad12f9643e19fe170e253 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Sun, 26 May 2024 14:18:19 +0200 Subject: [PATCH 33/42] feat: add clip skip handling (#2999) --- language/en.json | 1 + modules/async_worker.py | 6 ++++++ modules/config.py | 6 ++++++ modules/default_pipeline.py | 11 +++++++++++ modules/meta_parser.py | 18 ++++++++++-------- webui.py | 11 +++++++---- 6 files changed, 41 insertions(+), 12 deletions(-) diff --git a/language/en.json b/language/en.json index 90eaf2eee..a4056e1ef 100644 --- a/language/en.json +++ b/language/en.json @@ -320,6 +320,7 @@ "vae": "vae", "CFG Mimicking from TSNR": "CFG Mimicking from TSNR", "Enabling Fooocus's implementation of CFG mimicking for TSNR (effective when real CFG > mimicked CFG).": "Enabling Fooocus's implementation of CFG mimicking for TSNR (effective when real CFG > mimicked CFG).", + "CLIP Skip": "CLIP Skip", "Sampler": "Sampler", "dpmpp_2m_sde_gpu": "dpmpp_2m_sde_gpu", "Only effective in non-inpaint mode.": "Only effective in non-inpaint mode.", diff --git a/modules/async_worker.py b/modules/async_worker.py index 594886d28..d7d9b9fd7 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -174,6 +174,7 @@ def handler(async_task): adm_scaler_negative = args.pop() adm_scaler_end = args.pop() adaptive_cfg = args.pop() + clip_skip = args.pop() sampler_name = args.pop() scheduler_name = args.pop() vae_name = args.pop() @@ -297,6 +298,7 @@ def handler(async_task): adm_scaler_end = 0.0 print(f'[Parameters] Adaptive CFG = {adaptive_cfg}') + print(f'[Parameters] CLIP Skip = {clip_skip}') print(f'[Parameters] Sharpness = {sharpness}') print(f'[Parameters] ControlNet Softness = {controlnet_softness}') print(f'[Parameters] ADM Scale = ' @@ -466,6 +468,8 @@ def handler(async_task): loras=loras, base_model_additional_loras=base_model_additional_loras, use_synthetic_refiner=use_synthetic_refiner, vae_name=vae_name) + pipeline.set_clip_skip(clip_skip) + progressbar(async_task, 3, 'Processing prompts ...') tasks = [] @@ -924,6 +928,8 @@ def callback(step, x0, x, total_steps, y): d.append( ('CFG Mimicking from TSNR', 'adaptive_cfg', modules.patch.patch_settings[pid].adaptive_cfg)) + if clip_skip > 1: + d.append(('CLIP Skip', 'clip_skip', clip_skip)) d.append(('Sampler', 'sampler', sampler_name)) d.append(('Scheduler', 'scheduler', scheduler_name)) d.append(('VAE', 'vae', vae_name)) diff --git a/modules/config.py b/modules/config.py index 08ed99d7f..0aee27134 100644 --- a/modules/config.py +++ b/modules/config.py @@ -434,6 +434,11 @@ def init_temp_path(path: str | None, default_path: str) -> str: default_value=7.0, validator=lambda x: isinstance(x, numbers.Number) ) +default_clip_skip = get_config_item_or_set_default( + key='default_clip_skip', + default_value=1, + validator=lambda x: isinstance(x, numbers.Number) +) default_overwrite_step = get_config_item_or_set_default( key='default_overwrite_step', default_value=-1, @@ -488,6 +493,7 @@ def init_temp_path(path: str | None, default_path: str) -> str: "default_cfg_scale": "guidance_scale", "default_sample_sharpness": "sharpness", "default_cfg_tsnr": "adaptive_cfg", + "default_clip_skip": "clip_skip", "default_sampler": "sampler", "default_scheduler": "scheduler", "default_overwrite_step": "steps", diff --git a/modules/default_pipeline.py b/modules/default_pipeline.py index 38f914c57..494644d69 100644 --- a/modules/default_pipeline.py +++ b/modules/default_pipeline.py @@ -201,6 +201,17 @@ def clip_encode(texts, pool_top_k=1): return [[torch.cat(cond_list, dim=1), {"pooled_output": pooled_acc}]] +@torch.no_grad() +@torch.inference_mode() +def set_clip_skip(clip_skip: int): + global final_clip + + if final_clip is None: + return + + final_clip.clip_layer(-abs(clip_skip)) + return + @torch.no_grad() @torch.inference_mode() def clear_all_caches(): diff --git a/modules/meta_parser.py b/modules/meta_parser.py index 4ce12435c..586e62da2 100644 --- a/modules/meta_parser.py +++ b/modules/meta_parser.py @@ -34,16 +34,17 @@ def load_parameter_button_click(raw_metadata: dict | str, is_generating: bool): get_list('styles', 'Styles', loaded_parameter_dict, results) get_str('performance', 'Performance', loaded_parameter_dict, results) get_steps('steps', 'Steps', loaded_parameter_dict, results) - get_float('overwrite_switch', 'Overwrite Switch', loaded_parameter_dict, results) + get_number('overwrite_switch', 'Overwrite Switch', loaded_parameter_dict, results) get_resolution('resolution', 'Resolution', loaded_parameter_dict, results) - get_float('guidance_scale', 'Guidance Scale', loaded_parameter_dict, results) - get_float('sharpness', 'Sharpness', loaded_parameter_dict, results) + get_number('guidance_scale', 'Guidance Scale', loaded_parameter_dict, results) + get_number('sharpness', 'Sharpness', loaded_parameter_dict, results) get_adm_guidance('adm_guidance', 'ADM Guidance', loaded_parameter_dict, results) get_str('refiner_swap_method', 'Refiner Swap Method', loaded_parameter_dict, results) - get_float('adaptive_cfg', 'CFG Mimicking from TSNR', loaded_parameter_dict, results) + get_number('adaptive_cfg', 'CFG Mimicking from TSNR', loaded_parameter_dict, results) + get_number('clip_skip', 'CLIP Skip', loaded_parameter_dict, results, cast_type=int) get_str('base_model', 'Base Model', loaded_parameter_dict, results) get_str('refiner_model', 'Refiner Model', loaded_parameter_dict, results) - get_float('refiner_switch', 'Refiner Switch', loaded_parameter_dict, results) + get_number('refiner_switch', 'Refiner Switch', loaded_parameter_dict, results) get_str('sampler', 'Sampler', loaded_parameter_dict, results) get_str('scheduler', 'Scheduler', loaded_parameter_dict, results) get_str('vae', 'VAE', loaded_parameter_dict, results) @@ -83,11 +84,11 @@ def get_list(key: str, fallback: str | None, source_dict: dict, results: list, d results.append(gr.update()) -def get_float(key: str, fallback: str | None, source_dict: dict, results: list, default=None): +def get_number(key: str, fallback: str | None, source_dict: dict, results: list, default=None, cast_type=float): try: h = source_dict.get(key, source_dict.get(fallback, default)) assert h is not None - h = float(h) + h = cast_type(h) results.append(h) except: results.append(gr.update()) @@ -314,6 +315,7 @@ def get_scheme(self) -> MetadataScheme: 'adm_guidance': 'ADM Guidance', 'refiner_swap_method': 'Refiner Swap Method', 'adaptive_cfg': 'Adaptive CFG', + 'clip_skip': 'Clip skip', 'overwrite_switch': 'Overwrite Switch', 'freeu': 'FreeU', 'base_model': 'Model', @@ -458,7 +460,7 @@ def parse_string(self, metadata: dict) -> str: self.fooocus_to_a1111['refiner_model_hash']: self.refiner_model_hash } - for key in ['adaptive_cfg', 'overwrite_switch', 'refiner_swap_method', 'freeu']: + for key in ['adaptive_cfg', 'clip_skip', 'overwrite_switch', 'refiner_swap_method', 'freeu']: if key in data: generation_params[self.fooocus_to_a1111[key]] = data[key] diff --git a/webui.py b/webui.py index b475cd90c..d72eb2ecd 100644 --- a/webui.py +++ b/webui.py @@ -412,6 +412,9 @@ def update_history_link(): value=modules.config.default_cfg_tsnr, info='Enabling Fooocus\'s implementation of CFG mimicking for TSNR ' '(effective when real CFG > mimicked CFG).') + clip_skip = gr.Slider(label='CLIP Skip', minimum=1, maximum=10, step=1, + value=modules.config.default_clip_skip, + info='Bypass CLIP layers to avoid overfitting (use 1 to disable).') sampler_name = gr.Dropdown(label='Sampler', choices=flags.sampler_list, value=modules.config.default_sampler) scheduler_name = gr.Dropdown(label='Scheduler', choices=flags.scheduler_list, @@ -576,9 +579,9 @@ def refresh_files_clicked(): load_data_outputs = [advanced_checkbox, image_number, prompt, negative_prompt, style_selections, performance_selection, overwrite_step, overwrite_switch, aspect_ratios_selection, overwrite_width, overwrite_height, guidance_scale, sharpness, adm_scaler_positive, - adm_scaler_negative, adm_scaler_end, refiner_swap_method, adaptive_cfg, base_model, - refiner_model, refiner_switch, sampler_name, scheduler_name, vae_name, seed_random, - image_seed, generate_button, load_parameter_button] + freeu_ctrls + lora_ctrls + adm_scaler_negative, adm_scaler_end, refiner_swap_method, adaptive_cfg, clip_skip, + base_model, refiner_model, refiner_switch, sampler_name, scheduler_name, vae_name, + seed_random, image_seed, generate_button, load_parameter_button] + freeu_ctrls + lora_ctrls if not args_manager.args.disable_preset_selection: def preset_selection_change(preset, is_generating): @@ -663,7 +666,7 @@ def inpaint_mode_change(mode): ctrls += [uov_method, uov_input_image] ctrls += [outpaint_selections, inpaint_input_image, inpaint_additional_prompt, inpaint_mask_image] ctrls += [disable_preview, disable_intermediate_results, disable_seed_increment, black_out_nsfw] - ctrls += [adm_scaler_positive, adm_scaler_negative, adm_scaler_end, adaptive_cfg] + ctrls += [adm_scaler_positive, adm_scaler_negative, adm_scaler_end, adaptive_cfg, clip_skip] ctrls += [sampler_name, scheduler_name, vae_name] ctrls += [overwrite_step, overwrite_switch, overwrite_width, overwrite_height, overwrite_vary_strength] ctrls += [overwrite_upscale_strength, mixing_image_prompt_and_vary_upscale, mixing_image_prompt_and_inpaint] From 67289dd0fef248650cefd87729347032c443e0cc Mon Sep 17 00:00:00 2001 From: Manuel Schmid Date: Sun, 26 May 2024 15:11:40 +0200 Subject: [PATCH 34/42] release: bump version to 2.4.0, update changelog --- fooocus_version.py | 2 +- update_log.md | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/fooocus_version.py b/fooocus_version.py index 41556f902..ecc158079 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.4.0-rc2' +version = '2.4.0' diff --git a/update_log.md b/update_log.md index 62c4882bc..e9544da3c 100644 --- a/update_log.md +++ b/update_log.md @@ -1,3 +1,22 @@ +# [2.4.0](https://github.com/lllyasviel/Fooocus/releases/tag/v2.4.0) + +* Add clip skip slider +* Add select for custom VAE +* Add new style "Random Style" +* Update default anime model to animaPencilXL_v310 +* Add button to reconnect the UI after Fooocus crashed without having to configure everything again (no page reload required) +* Add performance "hyper-sd" (based on [Hyper-SDXL 4 step LoRA](https://huggingface.co/ByteDance/Hyper-SD/blob/main/Hyper-SDXL-4steps-lora.safetensors)) +* Add [AlignYourSteps](https://research.nvidia.com/labs/toronto-ai/AlignYourSteps/) scheduler by Nvidia, see +* Add [TCD](https://github.com/jabir-zheng/TCD) sampler and scheduler (based on sgm_uniform) +* Add NSFW image censoring (disables intermediate image preview while generating). Set config value `default_black_out_nsfw` to True to always enable. +* Add argument `--enable-describe-uov-image` to automatically describe uploaded images for upscaling +* Add inline lora prompt references with subfolder support, example prompt: `colorful bird ` +* Add size and aspect ratio recommendation on image describe +* Add inpaint brush color picker, helpful when image and mask brush have the same color +* Add automated Docker image build using Github Actions on each release. +* Add full raw prompts to history logs +* Change code ownership from @lllyasviel to @mashb1t for automated issue / MR notification + # [2.3.1](https://github.com/lllyasviel/Fooocus/releases/tag/2.3.1) * Remove positive prompt from anime prefix to not reset prompt after switching presets From 57d2f2a0ddf3eb44e562578283208b496505474c Mon Sep 17 00:00:00 2001 From: Alexdnk <83111151+Alexdnk@users.noreply.github.com> Date: Sun, 26 May 2024 23:10:29 +0700 Subject: [PATCH 35/42] feat: make ui settings more compact (#2590) * Slightly more compact ui settings Changed Radio to Dropdown. * feat: change preset from option to select, add accordion for resolution * feat: change title of aspect ratios accordion on load and update * refactor: reorder image number slider, code cleanup * fix: add missing scroll down for metadata tab * fix: adjust indent --------- Co-authored-by: Manuel Schmid Co-authored-by: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> --- css/style.css | 6 +++++- webui.py | 39 +++++++++++++++++++++++++++------------ 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/css/style.css b/css/style.css index 649f77c5d..18bacaaf6 100644 --- a/css/style.css +++ b/css/style.css @@ -107,10 +107,14 @@ progress::after { overflow: auto !important; } -.aspect_ratios label { +.performance_selection label { width: 140px !important; } +.aspect_ratios label { + flex: calc(50% - 5px) !important; +} + .aspect_ratios label span { white-space: nowrap !important; } diff --git a/webui.py b/webui.py index d72eb2ecd..1d5bec923 100644 --- a/webui.py +++ b/webui.py @@ -231,7 +231,7 @@ def trigger_show_image_properties(image): desc_input_image.upload(trigger_show_image_properties, inputs=desc_input_image, outputs=desc_image_size, show_progress=False, queue=False) - with gr.TabItem(label='Metadata') as load_tab: + with gr.TabItem(label='Metadata') as metadata_tab: with gr.Column(): metadata_input_image = grh.Image(label='For images created by Fooocus', source='upload', type='filepath') metadata_json = gr.JSON(label='Metadata') @@ -264,25 +264,40 @@ def trigger_metadata_preview(filepath): inpaint_tab.select(lambda: 'inpaint', outputs=current_tab, queue=False, _js=down_js, show_progress=False) ip_tab.select(lambda: 'ip', outputs=current_tab, queue=False, _js=down_js, show_progress=False) desc_tab.select(lambda: 'desc', outputs=current_tab, queue=False, _js=down_js, show_progress=False) + metadata_tab.select(lambda: 'metadata', outputs=current_tab, queue=False, _js=down_js, show_progress=False) with gr.Column(scale=1, visible=modules.config.default_advanced_checkbox) as advanced_column: with gr.Tab(label='Setting'): if not args_manager.args.disable_preset_selection: - preset_selection = gr.Radio(label='Preset', - choices=modules.config.available_presets, - value=args_manager.args.preset if args_manager.args.preset else "initial", - interactive=True) + preset_selection = gr.Dropdown(label='Preset', + choices=modules.config.available_presets, + value=args_manager.args.preset if args_manager.args.preset else "initial", + interactive=True) performance_selection = gr.Radio(label='Performance', choices=flags.Performance.list(), - value=modules.config.default_performance) - aspect_ratios_selection = gr.Radio(label='Aspect Ratios', choices=modules.config.available_aspect_ratios_labels, - value=modules.config.default_aspect_ratio, info='width × height', - elem_classes='aspect_ratios') + value=modules.config.default_performance, + elem_classes=['performance_selection']) + with gr.Accordion(label='Aspect Ratios', open=False) as aspect_ratios_accordion: + aspect_ratios_selection = gr.Radio(label='Aspect Ratios', show_label=False, + choices=modules.config.available_aspect_ratios_labels, + value=modules.config.default_aspect_ratio, + info='width × height', + elem_classes='aspect_ratios') + + def change_aspect_ratio(text): + import re + regex = re.compile('<.*?>') + cleaned_text = re.sub(regex, '', text) + return gr.update(label='Aspect Ratios ' + cleaned_text) + + aspect_ratios_selection.change(change_aspect_ratio, inputs=aspect_ratios_selection, outputs=aspect_ratios_accordion, queue=False, show_progress=False) + shared.gradio_root.load(change_aspect_ratio, inputs=aspect_ratios_selection, outputs=aspect_ratios_accordion, queue=False, show_progress=False) + image_number = gr.Slider(label='Image Number', minimum=1, maximum=modules.config.default_max_image_number, step=1, value=modules.config.default_image_number) output_format = gr.Radio(label='Output Format', - choices=flags.OutputFormat.list(), - value=modules.config.default_output_format) + choices=flags.OutputFormat.list(), + value=modules.config.default_output_format) negative_prompt = gr.Textbox(label='Negative Prompt', show_label=True, placeholder="Type prompt here.", info='Describing what you do not want to see.', lines=2, @@ -603,7 +618,7 @@ def preset_selection_change(preset, is_generating): return modules.meta_parser.load_parameter_button_click(json.dumps(preset_prepared), is_generating) preset_selection.change(preset_selection_change, inputs=[preset_selection, state_is_generating], outputs=load_data_outputs, queue=False, show_progress=True) \ - .then(fn=style_sorter.sort_styles, inputs=style_selections, outputs=style_selections, queue=False, show_progress=False) \ + .then(fn=style_sorter.sort_styles, inputs=style_selections, outputs=style_selections, queue=False, show_progress=False) performance_selection.change(lambda x: [gr.update(interactive=not flags.Performance.has_restricted_features(x))] * 11 + [gr.update(visible=not flags.Performance.has_restricted_features(x))] * 1 + From c227cf1f5676381c155f2609e75f9d259cc9ba4e Mon Sep 17 00:00:00 2001 From: Manuel Schmid Date: Sun, 26 May 2024 18:16:18 +0200 Subject: [PATCH 36/42] docs: update changelog --- update_log.md | 1 + 1 file changed, 1 insertion(+) diff --git a/update_log.md b/update_log.md index e9544da3c..77d70cb47 100644 --- a/update_log.md +++ b/update_log.md @@ -1,5 +1,6 @@ # [2.4.0](https://github.com/lllyasviel/Fooocus/releases/tag/v2.4.0) +* Change settings tab elements to be more compact * Add clip skip slider * Add select for custom VAE * Add new style "Random Style" From de34023c797aace9dbc8ddecb439eda84287fba1 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Sun, 26 May 2024 19:23:21 +0200 Subject: [PATCH 37/42] fix: use translation for aspect ratios label (#3001) use javascript code instead of python handling for updates for https://github.com/lllyasviel/Fooocus/pull/2590 --- javascript/localization.js | 6 ++++++ javascript/script.js | 5 +++++ webui.py | 12 +++--------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/javascript/localization.js b/javascript/localization.js index 0a8394ca2..9f13d6fcb 100644 --- a/javascript/localization.js +++ b/javascript/localization.js @@ -80,6 +80,12 @@ function refresh_style_localization() { processNode(document.querySelector('.style_selections')); } +function refresh_aspect_ratios_label(value) { + label = document.querySelector('#aspect_ratios_accordion div span[data-original-text="Aspect Ratios"]') + translation = getTranslation("Aspect Ratios") + label.textContent = translation + " " + htmlDecode(value) +} + function localizeWholePage() { processNode(gradioApp()); diff --git a/javascript/script.js b/javascript/script.js index d379a783f..21dd483d8 100644 --- a/javascript/script.js +++ b/javascript/script.js @@ -256,3 +256,8 @@ function set_theme(theme) { window.location.replace(gradioURL + '?__theme=' + theme); } } + +function htmlDecode(input) { + var doc = new DOMParser().parseFromString(input, "text/html"); + return doc.documentElement.textContent; +} \ No newline at end of file diff --git a/webui.py b/webui.py index 1d5bec923..edc9b6b15 100644 --- a/webui.py +++ b/webui.py @@ -277,21 +277,15 @@ def trigger_metadata_preview(filepath): choices=flags.Performance.list(), value=modules.config.default_performance, elem_classes=['performance_selection']) - with gr.Accordion(label='Aspect Ratios', open=False) as aspect_ratios_accordion: + with gr.Accordion(label='Aspect Ratios', open=False, elem_id='aspect_ratios_accordion') as aspect_ratios_accordion: aspect_ratios_selection = gr.Radio(label='Aspect Ratios', show_label=False, choices=modules.config.available_aspect_ratios_labels, value=modules.config.default_aspect_ratio, info='width × height', elem_classes='aspect_ratios') - def change_aspect_ratio(text): - import re - regex = re.compile('<.*?>') - cleaned_text = re.sub(regex, '', text) - return gr.update(label='Aspect Ratios ' + cleaned_text) - - aspect_ratios_selection.change(change_aspect_ratio, inputs=aspect_ratios_selection, outputs=aspect_ratios_accordion, queue=False, show_progress=False) - shared.gradio_root.load(change_aspect_ratio, inputs=aspect_ratios_selection, outputs=aspect_ratios_accordion, queue=False, show_progress=False) + aspect_ratios_selection.change(lambda x: None, inputs=aspect_ratios_selection, queue=False, show_progress=False, _js='(x)=>{refresh_aspect_ratios_label(x);}') + shared.gradio_root.load(lambda x: None, inputs=aspect_ratios_selection, queue=False, show_progress=False, _js='(x)=>{refresh_aspect_ratios_label(x);}') image_number = gr.Slider(label='Image Number', minimum=1, maximum=modules.config.default_max_image_number, step=1, value=modules.config.default_image_number) From 989a1ad52b209cc0712c610b69988e1bf14eddb6 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Sun, 26 May 2024 22:07:44 +0200 Subject: [PATCH 38/42] Revert "feat: add clip skip handling (#2999)" (#3008) This reverts commit cc58fe52706a5a9ec75ad12f9643e19fe170e253. --- language/en.json | 1 - modules/async_worker.py | 6 ------ modules/config.py | 6 ------ modules/default_pipeline.py | 11 ----------- modules/meta_parser.py | 18 ++++++++---------- webui.py | 11 ++++------- 6 files changed, 12 insertions(+), 41 deletions(-) diff --git a/language/en.json b/language/en.json index a4056e1ef..90eaf2eee 100644 --- a/language/en.json +++ b/language/en.json @@ -320,7 +320,6 @@ "vae": "vae", "CFG Mimicking from TSNR": "CFG Mimicking from TSNR", "Enabling Fooocus's implementation of CFG mimicking for TSNR (effective when real CFG > mimicked CFG).": "Enabling Fooocus's implementation of CFG mimicking for TSNR (effective when real CFG > mimicked CFG).", - "CLIP Skip": "CLIP Skip", "Sampler": "Sampler", "dpmpp_2m_sde_gpu": "dpmpp_2m_sde_gpu", "Only effective in non-inpaint mode.": "Only effective in non-inpaint mode.", diff --git a/modules/async_worker.py b/modules/async_worker.py index d7d9b9fd7..594886d28 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -174,7 +174,6 @@ def handler(async_task): adm_scaler_negative = args.pop() adm_scaler_end = args.pop() adaptive_cfg = args.pop() - clip_skip = args.pop() sampler_name = args.pop() scheduler_name = args.pop() vae_name = args.pop() @@ -298,7 +297,6 @@ def handler(async_task): adm_scaler_end = 0.0 print(f'[Parameters] Adaptive CFG = {adaptive_cfg}') - print(f'[Parameters] CLIP Skip = {clip_skip}') print(f'[Parameters] Sharpness = {sharpness}') print(f'[Parameters] ControlNet Softness = {controlnet_softness}') print(f'[Parameters] ADM Scale = ' @@ -468,8 +466,6 @@ def handler(async_task): loras=loras, base_model_additional_loras=base_model_additional_loras, use_synthetic_refiner=use_synthetic_refiner, vae_name=vae_name) - pipeline.set_clip_skip(clip_skip) - progressbar(async_task, 3, 'Processing prompts ...') tasks = [] @@ -928,8 +924,6 @@ def callback(step, x0, x, total_steps, y): d.append( ('CFG Mimicking from TSNR', 'adaptive_cfg', modules.patch.patch_settings[pid].adaptive_cfg)) - if clip_skip > 1: - d.append(('CLIP Skip', 'clip_skip', clip_skip)) d.append(('Sampler', 'sampler', sampler_name)) d.append(('Scheduler', 'scheduler', scheduler_name)) d.append(('VAE', 'vae', vae_name)) diff --git a/modules/config.py b/modules/config.py index 0aee27134..08ed99d7f 100644 --- a/modules/config.py +++ b/modules/config.py @@ -434,11 +434,6 @@ def init_temp_path(path: str | None, default_path: str) -> str: default_value=7.0, validator=lambda x: isinstance(x, numbers.Number) ) -default_clip_skip = get_config_item_or_set_default( - key='default_clip_skip', - default_value=1, - validator=lambda x: isinstance(x, numbers.Number) -) default_overwrite_step = get_config_item_or_set_default( key='default_overwrite_step', default_value=-1, @@ -493,7 +488,6 @@ def init_temp_path(path: str | None, default_path: str) -> str: "default_cfg_scale": "guidance_scale", "default_sample_sharpness": "sharpness", "default_cfg_tsnr": "adaptive_cfg", - "default_clip_skip": "clip_skip", "default_sampler": "sampler", "default_scheduler": "scheduler", "default_overwrite_step": "steps", diff --git a/modules/default_pipeline.py b/modules/default_pipeline.py index 494644d69..38f914c57 100644 --- a/modules/default_pipeline.py +++ b/modules/default_pipeline.py @@ -201,17 +201,6 @@ def clip_encode(texts, pool_top_k=1): return [[torch.cat(cond_list, dim=1), {"pooled_output": pooled_acc}]] -@torch.no_grad() -@torch.inference_mode() -def set_clip_skip(clip_skip: int): - global final_clip - - if final_clip is None: - return - - final_clip.clip_layer(-abs(clip_skip)) - return - @torch.no_grad() @torch.inference_mode() def clear_all_caches(): diff --git a/modules/meta_parser.py b/modules/meta_parser.py index 586e62da2..4ce12435c 100644 --- a/modules/meta_parser.py +++ b/modules/meta_parser.py @@ -34,17 +34,16 @@ def load_parameter_button_click(raw_metadata: dict | str, is_generating: bool): get_list('styles', 'Styles', loaded_parameter_dict, results) get_str('performance', 'Performance', loaded_parameter_dict, results) get_steps('steps', 'Steps', loaded_parameter_dict, results) - get_number('overwrite_switch', 'Overwrite Switch', loaded_parameter_dict, results) + get_float('overwrite_switch', 'Overwrite Switch', loaded_parameter_dict, results) get_resolution('resolution', 'Resolution', loaded_parameter_dict, results) - get_number('guidance_scale', 'Guidance Scale', loaded_parameter_dict, results) - get_number('sharpness', 'Sharpness', loaded_parameter_dict, results) + get_float('guidance_scale', 'Guidance Scale', loaded_parameter_dict, results) + get_float('sharpness', 'Sharpness', loaded_parameter_dict, results) get_adm_guidance('adm_guidance', 'ADM Guidance', loaded_parameter_dict, results) get_str('refiner_swap_method', 'Refiner Swap Method', loaded_parameter_dict, results) - get_number('adaptive_cfg', 'CFG Mimicking from TSNR', loaded_parameter_dict, results) - get_number('clip_skip', 'CLIP Skip', loaded_parameter_dict, results, cast_type=int) + get_float('adaptive_cfg', 'CFG Mimicking from TSNR', loaded_parameter_dict, results) get_str('base_model', 'Base Model', loaded_parameter_dict, results) get_str('refiner_model', 'Refiner Model', loaded_parameter_dict, results) - get_number('refiner_switch', 'Refiner Switch', loaded_parameter_dict, results) + get_float('refiner_switch', 'Refiner Switch', loaded_parameter_dict, results) get_str('sampler', 'Sampler', loaded_parameter_dict, results) get_str('scheduler', 'Scheduler', loaded_parameter_dict, results) get_str('vae', 'VAE', loaded_parameter_dict, results) @@ -84,11 +83,11 @@ def get_list(key: str, fallback: str | None, source_dict: dict, results: list, d results.append(gr.update()) -def get_number(key: str, fallback: str | None, source_dict: dict, results: list, default=None, cast_type=float): +def get_float(key: str, fallback: str | None, source_dict: dict, results: list, default=None): try: h = source_dict.get(key, source_dict.get(fallback, default)) assert h is not None - h = cast_type(h) + h = float(h) results.append(h) except: results.append(gr.update()) @@ -315,7 +314,6 @@ def get_scheme(self) -> MetadataScheme: 'adm_guidance': 'ADM Guidance', 'refiner_swap_method': 'Refiner Swap Method', 'adaptive_cfg': 'Adaptive CFG', - 'clip_skip': 'Clip skip', 'overwrite_switch': 'Overwrite Switch', 'freeu': 'FreeU', 'base_model': 'Model', @@ -460,7 +458,7 @@ def parse_string(self, metadata: dict) -> str: self.fooocus_to_a1111['refiner_model_hash']: self.refiner_model_hash } - for key in ['adaptive_cfg', 'clip_skip', 'overwrite_switch', 'refiner_swap_method', 'freeu']: + for key in ['adaptive_cfg', 'overwrite_switch', 'refiner_swap_method', 'freeu']: if key in data: generation_params[self.fooocus_to_a1111[key]] = data[key] diff --git a/webui.py b/webui.py index edc9b6b15..090604a0f 100644 --- a/webui.py +++ b/webui.py @@ -421,9 +421,6 @@ def update_history_link(): value=modules.config.default_cfg_tsnr, info='Enabling Fooocus\'s implementation of CFG mimicking for TSNR ' '(effective when real CFG > mimicked CFG).') - clip_skip = gr.Slider(label='CLIP Skip', minimum=1, maximum=10, step=1, - value=modules.config.default_clip_skip, - info='Bypass CLIP layers to avoid overfitting (use 1 to disable).') sampler_name = gr.Dropdown(label='Sampler', choices=flags.sampler_list, value=modules.config.default_sampler) scheduler_name = gr.Dropdown(label='Scheduler', choices=flags.scheduler_list, @@ -588,9 +585,9 @@ def refresh_files_clicked(): load_data_outputs = [advanced_checkbox, image_number, prompt, negative_prompt, style_selections, performance_selection, overwrite_step, overwrite_switch, aspect_ratios_selection, overwrite_width, overwrite_height, guidance_scale, sharpness, adm_scaler_positive, - adm_scaler_negative, adm_scaler_end, refiner_swap_method, adaptive_cfg, clip_skip, - base_model, refiner_model, refiner_switch, sampler_name, scheduler_name, vae_name, - seed_random, image_seed, generate_button, load_parameter_button] + freeu_ctrls + lora_ctrls + adm_scaler_negative, adm_scaler_end, refiner_swap_method, adaptive_cfg, base_model, + refiner_model, refiner_switch, sampler_name, scheduler_name, vae_name, seed_random, + image_seed, generate_button, load_parameter_button] + freeu_ctrls + lora_ctrls if not args_manager.args.disable_preset_selection: def preset_selection_change(preset, is_generating): @@ -675,7 +672,7 @@ def inpaint_mode_change(mode): ctrls += [uov_method, uov_input_image] ctrls += [outpaint_selections, inpaint_input_image, inpaint_additional_prompt, inpaint_mask_image] ctrls += [disable_preview, disable_intermediate_results, disable_seed_increment, black_out_nsfw] - ctrls += [adm_scaler_positive, adm_scaler_negative, adm_scaler_end, adaptive_cfg, clip_skip] + ctrls += [adm_scaler_positive, adm_scaler_negative, adm_scaler_end, adaptive_cfg] ctrls += [sampler_name, scheduler_name, vae_name] ctrls += [overwrite_step, overwrite_switch, overwrite_width, overwrite_height, overwrite_vary_strength] ctrls += [overwrite_upscale_strength, mixing_image_prompt_and_vary_upscale, mixing_image_prompt_and_inpaint] From dfff9b7dcfe447d3595d87c640af78ccf7389eae Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Mon, 27 May 2024 00:28:22 +0200 Subject: [PATCH 39/42] fix: adjust clip skip default value from 1 to 2 (#3011) * Revert "Revert "feat: add clip skip handling (#2999)" (#3008)" This reverts commit 989a1ad52b209cc0712c610b69988e1bf14eddb6. * feat: use clip skip 2 as default --- language/en.json | 2 ++ modules/async_worker.py | 6 ++++++ modules/config.py | 6 ++++++ modules/default_pipeline.py | 11 +++++++++++ modules/flags.py | 2 ++ modules/meta_parser.py | 18 ++++++++++-------- webui.py | 11 +++++++---- 7 files changed, 44 insertions(+), 12 deletions(-) diff --git a/language/en.json b/language/en.json index 90eaf2eee..5819f4eed 100644 --- a/language/en.json +++ b/language/en.json @@ -320,6 +320,8 @@ "vae": "vae", "CFG Mimicking from TSNR": "CFG Mimicking from TSNR", "Enabling Fooocus's implementation of CFG mimicking for TSNR (effective when real CFG > mimicked CFG).": "Enabling Fooocus's implementation of CFG mimicking for TSNR (effective when real CFG > mimicked CFG).", + "CLIP Skip": "CLIP Skip", + "Bypass CLIP layers to avoid overfitting (use 1 to not skip any layers, 2 is recommended).": "Bypass CLIP layers to avoid overfitting (use 1 to not skip any layers, 2 is recommended).", "Sampler": "Sampler", "dpmpp_2m_sde_gpu": "dpmpp_2m_sde_gpu", "Only effective in non-inpaint mode.": "Only effective in non-inpaint mode.", diff --git a/modules/async_worker.py b/modules/async_worker.py index 594886d28..d7d9b9fd7 100644 --- a/modules/async_worker.py +++ b/modules/async_worker.py @@ -174,6 +174,7 @@ def handler(async_task): adm_scaler_negative = args.pop() adm_scaler_end = args.pop() adaptive_cfg = args.pop() + clip_skip = args.pop() sampler_name = args.pop() scheduler_name = args.pop() vae_name = args.pop() @@ -297,6 +298,7 @@ def handler(async_task): adm_scaler_end = 0.0 print(f'[Parameters] Adaptive CFG = {adaptive_cfg}') + print(f'[Parameters] CLIP Skip = {clip_skip}') print(f'[Parameters] Sharpness = {sharpness}') print(f'[Parameters] ControlNet Softness = {controlnet_softness}') print(f'[Parameters] ADM Scale = ' @@ -466,6 +468,8 @@ def handler(async_task): loras=loras, base_model_additional_loras=base_model_additional_loras, use_synthetic_refiner=use_synthetic_refiner, vae_name=vae_name) + pipeline.set_clip_skip(clip_skip) + progressbar(async_task, 3, 'Processing prompts ...') tasks = [] @@ -924,6 +928,8 @@ def callback(step, x0, x, total_steps, y): d.append( ('CFG Mimicking from TSNR', 'adaptive_cfg', modules.patch.patch_settings[pid].adaptive_cfg)) + if clip_skip > 1: + d.append(('CLIP Skip', 'clip_skip', clip_skip)) d.append(('Sampler', 'sampler', sampler_name)) d.append(('Scheduler', 'scheduler', scheduler_name)) d.append(('VAE', 'vae', vae_name)) diff --git a/modules/config.py b/modules/config.py index 08ed99d7f..cb651c5b6 100644 --- a/modules/config.py +++ b/modules/config.py @@ -434,6 +434,11 @@ def init_temp_path(path: str | None, default_path: str) -> str: default_value=7.0, validator=lambda x: isinstance(x, numbers.Number) ) +default_clip_skip = get_config_item_or_set_default( + key='default_clip_skip', + default_value=2, + validator=lambda x: isinstance(x, int) and 1 <= x <= modules.flags.clip_skip_max +) default_overwrite_step = get_config_item_or_set_default( key='default_overwrite_step', default_value=-1, @@ -488,6 +493,7 @@ def init_temp_path(path: str | None, default_path: str) -> str: "default_cfg_scale": "guidance_scale", "default_sample_sharpness": "sharpness", "default_cfg_tsnr": "adaptive_cfg", + "default_clip_skip": "clip_skip", "default_sampler": "sampler", "default_scheduler": "scheduler", "default_overwrite_step": "steps", diff --git a/modules/default_pipeline.py b/modules/default_pipeline.py index 38f914c57..494644d69 100644 --- a/modules/default_pipeline.py +++ b/modules/default_pipeline.py @@ -201,6 +201,17 @@ def clip_encode(texts, pool_top_k=1): return [[torch.cat(cond_list, dim=1), {"pooled_output": pooled_acc}]] +@torch.no_grad() +@torch.inference_mode() +def set_clip_skip(clip_skip: int): + global final_clip + + if final_clip is None: + return + + final_clip.clip_layer(-abs(clip_skip)) + return + @torch.no_grad() @torch.inference_mode() def clear_all_caches(): diff --git a/modules/flags.py b/modules/flags.py index 89e1ea0f2..e48052e18 100644 --- a/modules/flags.py +++ b/modules/flags.py @@ -54,6 +54,8 @@ sampler_list = SAMPLER_NAMES scheduler_list = SCHEDULER_NAMES +clip_skip_max = 12 + default_vae = 'Default (model)' refiner_swap_method = 'joint' diff --git a/modules/meta_parser.py b/modules/meta_parser.py index 4ce12435c..586e62da2 100644 --- a/modules/meta_parser.py +++ b/modules/meta_parser.py @@ -34,16 +34,17 @@ def load_parameter_button_click(raw_metadata: dict | str, is_generating: bool): get_list('styles', 'Styles', loaded_parameter_dict, results) get_str('performance', 'Performance', loaded_parameter_dict, results) get_steps('steps', 'Steps', loaded_parameter_dict, results) - get_float('overwrite_switch', 'Overwrite Switch', loaded_parameter_dict, results) + get_number('overwrite_switch', 'Overwrite Switch', loaded_parameter_dict, results) get_resolution('resolution', 'Resolution', loaded_parameter_dict, results) - get_float('guidance_scale', 'Guidance Scale', loaded_parameter_dict, results) - get_float('sharpness', 'Sharpness', loaded_parameter_dict, results) + get_number('guidance_scale', 'Guidance Scale', loaded_parameter_dict, results) + get_number('sharpness', 'Sharpness', loaded_parameter_dict, results) get_adm_guidance('adm_guidance', 'ADM Guidance', loaded_parameter_dict, results) get_str('refiner_swap_method', 'Refiner Swap Method', loaded_parameter_dict, results) - get_float('adaptive_cfg', 'CFG Mimicking from TSNR', loaded_parameter_dict, results) + get_number('adaptive_cfg', 'CFG Mimicking from TSNR', loaded_parameter_dict, results) + get_number('clip_skip', 'CLIP Skip', loaded_parameter_dict, results, cast_type=int) get_str('base_model', 'Base Model', loaded_parameter_dict, results) get_str('refiner_model', 'Refiner Model', loaded_parameter_dict, results) - get_float('refiner_switch', 'Refiner Switch', loaded_parameter_dict, results) + get_number('refiner_switch', 'Refiner Switch', loaded_parameter_dict, results) get_str('sampler', 'Sampler', loaded_parameter_dict, results) get_str('scheduler', 'Scheduler', loaded_parameter_dict, results) get_str('vae', 'VAE', loaded_parameter_dict, results) @@ -83,11 +84,11 @@ def get_list(key: str, fallback: str | None, source_dict: dict, results: list, d results.append(gr.update()) -def get_float(key: str, fallback: str | None, source_dict: dict, results: list, default=None): +def get_number(key: str, fallback: str | None, source_dict: dict, results: list, default=None, cast_type=float): try: h = source_dict.get(key, source_dict.get(fallback, default)) assert h is not None - h = float(h) + h = cast_type(h) results.append(h) except: results.append(gr.update()) @@ -314,6 +315,7 @@ def get_scheme(self) -> MetadataScheme: 'adm_guidance': 'ADM Guidance', 'refiner_swap_method': 'Refiner Swap Method', 'adaptive_cfg': 'Adaptive CFG', + 'clip_skip': 'Clip skip', 'overwrite_switch': 'Overwrite Switch', 'freeu': 'FreeU', 'base_model': 'Model', @@ -458,7 +460,7 @@ def parse_string(self, metadata: dict) -> str: self.fooocus_to_a1111['refiner_model_hash']: self.refiner_model_hash } - for key in ['adaptive_cfg', 'overwrite_switch', 'refiner_swap_method', 'freeu']: + for key in ['adaptive_cfg', 'clip_skip', 'overwrite_switch', 'refiner_swap_method', 'freeu']: if key in data: generation_params[self.fooocus_to_a1111[key]] = data[key] diff --git a/webui.py b/webui.py index 090604a0f..49f00aaba 100644 --- a/webui.py +++ b/webui.py @@ -421,6 +421,9 @@ def update_history_link(): value=modules.config.default_cfg_tsnr, info='Enabling Fooocus\'s implementation of CFG mimicking for TSNR ' '(effective when real CFG > mimicked CFG).') + clip_skip = gr.Slider(label='CLIP Skip', minimum=1, maximum=flags.clip_skip_max, step=1, + value=modules.config.default_clip_skip, + info='Bypass CLIP layers to avoid overfitting (use 1 to not skip any layers, 2 is recommended).') sampler_name = gr.Dropdown(label='Sampler', choices=flags.sampler_list, value=modules.config.default_sampler) scheduler_name = gr.Dropdown(label='Scheduler', choices=flags.scheduler_list, @@ -585,9 +588,9 @@ def refresh_files_clicked(): load_data_outputs = [advanced_checkbox, image_number, prompt, negative_prompt, style_selections, performance_selection, overwrite_step, overwrite_switch, aspect_ratios_selection, overwrite_width, overwrite_height, guidance_scale, sharpness, adm_scaler_positive, - adm_scaler_negative, adm_scaler_end, refiner_swap_method, adaptive_cfg, base_model, - refiner_model, refiner_switch, sampler_name, scheduler_name, vae_name, seed_random, - image_seed, generate_button, load_parameter_button] + freeu_ctrls + lora_ctrls + adm_scaler_negative, adm_scaler_end, refiner_swap_method, adaptive_cfg, clip_skip, + base_model, refiner_model, refiner_switch, sampler_name, scheduler_name, vae_name, + seed_random, image_seed, generate_button, load_parameter_button] + freeu_ctrls + lora_ctrls if not args_manager.args.disable_preset_selection: def preset_selection_change(preset, is_generating): @@ -672,7 +675,7 @@ def inpaint_mode_change(mode): ctrls += [uov_method, uov_input_image] ctrls += [outpaint_selections, inpaint_input_image, inpaint_additional_prompt, inpaint_mask_image] ctrls += [disable_preview, disable_intermediate_results, disable_seed_increment, black_out_nsfw] - ctrls += [adm_scaler_positive, adm_scaler_negative, adm_scaler_end, adaptive_cfg] + ctrls += [adm_scaler_positive, adm_scaler_negative, adm_scaler_end, adaptive_cfg, clip_skip] ctrls += [sampler_name, scheduler_name, vae_name] ctrls += [overwrite_step, overwrite_switch, overwrite_width, overwrite_height, overwrite_vary_strength] ctrls += [overwrite_upscale_strength, mixing_image_prompt_and_vary_upscale, mixing_image_prompt_and_inpaint] From 0e621ae34ec1c8c0d81b83ce95eed51f0ab2d617 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Tue, 28 May 2024 00:09:39 +0200 Subject: [PATCH 40/42] fix: add type check for undefined, use fallback when no translation for aspect ratios was given (#3025) --- javascript/localization.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/javascript/localization.js b/javascript/localization.js index 9f13d6fcb..21b3b3332 100644 --- a/javascript/localization.js +++ b/javascript/localization.js @@ -81,9 +81,12 @@ function refresh_style_localization() { } function refresh_aspect_ratios_label(value) { - label = document.querySelector('#aspect_ratios_accordion div span[data-original-text="Aspect Ratios"]') - translation = getTranslation("Aspect Ratios") - label.textContent = translation + " " + htmlDecode(value) + label = document.querySelector('#aspect_ratios_accordion div span[data-original-text="Aspect Ratios"]'); + translation = getTranslation("Aspect Ratios"); + if (typeof translation == "undefined") { + translation = "Aspect Ratios"; + } + label.textContent = translation + " " + htmlDecode(value); } function localizeWholePage() { From 4a070a9d610a1955c90f7619055460729ae0ac60 Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Tue, 28 May 2024 00:49:47 +0200 Subject: [PATCH 41/42] feat: build docker image tagged "edge" on push to main branch (#3026) * feat: build docker image on push to main branch * feat: add tag "edge" for main when building the docker image * feat: update name of build container workflow --- .github/workflows/build_container.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_container.yml b/.github/workflows/build_container.yml index 1e118a1ff..eb70cda3d 100644 --- a/.github/workflows/build_container.yml +++ b/.github/workflows/build_container.yml @@ -1,9 +1,11 @@ -name: Create and publish a container image +name: Docker image build on: push: + branches: + - main tags: - - 'v*' + - v* jobs: build-and-push-image: @@ -33,6 +35,7 @@ jobs: type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} type=semver,pattern={{major}} + type=edge,branch=main - name: Build and push Docker image uses: docker/build-push-action@v5 From 725bf05c3129888c237a09dbfdc8ab751263492a Mon Sep 17 00:00:00 2001 From: Manuel Schmid <9307310+mashb1t@users.noreply.github.com> Date: Tue, 28 May 2024 01:10:45 +0200 Subject: [PATCH 42/42] release: bump version to 2.4.1, update changelog (#3027) --- fooocus_version.py | 2 +- update_log.md | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/fooocus_version.py b/fooocus_version.py index ecc158079..750114584 100644 --- a/fooocus_version.py +++ b/fooocus_version.py @@ -1 +1 @@ -version = '2.4.0' +version = '2.4.1' diff --git a/update_log.md b/update_log.md index 77d70cb47..733f077bb 100644 --- a/update_log.md +++ b/update_log.md @@ -1,3 +1,8 @@ +# [2.4.1](https://github.com/lllyasviel/Fooocus/releases/tag/v2.4.1) + +* Fix some small bugs (e.g. adjust clip skip default value from 1 to 2, add type check to aspect ratios js update function) +* Add automated docker build on push to main, tagged with `edge`. See [available docker images](https://github.com/lllyasviel/Fooocus/pkgs/container/fooocus). + # [2.4.0](https://github.com/lllyasviel/Fooocus/releases/tag/v2.4.0) * Change settings tab elements to be more compact