Update bitsandbytes

requirements.txt

@@ -1,6 +1,5 @@
 albumentations==1.3.0
 altair==4.2.2
-bitsandbytes==0.35.0
 dadaptation==3.1
 diffusers[torch]==0.10.2
 easygui==0.98.3

requirements_linux.txt

@@ -1,4 +1,4 @@
 torch==2.0.1+cu118 torchvision==0.15.2+cu118 --extra-index-url https://download.pytorch.org/whl/cu118 # no_verify leave this to specify not checking this a verification stage
-xformers==0.0.20
+xformers==0.0.20 bitsandbytes==0.35.0
 accelerate==0.19.0 tensorboard==2.12.1 tensorflow==2.12.0
 -r requirements.txt

requirements_macos_amd64.txt

@@ -1,31 +1,4 @@
 torch==2.0.0 torchvision==0.15.1 -f https://download.pytorch.org/whl/cpu/torch_stable.html
-xformers
+xformers bitsandbytes==0.35.0
 accelerate==0.19.0 tensorflow-macos tensorboard==2.12.1
 -r requirements.txt
-
-# accelerate==0.15.0
-# albumentations==1.3.0
-# altair==4.2.2
-# bitsandbytes==0.35.0
-# dadaptation==3.1
-# diffusers[torch]==0.10.2
-# easygui==0.98.3
-# einops==0.6.0
-# fairscale==0.4.13
-# ftfy==6.1.1
-# gradio==3.23.0
-# huggingface-hub==0.13.0
-# lion-pytorch==0.0.6
-# lycoris_lora==0.1.6
-# opencv-python==4.7.0.68
-# pytorch-lightning==1.9.0
-# rich==13.4.1
-# safetensors==0.2.6
-# timm==0.6.12
-# tk==0.1.0
-# toml==0.10.2
-# transformers==4.26.0
-# voluptuous==0.13.1
-# wandb==0.15.0
-# # for kohya_ss library
-# .

requirements_macos_arm64.txt

@@ -1,31 +1,4 @@
 torch==2.0.0 torchvision==0.15.1 -f https://download.pytorch.org/whl/cpu/torch_stable.html
-xformers
+xformers bitsandbytes==0.35.0
 accelerate==0.19.0 tensorflow-metal tensorboard==2.12.1
 -r requirements.txt
-
-# accelerate==0.15.0
-# albumentations==1.3.0
-# altair==4.2.2
-# bitsandbytes==0.35.0
-# dadaptation==3.1
-# diffusers[torch]==0.10.2
-# easygui==0.98.3
-# einops==0.6.0
-# fairscale==0.4.13
-# ftfy==6.1.1
-# gradio==3.23.0
-# huggingface-hub==0.13.0
-# lion-pytorch==0.0.6
-# lycoris_lora==0.1.6
-# opencv-python==4.7.0.68
-# pytorch-lightning==1.9.0
-# rich==13.4.1
-# safetensors==0.2.6
-# timm==0.6.12
-# tk==0.1.0
-# toml==0.10.2
-# transformers==4.26.0
-# voluptuous==0.13.1
-# wandb==0.15.0
-# # for kohya_ss library
-# .

requirements_windows_torch1.txt

@@ -1,5 +1,6 @@
 torch==1.12.1+cu116 torchvision==0.13.1+cu116 --index-url https://download.pytorch.org/whl/cu116 # no_verify
 https://github.com/C43H66N12O12S2/stable-diffusion-webui/releases/download/f/xformers-0.0.14.dev0-cp310-cp310-win_amd64.whl -U -I --no-deps # no_verify
+bitsandbytes-windows
 accelerate==0.15.0 tensorboard==2.10.1 tensorflow==2.10.1
 -r requirements.txt
 

requirements_windows_torch2.txt

@@ -1,5 +1,5 @@
 torch==2.0.1+cu118 torchvision==0.15.2+cu118 --index-url https://download.pytorch.org/whl/cu118 # no_verify
-xformers==0.0.20
+xformers==0.0.20 bitsandbytes-windows
 accelerate==0.19.0 tensorboard==2.12.3 tensorflow==2.12.0
 -r requirements.txt
 

setup/setup_windows.py

@@ -120,7 +120,7 @@ def install_kohya_ss_torch1():
     #     'xformers-0.0.14'
     # )
     setup_common.install_requirements('requirements_windows_torch1.txt', check_no_verify_flag=False)
-    sync_bits_and_bytes_files()
+    # sync_bits_and_bytes_files()
     setup_common.configure_accelerate(run_accelerate=True)
     # run_cmd(f'accelerate config')
 
@@ -144,7 +144,7 @@ def install_kohya_ss_torch2():
     # )
     setup_common.install_requirements('requirements_windows_torch2.txt', check_no_verify_flag=False)
     # install('https://huggingface.co/r4ziel/xformers_pre_built/resolve/main/triton-2.0.0-cp310-cp310-win_amd64.whl', 'triton', reinstall=reinstall)
-    sync_bits_and_bytes_files()
+    # sync_bits_and_bytes_files()
     setup_common.configure_accelerate(run_accelerate=True)
     # run_cmd(f'accelerate config')
 

test/config/dreambooth-AdamW8bit.json

@@ -0,0 +1,73 @@
+{
+  "adaptive_noise_scale": 0,
+  "additional_parameters": "",
+  "bucket_no_upscale": true,
+  "bucket_reso_steps": 64,
+  "cache_latents": true,
+  "cache_latents_to_disk": false,
+  "caption_dropout_every_n_epochs": 0.0,
+  "caption_dropout_rate": 0.05,
+  "caption_extension": "",
+  "clip_skip": 2,
+  "color_aug": false,
+  "enable_bucket": true,
+  "epoch": 1,
+  "flip_aug": false,
+  "full_fp16": false,
+  "gradient_accumulation_steps": 1.0,
+  "gradient_checkpointing": false,
+  "keep_tokens": "0",
+  "learning_rate": 1.0,
+  "logging_dir": "./test/logs",
+  "lr_scheduler": "constant",
+  "lr_warmup": 0,
+  "max_data_loader_n_workers": "0",
+  "max_resolution": "512,512",
+  "max_token_length": "75",
+  "max_train_epochs": "",
+  "mem_eff_attn": false,
+  "min_snr_gamma": 0,
+  "mixed_precision": "bf16",
+  "model_list": "runwayml/stable-diffusion-v1-5",
+  "multires_noise_discount": 0,
+  "multires_noise_iterations": 0,
+  "no_token_padding": false,
+  "noise_offset": "0.05",
+  "noise_offset_type": "Original",
+  "num_cpu_threads_per_process": 2,
+  "optimizer": "AdamW8bit",
+  "optimizer_args": "",
+  "output_dir": "./test/output",
+  "output_name": "db-8bit",
+  "persistent_data_loader_workers": false,
+  "pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
+  "prior_loss_weight": 1.0,
+  "random_crop": false,
+  "reg_data_dir": "",
+  "resume": "",
+  "sample_every_n_epochs": 0,
+  "sample_every_n_steps": 25,
+  "sample_prompts": "a painting of a gas mask , by darius kawasaki",
+  "sample_sampler": "euler_a",
+  "save_every_n_epochs": 1,
+  "save_every_n_steps": 0,
+  "save_last_n_steps": 0,
+  "save_last_n_steps_state": 0,
+  "save_model_as": "safetensors",
+  "save_precision": "fp16",
+  "save_state": false,
+  "scale_v_pred_loss_like_noise_pred": false,
+  "seed": "1234",
+  "shuffle_caption": false,
+  "stop_text_encoder_training": 0,
+  "train_batch_size": 4,
+  "train_data_dir": "./test/img",
+  "use_wandb": false,
+  "v2": false,
+  "v_parameterization": false,
+  "vae": "",
+  "vae_batch_size": 0,
+  "wandb_api_key": "",
+  "weighted_captions": false,
+  "xformers": true
+}