From 06eed6977b1f5f3cbaadc3115c571120e7293cc3 Mon Sep 17 00:00:00 2001 From: bmaltais Date: Wed, 6 Dec 2023 17:47:13 -0500 Subject: [PATCH] Add GLoRA support --- .release | 2 +- README.md | 3 ++- lora_gui.py | 43 +++++++++++++++++++++++++++++++++---------- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/.release b/.release index 6a112b855..d542ef359 100644 --- a/.release +++ b/.release @@ -1 +1 @@ -v22.2.2 +v22.3.0 diff --git a/README.md b/README.md index a8f880334..14244a46e 100644 --- a/README.md +++ b/README.md @@ -651,7 +651,7 @@ masterpiece, best quality, 1boy, in business suit, standing at street, looking b ## Change History -* 2023/12/05 (v22.3.0) +* 2023/12/06 (v22.3.0) - Merge sd-scripts updates: - `finetune\tag_images_by_wd14_tagger.py` now supports the separator other than `,` with `--caption_separator` option. Thanks to KohakuBlueleaf! PR [#913](https://github.com/kohya-ss/sd-scripts/pull/913) - Min SNR Gamma with V-predicition (SD 2.1) is fixed. Thanks to feffy380! PR[#934](https://github.com/kohya-ss/sd-scripts/pull/934) @@ -663,6 +663,7 @@ masterpiece, best quality, 1boy, in business suit, standing at street, looking b - `--ds_depth_1` and `--ds_depth_2` options denote the depth (block index) of the Deep Shrink for the first and second stages. - `--ds_ratio` option denotes the ratio of the Deep Shrink. `0.5` means the half of the original latent size for the Deep Shrink. - `--dst1`, `--dst2`, `--dsd1`, `--dsd2` and `--dsr` prompt options are also available. + - Add GLoRA support * 2023/12/03 (v22.2.2) - Update Lycoris module to 2.0.0 (https://github.com/KohakuBlueleaf/LyCORIS/blob/0006e2ffa05a48d8818112d9f70da74c0cd30b99/README.md) diff --git a/lora_gui.py b/lora_gui.py index c7b05b602..1a8199d0e 100644 --- a/lora_gui.py +++ b/lora_gui.py @@ -732,6 +732,17 @@ def train_model( run_cmd += f" --network_module=lycoris.kohya" run_cmd += f' --network_args "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "algo=locon"' + if LoRA_type == "LyCORIS/GLoRA": + try: + import lycoris + except ModuleNotFoundError: + log.info( + "\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program." + ) + return + run_cmd += f" --network_module=lycoris.kohya" + run_cmd += f' --network_args "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "algo=glora"' + if LoRA_type == "LyCORIS/LoHa": try: import lycoris @@ -741,7 +752,7 @@ def train_model( ) return run_cmd += f" --network_module=lycoris.kohya" - run_cmd += f' --network_args "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "use_cp={use_cp}" "algo=loha"' + run_cmd += f' --network_args "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "use_tucker={use_cp}" "algo=loha"' # This is a hack to fix a train_network LoHA logic issue if not network_dropout > 0.0: run_cmd += f' --network_dropout="{network_dropout}"' @@ -787,7 +798,7 @@ def train_model( # This is a hack to fix a train_network LoHA logic issue if not network_dropout > 0.0: run_cmd += f' --network_dropout="{network_dropout}"' - + if LoRA_type == "LyCORIS/Native Fine-Tuning": try: import lycoris @@ -797,11 +808,13 @@ def train_model( ) return run_cmd += f" --network_module=lycoris.kohya" - run_cmd += f' --network_args "preset={LyCORIS_preset}" "algo=full" "train_norm=True"' + run_cmd += ( + f' --network_args "preset={LyCORIS_preset}" "algo=full" "train_norm=True"' + ) # This is a hack to fix a train_network LoHA logic issue if not network_dropout > 0.0: run_cmd += f' --network_dropout="{network_dropout}"' - + if LoRA_type == "LyCORIS/Diag-OFT": try: import lycoris @@ -815,7 +828,6 @@ def train_model( # This is a hack to fix a train_network LoHA logic issue if not network_dropout > 0.0: run_cmd += f' --network_dropout="{network_dropout}"' - if LoRA_type in ["Kohya LoCon", "Standard"]: kohya_lora_var_list = [ @@ -967,7 +979,7 @@ def train_model( if full_bf16: run_cmd += f" --full_bf16" - + if debiased_estimation_loss: run_cmd += " --debiased_estimation_loss" @@ -1139,6 +1151,7 @@ def list_presets(path): "LyCORIS/DyLoRA", "LyCORIS/iA3", "LyCORIS/Diag-OFT", + "LyCORIS/GLoRA", "LyCORIS/LoCon", "LyCORIS/LoHa", "LyCORIS/LoKr", @@ -1158,7 +1171,8 @@ def list_presets(path): "unet-convblock-only", ], value="full", - visible=False, interactive=True + visible=False, + interactive=True # info="https://github.com/KohakuBlueleaf/LyCORIS/blob/0006e2ffa05a48d8818112d9f70da74c0cd30b99/docs/Preset.md" ) with gr.Box(): @@ -1235,7 +1249,7 @@ def list_presets(path): with gr.Row() as LoRA_dim_alpha: network_dim = gr.Slider( minimum=1, - maximum=100000, # 512 if not LoRA_type == "LyCORIS/LoKr" else 100000, + maximum=100000, # 512 if not LoRA_type == "LyCORIS/LoKr" else 100000, label="Network Rank (Dimension)", value=8, step=1, @@ -1254,7 +1268,7 @@ def list_presets(path): # locon= gr.Checkbox(label='Train a LoCon instead of a general LoRA (does not support v2 base models) (may not be able to some utilities now)', value=False) conv_dim = gr.Slider( minimum=0, - maximum=100000, # 512 if not LoRA_type == "LyCORIS/LoKr" else 100000, + maximum=100000, # 512 if not LoRA_type == "LyCORIS/LoKr" else 100000, value=1, step=1, label="Convolution Rank (Dimension)", @@ -1322,6 +1336,7 @@ def update_LoRA_settings(LoRA_type): "LoRA-FA", "LyCORIS/Diag-OFT", "LyCORIS/DyLoRA", + "LyCORIS/GLoRA", "LyCORIS/LoCon", "LyCORIS/LoHa", "LyCORIS/LoKr", @@ -1340,6 +1355,7 @@ def update_LoRA_settings(LoRA_type): "LyCORIS/LoHa", "LyCORIS/LoKr", "LyCORIS/LoCon", + "LyCORIS/GLoRA", }, gr.Row, ), @@ -1365,6 +1381,7 @@ def update_LoRA_settings(LoRA_type): "LoRA-FA", "LyCORIS/Diag-OFT", "LyCORIS/DyLoRA", + "LyCORIS/GLoRA", "LyCORIS/LoHa", "LyCORIS/LoCon", "LyCORIS/LoKr", @@ -1380,6 +1397,7 @@ def update_LoRA_settings(LoRA_type): "LoRA-FA", "LyCORIS/Diag-OFT", "LyCORIS/DyLoRA", + "LyCORIS/GLoRA", "LyCORIS/LoHa", "LyCORIS/LoCon", "LyCORIS/LoKr", @@ -1395,6 +1413,7 @@ def update_LoRA_settings(LoRA_type): "LoRA-FA", "LyCORIS/Diag-OFT", "LyCORIS/DyLoRA", + "LyCORIS/GLoRA", "LyCORIS/LoHa", "LyCORIS/LoCon", "LyCORIS/LoKr", @@ -1406,6 +1425,7 @@ def update_LoRA_settings(LoRA_type): { "LyCORIS/DyLoRA", "LyCORIS/LoHa", + "LyCORIS/GLoRA", "LyCORIS/LoCon", "LyCORIS/LoKr", }, @@ -1420,6 +1440,7 @@ def update_LoRA_settings(LoRA_type): "Kohya LoCon", "LoRA-FA", "LyCORIS/DyLoRA", + "LyCORIS/GLoRA", "LyCORIS/LoHa", "LyCORIS/LoCon", "LyCORIS/LoKr", @@ -1435,6 +1456,7 @@ def update_LoRA_settings(LoRA_type): "LoRA-FA", "LyCORIS/Diag-OFT", "LyCORIS/DyLoRA", + "LyCORIS/GLoRA", "LyCORIS/LoHa", "LyCORIS/LoCon", "LyCORIS/LoKr", @@ -1467,13 +1489,14 @@ def update_LoRA_settings(LoRA_type): "LyCORIS/DyLoRA", "LyCORIS/iA3", "LyCORIS/Diag-OFT", + "LyCORIS/GLoRA", "LyCORIS/LoCon", "LyCORIS/LoHa", "LyCORIS/LoKr", "LyCORIS/Native Fine-Tuning", }, gr.Dropdown, - ) + ), } results = []