Add GLoRA support

bmaltais · Dec 6, 2023 · 06eed69 · 06eed69
1 parent 638ec29
commit 06eed69
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 12 deletions.
diff --git a/.release b/.release
@@ -1 +1 @@
-v22.2.2
+v22.3.0
diff --git a/README.md b/README.md
@@ -651,7 +651,7 @@ masterpiece, best quality, 1boy, in business suit, standing at street, looking b
 
 
 ## Change History
-* 2023/12/05 (v22.3.0)
+* 2023/12/06 (v22.3.0)
 - Merge sd-scripts updates:
   - `finetune\tag_images_by_wd14_tagger.py` now supports the separator other than `,` with `--caption_separator` option. Thanks to KohakuBlueleaf! PR [#913](https://github.com/kohya-ss/sd-scripts/pull/913)
   - Min SNR Gamma with V-predicition (SD 2.1) is fixed. Thanks to feffy380! PR[#934](https://github.com/kohya-ss/sd-scripts/pull/934)
@@ -663,6 +663,7 @@ masterpiece, best quality, 1boy, in business suit, standing at street, looking b
     - `--ds_depth_1` and `--ds_depth_2` options denote the depth (block index) of the Deep Shrink for the first and second stages.
     - `--ds_ratio` option denotes the ratio of the Deep Shrink. `0.5` means the half of the original latent size for the Deep Shrink.
     - `--dst1`, `--dst2`, `--dsd1`, `--dsd2` and `--dsr` prompt options are also available.
+  - Add GLoRA support
 
 * 2023/12/03 (v22.2.2)
 - Update Lycoris module to 2.0.0 (https://github.com/KohakuBlueleaf/LyCORIS/blob/0006e2ffa05a48d8818112d9f70da74c0cd30b99/README.md)

diff --git a/lora_gui.py b/lora_gui.py
@@ -732,6 +732,17 @@ def train_model(
         run_cmd += f" --network_module=lycoris.kohya"
         run_cmd += f' --network_args "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "algo=locon"'
 
+    if LoRA_type == "LyCORIS/GLoRA":
+        try:
+            import lycoris
+        except ModuleNotFoundError:
+            log.info(
+                "\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program."
+            )
+            return
+        run_cmd += f" --network_module=lycoris.kohya"
+        run_cmd += f' --network_args "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "algo=glora"'
+
     if LoRA_type == "LyCORIS/LoHa":
         try:
             import lycoris
@@ -741,7 +752,7 @@ def train_model(
             )
             return
         run_cmd += f" --network_module=lycoris.kohya"
-        run_cmd += f' --network_args "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "use_cp={use_cp}" "algo=loha"'
+        run_cmd += f' --network_args "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "use_tucker={use_cp}" "algo=loha"'
         # This is a hack to fix a train_network LoHA logic issue
         if not network_dropout > 0.0:
             run_cmd += f' --network_dropout="{network_dropout}"'
@@ -787,7 +798,7 @@ def train_model(
         # This is a hack to fix a train_network LoHA logic issue
         if not network_dropout > 0.0:
             run_cmd += f' --network_dropout="{network_dropout}"'
-            
+
     if LoRA_type == "LyCORIS/Native Fine-Tuning":
         try:
             import lycoris
@@ -797,11 +808,13 @@ def train_model(
             )
             return
         run_cmd += f" --network_module=lycoris.kohya"
-        run_cmd += f' --network_args "preset={LyCORIS_preset}" "algo=full" "train_norm=True"'
+        run_cmd += (
+            f' --network_args "preset={LyCORIS_preset}" "algo=full" "train_norm=True"'
+        )
         # This is a hack to fix a train_network LoHA logic issue
         if not network_dropout > 0.0:
             run_cmd += f' --network_dropout="{network_dropout}"'
-    
+
     if LoRA_type == "LyCORIS/Diag-OFT":
         try:
             import lycoris
@@ -815,7 +828,6 @@ def train_model(
         # This is a hack to fix a train_network LoHA logic issue
         if not network_dropout > 0.0:
             run_cmd += f' --network_dropout="{network_dropout}"'
-
 
     if LoRA_type in ["Kohya LoCon", "Standard"]:
         kohya_lora_var_list = [
@@ -967,7 +979,7 @@ def train_model(
 
     if full_bf16:
         run_cmd += f" --full_bf16"
-        
+
     if debiased_estimation_loss:
         run_cmd += " --debiased_estimation_loss"
 
@@ -1139,6 +1151,7 @@ def list_presets(path):
                             "LyCORIS/DyLoRA",
                             "LyCORIS/iA3",
                             "LyCORIS/Diag-OFT",
+                            "LyCORIS/GLoRA",
                             "LyCORIS/LoCon",
                             "LyCORIS/LoHa",
                             "LyCORIS/LoKr",
@@ -1158,7 +1171,8 @@ def list_presets(path):
                             "unet-convblock-only",
                         ],
                         value="full",
-                        visible=False, interactive=True
+                        visible=False,
+                        interactive=True
                         # info="https://github.com/KohakuBlueleaf/LyCORIS/blob/0006e2ffa05a48d8818112d9f70da74c0cd30b99/docs/Preset.md"
                     )
                     with gr.Box():
@@ -1235,7 +1249,7 @@ def list_presets(path):
                 with gr.Row() as LoRA_dim_alpha:
                     network_dim = gr.Slider(
                         minimum=1,
-                        maximum=100000, # 512 if not LoRA_type == "LyCORIS/LoKr" else 100000,
+                        maximum=100000,  # 512 if not LoRA_type == "LyCORIS/LoKr" else 100000,
                         label="Network Rank (Dimension)",
                         value=8,
                         step=1,
@@ -1254,7 +1268,7 @@ def list_presets(path):
                     # locon= gr.Checkbox(label='Train a LoCon instead of a general LoRA (does not support v2 base models) (may not be able to some utilities now)', value=False)
                     conv_dim = gr.Slider(
                         minimum=0,
-                        maximum=100000, # 512 if not LoRA_type == "LyCORIS/LoKr" else 100000,
+                        maximum=100000,  # 512 if not LoRA_type == "LyCORIS/LoKr" else 100000,
                         value=1,
                         step=1,
                         label="Convolution Rank (Dimension)",
@@ -1322,6 +1336,7 @@ def update_LoRA_settings(LoRA_type):
                                     "LoRA-FA",
                                     "LyCORIS/Diag-OFT",
                                     "LyCORIS/DyLoRA",
+                                    "LyCORIS/GLoRA",
                                     "LyCORIS/LoCon",
                                     "LyCORIS/LoHa",
                                     "LyCORIS/LoKr",
@@ -1340,6 +1355,7 @@ def update_LoRA_settings(LoRA_type):
                                     "LyCORIS/LoHa",
                                     "LyCORIS/LoKr",
                                     "LyCORIS/LoCon",
+                                    "LyCORIS/GLoRA",
                                 },
                                 gr.Row,
                             ),
@@ -1365,6 +1381,7 @@ def update_LoRA_settings(LoRA_type):
                                     "LoRA-FA",
                                     "LyCORIS/Diag-OFT",
                                     "LyCORIS/DyLoRA",
+                                    "LyCORIS/GLoRA",
                                     "LyCORIS/LoHa",
                                     "LyCORIS/LoCon",
                                     "LyCORIS/LoKr",
@@ -1380,6 +1397,7 @@ def update_LoRA_settings(LoRA_type):
                                     "LoRA-FA",
                                     "LyCORIS/Diag-OFT",
                                     "LyCORIS/DyLoRA",
+                                    "LyCORIS/GLoRA",
                                     "LyCORIS/LoHa",
                                     "LyCORIS/LoCon",
                                     "LyCORIS/LoKr",
@@ -1395,6 +1413,7 @@ def update_LoRA_settings(LoRA_type):
                                     "LoRA-FA",
                                     "LyCORIS/Diag-OFT",
                                     "LyCORIS/DyLoRA",
+                                    "LyCORIS/GLoRA",
                                     "LyCORIS/LoHa",
                                     "LyCORIS/LoCon",
                                     "LyCORIS/LoKr",
@@ -1406,6 +1425,7 @@ def update_LoRA_settings(LoRA_type):
                                 {
                                     "LyCORIS/DyLoRA",
                                     "LyCORIS/LoHa",
+                                    "LyCORIS/GLoRA",
                                     "LyCORIS/LoCon",
                                     "LyCORIS/LoKr",
                                 },
@@ -1420,6 +1440,7 @@ def update_LoRA_settings(LoRA_type):
                                     "Kohya LoCon",
                                     "LoRA-FA",
                                     "LyCORIS/DyLoRA",
+                                    "LyCORIS/GLoRA",
                                     "LyCORIS/LoHa",
                                     "LyCORIS/LoCon",
                                     "LyCORIS/LoKr",
@@ -1435,6 +1456,7 @@ def update_LoRA_settings(LoRA_type):
                                     "LoRA-FA",
                                     "LyCORIS/Diag-OFT",
                                     "LyCORIS/DyLoRA",
+                                    "LyCORIS/GLoRA",
                                     "LyCORIS/LoHa",
                                     "LyCORIS/LoCon",
                                     "LyCORIS/LoKr",
@@ -1467,13 +1489,14 @@ def update_LoRA_settings(LoRA_type):
                                     "LyCORIS/DyLoRA",
                                     "LyCORIS/iA3",
                                     "LyCORIS/Diag-OFT",
+                                    "LyCORIS/GLoRA",
                                     "LyCORIS/LoCon",
                                     "LyCORIS/LoHa",
                                     "LyCORIS/LoKr",
                                     "LyCORIS/Native Fine-Tuning",
                                 },
                                 gr.Dropdown,
-                            )
+                            ),
                         }
 
                         results = []