diff --git a/.github/workflows/typos.yaml b/.github/workflows/typos.yaml index 11269e1e7..37f194d80 100644 --- a/.github/workflows/typos.yaml +++ b/.github/workflows/typos.yaml @@ -18,4 +18,4 @@ jobs: - uses: actions/checkout@v4 - name: typos-action - uses: crate-ci/typos@v1.16.21 + uses: crate-ci/typos@v1.16.23 diff --git a/.release b/.release index 8013ac773..6a112b855 100644 --- a/.release +++ b/.release @@ -1 +1 @@ -v22.2.1 +v22.2.2 diff --git a/Dockerfile b/Dockerfile index bfb282224..200370899 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,9 @@ ENV DEBIAN_FRONTEND=noninteractive ENV TZ=Europe/London RUN apt update && apt-get install -y software-properties-common -RUN add-apt-repository ppa:deadsnakes/ppa && \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + add-apt-repository ppa:deadsnakes/ppa && \ apt update && \ apt-get install -y git curl libgl1 libglib2.0-0 libgoogle-perftools-dev \ python3.10-dev python3.10-tk python3-html5lib python3-apt python3-pip python3.10-distutils && \ @@ -17,22 +19,22 @@ RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 3 RUN curl -sS https://bootstrap.pypa.io/get-pip.py | python3 WORKDIR /app -RUN python3 -m pip install wheel +RUN --mount=type=cache,target=/root/.cache/pip python3 -m pip install wheel # Todo: Install torch 2.1.0 for cu121 support (only available as nightly as of writing) -## RUN python3 -m pip install --pre torch ninja setuptools --extra-index-url https://download.pytorch.org/whl/nightly/cu121 +## RUN --mount=type=cache,target=/root/.cache/pip python3 -m pip install --pre torch ninja setuptools --extra-index-url https://download.pytorch.org/whl/nightly/cu121 # Todo: Install xformers nightly for Torch 2.1.0 support -## RUN python3 -m pip install -v -U git+https://github.com/facebookresearch/xformers.git@main#egg=xformers +## RUN --mount=type=cache,target=/root/.cache/pip python3 -m pip install -v -U git+https://github.com/facebookresearch/xformers.git@main#egg=xformers # Install requirements COPY ./requirements.txt ./requirements_linux_docker.txt ./ COPY ./setup/docker_setup.py ./setup.py -RUN python3 -m pip install -r ./requirements_linux_docker.txt -RUN python3 -m pip install -r ./requirements.txt +RUN --mount=type=cache,target=/root/.cache/pip python3 -m pip install -r ./requirements_linux_docker.txt +RUN --mount=type=cache,target=/root/.cache/pip python3 -m pip install -r ./requirements.txt # Replace pillow with pillow-simd -RUN python3 -m pip uninstall -y pillow && \ +RUN --mount=type=cache,target=/root/.cache/pip python3 -m pip uninstall -y pillow && \ CC="cc -mavx2" python3 -m pip install -U --force-reinstall pillow-simd # Fix missing libnvinfer7 diff --git a/README.md b/README.md index 828b63391..85dfd5cdd 100644 --- a/README.md +++ b/README.md @@ -651,7 +651,15 @@ masterpiece, best quality, 1boy, in business suit, standing at street, looking b ## Change History -* 2023/11/?? (v22.2.1) +* 2023/12/03 (v22.2.2) +- Update Lycoris module to 2.0.0 (https://github.com/KohakuBlueleaf/LyCORIS/blob/0006e2ffa05a48d8818112d9f70da74c0cd30b99/README.md) +- Update Lycoris merge and extract tools +- Remove anoying warning about local pip modules that is not necessary. +- Adding support for LyCORIS presets +- Adding Support for LyCORIS Native Fine-Tuning +- Adding support for Lycoris Diag-OFT + +* 2023/11/20 (v22.2.1) - Fix issue with `Debiased Estimation loss` not getting properly loaded from json file. Oups. * 2023/11/15 (v22.2.0) diff --git a/gui.bat b/gui.bat index 43b7d926a..7b4154150 100644 --- a/gui.bat +++ b/gui.bat @@ -4,7 +4,7 @@ call .\venv\Scripts\deactivate.bat :: Calling external python program to check for local modules -python .\setup\check_local_modules.py --no_question +:: python .\setup\check_local_modules.py --no_question :: Activate the virtual environment call .\venv\Scripts\activate.bat diff --git a/gui.ps1 b/gui.ps1 index 2947343db..1ef3932f0 100644 --- a/gui.ps1 +++ b/gui.ps1 @@ -4,24 +4,24 @@ if ($env:VIRTUAL_ENV) { & deactivate } -# Run pip freeze and capture the output -$pipOutput = & pip freeze +# # Run pip freeze and capture the output +# $pipOutput = & pip freeze -# Check if modules are found in the output -if ($pipOutput) { - Write-Host " " - Write-Host -ForegroundColor Yellow -Object "=============================================================" - Write-Host -ForegroundColor Yellow -Object "Modules installed outside the virtual environment were found." - Write-Host -ForegroundColor Yellow -Object "This can cause issues. Please review the installed modules." - Write-Host " " - Write-Host -ForegroundColor Yellow -Object "You can deinstall all the local modules with:" - Write-Host " " - Write-Host -ForegroundColor Blue -Object "deactivate" - Write-Host -ForegroundColor Blue -Object "pip freeze > uninstall.txt" - Write-Host -ForegroundColor Blue -Object "pip uninstall -y -r uninstall.txt" - Write-Host -ForegroundColor Yellow -Object "=============================================================" - Write-Host " " -} +# # Check if modules are found in the output +# if ($pipOutput) { +# Write-Host " " +# Write-Host -ForegroundColor Yellow -Object "=============================================================" +# Write-Host -ForegroundColor Yellow -Object "Modules installed outside the virtual environment were found." +# Write-Host -ForegroundColor Yellow -Object "This can cause issues. Please review the installed modules." +# Write-Host " " +# Write-Host -ForegroundColor Yellow -Object "You can deinstall all the local modules with:" +# Write-Host " " +# Write-Host -ForegroundColor Blue -Object "deactivate" +# Write-Host -ForegroundColor Blue -Object "pip freeze > uninstall.txt" +# Write-Host -ForegroundColor Blue -Object "pip uninstall -y -r uninstall.txt" +# Write-Host -ForegroundColor Yellow -Object "=============================================================" +# Write-Host " " +# } # Activate the virtual environment # Write-Host "Activating the virtual environment..." diff --git a/library/class_basic_training.py b/library/class_basic_training.py index ac796a345..fe92501ef 100644 --- a/library/class_basic_training.py +++ b/library/class_basic_training.py @@ -105,6 +105,7 @@ def __init__( "Lion", "Lion8bit", "PagedAdamW8bit", + "PagedAdamW32bit", "PagedLion8bit", "Prodigy", "SGDNesterov", diff --git a/library/extract_lycoris_locon_gui.py b/library/extract_lycoris_locon_gui.py index d3c19da6f..aec4cda05 100644 --- a/library/extract_lycoris_locon_gui.py +++ b/library/extract_lycoris_locon_gui.py @@ -13,11 +13,11 @@ # Set up logging log = setup_logging() -folder_symbol = '\U0001f4c2' # 📂 -refresh_symbol = '\U0001f504' # 🔄 -save_style_symbol = '\U0001f4be' # 💾 -document_symbol = '\U0001F4C4' # 📄 -PYTHON = 'python3' if os.name == 'posix' else './venv/Scripts/python.exe' +folder_symbol = "\U0001f4c2" # 📂 +refresh_symbol = "\U0001f504" # 🔄 +save_style_symbol = "\U0001f4be" # 💾 +document_symbol = "\U0001F4C4" # 📄 +PYTHON = "python3" if os.name == "posix" else "./venv/Scripts/python.exe" def extract_lycoris_locon( @@ -25,6 +25,7 @@ def extract_lycoris_locon( base_model, output_name, device, + is_sdxl, is_v2, mode, linear_dim, @@ -40,46 +41,48 @@ def extract_lycoris_locon( disable_cp, ): # Check for caption_text_input - if db_model == '': - msgbox('Invalid finetuned model file') + if db_model == "": + msgbox("Invalid finetuned model file") return - if base_model == '': - msgbox('Invalid base model file') + if base_model == "": + msgbox("Invalid base model file") return # Check if source model exist if not os.path.isfile(db_model): - msgbox('The provided finetuned model is not a file') + msgbox("The provided finetuned model is not a file") return if not os.path.isfile(base_model): - msgbox('The provided base model is not a file') + msgbox("The provided base model is not a file") return run_cmd = f'{PYTHON} "{os.path.join("tools","lycoris_locon_extract.py")}"' + if is_sdxl: + run_cmd += f" --is_sdxl" if is_v2: - run_cmd += f' --is_v2' - run_cmd += f' --device {device}' - run_cmd += f' --mode {mode}' - run_cmd += f' --safetensors' - if mode == 'fixed': - run_cmd += f' --linear_dim {linear_dim}' - run_cmd += f' --conv_dim {conv_dim}' - if mode == 'threshold': - run_cmd += f' --linear_threshold {linear_threshold}' - run_cmd += f' --conv_threshold {conv_threshold}' - if mode == 'ratio': - run_cmd += f' --linear_ratio {linear_ratio}' - run_cmd += f' --conv_ratio {conv_ratio}' - if mode == 'quantile': - run_cmd += f' --linear_quantile {linear_quantile}' - run_cmd += f' --conv_quantile {conv_quantile}' + run_cmd += f" --is_v2" + run_cmd += f" --device {device}" + run_cmd += f" --mode {mode}" + run_cmd += f" --safetensors" + if mode == "fixed": + run_cmd += f" --linear_dim {linear_dim}" + run_cmd += f" --conv_dim {conv_dim}" + if mode == "threshold": + run_cmd += f" --linear_threshold {linear_threshold}" + run_cmd += f" --conv_threshold {conv_threshold}" + if mode == "ratio": + run_cmd += f" --linear_ratio {linear_ratio}" + run_cmd += f" --conv_ratio {conv_ratio}" + if mode == "quantile": + run_cmd += f" --linear_quantile {linear_quantile}" + run_cmd += f" --conv_quantile {conv_quantile}" if use_sparse_bias: - run_cmd += f' --use_sparse_bias' - run_cmd += f' --sparsity {sparsity}' + run_cmd += f" --use_sparse_bias" + run_cmd += f" --sparsity {sparsity}" if disable_cp: - run_cmd += f' --disable_cp' + run_cmd += f" --disable_cp" run_cmd += f' "{base_model}"' run_cmd += f' "{db_model}"' run_cmd += f' "{output_name}"' @@ -87,10 +90,12 @@ def extract_lycoris_locon( log.info(run_cmd) # Run the command - if os.name == 'posix': + if os.name == "posix": os.system(run_cmd) else: subprocess.run(run_cmd) + + log.info('Done extracting...') ### @@ -110,7 +115,7 @@ def extract_lycoris_locon( def update_mode(mode): # Create a list of possible mode values - modes = ['fixed', 'threshold', 'ratio', 'quantile'] + modes = ["fixed", "threshold", "ratio", "quantile"] # Initialize an empty list to store visibility updates updates = [] @@ -125,26 +130,26 @@ def update_mode(mode): def gradio_extract_lycoris_locon_tab(headless=False): - with gr.Tab('Extract LyCORIS LoCON'): + with gr.Tab("Extract LyCORIS LoCON"): gr.Markdown( - 'This utility can extract a LyCORIS LoCon network from a finetuned model.' + "This utility can extract a LyCORIS LoCon network from a finetuned model." ) lora_ext = gr.Textbox( - value='*.safetensors', visible=False - ) # lora_ext = gr.Textbox(value='*.safetensors *.pt', visible=False) - lora_ext_name = gr.Textbox(value='LoRA model types', visible=False) - model_ext = gr.Textbox(value='*.safetensors *.ckpt', visible=False) - model_ext_name = gr.Textbox(value='Model types', visible=False) + value="*.safetensors", visible=False + ) # lora_ext = gr.Textbox(value='*.safetensors *.pt', visible=False) + lora_ext_name = gr.Textbox(value="LoRA model types", visible=False) + model_ext = gr.Textbox(value="*.safetensors *.ckpt", visible=False) + model_ext_name = gr.Textbox(value="Model types", visible=False) with gr.Row(): db_model = gr.Textbox( - label='Finetuned model', - placeholder='Path to the finetuned model to extract', + label="Finetuned model", + placeholder="Path to the finetuned model to extract", interactive=True, ) button_db_model_file = gr.Button( folder_symbol, - elem_id='open_folder_small', + elem_id="open_folder_small", visible=(not headless), ) button_db_model_file.click( @@ -155,13 +160,13 @@ def gradio_extract_lycoris_locon_tab(headless=False): ) base_model = gr.Textbox( - label='Stable Diffusion base model', - placeholder='Stable Diffusion original model: ckpt or safetensors file', + label="Stable Diffusion base model", + placeholder="Stable Diffusion original model: ckpt or safetensors file", interactive=True, ) button_base_model_file = gr.Button( folder_symbol, - elem_id='open_folder_small', + elem_id="open_folder_small", visible=(not headless), ) button_base_model_file.click( @@ -172,13 +177,13 @@ def gradio_extract_lycoris_locon_tab(headless=False): ) with gr.Row(): output_name = gr.Textbox( - label='Save to', - placeholder='path where to save the extracted LoRA model...', + label="Save to", + placeholder="path where to save the extracted LoRA model...", interactive=True, ) button_output_name = gr.Button( folder_symbol, - elem_id='open_folder_small', + elem_id="open_folder_small", visible=(not headless), ) button_output_name.click( @@ -188,109 +193,115 @@ def gradio_extract_lycoris_locon_tab(headless=False): show_progress=False, ) device = gr.Dropdown( - label='Device', + label="Device", choices=[ - 'cpu', - 'cuda', + "cpu", + "cuda", ], - value='cuda', + value="cuda", interactive=True, ) - is_v2 = gr.Checkbox(label='is v2', value=False, interactive=True) + + is_sdxl = gr.Checkbox(label="is SDXL", value=False, interactive=True) + + is_v2 = gr.Checkbox(label="is v2", value=False, interactive=True) mode = gr.Dropdown( - label='Mode', - choices=['fixed', 'threshold', 'ratio', 'quantile'], - value='fixed', + label="Mode", + choices=["fixed", "full", "quantile", "ratio", "threshold"], + value="fixed", interactive=True, ) with gr.Row(visible=True) as fixed: linear_dim = gr.Slider( minimum=1, maximum=1024, - label='Network Dimension', + label="Network Dimension", value=1, step=1, interactive=True, + info="network dim for linear layer in fixed mode", ) conv_dim = gr.Slider( minimum=1, maximum=1024, - label='Conv Dimension', + label="Conv Dimension", value=1, step=1, interactive=True, + info="network dim for conv layer in fixed mode", ) with gr.Row(visible=False) as threshold: linear_threshold = gr.Slider( minimum=0, maximum=1, - label='Linear threshold', + label="Linear threshold", value=0.65, step=0.01, interactive=True, - info='The higher the value, the smaller the file. Recommended starting value: 0.65', + info="The higher the value, the smaller the file. Recommended starting value: 0.65", ) conv_threshold = gr.Slider( minimum=0, maximum=1, - label='Conv threshold', + label="Conv threshold", value=0.65, step=0.01, interactive=True, - info='The higher the value, the smaller the file. Recommended starting value: 0.65', + info="The higher the value, the smaller the file. Recommended starting value: 0.65", ) with gr.Row(visible=False) as ratio: linear_ratio = gr.Slider( minimum=0, maximum=1, - label='Linear ratio', + label="Linear ratio", value=0.75, step=0.01, interactive=True, - info='The higher the value, the smaller the file. Recommended starting value: 0.75', + info="The higher the value, the smaller the file. Recommended starting value: 0.75", ) conv_ratio = gr.Slider( minimum=0, maximum=1, - label='Conv ratio', + label="Conv ratio", value=0.75, step=0.01, interactive=True, - info='The higher the value, the smaller the file. Recommended starting value: 0.75', + info="The higher the value, the smaller the file. Recommended starting value: 0.75", ) with gr.Row(visible=False) as quantile: linear_quantile = gr.Slider( minimum=0, maximum=1, - label='Linear quantile', + label="Linear quantile", value=0.75, step=0.01, interactive=True, - info='The higher the value, the larger the file. Recommended starting value: 0.75', + info="The higher the value, the larger the file. Recommended starting value: 0.75", ) conv_quantile = gr.Slider( minimum=0, maximum=1, - label='Conv quantile', + label="Conv quantile", value=0.75, step=0.01, interactive=True, - info='The higher the value, the larger the file. Recommended starting value: 0.75', + info="The higher the value, the larger the file. Recommended starting value: 0.75", ) with gr.Row(): use_sparse_bias = gr.Checkbox( - label='Use sparse biais', value=False, interactive=True + label="Use sparse biais", value=False, interactive=True ) sparsity = gr.Slider( minimum=0, maximum=1, - label='Sparsity', + label="Sparsity", + info="Sparsity for sparse bias", value=0.98, step=0.01, interactive=True, ) disable_cp = gr.Checkbox( - label='Disable CP decomposition', value=False, interactive=True + label="Disable CP decomposition", value=False, interactive=True ) mode.change( update_mode, @@ -303,7 +314,7 @@ def gradio_extract_lycoris_locon_tab(headless=False): ], ) - extract_button = gr.Button('Extract LyCORIS LoCon') + extract_button = gr.Button("Extract LyCORIS LoCon") extract_button.click( extract_lycoris_locon, @@ -312,6 +323,7 @@ def gradio_extract_lycoris_locon_tab(headless=False): base_model, output_name, device, + is_sdxl, is_v2, mode, linear_dim, diff --git a/library/merge_lycoris_gui.py b/library/merge_lycoris_gui.py index 7d56f1e07..bb084ffdc 100644 --- a/library/merge_lycoris_gui.py +++ b/library/merge_lycoris_gui.py @@ -26,6 +26,7 @@ def merge_lycoris( output_name, dtype, device, + is_sdxl, is_v2, ): log.info('Merge model...') @@ -37,6 +38,8 @@ def merge_lycoris( run_cmd += f' --weight {weight}' run_cmd += f' --device {device}' run_cmd += f' --dtype {dtype}' + if is_sdxl: + run_cmd += f' --is_sdxl' if is_v2: run_cmd += f' --is_v2' @@ -149,12 +152,13 @@ def gradio_merge_lycoris_tab(headless=False): label='Device', choices=[ 'cpu', - # 'cuda', + 'cuda', ], value='cpu', interactive=True, ) + is_sdxl = gr.Checkbox(label='is sdxl', value=False, interactive=True) is_v2 = gr.Checkbox(label='is v2', value=False, interactive=True) merge_button = gr.Button('Merge model') @@ -168,6 +172,7 @@ def gradio_merge_lycoris_tab(headless=False): output_name, dtype, device, + is_sdxl, is_v2, ], show_progress=False, diff --git a/lora_gui.py b/lora_gui.py index a0b9ba654..c7b05b602 100644 --- a/lora_gui.py +++ b/lora_gui.py @@ -171,6 +171,7 @@ def save_configuration( min_timestep, max_timestep, vae, + LyCORIS_preset, debiased_estimation_loss, ): # Get list of function parameters and values @@ -325,6 +326,7 @@ def open_configuration( min_timestep, max_timestep, vae, + LyCORIS_preset, debiased_estimation_loss, training_preset, ): @@ -499,6 +501,7 @@ def train_model( min_timestep, max_timestep, vae, + LyCORIS_preset, debiased_estimation_loss, ): # Get list of function parameters and values @@ -727,7 +730,7 @@ def train_model( ) return run_cmd += f" --network_module=lycoris.kohya" - run_cmd += f' --network_args "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "algo=locon"' + run_cmd += f' --network_args "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "algo=locon"' if LoRA_type == "LyCORIS/LoHa": try: @@ -738,7 +741,7 @@ def train_model( ) return run_cmd += f" --network_module=lycoris.kohya" - run_cmd += f' --network_args "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "use_cp={use_cp}" "algo=loha"' + run_cmd += f' --network_args "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "use_cp={use_cp}" "algo=loha"' # This is a hack to fix a train_network LoHA logic issue if not network_dropout > 0.0: run_cmd += f' --network_dropout="{network_dropout}"' @@ -752,7 +755,7 @@ def train_model( ) return run_cmd += f" --network_module=lycoris.kohya" - run_cmd += f' --network_args "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "train_on_input={train_on_input}" "algo=ia3"' + run_cmd += f' --network_args "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "train_on_input={train_on_input}" "algo=ia3"' # This is a hack to fix a train_network LoHA logic issue if not network_dropout > 0.0: run_cmd += f' --network_dropout="{network_dropout}"' @@ -766,7 +769,7 @@ def train_model( ) return run_cmd += f" --network_module=lycoris.kohya" - run_cmd += f' --network_args "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "use_cp={use_cp}" "block_size={unit}" "algo=dylora"' + run_cmd += f' --network_args "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "use_cp={use_cp}" "block_size={unit}" "algo=dylora"' # This is a hack to fix a train_network LoHA logic issue if not network_dropout > 0.0: run_cmd += f' --network_dropout="{network_dropout}"' @@ -780,10 +783,39 @@ def train_model( ) return run_cmd += f" --network_module=lycoris.kohya" - run_cmd += f' --network_args "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "factor={factor}" "use_cp={use_cp}" "algo=lokr"' + run_cmd += f' --network_args "preset={LyCORIS_preset}" "conv_dim={conv_dim}" "conv_alpha={conv_alpha}" "factor={factor}" "use_tucker={use_cp}" "algo=lokr"' # This is a hack to fix a train_network LoHA logic issue if not network_dropout > 0.0: run_cmd += f' --network_dropout="{network_dropout}"' + + if LoRA_type == "LyCORIS/Native Fine-Tuning": + try: + import lycoris + except ModuleNotFoundError: + log.info( + "\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program." + ) + return + run_cmd += f" --network_module=lycoris.kohya" + run_cmd += f' --network_args "preset={LyCORIS_preset}" "algo=full" "train_norm=True"' + # This is a hack to fix a train_network LoHA logic issue + if not network_dropout > 0.0: + run_cmd += f' --network_dropout="{network_dropout}"' + + if LoRA_type == "LyCORIS/Diag-OFT": + try: + import lycoris + except ModuleNotFoundError: + log.info( + "\033[1;31mError:\033[0m The required module 'lycoris_lora' is not installed. Please install by running \033[33mupgrade.ps1\033[0m before running this program." + ) + return + run_cmd += f" --network_module=lycoris.kohya" + run_cmd += f' --network_args "preset={LyCORIS_preset}" "algo=diag-oft" ' + # This is a hack to fix a train_network LoHA logic issue + if not network_dropout > 0.0: + run_cmd += f' --network_dropout="{network_dropout}"' + if LoRA_type in ["Kohya LoCon", "Standard"]: kohya_lora_var_list = [ @@ -1106,13 +1138,29 @@ def list_presets(path): "LoRA-FA", "LyCORIS/DyLoRA", "LyCORIS/iA3", + "LyCORIS/Diag-OFT", "LyCORIS/LoCon", "LyCORIS/LoHa", "LyCORIS/LoKr", + "LyCORIS/Native Fine-Tuning", "Standard", ], value="Standard", ) + LyCORIS_preset = gr.Dropdown( + label="LyCORIS Preset", + choices=[ + "attn-mlp", + "attn-only", + "full", + "full-lin", + "unet-transformer-only", + "unet-convblock-only", + ], + value="full", + visible=False, interactive=True + # info="https://github.com/KohakuBlueleaf/LyCORIS/blob/0006e2ffa05a48d8818112d9f70da74c0cd30b99/docs/Preset.md" + ) with gr.Box(): with gr.Row(): lora_network_weights = gr.Textbox( @@ -1187,7 +1235,7 @@ def list_presets(path): with gr.Row() as LoRA_dim_alpha: network_dim = gr.Slider( minimum=1, - maximum=1024, + maximum=100000, # 512 if not LoRA_type == "LyCORIS/LoKr" else 100000, label="Network Rank (Dimension)", value=8, step=1, @@ -1206,7 +1254,7 @@ def list_presets(path): # locon= gr.Checkbox(label='Train a LoCon instead of a general LoRA (does not support v2 base models) (may not be able to some utilities now)', value=False) conv_dim = gr.Slider( minimum=0, - maximum=512, + maximum=100000, # 512 if not LoRA_type == "LyCORIS/LoKr" else 100000, value=1, step=1, label="Convolution Rank (Dimension)", @@ -1272,6 +1320,7 @@ def update_LoRA_settings(LoRA_type): "Kohya DyLoRA", "Kohya LoCon", "LoRA-FA", + "LyCORIS/Diag-OFT", "LyCORIS/DyLoRA", "LyCORIS/LoCon", "LyCORIS/LoHa", @@ -1286,6 +1335,7 @@ def update_LoRA_settings(LoRA_type): "Kohya DyLoRA", "Kohya LoCon", "LoRA-FA", + "LyCORIS/Diag-OFT", "LyCORIS/DyLoRA", "LyCORIS/LoHa", "LyCORIS/LoKr", @@ -1313,6 +1363,7 @@ def update_LoRA_settings(LoRA_type): "Kohya DyLoRA", "Kohya LoCon", "LoRA-FA", + "LyCORIS/Diag-OFT", "LyCORIS/DyLoRA", "LyCORIS/LoHa", "LyCORIS/LoCon", @@ -1327,6 +1378,7 @@ def update_LoRA_settings(LoRA_type): "Kohya DyLoRA", "Kohya LoCon", "LoRA-FA", + "LyCORIS/Diag-OFT", "LyCORIS/DyLoRA", "LyCORIS/LoHa", "LyCORIS/LoCon", @@ -1341,6 +1393,7 @@ def update_LoRA_settings(LoRA_type): "Kohya DyLoRA", "Kohya LoCon", "LoRA-FA", + "LyCORIS/Diag-OFT", "LyCORIS/DyLoRA", "LyCORIS/LoHa", "LyCORIS/LoCon", @@ -1380,6 +1433,7 @@ def update_LoRA_settings(LoRA_type): "Kohya DyLoRA", "Kohya LoCon", "LoRA-FA", + "LyCORIS/Diag-OFT", "LyCORIS/DyLoRA", "LyCORIS/LoHa", "LyCORIS/LoCon", @@ -1408,6 +1462,18 @@ def update_LoRA_settings(LoRA_type): }, gr.Slider, ), + "LyCORIS_preset": ( + { + "LyCORIS/DyLoRA", + "LyCORIS/iA3", + "LyCORIS/Diag-OFT", + "LyCORIS/LoCon", + "LyCORIS/LoHa", + "LyCORIS/LoKr", + "LyCORIS/Native Fine-Tuning", + }, + gr.Dropdown, + ) } results = [] @@ -1498,6 +1564,7 @@ def update_LoRA_settings(LoRA_type): network_dropout, rank_dropout, module_dropout, + LyCORIS_preset, ], ) @@ -1650,6 +1717,7 @@ def update_LoRA_settings(LoRA_type): advanced_training.min_timestep, advanced_training.max_timestep, advanced_training.vae, + LyCORIS_preset, advanced_training.debiased_estimation_loss, ] diff --git a/requirements.txt b/requirements.txt index 216e96147..ffc2e3880 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ huggingface-hub==0.15.1 # for loading Diffusers' SDXL invisible-watermark==0.2.0 lion-pytorch==0.0.6 -lycoris_lora==1.9.0 +lycoris_lora==2.0.0 # for BLIP captioning # requests==2.28.2 # timm==0.6.12 diff --git a/tools/lycoris_locon_extract.py b/tools/lycoris_locon_extract.py index 671aa5de8..28b25eec5 100644 --- a/tools/lycoris_locon_extract.py +++ b/tools/lycoris_locon_extract.py @@ -1,4 +1,5 @@ import os, sys + sys.path.insert(0, os.getcwd()) import argparse @@ -6,87 +7,125 @@ def get_args(): parser = argparse.ArgumentParser() parser.add_argument( - "base_model", help="The model which use it to train the dreambooth model", - default='', type=str + "base_model", + help="The model which use it to train the dreambooth model", + default="", + type=str, + ) + parser.add_argument( + "db_model", + help="the dreambooth model you want to extract the locon", + default="", + type=str, ) parser.add_argument( - "db_model", help="the dreambooth model you want to extract the locon", - default='', type=str + "output_name", help="the output model", default="./out.pt", type=str ) parser.add_argument( - "output_name", help="the output model", - default='./out.pt', type=str + "--is_v2", + help="Your base/db model is sd v2 or not", + default=False, + action="store_true", ) parser.add_argument( - "--is_v2", help="Your base/db model is sd v2 or not", - default=False, action="store_true" + "--is_sdxl", + help="Your base/db model is sdxl or not", + default=False, + action="store_true", ) parser.add_argument( - "--device", help="Which device you want to use to extract the locon", - default='cpu', type=str + "--device", + help="Which device you want to use to extract the locon", + default="cpu", + type=str, ) parser.add_argument( - "--mode", + "--mode", help=( - 'extraction mode, can be "fixed", "threshold", "ratio", "quantile". ' + 'extraction mode, can be "full", "fixed", "threshold", "ratio", "quantile". ' 'If not "fixed", network_dim and conv_dim will be ignored' ), - default='fixed', type=str + default="fixed", + type=str, ) parser.add_argument( - "--safetensors", help='use safetensors to save locon model', - default=False, action="store_true" + "--safetensors", + help="use safetensors to save locon model", + default=False, + action="store_true", ) parser.add_argument( - "--linear_dim", help="network dim for linear layer in fixed mode", - default=1, type=int + "--linear_dim", + help="network dim for linear layer in fixed mode", + default=1, + type=int, ) parser.add_argument( - "--conv_dim", help="network dim for conv layer in fixed mode", - default=1, type=int + "--conv_dim", + help="network dim for conv layer in fixed mode", + default=1, + type=int, ) parser.add_argument( - "--linear_threshold", help="singular value threshold for linear layer in threshold mode", - default=0., type=float + "--linear_threshold", + help="singular value threshold for linear layer in threshold mode", + default=0.0, + type=float, ) parser.add_argument( - "--conv_threshold", help="singular value threshold for conv layer in threshold mode", - default=0., type=float + "--conv_threshold", + help="singular value threshold for conv layer in threshold mode", + default=0.0, + type=float, ) parser.add_argument( - "--linear_ratio", help="singular ratio for linear layer in ratio mode", - default=0., type=float + "--linear_ratio", + help="singular ratio for linear layer in ratio mode", + default=0.0, + type=float, ) parser.add_argument( - "--conv_ratio", help="singular ratio for conv layer in ratio mode", - default=0., type=float + "--conv_ratio", + help="singular ratio for conv layer in ratio mode", + default=0.0, + type=float, ) parser.add_argument( - "--linear_quantile", help="singular value quantile for linear layer quantile mode", - default=1., type=float + "--linear_quantile", + help="singular value quantile for linear layer quantile mode", + default=1.0, + type=float, ) parser.add_argument( - "--conv_quantile", help="singular value quantile for conv layer quantile mode", - default=1., type=float + "--conv_quantile", + help="singular value quantile for conv layer quantile mode", + default=1.0, + type=float, ) parser.add_argument( - "--use_sparse_bias", help="enable sparse bias", - default=False, action="store_true" + "--use_sparse_bias", + help="enable sparse bias", + default=False, + action="store_true", ) parser.add_argument( - "--sparsity", help="sparsity for sparse bias", - default=0.98, type=float + "--sparsity", help="sparsity for sparse bias", default=0.98, type=float ) parser.add_argument( - "--disable_cp", help="don't use cp decomposition", - default=False, action="store_true" + "--disable_cp", + help="don't use cp decomposition", + default=False, + action="store_true", ) return parser.parse_args() + + ARGS = get_args() from lycoris.utils import extract_diff from lycoris.kohya.model_utils import load_models_from_stable_diffusion_checkpoint +from lycoris.kohya.sdxl_model_util import load_models_from_sdxl_checkpoint import torch from safetensors.torch import save_file @@ -94,36 +133,58 @@ def get_args(): def main(): args = ARGS - base = load_models_from_stable_diffusion_checkpoint(args.is_v2, args.base_model) - db = load_models_from_stable_diffusion_checkpoint(args.is_v2, args.db_model) - + if args.is_sdxl: + base = load_models_from_sdxl_checkpoint(None, args.base_model, args.device) + db = load_models_from_sdxl_checkpoint(None, args.db_model, args.device) + else: + base = load_models_from_stable_diffusion_checkpoint(args.is_v2, args.base_model) + db = load_models_from_stable_diffusion_checkpoint(args.is_v2, args.db_model) + linear_mode_param = { - 'fixed': args.linear_dim, - 'threshold': args.linear_threshold, - 'ratio': args.linear_ratio, - 'quantile': args.linear_quantile, + "fixed": args.linear_dim, + "threshold": args.linear_threshold, + "ratio": args.linear_ratio, + "quantile": args.linear_quantile, + "full": None, }[args.mode] conv_mode_param = { - 'fixed': args.conv_dim, - 'threshold': args.conv_threshold, - 'ratio': args.conv_ratio, - 'quantile': args.conv_quantile, + "fixed": args.conv_dim, + "threshold": args.conv_threshold, + "ratio": args.conv_ratio, + "quantile": args.conv_quantile, + "full": None, }[args.mode] - + + if args.is_sdxl: + db_tes = [db[0], db[1]] + db_unet = db[3] + base_tes = [base[0], base[1]] + base_unet = base[3] + else: + db_tes = [db[0]] + db_unet = db[2] + base_tes = [base[0]] + base_unet = base[2] + state_dict = extract_diff( - base, db, + base_tes, + db_tes, + base_unet, + db_unet, args.mode, - linear_mode_param, conv_mode_param, - args.device, - args.use_sparse_bias, args.sparsity, - not args.disable_cp + linear_mode_param, + conv_mode_param, + args.device, + args.use_sparse_bias, + args.sparsity, + not args.disable_cp, ) - + if args.safetensors: save_file(state_dict, args.output_name) else: torch.save(state_dict, args.output_name) -if __name__ == '__main__': - main() \ No newline at end of file +if __name__ == "__main__": + main() diff --git a/tools/merge_lycoris.py b/tools/merge_lycoris.py index b29c8dc6b..25f673de2 100644 --- a/tools/merge_lycoris.py +++ b/tools/merge_lycoris.py @@ -1,4 +1,5 @@ import os, sys + sys.path.insert(0, os.getcwd()) import argparse @@ -6,80 +7,119 @@ def get_args(): parser = argparse.ArgumentParser() parser.add_argument( - "base_model", help="The model you want to merge with loha", - default='', type=str + "base_model", help="The model you want to merge with loha", default="", type=str ) parser.add_argument( - "lycoris_model", help="the lyco model you want to merge into sd model", - default='', type=str + "lycoris_model", + help="the lyco model you want to merge into sd model", + default="", + type=str, ) parser.add_argument( - "output_name", help="the output model", - default='./out.pt', type=str + "output_name", help="the output model", default="./out.pt", type=str ) parser.add_argument( - "--is_v2", help="Your base model is sd v2 or not", - default=False, action="store_true" + "--is_v2", + help="Your base model is sd v2 or not", + default=False, + action="store_true", ) parser.add_argument( - "--device", help="Which device you want to use to merge the weight", - default='cpu', type=str + "--is_sdxl", + help="Your base/db model is sdxl or not", + default=False, + action="store_true", ) parser.add_argument( - "--dtype", help='dtype to save', - default='float', type=str + "--device", + help="Which device you want to use to merge the weight", + default="cpu", + type=str, ) + parser.add_argument("--dtype", help="dtype to save", default="float", type=str) parser.add_argument( - "--weight", help='weight for the lyco model to merge', - default='1.0', type=float + "--weight", help="weight for the lyco model to merge", default="1.0", type=float ) return parser.parse_args() -ARGS = get_args() + + +args = ARGS = get_args() from lycoris.utils import merge from lycoris.kohya.model_utils import ( load_models_from_stable_diffusion_checkpoint, save_stable_diffusion_checkpoint, - load_file + load_file, +) +from lycoris.kohya.sdxl_model_util import ( + load_models_from_sdxl_checkpoint, + save_stable_diffusion_checkpoint as save_sdxl_checkpoint, ) import torch +@torch.no_grad() def main(): - base = load_models_from_stable_diffusion_checkpoint(ARGS.is_v2, ARGS.base_model) - if ARGS.lycoris_model.rsplit('.', 1)[-1] == 'safetensors': + if args.is_sdxl: + base = load_models_from_sdxl_checkpoint( + None, args.base_model, map_location=args.device + ) + else: + base = load_models_from_stable_diffusion_checkpoint(args.is_v2, args.base_model) + if ARGS.lycoris_model.rsplit(".", 1)[-1] == "safetensors": lyco = load_file(ARGS.lycoris_model) else: lyco = torch.load(ARGS.lycoris_model) - - dtype_str = ARGS.dtype.replace('fp', 'float').replace('bf', 'bfloat') + + dtype_str = ARGS.dtype.replace("fp", "float").replace("bf", "bfloat") dtype = { - 'float': torch.float, - 'float16': torch.float16, - 'float32': torch.float32, - 'float64': torch.float64, - 'bfloat': torch.bfloat16, - 'bfloat16': torch.bfloat16, + "float": torch.float, + "float16": torch.float16, + "float32": torch.float32, + "float64": torch.float64, + "bfloat": torch.bfloat16, + "bfloat16": torch.bfloat16, }.get(dtype_str, None) if dtype is None: raise ValueError(f'Cannot Find the dtype "{dtype}"') - - merge( - base, - lyco, - ARGS.weight, - ARGS.device - ) - - save_stable_diffusion_checkpoint( - ARGS.is_v2, ARGS.output_name, - base[0], base[2], - None, 0, 0, dtype, - base[1] - ) + + if args.is_sdxl: + base_tes = [base[0], base[1]] + base_unet = base[3] + else: + base_tes = [base[0]] + base_unet = base[2] + + merge(base_tes, base_unet, lyco, ARGS.weight, ARGS.device) + + if args.is_sdxl: + save_sdxl_checkpoint( + ARGS.output_name, + base[0].cpu(), + base[1].cpu(), + base[3].cpu(), + 0, + 0, + None, + base[2], + getattr(base[1], "logit_scale", None), + dtype, + ) + else: + save_stable_diffusion_checkpoint( + ARGS.is_v2, + ARGS.output_name, + base[0].cpu(), + base[2].cpu(), + None, + 0, + 0, + dtype, + base[1], + ) -if __name__ == '__main__': - main() \ No newline at end of file +if __name__ == "__main__": + main()