From 1bbb9214a105932f3f4f0a40b68ac6473631d295 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Thu, 5 Oct 2023 21:11:16 +0300 Subject: [PATCH 01/48] initial animatediff args filling --- .../deforum_helpers/deforum_animatediff.py | 74 +++++++++++++++++++ scripts/deforum_helpers/ui_left.py | 1 + 2 files changed, 75 insertions(+) create mode 100644 scripts/deforum_helpers/deforum_animatediff.py diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py new file mode 100644 index 000000000..930758b70 --- /dev/null +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -0,0 +1,74 @@ +# Copyright (C) 2023 Deforum LLC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +# Contact the authors: https://deforum.github.io/ + +# This helper script is responsible for AnimateDiff/Deforum integration +# https://github.com/continue-revolution/sd-webui-animatediff — animatediff repo + +import os +import copy +import gradio as gr +import scripts +from PIL import Image +import numpy as np +import importlib +from modules import scripts, shared +from .deforum_controlnet_gradio import hide_ui_by_cn_status, hide_file_textboxes, ToolButton +from .general_utils import count_files_in_folder, clean_gradio_path_strings # TODO: do it another way +from .video_audio_utilities import vid2frames, convert_image +from .animation_key_frames import AnimateDiffKeys +from .load_images import load_image +from .general_utils import debug_print + +#self.last_frame = last_frame +#self.latent_power_last = latent_power_last +#self.latent_scale_last = latent_scale_last + +cnet = None + +def find_animatediff(): + global cnet + if cnet: return cnet + try: + cnet = importlib.import_module('extensions.sd-webui-animatediff.scripts', 'animatediff') + except: + try: + cnet = importlib.import_module('extensions-builtin.sd-webui-animatediff.scripts', 'animatediff') + except: + pass + if cnet: + print(f"\033[0;32m*Deforum AnimateDiff support: enabled*\033[0m") + return True + return None + +def is_animatediff_enabled(animatediff_args): + if getattr(animatediff_args, f'animatediff_enabled', False): + return True + return False + +def animatediff_infotext(): + return """Requires the AnimateDiff extension to be installed.

+ If Deforum crashes due to AnimateDiff updates, go here and report your problem.

+ """ + +def animatediff_component_names(): + if not find_animatediff(): + return [] + + return [f'animatediff_{i}' for i in [ + 'enabled', 'model', 'video_length', 'fps', 'loop_number', + 'closed_loop', 'stride', 'overlap', 'interp', 'interp_x', 'reverse', + 'latent_power', 'latent_scale', 'threshold_b', 'resize_mode', 'control_mode', 'loopback_mode' + ]] diff --git a/scripts/deforum_helpers/ui_left.py b/scripts/deforum_helpers/ui_left.py index aa1886372..951459c5d 100644 --- a/scripts/deforum_helpers/ui_left.py +++ b/scripts/deforum_helpers/ui_left.py @@ -47,6 +47,7 @@ def setup_deforum_left_side_ui(): tab_keyframes_params = get_tab_keyframes(d, da, dloopArgs) # Keyframes tab tab_prompts_params = get_tab_prompts(da) # Prompts tab tab_init_params = get_tab_init(d, da, dp) # Init tab + controlnet_dict = animate_diff_ui() # AnimateDiff tab controlnet_dict = setup_controlnet_ui() # ControlNet tab tab_hybrid_params = get_tab_hybrid(da) # Hybrid tab tab_output_params = get_tab_output(da, dv) # Output tab From 2fdc3fa0d9df292c412d9684c22db55f8e732941 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Thu, 5 Oct 2023 21:23:03 +0300 Subject: [PATCH 02/48] animatediff ui preparation --- .../deforum_helpers/deforum_animatediff.py | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index 930758b70..0692d2e23 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -72,3 +72,90 @@ def animatediff_component_names(): 'closed_loop', 'stride', 'overlap', 'interp', 'interp_x', 'reverse', 'latent_power', 'latent_scale', 'threshold_b', 'resize_mode', 'control_mode', 'loopback_mode' ]] + +def setup_animatediff_ui_raw(): + + cnet = find_animatediff() + + model_dir = shared.opts.data.get("animatediff_model_path", os.path.join(scripts.basedir(), "model")) + + if not os.path.isdir(model_dir): + os.mkdir(model_dir) + + cn_models = [f for f in os.listdir(model_dir) if f != ".gitkeep"] + + def refresh_all_models(*inputs): + new_model_list = [ + f for f in os.listdir(model_dir) if f != ".gitkeep" + ] + dd = inputs[0] + if dd in new_model_list: + selected = dd + elif len(new_model_list) > 0: + selected = new_model_list[0] + else: + selected = None + return gr.Dropdown.update(choices=new_model_list, value=selected) + + refresh_symbol = '\U0001f504' # 🔄 + switch_values_symbol = '\U000021C5' # ⇅ + model_dropdowns = [] + infotext_fields = [] + + # TODO: unwrap + def create_model_in_tab_ui(cn_id): + with gr.Row(): + enabled = gr.Checkbox(label="Enable", value=False, interactive=True) + pixel_perfect = gr.Checkbox(label="Pixel Perfect", value=False, visible=False, interactive=True) + low_vram = gr.Checkbox(label="Low VRAM", value=False, visible=False, interactive=True) + overwrite_frames = gr.Checkbox(label='Overwrite input frames', value=True, visible=False, interactive=True) + with gr.Row(visible=False) as mod_row: + model = gr.Dropdown(cn_models, label=f"Model", value="None", interactive=True) + refresh_models = ToolButton(value=refresh_symbol) + refresh_models.click(refresh_all_models, model, model) + with gr.Row(visible=False) as weight_row: + weight = gr.Textbox(label="Weight schedule", lines=1, value='0:(1)', interactive=True) + with gr.Row(visible=False) as start_cs_row: + guidance_start = gr.Textbox(label="Starting Control Step schedule", lines=1, value='0:(0.0)', interactive=True) + with gr.Row(visible=False) as end_cs_row: + guidance_end = gr.Textbox(label="Ending Control Step schedule", lines=1, value='0:(1.0)', interactive=True) + model_dropdowns.append(model) + with gr.Column(visible=False) as advanced_column: + processor_res = gr.Slider(label="Annotator resolution", value=64, minimum=64, maximum=2048, interactive=False) + threshold_a = gr.Slider(label="Threshold A", value=64, minimum=64, maximum=1024, interactive=False) + threshold_b = gr.Slider(label="Threshold B", value=64, minimum=64, maximum=1024, interactive=False) + with gr.Row(visible=False) as vid_path_row: + vid_path = gr.Textbox(value='', label="ControlNet Input Video/ Image Path", interactive=True) + with gr.Row(visible=False) as mask_vid_path_row: # invisible temporarily since 26-04-23 until masks are fixed + mask_vid_path = gr.Textbox(value='', label="ControlNet Mask Video/ Image Path (*NOT WORKING, kept in UI for CN's devs testing!*)", interactive=True) + with gr.Row(visible=False) as control_mode_row: + control_mode = gr.Radio(choices=["Balanced", "My prompt is more important", "ControlNet is more important"], value="Balanced", label="Control Mode", interactive=True) + with gr.Row(visible=False) as env_row: + resize_mode = gr.Radio(choices=["Outer Fit (Shrink to Fit)", "Inner Fit (Scale to Fit)", "Just Resize"], value="Inner Fit (Scale to Fit)", label="Resize Mode", interactive=True) + with gr.Row(visible=False) as control_loopback_row: + loopback_mode = gr.Checkbox(label="LoopBack mode", value=False, interactive=True) + hide_output_list = [pixel_perfect, low_vram, mod_row, weight_row, start_cs_row, end_cs_row, env_row, overwrite_frames, vid_path_row, control_mode_row, mask_vid_path_row, + control_loopback_row] # add mask_vid_path_row when masks are working again + for cn_output in hide_output_list: + enabled.change(fn=hide_ui_by_cn_status, inputs=enabled, outputs=cn_output) + + # hide vid/image input fields + loopback_outs = [vid_path_row, mask_vid_path_row] + for loopback_output in loopback_outs: + loopback_mode.change(fn=hide_file_textboxes, inputs=loopback_mode, outputs=loopback_output) + infotext_fields.extend([ + (model, f"AnimateDiff Model"), + ]) + + return {key: value for key, value in locals().items() if key in + animatediff_component_names() + } + + with gr.TabItem('AnimateDiff'): + gr.HTML(animatediff_infotext()) + model_params = create_model_in_tab_ui(0) + + for key, value in model_params.items(): + locals()[f"animatediff_{key}"] = value + + return locals() \ No newline at end of file From 73098172261486371ad0ac6bd5a2a2db6541cf2d Mon Sep 17 00:00:00 2001 From: kabachuha Date: Thu, 5 Oct 2023 22:23:15 +0300 Subject: [PATCH 03/48] add animatediff params to UI --- .../deforum_helpers/deforum_animatediff.py | 54 +++++++------------ 1 file changed, 18 insertions(+), 36 deletions(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index 0692d2e23..b26685192 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -68,9 +68,10 @@ def animatediff_component_names(): return [] return [f'animatediff_{i}' for i in [ - 'enabled', 'model', 'video_length', 'fps', 'loop_number', - 'closed_loop', 'stride', 'overlap', 'interp', 'interp_x', 'reverse', - 'latent_power', 'latent_scale', 'threshold_b', 'resize_mode', 'control_mode', 'loopback_mode' + 'enabled', 'model', 'motion_lora_schedule', + 'window_length', # sliding window length (context batch size) + 'window_overlap', # how much do the contexts overlap. if -1, then batch_size // 4 + 'latent_power', 'latent_scale', ]] def setup_animatediff_ui_raw(): @@ -99,50 +100,31 @@ def refresh_all_models(*inputs): refresh_symbol = '\U0001f504' # 🔄 switch_values_symbol = '\U000021C5' # ⇅ - model_dropdowns = [] infotext_fields = [] # TODO: unwrap def create_model_in_tab_ui(cn_id): with gr.Row(): - enabled = gr.Checkbox(label="Enable", value=False, interactive=True) - pixel_perfect = gr.Checkbox(label="Pixel Perfect", value=False, visible=False, interactive=True) - low_vram = gr.Checkbox(label="Low VRAM", value=False, visible=False, interactive=True) - overwrite_frames = gr.Checkbox(label='Overwrite input frames', value=True, visible=False, interactive=True) + enabled = gr.Checkbox(label="Enable AnimateDiff", value=False, interactive=True) with gr.Row(visible=False) as mod_row: - model = gr.Dropdown(cn_models, label=f"Model", value="None", interactive=True) + model = gr.Dropdown(cn_models, label=f"Motion module", value="None", interactive=True, tooltip="Choose which motion module will be injected into the generation process.") refresh_models = ToolButton(value=refresh_symbol) refresh_models.click(refresh_all_models, model, model) - with gr.Row(visible=False) as weight_row: - weight = gr.Textbox(label="Weight schedule", lines=1, value='0:(1)', interactive=True) - with gr.Row(visible=False) as start_cs_row: - guidance_start = gr.Textbox(label="Starting Control Step schedule", lines=1, value='0:(0.0)', interactive=True) - with gr.Row(visible=False) as end_cs_row: - guidance_end = gr.Textbox(label="Ending Control Step schedule", lines=1, value='0:(1.0)', interactive=True) - model_dropdowns.append(model) - with gr.Column(visible=False) as advanced_column: - processor_res = gr.Slider(label="Annotator resolution", value=64, minimum=64, maximum=2048, interactive=False) - threshold_a = gr.Slider(label="Threshold A", value=64, minimum=64, maximum=1024, interactive=False) - threshold_b = gr.Slider(label="Threshold B", value=64, minimum=64, maximum=1024, interactive=False) - with gr.Row(visible=False) as vid_path_row: - vid_path = gr.Textbox(value='', label="ControlNet Input Video/ Image Path", interactive=True) - with gr.Row(visible=False) as mask_vid_path_row: # invisible temporarily since 26-04-23 until masks are fixed - mask_vid_path = gr.Textbox(value='', label="ControlNet Mask Video/ Image Path (*NOT WORKING, kept in UI for CN's devs testing!*)", interactive=True) - with gr.Row(visible=False) as control_mode_row: - control_mode = gr.Radio(choices=["Balanced", "My prompt is more important", "ControlNet is more important"], value="Balanced", label="Control Mode", interactive=True) - with gr.Row(visible=False) as env_row: - resize_mode = gr.Radio(choices=["Outer Fit (Shrink to Fit)", "Inner Fit (Scale to Fit)", "Just Resize"], value="Inner Fit (Scale to Fit)", label="Resize Mode", interactive=True) - with gr.Row(visible=False) as control_loopback_row: - loopback_mode = gr.Checkbox(label="LoopBack mode", value=False, interactive=True) - hide_output_list = [pixel_perfect, low_vram, mod_row, weight_row, start_cs_row, end_cs_row, env_row, overwrite_frames, vid_path_row, control_mode_row, mask_vid_path_row, - control_loopback_row] # add mask_vid_path_row when masks are working again + with gr.Row(visible=False) as motion_lora_row: + motion_lora_schedule = gr.Textbox(label="Motion lora", lines=1, value='0:("")', interactive=True) + with gr.Row(visible=False) as window_row: + window_length = gr.Textbox(label="Number of sliding window frames", lines=1, value='0:(16)', interactive=True) + with gr.Row(visible=False) as overlap_row: + # TODO: expose cadence as a variable + window_overlap = gr.Textbox(label="Number of overlapping frames", lines=1, value='0:(15)', interactive=True) + with gr.Row(visible=False) as latent_power_row: + latent_power = gr.Textbox(label="Latent power schedule", lines=1, value='0:(0.0)', interactive=True) + with gr.Row(visible=False) as latent_scale_row: + latent_scale = gr.Textbox(label="Latent scale schedule", lines=1, value='0:(1.0)', interactive=True) + hide_output_list = [enabled, motion_lora_row, mod_row, window_row, overlap_row, latent_power_row, latent_scale_row] for cn_output in hide_output_list: enabled.change(fn=hide_ui_by_cn_status, inputs=enabled, outputs=cn_output) - # hide vid/image input fields - loopback_outs = [vid_path_row, mask_vid_path_row] - for loopback_output in loopback_outs: - loopback_mode.change(fn=hide_file_textboxes, inputs=loopback_mode, outputs=loopback_output) infotext_fields.extend([ (model, f"AnimateDiff Model"), ]) From 2deed2a5d9a89fe0c8b6c7c2b535777b0914e32b Mon Sep 17 00:00:00 2001 From: kabachuha Date: Thu, 5 Oct 2023 22:29:37 +0300 Subject: [PATCH 04/48] add animatediff ui setup function --- .../deforum_helpers/deforum_animatediff.py | 21 ++++++++++++++++--- scripts/deforum_helpers/ui_left.py | 5 +++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index b26685192..f015e250d 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -118,9 +118,9 @@ def create_model_in_tab_ui(cn_id): # TODO: expose cadence as a variable window_overlap = gr.Textbox(label="Number of overlapping frames", lines=1, value='0:(15)', interactive=True) with gr.Row(visible=False) as latent_power_row: - latent_power = gr.Textbox(label="Latent power schedule", lines=1, value='0:(0.0)', interactive=True) + latent_power = gr.Textbox(label="Latent power schedule", lines=1, value='0:(1)', interactive=True) with gr.Row(visible=False) as latent_scale_row: - latent_scale = gr.Textbox(label="Latent scale schedule", lines=1, value='0:(1.0)', interactive=True) + latent_scale = gr.Textbox(label="Latent scale schedule", lines=1, value='0:(32)', interactive=True) hide_output_list = [enabled, motion_lora_row, mod_row, window_row, overlap_row, latent_power_row, latent_scale_row] for cn_output in hide_output_list: enabled.change(fn=hide_ui_by_cn_status, inputs=enabled, outputs=cn_output) @@ -140,4 +140,19 @@ def create_model_in_tab_ui(cn_id): for key, value in model_params.items(): locals()[f"animatediff_{key}"] = value - return locals() \ No newline at end of file + return locals() + +def setup_animatediff_ui(): + if not find_animatediff(): + gr.HTML("""AnimateDiff not found. Please install it :)""", elem_id='animatediff_not_found_html_msg') + return {} + + try: + return setup_animatediff_ui_raw() + except Exception as e: + print(f"'AnimateDiff UI setup failed with error: '{e}'!") + gr.HTML(f""" + Failed to setup AnimateDiff UI, check the reason in your commandline log. Please, downgrade your AnimateDiff extension and report the problem here (Deforum) or here (AnimateDiff). + """, elem_id='animatediff_not_found_html_msg') + return {} + \ No newline at end of file diff --git a/scripts/deforum_helpers/ui_left.py b/scripts/deforum_helpers/ui_left.py index 951459c5d..57b4df03a 100644 --- a/scripts/deforum_helpers/ui_left.py +++ b/scripts/deforum_helpers/ui_left.py @@ -20,6 +20,7 @@ from .gradio_funcs import change_css, handle_change_functions from .args import DeforumArgs, DeforumAnimArgs, ParseqArgs, DeforumOutputArgs, RootArgs, LoopArgs from .deforum_controlnet import setup_controlnet_ui +from .deforum_animatediff import setup_animatediff_ui from .ui_elements import get_tab_run, get_tab_keyframes, get_tab_prompts, get_tab_init, get_tab_hybrid, get_tab_output def set_arg_lists(): @@ -47,12 +48,12 @@ def setup_deforum_left_side_ui(): tab_keyframes_params = get_tab_keyframes(d, da, dloopArgs) # Keyframes tab tab_prompts_params = get_tab_prompts(da) # Prompts tab tab_init_params = get_tab_init(d, da, dp) # Init tab - controlnet_dict = animate_diff_ui() # AnimateDiff tab + animatediff_dict = setup_animatediff_ui() # AnimateDiff tab controlnet_dict = setup_controlnet_ui() # ControlNet tab tab_hybrid_params = get_tab_hybrid(da) # Hybrid tab tab_output_params = get_tab_output(da, dv) # Output tab # add returned gradio elements from main tabs to locals() - for key, value in {**tab_run_params, **tab_keyframes_params, **tab_prompts_params, **tab_init_params, **controlnet_dict, **tab_hybrid_params, **tab_output_params}.items(): + for key, value in {**tab_run_params, **tab_keyframes_params, **tab_prompts_params, **tab_init_params, **animatediff_dict, **controlnet_dict, **tab_hybrid_params, **tab_output_params}.items(): locals()[key] = value # Gradio's Change functions - hiding and renaming elements based on other elements From d52d372146d6d6a1abb892c819d2c4eaa965e610 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Thu, 5 Oct 2023 22:38:39 +0300 Subject: [PATCH 05/48] comment out import for now --- scripts/deforum_helpers/deforum_animatediff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index f015e250d..dbcf42055 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -28,7 +28,7 @@ from .deforum_controlnet_gradio import hide_ui_by_cn_status, hide_file_textboxes, ToolButton from .general_utils import count_files_in_folder, clean_gradio_path_strings # TODO: do it another way from .video_audio_utilities import vid2frames, convert_image -from .animation_key_frames import AnimateDiffKeys +#from .animation_key_frames import AnimateDiffKeys from .load_images import load_image from .general_utils import debug_print From 69e65086f658f11963da9c38549222ae6e4115f9 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Thu, 5 Oct 2023 22:43:19 +0300 Subject: [PATCH 06/48] fixup args order --- scripts/deforum_helpers/args.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/deforum_helpers/args.py b/scripts/deforum_helpers/args.py index e85c2a2a8..62847509c 100644 --- a/scripts/deforum_helpers/args.py +++ b/scripts/deforum_helpers/args.py @@ -23,6 +23,7 @@ import modules.shared as sh from modules.processing import get_fixed_seed from .defaults import get_guided_imgs_default_json, mask_fill_choices, get_samplers_list +from .deforum_animatediff import animatediff_component_names from .deforum_controlnet import controlnet_component_names from .general_utils import get_os, substitute_placeholders @@ -1118,7 +1119,7 @@ def DeforumOutputArgs(): def get_component_names(): return ['override_settings_with_file', 'custom_settings_file', *DeforumAnimArgs().keys(), 'animation_prompts', 'animation_prompts_positive', 'animation_prompts_negative', - *DeforumArgs().keys(), *DeforumOutputArgs().keys(), *ParseqArgs().keys(), *LoopArgs().keys(), *controlnet_component_names()] + *DeforumArgs().keys(), *DeforumOutputArgs().keys(), *ParseqArgs().keys(), *LoopArgs().keys(), *animatediff_component_names(), *controlnet_component_names()] def get_settings_component_names(): return [name for name in get_component_names()] @@ -1138,6 +1139,7 @@ def process_args(args_dict_main, run_id): video_args = SimpleNamespace(**{name: args_dict_main[name] for name in DeforumOutputArgs()}) parseq_args = SimpleNamespace(**{name: args_dict_main[name] for name in ParseqArgs()}) loop_args = SimpleNamespace(**{name: args_dict_main[name] for name in LoopArgs()}) + animatediff_args = SimpleNamespace(**{name: args_dict_main[name] for name in animatediff_component_names()}) controlnet_args = SimpleNamespace(**{name: args_dict_main[name] for name in controlnet_component_names()}) root.animation_prompts = json.loads(args_dict_main['animation_prompts']) From df1e699c95bb53528fb82b2b1b982d9a27332a1a Mon Sep 17 00:00:00 2001 From: kabachuha Date: Thu, 5 Oct 2023 22:58:32 +0300 Subject: [PATCH 07/48] fixup ui args locating --- scripts/deforum_helpers/deforum_animatediff.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index dbcf42055..3947b36b7 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -63,16 +63,19 @@ def animatediff_infotext(): If Deforum crashes due to AnimateDiff updates, go here and report your problem.

""" -def animatediff_component_names(): - if not find_animatediff(): - return [] - - return [f'animatediff_{i}' for i in [ +def animatediff_component_names_raw(): + return [ 'enabled', 'model', 'motion_lora_schedule', 'window_length', # sliding window length (context batch size) 'window_overlap', # how much do the contexts overlap. if -1, then batch_size // 4 'latent_power', 'latent_scale', - ]] + ] + +def animatediff_component_names(): + if not find_animatediff(): + return [] + + return [f'animatediff_{i}' for i in animatediff_component_names_raw()] def setup_animatediff_ui_raw(): @@ -130,7 +133,7 @@ def create_model_in_tab_ui(cn_id): ]) return {key: value for key, value in locals().items() if key in - animatediff_component_names() + animatediff_component_names_raw() } with gr.TabItem('AnimateDiff'): From e2bfd0201a602b3c4f5e4d8fefd7e01cc8cae8ea Mon Sep 17 00:00:00 2001 From: kabachuha Date: Thu, 5 Oct 2023 23:00:46 +0300 Subject: [PATCH 08/48] add "schedule" to motion lora param name --- scripts/deforum_helpers/deforum_animatediff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index 3947b36b7..23991da36 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -114,7 +114,7 @@ def create_model_in_tab_ui(cn_id): refresh_models = ToolButton(value=refresh_symbol) refresh_models.click(refresh_all_models, model, model) with gr.Row(visible=False) as motion_lora_row: - motion_lora_schedule = gr.Textbox(label="Motion lora", lines=1, value='0:("")', interactive=True) + motion_lora_schedule = gr.Textbox(label="Motion lora schedule", lines=1, value='0:("")', interactive=True) with gr.Row(visible=False) as window_row: window_length = gr.Textbox(label="Number of sliding window frames", lines=1, value='0:(16)', interactive=True) with gr.Row(visible=False) as overlap_row: From c5bff7c37a6f15c7229204ea1179146c5b088305 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Fri, 6 Oct 2023 00:18:48 +0300 Subject: [PATCH 09/48] wip progress with animatediff --- scripts/deforum_helpers/args.py | 4 +- .../deforum_helpers/deforum_animatediff.py | 66 ++++++++++++++++++- scripts/deforum_helpers/render.py | 8 ++- scripts/deforum_helpers/render_modes.py | 10 +-- scripts/deforum_helpers/run_deforum.py | 12 ++-- 5 files changed, 83 insertions(+), 17 deletions(-) diff --git a/scripts/deforum_helpers/args.py b/scripts/deforum_helpers/args.py index 62847509c..cb2750f2e 100644 --- a/scripts/deforum_helpers/args.py +++ b/scripts/deforum_helpers/args.py @@ -1146,7 +1146,7 @@ def process_args(args_dict_main, run_id): args_loaded_ok = True if override_settings_with_file: - args_loaded_ok = load_args(args_dict_main, args, anim_args, parseq_args, loop_args, controlnet_args, video_args, custom_settings_file, root, run_id) + args_loaded_ok = load_args(args_dict_main, args, anim_args, parseq_args, loop_args, animatediff_args, controlnet_args, video_args, custom_settings_file, root, run_id) positive_prompts = args_dict_main['animation_prompts_positive'] negative_prompts = args_dict_main['animation_prompts_negative'] @@ -1185,4 +1185,4 @@ def process_args(args_dict_main, run_id): default_img = default_img.resize((args.W,args.H)) root.default_img = default_img - return args_loaded_ok, root, args, anim_args, video_args, parseq_args, loop_args, controlnet_args + return args_loaded_ok, root, args, anim_args, video_args, parseq_args, loop_args, animatediff_args, controlnet_args diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index 23991da36..a311c8873 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -31,6 +31,9 @@ #from .animation_key_frames import AnimateDiffKeys from .load_images import load_image from .general_utils import debug_print +from modules.shared import opts, cmd_opts, state, sd_model + +import modules.paths as ph #self.last_frame = last_frame #self.latent_power_last = latent_power_last @@ -81,7 +84,7 @@ def setup_animatediff_ui_raw(): cnet = find_animatediff() - model_dir = shared.opts.data.get("animatediff_model_path", os.path.join(scripts.basedir(), "model")) + model_dir = ph.models_path + '/AnimateDiff' # shared.opts.data.get("animatediff_model_path", os.path.join(scripts.basedir(), "model")) if not os.path.isdir(model_dir): os.mkdir(model_dir) @@ -155,7 +158,64 @@ def setup_animatediff_ui(): except Exception as e: print(f"'AnimateDiff UI setup failed with error: '{e}'!") gr.HTML(f""" - Failed to setup AnimateDiff UI, check the reason in your commandline log. Please, downgrade your AnimateDiff extension and report the problem here (Deforum) or here (AnimateDiff). + Failed to setup AnimateDiff UI, check the reason in your commandline log. Please, downgrade your AnimateDiff extension to b192a2551a5ed66d4a3ce58d5d19a8872abc87ca and report the problem here (Deforum) or here (AnimateDiff). """, elem_id='animatediff_not_found_html_msg') return {} - \ No newline at end of file + +lora_hacker = None +cfg_hacker = None +cn_hacker = None + +# HACK: writing temp frames to a dir for animatediff processing +# TODO: +def write_temp_frames(): + #tmp_frame_dir = Path(f'{data_path}/tmp/animatediff-frames/') + #tmp_frame_dir.mkdir(parents=True, exist_ok=True) + ... + +def before_process(p, animatediff_args): + global lora_hacker, cfg_hacker, cn_hacker + + from scripts.animatediff_lora import AnimateDiffLora + from scripts.animatediff_infv2v import AnimateDiffInfV2V + from scripts.animatediff_cn import AnimateDiffControl + + from scripts.animatediff_mm import mm_animatediff as motion_module + + set_p(p, animatediff_args) + motion_module.inject(p.sd_model, animatediff_args.animatediff_model) + lora_hacker = AnimateDiffLora(motion_module.mm.using_v2) + lora_hacker.hack() + cfg_hacker = AnimateDiffInfV2V(p) + cfg_hacker.hack(animatediff_args) + cn_hacker = AnimateDiffControl(p) + cn_hacker.hack(animatediff_args) + +def postprocess( + p, animatediff_args + ): + global lora_hacker, cfg_hacker, cn_hacker + cn_hacker.restore() + cfg_hacker.restore() + lora_hacker.restore() + from scripts.animatediff_mm import mm_animatediff as motion_module + motion_module.restore(p.sd_model) + +def animatediff_process_from_args(animatediff_args): + args = animatediff_args + from scripts.animatediff_ui import AnimateDiffProcess + + return AnimateDiffProcess( + model = args.animatediff_model, + enable = args.animatediff_enabled, + video_length = args.animatediff_enabled, + ... + # FIXME + ) + +#def set_p(p, animatediff_args): +# if animatediff_args.animatediff_window_length < 1: +# return +# p.batch_size = animatediff_args.animatediff_window_length +# if animatediff_args.animatediff_window_overlap == -1: +# animatediff_args.animatediff_window_overlap = animatediff_args.animatediff_window_length // 4 diff --git a/scripts/deforum_helpers/render.py b/scripts/deforum_helpers/render.py index 964498c3a..2d5c2e4f8 100644 --- a/scripts/deforum_helpers/render.py +++ b/scripts/deforum_helpers/render.py @@ -52,11 +52,17 @@ from deforum_api import JobStatusTracker -def render_animation(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, root): +def render_animation(args, anim_args, video_args, parseq_args, loop_args, animatediff_args, controlnet_args, root): # initialise Parseq adapter + # TODO: @rewbs parseq_adapter = ParseqAdapter(parseq_args, anim_args, video_args, controlnet_args, loop_args) + if animatediff_args.enabled: + print("Rendering with AnimateDiff on.") + + + if opts.data.get("deforum_save_gen_info_as_srt", False): # create .srt file and set timeframe mechanism using FPS srt_filename = os.path.join(args.outdir, f"{root.timestring}.srt") srt_frame_duration = init_srt_file(srt_filename, video_args.fps) diff --git a/scripts/deforum_helpers/render_modes.py b/scripts/deforum_helpers/render_modes.py index 46730c991..49ddf7900 100644 --- a/scripts/deforum_helpers/render_modes.py +++ b/scripts/deforum_helpers/render_modes.py @@ -30,7 +30,7 @@ from .save_images import save_image from .settings import save_settings_from_animation_run -def render_input_video(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, root): +def render_input_video(args, anim_args, video_args, parseq_args, loop_args, animatediff_args, controlnet_args, root): # create a folder for the video input frames to live in video_in_frame_path = os.path.join(args.outdir, 'inputframes') os.makedirs(video_in_frame_path, exist_ok=True) @@ -61,10 +61,10 @@ def render_input_video(args, anim_args, video_args, parseq_args, loop_args, cont args.use_mask = True args.overlay_mask = True - render_animation(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, root) + render_animation(args, anim_args, video_args, parseq_args, loop_args, animatediff_args, controlnet_args, root) # Modified a copy of the above to allow using masking video with out a init video. -def render_animation_with_video_mask(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, root): +def render_animation_with_video_mask(args, anim_args, video_args, parseq_args, loop_args, animatediff_args, controlnet_args, root): # create a folder for the video input frames to live in mask_in_frame_path = os.path.join(args.outdir, 'maskframes') os.makedirs(mask_in_frame_path, exist_ok=True) @@ -80,7 +80,7 @@ def render_animation_with_video_mask(args, anim_args, video_args, parseq_args, l #args.use_init = True print(f"Loading {anim_args.max_frames} input frames from {mask_in_frame_path} and saving video frames to {args.outdir}") - render_animation(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, root) + render_animation(args, anim_args, video_args, parseq_args, loop_args, animatediff_args, controlnet_args, root) def get_parsed_value(value, frame_idx, max_f): pattern = r'`.*?`' @@ -93,7 +93,7 @@ def get_parsed_value(value, frame_idx, max_f): parsed_value = parsed_value.replace(matched_string, str(value)) return parsed_value -def render_interpolation(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, root): +def render_interpolation(args, anim_args, video_args, parseq_args, loop_args, animatediff_args, controlnet_args, root): # use parseq if manifest is provided parseq_adapter = ParseqAdapter(parseq_args, anim_args, video_args, controlnet_args, loop_args) diff --git a/scripts/deforum_helpers/run_deforum.py b/scripts/deforum_helpers/run_deforum.py index ca25d152b..5065c81eb 100644 --- a/scripts/deforum_helpers/run_deforum.py +++ b/scripts/deforum_helpers/run_deforum.py @@ -71,7 +71,7 @@ def run_deforum(*args): args_dict['self'] = None args_dict['p'] = p try: - args_loaded_ok, root, args, anim_args, video_args, parseq_args, loop_args, controlnet_args = process_args(args_dict, i) + args_loaded_ok, root, args, anim_args, video_args, parseq_args, loop_args, animatediff_args, controlnet_args = process_args(args_dict, i) except Exception as e: JobStatusTracker().fail_job(job_id, error_type="TERMINAL", message="Invalid arguments.") print("\n*START OF TRACEBACK*") @@ -111,13 +111,13 @@ def run_deforum(*args): JobStatusTracker().update_output_info(job_id, outdir=args.outdir, timestring=root.timestring) if anim_args.animation_mode == '2D' or anim_args.animation_mode == '3D': if anim_args.use_mask_video: - render_animation_with_video_mask(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, root) # allow mask video without an input video + render_animation_with_video_mask(args, anim_args, video_args, parseq_args, loop_args, animatediff_args, controlnet_args, root) # allow mask video without an input video else: - render_animation(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, root) + render_animation(args, anim_args, video_args, parseq_args, loop_args, animatediff_args, controlnet_args, root) elif anim_args.animation_mode == 'Video Input': - render_input_video(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, root)#TODO: prettify code + render_input_video(args, anim_args, video_args, parseq_args, loop_args, animatediff_args, controlnet_args, root)#TODO: prettify code elif anim_args.animation_mode == 'Interpolation': - render_interpolation(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, root) + render_interpolation(args, anim_args, video_args, parseq_args, loop_args, animatediff_args, controlnet_args, root) else: print('Other modes are not available yet!') except Exception as e: @@ -221,7 +221,7 @@ def run_deforum(*args): if shared.opts.data.get("deforum_enable_persistent_settings", False): persistent_sett_path = shared.opts.data.get("deforum_persistent_settings_path") - save_settings_from_animation_run(args, anim_args, parseq_args, loop_args, controlnet_args, video_args, root, persistent_sett_path) + save_settings_from_animation_run(args, anim_args, parseq_args, loop_args, animatediff_args, controlnet_args, video_args, root, persistent_sett_path) # Close the pipeline, not to interfere with ControlNet try: From d698d7551368cece005e61fa4007387f5ae9743c Mon Sep 17 00:00:00 2001 From: kabachuha Date: Sun, 8 Oct 2023 14:21:11 +0300 Subject: [PATCH 10/48] temp --- .../deforum_helpers/deforum_animatediff.py | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index a311c8873..180f8be87 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -173,7 +173,7 @@ def write_temp_frames(): #tmp_frame_dir.mkdir(parents=True, exist_ok=True) ... -def before_process(p, animatediff_args): +def before_process(p, animatediff_args, temp_video_path): global lora_hacker, cfg_hacker, cn_hacker from scripts.animatediff_lora import AnimateDiffLora @@ -182,7 +182,9 @@ def before_process(p, animatediff_args): from scripts.animatediff_mm import mm_animatediff as motion_module - set_p(p, animatediff_args) + animatediff_args = animatediff_process_from_args(animatediff_args, temp_video_path) + + animatediff_args.set_p(p) motion_module.inject(p.sd_model, animatediff_args.animatediff_model) lora_hacker = AnimateDiffLora(motion_module.mm.using_v2) lora_hacker.hack() @@ -201,16 +203,25 @@ def postprocess( from scripts.animatediff_mm import mm_animatediff as motion_module motion_module.restore(p.sd_model) -def animatediff_process_from_args(animatediff_args): +def animatediff_process_from_args(animatediff_args, temp_video_path): args = animatediff_args from scripts.animatediff_ui import AnimateDiffProcess return AnimateDiffProcess( - model = args.animatediff_model, - enable = args.animatediff_enabled, - video_length = args.animatediff_enabled, - ... - # FIXME + model=args.animatediff_model, + enable=args.animatediff_enabled, + video_length=args.animatediff_window_length, + fps=8, # Irrelevant? + loop_number=0, + closed_loop=False, + batch_size=args.animatediff_window_length, + stride=1, # From Deforum settings + overlap=-1, + format=['MP4'], + video_source=None, + video_path=temp_video_path, + latent_power=args.animatediff_latent_power, + latent_scale=args.animatediff_latent_scale, ) #def set_p(p, animatediff_args): From 1c255622142e6e75283e7ca40cd4c990ce084212 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Sun, 12 Nov 2023 19:25:01 +0300 Subject: [PATCH 11/48] don't use AD hackers --- .../deforum_helpers/deforum_animatediff.py | 79 ++----------------- 1 file changed, 8 insertions(+), 71 deletions(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index 180f8be87..610bccc61 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -84,7 +84,7 @@ def setup_animatediff_ui_raw(): cnet = find_animatediff() - model_dir = ph.models_path + '/AnimateDiff' # shared.opts.data.get("animatediff_model_path", os.path.join(scripts.basedir(), "model")) + model_dir = shared.opts.data.get("animatediff_model_path", os.path.join(scripts.basedir(), "model")) if not os.path.isdir(model_dir): os.mkdir(model_dir) @@ -127,7 +127,13 @@ def create_model_in_tab_ui(cn_id): latent_power = gr.Textbox(label="Latent power schedule", lines=1, value='0:(1)', interactive=True) with gr.Row(visible=False) as latent_scale_row: latent_scale = gr.Textbox(label="Latent scale schedule", lines=1, value='0:(32)', interactive=True) - hide_output_list = [enabled, motion_lora_row, mod_row, window_row, overlap_row, latent_power_row, latent_scale_row] + with gr.Row(visible=False) as rp_row: + closed_loop = gr.Radio( + choices=["N", "R-P", "R+P", "A"], + value="R-P", + label="Closed loop", + ) + hide_output_list = [enabled, motion_lora_row, mod_row, window_row, overlap_row, latent_power_row, latent_scale_row, closed_loop] for cn_output in hide_output_list: enabled.change(fn=hide_ui_by_cn_status, inputs=enabled, outputs=cn_output) @@ -161,72 +167,3 @@ def setup_animatediff_ui(): Failed to setup AnimateDiff UI, check the reason in your commandline log. Please, downgrade your AnimateDiff extension to b192a2551a5ed66d4a3ce58d5d19a8872abc87ca and report the problem here (Deforum) or here (AnimateDiff). """, elem_id='animatediff_not_found_html_msg') return {} - -lora_hacker = None -cfg_hacker = None -cn_hacker = None - -# HACK: writing temp frames to a dir for animatediff processing -# TODO: -def write_temp_frames(): - #tmp_frame_dir = Path(f'{data_path}/tmp/animatediff-frames/') - #tmp_frame_dir.mkdir(parents=True, exist_ok=True) - ... - -def before_process(p, animatediff_args, temp_video_path): - global lora_hacker, cfg_hacker, cn_hacker - - from scripts.animatediff_lora import AnimateDiffLora - from scripts.animatediff_infv2v import AnimateDiffInfV2V - from scripts.animatediff_cn import AnimateDiffControl - - from scripts.animatediff_mm import mm_animatediff as motion_module - - animatediff_args = animatediff_process_from_args(animatediff_args, temp_video_path) - - animatediff_args.set_p(p) - motion_module.inject(p.sd_model, animatediff_args.animatediff_model) - lora_hacker = AnimateDiffLora(motion_module.mm.using_v2) - lora_hacker.hack() - cfg_hacker = AnimateDiffInfV2V(p) - cfg_hacker.hack(animatediff_args) - cn_hacker = AnimateDiffControl(p) - cn_hacker.hack(animatediff_args) - -def postprocess( - p, animatediff_args - ): - global lora_hacker, cfg_hacker, cn_hacker - cn_hacker.restore() - cfg_hacker.restore() - lora_hacker.restore() - from scripts.animatediff_mm import mm_animatediff as motion_module - motion_module.restore(p.sd_model) - -def animatediff_process_from_args(animatediff_args, temp_video_path): - args = animatediff_args - from scripts.animatediff_ui import AnimateDiffProcess - - return AnimateDiffProcess( - model=args.animatediff_model, - enable=args.animatediff_enabled, - video_length=args.animatediff_window_length, - fps=8, # Irrelevant? - loop_number=0, - closed_loop=False, - batch_size=args.animatediff_window_length, - stride=1, # From Deforum settings - overlap=-1, - format=['MP4'], - video_source=None, - video_path=temp_video_path, - latent_power=args.animatediff_latent_power, - latent_scale=args.animatediff_latent_scale, - ) - -#def set_p(p, animatediff_args): -# if animatediff_args.animatediff_window_length < 1: -# return -# p.batch_size = animatediff_args.animatediff_window_length -# if animatediff_args.animatediff_window_overlap == -1: -# animatediff_args.animatediff_window_overlap = animatediff_args.animatediff_window_length // 4 From 76f281452682c71b64117482ca079c5298ebefc7 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Sun, 12 Nov 2023 20:29:54 +0300 Subject: [PATCH 12/48] animatediff seeding --- .../deforum_helpers/deforum_animatediff.py | 39 ++++++++++++++++++- scripts/deforum_helpers/deforum_controlnet.py | 9 ++++- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index 610bccc61..53d26a0db 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -121,7 +121,6 @@ def create_model_in_tab_ui(cn_id): with gr.Row(visible=False) as window_row: window_length = gr.Textbox(label="Number of sliding window frames", lines=1, value='0:(16)', interactive=True) with gr.Row(visible=False) as overlap_row: - # TODO: expose cadence as a variable window_overlap = gr.Textbox(label="Number of overlapping frames", lines=1, value='0:(15)', interactive=True) with gr.Row(visible=False) as latent_power_row: latent_power = gr.Textbox(label="Latent power schedule", lines=1, value='0:(1)', interactive=True) @@ -167,3 +166,41 @@ def setup_animatediff_ui(): Failed to setup AnimateDiff UI, check the reason in your commandline log. Please, downgrade your AnimateDiff extension to b192a2551a5ed66d4a3ce58d5d19a8872abc87ca and report the problem here (Deforum) or here (AnimateDiff). """, elem_id='animatediff_not_found_html_msg') return {} + +def find_animatediff_script(p): + animatediff_script = next((script for script in p.scripts.alwayson_scripts if "animatediff" in script.title().lower()), None) + if not animatediff_script: + raise Exception("AnimateDiff script not found.") + return animatediff_script + +def seed_animatediff(p, animatediff_args): + animatediff_script = find_animatediff_script(p) + # let's put it before ControlNet to cause less problems + p.scripts.alwayson_scripts = [animatediff_script] + p.scripts.alwayson_scripts + + args_dict = { + 'model': 'mm_sd_v15_v2.ckpt', # Motion module + 'format': ['GIF'], # Save format, 'GIF' | 'MP4' | 'PNG' | 'WEBP' | 'WEBM' | 'TXT' | 'Frame' + 'enable': True, # Enable AnimateDiff + 'video_length': 16, # Number of frames + 'fps': 8, # FPS + 'loop_number': 0, # Display loop number + 'closed_loop': 'R+P', # Closed loop, 'N' | 'R-P' | 'R+P' | 'A' + 'batch_size': 16, # Context batch size + 'stride': 1, # Stride + 'overlap': -1, # Overlap + 'interp': 'Off', # Frame interpolation, 'Off' | 'FILM' + 'interp_x': 10 # Interp X + 'video_source': 'path/to/video.mp4', # Video source + 'video_path': 'path/to/frames', # Video path + 'latent_power': 1, # Latent power + 'latent_scale': 32, # Latent scale + 'last_frame': None, # Optional last frame + 'latent_power_last': 1, # Optional latent power for last frame + 'latent_scale_last': 32,# Optional latent scale for last frame + 'request_id': '' # Optional request id. If provided, outputs will have request id as filename suffix + } + + args = list(args_dict.values()) + + p.script_args_value = args + p.script_args_value diff --git a/scripts/deforum_helpers/deforum_controlnet.py b/scripts/deforum_helpers/deforum_controlnet.py index cef20cbc8..ab2d4272f 100644 --- a/scripts/deforum_helpers/deforum_controlnet.py +++ b/scripts/deforum_helpers/deforum_controlnet.py @@ -38,6 +38,9 @@ max_models = shared.opts.data.get("control_net_unit_count", shared.opts.data.get("control_net_max_models_num", 5)) num_of_models = 5 if max_models <= 5 else max_models +# AnimateDiff support (it requires ControlNet anyway) +from .deforum_animatediff import seed_animatediff + def find_controlnet(): global cnet if cnet: return cnet @@ -217,7 +220,7 @@ def controlnet_component_names(): 'processor_res', 'threshold_a', 'threshold_b', 'resize_mode', 'control_mode', 'loopback_mode' ]] -def process_with_controlnet(p, args, anim_args, controlnet_args, root, parseq_adapter, is_img2img=True, frame_idx=0): +def process_with_controlnet(p, args, anim_args, controlnet_args, animatediff_args, root, parseq_adapter, is_img2img=True, frame_idx=0): CnSchKeys = ControlNetKeys(anim_args, controlnet_args) if not parseq_adapter.use_parseq else parseq_adapter.cn_keys def read_cn_data(cn_idx): @@ -289,6 +292,10 @@ def read_cn_data(cn_idx): # and all cn args will be replaced. p.script_args_value = [None] * controlnet_script.args_to + # Basically, launch AD on a number of previous frames once it hits the seed time + if frame_idx % animatediff_args.seed_time == 0: # TODO: make a trigger schedule + seed_animatediff(p, animatediff_args) + def create_cnu_dict(cn_args, prefix, img_np, mask_np, frame_idx, CnSchKeys): keys = [ From 5b1dde920cded8c1beaacea563a01b140f84089a Mon Sep 17 00:00:00 2001 From: kabachuha Date: Sun, 12 Nov 2023 20:52:13 +0300 Subject: [PATCH 13/48] better ui code --- .../deforum_helpers/deforum_animatediff.py | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index 53d26a0db..928ee0f5a 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -69,9 +69,12 @@ def animatediff_infotext(): def animatediff_component_names_raw(): return [ 'enabled', 'model', 'motion_lora_schedule', - 'window_length', # sliding window length (context batch size) - 'window_overlap', # how much do the contexts overlap. if -1, then batch_size // 4 - 'latent_power', 'latent_scale', + 'activation_schedule', + 'batch_size_schedule', + 'stride_schedule', + 'overlap_schedule', + 'latent_power_schedule', 'latent_scale_schedule', + 'closed_loop_schedule' ] def animatediff_component_names(): @@ -111,28 +114,31 @@ def refresh_all_models(*inputs): # TODO: unwrap def create_model_in_tab_ui(cn_id): with gr.Row(): + gr.Markdown('Note: AnimateDiff will work only if you have ControlNet installed as well') enabled = gr.Checkbox(label="Enable AnimateDiff", value=False, interactive=True) with gr.Row(visible=False) as mod_row: model = gr.Dropdown(cn_models, label=f"Motion module", value="None", interactive=True, tooltip="Choose which motion module will be injected into the generation process.") refresh_models = ToolButton(value=refresh_symbol) refresh_models.click(refresh_all_models, model, model) + with gr.Row(visible=False) as activation_row: + gr.Markdown('**Important!** This schedule sets up when AnimateDiff should run on the generated N previous frames. At the moment this is made with binary values: when the expression value is 0, it will make a pass, otherwise normal Deforum frames will be made') + activation_schedule = gr.Textbox(label="AnimateDiff activation schedule", lines=1, value='0:("t % ")', interactive=True) + gr.Markdown('Internal AnimateDiff settings, see its script in normal tabs') with gr.Row(visible=False) as motion_lora_row: motion_lora_schedule = gr.Textbox(label="Motion lora schedule", lines=1, value='0:("")', interactive=True) with gr.Row(visible=False) as window_row: - window_length = gr.Textbox(label="Number of sliding window frames", lines=1, value='0:(16)', interactive=True) + batch_size_schedule = gr.Textbox(label="Batch size", lines=1, value='0:(16)', interactive=True) + with gr.Row(visible=False) as stride_row: + stride_schedule = gr.Textbox(label="Stride", lines=1, value='0:(1)', interactive=True) with gr.Row(visible=False) as overlap_row: - window_overlap = gr.Textbox(label="Number of overlapping frames", lines=1, value='0:(15)', interactive=True) + overlap_schedule = gr.Textbox(label="Overlap", lines=1, value='0:(-1)', interactive=True) with gr.Row(visible=False) as latent_power_row: - latent_power = gr.Textbox(label="Latent power schedule", lines=1, value='0:(1)', interactive=True) + latent_power_schedule = gr.Textbox(label="Latent power schedule", lines=1, value='0:(1)', interactive=True) with gr.Row(visible=False) as latent_scale_row: - latent_scale = gr.Textbox(label="Latent scale schedule", lines=1, value='0:(32)', interactive=True) + latent_scale_schedule = gr.Textbox(label="Latent scale schedule", lines=1, value='0:(32)', interactive=True) with gr.Row(visible=False) as rp_row: - closed_loop = gr.Radio( - choices=["N", "R-P", "R+P", "A"], - value="R-P", - label="Closed loop", - ) - hide_output_list = [enabled, motion_lora_row, mod_row, window_row, overlap_row, latent_power_row, latent_scale_row, closed_loop] + closed_loop_schedule = gr.Textbox(label="Closed loop", lines=1, value='0:("R-P")', interactive=True) + hide_output_list = [enabled, activation_row, motion_lora_row, mod_row, window_row, stride_row, overlap_row, latent_power_row, latent_scale_row, rp_row] for cn_output in hide_output_list: enabled.change(fn=hide_ui_by_cn_status, inputs=enabled, outputs=cn_output) From fce1e041f04bafc0a611b624298235ad9be76866 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Sun, 12 Nov 2023 22:35:28 +0300 Subject: [PATCH 14/48] add AnimateDiffKeys --- .../deforum_helpers/animation_key_frames.py | 14 ++++ .../deforum_helpers/deforum_animatediff.py | 64 +++++++++++-------- scripts/deforum_helpers/deforum_controlnet.py | 3 +- 3 files changed, 53 insertions(+), 28 deletions(-) diff --git a/scripts/deforum_helpers/animation_key_frames.py b/scripts/deforum_helpers/animation_key_frames.py index ca4aa4310..27b31d861 100644 --- a/scripts/deforum_helpers/animation_key_frames.py +++ b/scripts/deforum_helpers/animation_key_frames.py @@ -87,6 +87,20 @@ def __init__(self, anim_args, controlnet_args): self.schedules[output_key] = self.fi.parse_inbetweens(getattr(controlnet_args, input_key), input_key) setattr(self, output_key, self.schedules[output_key]) +class AnimateDiffKeys(): + def __init__(self, animatediff_args, anim_args, seed): + self.fi = FrameInterpolater(anim_args.max_frames, seed) + self.enable = animatediff_args.enabled + self.model = animatediff_args.model + self.activation_schedule_series = self.fi.parse_inbetweens(animatediff_args.activation_schedule, 'activation_schedule') + self.motion_lora_schedule_series = self.fi.parse_inbetweens(animatediff_args.motion_lora_schedule, 'motion_lora_schedule', is_single_string = True) + self.video_length_schedule_series = self.fi.parse_inbetweens(animatediff_args.video_length_schedule, 'video_length_schedule') + self.batch_size_schedule_series = self.fi.parse_inbetweens(animatediff_args.batch_size_schedule, 'batch_size_schedule') + self.stride_schedule_series = self.fi.parse_inbetweens(animatediff_args.stride_schedule, 'stride_schedule') + self.overlap_schedule_series = self.fi.parse_inbetweens(animatediff_args.overlap_schedule, 'overlap_schedule') + self.latent_scale_schedule_series = self.fi.parse_inbetweens(animatediff_args.latent_scale_schedule, 'latent_scale_schedule') + self.closed_loop_schedule_series = self.fi.parse_inbetweens(animatediff_args.closed_loop_schedule, 'closed_loop_schedule', is_single_string = True) + class LooperAnimKeys(): def __init__(self, loop_args, anim_args, seed): self.fi = FrameInterpolater(anim_args.max_frames, seed) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index 928ee0f5a..1c533a9dc 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -28,7 +28,7 @@ from .deforum_controlnet_gradio import hide_ui_by_cn_status, hide_file_textboxes, ToolButton from .general_utils import count_files_in_folder, clean_gradio_path_strings # TODO: do it another way from .video_audio_utilities import vid2frames, convert_image -#from .animation_key_frames import AnimateDiffKeys +from .animation_key_frames import AnimateDiffKeys from .load_images import load_image from .general_utils import debug_print from modules.shared import opts, cmd_opts, state, sd_model @@ -68,8 +68,9 @@ def animatediff_infotext(): def animatediff_component_names_raw(): return [ - 'enabled', 'model', 'motion_lora_schedule', - 'activation_schedule', + 'enabled', 'model', 'activation_schedule', + 'motion_lora_schedule', + 'video_length_schedule', 'batch_size_schedule', 'stride_schedule', 'overlap_schedule', @@ -122,10 +123,12 @@ def create_model_in_tab_ui(cn_id): refresh_models.click(refresh_all_models, model, model) with gr.Row(visible=False) as activation_row: gr.Markdown('**Important!** This schedule sets up when AnimateDiff should run on the generated N previous frames. At the moment this is made with binary values: when the expression value is 0, it will make a pass, otherwise normal Deforum frames will be made') - activation_schedule = gr.Textbox(label="AnimateDiff activation schedule", lines=1, value='0:("t % ")', interactive=True) + activation_schedule = gr.Textbox(label="AnimateDiff activation schedule", lines=1, value='0:("t % 16")', interactive=True) gr.Markdown('Internal AnimateDiff settings, see its script in normal tabs') with gr.Row(visible=False) as motion_lora_row: motion_lora_schedule = gr.Textbox(label="Motion lora schedule", lines=1, value='0:("")', interactive=True) + with gr.Row(visible=False) as window_row: + video_length_schedule = gr.Textbox(label="N-back video length schedule", lines=1, value='0:(16)', interactive=True) with gr.Row(visible=False) as window_row: batch_size_schedule = gr.Textbox(label="Batch size", lines=1, value='0:(16)', interactive=True) with gr.Row(visible=False) as stride_row: @@ -179,33 +182,42 @@ def find_animatediff_script(p): raise Exception("AnimateDiff script not found.") return animatediff_script -def seed_animatediff(p, animatediff_args): +def seed_animatediff(p, animatediff_args, anim_args, frame_idx): + if find_animatediff() is None or not is_animatediff_enabled(animatediff_args): + return + + keys = AnimateDiffKeys(animatediff_args, anim_args) # if not parseq_adapter.use_parseq else parseq_adapter.cn_keys + + # Will do the back-render only on target frames + if int(keys.activation_schedule_series[frame_idx]) != 0: + return + animatediff_script = find_animatediff_script(p) # let's put it before ControlNet to cause less problems p.scripts.alwayson_scripts = [animatediff_script] + p.scripts.alwayson_scripts args_dict = { - 'model': 'mm_sd_v15_v2.ckpt', # Motion module - 'format': ['GIF'], # Save format, 'GIF' | 'MP4' | 'PNG' | 'WEBP' | 'WEBM' | 'TXT' | 'Frame' - 'enable': True, # Enable AnimateDiff - 'video_length': 16, # Number of frames - 'fps': 8, # FPS - 'loop_number': 0, # Display loop number - 'closed_loop': 'R+P', # Closed loop, 'N' | 'R-P' | 'R+P' | 'A' - 'batch_size': 16, # Context batch size - 'stride': 1, # Stride - 'overlap': -1, # Overlap - 'interp': 'Off', # Frame interpolation, 'Off' | 'FILM' - 'interp_x': 10 # Interp X - 'video_source': 'path/to/video.mp4', # Video source - 'video_path': 'path/to/frames', # Video path - 'latent_power': 1, # Latent power - 'latent_scale': 32, # Latent scale - 'last_frame': None, # Optional last frame - 'latent_power_last': 1, # Optional latent power for last frame - 'latent_scale_last': 32,# Optional latent scale for last frame - 'request_id': '' # Optional request id. If provided, outputs will have request id as filename suffix - } + 'model': keys.model, # Motion module + 'format': ['Frame'], # Save format, 'GIF' | 'MP4' | 'PNG' | 'WEBP' | 'WEBM' | 'TXT' | 'Frame' + 'enable': keys.enable, # Enable AnimateDiff + 'video_length': keys.video_length_schedule_series[frame_idx], # Number of frames + 'fps': 8, # FPS - don't care + 'loop_number': 0, # Display loop number + 'closed_loop': keys.closed_loop_schedule_series[frame_idx], # Closed loop, 'N' | 'R-P' | 'R+P' | 'A' + 'batch_size': keys.batch_size_schedule_series[frame_idx], # Context batch size + 'stride': keys.stride_schedule_series[frame_idx], # Stride + 'overlap': keys.overlap_schedule_series[frame_idx], # Overlap + 'interp': 'Off', # Frame interpolation, 'Off' | 'FILM' - don't care + 'interp_x': 10, # Interp X - don't care + 'video_source': '', # We don't use a video + 'video_path': 'path/to/frames', # Path with our selected video_length input frames + 'latent_power': keys.latent_power_schedule_series[frame_idx], # Latent power + 'latent_scale': keys.latent_scale_schedule_series[frame_idx], # Latent scale + 'last_frame': None, # Optional last frame + 'latent_power_last': 1, # Optional latent power for last frame + 'latent_scale_last': 32,# Optional latent scale for last frame + 'request_id': '' # Optional request id. If provided, outputs will have request id as filename suffix + } args = list(args_dict.values()) diff --git a/scripts/deforum_helpers/deforum_controlnet.py b/scripts/deforum_helpers/deforum_controlnet.py index ab2d4272f..acd3c2935 100644 --- a/scripts/deforum_helpers/deforum_controlnet.py +++ b/scripts/deforum_helpers/deforum_controlnet.py @@ -293,8 +293,7 @@ def read_cn_data(cn_idx): p.script_args_value = [None] * controlnet_script.args_to # Basically, launch AD on a number of previous frames once it hits the seed time - if frame_idx % animatediff_args.seed_time == 0: # TODO: make a trigger schedule - seed_animatediff(p, animatediff_args) + seed_animatediff(p, animatediff_args, frame_idx) def create_cnu_dict(cn_args, prefix, img_np, mask_np, frame_idx, CnSchKeys): From 3b82f207579e14354edfe3979c5aa4c613c57bd1 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 21:38:00 +0300 Subject: [PATCH 15/48] progress on animatediff logic --- .../deforum_helpers/deforum_animatediff.py | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index 1c533a9dc..fbcb96d99 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -24,6 +24,7 @@ from PIL import Image import numpy as np import importlib +import shutil from modules import scripts, shared from .deforum_controlnet_gradio import hide_ui_by_cn_status, hide_file_textboxes, ToolButton from .general_utils import count_files_in_folder, clean_gradio_path_strings # TODO: do it another way @@ -182,7 +183,7 @@ def find_animatediff_script(p): raise Exception("AnimateDiff script not found.") return animatediff_script -def seed_animatediff(p, animatediff_args, anim_args, frame_idx): +def seed_animatediff(p, animatediff_args, args, anim_args, frame_idx): if find_animatediff() is None or not is_animatediff_enabled(animatediff_args): return @@ -191,6 +192,20 @@ def seed_animatediff(p, animatediff_args, anim_args, frame_idx): # Will do the back-render only on target frames if int(keys.activation_schedule_series[frame_idx]) != 0: return + + video_length = int(keys.video_length_schedule_series[frame_idx]) + assert video_length > 1 + + # Managing the frames to be fed into AD: + # Create a temporal directory + animatediff_temp_dir = os.path.join(args.outdir, 'animatediff_temp') + if os.path.exists(animatediff_temp_dir): + shutil.rmtree(animatediff_temp_dir) + os.makedirs(animatediff_temp_dir) + # Copy the frames (except for the one which is being CN-made) into that dir + for offset in range(video_length - 1): + filename = f"{root.timestring}_{frame_idx - offset - 1:09}.png" + Image.open(os.path.join(args.outdir, filename)).save(os.path.join(f"{offset:09}.png"), "PNG") animatediff_script = find_animatediff_script(p) # let's put it before ControlNet to cause less problems @@ -200,17 +215,17 @@ def seed_animatediff(p, animatediff_args, anim_args, frame_idx): 'model': keys.model, # Motion module 'format': ['Frame'], # Save format, 'GIF' | 'MP4' | 'PNG' | 'WEBP' | 'WEBM' | 'TXT' | 'Frame' 'enable': keys.enable, # Enable AnimateDiff - 'video_length': keys.video_length_schedule_series[frame_idx], # Number of frames + 'video_length': video_length, # Number of frames 'fps': 8, # FPS - don't care 'loop_number': 0, # Display loop number 'closed_loop': keys.closed_loop_schedule_series[frame_idx], # Closed loop, 'N' | 'R-P' | 'R+P' | 'A' - 'batch_size': keys.batch_size_schedule_series[frame_idx], # Context batch size - 'stride': keys.stride_schedule_series[frame_idx], # Stride - 'overlap': keys.overlap_schedule_series[frame_idx], # Overlap + 'batch_size': int(keys.batch_size_schedule_series[frame_idx]), # Context batch size + 'stride': int(keys.stride_schedule_series[frame_idx]), # Stride + 'overlap': int(keys.overlap_schedule_series[frame_idx]), # Overlap 'interp': 'Off', # Frame interpolation, 'Off' | 'FILM' - don't care 'interp_x': 10, # Interp X - don't care 'video_source': '', # We don't use a video - 'video_path': 'path/to/frames', # Path with our selected video_length input frames + 'video_path': animatediff_temp_dir, # Path with our selected video_length input frames 'latent_power': keys.latent_power_schedule_series[frame_idx], # Latent power 'latent_scale': keys.latent_scale_schedule_series[frame_idx], # Latent scale 'last_frame': None, # Optional last frame From d9b6c46f39130a363b42f26266aaefde93d91604 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 21:39:23 +0300 Subject: [PATCH 16/48] add animatediff args to seed_animatediff in CN --- scripts/deforum_helpers/deforum_controlnet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deforum_helpers/deforum_controlnet.py b/scripts/deforum_helpers/deforum_controlnet.py index acd3c2935..72a4eea01 100644 --- a/scripts/deforum_helpers/deforum_controlnet.py +++ b/scripts/deforum_helpers/deforum_controlnet.py @@ -293,7 +293,7 @@ def read_cn_data(cn_idx): p.script_args_value = [None] * controlnet_script.args_to # Basically, launch AD on a number of previous frames once it hits the seed time - seed_animatediff(p, animatediff_args, frame_idx) + seed_animatediff(p, animatediff_args, args, anim_args, frame_idx) def create_cnu_dict(cn_args, prefix, img_np, mask_np, frame_idx, CnSchKeys): From 719aa3c57e59f2e27eafcdaa585e84418002a6a6 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 22:01:47 +0300 Subject: [PATCH 17/48] animatediff reaping logic --- .../deforum_helpers/deforum_animatediff.py | 29 ++++++++++++++++--- scripts/deforum_helpers/deforum_controlnet.py | 2 +- scripts/deforum_helpers/generate.py | 27 +++++++++-------- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index fbcb96d99..9e3e1263a 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -183,8 +183,14 @@ def find_animatediff_script(p): raise Exception("AnimateDiff script not found.") return animatediff_script -def seed_animatediff(p, animatediff_args, args, anim_args, frame_idx): - if find_animatediff() is None or not is_animatediff_enabled(animatediff_args): +def get_animatediff_temp_dir(args): + return os.path.join(args.outdir, 'animatediff_temp') + +def need_animatediff(animatediff_args): + return find_animatediff() is not None and is_animatediff_enabled(animatediff_args): + +def seed_animatediff(p, animatediff_args, args, anim_args, root, frame_idx): + if not need_animatediff(animatediff_args): return keys = AnimateDiffKeys(animatediff_args, anim_args) # if not parseq_adapter.use_parseq else parseq_adapter.cn_keys @@ -198,14 +204,14 @@ def seed_animatediff(p, animatediff_args, args, anim_args, frame_idx): # Managing the frames to be fed into AD: # Create a temporal directory - animatediff_temp_dir = os.path.join(args.outdir, 'animatediff_temp') + animatediff_temp_dir = get_animatediff_temp_dir(args) if os.path.exists(animatediff_temp_dir): shutil.rmtree(animatediff_temp_dir) os.makedirs(animatediff_temp_dir) # Copy the frames (except for the one which is being CN-made) into that dir for offset in range(video_length - 1): filename = f"{root.timestring}_{frame_idx - offset - 1:09}.png" - Image.open(os.path.join(args.outdir, filename)).save(os.path.join(f"{offset:09}.png"), "PNG") + Image.open(os.path.join(args.outdir, filename)).save(os.path.join(animatediff_temp_dir, f"{offset:09}.png"), "PNG") animatediff_script = find_animatediff_script(p) # let's put it before ControlNet to cause less problems @@ -237,3 +243,18 @@ def seed_animatediff(p, animatediff_args, args, anim_args, frame_idx): args = list(args_dict.values()) p.script_args_value = args + p.script_args_value + +def reap_animatediff(images, args, root, frame_idx): + if not need_animatediff(animatediff_args): + return + + animatediff_temp_dir = get_animatediff_temp_dir(args) + assert os.path.exists(animatediff_temp_dir) + + for offset in range(len(images)): + frame = images[-offset-1] + cur_frame_idx = frame_idx - offset + + # overwrite the results + filename = f"{root.timestring}_{cur_frame_idx:09}.png" + frame.save(os.path.join(args.outdir, filename), "PNG") diff --git a/scripts/deforum_helpers/deforum_controlnet.py b/scripts/deforum_helpers/deforum_controlnet.py index 72a4eea01..dc4a7039e 100644 --- a/scripts/deforum_helpers/deforum_controlnet.py +++ b/scripts/deforum_helpers/deforum_controlnet.py @@ -293,7 +293,7 @@ def read_cn_data(cn_idx): p.script_args_value = [None] * controlnet_script.args_to # Basically, launch AD on a number of previous frames once it hits the seed time - seed_animatediff(p, animatediff_args, args, anim_args, frame_idx) + seed_animatediff(p, animatediff_args, args, anim_args, root, frame_idx) def create_cnu_dict(cn_args, prefix, img_np, mask_np, frame_idx, CnSchKeys): diff --git a/scripts/deforum_helpers/generate.py b/scripts/deforum_helpers/generate.py index fe48055e3..4a786e4bc 100644 --- a/scripts/deforum_helpers/generate.py +++ b/scripts/deforum_helpers/generate.py @@ -36,6 +36,7 @@ from types import SimpleNamespace from .general_utils import debug_print +from .deforum_animatediff import reap_animatediff def load_mask_latent(mask_input, shape): # mask_input (str or PIL Image.Image): Path to the mask image or a PIL Image object @@ -70,14 +71,14 @@ def pairwise_repl(iterable): next(b, None) return zip(a, b) -def generate(args, keys, anim_args, loop_args, controlnet_args, root, parseq_adapter, frame=0, sampler_name=None): +def generate(args, keys, anim_args, loop_args, controlnet_args, animatediff_args, root, parseq_adapter, frame=0, sampler_name=None): if state.interrupted: return None if args.reroll_blank_frames == 'ignore': - return generate_inner(args, keys, anim_args, loop_args, controlnet_args, root, parseq_adapter, frame, sampler_name) + return generate_inner(args, keys, anim_args, loop_args, controlnet_args, animatediff_args, root, parseq_adapter, frame, sampler_name) - image, caught_vae_exception = generate_with_nans_check(args, keys, anim_args, loop_args, controlnet_args, root, parseq_adapter, frame, sampler_name) + image, caught_vae_exception = generate_with_nans_check(args, keys, anim_args, loop_args, controlnet_args, animatediff_args, root, parseq_adapter, frame, sampler_name) if caught_vae_exception or not image.getbbox(): patience = args.reroll_patience @@ -86,7 +87,7 @@ def generate(args, keys, anim_args, loop_args, controlnet_args, root, parseq_ada while caught_vae_exception or not image.getbbox(): print("Rerolling with +1 seed...") args.seed += 1 - image, caught_vae_exception = generate_with_nans_check(args, keys, anim_args, loop_args, controlnet_args, root, parseq_adapter, frame, sampler_name) + image, caught_vae_exception = generate_with_nans_check(args, keys, anim_args, loop_args, controlnet_args, animatediff_args, root, parseq_adapter, frame, sampler_name) patience -= 1 if patience == 0: print("Rerolling with +1 seed failed for 10 iterations! Try setting webui's precision to 'full' and if it fails, please report this to the devs! Interrupting...") @@ -100,12 +101,12 @@ def generate(args, keys, anim_args, loop_args, controlnet_args, root, parseq_ada return None return image -def generate_with_nans_check(args, keys, anim_args, loop_args, controlnet_args, root, parseq_adapter, frame=0, sampler_name=None): +def generate_with_nans_check(args, keys, anim_args, loop_args, controlnet_args, animatediff_args, root, parseq_adapter, frame=0, sampler_name=None): if cmd_opts.disable_nan_check: - image = generate_inner(args, keys, anim_args, loop_args, controlnet_args, root, parseq_adapter, frame, sampler_name) + image = generate_inner(args, keys, anim_args, loop_args, controlnet_args, animatediff_args, root, parseq_adapter, frame, sampler_name) else: try: - image = generate_inner(args, keys, anim_args, loop_args, controlnet_args, root, parseq_adapter, frame, sampler_name) + image = generate_inner(args, keys, anim_args, loop_args, controlnet_args, animatediff_args, root, parseq_adapter, frame, sampler_name) except Exception as e: if "A tensor with all NaNs was produced in VAE." in repr(e): print(e) @@ -114,7 +115,7 @@ def generate_with_nans_check(args, keys, anim_args, loop_args, controlnet_args, raise e return image, False -def generate_inner(args, keys, anim_args, loop_args, controlnet_args, root, parseq_adapter, frame=0, sampler_name=None): +def generate_inner(args, keys, anim_args, loop_args, controlnet_args, animatediff_args, root, parseq_adapter, frame=0, sampler_name=None): # Setup the pipeline p = get_webui_sd_pipeline(args, root) p.prompt, p.negative_prompt = split_weighted_subprompts(args.prompt, frame, anim_args.max_frames) @@ -235,7 +236,7 @@ def generate_inner(args, keys, anim_args, loop_args, controlnet_args, root, pars print_combined_table(args, anim_args, p_txt, keys, frame) # print dynamic table to cli if is_controlnet_enabled(controlnet_args): - process_with_controlnet(p_txt, args, anim_args, controlnet_args, root, parseq_adapter, is_img2img=False, frame_idx=frame) + process_with_controlnet(p_txt, args, anim_args, controlnet_args, animatediff_args, root, parseq_adapter, is_img2img=False, frame_idx=frame) with A1111OptionsOverrider({"control_net_detectedmap_dir" : os.path.join(args.outdir, "controlnet_detected_map")}): processed = processing.process_images(p_txt) @@ -277,7 +278,7 @@ def generate_inner(args, keys, anim_args, loop_args, controlnet_args, root, pars processed = mock_process_images(args, p, init_image) else: if is_controlnet_enabled(controlnet_args): - process_with_controlnet(p, args, anim_args, controlnet_args, root, parseq_adapter, is_img2img=True, frame_idx=frame) + process_with_controlnet(p, args, anim_args, controlnet_args, animatediff_args, root, parseq_adapter, is_img2img=True, frame_idx=frame) with A1111OptionsOverrider({"control_net_detectedmap_dir" : os.path.join(args.outdir, "controlnet_detected_map")}): processed = processing.process_images(p) @@ -287,9 +288,11 @@ def generate_inner(args, keys, anim_args, loop_args, controlnet_args, root, pars root.initial_info = processed.info if root.first_frame is None: - root.first_frame = processed.images[0] + root.first_frame = processed.images[-1] - results = processed.images[0] + results = processed.images[-1] # AD uses ascending order, so we need to get the last frame + + reap_animatediff(processed.images, args, root, frame) return results From 3793566ea675b8ae2516433f553d11e5e8336e0b Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 22:07:31 +0300 Subject: [PATCH 18/48] put animatediff_args in function calls --- scripts/deforum_helpers/render.py | 10 ++++------ scripts/deforum_helpers/render_modes.py | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/scripts/deforum_helpers/render.py b/scripts/deforum_helpers/render.py index 2d5c2e4f8..51eeb90a3 100644 --- a/scripts/deforum_helpers/render.py +++ b/scripts/deforum_helpers/render.py @@ -59,9 +59,7 @@ def render_animation(args, anim_args, video_args, parseq_args, loop_args, animat parseq_adapter = ParseqAdapter(parseq_args, anim_args, video_args, controlnet_args, loop_args) if animatediff_args.enabled: - print("Rendering with AnimateDiff on.") - - + print("*Rendering with AnimateDiff on.*") if opts.data.get("deforum_save_gen_info_as_srt", False): # create .srt file and set timeframe mechanism using FPS srt_filename = os.path.join(args.outdir, f"{root.timestring}.srt") @@ -553,7 +551,7 @@ def render_animation(args, anim_args, video_args, parseq_args, loop_args, animat args.seed = random.randint(0, 2 ** 32 - 1) print(f"Optical flow redo is diffusing and warping using {optical_flow_redo_generation} and seed {args.seed} optical flow before generation.") - disposable_image = generate(args, keys, anim_args, loop_args, controlnet_args, root, parseq_adapter, frame_idx, sampler_name=scheduled_sampler_name) + disposable_image = generate(args, keys, anim_args, loop_args, controlnet_args, animatediff_args, root, parseq_adapter, frame_idx, sampler_name=scheduled_sampler_name) disposable_image = cv2.cvtColor(np.array(disposable_image), cv2.COLOR_RGB2BGR) disposable_flow = get_flow_from_images(prev_img, disposable_image, optical_flow_redo_generation, raft_model) disposable_image = cv2.cvtColor(disposable_image, cv2.COLOR_BGR2RGB) @@ -569,7 +567,7 @@ def render_animation(args, anim_args, video_args, parseq_args, loop_args, animat for n in range(0, int(anim_args.diffusion_redo)): print(f"Redo generation {n + 1} of {int(anim_args.diffusion_redo)} before final generation") args.seed = random.randint(0, 2 ** 32 - 1) - disposable_image = generate(args, keys, anim_args, loop_args, controlnet_args, root, parseq_adapter, frame_idx, sampler_name=scheduled_sampler_name) + disposable_image = generate(args, keys, anim_args, loop_args, controlnet_args, animatediff_args, root, parseq_adapter, frame_idx, sampler_name=scheduled_sampler_name) disposable_image = cv2.cvtColor(np.array(disposable_image), cv2.COLOR_RGB2BGR) # color match on last one only if n == int(anim_args.diffusion_redo): @@ -580,7 +578,7 @@ def render_animation(args, anim_args, video_args, parseq_args, loop_args, animat gc.collect() # generation - image = generate(args, keys, anim_args, loop_args, controlnet_args, root, parseq_adapter, frame_idx, sampler_name=scheduled_sampler_name) + image = generate(args, keys, anim_args, loop_args, controlnet_args, animatediff_args, root, parseq_adapter, frame_idx, sampler_name=scheduled_sampler_name) if image is None: break diff --git a/scripts/deforum_helpers/render_modes.py b/scripts/deforum_helpers/render_modes.py index 49ddf7900..29d9c79cf 100644 --- a/scripts/deforum_helpers/render_modes.py +++ b/scripts/deforum_helpers/render_modes.py @@ -162,7 +162,7 @@ def render_interpolation(args, anim_args, video_args, parseq_args, loop_args, an args.seed = int(keys.seed_schedule_series[frame_idx]) if (args.seed_behavior == 'schedule' or parseq_adapter.manages_seed()) else args.seed opts.data["CLIP_stop_at_last_layers"] = scheduled_clipskip if scheduled_clipskip is not None else opts.data["CLIP_stop_at_last_layers"] - image = generate(args, keys, anim_args, loop_args, controlnet_args, root, parseq_adapter, frame_idx, sampler_name=scheduled_sampler_name) + image = generate(args, keys, anim_args, loop_args, controlnet_args, animatediff_args, root, parseq_adapter, frame_idx, sampler_name=scheduled_sampler_name) filename = f"{root.timestring}_{frame_idx:09}.png" save_image(image, 'PIL', filename, args, video_args, root) From f0b5ab86f63043bb876550beb5c2e128441bc15b Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 22:13:15 +0300 Subject: [PATCH 19/48] add animatediff args to settings saving --- scripts/deforum_helpers/render.py | 2 +- scripts/deforum_helpers/render_modes.py | 2 +- scripts/deforum_helpers/settings.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/deforum_helpers/render.py b/scripts/deforum_helpers/render.py index 51eeb90a3..4fca6cf28 100644 --- a/scripts/deforum_helpers/render.py +++ b/scripts/deforum_helpers/render.py @@ -97,7 +97,7 @@ def render_animation(args, anim_args, video_args, parseq_args, loop_args, animat print(f"Saving animation frames to:\n{args.outdir}") # save settings.txt file for the current run - save_settings_from_animation_run(args, anim_args, parseq_args, loop_args, controlnet_args, video_args, root) + save_settings_from_animation_run(args, anim_args, parseq_args, loop_args, controlnet_args, animatediff_args, video_args, root) # resume from timestring if anim_args.resume_from_timestring: diff --git a/scripts/deforum_helpers/render_modes.py b/scripts/deforum_helpers/render_modes.py index 29d9c79cf..388ecfa76 100644 --- a/scripts/deforum_helpers/render_modes.py +++ b/scripts/deforum_helpers/render_modes.py @@ -106,7 +106,7 @@ def render_interpolation(args, anim_args, video_args, parseq_args, loop_args, an print(f"Saving interpolation animation frames to {args.outdir}") # save settings.txt file for the current run - save_settings_from_animation_run(args, anim_args, parseq_args, loop_args, controlnet_args, video_args, root) + save_settings_from_animation_run(args, anim_args, parseq_args, loop_args, controlnet_args, animatediff_args, video_args, root) # Compute interpolated prompts if parseq_adapter.manages_prompts(): diff --git a/scripts/deforum_helpers/settings.py b/scripts/deforum_helpers/settings.py index bd1132d9b..214f5821f 100644 --- a/scripts/deforum_helpers/settings.py +++ b/scripts/deforum_helpers/settings.py @@ -58,7 +58,7 @@ def load_args(args_dict_main, args, anim_args, parseq_args, loop_args, controlne return True # save settings function that get calls when run_deforum is being called -def save_settings_from_animation_run(args, anim_args, parseq_args, loop_args, controlnet_args, video_args, root, full_out_file_path = None): +def save_settings_from_animation_run(args, anim_args, parseq_args, loop_args, controlnet_args, animatediff_args, video_args, root, full_out_file_path = None): if full_out_file_path: args.__dict__["seed"] = root.raw_seed args.__dict__["batch_name"] = root.raw_batch_name @@ -69,7 +69,7 @@ def save_settings_from_animation_run(args, anim_args, parseq_args, loop_args, co settings_filename = full_out_file_path if full_out_file_path else os.path.join(args.outdir, f"{root.timestring}_settings.txt") with open(settings_filename, "w+", encoding="utf-8") as f: s = {} - for d in (args.__dict__, anim_args.__dict__, parseq_args.__dict__, loop_args.__dict__, controlnet_args.__dict__, video_args.__dict__): + for d in (args.__dict__, anim_args.__dict__, parseq_args.__dict__, loop_args.__dict__, controlnet_args.__dict__, animatediff_args.__dict__, video_args.__dict__): s.update({k: v for k, v in d.items() if k not in exclude_keys}) s["sd_model_name"] = sh.sd_model.sd_checkpoint_info.name s["sd_model_hash"] = sh.sd_model.sd_checkpoint_info.hash From b54e26059034645318c032f24d8ace3801bfa9cc Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 22:18:25 +0300 Subject: [PATCH 20/48] syntax fixup --- scripts/deforum_helpers/deforum_animatediff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index 9e3e1263a..9d6d5b623 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -187,7 +187,7 @@ def get_animatediff_temp_dir(args): return os.path.join(args.outdir, 'animatediff_temp') def need_animatediff(animatediff_args): - return find_animatediff() is not None and is_animatediff_enabled(animatediff_args): + return find_animatediff() is not None and is_animatediff_enabled(animatediff_args) def seed_animatediff(p, animatediff_args, args, anim_args, root, frame_idx): if not need_animatediff(animatediff_args): From f79b037d918212a0b5c156217cf3c37a7c64e588 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 22:22:21 +0300 Subject: [PATCH 21/48] fix layout --- scripts/deforum_helpers/deforum_animatediff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index 9d6d5b623..801fbfd4d 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -128,7 +128,7 @@ def create_model_in_tab_ui(cn_id): gr.Markdown('Internal AnimateDiff settings, see its script in normal tabs') with gr.Row(visible=False) as motion_lora_row: motion_lora_schedule = gr.Textbox(label="Motion lora schedule", lines=1, value='0:("")', interactive=True) - with gr.Row(visible=False) as window_row: + with gr.Row(visible=False) as length_row: video_length_schedule = gr.Textbox(label="N-back video length schedule", lines=1, value='0:(16)', interactive=True) with gr.Row(visible=False) as window_row: batch_size_schedule = gr.Textbox(label="Batch size", lines=1, value='0:(16)', interactive=True) From 18ea77bfa40bcfb32af3b967becd451bff702c01 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 22:22:47 +0300 Subject: [PATCH 22/48] start from 1th frame in the ad schedule --- scripts/deforum_helpers/deforum_animatediff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index 801fbfd4d..0a293da1b 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -124,7 +124,7 @@ def create_model_in_tab_ui(cn_id): refresh_models.click(refresh_all_models, model, model) with gr.Row(visible=False) as activation_row: gr.Markdown('**Important!** This schedule sets up when AnimateDiff should run on the generated N previous frames. At the moment this is made with binary values: when the expression value is 0, it will make a pass, otherwise normal Deforum frames will be made') - activation_schedule = gr.Textbox(label="AnimateDiff activation schedule", lines=1, value='0:("t % 16")', interactive=True) + activation_schedule = gr.Textbox(label="AnimateDiff activation schedule", lines=1, value='0:("(abs(t-1)) % 16")', interactive=True) gr.Markdown('Internal AnimateDiff settings, see its script in normal tabs') with gr.Row(visible=False) as motion_lora_row: motion_lora_schedule = gr.Textbox(label="Motion lora schedule", lines=1, value='0:("")', interactive=True) From 39e9af5e8780ffbdaf623af294b2619df5c798fa Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 22:26:02 +0300 Subject: [PATCH 23/48] more layout fixes --- scripts/deforum_helpers/deforum_animatediff.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index 0a293da1b..da51b12f0 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -122,8 +122,9 @@ def create_model_in_tab_ui(cn_id): model = gr.Dropdown(cn_models, label=f"Motion module", value="None", interactive=True, tooltip="Choose which motion module will be injected into the generation process.") refresh_models = ToolButton(value=refresh_symbol) refresh_models.click(refresh_all_models, model, model) - with gr.Row(visible=False) as activation_row: + with gr.Row(visible=False) as inforow: gr.Markdown('**Important!** This schedule sets up when AnimateDiff should run on the generated N previous frames. At the moment this is made with binary values: when the expression value is 0, it will make a pass, otherwise normal Deforum frames will be made') + with gr.Row(visible=False) as activation_row: activation_schedule = gr.Textbox(label="AnimateDiff activation schedule", lines=1, value='0:("(abs(t-1)) % 16")', interactive=True) gr.Markdown('Internal AnimateDiff settings, see its script in normal tabs') with gr.Row(visible=False) as motion_lora_row: @@ -142,7 +143,7 @@ def create_model_in_tab_ui(cn_id): latent_scale_schedule = gr.Textbox(label="Latent scale schedule", lines=1, value='0:(32)', interactive=True) with gr.Row(visible=False) as rp_row: closed_loop_schedule = gr.Textbox(label="Closed loop", lines=1, value='0:("R-P")', interactive=True) - hide_output_list = [enabled, activation_row, motion_lora_row, mod_row, window_row, stride_row, overlap_row, latent_power_row, latent_scale_row, rp_row] + hide_output_list = [enabled, inforow, activation_row, motion_lora_row, mod_row, length_row, window_row, stride_row, overlap_row, latent_power_row, latent_scale_row, rp_row] for cn_output in hide_output_list: enabled.change(fn=hide_ui_by_cn_status, inputs=enabled, outputs=cn_output) From e802095917545c59b1d69f4ea25e2fb127bd6dd6 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 22:32:05 +0300 Subject: [PATCH 24/48] add animatediff_ to ad args references where missed --- .../deforum_helpers/animation_key_frames.py | 20 +++++++++---------- scripts/deforum_helpers/render.py | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/deforum_helpers/animation_key_frames.py b/scripts/deforum_helpers/animation_key_frames.py index 27b31d861..a1222d3c3 100644 --- a/scripts/deforum_helpers/animation_key_frames.py +++ b/scripts/deforum_helpers/animation_key_frames.py @@ -90,16 +90,16 @@ def __init__(self, anim_args, controlnet_args): class AnimateDiffKeys(): def __init__(self, animatediff_args, anim_args, seed): self.fi = FrameInterpolater(anim_args.max_frames, seed) - self.enable = animatediff_args.enabled - self.model = animatediff_args.model - self.activation_schedule_series = self.fi.parse_inbetweens(animatediff_args.activation_schedule, 'activation_schedule') - self.motion_lora_schedule_series = self.fi.parse_inbetweens(animatediff_args.motion_lora_schedule, 'motion_lora_schedule', is_single_string = True) - self.video_length_schedule_series = self.fi.parse_inbetweens(animatediff_args.video_length_schedule, 'video_length_schedule') - self.batch_size_schedule_series = self.fi.parse_inbetweens(animatediff_args.batch_size_schedule, 'batch_size_schedule') - self.stride_schedule_series = self.fi.parse_inbetweens(animatediff_args.stride_schedule, 'stride_schedule') - self.overlap_schedule_series = self.fi.parse_inbetweens(animatediff_args.overlap_schedule, 'overlap_schedule') - self.latent_scale_schedule_series = self.fi.parse_inbetweens(animatediff_args.latent_scale_schedule, 'latent_scale_schedule') - self.closed_loop_schedule_series = self.fi.parse_inbetweens(animatediff_args.closed_loop_schedule, 'closed_loop_schedule', is_single_string = True) + self.enable = animatediff_args.animatediff_enabled + self.model = animatediff_args.animatediff_model + self.activation_schedule_series = self.fi.parse_inbetweens(animatediff_args.animatediff_activation_schedule, 'activation_schedule') + self.motion_lora_schedule_series = self.fi.parse_inbetweens(animatediff_args.animatediff_motion_lora_schedule, 'motion_lora_schedule', is_single_string = True) + self.video_length_schedule_series = self.fi.parse_inbetweens(animatediff_args.animatediff_video_length_schedule, 'video_length_schedule') + self.batch_size_schedule_series = self.fi.parse_inbetweens(animatediff_args.animatediff_batch_size_schedule, 'batch_size_schedule') + self.stride_schedule_series = self.fi.parse_inbetweens(animatediff_args.animatediff_stride_schedule, 'stride_schedule') + self.overlap_schedule_series = self.fi.parse_inbetweens(animatediff_args.animatediff_overlap_schedule, 'overlap_schedule') + self.latent_scale_schedule_series = self.fi.parse_inbetweens(animatediff_args.animatediff_latent_scale_schedule, 'latent_scale_schedule') + self.closed_loop_schedule_series = self.fi.parse_inbetweens(animatediff_args.animatediff_closed_loop_schedule, 'closed_loop_schedule', is_single_string = True) class LooperAnimKeys(): def __init__(self, loop_args, anim_args, seed): diff --git a/scripts/deforum_helpers/render.py b/scripts/deforum_helpers/render.py index 4fca6cf28..247a5ef80 100644 --- a/scripts/deforum_helpers/render.py +++ b/scripts/deforum_helpers/render.py @@ -58,8 +58,8 @@ def render_animation(args, anim_args, video_args, parseq_args, loop_args, animat # TODO: @rewbs parseq_adapter = ParseqAdapter(parseq_args, anim_args, video_args, controlnet_args, loop_args) - if animatediff_args.enabled: - print("*Rendering with AnimateDiff on.*") + if animatediff_args.animatediff_enabled: + print("*Rendering with AnimateDiff turned on. (Experimental!)*") if opts.data.get("deforum_save_gen_info_as_srt", False): # create .srt file and set timeframe mechanism using FPS srt_filename = os.path.join(args.outdir, f"{root.timestring}.srt") From a01f1306ff6bc83e1431770f1abc7cea7a7d2838 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 22:38:11 +0300 Subject: [PATCH 25/48] fixup --- scripts/deforum_helpers/generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deforum_helpers/generate.py b/scripts/deforum_helpers/generate.py index 4a786e4bc..f353a1026 100644 --- a/scripts/deforum_helpers/generate.py +++ b/scripts/deforum_helpers/generate.py @@ -292,7 +292,7 @@ def generate_inner(args, keys, anim_args, loop_args, controlnet_args, animatedif results = processed.images[-1] # AD uses ascending order, so we need to get the last frame - reap_animatediff(processed.images, args, root, frame) + reap_animatediff(processed.images, animatediff_args, args, root, frame) return results From 6761e6ec5ec000f6884ad47fa3d753cc7f8f68a7 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 22:38:22 +0300 Subject: [PATCH 26/48] activate controlnet when AD is on --- scripts/deforum_helpers/generate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/deforum_helpers/generate.py b/scripts/deforum_helpers/generate.py index f353a1026..8f13b03b3 100644 --- a/scripts/deforum_helpers/generate.py +++ b/scripts/deforum_helpers/generate.py @@ -235,7 +235,7 @@ def generate_inner(args, keys, anim_args, loop_args, controlnet_args, animatedif print_combined_table(args, anim_args, p_txt, keys, frame) # print dynamic table to cli - if is_controlnet_enabled(controlnet_args): + if is_controlnet_enabled(controlnet_args) or is_animatediff_enabled(animatediff_args): process_with_controlnet(p_txt, args, anim_args, controlnet_args, animatediff_args, root, parseq_adapter, is_img2img=False, frame_idx=frame) with A1111OptionsOverrider({"control_net_detectedmap_dir" : os.path.join(args.outdir, "controlnet_detected_map")}): @@ -277,7 +277,7 @@ def generate_inner(args, keys, anim_args, loop_args, controlnet_args, animatedif if args.motion_preview_mode: processed = mock_process_images(args, p, init_image) else: - if is_controlnet_enabled(controlnet_args): + if is_controlnet_enabled(controlnet_args) or is_animatediff_enabled(animatediff_args): process_with_controlnet(p, args, anim_args, controlnet_args, animatediff_args, root, parseq_adapter, is_img2img=True, frame_idx=frame) with A1111OptionsOverrider({"control_net_detectedmap_dir" : os.path.join(args.outdir, "controlnet_detected_map")}): From 66f0f44599b436ce1226328c8783c004a9f75843 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 22:40:08 +0300 Subject: [PATCH 27/48] fix refs to is_animatediff_enabled --- scripts/deforum_helpers/deforum_animatediff.py | 2 +- scripts/deforum_helpers/generate.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index da51b12f0..46ee5052d 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -245,7 +245,7 @@ def seed_animatediff(p, animatediff_args, args, anim_args, root, frame_idx): p.script_args_value = args + p.script_args_value -def reap_animatediff(images, args, root, frame_idx): +def reap_animatediff(images, animatediff_args, args, root, frame_idx): if not need_animatediff(animatediff_args): return diff --git a/scripts/deforum_helpers/generate.py b/scripts/deforum_helpers/generate.py index 8f13b03b3..ec8dbad07 100644 --- a/scripts/deforum_helpers/generate.py +++ b/scripts/deforum_helpers/generate.py @@ -36,7 +36,7 @@ from types import SimpleNamespace from .general_utils import debug_print -from .deforum_animatediff import reap_animatediff +from .deforum_animatediff import reap_animatediff, is_animatediff_enabled def load_mask_latent(mask_input, shape): # mask_input (str or PIL Image.Image): Path to the mask image or a PIL Image object From a8ce8cd810ae0e9fb33cf733bb795c5985f09c36 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 22:41:45 +0300 Subject: [PATCH 28/48] better text --- scripts/deforum_helpers/deforum_animatediff.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index 46ee5052d..b9d9bb39e 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -63,9 +63,9 @@ def is_animatediff_enabled(animatediff_args): return False def animatediff_infotext(): - return """Requires the AnimateDiff extension to be installed.

- If Deforum crashes due to AnimateDiff updates, go here and report your problem.

- """ + return """**Experimental!** +Requires the AnimateDiff extension to be installed.

+""" def animatediff_component_names_raw(): return [ From 7809a7baf3aebadbd7c9f4140ad1e2e9309f1989 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 22:44:18 +0300 Subject: [PATCH 29/48] don't need seed in AnimateDiffKeys --- scripts/deforum_helpers/animation_key_frames.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/deforum_helpers/animation_key_frames.py b/scripts/deforum_helpers/animation_key_frames.py index a1222d3c3..dfbde2818 100644 --- a/scripts/deforum_helpers/animation_key_frames.py +++ b/scripts/deforum_helpers/animation_key_frames.py @@ -88,8 +88,8 @@ def __init__(self, anim_args, controlnet_args): setattr(self, output_key, self.schedules[output_key]) class AnimateDiffKeys(): - def __init__(self, animatediff_args, anim_args, seed): - self.fi = FrameInterpolater(anim_args.max_frames, seed) + def __init__(self, animatediff_args, anim_args): + self.fi = FrameInterpolater(anim_args.max_frames) self.enable = animatediff_args.animatediff_enabled self.model = animatediff_args.animatediff_model self.activation_schedule_series = self.fi.parse_inbetweens(animatediff_args.animatediff_activation_schedule, 'activation_schedule') From 0e2a3a24a7b22045235bec40c99f7cd0a08d52d3 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 22:49:36 +0300 Subject: [PATCH 30/48] no quotes in math --- scripts/deforum_helpers/deforum_animatediff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index b9d9bb39e..11c31d453 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -125,7 +125,7 @@ def create_model_in_tab_ui(cn_id): with gr.Row(visible=False) as inforow: gr.Markdown('**Important!** This schedule sets up when AnimateDiff should run on the generated N previous frames. At the moment this is made with binary values: when the expression value is 0, it will make a pass, otherwise normal Deforum frames will be made') with gr.Row(visible=False) as activation_row: - activation_schedule = gr.Textbox(label="AnimateDiff activation schedule", lines=1, value='0:("(abs(t-1)) % 16")', interactive=True) + activation_schedule = gr.Textbox(label="AnimateDiff activation schedule", lines=1, value='0:((abs(t-1)) % 16)', interactive=True) gr.Markdown('Internal AnimateDiff settings, see its script in normal tabs') with gr.Row(visible=False) as motion_lora_row: motion_lora_schedule = gr.Textbox(label="Motion lora schedule", lines=1, value='0:("")', interactive=True) From 3b1e6e2ac2385b918f2b10228fc74c566c5d3ecf Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 22:52:54 +0300 Subject: [PATCH 31/48] reap only if > 1 image --- scripts/deforum_helpers/generate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/deforum_helpers/generate.py b/scripts/deforum_helpers/generate.py index ec8dbad07..261a63987 100644 --- a/scripts/deforum_helpers/generate.py +++ b/scripts/deforum_helpers/generate.py @@ -292,7 +292,8 @@ def generate_inner(args, keys, anim_args, loop_args, controlnet_args, animatedif results = processed.images[-1] # AD uses ascending order, so we need to get the last frame - reap_animatediff(processed.images, animatediff_args, args, root, frame) + if len(processed.images) > 1: + reap_animatediff(processed.images, animatediff_args, args, root, frame) return results From d0edde4e73014616be68e9c0e9ba9512eaa1ee85 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 22:56:35 +0300 Subject: [PATCH 32/48] better default activation schedule --- scripts/deforum_helpers/deforum_animatediff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index 11c31d453..c9d643788 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -125,7 +125,7 @@ def create_model_in_tab_ui(cn_id): with gr.Row(visible=False) as inforow: gr.Markdown('**Important!** This schedule sets up when AnimateDiff should run on the generated N previous frames. At the moment this is made with binary values: when the expression value is 0, it will make a pass, otherwise normal Deforum frames will be made') with gr.Row(visible=False) as activation_row: - activation_schedule = gr.Textbox(label="AnimateDiff activation schedule", lines=1, value='0:((abs(t-1)) % 16)', interactive=True) + activation_schedule = gr.Textbox(label="AnimateDiff activation schedule", lines=1, value='0:(1), 2:((t-1) % 16)', interactive=True) gr.Markdown('Internal AnimateDiff settings, see its script in normal tabs') with gr.Row(visible=False) as motion_lora_row: motion_lora_schedule = gr.Textbox(label="Motion lora schedule", lines=1, value='0:("")', interactive=True) From d268d0346b2b47be6753aeb6dfaf0e8230cdfa74 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 23:00:45 +0300 Subject: [PATCH 33/48] carryover prev_always_on_scripts --- scripts/deforum_helpers/deforum_animatediff.py | 8 ++++---- scripts/deforum_helpers/deforum_controlnet.py | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index c9d643788..fc6c1674a 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -178,8 +178,8 @@ def setup_animatediff_ui(): """, elem_id='animatediff_not_found_html_msg') return {} -def find_animatediff_script(p): - animatediff_script = next((script for script in p.scripts.alwayson_scripts if "animatediff" in script.title().lower()), None) +def find_animatediff_script(prev_always_on_scripts): + animatediff_script = next((script for script in prev_always_on_scripts if "animatediff" in script.title().lower()), None) if not animatediff_script: raise Exception("AnimateDiff script not found.") return animatediff_script @@ -190,7 +190,7 @@ def get_animatediff_temp_dir(args): def need_animatediff(animatediff_args): return find_animatediff() is not None and is_animatediff_enabled(animatediff_args) -def seed_animatediff(p, animatediff_args, args, anim_args, root, frame_idx): +def seed_animatediff(p, prev_always_on_scripts, animatediff_args, args, anim_args, root, frame_idx): if not need_animatediff(animatediff_args): return @@ -214,7 +214,7 @@ def seed_animatediff(p, animatediff_args, args, anim_args, root, frame_idx): filename = f"{root.timestring}_{frame_idx - offset - 1:09}.png" Image.open(os.path.join(args.outdir, filename)).save(os.path.join(animatediff_temp_dir, f"{offset:09}.png"), "PNG") - animatediff_script = find_animatediff_script(p) + animatediff_script = find_animatediff_script(prev_always_on_scripts) # let's put it before ControlNet to cause less problems p.scripts.alwayson_scripts = [animatediff_script] + p.scripts.alwayson_scripts diff --git a/scripts/deforum_helpers/deforum_controlnet.py b/scripts/deforum_helpers/deforum_controlnet.py index dc4a7039e..ad15b1907 100644 --- a/scripts/deforum_helpers/deforum_controlnet.py +++ b/scripts/deforum_helpers/deforum_controlnet.py @@ -287,13 +287,14 @@ def read_cn_data(cn_idx): # p.scripts = copy.copy(scripts.scripts_img2img if is_img2img else scripts.scripts_txt2img) controlnet_script = find_controlnet_script(p) + prev_always_on_scripts = p.scripts.alwayson_scripts p.scripts.alwayson_scripts = [controlnet_script] # Filling the list with None is safe because only the length will be considered, # and all cn args will be replaced. p.script_args_value = [None] * controlnet_script.args_to # Basically, launch AD on a number of previous frames once it hits the seed time - seed_animatediff(p, animatediff_args, args, anim_args, root, frame_idx) + seed_animatediff(p, prev_always_on_scripts, animatediff_args, args, anim_args, root, frame_idx) def create_cnu_dict(cn_args, prefix, img_np, mask_np, frame_idx, CnSchKeys): From 165b9111dda8295d72b8f407a497c9b28d7007e2 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 23:03:38 +0300 Subject: [PATCH 34/48] fix up missed series --- scripts/deforum_helpers/animation_key_frames.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/deforum_helpers/animation_key_frames.py b/scripts/deforum_helpers/animation_key_frames.py index dfbde2818..af3126252 100644 --- a/scripts/deforum_helpers/animation_key_frames.py +++ b/scripts/deforum_helpers/animation_key_frames.py @@ -99,6 +99,7 @@ def __init__(self, animatediff_args, anim_args): self.stride_schedule_series = self.fi.parse_inbetweens(animatediff_args.animatediff_stride_schedule, 'stride_schedule') self.overlap_schedule_series = self.fi.parse_inbetweens(animatediff_args.animatediff_overlap_schedule, 'overlap_schedule') self.latent_scale_schedule_series = self.fi.parse_inbetweens(animatediff_args.animatediff_latent_scale_schedule, 'latent_scale_schedule') + self.latent_power_schedule_series = self.fi.parse_inbetweens(animatediff_args.animatediff_latent_power_schedule, 'latent_power_schedule') self.closed_loop_schedule_series = self.fi.parse_inbetweens(animatediff_args.animatediff_closed_loop_schedule, 'closed_loop_schedule', is_single_string = True) class LooperAnimKeys(): From b7c0ce3dd1a38012733c6d9f95ddd14d7fc570ad Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 23:06:32 +0300 Subject: [PATCH 35/48] hide CN warning message when AD --- scripts/deforum_helpers/deforum_controlnet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/deforum_helpers/deforum_controlnet.py b/scripts/deforum_helpers/deforum_controlnet.py index ad15b1907..a893a8f7f 100644 --- a/scripts/deforum_helpers/deforum_controlnet.py +++ b/scripts/deforum_helpers/deforum_controlnet.py @@ -39,7 +39,7 @@ num_of_models = 5 if max_models <= 5 else max_models # AnimateDiff support (it requires ControlNet anyway) -from .deforum_animatediff import seed_animatediff +from .deforum_animatediff import seed_animatediff, is_animatediff_enabled def find_controlnet(): global cnet @@ -267,7 +267,7 @@ def read_cn_data(cn_idx): cn_inputframes_list = [os.path.join(args.outdir, f'controlnet_{i}_inputframes') for i in range(1, num_of_models + 1)] - if not any(os.path.exists(cn_inputframes) for cn_inputframes in cn_inputframes_list) and not any_loopback_mode: + if not any(os.path.exists(cn_inputframes) for cn_inputframes in cn_inputframes_list) and not any_loopback_mode and not is_animatediff_enabled(animatediff_args): print(f'\033[33mNeither the base nor the masking frames for ControlNet were found. Using the regular pipeline\033[0m') # Remove all scripts except controlnet. From 6b60dbafd8d3f4696035fc4384569d87f7de871b Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 23:08:51 +0300 Subject: [PATCH 36/48] p.is_api = True when animatediff --- scripts/deforum_helpers/deforum_animatediff.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index fc6c1674a..d88e03cf1 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -216,6 +216,7 @@ def seed_animatediff(p, prev_always_on_scripts, animatediff_args, args, anim_arg animatediff_script = find_animatediff_script(prev_always_on_scripts) # let's put it before ControlNet to cause less problems + p.is_api = True # to parse the params internally p.scripts.alwayson_scripts = [animatediff_script] + p.scripts.alwayson_scripts args_dict = { From 58439c071c369135216ccb368c593ec7f32cbd54 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 23:12:54 +0300 Subject: [PATCH 37/48] send args_dict as a list containing the dict --- scripts/deforum_helpers/deforum_animatediff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index d88e03cf1..cd237b485 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -242,7 +242,7 @@ def seed_animatediff(p, prev_always_on_scripts, animatediff_args, args, anim_arg 'request_id': '' # Optional request id. If provided, outputs will have request id as filename suffix } - args = list(args_dict.values()) + args = [args_dict] p.script_args_value = args + p.script_args_value From 698b63a3cd00def9ba3fbf65feb49fa10814771e Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 23:22:08 +0300 Subject: [PATCH 38/48] try putting animatediff script after cn --- scripts/deforum_helpers/deforum_animatediff.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index cd237b485..efd3286c3 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -215,9 +215,8 @@ def seed_animatediff(p, prev_always_on_scripts, animatediff_args, args, anim_arg Image.open(os.path.join(args.outdir, filename)).save(os.path.join(animatediff_temp_dir, f"{offset:09}.png"), "PNG") animatediff_script = find_animatediff_script(prev_always_on_scripts) - # let's put it before ControlNet to cause less problems p.is_api = True # to parse the params internally - p.scripts.alwayson_scripts = [animatediff_script] + p.scripts.alwayson_scripts + p.scripts.alwayson_scripts.append(animatediff_script) args_dict = { 'model': keys.model, # Motion module @@ -242,9 +241,7 @@ def seed_animatediff(p, prev_always_on_scripts, animatediff_args, args, anim_arg 'request_id': '' # Optional request id. If provided, outputs will have request id as filename suffix } - args = [args_dict] - - p.script_args_value = args + p.script_args_value + p.script_args_value.append(args_dict) def reap_animatediff(images, animatediff_args, args, root, frame_idx): if not need_animatediff(animatediff_args): From 94c5583ad9c002dbea6584730c3c3d486f7df790 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 23:26:02 +0300 Subject: [PATCH 39/48] test with cn disabled --- scripts/deforum_helpers/deforum_controlnet.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/deforum_helpers/deforum_controlnet.py b/scripts/deforum_helpers/deforum_controlnet.py index a893a8f7f..49b718357 100644 --- a/scripts/deforum_helpers/deforum_controlnet.py +++ b/scripts/deforum_helpers/deforum_controlnet.py @@ -296,6 +296,9 @@ def read_cn_data(cn_idx): # Basically, launch AD on a number of previous frames once it hits the seed time seed_animatediff(p, prev_always_on_scripts, animatediff_args, args, anim_args, root, frame_idx) + if is_animatediff_enabled(animatediff_args): + return # FIXME temporary disable CN + def create_cnu_dict(cn_args, prefix, img_np, mask_np, frame_idx, CnSchKeys): keys = [ From 0b27a4577943d3e67fe4e886ca5252bb2c53f158 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 23:27:57 +0300 Subject: [PATCH 40/48] try putting AD after CN --- scripts/deforum_helpers/deforum_controlnet.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/scripts/deforum_helpers/deforum_controlnet.py b/scripts/deforum_helpers/deforum_controlnet.py index 49b718357..913fddc0a 100644 --- a/scripts/deforum_helpers/deforum_controlnet.py +++ b/scripts/deforum_helpers/deforum_controlnet.py @@ -293,12 +293,6 @@ def read_cn_data(cn_idx): # and all cn args will be replaced. p.script_args_value = [None] * controlnet_script.args_to - # Basically, launch AD on a number of previous frames once it hits the seed time - seed_animatediff(p, prev_always_on_scripts, animatediff_args, args, anim_args, root, frame_idx) - - if is_animatediff_enabled(animatediff_args): - return # FIXME temporary disable CN - def create_cnu_dict(cn_args, prefix, img_np, mask_np, frame_idx, CnSchKeys): keys = [ @@ -327,6 +321,9 @@ def create_cnu_dict(cn_args, prefix, img_np, mask_np, frame_idx, CnSchKeys): cnet.update_cn_script_in_processing(p, cn_units, is_img2img=is_img2img, is_ui=False) + # Basically, launch AD on a number of previous frames once it hits the seed time + seed_animatediff(p, prev_always_on_scripts, animatediff_args, args, anim_args, root, frame_idx) + def find_controlnet_script(p): controlnet_script = next((script for script in p.scripts.alwayson_scripts if script.title().lower() == "controlnet"), None) if not controlnet_script: From d267e0a6b4a8fb06e344fc8262eec5136e78600f Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 23:35:06 +0300 Subject: [PATCH 41/48] Revert "try putting AD after CN" This reverts commit 0b27a4577943d3e67fe4e886ca5252bb2c53f158. --- scripts/deforum_helpers/deforum_controlnet.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/deforum_helpers/deforum_controlnet.py b/scripts/deforum_helpers/deforum_controlnet.py index 913fddc0a..49b718357 100644 --- a/scripts/deforum_helpers/deforum_controlnet.py +++ b/scripts/deforum_helpers/deforum_controlnet.py @@ -293,6 +293,12 @@ def read_cn_data(cn_idx): # and all cn args will be replaced. p.script_args_value = [None] * controlnet_script.args_to + # Basically, launch AD on a number of previous frames once it hits the seed time + seed_animatediff(p, prev_always_on_scripts, animatediff_args, args, anim_args, root, frame_idx) + + if is_animatediff_enabled(animatediff_args): + return # FIXME temporary disable CN + def create_cnu_dict(cn_args, prefix, img_np, mask_np, frame_idx, CnSchKeys): keys = [ @@ -321,9 +327,6 @@ def create_cnu_dict(cn_args, prefix, img_np, mask_np, frame_idx, CnSchKeys): cnet.update_cn_script_in_processing(p, cn_units, is_img2img=is_img2img, is_ui=False) - # Basically, launch AD on a number of previous frames once it hits the seed time - seed_animatediff(p, prev_always_on_scripts, animatediff_args, args, anim_args, root, frame_idx) - def find_controlnet_script(p): controlnet_script = next((script for script in p.scripts.alwayson_scripts if script.title().lower() == "controlnet"), None) if not controlnet_script: From 5d040b52520b522aaafa3e91ca8514eba2fda383 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 23:35:09 +0300 Subject: [PATCH 42/48] Revert "test with cn disabled" This reverts commit 94c5583ad9c002dbea6584730c3c3d486f7df790. --- scripts/deforum_helpers/deforum_controlnet.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/deforum_helpers/deforum_controlnet.py b/scripts/deforum_helpers/deforum_controlnet.py index 49b718357..a893a8f7f 100644 --- a/scripts/deforum_helpers/deforum_controlnet.py +++ b/scripts/deforum_helpers/deforum_controlnet.py @@ -296,9 +296,6 @@ def read_cn_data(cn_idx): # Basically, launch AD on a number of previous frames once it hits the seed time seed_animatediff(p, prev_always_on_scripts, animatediff_args, args, anim_args, root, frame_idx) - if is_animatediff_enabled(animatediff_args): - return # FIXME temporary disable CN - def create_cnu_dict(cn_args, prefix, img_np, mask_np, frame_idx, CnSchKeys): keys = [ From 3cdaeef791cf39404ee0e7f0dec907bf508c13df Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 23:35:14 +0300 Subject: [PATCH 43/48] Revert "try putting animatediff script after cn" This reverts commit 698b63a3cd00def9ba3fbf65feb49fa10814771e. --- scripts/deforum_helpers/deforum_animatediff.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index efd3286c3..cd237b485 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -215,8 +215,9 @@ def seed_animatediff(p, prev_always_on_scripts, animatediff_args, args, anim_arg Image.open(os.path.join(args.outdir, filename)).save(os.path.join(animatediff_temp_dir, f"{offset:09}.png"), "PNG") animatediff_script = find_animatediff_script(prev_always_on_scripts) + # let's put it before ControlNet to cause less problems p.is_api = True # to parse the params internally - p.scripts.alwayson_scripts.append(animatediff_script) + p.scripts.alwayson_scripts = [animatediff_script] + p.scripts.alwayson_scripts args_dict = { 'model': keys.model, # Motion module @@ -241,7 +242,9 @@ def seed_animatediff(p, prev_always_on_scripts, animatediff_args, args, anim_arg 'request_id': '' # Optional request id. If provided, outputs will have request id as filename suffix } - p.script_args_value.append(args_dict) + args = [args_dict] + + p.script_args_value = args + p.script_args_value def reap_animatediff(images, animatediff_args, args, root, frame_idx): if not need_animatediff(animatediff_args): From a6720ceac061dc488a99da5ca5ece1bb83d6ecc5 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 23:35:22 +0300 Subject: [PATCH 44/48] Revert "send args_dict as a list containing the dict" This reverts commit 58439c071c369135216ccb368c593ec7f32cbd54. --- scripts/deforum_helpers/deforum_animatediff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index cd237b485..d88e03cf1 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -242,7 +242,7 @@ def seed_animatediff(p, prev_always_on_scripts, animatediff_args, args, anim_arg 'request_id': '' # Optional request id. If provided, outputs will have request id as filename suffix } - args = [args_dict] + args = list(args_dict.values()) p.script_args_value = args + p.script_args_value From 13d01b9b8de7830e5c1bbf1142744168317b4570 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 23:38:17 +0300 Subject: [PATCH 45/48] hardcoded script args offset Update deforum_animatediff.py --- scripts/deforum_helpers/deforum_animatediff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index d88e03cf1..e63dd4afe 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -242,7 +242,7 @@ def seed_animatediff(p, prev_always_on_scripts, animatediff_args, args, anim_arg 'request_id': '' # Optional request id. If provided, outputs will have request id as filename suffix } - args = list(args_dict.values()) + args = [None] * 10 + [args_dict] # HACK hardcoded args offset p.script_args_value = args + p.script_args_value From cf343a4e113fe47230b3655829e90dcb47e6e82a Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 23:45:13 +0300 Subject: [PATCH 46/48] add PNG to save format --- scripts/deforum_helpers/deforum_animatediff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index e63dd4afe..3a0924715 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -221,7 +221,7 @@ def seed_animatediff(p, prev_always_on_scripts, animatediff_args, args, anim_arg args_dict = { 'model': keys.model, # Motion module - 'format': ['Frame'], # Save format, 'GIF' | 'MP4' | 'PNG' | 'WEBP' | 'WEBM' | 'TXT' | 'Frame' + 'format': ['PNG', 'Frame'], # Save format, 'GIF' | 'MP4' | 'PNG' | 'WEBP' | 'WEBM' | 'TXT' | 'Frame' 'enable': keys.enable, # Enable AnimateDiff 'video_length': video_length, # Number of frames 'fps': 8, # FPS - don't care From c57244849fea140a2df87b344dab31012d47a322 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Mon, 13 Nov 2023 23:52:53 +0300 Subject: [PATCH 47/48] do_not_save_grid when CN/AD --- scripts/deforum_helpers/deforum_controlnet.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/deforum_helpers/deforum_controlnet.py b/scripts/deforum_helpers/deforum_controlnet.py index a893a8f7f..729b85036 100644 --- a/scripts/deforum_helpers/deforum_controlnet.py +++ b/scripts/deforum_helpers/deforum_controlnet.py @@ -221,6 +221,7 @@ def controlnet_component_names(): ]] def process_with_controlnet(p, args, anim_args, controlnet_args, animatediff_args, root, parseq_adapter, is_img2img=True, frame_idx=0): + p.do_not_save_grid = True CnSchKeys = ControlNetKeys(anim_args, controlnet_args) if not parseq_adapter.use_parseq else parseq_adapter.cn_keys def read_cn_data(cn_idx): From 7ce84abff752e10f7a4e4c2551fcbfcb7c2a8f96 Mon Sep 17 00:00:00 2001 From: kabachuha Date: Tue, 14 Nov 2023 00:02:14 +0300 Subject: [PATCH 48/48] note about motion lora not supported yet --- scripts/deforum_helpers/deforum_animatediff.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deforum_helpers/deforum_animatediff.py b/scripts/deforum_helpers/deforum_animatediff.py index 3a0924715..5091586d1 100644 --- a/scripts/deforum_helpers/deforum_animatediff.py +++ b/scripts/deforum_helpers/deforum_animatediff.py @@ -128,7 +128,7 @@ def create_model_in_tab_ui(cn_id): activation_schedule = gr.Textbox(label="AnimateDiff activation schedule", lines=1, value='0:(1), 2:((t-1) % 16)', interactive=True) gr.Markdown('Internal AnimateDiff settings, see its script in normal tabs') with gr.Row(visible=False) as motion_lora_row: - motion_lora_schedule = gr.Textbox(label="Motion lora schedule", lines=1, value='0:("")', interactive=True) + motion_lora_schedule = gr.Textbox(label="Motion lora schedule (not supported atm, stay tuned!)", lines=1, value='0:("")', interactive=True) with gr.Row(visible=False) as length_row: video_length_schedule = gr.Textbox(label="N-back video length schedule", lines=1, value='0:(16)', interactive=True) with gr.Row(visible=False) as window_row: