.gitignore

@@ -29,7 +29,7 @@ notification.mp3
 /SwinIR
 /textual_inversion
 .vscode
-/extensions
+# /extensions
 /test/stdout.txt
 /test/stderr.txt
 /cache.json

extensions/deforum/.github/FUNDING.yml

@@ -0,0 +1,13 @@
+# These are supported funding model platforms
+
+github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: deforum
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

extensions/deforum/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,97 @@
+name: Bug Report
+description: Create a bug report for the Deforum extension
+title: "[Bug]: "
+labels: ["bug-report"]
+
+body:
+  - type: checkboxes
+    attributes:
+      label: Have you read the latest version of the FAQ?
+      description: Please visit the page called FAQ & Troubleshooting on the Deforum wiki in this repository and see if your problem has already been described there.
+      options:
+        - label: I have visited the FAQ page right now and my issue is not present there
+          required: true
+  - type: checkboxes
+    attributes:
+      label: Is there an existing issue for this?
+      description: Please search to see if an issue already exists for the bug you encountered (including the closed issues).
+      options:
+        - label: I have searched the existing issues and checked the recent builds/commits of both this extension and the webui
+          required: true
+  - type: checkboxes
+    attributes:
+      label: Are you using the latest version of the Deforum extension?
+      description: Please, check if your Deforum is based on the latest repo commit (git log) or update it through the 'Extensions' tab and check if the issue still persist. Otherwise, check this box.
+      options:
+        - label: I have Deforum updated to the lastest version and I still have the issue.
+          required: true
+  - type: markdown
+    attributes:
+      value: |
+        *Please fill this form with as much information as possible, don't forget to fill "What OS..." and "What browsers" and *provide screenshots if possible**
+  - type: textarea
+    id: what-did
+    attributes:
+      label: What happened?
+      description: Tell us what happened in a very clear and simple way
+    validations:
+      required: true
+  - type: textarea
+    id: steps
+    attributes:
+      label: Steps to reproduce the problem
+      description: Please provide us with precise step by step information on how to reproduce the bug
+      value: |
+        1. Go to .... 
+        2. Press ....
+        3. ...
+    validations:
+      required: true
+  - type: textarea
+    id: what-should
+    attributes:
+      label: What should have happened?
+      description: Tell what you think the normal behavior should be
+  - type: textarea
+    id: commits
+    attributes:
+      label: WebUI and Deforum extension Commit IDs
+      description: Which commit of the webui/deforum extension are you running on? (Do not write *Latest version/repo/commit*, as this means nothing and will have changed by the time we read your issue. Rather, copy the **Commit** link at the bottom of the UI, or if you can't launch the webui at all, enter your cmd/terminal, CD into the main webui folder to get the webui commit id, and cd into the extensions/deforum folder to get the deforum commit id, both using the command 'git rev-parse HEAD'.)
+      value: |
+             webui commit id - 
+             deforum exten commit id - 
+    validations:
+      required: true
+  - type: dropdown
+    id: where
+    attributes:
+      label: On which platform are you launching the webui with the extension?
+      multiple: true
+      options:
+        - Local PC setup (Windows)
+        - Local PC setup (Linux)
+        - Local PC setup (Mac)
+        - Google Colab (The Last Ben's)
+        - Google Colab (Other)
+        - Cloud server (Linux)
+        - Other (please specify in "additional information")
+  - type: textarea
+    id: customsettings
+    attributes:
+      label: Webui core settings
+      description: Send here a link to your ui-config.json file in the core 'stable-diffusion-webui' folder (ideally, upload it to GitHub gists). Friendly reminder - if you have 'With img2img, do exactly the amount of steps the slider specified' checked, your issue will be discarded immediately. 😉
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Console logs
+      description: Please provide **full** cmd/terminal logs from the moment you started UI to the end of it, after your bug happened. If it's very long, provide a link to GitHub gists or similar service.
+      render: Shell
+    validations:
+      required: true
+  - type: textarea
+    id: misc
+    attributes:
+      label: Additional information
+      description: Please provide us with any relevant additional info or context.

extensions/deforum/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Deforum Github discussions
+    url: https://github.com/deforum-art/deforum-for-automatic1111-webui/discussions
+    about: Please ask and answer questions here.
+  - name: Deforum Discord
+    url: https://discord.gg/deforum
+    about: Or here :)

extensions/deforum/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,40 @@
+name: Feature request
+description: Suggest an idea for the Deforum extension
+title: "[Feature Request]: "
+labels: ["enhancement"]
+
+body:
+  - type: checkboxes
+    attributes:
+      label: Is there an existing issue for this?
+      description: Please search to see if an issue already exists for the feature you want, and that it's not implemented in a recent build/commit.
+      options:
+        - label: I have searched the existing issues and checked the recent builds/commits
+          required: true
+  - type: markdown
+    attributes:
+      value: |
+        *Please fill this form with as much information as possible, provide screenshots and/or illustrations of the feature if possible*
+  - type: textarea
+    id: feature
+    attributes:
+      label: What would your feature do ?
+      description: Tell us about your feature in a very clear and simple way, and what problem it would solve
+    validations:
+      required: true
+  - type: textarea
+    id: workflow
+    attributes:
+      label: Proposed workflow
+      description: Please provide us with step by step information on how you'd like the feature to be accessed and used
+      value: |
+        1. Go to .... 
+        2. Press ....
+        3. ...
+    validations:
+      required: true
+  - type: textarea
+    id: misc
+    attributes:
+      label: Additional information
+      description: Add any other context or screenshots about the feature request here.

extensions/deforum/.gitignore

@@ -0,0 +1,10 @@
+# Unnecessary compiled python files.
+__pycache__
+*.pyc
+*.pyo
+
+# Output Images
+outputs
+
+# Log files for colab-convert
+cc-outputs.log

extensions/deforum/CONTRIBUTING.md

@@ -0,0 +1,7 @@
+# Contributing
+
+As a part of the Deforum team I (kabachuha) want this script extension to remain a part of the Deforum project.
+
+Thus, if you want to submit feature request or bugfix, unless it only relates to automatic1111's porting issues, consider making a PR first to the parent repository notebook https://github.com/deforum/stable-diffusion.
+
+Also, you may want to inforum the dev team about your work via Discord https://discord.gg/deforum to ensure that no one else is working on the same stuff.

extensions/deforum/README.md

@@ -0,0 +1,81 @@
+
+# Deforum Stable Diffusion — official extension for AUTOMATIC1111's webui
+
+<p align="left">
+    <a href="https://github.com/deforum-art/deforum-for-automatic1111-webui/commits"><img alt="Last Commit" src="https://img.shields.io/github/last-commit/deforum-art/deforum-for-automatic1111-webui"></a>
+    <a href="https://github.com/deforum-art/deforum-for-automatic1111-webui/issues"><img alt="GitHub issues" src="https://img.shields.io/github/issues/deforum-art/deforum-for-automatic1111-webui"></a>
+    <a href="https://github.com/deforum-art/deforum-for-automatic1111-webui/stargazers"><img alt="GitHub stars" src="https://img.shields.io/github/stars/deforum-art/deforum-for-automatic1111-webui"></a>
+    <a href="https://github.com/deforum-art/deforum-for-automatic1111-webui/network"><img alt="GitHub forks" src="https://img.shields.io/github/forks/deforum-art/deforum-for-automatic1111-webui"></a>
+    </a>
+</p>
+
+## Before Starting
+
+**Important note about versions updating:** <br>
+As auto's webui is getting updated multiple times a day, every day, things tend to break with regards to extensions compatability.
+Therefore, it is best recommended to keep two folders:
+1. "Stable" folder that you don't regularly update, with versions that you know *work* together (we will provide info on this soon). 
+2. "Experimental" folder in which you can add 'git pull' to your webui-user.bat, update deforum every day, etc. Keep it wild - but be prepared for bugs. 
+
+
+## Getting Started
+
+1. Install [AUTOMATIC1111's webui](https://github.com/AUTOMATIC1111/stable-diffusion-webui/). <br>If the repo link doesn't work, please use the alternate official download source: [https://gitgud.io/AUTOMATIC1111/stable-diffusion-webui](https://gitgud.io/AUTOMATIC1111/stable-diffusion-webui). To change your existing webui's installation origin, execute `git remote set-url origin https://gitgud.io/AUTOMATIC1111/stable-diffusion-webui` in the webui starting folder.
+
+2. Now two ways: either clone the repo into the `extensions` directory via git commandline launched within in the `stable-diffusion-webui` folder
+
+```sh
+git clone https://github.com/deforum-art/deforum-for-automatic1111-webui extensions/deforum
+```
+
+Or download this repository, locate the `extensions` folder within your WebUI installation, create a folder named `deforum` and put the contents of the downloaded directory inside of it. Then restart WebUI. **Warning: the extension folder has to be named 'deforum' or 'deforum-for-automatic1111-webui', otherwise it will fail to locate the 3D modules as the PATH addition is hardcoded**
+
+3. Open the webui, find the Deforum tab at the top of the page.
+
+4. Enter the animation settings. Refer to [this general guide](https://docs.google.com/document/d/1pEobUknMFMkn8F5TMsv8qRzamXX_75BShMMXV8IFslI/edit) and [this guide to math keyframing functions in Deforum](https://docs.google.com/document/d/1pfW1PwbDIuW0cv-dnuyYj1UzPqe23BlSLTJsqazffXM/edit?usp=sharing). However, **in this version prompt weights less than zero don't just like in original Deforum!** Split the positive and the negative prompt in the json section using --neg argument like this "apple:\`where(cos(t)>=0, cos(t), 0)\`, snow --neg strawberry:\`where(cos(t)<0, -cos(t), 0)\`"
+
+5. To view animation frames as they're being made, without waiting for the completion of an animation, go to the 'Settings' tab and set the value of this toolbar **above zero**. Warning: it may slow down the generation process. If you have 'Do exactly the amount of steps the slider specifies' checkbox selected in the tab, unselect it as it won't allow you to use Deforum schedules and you will get adrupt frame changes without transitions. Then click 'Apply settings' at the top of the page. Now return to the 'Deforum' tab.
+
+![adsdasunknown](https://user-images.githubusercontent.com/14872007/196064311-1b79866a-e55b-438a-84a7-004ff30829ad.png)
+
+
+6. Run the script and see if you got it working or even got something. **In 3D mode a large delay is expected at first** as the script loads the depth models. In the end, using the default settings the whole thing should consume 6.4 GBs of VRAM at 3D mode peaks and no more than 3.8 GB VRAM in 3D mode if you launch the webui with the '--lowvram' command line argument.
+
+7. After the generation process is completed, click the button with the self-describing name to show the video or gif result right in the GUI!
+
+8. Join our Discord where you can post generated stuff, ask questions and more: https://discord.gg/deforum. <br>
+* There's also the 'Issues' tab in the repo, for well... reporting issues ;) 
+
+9. Profit!
+
+## Known issues
+
+* This port is not fully backward-compatible with the notebook and the local version both due to the changes in how AUTOMATIC1111's webui handles Stable Diffusion models and the changes in this script to get it to work in the new environment. *Expect* that you may not get exactly the same result or that the thing may break down because of the older settings.
+
+## Screenshots
+
+https://user-images.githubusercontent.com/121192995/215522284-d6fbedd5-09e2-4d2c-bd10-f9bbb4a20f82.mp4
+
+Main extension tab:
+
+![maintab](https://user-images.githubusercontent.com/121192995/215362176-4e5599c1-9cb6-4bf9-964d-0ff882661993.png)
+
+Keyframes tab:
+
+![keyframes](https://user-images.githubusercontent.com/121192995/215362228-c239c43a-d565-4862-b490-d18b19eaaaa5.png)
+
+Math evaluation:
+
+![math-eval](https://user-images.githubusercontent.com/121192995/215362467-481127a4-247a-4b0d-924a-d10719aa4c01.png)
+
+
+## Benchmarks
+
+3D mode without additional WebUI flags
+
+![image](https://user-images.githubusercontent.com/14872007/196294447-7817f138-ec4b-4001-885f-454f8667100d.png)
+
+3D mode when WebUI is launched with '--lowvram'
+
+![image](https://user-images.githubusercontent.com/14872007/196294517-125fbb27-c06d-4c4b-bcbc-7c743103eff6.png)
+

extensions/deforum/install.py

@@ -0,0 +1,14 @@
+import launch
+import os
+import sys
+
+req_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "requirements.txt")
+
+with open(req_file) as file:
+    for lib in file:
+        lib = lib.strip()
+        if not launch.is_installed(lib):
+            if lib == 'rich':
+                launch.run(f'"{sys.executable}" -m pip install {lib}', desc=f"Installing Deforum requirement: {lib}", errdesc=f"Couldn't install {lib}")
+            else:
+                launch.run_pip(f"install {lib}", f"Deforum requirement: {lib}")

extensions/deforum/javascript/deforum-hints.js

@@ -0,0 +1,191 @@
+// mouseover tooltips for various UI elements
+
+deforum_titles = {
+    //Run
+    "Override settings": "specify a custom settings file and ignore settings displayed in the interface",
+	"Custom settings file": "the path to a custom settings file",
+    "Width": "The width of the output images, in pixels (must be a multiple of 64)",
+    "Height": "The height of the output images, in pixels (must be a multiple of 64)",
+    "Restore faces": "Restore low quality faces using GFPGAN neural network",
+    "Tiling": "Produce an image that can be tiled.",
+    "Highres. fix": "Use a two step process to partially create an image at smaller resolution, upscale, and then improve details in it without changing composition",
+    "Seed": "A value that determines the output of random number generator - if you create an image with same parameters and seed as another image, you'll get the same result",
+    "Sampler": "Which algorithm to use to produce the image",
+    "Enable extras": "enable additional seed settings",
+    "Subseed": "Seed of a different picture to be mixed into the generation.",
+    "Subseed strength": "How strong of a variation to produce. At 0, there will be no effect. At 1, you will get the complete picture with variation seed (except for ancestral samplers, where you will just get something).",
+    "Resize seed from width": "Normally, changing the resolution will completely change an image, even when using the same seed. If you generated an image with a particular seed and then changed the resolution, put the original resolution here to get an image that more closely resemles the original",
+    "Resize seed from height": "Normally, changing the resolution will completely change an image, even when using the same seed. If you generated an image with a particular seed and then changed the resolution, put the original resolution here to get an image that more closely resemles the original",
+    "Steps": "How many times to improve the generated image iteratively; higher values take longer; very low values can produce bad results",
+    //"ddim_eta": "";
+    //"n_batch": "",
+    //"make_grid": "",
+    //"grid_rows": "",
+    //"save_settings": "",
+    //"save_samples": "",
+    "Batch name": "output images will be placed in a folder with this name, inside of the img2img output folder",
+	"Pix2Pix img CFG schedule": "*Only in use with pix2pix checkpoints!*",
+    "Filename format": "specify the format of the filename for output images",
+    "Seed behavior": "defines the seed behavior that is used for animations",
+        "iter": "the seed value will increment by 1 for each subsequent frame of the animation",
+        "fixed": "the seed will remain fixed across all frames of animation",
+        "random": "a random seed will be used on each frame of the animation",
+		"schedule": "specify your own seed schedule (found on the Keyframes page)",
+
+    //Keyframes
+    "Animation mode": "selects the type of animation",
+        "2D": "only 2D motion parameters will be used, but this mode uses the least amount of VRAM. You can optionally enable flip_2d_perspective to enable some psuedo-3d animation parameters while in 2D mode.",
+        "3D": "enables all 3D motion parameters.",
+        "Video Input": "will ignore all motion parameters and attempt to reference a video loaded into the runtime, specified by the video_init_path. Max_frames is ignored during video_input mode, and instead, follows the number of frames pulled from the video’s length. Resume_from_timestring is NOT available with Video_Input mode.",
+    "Max frames": "the maximum number of output images to be created",
+    "Border": "controls handling method of pixels to be generated when the image is smaller than the frame.",
+        "wrap": "pulls pixels from the opposite edge of the image",
+        "replicate": "repeats the edge of the pixels, and extends them. Animations with quick motion may yield lines where this border function was attempting to populate pixels into the empty space created.",
+    "Angle": "2D operator to rotate canvas clockwise/anticlockwise in degrees per frame",
+    "Zoom": "2D operator that scales the canvas size, multiplicatively. [static = 1.0]",
+    "Translation X": "2D & 3D operator to move canvas left/right in pixels per frame",
+    "Translation Y": "2D & 3D operator to move canvas up/down in pixels per frame", 
+    "Translation Z": "3D operator to move canvas towards/away from view [speed set by FOV]",
+    "Rotation 3D X": "3D operator to tilt canvas up/down in degrees per frame",
+    "Rotation 3D Y": "3D operator to pan canvas left/right in degrees per frame",
+    "Rotation 3D Z": "3D operator to roll canvas clockwise/anticlockwise",
+    "Enable perspective flip": "enables 2D mode functions to simulate faux 3D movement",
+    "Perspective flip theta": "the roll effect angle",
+    "Perspective flip phi": "the tilt effect angle",
+    "Perspective flip gamma": "the pan effect angle",
+    "Perspective flip fv": "the 2D vanishing point of perspective (recommended range 30-160)",
+    "Noise schedule": "amount of graininess to add per frame for diffusion diversity",
+    "Strength schedule": "amount of presence of previous frame to influence next frame, also controls steps in the following formula [steps - (strength_schedule * steps)]",
+    "Sampler schedule": "controls which sampler to use at a specific scheduled frame",
+    "Contrast schedule": "adjusts the overall contrast per frame [default neutral at 1.0]",
+    "CFG scale schedule": "how closely the image should conform to the prompt. Lower values produce more creative results. (recommended range 5-15)",
+    "FOV schedule": "adjusts the scale at which the canvas is moved in 3D by the translation_z value. [maximum range -180 to +180, with 0 being undefined. Values closer to 180 will make the image have less depth, while values closer to 0 will allow more depth]",
+    //"near_schedule": "",
+    //"far_schedule":  "",
+    "Seed schedule": "allows you to specify seeds at a specific schedule, if seed_behavior is set to schedule.",
+    "Color coherence": "The color coherence will attempt to sample the overall pixel color information, and trend those values analyzed in the first frame to be applied to future frames.",
+        // "None": "Disable color coherence",
+        "Match Frame 0 HSV": "HSV is a good method for balancing presence of vibrant colors, but may produce unrealistic results - (ie.blue apples)",
+        "Match Frame 0 LAB": "LAB is a more linear approach to mimic human perception of color space - a good default setting for most users.",
+        "Match Frame 0 RGB": "RGB is good for enforcing unbiased amounts of color in each red, green and blue channel - some images may yield colorized artifacts if sampling is too low.",
+    "Cadence": "A setting of 1 will cause every frame to receive diffusion in the sequence of image outputs. A setting of 2 will only diffuse on every other frame, yet motion will still be in effect. The output of images during the cadence sequence will be automatically blended, additively and saved to the specified drive. This may improve the illusion of coherence in some workflows as the content and context of an image will not change or diffuse during frames that were skipped. Higher values of 4-8 cadence will skip over a larger amount of frames and only diffuse the “Nth” frame as set by the diffusion_cadence value. This may produce more continuity in an animation, at the cost of little opportunity to add more diffused content. In extreme examples, motion within a frame will fail to produce diverse prompt context, and the space will be filled with lines or approximations of content - resulting in unexpected animation patterns and artifacts. Video Input & Interpolation modes are not affected by diffusion_cadence.",
+    "Noise type": "Selects the type of noise being added to each frame",
+        "uniform": "Uniform noise covers the entire frame. It somewhat flattens and sharpens the video over time, but may be good for cartoonish look. This is the old default setting.",
+        "perlin": "Perlin noise is a more natural looking noise. It is heterogeneous and less sharp than uniform noise, this way it is more likely that new details will appear in a more coherent way. This is the new default setting.",
+    "Perlin W": "The width of the Perlin sample. Lower values will make larger noise regions. Think of it as inverse brush stroke width. The greater this setting, the smaller details it will affect.",
+    "Perlin H": "The height of the Perlin sample. Lower values will make larger noise regions. Think of it as inverse brush stroke width. The greater this setting, the smaller details it will affect.",
+    "Perlin octaves": "The number of Perlin noise octaves, that is the count of P-noise iterations. Higher values will make the noise more soft and smoke-like, whereas lower values will make it look more organic and spotty. It is limited by 8 octaves as the resulting gain will run out of bounds.",
+    "Perlin persistence": "How much of noise from each octave is added on each iteration. Higher values will make it more straighter and sharper, while lower values will make it rounder and smoother. It is limited by 1.0 as the resulting gain fill the frame completely with noise.",
+    "Use depth warping": "enables instructions to warp an image dynamically in 3D mode only.",
+    "MiDaS weight": "sets a midpoint at which a depthmap is to be drawn: range [-1 to +1]",
+    "Padding mode": "instructs the handling of pixels outside the field of view as they come into the scene.",
+	    //"border": "Border will attempt to use the edges of the canvas as the pixels to be drawn", //duplicate name as another property
+	    "reflection": "reflection will attempt to approximate the image and tile/repeat pixels",
+	    "zeros": "zeros will not add any new pixel information",
+	"sampling_mode": "choose from Bicubic, Bilinear or Nearest modes. (Recommended: Bicubic)",
+    "Save depth maps": "will output a greyscale depth map image alongside the output images.",
+    
+	// Prompts
+	"Prompts": "prompts for your animation in a JSON format. Use --neg words to add 'words' as negative prompt",
+	"Prompts positive": "positive prompt to be appended to *all* prompts",
+	"Prompts negative": "negative prompt to be appended to *all* prompts. DON'T use --neg here!",
+	
+    //Init
+	"Use init": "Diffuse the first frame based on an image, similar to img2img.",
+    "Strength": "Controls the strength of the diffusion on the init image. 0 = disabled",
+    "Strength 0 no init": "Set the strength to 0 automatically when no init image is used",
+    "Init image": "the path to your init image",
+    "Use mask": "Use a grayscale image as a mask on your init image. Whiter areas of the mask are areas that change more.",
+    "Use alpha as mask": "use the alpha channel of the init image as the mask",
+    "Mask file": "the path to your mask image",
+    "Invert mask": "Inverts the colors of the mask",
+    "Mask brightness adjust": "adjust the brightness of the mask. Should be a positive number, with 1.0 meaning no adjustment.",
+    "Mask contrast adjust": "adjust the brightness of the mask. Should be a positive number, with 1.0 meaning no adjustment.",
+    "overlay mask": "Overlay the masked image at the end of the generation so it does not get degraded by encoding and decoding",
+    "Mask overlay blur": "Blur edges of final overlay mask, if used. Minimum = 0 (no blur)",
+    "Video init path": "the directory \/ URL at which your video file is located for Video Input mode only",
+    "Extract nth frame": "during the run sequence, only frames specified by this value will be extracted, saved, and diffused upon. A value of 1 indicates that every frame is to be accounted for. Values of 2 will use every other frame for the sequence. Higher values will skip that number of frames respectively.",
+	"Extract from frame":"start extracting the input video only from this frame number",
+	"Extract to frame": "stop the extraction of the video at this frame number. -1 for no limits",
+    "Overwrite extracted frames": "when enabled, will re-extract video frames each run. When using video_input mode, the run will be instructed to write video frames to the drive. If you’ve already populated the frames needed, uncheck this box to skip past redundant extraction, and immediately start the render. If you have not extracted frames, you must run at least once with this box checked to write the necessary frames.",
+    "Use mask video": "video_input mode only, enables the extraction and use of a separate video file intended for use as a mask. White areas of the extracted video frames will not be affected by diffusion, while black areas will be fully effected. Lighter/darker areas are affected dynamically.",
+    "Video mask path": "the directory in which your mask video is located.",
+    "Interpolate key frames": "selects whether to ignore prompt schedule or _x_frames.",
+    "Interpolate x frames": "the number of frames to transition thru between prompts (when interpolate_key_frames = true, then the numbers in front of the animation prompts will dynamically guide the images based on their value. If set to false, will ignore the prompt numbers and force interpole_x_frames value regardless of prompt number)",
+    "Resume from timestring": "instructs the run to start from a specified point",
+    "Resume timestring": "the required timestamp to reference when resuming. Currently only available in 2D & 3D mode, the timestamp is saved as the settings .txt file name as well as images produced during your previous run. The format follows: yyyymmddhhmmss - a timestamp of when the run was started to diffuse.",
+	
+    //Video Output
+    "Skip video for run all": "when checked, do not output a video",
+	"Make GIF": "create a gif in addition to .mp4 file. supports up to 30 fps, will self-disable at higher fps values",
+	"Upscale":"upscale the images of the next run once it's finished + make a video out of them",
+	"Upscale model":"model of the upscaler to use. 'realesr-animevideov3' is much faster but yields smoother, less detailed results. the other models only do x4",
+	"Upscale factor":"how many times to upscale, actual options depend on the chosen upscale model",
+    "FPS": "The frames per second that the video will run at",
+    "Output format": "select the type of video file to output",
+        "PIL gif": "create an animated GIF",
+        "FFMPEG mp4": "create an MP4 video file",
+    "FFmpeg location": "the path to where ffmpeg is located. Leave at default 'ffmpeg' if ffmpeg is in your PATH!",
+	"FFmpeg crf": "controls quality where lower is better, less compressed. values: 0 to 51, default 17",
+	"FFmpeg preset": "controls how good the compression is, and the operation speed. If you're not in a rush keep it at 'veryslow'",
+    "Add soundtrack": "when this box is checked, and FFMPEG mp4 is selected as the output format, an audio file will be multiplexed with the video.",
+    "Soundtrack path": "the path\/ URL to an audio file to accompany the video",
+    "Use manual settings": "when this is unchecked, the video will automatically be created in the same output folder as the images. Check this box to specify different settings for the creation of the video, specified by the following options",
+    "Render steps": "render each step of diffusion as a separate frame",
+    "Max video frames": "the maximum number of frames to include in the video, when use_manual_settings is checked",
+    //"path_name_modifier": "",
+    "Image path": "the location of images to create the video from, when use_manual_settings is checked",
+    "MP4 path": "the output location of the mp4 file, when use_manual_settings is checked",
+	"Engine": "choose the frame interpolation engine and version",
+	"Interp X":"how many times to interpolate the source video. e.g source video fps of 12 and a value of x2 will yield a 24fps interpolated video",
+	"Slow-Mo X":"how many times to slow-down the video. *Naturally affects output fps as well",
+	"Keep Imgs": "delete or keep raw affected (interpolated/ upscaled depending on the UI section) png imgs",
+	"Interpolate an existing video":"This feature allows you to interpolate any video with a dedicated button. Video could be completly unrelated to deforum",
+	"In Frame Count": "uploaded video total frame count",
+	"In FPS":"uploaded video FPS",
+	"Interpolated Vid FPS":"calculated output-interpolated video FPS",
+	"In Res":"uploaded video resolution",
+	"Out Res":"output video resolution",
+
+    // Looper Args
+    // "use_looper": "",
+	"Enable guided images mode": "check this box to enable guided images mode",
+    "Images to use for keyframe guidance": "images you iterate over, you can do local or web paths (no single backslashes!)",
+    "Image strength schedule": "how much the image should look like the previou one and new image frame init. strength schedule might be better if this is higher, around .75 during the keyfames you want to switch on",
+    "Blend factor max": "blendFactor = blendFactorMax - blendFactorSlope * cos((frame % tweening_frames_schedule) / (tweening_frames_schedule / 2))",
+    "Blend factor slope": "blendFactor = blendFactorMax - blendFactorSlope * cos((frame % tweening_frames_schedule) / (tweening_frames_schedule / 2))",
+    "Tweening frames schedule": "number of the frames that we will blend between current imagined image and input frame image",
+    "Color correction factor": "how close to get to the colors of the input frame image/ the amount each frame during a tweening step to use the new images colors"
+}
+
+
+onUiUpdate(function(){
+	gradioApp().querySelectorAll('span, button, select, p').forEach(function(span){
+		tooltip = deforum_titles[span.textContent];
+
+		if(!tooltip){
+		    tooltip = deforum_titles[span.value];
+		}
+
+		if(!tooltip){
+			for (const c of span.classList) {
+				if (c in deforum_titles) {
+					tooltip = deforum_titles[c];
+					break;
+				}
+			}
+		}
+
+		if(tooltip){
+			span.title = tooltip;
+		}
+	})
+
+	gradioApp().querySelectorAll('select').forEach(function(select){
+	    if (select.onchange != null) return;
+
+	    select.onchange = function(){
+            select.title = deforum_titles[select.value] || "";
+	    }
+	})
+})

extensions/deforum/javascript/deforum.js

@@ -0,0 +1,21 @@
+function submit_deforum(){
+	// alert('Hello, Deforum!')
+    rememberGallerySelection('deforum_gallery')
+    showSubmitButtons('deforum', false)
+
+    var id = randomId()
+    requestProgress(id, gradioApp().getElementById('deforum_gallery_container'), gradioApp().getElementById('deforum_gallery'), function(){
+        showSubmitButtons('deforum', true)
+    })
+
+    var res = create_submit_args(arguments)
+
+    res[0] = id
+	// res[1] = get_tab_index('deforum')
+
+    return res
+}
+
+onUiUpdate(function(){
+    check_gallery('deforum_gallery')
+})

extensions/deforum/requirements.txt

@@ -0,0 +1,7 @@
+numexpr
+matplotlib
+pandas
+av
+pims
+imageio_ffmpeg
+rich

extensions/deforum/scripts/deforum.py

@@ -0,0 +1,318 @@
+# Detach 'deforum_helpers' from 'scripts' to prevent "No module named 'scripts.deforum_helpers'"  error 
+# causing Deforum's tab not show up in some cases when you've might've broken the environment with webui packages updates
+import sys, os, shutil
+
+basedirs = [os.getcwd()]
+if 'google.colab' in sys.modules:
+    basedirs.append('/content/gdrive/MyDrive/sd/stable-diffusion-webui') #hardcode as TheLastBen's colab seems to be the primal source
+
+for basedir in basedirs:
+    deforum_paths_to_ensure = [basedir + '/extensions/deforum-for-automatic1111-webui/scripts', basedir + '/extensions/sd-webui-controlnet', basedir + '/extensions/deforum/scripts', basedir + '/scripts/deforum_helpers/src', basedir + '/extensions/deforum/scripts/deforum_helpers/src', basedir +'/extensions/deforum-for-automatic1111-webui/scripts/deforum_helpers/src',basedir]
+
+    for deforum_scripts_path_fix in deforum_paths_to_ensure:
+        if not deforum_scripts_path_fix in sys.path:
+            sys.path.extend([deforum_scripts_path_fix])
+
+# Main deforum stuff
+import deforum_helpers.args as deforum_args
+import deforum_helpers.settings as deforum_settings
+from deforum_helpers.save_images import dump_frames_cache, reset_frames_cache
+from deforum_helpers.frame_interpolation import process_video_interpolation
+
+import modules.scripts as wscripts
+from modules import script_callbacks
+import gradio as gr
+import json
+
+from modules.processing import Processed, StableDiffusionProcessingImg2Img, process_images
+from PIL import Image
+from deforum_helpers.video_audio_utilities import ffmpeg_stitch_video, make_gifski_gif
+from deforum_helpers.upscaling import make_upscale_v2
+import gc
+import torch
+from webui import wrap_gradio_gpu_call
+import modules.shared as shared
+from modules.shared import opts, cmd_opts, state
+from modules.ui import create_output_panel, plaintext_to_html, wrap_gradio_call
+from types import SimpleNamespace
+
+def run_deforum(*args, **kwargs):
+    args_dict = {deforum_args.component_names[i]: args[i+2] for i in range(0, len(deforum_args.component_names))}
+    p = StableDiffusionProcessingImg2Img(
+        sd_model=shared.sd_model,
+        outpath_samples = opts.outdir_samples or opts.outdir_img2img_samples,
+        outpath_grids = opts.outdir_grids or opts.outdir_img2img_grids,
+        #we'll setup the rest later
+    )
+    
+    print("\033[4;33mDeforum extension for auto1111 webui, v2.2b\033[0m")
+    args_dict['self'] = None
+    args_dict['p'] = p
+    
+    root, args, anim_args, video_args, parseq_args, loop_args, controlnet_args = deforum_args.process_args(args_dict)
+    root.clipseg_model = None
+    root.initial_clipskip = opts.data["CLIP_stop_at_last_layers"]
+    root.basedirs = basedirs
+
+    for basedir in basedirs:
+        sys.path.extend([
+            basedir + '/scripts/deforum_helpers/src',
+            basedir + '/extensions/deforum/scripts/deforum_helpers/src',
+            basedir + '/extensions/deforum-for-automatic1111-webui/scripts/deforum_helpers/src',
+        ])
+    
+    # clean up unused memory
+    reset_frames_cache(root)
+    gc.collect()
+    torch.cuda.empty_cache()
+    
+    from deforum_helpers.render import render_animation
+    from deforum_helpers.render_modes import render_input_video, render_animation_with_video_mask, render_interpolation
+
+    tqdm_backup = shared.total_tqdm
+    shared.total_tqdm = deforum_settings.DeforumTQDM(args, anim_args, parseq_args)
+    try:
+        # dispatch to appropriate renderer
+        if anim_args.animation_mode == '2D' or anim_args.animation_mode == '3D':
+            if anim_args.use_mask_video: 
+                render_animation_with_video_mask(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, root.animation_prompts, root) # allow mask video without an input video
+            else:    
+                render_animation(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, root.animation_prompts, root)
+        elif anim_args.animation_mode == 'Video Input':
+            render_input_video(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, root.animation_prompts, root)#TODO: prettify code
+        elif anim_args.animation_mode == 'Interpolation':
+            render_interpolation(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, root.animation_prompts, root)
+        else:
+            print('Other modes are not available yet!')
+    finally:
+        shared.total_tqdm = tqdm_backup
+        opts.data["CLIP_stop_at_last_layers"] = root.initial_clipskip
+    
+    if video_args.store_frames_in_ram:
+        dump_frames_cache(root)
+    
+    from base64 import b64encode
+    
+    real_audio_track = None
+    if video_args.add_soundtrack != 'None':
+        real_audio_track = anim_args.video_init_path if video_args.add_soundtrack == 'Init Video' else video_args.soundtrack_path
+    
+    # Delete folder with duplicated imgs from OS temp folder
+    shutil.rmtree(root.tmp_deforum_run_duplicated_folder, ignore_errors=True)
+
+    # Decide whether or not we need to try and frame interpolate laters
+    need_to_frame_interpolate = False
+    if video_args.frame_interpolation_engine != "None" and not video_args.skip_video_for_run_all and not video_args.store_frames_in_ram:
+        need_to_frame_interpolate = True
+        
+    if video_args.skip_video_for_run_all:
+        print('Skipping video creation, uncheck skip_video_for_run_all if you want to run it')
+    else:
+        import subprocess
+
+        path_name_modifier = video_args.path_name_modifier
+        if video_args.render_steps: # render steps from a single image
+            fname = f"{path_name_modifier}_%05d.png"
+            all_step_dirs = [os.path.join(args.outdir, d) for d in os.listdir(args.outdir) if os.path.isdir(os.path.join(args.outdir,d))]
+            newest_dir = max(all_step_dirs, key=os.path.getmtime)
+            image_path = os.path.join(newest_dir, fname)
+            print(f"Reading images from {image_path}")
+            mp4_path = os.path.join(newest_dir, f"{args.timestring}_{path_name_modifier}.mp4")
+            max_video_frames = args.steps
+        else: # render images for a video
+            image_path = os.path.join(args.outdir, f"{args.timestring}_%05d.png")
+            mp4_path = os.path.join(args.outdir, f"{args.timestring}.mp4")
+            max_video_frames = anim_args.max_frames
+
+        exclude_keys = deforum_settings.get_keys_to_exclude('video')
+        video_settings_filename = os.path.join(args.outdir, f"{args.timestring}_video-settings.txt")
+        with open(video_settings_filename, "w+", encoding="utf-8") as f:
+            s = {}
+            for key, value in dict(video_args.__dict__).items():
+                if key not in exclude_keys:
+                    s[key] = value
+            json.dump(s, f, ensure_ascii=False, indent=4)
+
+        # Stitch video using ffmpeg!
+        try:
+            ffmpeg_stitch_video(ffmpeg_location=video_args.ffmpeg_location, fps=video_args.fps, outmp4_path=mp4_path, stitch_from_frame=0, stitch_to_frame=max_video_frames, imgs_path=image_path, add_soundtrack=video_args.add_soundtrack, audio_path=real_audio_track, crf=video_args.ffmpeg_crf, preset=video_args.ffmpeg_preset)
+            mp4 = open(mp4_path,'rb').read()
+            data_url = "data:video/mp4;base64," + b64encode(mp4).decode()
+            deforum_args.i1_store = f'<p style=\"font-weight:bold;margin-bottom:0em\">Deforum v0.5-webui-beta</p><video controls loop><source src="{data_url}" type="video/mp4"></video>'
+        except Exception as e:
+            if need_to_frame_interpolate:
+                print(f"FFMPEG DID NOT STITCH ANY VIDEO. However, you requested to frame interpolate  - so we will continue to frame interpolation, but you'll be left only with the interpolated frames and not a video, since ffmpeg couldn't run. Original ffmpeg error: {e}")
+            else:
+                print(f"** FFMPEG DID NOT STITCH ANY VIDEO ** Error: {e}")
+            pass
+            
+    if root.initial_info is None:
+        root.initial_info = "An error has occured and nothing has been generated!"
+        root.initial_info += "\nPlease, report the bug to https://github.com/deforum-art/deforum-for-automatic1111-webui/issues"
+        import numpy as np
+        a = np.random.rand(args.W, args.H, 3)*255
+        root.first_frame = Image.fromarray(a.astype('uint8')).convert('RGB')
+        root.initial_seed = 6934
+    # FRAME INTERPOLATION TIME
+    if need_to_frame_interpolate: 
+        print(f"Got a request to *frame interpolate* using {video_args.frame_interpolation_engine}")
+        process_video_interpolation(frame_interpolation_engine=video_args.frame_interpolation_engine, frame_interpolation_x_amount=video_args.frame_interpolation_x_amount,frame_interpolation_slow_mo_enabled=video_args.frame_interpolation_slow_mo_enabled, frame_interpolation_slow_mo_amount=video_args.frame_interpolation_slow_mo_amount, orig_vid_fps=video_args.fps, deforum_models_path=root.models_path, real_audio_track=real_audio_track, raw_output_imgs_path=args.outdir, img_batch_id=args.timestring, ffmpeg_location=video_args.ffmpeg_location, ffmpeg_crf=video_args.ffmpeg_crf, ffmpeg_preset=video_args.ffmpeg_preset, keep_interp_imgs=video_args.frame_interpolation_keep_imgs, orig_vid_name=None, resolution=None)
+    
+    if video_args.make_gif and not video_args.skip_video_for_run_all and not video_args.store_frames_in_ram:
+        make_gifski_gif(imgs_raw_path = args.outdir, imgs_batch_id = args.timestring, fps = video_args.fps, models_folder = root.models_path, current_user_os = root.current_user_os)
+    
+    # Upscale video once generation is done:
+    if video_args.r_upscale_video and not video_args.skip_video_for_run_all and not video_args.store_frames_in_ram:
+        
+        # out mp4 path is defined in make_upscale func
+        make_upscale_v2(upscale_factor = video_args.r_upscale_factor, upscale_model = video_args.r_upscale_model, keep_imgs = video_args.r_upscale_keep_imgs, imgs_raw_path = args.outdir, imgs_batch_id = args.timestring, fps = video_args.fps, deforum_models_path = root.models_path, current_user_os = root.current_user_os, ffmpeg_location=video_args.ffmpeg_location, stitch_from_frame=0, stitch_to_frame=max_video_frames, ffmpeg_crf=video_args.ffmpeg_crf, ffmpeg_preset=video_args.ffmpeg_preset, add_soundtrack = video_args.add_soundtrack ,audio_path=real_audio_track)
+        
+    root.initial_info += "\n The animation is stored in " + args.outdir
+    root.initial_info += "\n Timestring = " + args.timestring + '\n'
+    root.initial_info += "Only the first frame is shown in webui not to clutter the memory"
+    reset_frames_cache(root) # cleanup the RAM in any case
+    processed = Processed(p, [root.first_frame], root.initial_seed, root.initial_info)
+    
+    if processed is None:
+        processed = process_images(p)
+
+    shared.total_tqdm.clear()
+
+    generation_info_js = processed.js()
+    if opts.samples_log_stdout:
+        print(generation_info_js)
+
+    if opts.do_not_show_images:
+        processed.images = []
+
+    return processed.images, generation_info_js, plaintext_to_html(processed.info), plaintext_to_html('')
+
+def on_ui_tabs():
+    with gr.Blocks(analytics_enabled=False) as deforum_interface:
+        components = {}
+        dummy_component = gr.Label(visible=False)
+        with gr.Row(elem_id='deforum_progress_row').style(equal_height=False):
+            with gr.Column(scale=1, variant='panel'):
+                components = deforum_args.setup_deforum_setting_dictionary(None, True, True)
+        
+            with gr.Column(scale=1):
+                with gr.Row():
+                    btn = gr.Button("Click here after the generation to show the video")
+                    components['btn'] = btn
+                    close_btn = gr.Button("Close the video", visible=False)
+                with gr.Row():
+                    i1 = gr.HTML(deforum_args.i1_store, elem_id='deforum_header')
+                    components['i1'] = i1
+                    # Show video
+                    def show_vid():
+                        return {
+                            i1: gr.update(value=deforum_args.i1_store, visible=True),
+                            close_btn: gr.update(visible=True),
+                            btn: gr.update(value="Update the video", visible=True),
+                        }
+                
+                    btn.click(
+                        show_vid,
+                        [],
+                        [i1, close_btn, btn],
+                        )
+                    # Close video
+                    def close_vid():
+                        return {
+                            i1: gr.update(value=deforum_args.i1_store_backup, visible=True),
+                            close_btn: gr.update(visible=False),
+                            btn: gr.update(value="Click here after the generation to show the video", visible=True),
+                        }
+                    
+                    close_btn.click(
+                        close_vid,
+                        [],
+                        [i1, close_btn, btn],
+                        )
+                id_part = 'deforum'
+                with gr.Row(elem_id=f"{id_part}_generate_box"):
+                    skip = gr.Button('Skip', elem_id=f"{id_part}_skip", visible=False)
+                    interrupt = gr.Button('Interrupt', elem_id=f"{id_part}_interrupt", visible=True)
+                    submit = gr.Button('Generate', elem_id=f"{id_part}_generate", variant='primary')
+
+                    skip.click(
+                        fn=lambda: state.skip(),
+                        inputs=[],
+                        outputs=[],
+                    )
+
+                    interrupt.click(
+                        fn=lambda: state.interrupt(),
+                        inputs=[],
+                        outputs=[],
+                    )
+                
+                deforum_gallery, generation_info, html_info, html_log = create_output_panel("deforum", opts.outdir_img2img_samples)
+
+                gr.HTML("<p>* Paths can be relative to webui folder OR full - absolute </p>")
+                with gr.Row():
+                    settings_path = gr.Textbox("deforum_settings.txt", elem_id='deforum_settings_path', label="General Settings File")
+                    #reuse_latest_settings_btn = gr.Button('Reuse Latest', elem_id='deforum_reuse_latest_settings_btn')#TODO
+                with gr.Row():
+                    save_settings_btn = gr.Button('Save Settings', elem_id='deforum_save_settings_btn')
+                    load_settings_btn = gr.Button('Load Settings', elem_id='deforum_load_settings_btn')
+                with gr.Row():
+                    video_settings_path = gr.Textbox("deforum_video-settings.txt", elem_id='deforum_video_settings_path', label="Video Settings File")
+                    #reuse_latest_video_settings_btn = gr.Button('Reuse Latest', elem_id='deforum_reuse_latest_video_settings_btn')#TODO
+                with gr.Row():
+                    save_video_settings_btn = gr.Button('Save Video Settings', elem_id='deforum_save_video_settings_btn')
+                    load_video_settings_btn = gr.Button('Load Video Settings', elem_id='deforum_load_video_settings_btn')
+
+                # components['prompts'].visible = False#hide prompts for the time being
+                #TODO clean up the code
+                components['save_sample_per_step'].visible = False
+                components['show_sample_per_step'].visible = False
+                components['display_samples'].visible = False
+
+        component_list = [components[name] for name in deforum_args.component_names]
+
+        submit.click(
+                    fn=wrap_gradio_gpu_call(run_deforum, extra_outputs=[None, '', '']),
+                    _js="submit_deforum",
+                    inputs=[dummy_component, dummy_component] + component_list,
+                    outputs=[
+                         deforum_gallery,
+                         generation_info,
+                         html_info,
+                         html_log,
+                    ],
+                )
+        
+        settings_component_list = [components[name] for name in deforum_args.settings_component_names]
+        video_settings_component_list = [components[name] for name in deforum_args.video_args_names]
+        stuff = gr.HTML("") # wrap gradio call garbage
+        stuff.visible = False
+        
+        save_settings_btn.click(
+                    fn=wrap_gradio_call(deforum_settings.save_settings),
+                    inputs=[settings_path] + settings_component_list,
+                    outputs=[stuff],
+                )
+        
+        load_settings_btn.click(
+                    fn=wrap_gradio_call(deforum_settings.load_settings),
+                    inputs=[settings_path]+ settings_component_list,
+                    outputs=settings_component_list + [stuff],
+                )
+        
+        save_video_settings_btn.click(
+                    fn=wrap_gradio_call(deforum_settings.save_video_settings),
+                    inputs=[video_settings_path] + video_settings_component_list,
+                    outputs=[stuff],
+                )
+        
+        load_video_settings_btn.click(
+                    fn=wrap_gradio_call(deforum_settings.load_video_settings),
+                    inputs=[video_settings_path] + video_settings_component_list,
+                    outputs=video_settings_component_list + [stuff],
+                )
+
+
+    return [(deforum_interface, "Deforum", "deforum_interface")]
+
+script_callbacks.on_ui_tabs(on_ui_tabs)

extensions/deforum/scripts/deforum_helpers/__init__.py

@@ -0,0 +1,7 @@
+"""
+from .save_images import save_samples, get_output_folder
+from .depth import DepthModel
+from .prompt import sanitize
+from .animation import construct_RotationMatrixHomogenous, getRotationMatrixManual, getPoints_for_PerspectiveTranformEstimation, warpMatrix, anim_frame_warp_2d, anim_frame_warp_3d
+from .generate import add_noise, load_img, load_mask_latent, prepare_mask
+"""

extensions/deforum/scripts/deforum_helpers/animation.py

@@ -0,0 +1,258 @@
+import numpy as np
+import cv2
+from functools import reduce
+import math
+import py3d_tools as p3d
+import torch
+from einops import rearrange
+from .prompt import check_is_number
+
+# Webui
+from modules.shared import state
+
+def sample_from_cv2(sample: np.ndarray) -> torch.Tensor:
+    sample = ((sample.astype(float) / 255.0) * 2) - 1
+    sample = sample[None].transpose(0, 3, 1, 2).astype(np.float16)
+    sample = torch.from_numpy(sample)
+    return sample
+
+def sample_to_cv2(sample: torch.Tensor, type=np.uint8) -> np.ndarray:
+    sample_f32 = rearrange(sample.squeeze().cpu().numpy(), "c h w -> h w c").astype(np.float32)
+    sample_f32 = ((sample_f32 * 0.5) + 0.5).clip(0, 1)
+    sample_int8 = (sample_f32 * 255)
+    return sample_int8.astype(type)
+
+def construct_RotationMatrixHomogenous(rotation_angles):
+    assert(type(rotation_angles)==list and len(rotation_angles)==3)
+    RH = np.eye(4,4)
+    cv2.Rodrigues(np.array(rotation_angles), RH[0:3, 0:3])
+    return RH
+
+# https://en.wikipedia.org/wiki/Rotation_matrix
+def getRotationMatrixManual(rotation_angles):
+	
+    rotation_angles = [np.deg2rad(x) for x in rotation_angles]
+    
+    phi         = rotation_angles[0] # around x
+    gamma       = rotation_angles[1] # around y
+    theta       = rotation_angles[2] # around z
+    
+    # X rotation
+    Rphi        = np.eye(4,4)
+    sp          = np.sin(phi)
+    cp          = np.cos(phi)
+    Rphi[1,1]   = cp
+    Rphi[2,2]   = Rphi[1,1]
+    Rphi[1,2]   = -sp
+    Rphi[2,1]   = sp
+    
+    # Y rotation
+    Rgamma        = np.eye(4,4)
+    sg            = np.sin(gamma)
+    cg            = np.cos(gamma)
+    Rgamma[0,0]   = cg
+    Rgamma[2,2]   = Rgamma[0,0]
+    Rgamma[0,2]   = sg
+    Rgamma[2,0]   = -sg
+    
+    # Z rotation (in-image-plane)
+    Rtheta      = np.eye(4,4)
+    st          = np.sin(theta)
+    ct          = np.cos(theta)
+    Rtheta[0,0] = ct
+    Rtheta[1,1] = Rtheta[0,0]
+    Rtheta[0,1] = -st
+    Rtheta[1,0] = st
+    
+    R           = reduce(lambda x,y : np.matmul(x,y), [Rphi, Rgamma, Rtheta]) 
+    
+    return R
+
+def getPoints_for_PerspectiveTranformEstimation(ptsIn, ptsOut, W, H, sidelength):
+    
+    ptsIn2D      =  ptsIn[0,:]
+    ptsOut2D     =  ptsOut[0,:]
+    ptsOut2Dlist =  []
+    ptsIn2Dlist  =  []
+    
+    for i in range(0,4):
+        ptsOut2Dlist.append([ptsOut2D[i,0], ptsOut2D[i,1]])
+        ptsIn2Dlist.append([ptsIn2D[i,0], ptsIn2D[i,1]])
+    
+    pin  =  np.array(ptsIn2Dlist)   +  [W/2.,H/2.]
+    pout = (np.array(ptsOut2Dlist)  +  [1.,1.]) * (0.5*sidelength)
+    pin  = pin.astype(np.float32)
+    pout = pout.astype(np.float32)
+    
+    return pin, pout
+
+
+def warpMatrix(W, H, theta, phi, gamma, scale, fV):
+    
+    # M is to be estimated
+    M          = np.eye(4, 4)
+    
+    fVhalf     = np.deg2rad(fV/2.)
+    d          = np.sqrt(W*W+H*H)
+    sideLength = scale*d/np.cos(fVhalf)
+    h          = d/(2.0*np.sin(fVhalf))
+    n          = h-(d/2.0)
+    f          = h+(d/2.0)
+    
+    # Translation along Z-axis by -h
+    T       = np.eye(4,4)
+    T[2,3]  = -h
+    
+    # Rotation matrices around x,y,z
+    R = getRotationMatrixManual([phi, gamma, theta])
+    
+    
+    # Projection Matrix 
+    P       = np.eye(4,4)
+    P[0,0]  = 1.0/np.tan(fVhalf)
+    P[1,1]  = P[0,0]
+    P[2,2]  = -(f+n)/(f-n)
+    P[2,3]  = -(2.0*f*n)/(f-n)
+    P[3,2]  = -1.0
+    
+    # pythonic matrix multiplication
+    F       = reduce(lambda x,y : np.matmul(x,y), [P, T, R]) 
+    
+    # shape should be 1,4,3 for ptsIn and ptsOut since perspectiveTransform() expects data in this way. 
+    # In C++, this can be achieved by Mat ptsIn(1,4,CV_64FC3);
+    ptsIn = np.array([[
+                 [-W/2., H/2., 0.],[ W/2., H/2., 0.],[ W/2.,-H/2., 0.],[-W/2.,-H/2., 0.]
+                 ]])
+    ptsOut  = np.array(np.zeros((ptsIn.shape), dtype=ptsIn.dtype))
+    ptsOut  = cv2.perspectiveTransform(ptsIn, F)
+    
+    ptsInPt2f, ptsOutPt2f = getPoints_for_PerspectiveTranformEstimation(ptsIn, ptsOut, W, H, sideLength)
+    
+    # check float32 otherwise OpenCV throws an error
+    assert(ptsInPt2f.dtype  == np.float32)
+    assert(ptsOutPt2f.dtype == np.float32)
+    M33 = cv2.getPerspectiveTransform(ptsInPt2f,ptsOutPt2f)
+
+    return M33, sideLength
+
+def get_flip_perspective_matrix(W, H, keys, frame_idx):
+    perspective_flip_theta = keys.perspective_flip_theta_series[frame_idx]
+    perspective_flip_phi = keys.perspective_flip_phi_series[frame_idx]
+    perspective_flip_gamma = keys.perspective_flip_gamma_series[frame_idx]
+    perspective_flip_fv = keys.perspective_flip_fv_series[frame_idx]
+    M,sl = warpMatrix(W, H, perspective_flip_theta, perspective_flip_phi, perspective_flip_gamma, 1., perspective_flip_fv);
+    post_trans_mat = np.float32([[1, 0, (W-sl)/2], [0, 1, (H-sl)/2]])
+    post_trans_mat = np.vstack([post_trans_mat, [0,0,1]])
+    bM = np.matmul(M, post_trans_mat)
+    return bM
+
+def flip_3d_perspective(anim_args, prev_img_cv2, keys, frame_idx):
+    W, H = (prev_img_cv2.shape[1], prev_img_cv2.shape[0])
+    return cv2.warpPerspective(
+        prev_img_cv2,
+        get_flip_perspective_matrix(W, H, keys, frame_idx),
+        (W, H),
+        borderMode=cv2.BORDER_WRAP if anim_args.border == 'wrap' else cv2.BORDER_REPLICATE
+    )
+
+def anim_frame_warp(prev_img_cv2, args, anim_args, keys, frame_idx, depth_model=None, depth=None, device='cuda', half_precision = False):
+
+    if anim_args.use_depth_warping:
+        if depth is None and depth_model is not None:
+            depth = depth_model.predict(prev_img_cv2, anim_args, half_precision)
+    else:
+        depth = None
+
+    if anim_args.animation_mode == '2D':
+        prev_img = anim_frame_warp_2d(prev_img_cv2, args, anim_args, keys, frame_idx)
+    else: # '3D'
+        prev_img = anim_frame_warp_3d(device, prev_img_cv2, depth, anim_args, keys, frame_idx)
+                
+    return prev_img, depth
+
+def anim_frame_warp_2d(prev_img_cv2, args, anim_args, keys, frame_idx):
+    angle = keys.angle_series[frame_idx]
+    zoom = keys.zoom_series[frame_idx]
+    translation_x = keys.translation_x_series[frame_idx]
+    translation_y = keys.translation_y_series[frame_idx]
+
+    center = (args.W // 2, args.H // 2)
+    trans_mat = np.float32([[1, 0, translation_x], [0, 1, translation_y]])
+    rot_mat = cv2.getRotationMatrix2D(center, angle, zoom)
+    trans_mat = np.vstack([trans_mat, [0,0,1]])
+    rot_mat = np.vstack([rot_mat, [0,0,1]])
+    if anim_args.enable_perspective_flip:
+        bM = get_flip_perspective_matrix(args.W, args.H, keys, frame_idx)
+        rot_mat = np.matmul(bM, rot_mat, trans_mat)
+    else:
+        rot_mat = np.matmul(rot_mat, trans_mat)
+    return cv2.warpPerspective(
+        prev_img_cv2,
+        rot_mat,
+        (prev_img_cv2.shape[1], prev_img_cv2.shape[0]),
+        borderMode=cv2.BORDER_WRAP if anim_args.border == 'wrap' else cv2.BORDER_REPLICATE
+    )
+
+def anim_frame_warp_3d(device, prev_img_cv2, depth, anim_args, keys, frame_idx):
+    TRANSLATION_SCALE = 1.0/200.0 # matches Disco
+    translate_xyz = [
+        -keys.translation_x_series[frame_idx] * TRANSLATION_SCALE, 
+        keys.translation_y_series[frame_idx] * TRANSLATION_SCALE, 
+        -keys.translation_z_series[frame_idx] * TRANSLATION_SCALE
+    ]
+    rotate_xyz = [
+        math.radians(keys.rotation_3d_x_series[frame_idx]), 
+        math.radians(keys.rotation_3d_y_series[frame_idx]), 
+        math.radians(keys.rotation_3d_z_series[frame_idx])
+    ]
+    if anim_args.enable_perspective_flip:
+        prev_img_cv2 = flip_3d_perspective(anim_args, prev_img_cv2, keys, frame_idx)
+    rot_mat = p3d.euler_angles_to_matrix(torch.tensor(rotate_xyz, device=device), "XYZ").unsqueeze(0)
+    result = transform_image_3d(device if not device.type.startswith('mps') else torch.device('cpu'), prev_img_cv2, depth, rot_mat, translate_xyz, anim_args, keys, frame_idx)
+    torch.cuda.empty_cache()
+    return result
+
+def transform_image_3d(device, prev_img_cv2, depth_tensor, rot_mat, translate, anim_args, keys, frame_idx):
+    # adapted and optimized version of transform_image_3d from Disco Diffusion https://github.com/alembics/disco-diffusion 
+    w, h = prev_img_cv2.shape[1], prev_img_cv2.shape[0]
+
+    aspect_ratio = float(w)/float(h)
+    near = keys.near_series[frame_idx]
+    far = keys.far_series[frame_idx]
+    fov_deg = keys.fov_series[frame_idx]
+    persp_cam_old = p3d.FoVPerspectiveCameras(near, far, aspect_ratio, fov=fov_deg, degrees=True, device=device)
+    persp_cam_new = p3d.FoVPerspectiveCameras(near, far, aspect_ratio, fov=fov_deg, degrees=True, R=rot_mat, T=torch.tensor([translate]), device=device)
+
+    # range of [-1,1] is important to torch grid_sample's padding handling
+    y,x = torch.meshgrid(torch.linspace(-1.,1.,h,dtype=torch.float32,device=device),torch.linspace(-1.,1.,w,dtype=torch.float32,device=device))
+    if depth_tensor is None:
+        z = torch.ones_like(x)
+    else:
+        z = torch.as_tensor(depth_tensor, dtype=torch.float32, device=device)
+    xyz_old_world = torch.stack((x.flatten(), y.flatten(), z.flatten()), dim=1)
+
+    xyz_old_cam_xy = persp_cam_old.get_full_projection_transform().transform_points(xyz_old_world)[:,0:2]
+    xyz_new_cam_xy = persp_cam_new.get_full_projection_transform().transform_points(xyz_old_world)[:,0:2]
+
+    offset_xy = xyz_new_cam_xy - xyz_old_cam_xy
+    # affine_grid theta param expects a batch of 2D mats. Each is 2x3 to do rotation+translation.
+    identity_2d_batch = torch.tensor([[1.,0.,0.],[0.,1.,0.]], device=device).unsqueeze(0)
+    # coords_2d will have shape (N,H,W,2).. which is also what grid_sample needs.
+    coords_2d = torch.nn.functional.affine_grid(identity_2d_batch, [1,1,h,w], align_corners=False)
+    offset_coords_2d = coords_2d - torch.reshape(offset_xy, (h,w,2)).unsqueeze(0)
+
+    image_tensor = rearrange(torch.from_numpy(prev_img_cv2.astype(np.float32)), 'h w c -> c h w').to(device)
+    new_image = torch.nn.functional.grid_sample(
+        image_tensor.add(1/512 - 0.0001).unsqueeze(0), 
+        offset_coords_2d, 
+        mode=anim_args.sampling_mode, 
+        padding_mode=anim_args.padding_mode, 
+        align_corners=False
+    )
+
+    # convert back to cv2 style numpy array
+    result = rearrange(
+        new_image.squeeze().clamp(0,255), 
+        'c h w -> h w c'
+    ).cpu().numpy().astype(prev_img_cv2.dtype)
+    return result

extensions/deforum/scripts/deforum_helpers/animation_key_frames.py

@@ -0,0 +1,106 @@
+import re
+import numpy as np
+import numexpr
+import pandas as pd
+from .prompt import check_is_number
+
+class DeformAnimKeys():
+    def __init__(self, anim_args):
+        self.angle_series = get_inbetweens(parse_key_frames(anim_args.angle), anim_args.max_frames)
+        self.zoom_series = get_inbetweens(parse_key_frames(anim_args.zoom), anim_args.max_frames)
+        self.translation_x_series = get_inbetweens(parse_key_frames(anim_args.translation_x), anim_args.max_frames)
+        self.translation_y_series = get_inbetweens(parse_key_frames(anim_args.translation_y), anim_args.max_frames)
+        self.translation_z_series = get_inbetweens(parse_key_frames(anim_args.translation_z), anim_args.max_frames)
+        self.rotation_3d_x_series = get_inbetweens(parse_key_frames(anim_args.rotation_3d_x), anim_args.max_frames)
+        self.rotation_3d_y_series = get_inbetweens(parse_key_frames(anim_args.rotation_3d_y), anim_args.max_frames)
+        self.rotation_3d_z_series = get_inbetweens(parse_key_frames(anim_args.rotation_3d_z), anim_args.max_frames)
+        self.perspective_flip_theta_series = get_inbetweens(parse_key_frames(anim_args.perspective_flip_theta), anim_args.max_frames)
+        self.perspective_flip_phi_series = get_inbetweens(parse_key_frames(anim_args.perspective_flip_phi), anim_args.max_frames)
+        self.perspective_flip_gamma_series = get_inbetweens(parse_key_frames(anim_args.perspective_flip_gamma), anim_args.max_frames)
+        self.perspective_flip_fv_series = get_inbetweens(parse_key_frames(anim_args.perspective_flip_fv), anim_args.max_frames)
+        self.noise_schedule_series = get_inbetweens(parse_key_frames(anim_args.noise_schedule), anim_args.max_frames)
+        self.strength_schedule_series = get_inbetweens(parse_key_frames(anim_args.strength_schedule), anim_args.max_frames)
+        self.contrast_schedule_series = get_inbetweens(parse_key_frames(anim_args.contrast_schedule), anim_args.max_frames)
+        self.cfg_scale_schedule_series = get_inbetweens(parse_key_frames(anim_args.cfg_scale_schedule), anim_args.max_frames)
+        self.pix2pix_img_cfg_scale_series = get_inbetweens(parse_key_frames(anim_args.pix2pix_img_cfg_scale_schedule), anim_args.max_frames)
+        self.subseed_schedule_series = get_inbetweens(parse_key_frames(anim_args.subseed_schedule), anim_args.max_frames)
+        self.subseed_strength_schedule_series = get_inbetweens(parse_key_frames(anim_args.subseed_strength_schedule), anim_args.max_frames)
+        self.checkpoint_schedule_series = get_inbetweens(parse_key_frames(anim_args.checkpoint_schedule), anim_args.max_frames, is_single_string = True)
+        self.steps_schedule_series = get_inbetweens(parse_key_frames(anim_args.steps_schedule), anim_args.max_frames)
+        self.seed_schedule_series = get_inbetweens(parse_key_frames(anim_args.seed_schedule), anim_args.max_frames)
+        self.sampler_schedule_series = get_inbetweens(parse_key_frames(anim_args.sampler_schedule), anim_args.max_frames, is_single_string = True)
+        self.clipskip_schedule_series = get_inbetweens(parse_key_frames(anim_args.clipskip_schedule), anim_args.max_frames)
+        self.mask_schedule_series = get_inbetweens(parse_key_frames(anim_args.mask_schedule), anim_args.max_frames, is_single_string = True)
+        self.noise_mask_schedule_series = get_inbetweens(parse_key_frames(anim_args.noise_mask_schedule), anim_args.max_frames, is_single_string = True)
+        self.kernel_schedule_series = get_inbetweens(parse_key_frames(anim_args.kernel_schedule), anim_args.max_frames)
+        self.sigma_schedule_series = get_inbetweens(parse_key_frames(anim_args.sigma_schedule), anim_args.max_frames)
+        self.amount_schedule_series = get_inbetweens(parse_key_frames(anim_args.amount_schedule), anim_args.max_frames)
+        self.threshold_schedule_series = get_inbetweens(parse_key_frames(anim_args.threshold_schedule), anim_args.max_frames)
+        self.fov_series = get_inbetweens(parse_key_frames(anim_args.fov_schedule), anim_args.max_frames)
+        self.near_series = get_inbetweens(parse_key_frames(anim_args.near_schedule), anim_args.max_frames)
+        self.far_series = get_inbetweens(parse_key_frames(anim_args.far_schedule), anim_args.max_frames)
+        self.hybrid_comp_alpha_schedule_series = get_inbetweens(parse_key_frames(anim_args.hybrid_comp_alpha_schedule), anim_args.max_frames)
+        self.hybrid_comp_mask_blend_alpha_schedule_series = get_inbetweens(parse_key_frames(anim_args.hybrid_comp_mask_blend_alpha_schedule), anim_args.max_frames)
+        self.hybrid_comp_mask_contrast_schedule_series = get_inbetweens(parse_key_frames(anim_args.hybrid_comp_mask_contrast_schedule), anim_args.max_frames)
+        self.hybrid_comp_mask_auto_contrast_cutoff_high_schedule_series = get_inbetweens(parse_key_frames(anim_args.hybrid_comp_mask_auto_contrast_cutoff_high_schedule), anim_args.max_frames)
+        self.hybrid_comp_mask_auto_contrast_cutoff_low_schedule_series = get_inbetweens(parse_key_frames(anim_args.hybrid_comp_mask_auto_contrast_cutoff_low_schedule), anim_args.max_frames)
+
+class LooperAnimKeys():
+    def __init__(self, loop_args, anim_args):
+        self.use_looper = loop_args.use_looper
+        self.imagesToKeyframe = loop_args.init_images
+        self.image_strength_schedule_series = get_inbetweens(parse_key_frames(loop_args.image_strength_schedule), anim_args.max_frames)
+        self.blendFactorMax_series = get_inbetweens(parse_key_frames(loop_args.blendFactorMax), anim_args.max_frames)
+        self.blendFactorSlope_series = get_inbetweens(parse_key_frames(loop_args.blendFactorSlope), anim_args.max_frames)
+        self.tweening_frames_schedule_series = get_inbetweens(parse_key_frames(loop_args.tweening_frames_schedule), anim_args.max_frames)
+        self.color_correction_factor_series = get_inbetweens(parse_key_frames(loop_args.color_correction_factor), anim_args.max_frames)
+
+def get_inbetweens(key_frames, max_frames, integer=False, interp_method='Linear', is_single_string = False):
+    key_frame_series = pd.Series([np.nan for a in range(max_frames)])
+    for i in range(0, max_frames):
+        if i in key_frames:
+            value = key_frames[i]
+            value_is_number = check_is_number(value)
+            # if it's only a number, leave the rest for the default interpolation
+            if value_is_number:
+                t = i
+                key_frame_series[i] = value
+        if not value_is_number:
+            t = i
+            if is_single_string:
+                if value.find("'") > -1:
+                    value = value.replace("'","")
+                if value.find('"') > -1:
+                    value = value.replace('"',"")
+            key_frame_series[i] = numexpr.evaluate(value) if not is_single_string else value # workaround for values formatted like 0:("I am test") //used for sampler schedules
+    key_frame_series = key_frame_series.astype(float) if not is_single_string else key_frame_series # as string
+    
+    if interp_method == 'Cubic' and len(key_frames.items()) <= 3:
+        interp_method = 'Quadratic'    
+    if interp_method == 'Quadratic' and len(key_frames.items()) <= 2:
+        interp_method = 'Linear'
+          
+    key_frame_series[0] = key_frame_series[key_frame_series.first_valid_index()]
+    key_frame_series[max_frames-1] = key_frame_series[key_frame_series.last_valid_index()]
+    key_frame_series = key_frame_series.interpolate(method=interp_method.lower(), limit_direction='both')
+    if integer:
+        return key_frame_series.astype(int)
+    return key_frame_series
+
+def parse_key_frames(string, prompt_parser=None):
+    # because math functions (i.e. sin(t)) can utilize brackets 
+    # it extracts the value in form of some stuff
+    # which has previously been enclosed with brackets and
+    # with a comma or end of line existing after the closing one
+    pattern = r'((?P<frame>[0-9]+):[\s]*\((?P<param>[\S\s]*?)\)([,][\s]?|[\s]?$))'
+    frames = dict()
+    for match_object in re.finditer(pattern, string):
+        frame = int(match_object.groupdict()['frame'])
+        param = match_object.groupdict()['param']
+        if prompt_parser:
+            frames[frame] = prompt_parser(param)
+        else:
+            frames[frame] = param
+    if frames == {} and len(string) != 0:
+        raise RuntimeError('Key Frame string not correctly formatted')
+    return frames

extensions/deforum/scripts/deforum_helpers/args.py

@@ -0,0 +1,1214 @@
+from modules.shared import cmd_opts
+from modules.processing import get_fixed_seed
+from modules.ui_components import FormRow
+import modules.shared as sh
+import modules.paths as ph
+import os
+from .frame_interpolation import set_interp_out_fps, gradio_f_interp_get_fps_and_fcount, process_interp_vid_upload_logic
+from .upscaling import process_upscale_vid_upload_logic, process_ncnn_upscale_vid_upload_logic
+from .video_audio_utilities import find_ffmpeg_binary, ffmpeg_stitch_video, direct_stitch_vid_from_frames, get_quick_vid_info, extract_number
+from .gradio_funcs import *
+from .general_utils import get_os
+from .deforum_controlnet import controlnet_component_names, setup_controlnet_ui
+import tempfile
+        
+def Root():
+    device = sh.device
+    models_path = ph.models_path + '/Deforum'
+    half_precision = not cmd_opts.no_half
+    mask_preset_names = ['everywhere','init_mask','video_mask']
+    p = None
+    frames_cache = []
+    initial_seed = None
+    initial_info = None
+    first_frame = None
+    outpath_samples = ""
+    animation_prompts = None
+    color_corrections = None 
+    initial_clipskip = None
+    current_user_os = get_os()
+    tmp_deforum_run_duplicated_folder = os.path.join(tempfile.gettempdir(), 'tmp_run_deforum')
+    return locals()
+
+def DeforumAnimArgs():
+
+    #@markdown ####**Animation:**
+    animation_mode = '2D' #@param ['None', '2D', '3D', 'Video Input', 'Interpolation'] {type:'string'}
+    max_frames = 120 #@param {type:"number"}
+    border = 'replicate' #@param ['wrap', 'replicate'] {type:'string'}
+    #@markdown ####**Motion Parameters:**
+    angle = "0:(0)"#@param {type:"string"}
+    zoom = "0:(1.0025+0.002*sin(1.25*3.14*t/30))"#@param {type:"string"}
+    translation_x = "0:(0)"#@param {type:"string"}
+    translation_y = "0:(0)"#@param {type:"string"}
+    translation_z = "0:(1.75)"#@param {type:"string"}
+    rotation_3d_x = "0:(0)"#@param {type:"string"}
+    rotation_3d_y = "0:(0)"#@param {type:"string"}
+    rotation_3d_z = "0:(0)"#@param {type:"string"}
+    enable_perspective_flip = False #@param {type:"boolean"}
+    perspective_flip_theta = "0:(0)"#@param {type:"string"}
+    perspective_flip_phi = "0:(0)"#@param {type:"string"}
+    perspective_flip_gamma = "0:(0)"#@param {type:"string"}
+    perspective_flip_fv = "0:(53)"#@param {type:"string"}
+    noise_schedule = "0: (0.065)"#@param {type:"string"}
+    strength_schedule = "0: (0.65)"#@param {type:"string"}
+    contrast_schedule = "0: (1.0)"#@param {type:"string"}
+    cfg_scale_schedule = "0: (7)"
+    enable_steps_scheduling = False#@param {type:"boolean"}
+    steps_schedule = "0: (25)"#@param {type:"string"}
+    fov_schedule = "0: (70)"
+    near_schedule = "0: (200)"
+    far_schedule = "0: (10000)"
+    seed_schedule = "0:(5), 1:(-1), 219:(-1), 220:(5)"
+    pix2pix_img_cfg_scale = "1.5"
+    pix2pix_img_cfg_scale_schedule = "0:(1.5)"
+    enable_subseed_scheduling = False
+    subseed_schedule = "0:(1)"
+    subseed_strength_schedule = "0:(0)"
+    
+    # Sampler Scheduling
+    enable_sampler_scheduling = False #@param {type:"boolean"}
+    sampler_schedule = '0: ("Euler a")'
+
+    # Composable mask scheduling
+    use_noise_mask = False
+    mask_schedule = '0: ("!({everywhere}^({init_mask}|{video_mask}) ) ")'
+    noise_mask_schedule = '0: ("!({everywhere}^({init_mask}|{video_mask}) ) ")'
+    # Checkpoint Scheduling
+    enable_checkpoint_scheduling = False#@param {type:"boolean"}
+    checkpoint_schedule = '0: ("model1.ckpt"), 100: ("model2.ckpt")'
+    
+    # CLIP skip Scheduling
+    enable_clipskip_scheduling = False #@param {type:"boolean"}
+    clipskip_schedule = '0: (2)'
+
+    # Anti-blur
+    kernel_schedule = "0: (5)"
+    sigma_schedule = "0: (1.0)"
+    amount_schedule = "0: (0.35)"
+    threshold_schedule = "0: (0.0)"
+    # Hybrid video
+    hybrid_comp_alpha_schedule = "0:(1)" #@param {type:"string"}
+    hybrid_comp_mask_blend_alpha_schedule = "0:(0.5)" #@param {type:"string"}
+    hybrid_comp_mask_contrast_schedule = "0:(1)" #@param {type:"string"}
+    hybrid_comp_mask_auto_contrast_cutoff_high_schedule =  "0:(100)" #@param {type:"string"}
+    hybrid_comp_mask_auto_contrast_cutoff_low_schedule =  "0:(0)" #@param {type:"string"}
+
+    #@markdown ####**Coherence:**
+    color_coherence = 'Match Frame 0 LAB' #@param ['None', 'Match Frame 0 HSV', 'Match Frame 0 LAB', 'Match Frame 0 RGB', 'Video Input'] {type:'string'}
+    color_coherence_video_every_N_frames = 1 #@param {type:"integer"}
+    color_force_grayscale = False #@param {type:"boolean"}
+    diffusion_cadence = '2' #@param ['1','2','3','4','5','6','7','8'] {type:'string'}
+
+    #@markdown ####**Noise settings:**
+    noise_type = 'perlin' #@param ['uniform', 'perlin'] {type:'string'}
+    # Perlin params
+    perlin_w = 8 #@param {type:"number"}
+    perlin_h = 8 #@param {type:"number"}
+    perlin_octaves = 4 #@param {type:"number"}
+    perlin_persistence = 0.5 #@param {type:"number"}
+
+    #@markdown ####**3D Depth Warping:**
+    use_depth_warping = True #@param {type:"boolean"}
+    midas_weight = 0.2 #@param {type:"number"}
+
+    padding_mode = 'border'#@param ['border', 'reflection', 'zeros'] {type:'string'}
+    sampling_mode = 'bicubic'#@param ['bicubic', 'bilinear', 'nearest'] {type:'string'}
+    save_depth_maps = False #@param {type:"boolean"}
+
+    #@markdown ####**Video Input:**
+    video_init_path ='https://github.com/hithereai/d/releases/download/m/vid.mp4' #@param {type:"string"}
+    extract_nth_frame = 1#@param {type:"number"}
+    extract_from_frame = 0 #@param {type:"number"}
+    extract_to_frame = -1 #@param {type:"number"} minus 1 for unlimited frames
+    overwrite_extracted_frames = True #@param {type:"boolean"}
+    use_mask_video = False #@param {type:"boolean"}
+    video_mask_path ='/content/video_in.mp4'#@param {type:"string"}
+
+    #@markdown ####**Hybrid Video for 2D/3D Animation Mode:**
+    hybrid_generate_inputframes = False #@param {type:"boolean"}
+    hybrid_generate_human_masks = "None" #@param ['None','PNGs','Video', 'Both']
+    hybrid_use_first_frame_as_init_image = True #@param {type:"boolean"}
+    hybrid_motion = "None" #@param ['None','Optical Flow','Perspective','Affine']
+    hybrid_motion_use_prev_img = False #@param {type:"boolean"}
+    hybrid_flow_method = "Farneback" #@param ['DIS Medium','Farneback']
+    hybrid_composite = False #@param {type:"boolean"}
+    hybrid_comp_mask_type = "None" #@param ['None', 'Depth', 'Video Depth', 'Blend', 'Difference']
+    hybrid_comp_mask_inverse = False #@param {type:"boolean"}
+    hybrid_comp_mask_equalize = "None" #@param  ['None','Before','After','Both']
+    hybrid_comp_mask_auto_contrast = False #@param {type:"boolean"}
+    hybrid_comp_save_extra_frames = False #@param {type:"boolean"}
+
+    #@markdown ####**Resume Animation:**
+    resume_from_timestring = False #@param {type:"boolean"}
+    resume_timestring = "20220829210106" #@param {type:"string"}
+
+    return locals()
+
+# def DeforumPrompts():
+    # return
+    
+def DeforumAnimPrompts():
+    return r"""{
+    "0": "tiny cute swamp bunny, highly detailed, intricate, ultra hd, sharp photo, crepuscular rays, in focus, by tomasz alen kopera",
+    "30": "anthropomorphic clean cat, surrounded by fractals, epic angle and pose, symmetrical, 3d, depth of field, ruan jia and fenghua zhong",
+    "60": "a beautiful coconut --neg photo, realistic",
+    "90": "a beautiful durian, trending on Artstation"
+}
+    """
+
+def DeforumArgs():
+    #@markdown **Image Settings**
+    W = 512 #@param
+    H = 512 #@param
+    W, H = map(lambda x: x - x % 64, (W, H))  # resize to integer multiple of 64
+
+    #@markdonw **Webui stuff**
+    tiling = False
+    restore_faces = False
+    seed_enable_extras = False
+    subseed = -1
+    subseed_strength = 0
+    seed_resize_from_w = 0
+    seed_resize_from_h = 0
+    
+    #@markdown **Sampling Settings**
+    seed = -1 #@param
+    sampler = 'euler_ancestral' #@param ["klms","dpm2","dpm2_ancestral","heun","euler","euler_ancestral","plms", "ddim"]
+    steps = 25 #@param
+    scale = 7 #@param
+    ddim_eta = 0.0 #@param
+    dynamic_threshold = None
+    static_threshold = None
+
+    #@markdown **Save & Display Settings**
+    save_samples = True #@param {type:"boolean"}
+    save_settings = True #@param {type:"boolean"}
+    display_samples = True #@param {type:"boolean"}
+    save_sample_per_step = False #@param {type:"boolean"}
+    show_sample_per_step = False #@param {type:"boolean"}
+
+    #@markdown **Prompt Settings**
+    prompt_weighting = False #@param {type:"boolean"}
+    normalize_prompt_weights = True #@param {type:"boolean"}
+    log_weighted_subprompts = False #@param {type:"boolean"}
+
+    #@markdown **Batch Settings**
+    n_batch = 1 #@param
+    batch_name = "Deforum" #@param {type:"string"}
+    filename_format = "{timestring}_{index}_{prompt}.png" #@param ["{timestring}_{index}_{seed}.png","{timestring}_{index}_{prompt}.png"]
+    seed_behavior = "iter" #@param ["iter","fixed","random","ladder","alternate","schedule"]
+    seed_iter_N = 1 #@param {type:'integer'}
+    # make_grid = False #@param {type:"boolean"}
+    # grid_rows = 2 #@param 
+    outdir = ""#get_output_folder(output_path, batch_name)
+
+    #@markdown **Init Settings**
+    use_init = False #@param {type:"boolean"}
+    strength = 0.0 #@param {type:"number"}
+    strength_0_no_init = True # Set the strength to 0 automatically when no init image is used
+    init_image = "https://github.com/hithereai/d/releases/download/m/kaba.png" #@param {type:"string"}
+    # Whiter areas of the mask are areas that change more
+    use_mask = False #@param {type:"boolean"}
+    use_alpha_as_mask = False # use the alpha channel of the init image as the mask
+    mask_file = "https://github.com/hithereai/d/releases/download/m/mask.jpg" #@param {type:"string"}
+    invert_mask = False #@param {type:"boolean"}
+    # Adjust mask image, 1.0 is no adjustment. Should be positive numbers.
+    mask_contrast_adjust = 1.0  #@param {type:"number"}
+    mask_brightness_adjust = 1.0  #@param {type:"number"}
+    # Overlay the masked image at the end of the generation so it does not get degraded by encoding and decoding
+    overlay_mask = True  # {type:"boolean"}
+    # Blur edges of final overlay mask, if used. Minimum = 0 (no blur)
+    mask_overlay_blur = 4 # {type:"number"}
+
+    fill = 1 #MASKARGSEXPANSION Todo : Rename and convert to same formatting as used in img2img masked content
+    full_res_mask = True
+    full_res_mask_padding = 4
+    reroll_blank_frames = 'reroll' # reroll, interrupt, or ignore
+
+    n_samples = 1 # doesnt do anything
+    precision = 'autocast' 
+    C = 4
+    f = 8
+
+    prompt = ""
+    timestring = ""
+    init_latent = None
+    init_sample = None
+    init_c = None
+    mask_image = None
+    noise_mask = None
+    seed_internal = 0
+
+    return locals()
+
+def keyframeExamples():
+    return '''{
+    "0": "https://user-images.githubusercontent.com/121192995/215279228-1673df8a-f919-4380-b04c-19379b2041ff.png",
+    "50": "https://user-images.githubusercontent.com/121192995/215279281-7989fd6f-4b9b-4d90-9887-b7960edd59f8.png",
+    "100": "https://user-images.githubusercontent.com/121192995/215279284-afc14543-d220-4142-bbf4-503776ca2b8b.png",
+    "150": "https://user-images.githubusercontent.com/121192995/215279286-23378635-85b3-4457-b248-23e62c048049.jpg",
+    "200": "https://user-images.githubusercontent.com/121192995/215279228-1673df8a-f919-4380-b04c-19379b2041ff.png"
+}'''
+
+def LoopArgs():
+    use_looper = False
+    init_images = keyframeExamples()
+    image_strength_schedule = "0:(0.75)"
+    blendFactorMax = "0:(0.35)"
+    blendFactorSlope = "0:(0.25)"
+    tweening_frames_schedule = "0:(20)"
+    color_correction_factor = "0:(0.075)"
+    return locals()
+
+def ParseqArgs():
+    parseq_manifest = None
+    parseq_use_deltas = True
+    return locals()
+    
+def DeforumOutputArgs():
+    skip_video_for_run_all = False #@param {type: 'boolean'}
+    fps = 15 #@param {type:"number"}
+    make_gif = False
+    image_path = "C:/SD/20230124234916_%05d.png" #@param {type:"string"}
+    mp4_path = "testvidmanualsettings.mp4" #@param {type:"string"}
+    ffmpeg_location = find_ffmpeg_binary()
+    ffmpeg_crf = '17'
+    ffmpeg_preset = 'slow'
+    add_soundtrack = 'None' #@param ["File","Init Video"]
+    soundtrack_path = "https://freetestdata.com/wp-content/uploads/2021/09/Free_Test_Data_1MB_MP3.mp3"
+    # End-Run upscaling
+    r_upscale_video = False
+    r_upscale_factor = 'x2' # ['2x', 'x3', 'x4']
+    # **model below** - 'realesr-animevideov3' (default of realesrgan engine, does 2-4x), the rest do only 4x: 'realesrgan-x4plus', 'realesrgan-x4plus-anime'
+    r_upscale_model = 'realesr-animevideov3' 
+    r_upscale_keep_imgs = True
+    
+    render_steps = False  #@param {type: 'boolean'}
+    path_name_modifier = "x0_pred" #@param ["x0_pred","x"]
+    # max_video_frames = 200 #@param {type:"string"}
+    store_frames_in_ram = False #@param {type: 'boolean'}
+    #@markdown **Interpolate Video Settings**
+    # todo: change them to support FILM interpolation as well
+    frame_interpolation_engine = "None" #@param ["None", "RIFE v4.6", "FILM"]
+    frame_interpolation_x_amount = 2 # [2 to 1000 depends on the engine]
+    frame_interpolation_slow_mo_enabled = False
+    frame_interpolation_slow_mo_amount = 2 #@param [2 to 10]
+    frame_interpolation_keep_imgs = False #@param {type: 'boolean'}
+    return locals()
+    
+import gradio as gr
+import os
+import time
+from types import SimpleNamespace
+
+i1_store_backup = "<p style=\"text-align:center;font-weight:bold;margin-bottom:0em\">Deforum extension for auto1111 — version 2.2b</p>"
+i1_store = i1_store_backup
+
+mask_fill_choices=['fill', 'original', 'latent noise', 'latent nothing']
+
+def setup_deforum_setting_dictionary(self, is_img2img, is_extension = True):
+    d = SimpleNamespace(**DeforumArgs()) #default args
+    da = SimpleNamespace(**DeforumAnimArgs()) #default anim args
+    dp = SimpleNamespace(**ParseqArgs()) #default parseq ars
+    dv = SimpleNamespace(**DeforumOutputArgs()) #default video args
+    dr = SimpleNamespace(**Root()) # ROOT args
+    dloopArgs = SimpleNamespace(**LoopArgs())
+    if not is_extension:
+        with gr.Row():
+            btn = gr.Button("Click here after the generation to show the video")
+        with gr.Row():
+            i1 = gr.HTML(i1_store, elem_id='deforum_header')
+    else:
+        btn = i1 = gr.HTML("")
+       
+    # MAIN (TOP) EXTENSION INFO ACCORD
+    with gr.Accordion("Info, Links and Help", open=False, elem_id='main_top_info_accord'):
+            gr.HTML("""<strong>Made by <a href="https://deforum.github.io">deforum.github.io</a>, port for AUTOMATIC1111's webui maintained by <a href="https://github.com/kabachuha">kabachuha</a></strong>""")
+            gr.HTML("""<a  style="color:SteelBlue" href="https://github.com/deforum-art/deforum-for-automatic1111-webui/wiki/FAQ-&-Troubleshooting">FOR HELP CLICK HERE</a""", elem_id="for_help_click_here")
+            gr.HTML("""<ul style="list-style-type:circle; margin-left:1em">
+            <li>The code for this extension: <a  style="color:SteelBlue" href="https://github.com/deforum-art/deforum-for-automatic1111-webui">here</a>.</li>
+            <li>Join the <a style="color:SteelBlue" href="https://discord.gg/deforum">official Deforum Discord</a> to share your creations and suggestions.</li>
+            <li>Official Deforum Wiki: <a style="color:SteelBlue" href="https://github.com/deforum-art/deforum-for-automatic1111-webui/wiki">here</a>.</li>
+            <li>Anime-inclined great guide (by FizzleDorf) with lots of examples: <a style="color:SteelBlue" href="https://rentry.org/AnimAnon-Deforum">here</a>.</li>
+            <li>For advanced keyframing with Math functions, see <a style="color:SteelBlue" href="https://github.com/deforum-art/deforum-for-automatic1111-webui/wiki/Maths-in-Deforum">here</a>.</li>
+            <li>Alternatively, use <a style="color:SteelBlue" href="https://sd-parseq.web.app/deforum">sd-parseq</a> as a UI to define your animation schedules (see the Parseq section in the Keyframes tab).</li>
+            <li><a style="color:SteelBlue" href="https://www.framesync.xyz/">framesync.xyz</a> is also a good option, it makes compact math formulae for Deforum keyframes by selecting various waveforms.</li>
+            <li>The other site allows for making keyframes using <a style="color:SteelBlue" href="https://www.chigozie.co.uk/keyframe-string-generator/">interactive splines and Bezier curves</a> (select Disco output format).</li>
+            <li>If you want to use Width/Height which are not multiples of 64, please change noise_type to 'Uniform', in Keyframes --> Noise.</li>
+            </ul>
+            <italic>If you liked this extension, please <a style="color:SteelBlue" href="https://github.com/deforum-art/deforum-for-automatic1111-webui">give it a star on GitHub</a>!</italic> 😊""")
+    if not is_extension:
+        def show_vid():
+            return {
+                i1: gr.update(value=i1_store, visible=True)
+            }
+        
+        btn.click(
+            show_vid,
+            [],
+            [i1]
+            )
+            
+    with gr.Blocks():
+        # RUN TAB
+        with gr.Tab('Run'):
+            from modules.sd_samplers import samplers_for_img2img
+            with gr.Row(variant='compact'):
+                sampler = gr.Dropdown(label="Sampler", choices=[x.name for x in samplers_for_img2img], value=samplers_for_img2img[0].name, type="value", elem_id="sampler", interactive=True)
+                steps = gr.Slider(label="Steps", minimum=0, maximum=200, step=1, value=d.steps, interactive=True)
+            with gr.Row(variant='compact'):
+                W = gr.Slider(label="Width", minimum=64, maximum=2048, step=64, value=d.W, interactive=True)
+                H = gr.Slider(label="Height", minimum=64, maximum=2048, step=64, value=d.H, interactive=True) 
+            with gr.Row(variables='compact'):
+                seed = gr.Number(label="Seed", value=d.seed, interactive=True, precision=0)
+                batch_name = gr.Textbox(label="Batch name", lines=1, interactive=True, value = d.batch_name)
+            with gr.Accordion('Restore Faces, Tiling & more', open=False) as run_more_settings_accord:
+                with gr.Row(variant='compact'):
+                    restore_faces = gr.Checkbox(label='Restore Faces', value=d.restore_faces)
+                    tiling = gr.Checkbox(label='Tiling', value=False)
+                    ddim_eta = gr.Number(label="DDIM Eta", value=d.ddim_eta, interactive=True)
+                with gr.Row() as pix2pix_img_cfg_scale_row:
+                    pix2pix_img_cfg_scale_schedule = gr.Textbox(label="Pix2Pix img CFG schedule", value=da.pix2pix_img_cfg_scale_schedule, interactive=True)    
+            # RUN FROM SETTING FILE ACCORD
+            with gr.Accordion('Resume & Run from file', open=False):
+                with gr.Tab('Run from Settings file'):
+                    with gr.Row(variant='compact'):
+                        override_settings_with_file = gr.Checkbox(label="Override settings", value=False, interactive=True, elem_id='override_settings')
+                        custom_settings_file = gr.Textbox(label="Custom settings file", lines=1, interactive=True, elem_id='custom_settings_file')
+                # RESUME ANIMATION ACCORD
+                with gr.Tab('Resume Animation'):
+                    with gr.Row(variant='compact'):
+                        resume_from_timestring = gr.Checkbox(label="Resume from timestring", value=da.resume_from_timestring, interactive=True)
+                        resume_timestring = gr.Textbox(label="Resume timestring", lines=1, value = da.resume_timestring, interactive=True)
+        # KEYFRAMES TAB
+        with gr.Tab('Keyframes'): #TODO make a some sort of the original dictionary parsing
+            with gr.Row(variant='compact'):
+                with gr.Column(scale=2):
+                    animation_mode = gr.Radio(['2D', '3D', 'Interpolation', 'Video Input'], label="Animation mode", value=da.animation_mode, elem_id="animation_mode")
+                with gr.Column(scale=1, min_width=180):
+                    border = gr.Radio(['replicate', 'wrap'], label="Border", value=da.border, elem_id="border")
+            with gr.Row(variant='compact'):
+                diffusion_cadence = gr.Slider(label="Cadence", minimum=1, maximum=50, step=1, value=da.diffusion_cadence, interactive=True)
+                max_frames = gr.Number(label="Max frames", lines=1, value = da.max_frames, interactive=True, precision=0)
+            # GUIDED IMAGES ACCORD
+            with gr.Accordion('Guided Images', open=False, elem_id='guided_images_accord') as guided_images_accord:
+                # GUIDED IMAGES INFO ACCORD
+                with gr.Accordion('*READ ME before you use this mode!*', open=False):
+                    gr.HTML("""You can use this as a guided image tool or as a looper depending on your settings in the keyframe images field. 
+                               Set the keyframes and the images that you want to show up. 
+                               Note: the number of frames between each keyframe should be greater than the tweening frames.""")
+                    #    In later versions this should be also in the strength schedule, but for now you need to set it.
+                    gr.HTML("""Prerequisites and Important Info: 
+                               <ul style="list-style-type:circle; margin-left:2em; margin-bottom:0em">
+                                   <li>This mode works ONLY with 2D/3D animation modes. Interpolation and Video Input modes aren't supported.</ li>
+                                   <li>Set Init tab's strength slider greater than 0. Recommended value (.65 - .80).</ li>
+                                   <li>Set 'seed_behavior' to 'schedule' under the Seed Scheduling section below.</li>
+                                </ul>
+                            """)
+                    gr.HTML("""Looping recommendations: 
+                                <ul style="list-style-type:circle; margin-left:2em; margin-bottom:0em">
+                                    <li>seed_schedule should start and end on the same seed. <br />
+                                        Example: seed_schedule could use 0:(5), 1:(-1), 219:(-1), 220:(5)</li>
+                                    <li>The 1st and last keyframe images should match.</li>
+                                    <li>Set your total number of keyframes to be 21 more than the last inserted keyframe image. <br />
+                                        Example: Default args should use 221 as total keyframes.</li>
+                                    <li>Prompts are stored in JSON format. If you've got an error, check it in validator, <a style="color:SteelBlue" href="https://odu.github.io/slingjsonlint/">like here</a></li>
+                                </ul>
+                            """)
+                with gr.Row():
+                    use_looper = gr.Checkbox(label="Enable guided images mode", value=dloopArgs.use_looper, interactive=True)
+                with gr.Row():
+                    init_images = gr.Textbox(label="Images to use for keyframe guidance", lines=9, value = keyframeExamples(), interactive=True)
+                # GUIDED IMAGES SCHEDULES ACCORD
+                with gr.Accordion('Guided images schedules', open=False):
+                    with gr.Row():
+                        image_strength_schedule = gr.Textbox(label="Image strength schedule", lines=1, value = dloopArgs.image_strength_schedule, interactive=True)
+                    with gr.Row():
+                        blendFactorMax = gr.Textbox(label="Blend factor max", lines=1, value = dloopArgs.blendFactorMax, interactive=True)
+                    with gr.Row():
+                        blendFactorSlope = gr.Textbox(label="Blend factor slope", lines=1, value = dloopArgs.blendFactorSlope, interactive=True)
+                    with gr.Row():
+                        tweening_frames_schedule = gr.Textbox(label="Tweening frames schedule", lines=1, value = dloopArgs.tweening_frames_schedule, interactive=True)
+                    with gr.Row():
+                        color_correction_factor = gr.Textbox(label="Color correction factor", lines=1, value = dloopArgs.color_correction_factor, interactive=True)
+            # EXTA SCHEDULES TABS
+            with gr.Tabs(elem_id='extra_schedules'):
+                with gr.TabItem('Strength'):
+                    strength_schedule = gr.Textbox(label="Strength schedule", lines=1, value = da.strength_schedule, interactive=True)
+                with gr.TabItem('CFG'):
+                    cfg_scale_schedule = gr.Textbox(label="CFG scale schedule", lines=1, value = da.cfg_scale_schedule, interactive=True)
+                with gr.TabItem('Seed') as a3:
+                    with gr.Row():
+                        seed_behavior = gr.Radio(['iter', 'fixed', 'random', 'ladder', 'alternate', 'schedule'], label="Seed behavior", value=d.seed_behavior, elem_id="seed_behavior")
+                    with gr.Row() as seed_iter_N_row:
+                        seed_iter_N = gr.Number(label="Seed iter N", value=d.seed_iter_N, interactive=True, precision=0)
+                    with gr.Row(visible=False) as seed_schedule_row:
+                        seed_schedule = gr.Textbox(label="Seed schedule", lines=1, value = da.seed_schedule, interactive=True)
+                with gr.TabItem('SubSeed', open=False) as subseed_sch_tab:
+                    enable_subseed_scheduling = gr.Checkbox(label="Enable Subseed scheduling", value=da.enable_subseed_scheduling, interactive=True)
+                    subseed_schedule = gr.Textbox(label="Subseed schedule", lines=1, value = da.subseed_schedule, interactive=True)
+                    subseed_strength_schedule = gr.Textbox(label="Subseed strength schedule", lines=1, value = da.subseed_strength_schedule, interactive=True)
+                    with gr.Row(variant='compact'):
+                        seed_resize_from_w = gr.Slider(minimum=0, maximum=2048, step=64, label="Resize seed from width", value=0)
+                        seed_resize_from_h = gr.Slider(minimum=0, maximum=2048, step=64, label="Resize seed from height", value=0)
+                # Steps Scheduling
+                with gr.TabItem('Step') as a13:
+                    with gr.Row():
+                        enable_steps_scheduling = gr.Checkbox(label="Enable steps scheduling", value=da.enable_steps_scheduling, interactive=True)
+                    with gr.Row():
+                        steps_schedule = gr.Textbox(label="Steps schedule", lines=1, value = da.steps_schedule, interactive=True)
+                # Sampler Scheduling
+                with gr.TabItem('Sampler') as a14:
+                    with gr.Row():
+                        enable_sampler_scheduling = gr.Checkbox(label="Enable sampler scheduling", value=da.enable_sampler_scheduling, interactive=True)
+                    with gr.Row():
+                        sampler_schedule = gr.Textbox(label="Sampler schedule", lines=1, value = da.sampler_schedule, interactive=True)
+                # Checkpoint Scheduling
+                with gr.TabItem('Checkpoint') as a15:
+                    with gr.Row():
+                        enable_checkpoint_scheduling = gr.Checkbox(label="Enable checkpoint scheduling", value=da.enable_checkpoint_scheduling, interactive=True)
+                    with gr.Row():
+                        checkpoint_schedule = gr.Textbox(label="Checkpoint schedule", lines=1, value = da.checkpoint_schedule, interactive=True)
+                with gr.TabItem('CLIP Skip', open=False) as a16:
+                    with gr.Row():
+                        enable_clipskip_scheduling = gr.Checkbox(label="Enable CLIP skip scheduling", value=da.enable_clipskip_scheduling, interactive=True)
+                    with gr.Row():
+                        clipskip_schedule = gr.Textbox(label="CLIP skip schedule", lines=1, value = da.clipskip_schedule, interactive=True)
+            # MOTION INNER TAB
+            with gr.Tab('Motion') as motion_tab:
+                with gr.Column(visible=True) as only_2d_motion_column:
+                    with gr.Row(variant='compact'):
+                        angle = gr.Textbox(label="Angle", lines=1, value = da.angle, interactive=True)
+                    with gr.Row(variant='compact'):
+                        zoom = gr.Textbox(label="Zoom", lines=1, value = da.zoom, interactive=True)
+                with gr.Column(visible=True) as both_anim_mode_motion_params_column:
+                    with gr.Row(variant='compact'):
+                        translation_x = gr.Textbox(label="Translation X", lines=1, value = da.translation_x, interactive=True)
+                    with gr.Row(variant='compact'):
+                        translation_y = gr.Textbox(label="Translation Y", lines=1, value = da.translation_y, interactive=True)
+                with gr.Column(visible=False) as only_3d_motion_column:
+                    with gr.Row(variant='compact'):
+                        translation_z = gr.Textbox(label="Translation Z", lines=1, value = da.translation_z, interactive=True)
+                    with gr.Row(variant='compact'):
+                        rotation_3d_x = gr.Textbox(label="Rotation 3D X", lines=1, value = da.rotation_3d_x, interactive=True)
+                    with gr.Row(variant='compact'):
+                        rotation_3d_y = gr.Textbox(label="Rotation 3D Y", lines=1, value = da.rotation_3d_y, interactive=True)
+                    with gr.Row(variant='compact'):
+                        rotation_3d_z = gr.Textbox(label="Rotation 3D Z", lines=1, value = da.rotation_3d_z, interactive=True)
+                # 3D DEPTH & FOV ACCORD
+                with gr.Accordion('Depth Warping & FOV', visible=False, open=False) as depth_3d_warping_accord:
+                    with gr.Tab('Depth Warping'): 
+                        with gr.Row(variant='compact'):
+                            use_depth_warping = gr.Checkbox(label="Use depth warping", value=da.use_depth_warping, interactive=True)
+                            midas_weight = gr.Number(label="MiDaS weight", value=da.midas_weight, interactive=True)
+                        with gr.Row(variant='compact'):
+                            padding_mode = gr.Radio(['border', 'reflection', 'zeros'], label="Padding mode", value=da.padding_mode, elem_id="padding_mode")
+                            sampling_mode = gr.Radio(['bicubic', 'bilinear', 'nearest'], label="Sampling mode", value=da.sampling_mode, elem_id="sampling_mode")
+                    with gr.Tab('Field Of View', visible=False, open=False) as fov_accord:
+                        with gr.Row(variant='compact'):
+                            fov_schedule = gr.Textbox(label="FOV schedule", lines=1, value = da.fov_schedule, interactive=True)
+                        with gr.Row():
+                            near_schedule = gr.Textbox(label="Near schedule", lines=1, value = da.near_schedule, interactive=True)
+                        with gr.Row():
+                            far_schedule = gr.Textbox(label="Far schedule", lines=1, value = da.far_schedule, interactive=True)
+                # PERSPECTIVE FLIP ACCORD
+                with gr.Accordion('Perspective Flip', open=False) as perspective_flip_accord:
+                    with gr.Row():
+                        enable_perspective_flip = gr.Checkbox(label="Enable perspective flip", value=da.enable_perspective_flip, interactive=True)
+                    with gr.Row():
+                        perspective_flip_theta = gr.Textbox(label="Perspective flip theta", lines=1, value = da.perspective_flip_theta, interactive=True)
+                    with gr.Row():
+                        perspective_flip_phi = gr.Textbox(label="Perspective flip phi", lines=1, value = da.perspective_flip_phi, interactive=True)
+                    with gr.Row():
+                        perspective_flip_gamma = gr.Textbox(label="Perspective flip gamma", lines=1, value = da.perspective_flip_gamma, interactive=True)
+                    with gr.Row():
+                        perspective_flip_fv = gr.Textbox(label="Perspective flip fv", lines=1, value = da.perspective_flip_fv, interactive=True)
+            # NOISE INNER TAB
+            with gr.Tab('Noise', open=True) as a8:
+                with gr.Row():
+                    noise_type = gr.Radio(['uniform', 'perlin'], label="Noise type", value=da.noise_type, elem_id="noise_type")
+                with gr.Row():
+                    noise_schedule = gr.Textbox(label="Noise schedule", lines=1, value = da.noise_schedule, interactive=True)
+                with gr.Row() as perlin_row:
+                    with gr.Column(min_width=220):
+                        perlin_octaves = gr.Slider(label="Perlin octaves", minimum=1, maximum=7, value=da.perlin_octaves, step=1, interactive=True)
+                    with gr.Column(min_width=220):
+                        perlin_persistence = gr.Slider(label="Perlin persistence", minimum=0, maximum=1, value=da.perlin_persistence, step=0.02, interactive=True)
+            # COHERENCE INNER TAB
+            with gr.Tab('Coherence', open=False) as coherence_accord:
+                with gr.Row(equal_height=True):
+                    # Future TODO: remove 'match frame 0' prefix (after we manage the deprecated-names settings import), then convert from Dropdown to Radio!
+                    color_coherence = gr.Dropdown(label="Color coherence", choices=['None', 'Match Frame 0 HSV', 'Match Frame 0 LAB', 'Match Frame 0 RGB', 'Video Input'], value=da.color_coherence, type="value", elem_id="color_coherence", interactive=True)
+                    with gr.Column() as force_grayscale_column:
+                        color_force_grayscale = gr.Checkbox(label="Color force Grayscale", value=da.color_force_grayscale, interactive=True)
+                with gr.Row(visible=False) as color_coherence_video_every_N_frames_row:
+                    color_coherence_video_every_N_frames = gr.Number(label="Color coherence video every N frames", value=1, interactive=True)
+                with gr.Row():
+                    contrast_schedule = gr.Textbox(label="Contrast schedule", lines=1, value = da.contrast_schedule, interactive=True)
+                with gr.Row():
+                    # what to do with blank frames (they may result from glitches or the NSFW filter being turned on): reroll with +1 seed, interrupt the animation generation, or do nothing
+                    reroll_blank_frames = gr.Radio(['reroll', 'interrupt', 'ignore'], label="Reroll blank frames", value=d.reroll_blank_frames, elem_id="reroll_blank_frames")
+            # ANTI BLUR INNER TAB  
+            with gr.Tab('Anti Blur', open=False, elem_id='anti_blur_accord') as anti_blur_tab:
+                with gr.Row(variant='compact'):
+                    kernel_schedule = gr.Textbox(label="Kernel schedule", lines=1, value = da.kernel_schedule, interactive=True)
+                with gr.Row(variant='compact'):
+                    sigma_schedule = gr.Textbox(label="Sigma schedule", lines=1, value = da.sigma_schedule, interactive=True)
+                with gr.Row(variant='compact'):
+                    amount_schedule = gr.Textbox(label="Amount schedule", lines=1, value = da.amount_schedule, interactive=True)
+                with gr.Row(variant='compact'):
+                    threshold_schedule = gr.Textbox(label="Threshold schedule", lines=1, value = da.threshold_schedule, interactive=True)
+        # PROMPTS TAB    
+        with gr.Tab('Prompts'):
+            # PROMPTS INFO ACCORD  
+            with gr.Accordion(label='*Important* notes on Prompts', elem_id='prompts_info_accord', open=False, visible=True) as prompts_info_accord:
+                gr.HTML("""
+                    <ul style="list-style-type:circle; margin-left:0.75em; margin-bottom:0.2em">
+                    <li>Please always keep values in math functions above 0.</li>
+                    <li>There is *no* Batch mode like in vanilla deforum. Please Use the txt2img tab for that.</li>
+                    <li>For negative prompts, please write your positive prompt, then --neg ugly, text, assymetric, or any other negative tokens of your choice. OR:</li>
+                    <li>Use the negative_prompts field to automatically append all words as a negative prompt. *Don't* add --neg in the negative_prompts field!</li>
+                    <li>Prompts are stored in JSON format. If you've got an error, check it in a <a style="color:SteelBlue" href="https://odu.github.io/slingjsonlint/">JSON Validator</a></li>
+                    </ul>
+                    """)
+            with gr.Row():
+                animation_prompts = gr.Textbox(label="Prompts", lines=8, interactive=True, value = DeforumAnimPrompts())
+            with gr.Row():
+                animation_prompts_positive = gr.Textbox(label="Prompts positive", lines=1, interactive=True, value = "")
+            with gr.Row():
+                animation_prompts_negative = gr.Textbox(label="Prompts negative", lines=1, interactive=True, value = "")
+            # COMPOSABLE MASK SCHEDULING ACCORD
+            with gr.Accordion('Composable Mask scheduling', open=False):
+                gr.HTML("""
+                        <ul style="list-style-type:circle; margin-left:0.75em; margin-bottom:0.2em">
+                        <li>To enable, check use_mask in the Init tab</li>
+                        <li>Supports boolean operations: (! - negation, & - and, | - or, ^ - xor, \ - difference, () - nested operations)</li>
+                        <li>default variables: in \{\}, like \{init_mask\}, \{video_mask\}, \{everywhere\}</li>
+                        <li>masks from files: in [], like [mask1.png]</li>
+                        <li>description-based: <i>word masks</i> in &lt;&gt;, like &lt;apple&gt;, &lt;hair&gt</li>
+                        </ul>
+                        """)
+                with gr.Row():
+                    mask_schedule = gr.Textbox(label="Mask schedule", lines=1, value = da.mask_schedule, interactive=True)
+                with gr.Row():
+                    use_noise_mask = gr.Checkbox(label="Use noise mask", value=da.use_noise_mask, interactive=True)
+                with gr.Row():
+                    noise_mask_schedule = gr.Textbox(label="Noise mask schedule", lines=1, value = da.noise_mask_schedule, interactive=True)
+        # INIT MAIN TAB
+        with gr.Tab('Init'):
+            # IMAGE INIT INNER-TAB
+            with gr.Tab('Image Init'):
+                with gr.Row():
+                    with gr.Column(min_width=150):
+                        use_init = gr.Checkbox(label="Use init", value=d.use_init, interactive=True, visible=True)
+                    with gr.Column(min_width=150):
+                        strength_0_no_init = gr.Checkbox(label="Strength 0 no init", value=True, interactive=True)
+                    with gr.Column(min_width=170):
+                        strength = gr.Slider(label="Strength", minimum=0, maximum=1, step=0.01, value=0, interactive=True)
+                with gr.Row():
+                    init_image = gr.Textbox(label="Init image", lines=1, interactive=True, value = d.init_image)
+            # VIDEO INIT INNER-TAB
+            with gr.Tab('Video Init'):
+                with gr.Row():
+                    video_init_path = gr.Textbox(label="Video init path", lines=1, value = da.video_init_path, interactive=True)
+                with gr.Row():
+                    extract_from_frame = gr.Number(label="Extract from frame", value=da.extract_from_frame, interactive=True, precision=0)
+                    extract_to_frame = gr.Number(label="Extract to frame", value=da.extract_to_frame, interactive=True, precision=0)
+                    extract_nth_frame = gr.Number(label="Extract nth frame", value=da.extract_nth_frame, interactive=True, precision=0)
+                    overwrite_extracted_frames = gr.Checkbox(label="Overwrite extracted frames", value=False, interactive=True)
+                    use_mask_video = gr.Checkbox(label="Use mask video", value=False, interactive=True)
+                with gr.Row():
+                    video_mask_path = gr.Textbox(label="Video mask path", lines=1, value = da.video_mask_path, interactive=True)
+            # MASK INIT INNER-TAB
+            with gr.Tab('Mask Init'):
+                with gr.Row():
+                    use_mask = gr.Checkbox(label="Use mask", value=d.use_mask, interactive=True)
+                    use_alpha_as_mask = gr.Checkbox(label="Use alpha as mask", value=d.use_alpha_as_mask, interactive=True)
+                    invert_mask = gr.Checkbox(label="Invert mask", value=d.invert_mask, interactive=True)
+                    overlay_mask = gr.Checkbox(label="Overlay mask", value=d.overlay_mask, interactive=True)
+                with gr.Row():
+                    mask_file = gr.Textbox(label="Mask file", lines=1, interactive=True, value = d.mask_file)
+                with gr.Row():
+                    mask_overlay_blur = gr.Slider(label="Mask overlay blur", minimum=0, maximum=64, step=1, value=d.mask_overlay_blur, interactive=True)
+                with gr.Row():
+                    choice = mask_fill_choices[d.fill]
+                    fill = gr.Radio(label='Mask fill', choices=mask_fill_choices, value=choice, type="index")
+                with gr.Row():
+                    full_res_mask = gr.Checkbox(label="Full res mask", value=d.full_res_mask, interactive=True)
+                    full_res_mask_padding = gr.Slider(minimum=0, maximum=512, step=1, label="Full res mask padding", value=d.full_res_mask_padding, interactive=True)
+            # PARSEQ ACCORD
+            with gr.Accordion('Parseq', open=False):
+                gr.HTML("""
+                Use an <a style='color:SteelBlue;' target='_blank' href='https://sd-parseq.web.app/deforum'>sd-parseq manifest</a> for your animation (leave blank to ignore).</p>
+                <p style="margin-top:1em">
+                    Note that parseq overrides:
+                    <ul style="list-style-type:circle; margin-left:2em; margin-bottom:1em">
+                        <li>Run: seed, subseed, subseed strength.</li>
+                        <li>Keyframes: generation settings (noise, strength, contrast, scale).</li>
+                        <li>Keyframes: motion parameters for 2D and 3D (angle, zoom, translation, rotation, perspective flip).</li>
+                    </ul>
+                </p>
+                <p">
+                    Parseq does <strong><em>not</em></strong> override:
+                    <ul style="list-style-type:circle; margin-left:2em; margin-bottom:1em">
+                        <li>Run: Sampler, Width, Height, tiling, resize seed.</li>
+                        <li>Keyframes: animation settings (animation mode, max frames, border) </li>
+                        <li>Keyframes: coherence (color coherence & cadence) </li>
+                        <li>Keyframes: depth warping</li>
+                        <li>Output settings: all settings (including fps and max frames)</li>
+                    </ul>
+                </p>
+                """)
+                with gr.Row():
+                    parseq_manifest = gr.Textbox(label="Parseq Manifest (JSON or URL)", lines=4, value = dp.parseq_manifest, interactive=True)
+                with gr.Row():
+                    parseq_use_deltas = gr.Checkbox(label="Use delta values for movement parameters", value=dp.parseq_use_deltas, interactive=True)            
+        def show_hybrid_html_msg(choice):
+            if choice not in ['2D','3D']:
+                return gr.update(visible=True) 
+            else:
+                return gr.update(visible=False)
+        def change_hybrid_tab_status(choice):
+            if choice in ['2D','3D']:
+                return gr.update(visible=True) 
+            else:
+                return gr.update(visible=False)
+        # CONTROLNET TAB
+        with gr.Tab('ControlNet'):
+                gr.HTML("""
+                Requires the <a style='color:SteelBlue;' target='_blank' href='https://github.com/Mikubill/sd-webui-controlnet'>ControlNet</a> extension to be installed.</p>
+                <p style="margin-top:0.2em">
+                    *Work In Progress*. All params below are going to be keyframable at some point. If you want to speedup the integration, join Deforum's development. &#128521;
+                </p>
+                <p">
+                    Due to ControlNet base extension's inner works it needs its models to be located at 'extensions/deforum-for-automatic1111-webui/models'. So copy, symlink or move them there until a more elegant solution is found. And, as of now, it requires use_init checked for the first run. The ControlNet extension version used in the dev process is a24089a62e70a7fae44b7bf35b51fd584dd55e25, if even with all the other options above used it still breaks, upgrade/downgrade your CN version to this one.
+                </p>
+                """)
+                controlnet_dict = setup_controlnet_ui()
+        # HYBRID VIDEO TAB
+        with gr.Tab('Hybrid Video'):
+            # this html only shows when not in 2d/3d mode
+            hybrid_msg_html = gr.HTML(value='Please, change animation mode to 2D or 3D to enable Hybrid Mode',visible=False, elem_id='hybrid_msg_html')
+            # HYBRID INFO ACCORD
+            with gr.Accordion("Info & Help", open=False):
+                hybrid_html = "<p style=\"padding-bottom:0\"><b style=\"text-shadow: blue -1px -1px;\">Hybrid Video Compositing in 2D/3D Mode</b><span style=\"color:#DDD;font-size:0.7rem;text-shadow: black -1px -1px;margin-left:10px;\">by <a href=\"https://github.com/reallybigname\">reallybigname</a></span></p>"
+                hybrid_html += "<ul style=\"list-style-type:circle; margin-left:1em; margin-bottom:1em;\"><li>Composite video with previous frame init image in <b>2D or 3D animation_mode</b> <i>(not for Video Input mode)</i></li>"
+                hybrid_html += "<li>Uses your <b>Init</b> settings for <b>video_init_path, extract_nth_frame, overwrite_extracted_frames</b></li>"
+                hybrid_html += "<li>In Keyframes tab, you can also set <b>color_coherence</b> = '<b>Video Input</b>'</li>"
+                hybrid_html += "<li><b>color_coherence_video_every_N_frames</b> lets you only match every N frames</li>"
+                hybrid_html += "<li>Color coherence may be used with hybrid composite off, to just use video color.</li>"
+                hybrid_html += "<li>Hybrid motion may be used with hybrid composite off, to just use video motion.</li></ul>"
+                hybrid_html += "Hybrid Video Schedules"
+                hybrid_html += "<ul style=\"list-style-type:circle; margin-left:1em; margin-bottom:1em;\"><li>The alpha schedule controls overall alpha for video mix, whether using a composite mask or not.</li>"
+                hybrid_html += "<li>The <b>hybrid_comp_mask_blend_alpha_schedule</b> only affects the 'Blend' <b>hybrid_comp_mask_type</b>.</li>"
+                hybrid_html += "<li>Mask contrast schedule is from 0-255. Normal is 1. Affects all masks.</li>"
+                hybrid_html += "<li>Autocontrast low/high cutoff schedules 0-100. Low 0 High 100 is full range. <br>(<i><b>hybrid_comp_mask_auto_contrast</b> must be enabled</i>)</li></ul>"            
+                hybrid_html += "<a style='color:SteelBlue;' target='_blank' href='https://github.com/deforum-art/deforum-for-automatic1111-webui/wiki/Animation-Settings#hybrid-video-mode-for-2d3d-animations'>Click Here</a> for more info/ a Guide."      
+                gr.HTML(hybrid_html)
+            # HYBRID SETTINGS ACCORD
+            with gr.Accordion("Hybrid Settings", open=True) as hybrid_settings_accord:
+                with gr.Row(variant='compact'):
+                    with gr.Column(min_width=340):
+                        with gr.Row(variant='compact'):
+                            hybrid_generate_inputframes = gr.Checkbox(label="Generate inputframes", value=False, interactive=True)
+                            hybrid_composite = gr.Checkbox(label="Hybrid composite", value=False, interactive=True)
+                    with gr.Column(min_width=340) as hybrid_2nd_column:
+                        with gr.Row(variant='compact'):
+                            hybrid_use_first_frame_as_init_image = gr.Checkbox(label="First frame as init image", value=da.hybrid_use_first_frame_as_init_image, interactive=True, visible=False)
+                            hybrid_motion_use_prev_img = gr.Checkbox(label="Motion use prev img", value=False, interactive=True, visible=False)
+                with gr.Row() as hybrid_flow_row:
+                    with gr.Column(variant='compact'):
+                        with gr.Row(variant='compact'):
+                            hybrid_motion = gr.Radio(['None', 'Optical Flow', 'Perspective', 'Affine'], label="Hybrid motion", value=da.hybrid_motion, elem_id="hybrid_motion")
+                    with gr.Column(variant='compact'):
+                        with gr.Row(variant='compact'):
+                            with gr.Column(scale=1):
+                                hybrid_flow_method = gr.Radio(['DIS Medium', 'Farneback'], label="Flow method", value=da.hybrid_flow_method, elem_id="hybrid_flow_method", visible=False)
+                                hybrid_comp_mask_type = gr.Radio(['None', 'Depth', 'Video Depth', 'Blend', 'Difference'], label="Comp mask type", value=da.hybrid_comp_mask_type, elem_id="hybrid_comp_mask_type", visible=False)
+                with gr.Row(visible=False, variant='compact') as hybrid_comp_mask_row:
+                    hybrid_comp_mask_equalize = gr.Radio(['None', 'Before', 'After', 'Both'], label="Comp mask equalize", value=da.hybrid_comp_mask_equalize, elem_id="hybrid_comp_mask_equalize")
+                    with gr.Column(variant='compact'):
+                        hybrid_comp_mask_auto_contrast = gr.Checkbox(label="Comp mask auto contrast", value=False, interactive=True)
+                        hybrid_comp_mask_inverse = gr.Checkbox(label="Comp mask inverse", value=False, interactive=True)
+                with gr.Row(variant='compact'):
+                        hybrid_comp_save_extra_frames = gr.Checkbox(label="Comp save extra frames", value=False, interactive=True)
+            # HYBRID SCHEDULES ACCORD
+            with gr.Accordion("Hybrid Schedules", open=False, visible=False) as hybrid_sch_accord:
+                with gr.Row(variant='compact') as hybrid_comp_alpha_schedule_row:
+                    hybrid_comp_alpha_schedule = gr.Textbox(label="Comp alpha schedule", lines=1, value = da.hybrid_comp_alpha_schedule, interactive=True)
+                with gr.Row(variant='compact', visible=False) as hybrid_comp_mask_blend_alpha_schedule_row:
+                    hybrid_comp_mask_blend_alpha_schedule = gr.Textbox(label="Comp mask blend alpha schedule", lines=1, value = da.hybrid_comp_mask_blend_alpha_schedule, interactive=True, elem_id="hybridelemtest")
+                with gr.Row(variant='compact', visible=False) as hybrid_comp_mask_contrast_schedule_row:
+                    hybrid_comp_mask_contrast_schedule = gr.Textbox(label="Comp mask contrast schedule", lines=1, value = da.hybrid_comp_mask_contrast_schedule, interactive=True)
+                with gr.Row(variant='compact', visible=False) as hybrid_comp_mask_auto_contrast_cutoff_high_schedule_row :
+                    hybrid_comp_mask_auto_contrast_cutoff_high_schedule = gr.Textbox(label="Comp mask auto contrast cutoff high schedule", lines=1, value = da.hybrid_comp_mask_auto_contrast_cutoff_high_schedule, interactive=True)
+                with gr.Row(variant='compact', visible=False) as hybrid_comp_mask_auto_contrast_cutoff_low_schedule_row:
+                    hybrid_comp_mask_auto_contrast_cutoff_low_schedule = gr.Textbox(label="Comp mask auto contrast cutoff low schedule", lines=1, value = da.hybrid_comp_mask_auto_contrast_cutoff_low_schedule, interactive=True)
+            # HUMANS MASKING ACCORD
+            with gr.Accordion("Humans Masking", open=False, visible=False) as humans_masking_accord:
+                with gr.Row(variant='compact'):
+                    hybrid_generate_human_masks = gr.Radio(['None', 'PNGs', 'Video', 'Both'], label="Generate human masks", value=da.hybrid_generate_human_masks, elem_id="hybrid_generate_human_masks")
+        # OUTPUT TAB
+        with gr.Tab('Output'):
+            # VID OUTPUT ACCORD
+            with gr.Accordion('Video Output Settings', open=True):
+                with gr.Row(variant='compact') as fps_out_format_row:
+                    fps = gr.Slider(label="FPS", value=dv.fps, minimum=1, maximum=240, step=1)
+                    # NOT VISIBLE AS OF 11-02-23 moving to ffmpeg-only!
+                    output_format = gr.Dropdown(visible=False, label="Output format", choices=['FFMPEG mp4'], value='FFMPEG mp4', type="value", elem_id="output_format", interactive=True)
+                with gr.Column(variant='compact'):
+                    with gr.Row(variant='compact') as soundtrack_row:
+                        add_soundtrack = gr.Radio(['None', 'File', 'Init Video'], label="Add soundtrack", value=dv.add_soundtrack)
+                        soundtrack_path = gr.Textbox(label="Soundtrack path", lines=1, interactive=True, value = dv.soundtrack_path)
+                    with gr.Row(variant='compact'):
+                        skip_video_for_run_all = gr.Checkbox(label="Skip video for run all", value=dv.skip_video_for_run_all, interactive=True)
+                        store_frames_in_ram = gr.Checkbox(label="Store frames in ram", value=dv.store_frames_in_ram, interactive=True)
+                        save_depth_maps = gr.Checkbox(label="Save depth maps", value=da.save_depth_maps, interactive=True)
+                        # the following param only shows for windows and linux users!
+                        make_gif = gr.Checkbox(label="Make GIF", value=dv.make_gif, interactive=True)
+                with gr.Row(equal_height=True, variant='compact', visible=(True if dr.current_user_os in ["Windows", "Linux", "Mac"] else False)) as r_upscale_row:
+                    r_upscale_video = gr.Checkbox(label="Upscale", value=dv.r_upscale_video, interactive=True)
+                    r_upscale_model = gr.Dropdown(label="Upscale model", choices=['realesr-animevideov3', 'realesrgan-x4plus', 'realesrgan-x4plus-anime'], interactive=True, value = dv.r_upscale_model, type="value")
+                    r_upscale_factor =  gr.Dropdown(choices=['x2', 'x3', 'x4'], label="Upscale factor", interactive=True, value=dv.r_upscale_factor, type="value")
+                    r_upscale_keep_imgs = gr.Checkbox(label="Keep Imgs", value=dv.r_upscale_keep_imgs, interactive=True)
+                with gr.Accordion('FFmpeg settings', visible=True, open=False) as ffmpeg_quality_accordion:
+                    with gr.Row(equal_height=True, variant='compact', visible=True) as ffmpeg_set_row:
+                        ffmpeg_crf = gr.Slider(minimum=0, maximum=51, step=1, label="CRF", value=dv.ffmpeg_crf, interactive=True)
+                        ffmpeg_preset = gr.Dropdown(label="Preset", choices=['veryslow', 'slower', 'slow', 'medium', 'fast', 'faster', 'veryfast', 'superfast', 'ultrafast'], interactive=True, value = dv.ffmpeg_preset, type="value")
+                    with gr.Row(equal_height=True, variant='compact', visible=True) as ffmpeg_location_row:
+                        ffmpeg_location = gr.Textbox(label="Location", lines=1, interactive=True, value = dv.ffmpeg_location)
+            # FRAME INTERPOLATION TAB
+            with gr.Tab('Frame Interoplation') as frame_interp_tab:
+                with gr.Accordion('Important notes and Help', open=False):
+                    gr.HTML("""
+                    Use <a href="https://github.com/megvii-research/ECCV2022-RIFE">RIFE</a> / <a href="https://film-net.github.io/">FILM</a> Frame Interpolation to smooth out, slow-mo (or both) any video.</p>
+                     <p style="margin-top:1em">
+                        Supported engines:
+                        <ul style="list-style-type:circle; margin-left:1em; margin-bottom:1em">
+                            <li>RIFE v4.6 and FILM.</li>
+                        </ul>
+                    </p>
+                     <p style="margin-top:1em">
+                        Important notes:
+                        <ul style="list-style-type:circle; margin-left:1em; margin-bottom:1em">
+                            <li>Frame Interpolation will *not* run if any of the following are enabled: 'Store frames in ram' / 'Skip video for run all'.</li>
+                            <li>Audio (if provided) will *not* be transferred to the interpolated video if Slow-Mo is enabled.</li>
+                            <li>'add_soundtrack' and 'soundtrack_path' aren't being honoured in "Interpolate an existing video" mode. Original vid audio will be used instead with the same slow-mo rules above.</li>
+                        </ul>
+                    </p>
+                    """)
+                with gr.Column(variant='compact'):
+                    with gr.Row(variant='compact'):
+                        # Interpolation Engine
+                        frame_interpolation_engine = gr.Dropdown(label="Engine", choices=['None','RIFE v4.6','FILM'], value=dv.frame_interpolation_engine, type="value", elem_id="frame_interpolation_engine", interactive=True)
+                        frame_interpolation_slow_mo_enabled = gr.Checkbox(label="Slow Mo", elem_id="frame_interpolation_slow_mo_enabled", value=dv.frame_interpolation_slow_mo_enabled, interactive=True, visible=False)
+                        # If this is set to True, we keep all of the interpolated frames in a folder. Default is False - means we delete them at the end of the run
+                        frame_interpolation_keep_imgs = gr.Checkbox(label="Keep Imgs", elem_id="frame_interpolation_keep_imgs", value=dv.frame_interpolation_keep_imgs, interactive=True, visible=False)
+                    with gr.Row(variant='compact', visible=False) as frame_interp_amounts_row:
+                        with gr.Column(min_width=180) as frame_interp_x_amount_column:
+                            # How many times to interpolate (interp X)
+                            frame_interpolation_x_amount = gr.Slider(minimum=2, maximum=10, step=1, label="Interp X", value=dv.frame_interpolation_x_amount, interactive=True)
+                        with gr.Column(min_width=180, visible=False) as frame_interp_slow_mo_amount_column:
+                            # Interp Slow-Mo (setting final output fps, not really doing anything direclty with RIFE/FILM)
+                            frame_interpolation_slow_mo_amount =  gr.Slider(minimum=2, maximum=10, step=1, label="Slow-Mo X", value=dv.frame_interpolation_x_amount, interactive=True)
+                    # TODO: move these from here when done
+                    def hide_slow_mo(choice):
+                        return gr.update(visible=True) if choice else gr.update(visible=False)
+                    def hide_interp_by_interp_status(choice):
+                        return gr.update(visible=False) if choice == 'None' else gr.update(visible=True)
+                    def change_interp_x_max_limit(engine_name, current_value):
+                        if engine_name == 'FILM':
+                            return gr.update(maximum=300)
+                        elif current_value > 10:
+                            return gr.update(maximum=10, value=2)
+                        return gr.update(maximum=10)
+                    frame_interpolation_slow_mo_enabled.change(fn=hide_slow_mo,inputs=frame_interpolation_slow_mo_enabled,outputs=frame_interp_slow_mo_amount_column)
+                    interp_hide_list = [frame_interpolation_slow_mo_enabled,frame_interpolation_keep_imgs,frame_interp_amounts_row]
+                    for output in interp_hide_list:
+                        frame_interpolation_engine.change(fn=hide_interp_by_interp_status,inputs=frame_interpolation_engine,outputs=output)
+                    frame_interpolation_engine.change(fn=change_interp_x_max_limit,inputs=[frame_interpolation_engine,frame_interpolation_x_amount],outputs=frame_interpolation_x_amount)
+                    with gr.Row(visible=False) as interp_existing_video_row:
+                        # Intrpolate any existing video from the connected PC
+                        with gr.Accordion('Interpolate an existing video', open=False) as interp_existing_video_accord:
+                            # A drag-n-drop UI box to which the user uploads a *single* (at this stage) video
+                            vid_to_interpolate_chosen_file = gr.File(label="Video to Interpolate", interactive=True, file_count="single", file_types=["video"], elem_id="vid_to_interpolate_chosen_file")
+                            with gr.Row(variant='compact'):
+                                # Non interactive textbox showing uploaded input vid total Frame Count
+                                in_vid_frame_count_window = gr.Textbox(label="In Frame Count", lines=1, interactive=False, value='---')
+                                # Non interactive textbox showing uploaded input vid FPS
+                                in_vid_fps_ui_window = gr.Textbox(label="In FPS", lines=1, interactive=False, value='---')
+                                # Non interactive textbox showing expected output interpolated video FPS
+                                out_interp_vid_estimated_fps = gr.Textbox(label="Interpolated Vid FPS", value='---')
+                            # This is the actual button that's pressed to initiate the interpolation:
+                            interpolate_button = gr.Button(value="*Interpolate uploaded video*")
+                            # Show a text about CLI outputs:
+                            gr.HTML("* check your CLI for outputs")
+                            # make the functin call when the interpolation button is clicked
+                            interpolate_button.click(upload_vid_to_interpolate,inputs=[vid_to_interpolate_chosen_file, frame_interpolation_engine, frame_interpolation_x_amount, frame_interpolation_slow_mo_enabled, frame_interpolation_slow_mo_amount, frame_interpolation_keep_imgs, ffmpeg_location, ffmpeg_crf, ffmpeg_preset, in_vid_fps_ui_window])
+                            [change_fn.change(set_interp_out_fps, inputs=[frame_interpolation_x_amount, frame_interpolation_slow_mo_enabled, frame_interpolation_slow_mo_amount, in_vid_fps_ui_window], outputs=out_interp_vid_estimated_fps) for change_fn in [frame_interpolation_x_amount, frame_interpolation_slow_mo_amount, frame_interpolation_slow_mo_enabled]]
+                            # Populate the above FPS and FCount values as soon as a video is uploaded to the FileUploadBox (vid_to_interpolate_chosen_file)
+                            vid_to_interpolate_chosen_file.change(gradio_f_interp_get_fps_and_fcount,inputs=[vid_to_interpolate_chosen_file, frame_interpolation_x_amount, frame_interpolation_slow_mo_enabled, frame_interpolation_slow_mo_amount],outputs=[in_vid_fps_ui_window,in_vid_frame_count_window, out_interp_vid_estimated_fps])
+                    #TODO: move this from here
+                    interp_hide_list = [frame_interpolation_slow_mo_enabled,frame_interpolation_keep_imgs,frame_interp_amounts_row,interp_existing_video_row]
+                    for output in interp_hide_list:
+                        frame_interpolation_engine.change(fn=hide_interp_by_interp_status,inputs=frame_interpolation_engine,outputs=output)
+            # TODO: add upscalers parameters to the settings and make them a part of the pipeline
+            # VIDEO UPSCALE TAB
+            with gr.Tab('Video Upscaling'):
+                vid_to_upscale_chosen_file = gr.File(label="Video to Upscale", interactive=True, file_count="single", file_types=["video"], elem_id="vid_to_upscale_chosen_file")
+                with gr.Column():
+                    # NCNN UPSCALE TAB
+                    with gr.Tab('Upscale V2') as ncnn_upscale_tab:
+                        with gr.Row(variant='compact') as ncnn_upload_vid_stats_row:
+                            # Non interactive textbox showing uploaded input vid total Frame Count
+                            ncnn_upscale_in_vid_frame_count_window = gr.Textbox(label="In Frame Count", lines=1, interactive=False, value='---')
+                            # Non interactive textbox showing uploaded input vid FPS
+                            ncnn_upscale_in_vid_fps_ui_window = gr.Textbox(label="In FPS", lines=1, interactive=False, value='---')
+                            # Non interactive textbox showing uploaded input resolution
+                            ncnn_upscale_in_vid_res = gr.Textbox(label="In Res", lines=1, interactive=False, value='---')
+                            # Non interactive textbox showing expected output resolution
+                            ncnn_upscale_out_vid_res = gr.Textbox(label="Out Res", value='---')
+                        with gr.Column():
+                            with gr.Row(variant='compact', visible=(True if dr.current_user_os in ["Windows", "Linux", "Mac"] else False)) as ncnn_actual_upscale_row:
+                                ncnn_upscale_model = gr.Dropdown(label="Upscale model", choices=['realesr-animevideov3', 'realesrgan-x4plus', 'realesrgan-x4plus-anime'], interactive=True, value = "realesr-animevideov3", type="value")
+                                ncnn_upscale_factor =  gr.Dropdown(choices=['x2', 'x3', 'x4'], label="Upscale factor", interactive=True, value="x2", type="value")
+                                ncnn_upscale_keep_imgs = gr.Checkbox(label="Keep Imgs", value=True, interactive=True) # fix value
+                        ncnn_upscale_btn = gr.Button(value="*Upscale uploaded video*")
+                        ncnn_upscale_btn.click(ncnn_upload_vid_to_upscale,inputs=[vid_to_upscale_chosen_file, ncnn_upscale_in_vid_fps_ui_window, ncnn_upscale_in_vid_res, ncnn_upscale_out_vid_res, ncnn_upscale_model, ncnn_upscale_factor, ncnn_upscale_keep_imgs, ffmpeg_location, ffmpeg_crf, ffmpeg_preset])
+                    with gr.Tab('Upscale V1'):
+                        with gr.Column():
+                            selected_tab = gr.State(value=0)
+                            with gr.Tabs(elem_id="extras_resize_mode"):
+                                with gr.TabItem('Scale by', elem_id="extras_scale_by_tab") as tab_scale_by:
+                                    upscaling_resize = gr.Slider(minimum=1.0, maximum=8.0, step=0.05, label="Resize", value=2, elem_id="extras_upscaling_resize")
+                                with gr.TabItem('Scale to', elem_id="extras_scale_to_tab") as tab_scale_to:
+                                    with FormRow():
+                                        upscaling_resize_w = gr.Slider(label="Width", minimum=1, maximum=7680, step=1, value=512, elem_id="extras_upscaling_resize_w")
+                                        upscaling_resize_h = gr.Slider(label="Height", minimum=1, maximum=7680, step=1, value=512, elem_id="extras_upscaling_resize_h")
+                                        upscaling_crop = gr.Checkbox(label='Crop to fit', value=True, elem_id="extras_upscaling_crop")
+                            with FormRow():
+                                extras_upscaler_1 = gr.Dropdown(label='Upscaler 1', elem_id="extras_upscaler_1", choices=[x.name for x in sh.sd_upscalers], value=sh.sd_upscalers[3].name)
+                                extras_upscaler_2 = gr.Dropdown(label='Upscaler 2', elem_id="extras_upscaler_2", choices=[x.name for x in sh.sd_upscalers], value=sh.sd_upscalers[0].name)
+                            with FormRow():
+                                with gr.Column(scale=3):
+                                    extras_upscaler_2_visibility = gr.Slider(minimum=0.0, maximum=1.0, step=0.001, label="Upscaler 2 visibility", value=0.0, elem_id="extras_upscaler_2_visibility")
+                                with gr.Column(scale=1, min_width=80):
+                                    upscale_keep_imgs = gr.Checkbox(label="Keep Imgs", elem_id="upscale_keep_imgs", value=True, interactive=True)
+                            tab_scale_by.select(fn=lambda: 0, inputs=[], outputs=[selected_tab])
+                            tab_scale_to.select(fn=lambda: 1, inputs=[], outputs=[selected_tab])
+                            # This is the actual button that's pressed to initiate the Upscaling:
+                            upscale_btn = gr.Button(value="*Upscale uploaded video*")
+                            # Show a text about CLI outputs:
+                            gr.HTML("* check your CLI for outputs")
+                            # make the function call when the UPSCALE button is clicked
+                            upscale_btn.click(upload_vid_to_upscale,inputs=[vid_to_upscale_chosen_file, selected_tab, upscaling_resize, upscaling_resize_w, upscaling_resize_h, upscaling_crop, extras_upscaler_1, extras_upscaler_2, extras_upscaler_2_visibility, upscale_keep_imgs, ffmpeg_location, ffmpeg_crf, ffmpeg_preset])
+            # STITCH FRAMES TO VID TAB
+            with gr.Tab('Frames to Video') as stitch_imgs_to_vid_row:
+                with gr.Row(visible=False):
+                    path_name_modifier = gr.Dropdown(label="Path name modifier", choices=['x0_pred', 'x'], value=dv.path_name_modifier, type="value", elem_id="path_name_modifier", interactive=True, visible=False) 
+                gr.HTML("""
+                 <p style="margin-top:0em">
+                    Important Notes:
+                    <ul style="list-style-type:circle; margin-left:1em; margin-bottom:0.25em">
+                        <li>Enter relative to webui folder or Full-Absolute path, and make sure it ends with something like this: '20230124234916_%05d.png', just replace 20230124234916 with your batch ID. The %05d is important, don't forget it!</li>
+                    </ul>
+                    """)
+                with gr.Row(variant='compact'):
+                      image_path = gr.Textbox(label="Image path", lines=1, interactive=True, value = dv.image_path)
+                with gr.Row(visible=False):
+                    mp4_path = gr.Textbox(label="MP4 path", lines=1, interactive=True, value = dv.mp4_path)
+                # not visible as of 06-02-23 since render_steps is disabled as well and they work together. Need to fix both.
+                with gr.Row(visible=False):
+                    # rend_step Never worked - set to visible false 28-1-23 # MOVE OUT FROM HERE!
+                    render_steps = gr.Checkbox(label="Render steps", value=dv.render_steps, interactive=True, visible=False)
+                ffmpeg_stitch_imgs_but = gr.Button(value="*Stitch frames to video*")
+                ffmpeg_stitch_imgs_but.click(direct_stitch_vid_from_frames,inputs=[image_path, fps, ffmpeg_location, ffmpeg_crf, ffmpeg_preset, add_soundtrack, soundtrack_path])
+            # **OLD + NON ACTIVES AREA**
+            with gr.Accordion(visible=False, label='INVISIBLE') as not_in_use_accordion:
+                # NOT VISIBLE AS OF 09-02-23
+                mask_contrast_adjust = gr.Slider(label="Mask contrast adjust", minimum=0, maximum=1, step=0.01, value=d.mask_contrast_adjust, interactive=True)
+                mask_brightness_adjust = gr.Slider(label="Mask brightness adjust", minimum=0, maximum=1, step=0.01, value=d.mask_brightness_adjust, interactive=True)
+                from_img2img_instead_of_link = gr.Checkbox(label="from_img2img_instead_of_link", value=False, interactive=False, visible=False)
+                # INVISIBLE AS OF 08-02 (with static value of 8 for both W and H). Was in Perlin section before Perlin Octaves/Persistence
+                with gr.Column(min_width=200, visible=False):
+                    perlin_w = gr.Slider(label="Perlin W", minimum=0.1, maximum=16, step=0.1, value=da.perlin_w, interactive=True)
+                    perlin_h = gr.Slider(label="Perlin H", minimum=0.1, maximum=16, step=0.1, value=da.perlin_h, interactive=True)
+                with gr.Row(visible=False):
+                    filename_format = gr.Textbox(label="Filename format", lines=1, interactive=True, value = d.filename_format, visible=False)
+                with gr.Row(visible=False):
+                    save_settings = gr.Checkbox(label="save_settings", value=d.save_settings, interactive=True)
+                with gr.Row(visible=False):
+                    save_samples = gr.Checkbox(label="save_samples", value=d.save_samples, interactive=True)
+                    display_samples = gr.Checkbox(label="display_samples", value=False, interactive=False)
+            # NOT VISIBLE 11-02-23 htai
+            with gr.Accordion('Subseed controls & More', open=False, visible=False):
+                # Not visible until fixed, 06-02-23
+                # NOT VISIBLE as of 11-02 - we have sch now. will delete the actual params in a later date
+                with gr.Row(variant='compact', visible=False):
+                    seed_enable_extras = gr.Checkbox(label="Enable subseed controls", value=False)
+                    n_batch = gr.Number(label="N Batch", value=d.n_batch, interactive=True, precision=0, visible=False)
+            with gr.Row(visible=False):
+                save_sample_per_step = gr.Checkbox(label="Save sample per step", value=d.save_sample_per_step, interactive=True)
+                show_sample_per_step = gr.Checkbox(label="Show sample per step", value=d.show_sample_per_step, interactive=True)
+    # Gradio's Change functions - hiding and renaming elements based on other elements
+    fps.change(fn=change_gif_button_visibility, inputs=fps, outputs=make_gif)
+    r_upscale_model.change(fn=update_r_upscale_factor, inputs=r_upscale_model, outputs=r_upscale_factor)
+    ncnn_upscale_model.change(fn=update_r_upscale_factor, inputs=ncnn_upscale_model, outputs=ncnn_upscale_factor)
+    ncnn_upscale_model.change(update_upscale_out_res_by_model_name, inputs=[ncnn_upscale_in_vid_res, ncnn_upscale_model], outputs=ncnn_upscale_out_vid_res)
+    ncnn_upscale_factor.change(update_upscale_out_res, inputs=[ncnn_upscale_in_vid_res, ncnn_upscale_factor], outputs=ncnn_upscale_out_vid_res)
+    vid_to_upscale_chosen_file.change(vid_upscale_gradio_update_stats,inputs=[vid_to_upscale_chosen_file, ncnn_upscale_factor],outputs=[ncnn_upscale_in_vid_fps_ui_window, ncnn_upscale_in_vid_frame_count_window, ncnn_upscale_in_vid_res, ncnn_upscale_out_vid_res])
+    animation_mode.change(fn=change_max_frames_visibility, inputs=animation_mode, outputs=max_frames)
+    animation_mode.change(fn=change_diffusion_cadence_visibility, inputs=animation_mode, outputs=diffusion_cadence)
+    animation_mode.change(fn=disble_3d_related_stuff, inputs=animation_mode, outputs=depth_3d_warping_accord)
+    animation_mode.change(fn=disble_3d_related_stuff, inputs=animation_mode, outputs=fov_accord)
+    animation_mode.change(fn=disble_3d_related_stuff, inputs=animation_mode, outputs=only_3d_motion_column)
+    animation_mode.change(fn=enable_2d_related_stuff, inputs=animation_mode, outputs=only_2d_motion_column) 
+    animation_mode.change(fn=disable_by_interpolation, inputs=animation_mode, outputs=force_grayscale_column)
+    animation_mode.change(fn=disable_pers_flip_accord, inputs=animation_mode, outputs=perspective_flip_accord)    
+    animation_mode.change(fn=disable_pers_flip_accord, inputs=animation_mode, outputs=both_anim_mode_motion_params_column)
+    #Hybrid related:
+    animation_mode.change(fn=show_hybrid_html_msg, inputs=animation_mode, outputs=hybrid_msg_html)
+    animation_mode.change(fn=change_hybrid_tab_status, inputs=animation_mode, outputs=hybrid_sch_accord)
+    animation_mode.change(fn=change_hybrid_tab_status, inputs=animation_mode, outputs=hybrid_settings_accord)
+    animation_mode.change(fn=change_hybrid_tab_status, inputs=animation_mode, outputs=humans_masking_accord)
+    hybrid_comp_mask_type.change(fn=change_comp_mask_x_visibility, inputs=hybrid_comp_mask_type, outputs=hybrid_comp_mask_row)
+    hybrid_motion.change(fn=disable_by_non_optical_flow, inputs=hybrid_motion, outputs=hybrid_flow_method)
+    hybrid_motion.change(fn=disable_by_comp_mask, inputs=hybrid_motion, outputs=hybrid_motion_use_prev_img)
+    hybrid_composite.change(fn=disable_by_hybrid_composite_dynamic, inputs=[hybrid_composite, hybrid_comp_mask_type], outputs=hybrid_comp_mask_row)
+    hybrid_composite_outputs = [humans_masking_accord, hybrid_sch_accord, hybrid_comp_mask_type, hybrid_use_first_frame_as_init_image]
+    for output in hybrid_composite_outputs:
+        hybrid_composite.change(fn=disable_by_hybrid_composite, inputs=hybrid_composite, outputs=output)  
+    hybrid_comp_mask_type_outputs = [hybrid_comp_mask_blend_alpha_schedule_row, hybrid_comp_mask_contrast_schedule_row, hybrid_comp_mask_auto_contrast_cutoff_high_schedule_row, hybrid_comp_mask_auto_contrast_cutoff_low_schedule_row]
+    for output in hybrid_comp_mask_type_outputs:
+        hybrid_comp_mask_type.change(fn=disable_by_comp_mask, inputs=hybrid_comp_mask_type, outputs=output)
+    # End of hybrid related
+    seed_behavior.change(fn=change_seed_iter_visibility, inputs=seed_behavior, outputs=seed_iter_N_row) 
+    seed_behavior.change(fn=change_seed_schedule_visibility, inputs=seed_behavior, outputs=seed_schedule_row)
+    color_coherence.change(fn=change_color_coherence_video_every_N_frames_visibility, inputs=color_coherence, outputs=color_coherence_video_every_N_frames_row)
+    noise_type.change(fn=change_perlin_visibility, inputs=noise_type, outputs=perlin_row)
+    skip_video_for_run_all_outputs = [fps_out_format_row, soundtrack_row, ffmpeg_quality_accordion, store_frames_in_ram, make_gif, r_upscale_row]
+    for output in skip_video_for_run_all_outputs:
+        skip_video_for_run_all.change(fn=change_visibility_from_skip_video, inputs=skip_video_for_run_all, outputs=output)  
+    # END OF UI TABS
+    stuff = locals()
+    stuff = {**stuff, **controlnet_dict}
+    stuff.pop('controlnet_dict')
+    return stuff
+
+### SETTINGS STORAGE UPDATE! 2023-01-27
+### To Reduce The Number Of Settings Overrides,
+### They Are Being Passed As Dictionaries
+### It Would Have Been Also Nice To Retrieve Them
+### From Functions Like Deforumoutputargs(),
+### But Over Time There Was Some Cross-Polination,
+### So They Are Now Hardcoded As 'List'-Strings Below
+### If you're adding a new setting, add it to one of the lists
+### besides writing it in the setup functions above
+
+anim_args_names =   str(r'''animation_mode, max_frames, border,
+                        angle, zoom, translation_x, translation_y, translation_z,
+                        rotation_3d_x, rotation_3d_y, rotation_3d_z,
+                        enable_perspective_flip,
+                        perspective_flip_theta, perspective_flip_phi, perspective_flip_gamma, perspective_flip_fv,
+                        noise_schedule, strength_schedule, contrast_schedule, cfg_scale_schedule, pix2pix_img_cfg_scale_schedule,
+                        enable_subseed_scheduling, subseed_schedule, subseed_strength_schedule,
+                        enable_steps_scheduling, steps_schedule,
+                        fov_schedule, near_schedule, far_schedule,
+                        seed_schedule,
+                        enable_sampler_scheduling, sampler_schedule,
+                        mask_schedule, use_noise_mask, noise_mask_schedule,
+                        enable_checkpoint_scheduling, checkpoint_schedule,
+                        enable_clipskip_scheduling, clipskip_schedule,
+                        kernel_schedule, sigma_schedule, amount_schedule, threshold_schedule,
+                        color_coherence, color_coherence_video_every_N_frames, color_force_grayscale,
+                        diffusion_cadence,
+                        noise_type, perlin_w, perlin_h, perlin_octaves, perlin_persistence,
+                        use_depth_warping, midas_weight,
+                        padding_mode, sampling_mode, save_depth_maps,
+                        video_init_path, extract_nth_frame, extract_from_frame, extract_to_frame, overwrite_extracted_frames,
+                        use_mask_video, video_mask_path,
+                        resume_from_timestring, resume_timestring'''
+                    ).replace("\n", "").replace("\r", "").replace(" ", "").split(',')
+hybrid_args_names =   str(r'''hybrid_generate_inputframes, hybrid_generate_human_masks, hybrid_use_first_frame_as_init_image,
+                        hybrid_motion, hybrid_motion_use_prev_img, hybrid_flow_method, hybrid_composite, hybrid_comp_mask_type, hybrid_comp_mask_inverse,
+                        hybrid_comp_mask_equalize, hybrid_comp_mask_auto_contrast, hybrid_comp_save_extra_frames,
+                        hybrid_comp_alpha_schedule, hybrid_comp_mask_blend_alpha_schedule, hybrid_comp_mask_contrast_schedule,
+                        hybrid_comp_mask_auto_contrast_cutoff_high_schedule, hybrid_comp_mask_auto_contrast_cutoff_low_schedule'''
+                    ).replace("\n", "").replace("\r", "").replace(" ", "").split(',')
+args_names =    str(r'''W, H, tiling, restore_faces,
+                        seed, sampler,
+                        seed_enable_extras, seed_resize_from_w, seed_resize_from_h,
+                        steps, ddim_eta,
+                        n_batch,
+                        save_settings, save_samples, display_samples,
+                        save_sample_per_step, show_sample_per_step, 
+                        batch_name, filename_format,
+                        seed_behavior, seed_iter_N,
+                        use_init, from_img2img_instead_of_link, strength_0_no_init, strength, init_image,
+                        use_mask, use_alpha_as_mask, invert_mask, overlay_mask,
+                        mask_file, mask_contrast_adjust, mask_brightness_adjust, mask_overlay_blur,
+                        fill, full_res_mask, full_res_mask_padding,
+                        reroll_blank_frames'''
+                    ).replace("\n", "").replace("\r", "").replace(" ", "").split(',')
+video_args_names =  str(r'''skip_video_for_run_all,
+                            fps, make_gif, output_format, ffmpeg_location, ffmpeg_crf, ffmpeg_preset,
+                            add_soundtrack, soundtrack_path,
+                            r_upscale_video, r_upscale_model, r_upscale_factor, r_upscale_keep_imgs,
+                            render_steps,
+                            path_name_modifier, image_path, mp4_path, store_frames_in_ram,
+                            frame_interpolation_engine, frame_interpolation_x_amount, frame_interpolation_slow_mo_enabled, frame_interpolation_slow_mo_amount,
+                            frame_interpolation_keep_imgs'''
+                    ).replace("\n", "").replace("\r", "").replace(" ", "").split(',')
+parseq_args_names = str(r'''parseq_manifest, parseq_use_deltas'''
+                    ).replace("\n", "").replace("\r", "").replace(" ", "").split(',')
+loop_args_names = str(r'''use_looper, init_images, image_strength_schedule, blendFactorMax, blendFactorSlope, 
+                          tweening_frames_schedule, color_correction_factor'''
+                    ).replace("\n", "").replace("\r", "").replace(" ", "").split(',')
+
+component_names =   ['override_settings_with_file', 'custom_settings_file'] + anim_args_names +['animation_prompts', 'animation_prompts_positive', 'animation_prompts_negative'] + args_names + video_args_names + parseq_args_names + hybrid_args_names + loop_args_names + controlnet_component_names()
+settings_component_names = [name for name in component_names if name not in video_args_names]
+
+def setup_deforum_setting_ui(self, is_img2img, is_extension = True):
+    ds = setup_deforum_setting_dictionary(self, is_img2img, is_extension)
+    return [ds[name] for name in (['btn'] + component_names)]
+
+def pack_anim_args(args_dict):
+    return {name: args_dict[name] for name in (anim_args_names + hybrid_args_names)}
+
+def pack_args(args_dict):
+    args_dict = {name: args_dict[name] for name in args_names}
+    args_dict['precision'] = 'autocast' 
+    args_dict['scale'] = 7
+    args_dict['subseed'] = -1
+    args_dict['subseed_strength'] = 0
+    args_dict['C'] = 4
+    args_dict['f'] = 8
+    args_dict['timestring'] = ""
+    args_dict['init_latent'] = None
+    args_dict['init_sample'] = None
+    args_dict['init_c'] = None
+    args_dict['noise_mask'] = None
+    args_dict['seed_internal'] = 0
+    return args_dict
+    
+def pack_video_args(args_dict):
+    return {name: args_dict[name] for name in video_args_names}
+
+def pack_parseq_args(args_dict):
+    return {name: args_dict[name] for name in parseq_args_names}
+    
+def pack_loop_args(args_dict):
+    return {name: args_dict[name] for name in loop_args_names}
+
+def pack_controlnet_args(args_dict):
+    return {name: args_dict[name] for name in controlnet_component_names()}
+
+def process_args(args_dict_main):
+    override_settings_with_file = args_dict_main['override_settings_with_file']
+    custom_settings_file = args_dict_main['custom_settings_file']
+    args_dict = pack_args(args_dict_main)
+    anim_args_dict = pack_anim_args(args_dict_main)
+    video_args_dict = pack_video_args(args_dict_main)
+    parseq_args_dict = pack_parseq_args(args_dict_main)
+    loop_args_dict = pack_loop_args(args_dict_main)
+    controlnet_args_dict = pack_controlnet_args(args_dict_main)
+
+    import json
+    
+    root = SimpleNamespace(**Root())
+    root.p = args_dict_main['p']
+    p = root.p
+    root.animation_prompts = json.loads(args_dict_main['animation_prompts'])
+    positive_prompts = args_dict_main['animation_prompts_positive']
+    negative_prompts = args_dict_main['animation_prompts_negative']
+    # remove --neg from negative_prompts if recieved by mistake
+    negative_prompts = negative_prompts.replace('--neg', '')
+    for key in root.animation_prompts:
+        animationPromptCurr = root.animation_prompts[key]
+        root.animation_prompts[key] = f"{positive_prompts} {animationPromptCurr} {'' if '--neg' in animationPromptCurr else '--neg'} {negative_prompts}"
+    from deforum_helpers.settings import load_args
+    
+    if override_settings_with_file:
+        load_args(args_dict, anim_args_dict, parseq_args_dict, loop_args_dict, controlnet_args_dict, custom_settings_file, root)
+    
+    if not os.path.exists(root.models_path):
+        os.mkdir(root.models_path)
+
+    args = SimpleNamespace(**args_dict)
+    anim_args = SimpleNamespace(**anim_args_dict)
+    video_args = SimpleNamespace(**video_args_dict)
+    parseq_args = SimpleNamespace(**parseq_args_dict)
+    loop_args = SimpleNamespace(**loop_args_dict)
+    controlnet_args = SimpleNamespace(**controlnet_args_dict)
+
+    p.width, p.height = map(lambda x: x - x % 64, (args.W, args.H))
+    p.steps = args.steps
+    p.seed = args.seed
+    p.sampler_name = args.sampler
+    p.batch_size = args.n_batch
+    p.tiling = args.tiling
+    p.restore_faces = args.restore_faces
+    p.seed_enable_extras = args.seed_enable_extras
+    p.subseed = args.subseed
+    p.subseed_strength = args.subseed_strength
+    p.seed_resize_from_w = args.seed_resize_from_w
+    p.seed_resize_from_h = args.seed_resize_from_h
+    p.fill = args.fill
+    p.ddim_eta = args.ddim_eta
+
+    # TODO: Handle batch name dynamically?
+    current_arg_list = [args, anim_args, video_args, parseq_args]
+    args.outdir = os.path.join(p.outpath_samples, args.batch_name)
+    root.outpath_samples = args.outdir
+    args.outdir = os.path.join(os.getcwd(), args.outdir)
+    if not os.path.exists(args.outdir):
+        os.makedirs(args.outdir)
+    
+    args.seed = get_fixed_seed(args.seed)
+        
+    args.timestring = time.strftime('%Y%m%d%H%M%S')
+    args.strength = max(0.0, min(1.0, args.strength))
+
+    if not args.use_init:
+        args.init_image = None
+        
+    if anim_args.animation_mode == 'None':
+        anim_args.max_frames = 1
+    elif anim_args.animation_mode == 'Video Input':
+        args.use_init = True
+    
+    return root, args, anim_args, video_args, parseq_args, loop_args, controlnet_args
+
+def print_args(args):
+    print("ARGS: /n")
+    for key, value in args.__dict__.items():
+        print(f"{key}: {value}")
+ 
+# Local gradio-to-frame-interoplation function. *Needs* to stay here since we do Root() and use gradio elements directly, to be changed in the future
+def upload_vid_to_interpolate(file, engine, x_am, sl_enabled, sl_am, keep_imgs, f_location, f_crf, f_preset, in_vid_fps):
+    # print msg and do nothing if vid not uploaded or interp_x not provided
+    if not file or engine == 'None':
+        return print("Please upload a video and set a proper value for 'Interp X'. Can't interpolate x0 times :)")
+
+    root_params = Root()
+    f_models_path = root_params['models_path']
+
+    process_interp_vid_upload_logic(file, engine, x_am, sl_enabled, sl_am, keep_imgs, f_location, f_crf, f_preset, in_vid_fps, f_models_path, file.orig_name)
+
+# Local gradio-to-upscalers function. *Needs* to stay here since we do Root() and use gradio elements directly, to be changed in the future
+def upload_vid_to_upscale(vid_to_upscale_chosen_file, selected_tab, upscaling_resize, upscaling_resize_w, upscaling_resize_h, upscaling_crop, extras_upscaler_1, extras_upscaler_2, extras_upscaler_2_visibility, upscale_keep_imgs, ffmpeg_location, ffmpeg_crf, ffmpeg_preset):
+    # print msg and do nothing if vid not uploaded
+    if not vid_to_upscale_chosen_file:
+        return print("Please upload a video :)")
+    
+    process_upscale_vid_upload_logic(vid_to_upscale_chosen_file, selected_tab, upscaling_resize, upscaling_resize_w, upscaling_resize_h, upscaling_crop, extras_upscaler_1, extras_upscaler_2, extras_upscaler_2_visibility, vid_to_upscale_chosen_file.orig_name, upscale_keep_imgs, ffmpeg_location, ffmpeg_crf, ffmpeg_preset)
+
+def ncnn_upload_vid_to_upscale(vid_path, in_vid_fps, in_vid_res, out_vid_res, upscale_model, upscale_factor, keep_imgs, f_location, f_crf, f_preset):
+    if vid_path is None:
+        print("Please upload a video :)")
+        return
+    root_params = Root()
+    f_models_path = root_params['models_path']
+    current_user = root_params['current_user_os']
+    process_ncnn_upscale_vid_upload_logic(vid_path, in_vid_fps, in_vid_res, out_vid_res, f_models_path, upscale_model, upscale_factor, keep_imgs, f_location, f_crf, f_preset, current_user)

extensions/deforum/scripts/deforum_helpers/blank_frame_reroll.py

@@ -0,0 +1,24 @@
+from .generate import generate
+#WebUI
+from modules.shared import opts, cmd_opts, state
+
+def blank_frame_reroll(image, args, root, frame_idx):
+    patience = 10
+    print("Blank frame detected! If you don't have the NSFW filter enabled, this may be due to a glitch!")
+    if args.reroll_blank_frames == 'reroll':
+        while not image.getbbox():
+            print("Rerolling with +1 seed...")
+            args.seed += 1
+            image = generate(args, root, frame_idx)
+            patience -= 1
+            if patience == 0:
+                print("Rerolling with +1 seed failed for 10 iterations! Try setting webui's precision to 'full' and if it fails, please report this to the devs! Interrupting...")
+                state.interrupted = True
+                state.current_image = image
+                return None
+    elif args.reroll_blank_frames == 'interrupt':
+        print("Interrupting to save your eyes...")
+        state.interrupted = True
+        state.current_image = image
+        return None
+    return image

extensions/deforum/scripts/deforum_helpers/colors.py

@@ -0,0 +1,16 @@
+from skimage.exposure import match_histograms
+import cv2
+
+def maintain_colors(prev_img, color_match_sample, mode):
+    if mode == 'Match Frame 0 RGB':
+        return match_histograms(prev_img, color_match_sample, multichannel=True)
+    elif mode == 'Match Frame 0 HSV':
+        prev_img_hsv = cv2.cvtColor(prev_img, cv2.COLOR_RGB2HSV)
+        color_match_hsv = cv2.cvtColor(color_match_sample, cv2.COLOR_RGB2HSV)
+        matched_hsv = match_histograms(prev_img_hsv, color_match_hsv, multichannel=True)
+        return cv2.cvtColor(matched_hsv, cv2.COLOR_HSV2RGB)
+    else: # Match Frame 0 LAB
+        prev_img_lab = cv2.cvtColor(prev_img, cv2.COLOR_RGB2LAB)
+        color_match_lab = cv2.cvtColor(color_match_sample, cv2.COLOR_RGB2LAB)
+        matched_lab = match_histograms(prev_img_lab, color_match_lab, multichannel=True)
+        return cv2.cvtColor(matched_lab, cv2.COLOR_LAB2RGB)

extensions/deforum/scripts/deforum_helpers/composable_masks.py

@@ -0,0 +1,198 @@
+# At the moment there are three types of masks: mask from variable, file mask and word mask
+# Variable masks include init_mask for the predefined whole-video mask, frame_mask from video-masking system
+# and human_mask for a model which better segments people in the background video
+# They are put in {}-brackets
+# Word masks are framed with <>-bracets, like: <cat>, <anime girl>
+# File masks are put in []-brackes
+# Empty strings are counted as the whole frame
+# We want to put them all into a sequence of boolean operations
+
+# Example:
+# \ <armor>
+# (({human_mask} & [mask1.png]) ^ <apple>)
+
+# Writing the parser for the boolean sequence
+# using regex and PIL operations
+import re
+from .load_images import get_mask_from_file, check_mask_for_errors, blank_if_none
+from .word_masking import get_word_mask
+from torch import Tensor
+import PIL
+from PIL import Image, ImageChops
+
+# val_masks: name, PIL Image mask
+# Returns an image in mode '1' (needed for bool ops), convert to 'L' in the sender function
+def compose_mask(root, args, mask_seq, val_masks, frame_image, inner_idx:int = 0):
+    # Compose_mask recursively: go to inner brackets, then b-op it and go upstack
+    
+    # Step 1:
+    # recursive parenthesis pass
+    # regex is not powerful here
+
+    seq = ""
+    inner_seq = ""
+    parentheses_counter = 0
+
+    for c in mask_seq:
+        if c == ')':
+            parentheses_counter = parentheses_counter - 1
+        if parentheses_counter > 0:
+            inner_seq += c
+        if c == '(':
+            parentheses_counter = parentheses_counter + 1
+        if parentheses_counter == 0:
+            if len(inner_seq) > 0:
+                inner_idx += 1
+                seq += compose_mask(root, args, inner_seq, val_masks, frame_image, inner_idx)
+                inner_seq = ""
+            else:
+                seq += c
+    
+    if parentheses_counter != 0:
+        raise Exception('Mismatched parentheses in {mask_seq}!')
+
+    mask_seq = seq
+
+    # Step 2:
+    # Load the word masks and file masks as vars
+    
+    # File masks
+    pattern = r'\[(?P<inner>[\S\s]*?)\]'
+    
+    def parse(match_object):
+        nonlocal inner_idx
+        inner_idx += 1
+        content = match_object.groupdict()['inner']
+        val_masks[str(inner_idx)] = get_mask_from_file(content, args).convert('1') # TODO: add caching
+        return f"{{{inner_idx}}}"
+    
+    mask_seq = re.sub(pattern, parse, mask_seq)
+    
+    # Word masks
+    pattern = r'<(?P<inner>[\S\s]*?)>'
+    
+    def parse(match_object):
+        nonlocal inner_idx
+        inner_idx += 1
+        content = match_object.groupdict()['inner']
+        val_masks[str(inner_idx)] = get_word_mask(root, frame_image, content).convert('1')
+        return f"{{{inner_idx}}}"
+    
+    mask_seq = re.sub(pattern, parse, mask_seq)
+    
+    # Now that all inner parenthesis are eliminated we're left with a linear string
+    
+    # Step 3:
+    # Boolean operations with masks
+    # Operators: invert !, and &, or |, xor ^, difference \
+    
+    # Invert vars with '!'
+    pattern = r'![\S\s]*{(?P<inner>[\S\s]*?)}'
+    def parse(match_object):
+        nonlocal inner_idx
+        inner_idx += 1
+        content = match_object.groupdict()['inner']
+        savename = content
+        if content in root.mask_preset_names:
+            inner_idx += 1
+            savename = str(inner_idx)
+        val_masks[savename] = ImageChops.invert(val_masks[content])
+        return f"{{{savename}}}"
+    
+    mask_seq = re.sub(pattern, parse, mask_seq)
+    
+    # Multiply neighbouring vars with '&'
+    # Wait for replacements stall (like in Markov chains)
+    while True:
+        pattern = r'{(?P<inner1>[\S\s]*?)}[\s]*&[\s]*{(?P<inner2>[\S\s]*?)}'
+        def parse(match_object):
+            nonlocal inner_idx
+            inner_idx += 1
+            content = match_object.groupdict()['inner1']
+            content_second = match_object.groupdict()['inner2']
+            savename = content
+            if content in root.mask_preset_names:
+                inner_idx += 1
+                savename = str(inner_idx)
+            val_masks[savename] = ImageChops.logical_and(val_masks[content], val_masks[content_second])
+            return f"{{{savename}}}"
+        
+        prev_mask_seq = mask_seq
+        mask_seq = re.sub(pattern, parse, mask_seq)
+        if mask_seq is prev_mask_seq:
+            break
+    
+    # Add neighbouring vars with '|'
+    while True:
+        pattern = r'{(?P<inner1>[\S\s]*?)}[\s]*?\|[\s]*?{(?P<inner2>[\S\s]*?)}'
+        def parse(match_object):
+            nonlocal inner_idx
+            inner_idx += 1
+            content = match_object.groupdict()['inner1']
+            content_second = match_object.groupdict()['inner2']
+            savename = content
+            if content in root.mask_preset_names:
+                inner_idx += 1
+                savename = str(inner_idx)
+            val_masks[savename] = ImageChops.logical_or(val_masks[content], val_masks[content_second])
+            return f"{{{savename}}}"
+        
+        prev_mask_seq = mask_seq
+        mask_seq = re.sub(pattern, parse, mask_seq)
+        if mask_seq is prev_mask_seq:
+            break
+    
+    # Mutually exclude neighbouring vars with '^'
+    while True:
+        pattern = r'{(?P<inner1>[\S\s]*?)}[\s]*\^[\s]*{(?P<inner2>[\S\s]*?)}'
+        def parse(match_object):
+            nonlocal inner_idx
+            inner_idx += 1
+            content = match_object.groupdict()['inner1']
+            content_second = match_object.groupdict()['inner2']
+            savename = content
+            if content in root.mask_preset_names:
+                inner_idx += 1
+                savename = str(inner_idx)
+            val_masks[savename] = ImageChops.logical_xor(val_masks[content], val_masks[content_second])
+            return f"{{{savename}}}"
+        
+        prev_mask_seq = mask_seq
+        mask_seq = re.sub(pattern, parse, mask_seq)
+        if mask_seq is prev_mask_seq:
+            break
+    
+    # Set-difference the regions with '\'
+    while True:
+        pattern = r'{(?P<inner1>[\S\s]*?)}[\s]*\\[\s]*{(?P<inner2>[\S\s]*?)}'
+        def parse(match_object):
+            content = match_object.groupdict()['inner1']
+            content_second = match_object.groupdict()['inner2']
+            savename = content
+            if content in root.mask_preset_names:
+                nonlocal inner_idx
+                inner_idx += 1
+                savename = str(inner_idx)
+            val_masks[savename] = ImageChops.logical_and(val_masks[content], ImageChops.invert(val_masks[content_second]))
+            return f"{{{savename}}}"
+        
+        prev_mask_seq = mask_seq
+        mask_seq = re.sub(pattern, parse, mask_seq)
+        if mask_seq is prev_mask_seq:
+            break
+    
+    # Step 4:
+    # Output
+    # Now we should have a single var left to return. If not, raise an error message
+    pattern = r'{(?P<inner>[\S\s]*?)}'
+    matches = re.findall(pattern, mask_seq)
+    
+    if len(matches) != 1:
+        raise Exception(f'Wrong composable mask expression format! Broken mask sequence: {mask_seq}')
+    
+    return f"{{{matches[0]}}}"
+
+def compose_mask_with_check(root, args, mask_seq, val_masks, frame_image):
+    for k, v in val_masks.items():
+        val_masks[k] = blank_if_none(v, args.W, args.H, '1').convert('1')
+    return check_mask_for_errors(val_masks[compose_mask(root, args, mask_seq, val_masks, frame_image, 0)[1:-1]].convert('L'))

extensions/deforum/scripts/deforum_helpers/deforum_controlnet.py

@@ -0,0 +1,462 @@
+# This helper script is responsible for ControlNet/Deforum integration
+# https://github.com/Mikubill/sd-webui-controlnet — controlnet repo
+
+import os, sys
+import gradio as gr
+import scripts
+import modules.scripts as scrpts
+from PIL import Image
+import numpy as np
+from modules.processing import process_images
+from .rich import console
+from rich.table import Table
+from rich import box
+
+has_controlnet = None
+
+def find_controlnet():
+    global has_controlnet
+    if has_controlnet is not None:
+        return has_controlnet
+    
+    try:
+        from scripts import controlnet
+    except Exception as e:
+        print(f'\033[33mFailed to import controlnet! The exact error is {e}. Deforum support for ControlNet will not be activated\033[0m')
+        has_controlnet = False
+        return False
+    has_controlnet = True
+    print(f"\033[0;32m*Deforum ControlNet support: enabled*\033[0m")
+    return True
+
+# The most parts below are plainly copied from controlnet.py
+# TODO: come up with a cleaner way
+
+gradio_compat = True
+try:
+    from distutils.version import LooseVersion
+    from importlib_metadata import version
+    if LooseVersion(version("gradio")) < LooseVersion("3.10"):
+        gradio_compat = False
+except ImportError:
+    pass
+
+# svgsupports
+svgsupport = False
+try:
+    import io
+    import base64
+    from svglib.svglib import svg2rlg
+    from reportlab.graphics import renderPM
+    svgsupport = True
+except ImportError:
+    pass
+
+def ControlnetArgs():
+    controlnet_enabled = False
+    controlnet_scribble_mode = False
+    controlnet_rgbbgr_mode = False
+    controlnet_lowvram = False
+    controlnet_module = "none"
+    controlnet_model = "None"
+    controlnet_weight = 1.0
+    controlnet_guidance_strength = 1.0
+    blendFactorMax = "0:(0.35)"
+    blendFactorSlope = "0:(0.25)"
+    tweening_frames_schedule = "0:(20)"
+    color_correction_factor = "0:(0.075)"
+    return locals()
+
+def setup_controlnet_ui_raw():
+    # Already under an accordion
+    from scripts import controlnet
+    from scripts.controlnet import update_cn_models, cn_models, cn_models_names
+
+    refresh_symbol = '\U0001f504'  # 🔄
+    switch_values_symbol = '\U000021C5' # ⇅
+    model_dropdowns = []
+    infotext_fields = []
+        # Main part
+    class ToolButton(gr.Button, gr.components.FormComponent):
+        """Small button with single emoji as text, fits inside gradio forms"""
+
+        def __init__(self, **kwargs):
+            super().__init__(variant="tool", **kwargs)
+
+        def get_block_name(self):
+            return "button"
+            
+    from scripts.processor import canny, midas, midas_normal, leres, hed, mlsd, openpose, pidinet, simple_scribble, fake_scribble, uniformer
+
+    preprocessor = {
+        "none": lambda x, *args, **kwargs: x,
+        "canny": canny,
+        "depth": midas,
+        "depth_leres": leres,
+        "hed": hed,
+        "mlsd": mlsd,
+        "normal_map": midas_normal,
+        "openpose": openpose,
+        # "openpose_hand": openpose_hand,
+        "pidinet": pidinet,
+        # "scribble": simple_scribble,
+        "fake_scribble": fake_scribble,
+        "segmentation": uniformer,
+    }
+
+    # Copying the main ControlNet widgets while getting rid of static elements such as the scribble pad
+    with gr.Row():
+        controlnet_enabled = gr.Checkbox(label='Enable', value=False)
+        controlnet_scribble_mode = gr.Checkbox(label='Scribble Mode (Invert colors)', value=False, visible=False)
+        controlnet_rgbbgr_mode = gr.Checkbox(label='RGB to BGR', value=False, visible=False)
+        controlnet_lowvram = gr.Checkbox(label='Low VRAM', value=False, visible=False)
+
+    def refresh_all_models(*inputs):
+        update_cn_models()
+        
+        dd = inputs[0]
+        selected = dd if dd in cn_models else "None"
+        return gr.Dropdown.update(value=selected, choices=list(cn_models.keys()))
+
+    with gr.Row(visible=False) as cn_mod_row:
+        controlnet_module = gr.Dropdown(list(preprocessor.keys()), label=f"Preprocessor", value="none")
+        controlnet_model = gr.Dropdown(list(cn_models.keys()), label=f"Model", value="None")
+        refresh_models = ToolButton(value=refresh_symbol)
+        refresh_models.click(refresh_all_models, controlnet_model, controlnet_model)
+        # ctrls += (refresh_models, )
+    with gr.Row(visible=False) as cn_weight_row:
+        controlnet_weight = gr.Slider(label=f"Weight", value=1.0, minimum=0.0, maximum=2.0, step=.05)
+        controlnet_guidance_strength =  gr.Slider(label="Guidance strength (T)", value=1.0, minimum=0.0, maximum=1.0, interactive=True)
+        # ctrls += (module, model, weight,)
+        # model_dropdowns.append(model)
+  
+    # advanced options    
+    controlnet_advanced = gr.Column(visible=False)
+    with controlnet_advanced:
+        controlnet_processor_res = gr.Slider(label="Annotator resolution", value=64, minimum=64, maximum=2048, interactive=False)
+        controlnet_threshold_a =  gr.Slider(label="Threshold A", value=64, minimum=64, maximum=1024, interactive=False)
+        controlnet_threshold_b =  gr.Slider(label="Threshold B", value=64, minimum=64, maximum=1024, interactive=False)
+    
+    if gradio_compat:    
+        controlnet_module.change(build_sliders, inputs=[controlnet_module], outputs=[controlnet_processor_res, controlnet_threshold_a, controlnet_threshold_b, controlnet_advanced])
+        
+    infotext_fields.extend([
+        (controlnet_module, f"ControlNet Preprocessor"),
+        (controlnet_model, f"ControlNet Model"),
+        (controlnet_weight, f"ControlNet Weight"),
+    ])
+
+    with gr.Row(visible=False) as cn_env_row:
+        controlnet_resize_mode = gr.Radio(choices=["Envelope (Outer Fit)", "Scale to Fit (Inner Fit)", "Just Resize"], value="Scale to Fit (Inner Fit)", label="Resize Mode")
+
+    # Video input to be fed into ControlNet
+    #input_video_url = gr.Textbox(source='upload', type='numpy', tool='sketch') # TODO
+    controlnet_input_video_chosen_file = gr.File(label="ControlNet Video Input", interactive=True, file_count="single", file_types=["video"], elem_id="controlnet_input_video_chosen_file", visible=False)
+    controlnet_input_video_mask_chosen_file = gr.File(label="ControlNet Video Mask Input", interactive=True, file_count="single", file_types=["video"], elem_id="controlnet_input_video_mask_chosen_file", visible=False)
+   
+    cn_hide_output_list = [controlnet_scribble_mode,controlnet_rgbbgr_mode,controlnet_lowvram,cn_mod_row,cn_weight_row,cn_env_row,controlnet_input_video_chosen_file,controlnet_input_video_mask_chosen_file] 
+    for cn_output in cn_hide_output_list:
+        controlnet_enabled.change(fn=hide_ui_by_cn_status, inputs=controlnet_enabled,outputs=cn_output)
+        
+    return locals()
+
+            
+def setup_controlnet_ui():
+    if not find_controlnet():
+        gr.HTML("""
+                <a style='target='_blank' href='https://github.com/Mikubill/sd-webui-controlnet'>ControlNet not found. Please install it :)</a>
+                """, elem_id='controlnet_not_found_html_msg')
+        return {}
+
+    return setup_controlnet_ui_raw()
+
+def controlnet_component_names():
+    if not find_controlnet():
+        return []
+
+    controlnet_args_names = str(r'''controlnet_input_video_chosen_file, controlnet_input_video_mask_chosen_file,
+controlnet_enabled, controlnet_scribble_mode, controlnet_rgbbgr_mode, controlnet_lowvram,
+controlnet_module, controlnet_model,
+controlnet_weight, controlnet_guidance_strength,
+controlnet_processor_res, 
+controlnet_threshold_a, controlnet_threshold_b, controlnet_resize_mode'''
+    ).replace("\n", "").replace("\r", "").replace(" ", "").split(',')
+    
+    return controlnet_args_names
+
+def is_controlnet_enabled(controlnet_args):
+    return 'controlnet_enabled' in vars(controlnet_args) and controlnet_args.controlnet_enabled
+
+def process_txt2img_with_controlnet(p, args, anim_args, loop_args, controlnet_args, root, frame_idx = 1):
+    # TODO: use init image and mask here
+    p.control_net_enabled = False # we don't want to cause concurrence
+    p.init_images = []
+    controlnet_frame_path = os.path.join(args.outdir, 'controlnet_inputframes', f"{frame_idx:05}.jpg")
+    controlnet_mask_frame_path = os.path.join(args.outdir, 'controlnet_maskframes', f"{frame_idx:05}.jpg")
+    cn_mask_np = None
+    cn_image_np = None
+
+    if not os.path.exists(controlnet_frame_path) and not os.path.exists(controlnet_mask_frame_path):
+        print(f'\033[33mNeither the base nor the masking frames for ControlNet were found. Using the regular pipeline\033[0m')
+        from .deforum_controlnet_hardcode import restore_networks
+        unet = p.sd_model.model.diffusion_model
+        restore_networks(unet)
+        return process_images(p)
+    
+    if os.path.exists(controlnet_frame_path):
+        cn_image_np = Image.open(controlnet_frame_path).convert("RGB")
+    
+    if os.path.exists(controlnet_mask_frame_path):
+        cn_mask_np = Image.open(controlnet_mask_frame_path).convert("RGB")
+
+    cn_args = {
+        "enabled": True,
+        "module": controlnet_args.controlnet_module,
+        "model": controlnet_args.controlnet_model,
+        "weight": controlnet_args.controlnet_weight,
+        "input_image": {'image': cn_image_np, 'mask': cn_mask_np},
+        "scribble_mode": controlnet_args.controlnet_scribble_mode,
+        "resize_mode": controlnet_args.controlnet_resize_mode,
+        "rgbbgr_mode": controlnet_args.controlnet_rgbbgr_mode,
+        "lowvram": controlnet_args.controlnet_lowvram,
+        "processor_res": controlnet_args.controlnet_processor_res,
+        "threshold_a": controlnet_args.controlnet_threshold_a,
+        "threshold_b": controlnet_args.controlnet_threshold_b,
+        "guidance_strength": controlnet_args.controlnet_guidance_strength,"guidance_strength": controlnet_args.controlnet_guidance_strength,
+    }
+
+    from .deforum_controlnet_hardcode import process
+    p.script_args = (
+        cn_args["enabled"],
+        cn_args["module"],
+        cn_args["model"],
+        cn_args["weight"],
+        cn_args["input_image"],
+        cn_args["scribble_mode"],
+        cn_args["resize_mode"],
+        cn_args["rgbbgr_mode"],
+        cn_args["lowvram"],
+        cn_args["processor_res"],
+        cn_args["threshold_a"],
+        cn_args["threshold_b"],
+        cn_args["guidance_strength"],
+    )
+
+    table = Table(title="ControlNet params",padding=0, box=box.ROUNDED)
+
+    field_names = []
+    field_names += ["module", "model", "weight", "guidance", "scribble", "resize", "rgb->bgr", "proc res", "thr a", "thr b"]
+    for field_name in field_names:
+        table.add_column(field_name, justify="center")
+    
+    rows = []
+    rows += [cn_args["module"], cn_args["model"], cn_args["weight"], cn_args["guidance_strength"], cn_args["scribble_mode"], cn_args["resize_mode"], cn_args["rgbbgr_mode"], cn_args["processor_res"], cn_args["threshold_a"], cn_args["threshold_b"]]
+    rows = [str(x) for x in rows]
+
+    table.add_row(*rows)
+    
+    console.print(table)
+
+    processed = process(p, *(p.script_args))
+
+    if processed is None: # the script just swaps the pipeline, so failing is OK for the first time
+        processed = process_images(p)
+    
+    if processed is None: # now it's definitely not OK
+        raise Exception("\033[31mFailed to process a frame with ControlNet enabled!\033[0m")
+    
+    p.close()
+
+    return processed
+
+def process_img2img_with_controlnet(p, args, anim_args, loop_args, controlnet_args, root, frame_idx = 0):
+    p.control_net_enabled = False # we don't want to cause concurrence
+    controlnet_frame_path = os.path.join(args.outdir, 'controlnet_inputframes', f"{frame_idx:05}.jpg")
+    controlnet_mask_frame_path = os.path.join(args.outdir, 'controlnet_maskframes', f"{frame_idx:05}.jpg")
+
+    print(f'Reading ControlNet base frame {frame_idx} at {controlnet_frame_path}')
+    print(f'Reading ControlNet mask frame {frame_idx} at {controlnet_mask_frame_path}')
+
+    cn_mask_np = None
+    cn_image_np = None
+
+    if not os.path.exists(controlnet_frame_path) and not os.path.exists(controlnet_mask_frame_path):
+        print(f'\033[33mNeither the base nor the masking frames for ControlNet were found. Using the regular pipeline\033[0m')
+        return process_images(p)
+    
+    if os.path.exists(controlnet_frame_path):
+        cn_image_np = np.array(Image.open(controlnet_frame_path).convert("RGB")).astype('uint8')
+    
+    if os.path.exists(controlnet_mask_frame_path):
+        cn_mask_np = np.array(Image.open(controlnet_mask_frame_path).convert("RGB")).astype('uint8')
+
+    cn_args = {
+        "enabled": True,
+        "module": controlnet_args.controlnet_module,
+        "model": controlnet_args.controlnet_model,
+        "weight": controlnet_args.controlnet_weight,
+        "input_image": {'image': cn_image_np, 'mask': cn_mask_np},
+        "scribble_mode": controlnet_args.controlnet_scribble_mode,
+        "resize_mode": controlnet_args.controlnet_resize_mode,
+        "rgbbgr_mode": controlnet_args.controlnet_rgbbgr_mode,
+        "lowvram": controlnet_args.controlnet_lowvram,
+        "processor_res": controlnet_args.controlnet_processor_res,
+        "threshold_a": controlnet_args.controlnet_threshold_a,
+        "threshold_b": controlnet_args.controlnet_threshold_b,
+        "guidance_strength": controlnet_args.controlnet_guidance_strength,
+    }
+
+    from .deforum_controlnet_hardcode import process
+    p.script_args = (
+        cn_args["enabled"],
+        cn_args["module"],
+        cn_args["model"],
+        cn_args["weight"],
+        cn_args["input_image"],
+        cn_args["scribble_mode"],
+        cn_args["resize_mode"],
+        cn_args["rgbbgr_mode"],
+        cn_args["lowvram"],
+        cn_args["processor_res"],
+        cn_args["threshold_a"],
+        cn_args["threshold_b"],
+        cn_args["guidance_strength"],
+    )
+
+    table = Table(title="ControlNet params",padding=0, box=box.ROUNDED)
+
+    field_names = []
+    field_names += ["module", "model", "weight", "guidance", "scribble", "resize", "rgb->bgr", "proc res", "thr a", "thr b"]
+    for field_name in field_names:
+        table.add_column(field_name, justify="center")
+    
+    rows = []
+    rows += [cn_args["module"], cn_args["model"], cn_args["weight"], cn_args["guidance_strength"], cn_args["scribble_mode"], cn_args["resize_mode"], cn_args["rgbbgr_mode"], cn_args["processor_res"], cn_args["threshold_a"], cn_args["threshold_b"]]
+    rows = [str(x) for x in rows]
+
+    table.add_row(*rows)
+    
+    console.print(table)
+
+    processed = process(p, *(p.script_args))
+
+    if processed is None: # the script just swaps the pipeline, so failing is OK for the first time
+        processed = process_images(p)
+    
+    if processed is None: # now it's definitely not OK
+        raise Exception("\033[31mFailed to process a frame with ControlNet enabled!\033[0m")
+    
+    p.close()
+
+    return processed
+
+import pathlib
+from .video_audio_utilities import vid2frames
+
+def unpack_controlnet_vids(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, animation_prompts, root):
+    if controlnet_args.controlnet_input_video_chosen_file is not None and len(controlnet_args.controlnet_input_video_chosen_file.name) > 0:
+        print(f'Unpacking ControlNet base video')
+        # create a folder for the video input frames to live in
+        mask_in_frame_path = os.path.join(args.outdir, 'controlnet_inputframes') 
+        os.makedirs(mask_in_frame_path, exist_ok=True)
+
+        # save the video frames from mask video
+        print(f"Exporting Video Frames (1 every {anim_args.extract_nth_frame}) frames to {mask_in_frame_path}...")
+        vid2frames(video_path=controlnet_args.controlnet_input_video_chosen_file.name, video_in_frame_path=mask_in_frame_path, n=anim_args.extract_nth_frame, overwrite=anim_args.overwrite_extracted_frames, extract_from_frame=anim_args.extract_from_frame, extract_to_frame=anim_args.extract_to_frame, numeric_files_output=True)
+
+        print(f"Loading {anim_args.max_frames} input frames from {mask_in_frame_path} and saving video frames to {args.outdir}")
+        print(f'ControlNet base video unpacked!')
+    
+    if controlnet_args.controlnet_input_video_mask_chosen_file is not None and len(controlnet_args.controlnet_input_video_mask_chosen_file.name) > 0:
+        print(f'Unpacking ControlNet video mask')
+        # create a folder for the video input frames to live in
+        mask_in_frame_path = os.path.join(args.outdir, 'controlnet_maskframes') 
+        os.makedirs(mask_in_frame_path, exist_ok=True)
+
+        # save the video frames from mask video
+        print(f"Exporting Video Frames (1 every {anim_args.extract_nth_frame}) frames to {mask_in_frame_path}...")
+        vid2frames(video_path=controlnet_args.controlnet_input_video_mask_chosen_file.name, video_in_frame_path=mask_in_frame_path, n=anim_args.extract_nth_frame, overwrite=anim_args.overwrite_extracted_frames, extract_from_frame=anim_args.extract_from_frame, extract_to_frame=anim_args.extract_to_frame, numeric_files_output=True)
+
+        print(f"Loading {anim_args.max_frames} input frames from {mask_in_frame_path} and saving video frames to {args.outdir}")
+        print(f'ControlNet video mask unpacked!')
+
+def hide_ui_by_cn_status(choice):
+    return gr.update(visible=True) if choice else gr.update(visible=False)
+    
+def build_sliders(cn_model):
+        if cn_model == "canny":
+            return [
+                gr.update(label="Annotator resolution", value=512, minimum=64, maximum=2048, step=1, interactive=True),
+                gr.update(label="Canny low threshold", minimum=1, maximum=255, value=100, step=1, interactive=True),
+                gr.update(label="Canny high threshold", minimum=1, maximum=255, value=200, step=1, interactive=True),
+                gr.update(visible=True)
+            ]
+        elif cn_model == "mlsd": #Hough
+            return [
+                gr.update(label="Hough Resolution", minimum=64, maximum=2048, value=512, step=1, interactive=True),
+                gr.update(label="Hough value threshold (MLSD)", minimum=0.01, maximum=2.0, value=0.1, step=0.01, interactive=True),
+                gr.update(label="Hough distance threshold (MLSD)", minimum=0.01, maximum=20.0, value=0.1, step=0.01, interactive=True),
+                gr.update(visible=True)
+            ]
+        elif cn_model in ["hed", "fake_scribble"]:
+            return [
+                gr.update(label="HED Resolution", minimum=64, maximum=2048, value=512, step=1, interactive=True),
+                gr.update(label="Threshold A", value=64, minimum=64, maximum=1024, interactive=False),
+                gr.update(label="Threshold B", value=64, minimum=64, maximum=1024, interactive=False),
+                gr.update(visible=True)
+            ]
+        elif cn_model in ["openpose", "openpose_hand", "segmentation"]:
+            return [
+                gr.update(label="Annotator Resolution", minimum=64, maximum=2048, value=512, step=1, interactive=True),
+                gr.update(label="Threshold A", value=64, minimum=64, maximum=1024, interactive=False),
+                gr.update(label="Threshold B", value=64, minimum=64, maximum=1024, interactive=False),
+                gr.update(visible=True)
+            ]
+        elif cn_model == "depth":
+            return [
+                gr.update(label="Midas Resolution", minimum=64, maximum=2048, value=384, step=1, interactive=True),
+                gr.update(label="Threshold A", value=64, minimum=64, maximum=1024, interactive=False),
+                gr.update(label="Threshold B", value=64, minimum=64, maximum=1024, interactive=False),
+                gr.update(visible=True)
+            ]
+        elif cn_model == "depth_leres":
+            return [
+                gr.update(label="LeReS Resolution", minimum=64, maximum=2048, value=512, step=1, interactive=True),
+                gr.update(label="Remove Near %", value=0, minimum=0, maximum=100, step=0.1, interactive=True),
+                gr.update(label="Remove Background %", value=0, minimum=0, maximum=100, step=0.1, interactive=True),
+                gr.update(visible=True)
+            ]
+        elif cn_model == "normal_map":
+            return [
+                gr.update(label="Normal Resolution", minimum=64, maximum=2048, value=512, step=1, interactive=True),
+                gr.update(label="Normal background threshold", minimum=0.0, maximum=1.0, value=0.4, step=0.01, interactive=True),
+                gr.update(label="Threshold B", value=64, minimum=64, maximum=1024, interactive=False),
+                gr.update(visible=True)
+            ]
+        elif cn_model == "none":
+            return [
+                gr.update(label="Normal Resolution", value=64, minimum=64, maximum=2048, interactive=False),
+                gr.update(label="Threshold A", value=64, minimum=64, maximum=1024, interactive=False),
+                gr.update(label="Threshold B", value=64, minimum=64, maximum=1024, interactive=False),
+                gr.update(visible=False)
+            ]
+        else:
+            return [
+                gr.update(label="Annotator resolution", value=512, minimum=64, maximum=2048, step=1, interactive=True),
+                gr.update(label="Threshold A", value=64, minimum=64, maximum=1024, interactive=False),
+                gr.update(label="Threshold B", value=64, minimum=64, maximum=1024, interactive=False),
+                gr.update(visible=True)
+            ]
+
+    # def svgPreprocess(inputs):
+    #     if (inputs):
+    #         if (inputs['image'].startswith("data:image/svg+xml;base64,") and svgsupport):
+    #             svg_data = base64.b64decode(inputs['image'].replace('data:image/svg+xml;base64,',''))
+    #             drawing = svg2rlg(io.BytesIO(svg_data))
+    #             png_data = renderPM.drawToString(drawing, fmt='PNG')
+    #             encoded_string = base64.b64encode(png_data)
+    #             base64_str = str(encoded_string, "utf-8")
+    #             base64_str = "data:image/png;base64,"+ base64_str
+    #             inputs['image'] = base64_str
+    #         return input_image.orgpreprocess(inputs)
+    #     return None

extensions/deforum/scripts/deforum_helpers/deforum_controlnet_hardcode.py

@@ -0,0 +1,193 @@
+# TODO HACK FIXME HARDCODE — as using the scripts doesn't seem to work for some reason
+deforum_latest_network = None
+deforum_latest_params = (None, 'placeholder to trigger the model loading')
+deforum_input_image = None
+from scripts.processor import unload_hed, unload_mlsd, unload_midas, unload_leres, unload_pidinet, unload_openpose, unload_uniformer, HWC3
+import modules.shared as shared
+import modules.devices as devices
+import modules.processing as processing
+from modules.processing import StableDiffusionProcessingImg2Img, StableDiffusionProcessingTxt2Img
+import numpy as np
+from scripts.controlnet import update_cn_models, cn_models, cn_models_names
+import os
+import modules.scripts as scrpts
+import torch
+from scripts.cldm import PlugableControlModel
+from scripts.adapter import PlugableAdapter
+from scripts.utils import load_state_dict
+from torchvision.transforms import Resize, InterpolationMode, CenterCrop, Compose
+from einops import rearrange
+cn_models_dir = os.path.join(scrpts.basedir(), "models")
+default_conf_adapter = os.path.join(cn_models_dir, "sketch_adapter_v14.yaml")
+default_conf = os.path.join(cn_models_dir, "cldm_v15.yaml")
+unloadable = {
+    "hed": unload_hed,
+    "fake_scribble": unload_hed,
+    "mlsd": unload_mlsd,
+    "depth": unload_midas,
+    "depth_leres": unload_leres,
+    "normal_map": unload_midas,
+    "pidinet": unload_pidinet,
+    "openpose": unload_openpose,
+    "openpose_hand": unload_openpose,
+    "segmentation": unload_uniformer,
+}
+deforum_latest_model_hash = ""
+
+def restore_networks(unet):
+    global deforum_latest_network
+    global deforum_latest_params
+    if deforum_latest_network is not None:
+        print("restoring last networks")
+        deforum_input_image = None
+        deforum_latest_network.restore(unet)
+        deforum_latest_network = None
+
+    last_module = deforum_latest_params[0]
+    if last_module is not None:
+        unloadable.get(last_module, lambda:None)()
+
+def process(p, *args):
+
+    global deforum_latest_network
+    global deforum_latest_params
+    global deforum_input_image
+    global deforum_latest_model_hash
+    
+    unet = p.sd_model.model.diffusion_model
+
+    enabled, module, model, weight, image, scribble_mode, \
+        resize_mode, rgbbgr_mode, lowvram, pres, pthr_a, pthr_b, guidance_strength = args
+
+    if not enabled:
+        restore_networks(unet)
+        return
+
+    models_changed = deforum_latest_params[1] != model \
+        or deforum_latest_model_hash != p.sd_model.sd_model_hash or deforum_latest_network == None \
+        or (deforum_latest_network is not None and deforum_latest_network.lowvram != lowvram)
+
+    deforum_latest_params = (module, model)
+    deforum_latest_model_hash = p.sd_model.sd_model_hash
+    if models_changed:
+        restore_networks(unet)
+        model_path = cn_models.get(model, None)
+
+        if model_path is None:
+            raise RuntimeError(f"model not found: {model}")
+
+        # trim '"' at start/end
+        if model_path.startswith("\"") and model_path.endswith("\""):
+            model_path = model_path[1:-1]
+
+        if not os.path.exists(model_path):
+            raise ValueError(f"file not found: {model_path}")
+
+        print(f"Loading preprocessor: {module}, model: {model}")
+        state_dict = load_state_dict(model_path)
+        network_module = PlugableControlModel
+        network_config = shared.opts.data.get("control_net_model_config", default_conf)
+        if any([k.startswith("body.") for k, v in state_dict.items()]):
+            # adapter model     
+            network_module = PlugableAdapter
+            network_config = shared.opts.data.get("control_net_model_adapter_config", default_conf_adapter)
+
+        network = network_module(
+            state_dict=state_dict, 
+            config_path=network_config, 
+            weight=weight, 
+            lowvram=lowvram,
+            base_model=unet,
+        )
+        network.to(p.sd_model.device, dtype=p.sd_model.dtype)
+        network.hook(unet, p.sd_model)
+
+        print(f"ControlNet model {model} loaded.")
+        deforum_latest_network = network
+        
+    if image is not None:
+        deforum_input_image = HWC3(image['image'])
+        if 'mask' in image and image['mask'] is not None and not ((image['mask'][:, :, 0]==0).all() or (image['mask'][:, :, 0]==255).all()):
+            print("using mask as input")
+            deforum_input_image = HWC3(image['mask'][:, :, 0])
+            scribble_mode = True
+    else:
+        # use img2img init_image as default
+        deforum_input_image = getattr(p, "init_images", [None])[0]
+        if deforum_input_image is None:
+            raise ValueError('controlnet is enabled but no input image is given')
+        deforum_input_image = HWC3(np.asarray(deforum_input_image))
+            
+    if scribble_mode:
+        detected_map = np.zeros_like(deforum_input_image, dtype=np.uint8)
+        detected_map[np.min(deforum_input_image, axis=2) < 127] = 255
+        deforum_input_image = detected_map
+    
+    from scripts.processor import canny, midas, midas_normal, leres, hed, mlsd, openpose, pidinet, simple_scribble, fake_scribble, uniformer
+    
+    preprocessor = {
+        "none": lambda x, *args, **kwargs: x,
+        "canny": canny,
+        "depth": midas,
+        "depth_leres": leres,
+        "hed": hed,
+        "mlsd": mlsd,
+        "normal_map": midas_normal,
+        "openpose": openpose,
+        # "openpose_hand": openpose_hand,
+        "pidinet": pidinet,
+        "scribble": simple_scribble,
+        "fake_scribble": fake_scribble,
+        "segmentation": uniformer,
+    }
+            
+    preprocessor = preprocessor[deforum_latest_params[0]]
+    h, w, bsz = p.height, p.width, p.batch_size
+    if pres > 64:
+        detected_map = preprocessor(deforum_input_image, res=pres, thr_a=pthr_a, thr_b=pthr_b)
+    else:
+        detected_map = preprocessor(deforum_input_image)
+    detected_map = HWC3(detected_map)
+    
+    if module == "normal_map" or rgbbgr_mode:
+        control = torch.from_numpy(detected_map[:, :, ::-1].copy()).float().to(devices.get_device_for("controlnet")) / 255.0
+    else:
+        control = torch.from_numpy(detected_map.copy()).float().to(devices.get_device_for("controlnet")) / 255.0
+    
+    control = rearrange(control, 'h w c -> c h w')
+    detected_map = rearrange(torch.from_numpy(detected_map), 'h w c -> c h w')
+    if resize_mode == "Scale to Fit (Inner Fit)":
+        transform = Compose([
+            Resize(h if h<w else w, interpolation=InterpolationMode.BICUBIC),
+            CenterCrop(size=(h, w))
+        ]) 
+        control = transform(control)
+        detected_map = transform(detected_map)
+    elif resize_mode == "Envelope (Outer Fit)":
+        transform = Compose([
+            Resize(h if h>w else w, interpolation=InterpolationMode.BICUBIC),
+            CenterCrop(size=(h, w))
+        ]) 
+        control = transform(control)
+        detected_map = transform(detected_map)
+    else:
+        control = Resize((h,w), interpolation=InterpolationMode.BICUBIC)(control)
+        detected_map = Resize((h,w), interpolation=InterpolationMode.BICUBIC)(detected_map)
+        
+    # for log use
+    detected_map = rearrange(detected_map, 'c h w -> h w c').numpy().astype(np.uint8)
+        
+    # control = torch.stack([control for _ in range(bsz)], dim=0)
+    deforum_latest_network.notify(control, weight, guidance_strength)
+
+    if shared.opts.data.get("control_net_skip_img2img_processing") and hasattr(p, "init_images"):
+        swap_img2img_pipeline(p)
+
+def swap_img2img_pipeline(p: processing.StableDiffusionProcessingImg2Img):
+    p.__class__ = processing.StableDiffusionProcessingTxt2Img
+    dummy = processing.StableDiffusionProcessingTxt2Img()
+    for k,v in dummy.__dict__.items():
+        if hasattr(p, k):
+            continue
+        setattr(p, k, v)
+

extensions/deforum/scripts/deforum_helpers/deprecation_utils.py

@@ -0,0 +1,20 @@
+# This file is used to map deprecated setting names in a dictionary
+# and print a message containing the old and the new names
+# if the latter is removed completely, put a warning
+
+# as of 2023-02-05
+# "histogram_matching" -> None
+
+deprecation_map = {
+    "histogram_matching": None,
+    "flip_2d_perspective": "enable_perspective_flip"
+}
+
+def handle_deprecated_settings(settings_json):
+    for old_name, new_name in deprecation_map.items():
+        if old_name in settings_json:
+            if new_name is None:
+                print(f"WARNING: Setting '{old_name}' has been removed. It will be discarded and the default value used instead!")
+            else:
+                print(f"WARNING: Setting '{old_name}' has been renamed to '{new_name}'. The saved settings file will reflect the change")
+                settings_json[new_name] = settings_json.pop(old_name)

extensions/deforum/scripts/deforum_helpers/depth.py

@@ -0,0 +1,166 @@
+import math, os, subprocess
+import cv2
+import hashlib
+import numpy as np
+import torch
+import gc
+import torchvision.transforms as T
+from einops import rearrange, repeat
+from PIL import Image
+from infer import InferenceHelper
+from midas.dpt_depth import DPTDepthModel
+from midas.transforms import Resize, NormalizeImage, PrepareForNet
+import torchvision.transforms.functional as TF
+from .general_utils import checksum
+
+class DepthModel():
+    def __init__(self, device):
+        self.adabins_helper = None
+        self.depth_min = 1000
+        self.depth_max = -1000
+        self.device = device
+        self.midas_model = None
+        self.midas_transform = None
+    
+    def load_adabins(self, models_path):
+        if not os.path.exists(os.path.join(models_path,'AdaBins_nyu.pt')):
+            from basicsr.utils.download_util import load_file_from_url
+            load_file_from_url(r"https://cloudflare-ipfs.com/ipfs/Qmd2mMnDLWePKmgfS8m6ntAg4nhV5VkUyAydYBp8cWWeB7/AdaBins_nyu.pt", models_path)
+            if checksum(os.path.join(models_path,'AdaBins_nyu.pt')) != "643db9785c663aca72f66739427642726b03acc6c4c1d3755a4587aa2239962746410d63722d87b49fc73581dbc98ed8e3f7e996ff7b9c0d56d0fbc98e23e41a":
+                raise Exception(r"Error while downloading AdaBins_nyu.pt. Please download from here: https://drive.google.com/file/d/1lvyZZbC9NLcS8a__YPcUP7rDiIpbRpoF and place in: " + models_path)
+        self.adabins_helper = InferenceHelper(models_path=models_path, dataset='nyu', device=self.device)
+
+    def load_midas(self, models_path, half_precision=True):
+        if not os.path.exists(os.path.join(models_path, 'dpt_large-midas-2f21e586.pt')):
+            from basicsr.utils.download_util import load_file_from_url
+            load_file_from_url(r"https://github.com/intel-isl/DPT/releases/download/1_0/dpt_large-midas-2f21e586.pt", models_path)
+            if checksum(os.path.join(models_path,'dpt_large-midas-2f21e586.pt')) != "fcc4829e65d00eeed0a38e9001770676535d2e95c8a16965223aba094936e1316d569563552a852d471f310f83f597e8a238987a26a950d667815e08adaebc06":
+                raise Exception(r"Error while downloading dpt_large-midas-2f21e586.pt. Please download from here: https://github.com/intel-isl/DPT/releases/download/1_0/dpt_large-midas-2f21e586.pt and place in: " + models_path)
+
+        self.midas_model = DPTDepthModel(
+            path=f"{models_path}/dpt_large-midas-2f21e586.pt",
+            backbone="vitl16_384",
+            non_negative=True,
+        )
+        normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
+
+        self.midas_transform = T.Compose([
+            Resize(
+                384, 384,
+                resize_target=None,
+                keep_aspect_ratio=True,
+                ensure_multiple_of=32,
+                resize_method="minimal",
+                image_interpolation_method=cv2.INTER_CUBIC,
+            ),
+            normalization,
+            PrepareForNet()
+        ])
+
+        self.midas_model.eval()    
+        if self.device == torch.device("cuda"):
+            self.midas_model = self.midas_model.to(memory_format=torch.channels_last)
+        if half_precision:
+            self.midas_model = self.midas_model.half()
+        self.midas_model.to(self.device)
+
+    def predict(self, prev_img_cv2, anim_args, half_precision) -> torch.Tensor:
+        w, h = prev_img_cv2.shape[1], prev_img_cv2.shape[0]
+
+        # predict depth with AdaBins    
+        use_adabins = anim_args.midas_weight < 1.0 and self.adabins_helper is not None
+        if use_adabins:
+            MAX_ADABINS_AREA = 500000
+            MIN_ADABINS_AREA = 448*448
+
+            # resize image if too large or too small
+            img_pil = Image.fromarray(cv2.cvtColor(prev_img_cv2.astype(np.uint8), cv2.COLOR_RGB2BGR))
+            image_pil_area = w*h
+            resized = True
+            if image_pil_area > MAX_ADABINS_AREA:
+                scale = math.sqrt(MAX_ADABINS_AREA) / math.sqrt(image_pil_area)
+                depth_input = img_pil.resize((int(w*scale), int(h*scale)), Image.LANCZOS) # LANCZOS is good for downsampling
+                print(f"  resized to {depth_input.width}x{depth_input.height}")
+            elif image_pil_area < MIN_ADABINS_AREA:
+                scale = math.sqrt(MIN_ADABINS_AREA) / math.sqrt(image_pil_area)
+                depth_input = img_pil.resize((int(w*scale), int(h*scale)), Image.BICUBIC)
+                print(f"  resized to {depth_input.width}x{depth_input.height}")
+            else:
+                depth_input = img_pil
+                resized = False
+
+            # predict depth and resize back to original dimensions
+            try:
+                with torch.no_grad():
+                    _, adabins_depth = self.adabins_helper.predict_pil(depth_input)
+                if resized:
+                    adabins_depth = TF.resize(
+                        torch.from_numpy(adabins_depth), 
+                        torch.Size([h, w]),
+                        interpolation=TF.InterpolationMode.BICUBIC
+                    )
+                    adabins_depth = adabins_depth.cpu().numpy()
+                adabins_depth = adabins_depth.squeeze()
+            except:
+                print(f"  exception encountered, falling back to pure MiDaS")
+                use_adabins = False
+            torch.cuda.empty_cache()
+
+        if self.midas_model is not None:
+            # convert image from 0->255 uint8 to 0->1 float for feeding to MiDaS
+            img_midas = prev_img_cv2.astype(np.float32) / 255.0
+            img_midas_input = self.midas_transform({"image": img_midas})["image"]
+
+            # MiDaS depth estimation implementation
+            sample = torch.from_numpy(img_midas_input).float().to(self.device).unsqueeze(0)
+            if self.device == torch.device("cuda"):
+                sample = sample.to(memory_format=torch.channels_last)  
+                if half_precision:
+                    sample = sample.half()
+            with torch.no_grad():            
+                midas_depth = self.midas_model.forward(sample)
+            midas_depth = torch.nn.functional.interpolate(
+                midas_depth.unsqueeze(1),
+                size=img_midas.shape[:2],
+                mode="bicubic",
+                align_corners=False,
+            ).squeeze()
+            midas_depth = midas_depth.cpu().numpy()
+            torch.cuda.empty_cache()
+
+            # MiDaS makes the near values greater, and the far values lesser. Let's reverse that and try to align with AdaBins a bit better.
+            midas_depth = np.subtract(50.0, midas_depth)
+            midas_depth = midas_depth / 19.0
+
+            # blend between MiDaS and AdaBins predictions
+            if use_adabins:
+                depth_map = midas_depth*anim_args.midas_weight + adabins_depth*(1.0-anim_args.midas_weight)
+            else:
+                depth_map = midas_depth
+
+            depth_map = np.expand_dims(depth_map, axis=0)
+            depth_tensor = torch.from_numpy(depth_map).squeeze().to(self.device)
+        else:
+            depth_tensor = torch.ones((h, w), device=self.device)
+        
+        return depth_tensor
+
+    def save(self, filename: str, depth: torch.Tensor):
+        depth = depth.cpu().numpy()
+        if len(depth.shape) == 2:
+            depth = np.expand_dims(depth, axis=0)
+        self.depth_min = min(self.depth_min, depth.min())
+        self.depth_max = max(self.depth_max, depth.max())
+        print(f"  depth min:{depth.min()} max:{depth.max()}")
+        denom = max(1e-8, self.depth_max - self.depth_min)
+        temp = rearrange((depth - self.depth_min) / denom * 255, 'c h w -> h w c')
+        temp = repeat(temp, 'h w 1 -> h w c', c=3)
+        Image.fromarray(temp.astype(np.uint8)).save(filename)
+    
+    def to(self, device):
+        self.device = device
+        self.midas_model.to(device)
+        if self.adabins_helper is not None:
+            self.adabins_helper.to(device)
+        gc.collect()
+        torch.cuda.empty_cache()

extensions/deforum/scripts/deforum_helpers/frame_interpolation.py

@@ -0,0 +1,192 @@
+import os
+from pathlib import Path
+from rife.inference_video import run_rife_new_video_infer
+from .video_audio_utilities import get_quick_vid_info, vid2frames, media_file_has_audio, extract_number, ffmpeg_stitch_video
+from film_interpolation.film_inference import run_film_interp_infer
+from .general_utils import duplicate_pngs_from_folder, checksum
+
+# gets 'RIFE v4.3', returns: 'RIFE43'   
+def extract_rife_name(string):
+    parts = string.split()
+    if len(parts) != 2 or parts[0] != "RIFE" or (parts[1][0] != "v" or not parts[1][1:].replace('.','').isdigit()):
+        raise ValueError("Input string should contain exactly 2 words, first word should be 'RIFE' and second word should start with 'v' followed by 2 numbers")
+    return "RIFE"+parts[1][1:].replace('.','')
+
+# This function usually gets a filename, and converts it to a legal linux/windows *folder* name
+def clean_folder_name(string):
+    illegal_chars = ["/", "\\", "<", ">", ":", "\"", "|", "?", "*", "."]
+    for char in illegal_chars:
+        string = string.replace(char, "_")
+    return string
+
+def set_interp_out_fps(interp_x, slow_x_enabled, slom_x, in_vid_fps):
+    if interp_x == 'Disabled' or in_vid_fps in ('---', None, '', 'None'):
+        return '---'
+
+    # clean_interp_x = extract_number(interp_x)
+    # clean_slom_x = extract_number(slom_x)
+    fps = float(in_vid_fps) * int(interp_x)
+    # if slom_x != -1:
+    if slow_x_enabled:
+        fps /= int(slom_x)
+    return int(fps) if fps.is_integer() else fps
+    
+# get uploaded video frame count, fps, and return 3 valuees for the gradio UI: in fcount, in fps, out fps (using the set_interp_out_fps function above)
+def gradio_f_interp_get_fps_and_fcount(vid_path, interp_x, slow_x_enabled, slom_x):
+    if vid_path is None:
+        return '---', '---', '---'
+    fps, fcount, resolution = get_quick_vid_info(vid_path.name)
+    expected_out_fps = set_interp_out_fps(interp_x, slow_x_enabled, slom_x, fps)
+    return (str(round(fps,2)) if fps is not None else '---', (round(fcount,2)) if fcount is not None else '---', round(expected_out_fps,2))
+
+# handle call to interpolate an uploaded video from gradio button in args.py (the function that calls this func is named 'upload_vid_to_rife')
+def process_interp_vid_upload_logic(file, engine, x_am, sl_enabled, sl_am, keep_imgs, f_location, f_crf, f_preset, in_vid_fps, f_models_path, vid_file_name):
+
+    print("got a request to *frame interpolate* an existing video.")
+
+    _, _, resolution = get_quick_vid_info(file.name)
+    folder_name = clean_folder_name(Path(vid_file_name).stem)
+    outdir_no_tmp = os.path.join(os.getcwd(), 'outputs', 'frame-interpolation', folder_name)
+    i = 1
+    while os.path.exists(outdir_no_tmp):
+        outdir_no_tmp = os.path.join(os.getcwd(), 'outputs', 'frame-interpolation', folder_name + '_' + str(i))
+        i += 1
+
+    outdir = os.path.join(outdir_no_tmp, 'tmp_input_frames')
+    os.makedirs(outdir, exist_ok=True)
+    
+    vid2frames(video_path=file.name, video_in_frame_path=outdir, overwrite=True, extract_from_frame=0, extract_to_frame=-1, numeric_files_output=True, out_img_format='png')
+    
+    # check if the uploaded vid has an audio stream. If it doesn't, set audio param to None so that ffmpeg won't try to add non-existing audio to final video.
+    audio_file_to_pass = None
+    if media_file_has_audio(file.name, f_location):
+        audio_file_to_pass = file.name
+    
+    process_video_interpolation(frame_interpolation_engine=engine, frame_interpolation_x_amount=x_am, frame_interpolation_slow_mo_enabled = sl_enabled,frame_interpolation_slow_mo_amount=sl_am, orig_vid_fps=in_vid_fps, deforum_models_path=f_models_path, real_audio_track=audio_file_to_pass, raw_output_imgs_path=outdir, img_batch_id=None, ffmpeg_location=f_location, ffmpeg_crf=f_crf, ffmpeg_preset=f_preset, keep_interp_imgs=keep_imgs, orig_vid_name=folder_name, resolution=resolution)
+
+# handle params before talking with the actual interpolation module (rifee/film, more to be added)
+def process_video_interpolation(frame_interpolation_engine, frame_interpolation_x_amount, frame_interpolation_slow_mo_enabled, frame_interpolation_slow_mo_amount, orig_vid_fps, deforum_models_path, real_audio_track, raw_output_imgs_path, img_batch_id, ffmpeg_location, ffmpeg_crf, ffmpeg_preset, keep_interp_imgs, orig_vid_name, resolution):
+
+     # set initial output vid fps
+    fps = float(orig_vid_fps) * frame_interpolation_x_amount
+    
+    # re-calculate fps param to pass if slow_mo mode is enabled
+    if frame_interpolation_slow_mo_enabled:
+        fps = float(orig_vid_fps) * frame_interpolation_x_amount / int(frame_interpolation_slow_mo_amount)
+    # disable audio-adding by setting real_audio_track to None if slow-mo is enabled
+    if real_audio_track is not None and frame_interpolation_slow_mo_enabled:
+        real_audio_track = None  
+            
+    if frame_interpolation_engine == 'None':
+        return
+    elif frame_interpolation_engine.startswith("RIFE"):
+        # make sure interp_x is valid and in range
+        if frame_interpolation_x_amount not in range(2, 11):
+            raise Error("frame_interpolation_x_amount must be between 2x and 10x")
+  
+        # set UHD to True if res' is 2K or higher
+        if resolution:
+            UHD = resolution[0] >= 2048 and resolution[1] >= 2048
+        else:
+            UHD = False
+        # e.g from "RIFE v2.3 to RIFE23"
+        actual_model_folder_name = extract_rife_name(frame_interpolation_engine)
+        
+        # run actual rife interpolation and video stitching etc - the whole suite
+        run_rife_new_video_infer(interp_x_amount=frame_interpolation_x_amount, slow_mo_enabled = frame_interpolation_slow_mo_enabled, slow_mo_x_amount=frame_interpolation_slow_mo_amount, model=actual_model_folder_name, fps=fps, deforum_models_path=deforum_models_path, audio_track=real_audio_track, raw_output_imgs_path=raw_output_imgs_path, img_batch_id=img_batch_id, ffmpeg_location=ffmpeg_location, ffmpeg_crf=ffmpeg_crf, ffmpeg_preset=ffmpeg_preset, keep_imgs=keep_interp_imgs, orig_vid_name=orig_vid_name, UHD=UHD)
+    elif frame_interpolation_engine == 'FILM':
+        prepare_film_inference(deforum_models_path=deforum_models_path, x_am=frame_interpolation_x_amount, sl_enabled=frame_interpolation_slow_mo_enabled, sl_am=frame_interpolation_slow_mo_amount, keep_imgs=keep_interp_imgs, raw_output_imgs_path=raw_output_imgs_path, img_batch_id=img_batch_id, f_location=ffmpeg_location, f_crf=ffmpeg_crf, f_preset=ffmpeg_preset, fps=fps, audio_track=real_audio_track, orig_vid_name=orig_vid_name)
+    else:
+        print("Unknown Frame Interpolation engine chosen. Doing nothing.")
+        return
+        
+def prepare_film_inference(deforum_models_path, x_am, sl_enabled, sl_am, keep_imgs, raw_output_imgs_path, img_batch_id, f_location, f_crf, f_preset, fps, audio_track, orig_vid_name):
+    import shutil 
+    
+    parent_folder = os.path.dirname(raw_output_imgs_path)
+    grandparent_folder = os.path.dirname(parent_folder)
+    if orig_vid_name is not None:
+        interp_vid_path = os.path.join(parent_folder, str(orig_vid_name) +'_FILM_x' + str(x_am))
+    else:
+        interp_vid_path = os.path.join(raw_output_imgs_path, str(img_batch_id) +'_FILM_x' + str(x_am))
+    
+    film_model_name = 'film_net_fp16.pt'
+    film_model_folder = os.path.join(deforum_models_path,'film_interpolation')
+    film_model_path = os.path.join(film_model_folder, film_model_name) # actual full path to the film .pt model file
+    output_interp_imgs_folder = os.path.join(raw_output_imgs_path, 'interpolated_frames_film')
+    # set custom name depending on if we interpolate after a run, or interpolate a video (related/unrelated to deforum, we don't know) directly from within the interpolation tab
+    # interpolated_path = os.path.join(args.raw_output_imgs_path, 'interpolated_frames_rife')
+    if orig_vid_name is not None: # interpolating a video (deforum or unrelated)
+        custom_interp_path = "{}_{}".format(output_interp_imgs_folder, orig_vid_name)
+    else: # interpolating after a deforum run:
+        custom_interp_path = "{}_{}".format(output_interp_imgs_folder, img_batch_id)
+
+    # interp_vid_path = os.path.join(raw_output_imgs_path, str(img_batch_id) + '_FILM_x' + str(x_am))
+    img_path_for_ffmpeg = os.path.join(custom_interp_path, "frame_%05d.png")
+
+    if sl_enabled:
+        interp_vid_path = interp_vid_path + '_slomo_x' + str(sl_am)
+    interp_vid_path = interp_vid_path + '.mp4'
+
+    # In this folder we temporarily keep the original frames (converted/ copy-pasted and img format depends on scenario)
+    # the convertion case is done to avert a problem with 24 and 32 mixed outputs from the same animation run
+    temp_convert_raw_png_path = os.path.join(raw_output_imgs_path, "tmp_film_folder")
+    total_frames = duplicate_pngs_from_folder(raw_output_imgs_path, temp_convert_raw_png_path, img_batch_id, None)
+    check_and_download_film_model('film_net_fp16.pt', film_model_folder) # TODO: split this part
+    
+    # get number of in-between-frames to provide to FILM - mimics how RIFE works, we should get the same amount of total frames in the end
+    film_in_between_frames_count = calculate_frames_to_add(total_frames, x_am)
+    # Run actual FILM inference
+    run_film_interp_infer(
+    model_path = film_model_path,
+    input_folder = temp_convert_raw_png_path,
+    save_folder = custom_interp_path, # output folder is created in the infer part
+    inter_frames = film_in_between_frames_count)
+
+    add_soundtrack = 'None'
+    if not audio_track is None:
+        add_soundtrack = 'File'
+
+    print (f"*Passing interpolated frames to ffmpeg...*")
+    exception_raised = False
+    try:
+        ffmpeg_stitch_video(ffmpeg_location=f_location, fps=fps, outmp4_path=interp_vid_path, stitch_from_frame=0, stitch_to_frame=999999, imgs_path=img_path_for_ffmpeg, add_soundtrack=add_soundtrack, audio_path=audio_track, crf=f_crf, preset=f_preset)
+    except Exception as e:
+        exception_raised = True
+        print(f"An error occurred while stitching the video: {e}")
+
+    if orig_vid_name and (keep_imgs or exception_raised):
+        shutil.move(custom_interp_path, parent_folder) 
+    if not keep_imgs and not exception_raised:
+        if fps <= 450: # keep interp frames automatically if out_vid fps is above 450
+            shutil.rmtree(custom_interp_path, ignore_errors=True)
+    # delete duplicated raw non-interpolated frames
+    shutil.rmtree(temp_convert_raw_png_path, ignore_errors=True)
+    # remove folder with raw (non-interpolated) vid input frames in case of input VID and not PNGs
+    if orig_vid_name:
+        shutil.rmtree(raw_output_imgs_path, ignore_errors=True)
+
+def check_and_download_film_model(model_name, model_dest_folder):
+    from basicsr.utils.download_util import load_file_from_url
+    if model_name == 'film_net_fp16.pt':
+        model_dest_path = os.path.join(model_dest_folder, model_name)
+        download_url = 'https://github.com/hithereai/frame-interpolation-pytorch/releases/download/film_net_fp16.pt/film_net_fp16.pt'
+        film_model_hash = '0a823815b111488ac2b7dd7fe6acdd25d35a22b703e8253587764cf1ee3f8f93676d24154d9536d2ce5bc3b2f102fb36dfe0ca230dfbe289d5cd7bde5a34ec12'
+    else: # Unknown FILM model
+        raise Exception("Got a request to download an unknown FILM model. Can't proceed.")
+    if os.path.exists(model_dest_path):
+        return
+    try:
+        os.makedirs(model_dest_folder, exist_ok=True)
+        # download film model from url
+        load_file_from_url(download_url, model_dest_folder)
+        # verify checksum
+        if checksum(model_dest_path) != film_model_hash:
+            raise Exception(f"Error while downloading {model_name}. Please download from: {download_url}, and put in: {model_dest_folder}")
+    except Exception as e:
+        raise Exception(f"Error while downloading {model_name}. Please download from: {download_url}, and put in: {model_dest_folder}")
+        
+# get film no. of frames to add after each pic from tot frames in interp_x values
+def calculate_frames_to_add(total_frames, interp_x):
+    frames_to_add = (total_frames * interp_x - total_frames) / (total_frames - 1)
+    return int(round(frames_to_add))

extensions/deforum/scripts/deforum_helpers/general_utils.py

@@ -0,0 +1,32 @@
+import hashlib
+def checksum(filename, hash_factory=hashlib.blake2b, chunk_num_blocks=128):
+    h = hash_factory()
+    with open(filename,'rb') as f: 
+        while chunk := f.read(chunk_num_blocks*h.block_size): 
+            h.update(chunk)
+    return h.hexdigest()
+
+def get_os():
+    import platform
+    return {"Windows": "Windows", "Linux": "Linux", "Darwin": "Mac"}.get(platform.system(), "Unknown")
+
+# used in src/rife/inference_video.py and more, soon
+def duplicate_pngs_from_folder(from_folder, to_folder, img_batch_id, orig_vid_name):
+    import os, cv2, shutil #, subprocess
+    #TODO: don't copy-paste at all if the input is a video (now it copy-pastes, and if input is deforum run is also converts to make sure no errors rise cuz of 24-32 bit depth differences)
+    temp_convert_raw_png_path = os.path.join(from_folder, to_folder)
+    if not os.path.exists(temp_convert_raw_png_path):
+                os.makedirs(temp_convert_raw_png_path)
+                
+    frames_handled = 0
+    for f in os.listdir(from_folder):
+        if ('png' in f or 'jpg' in f) and '-' not in f and '_depth_' not in f and ((img_batch_id is not None and f.startswith(img_batch_id) or img_batch_id is None)):
+            frames_handled +=1
+            original_img_path = os.path.join(from_folder, f)
+            if orig_vid_name is not None:
+                shutil.copy(original_img_path, temp_convert_raw_png_path)
+            else:
+                image = cv2.imread(original_img_path)
+                new_path = os.path.join(temp_convert_raw_png_path, f)
+                cv2.imwrite(new_path, image, [cv2.IMWRITE_PNG_COMPRESSION, 0])
+    return frames_handled

extensions/deforum/scripts/deforum_helpers/generate.py

@@ -0,0 +1,244 @@
+import numpy as np
+import cv2
+from PIL import Image
+from .prompt import split_weighted_subprompts
+from .load_images import load_img, prepare_mask, check_mask_for_errors
+from .webui_sd_pipeline import get_webui_sd_pipeline
+from .animation import sample_from_cv2, sample_to_cv2
+from .rich import console
+#Webui
+import cv2
+from .animation import sample_from_cv2, sample_to_cv2
+from modules import processing, sd_models
+from modules.shared import opts, sd_model
+from modules.processing import process_images, StableDiffusionProcessingTxt2Img
+from .deforum_controlnet import is_controlnet_enabled, process_txt2img_with_controlnet, process_img2img_with_controlnet
+
+import math, json, itertools
+import requests
+
+def load_mask_latent(mask_input, shape):
+    # mask_input (str or PIL Image.Image): Path to the mask image or a PIL Image object
+    # shape (list-like len(4)): shape of the image to match, usually latent_image.shape
+    
+    if isinstance(mask_input, str): # mask input is probably a file name
+        if mask_input.startswith('http://') or mask_input.startswith('https://'):
+            mask_image = Image.open(requests.get(mask_input, stream=True).raw).convert('RGBA')
+        else:
+            mask_image = Image.open(mask_input).convert('RGBA')
+    elif isinstance(mask_input, Image.Image):
+        mask_image = mask_input
+    else:
+        raise Exception("mask_input must be a PIL image or a file name")
+
+    mask_w_h = (shape[-1], shape[-2])
+    mask = mask_image.resize(mask_w_h, resample=Image.LANCZOS)
+    mask = mask.convert("L")
+    return mask
+
+def isJson(myjson):
+    try:
+        json.loads(myjson)
+    except ValueError as e:
+        return False
+    return True
+
+# Add pairwise implementation here not to upgrade
+# the whole python to 3.10 just for one function
+def pairwise_repl(iterable):
+    a, b = itertools.tee(iterable)
+    next(b, None)
+    return zip(a, b)
+
+def generate(args, anim_args, loop_args, controlnet_args, root, frame = 0, return_sample=False, sampler_name=None):
+    assert args.prompt is not None
+    
+    # Setup the pipeline
+    p = get_webui_sd_pipeline(args, root, frame)
+    p.prompt, p.negative_prompt = split_weighted_subprompts(args.prompt, frame)
+    
+    if not args.use_init and args.strength > 0 and args.strength_0_no_init:
+        print("\nNo init image, but strength > 0. Strength has been auto set to 0, since use_init is False.")
+        print("If you want to force strength > 0 with no init, please set strength_0_no_init to False.\n")
+        args.strength = 0
+    processed = None
+    mask_image = None
+    init_image = None
+    image_init0 = None
+
+    if loop_args.use_looper:
+        # TODO find out why we need to set this in the init tab
+        if args.strength == 0:
+            raise RuntimeError("Strength needs to be greater than 0 in Init tab and strength_0_no_init should *not* be checked")
+        if args.seed_behavior != "schedule":
+            raise RuntimeError("seed_behavior needs to be set to schedule in under 'Keyframes' tab --> 'Seed scheduling'")
+        if not isJson(loop_args.imagesToKeyframe):
+            raise RuntimeError("The images set for use with keyframe-guidance are not in a proper JSON format")
+        args.strength = loop_args.imageStrength
+        tweeningFrames = loop_args.tweeningFrameSchedule
+        blendFactor = .07
+        colorCorrectionFactor = loop_args.colorCorrectionFactor
+        jsonImages = json.loads(loop_args.imagesToKeyframe)
+        framesToImageSwapOn = list(map(int, list(jsonImages.keys())))
+        # find which image to show
+        frameToChoose = 0
+        for swappingFrame in framesToImageSwapOn[1:]:
+            frameToChoose += (frame >= int(swappingFrame))
+        
+        #find which frame to do our swapping on for tweening
+        skipFrame = 25
+        for fs, fe in pairwise_repl(framesToImageSwapOn):
+            if fs <= frame <= fe:
+                skipFrame = fe - fs
+
+        if frame % skipFrame <= tweeningFrames: # number of tweening frames
+            blendFactor = loop_args.blendFactorMax - loop_args.blendFactorSlope*math.cos((frame % tweeningFrames) / (tweeningFrames / 2))
+        init_image2, _ = load_img(list(jsonImages.values())[frameToChoose],
+                                shape=(args.W, args.H),
+                                use_alpha_as_mask=args.use_alpha_as_mask)
+        image_init0 = list(jsonImages.values())[0]
+            
+    else: # they passed in a single init image
+        image_init0 = args.init_image
+
+
+    available_samplers = { 
+        'euler a':'Euler a',
+        'euler':'Euler',
+        'lms':'LMS',
+        'heun':'Heun',
+        'dpm2':'DPM2',
+        'dpm2 a':'DPM2 a',
+        'dpm++ 2s a':'DPM++ 2S a',
+        'dpm++ 2m':'DPM++ 2M',
+        'dpm++ sde':'DPM++ SDE',
+        'dpm fast':'DPM fast',
+        'dpm adaptive':'DPM adaptive',
+        'lms karras':'LMS Karras' ,
+        'dpm2 karras':'DPM2 Karras',
+        'dpm2 a karras':'DPM2 a Karras',
+        'dpm++ 2s a karras':'DPM++ 2S a Karras',
+        'dpm++ 2m karras':'DPM++ 2M Karras',
+        'dpm++ sde karras':'DPM++ SDE Karras'
+    }
+    if sampler_name is not None:
+        if sampler_name in available_samplers.keys():
+            args.sampler = available_samplers[sampler_name]
+
+    if args.checkpoint is not None:
+        info = sd_models.get_closet_checkpoint_match(args.checkpoint)
+        if info is None:
+            raise RuntimeError(f"Unknown checkpoint: {args.checkpoint}")
+        sd_models.reload_model_weights(info=info)
+    
+    if args.init_sample is not None:
+        # TODO: cleanup init_sample remains later
+        img = args.init_sample
+        init_image = img
+        image_init0 = img
+        if loop_args.use_looper and isJson(loop_args.imagesToKeyframe):
+            init_image = Image.blend(init_image, init_image2, blendFactor)
+            correction_colors = Image.blend(init_image, init_image2, colorCorrectionFactor)
+            p.color_corrections = [processing.setup_color_correction(correction_colors)]
+
+    # this is the first pass
+    elif loop_args.use_looper or (args.use_init and ((args.init_image != None and args.init_image != ''))):
+        init_image, mask_image = load_img(image_init0, # initial init image
+                                          shape=(args.W, args.H),  
+                                          use_alpha_as_mask=args.use_alpha_as_mask)
+                                          
+    else:
+        
+        if anim_args.animation_mode != 'Interpolation':
+            print(f"Not using an init image (doing pure txt2img)")
+        p_txt = StableDiffusionProcessingTxt2Img(
+                sd_model=sd_model,
+                outpath_samples=root.tmp_deforum_run_duplicated_folder,
+                outpath_grids=root.tmp_deforum_run_duplicated_folder,
+                prompt=p.prompt,
+                styles=p.styles,
+                negative_prompt=p.negative_prompt,
+                seed=p.seed,
+                subseed=p.subseed,
+                subseed_strength=p.subseed_strength,
+                seed_resize_from_h=p.seed_resize_from_h,
+                seed_resize_from_w=p.seed_resize_from_w,
+                sampler_name=p.sampler_name,
+                batch_size=p.batch_size,
+                n_iter=p.n_iter,
+                steps=p.steps,
+                cfg_scale=p.cfg_scale,
+                width=p.width,
+                height=p.height,
+                restore_faces=p.restore_faces,
+                tiling=p.tiling,
+                enable_hr=None,
+                denoising_strength=None,
+            )
+        # print dynamic table to cli
+        print_generate_table(args, anim_args, p_txt)
+
+        if is_controlnet_enabled(controlnet_args):
+            processed = process_txt2img_with_controlnet(p, args, anim_args, loop_args, controlnet_args, root, frame)
+        else:
+            processed = processing.process_images(p_txt)
+
+    if processed is None:
+        # Mask functions
+        if args.use_mask:
+            mask = args.mask_image
+            #assign masking options to pipeline
+            if mask is not None:
+                p.inpainting_mask_invert = args.invert_mask
+                p.inpainting_fill = args.fill 
+                p.inpaint_full_res= args.full_res_mask 
+                p.inpaint_full_res_padding = args.full_res_mask_padding
+        else:
+            mask = None
+
+        assert not ( (mask is not None and args.use_mask and args.overlay_mask) and (args.init_sample is None and init_image is None)), "Need an init image when use_mask == True and overlay_mask == True"
+        
+        p.init_images = [init_image]
+        p.image_mask = mask
+        p.image_cfg_scale = args.pix2pix_img_cfg_scale
+        
+        # print dynamic table to cli
+        print_generate_table(args, anim_args, p)
+       
+        if is_controlnet_enabled(controlnet_args):
+            processed = process_img2img_with_controlnet(p, args, anim_args, loop_args, controlnet_args, root, frame)
+        else:
+            processed = processing.process_images(p)
+    
+    if root.initial_info == None:
+        root.initial_seed = processed.seed
+        root.initial_info = processed.info
+        
+    if root.first_frame == None:
+        root.first_frame = processed.images[0]
+    
+    results = processed.images[0]
+    
+    return results
+
+def print_generate_table(args, anim_args, p):
+    from rich.table import Table
+    from rich import box
+    table = Table(padding=0, box=box.ROUNDED)
+    field_names = ["Steps", "CFG"]
+    if anim_args.animation_mode != 'Interpolation':
+        field_names.append("Denoise")
+    field_names += ["Subseed", "Subs. str"] * (anim_args.enable_subseed_scheduling)
+    field_names += ["Sampler"] * anim_args.enable_sampler_scheduling
+    field_names += ["Checkpoint"] * anim_args.enable_checkpoint_scheduling
+    for field_name in field_names:
+        table.add_column(field_name, justify="center")
+    rows = [str(p.steps), str(p.cfg_scale)]
+    if anim_args.animation_mode != 'Interpolation':
+        rows.append(str(p.denoising_strength))
+    rows += [str(p.subseed), str(p.subseed_strength)] * (anim_args.enable_subseed_scheduling)
+    rows += [p.sampler_name] * anim_args.enable_sampler_scheduling
+    rows += [str(args.checkpoint)] * anim_args.enable_checkpoint_scheduling
+    table.add_row(*rows)
+
+    console.print(table)

extensions/deforum/scripts/deforum_helpers/gradio_funcs.py

@@ -0,0 +1,83 @@
+import gradio as gr
+from .video_audio_utilities import extract_number, get_quick_vid_info
+
+def change_visibility_from_skip_video(choice):
+    return gr.update(visible=False) if choice else gr.update(visible=True) 
+
+def update_r_upscale_factor(choice):
+    return gr.update(value='x4', choices = ['x4']) if choice != 'realesr-animevideov3' else gr.update(value='x2', choices = ['x2', 'x3', 'x4'])
+
+def change_perlin_visibility(choice):
+    return gr.update(visible=choice=="perlin")
+
+def change_color_coherence_video_every_N_frames_visibility(choice):
+    return gr.update(visible=choice=="Video Input")
+
+def change_seed_iter_visibility(choice):
+    return gr.update(visible=choice=="iter")
+    
+def change_seed_schedule_visibility(choice):
+    return gr.update(visible=choice=="schedule")
+
+def disable_pers_flip_accord(choice):
+    return gr.update(visible=True) if choice in ['2D','3D'] else gr.update(visible=False)
+
+def change_max_frames_visibility(choice):
+    return gr.update(visible=choice != "Video Input")
+    
+def change_diffusion_cadence_visibility(choice):
+    return gr.update(visible=choice not in ['Video Input', 'Interpolation'])
+    
+def disble_3d_related_stuff(choice):
+    return gr.update(visible=False) if choice != '3D' else gr.update(visible=True)
+    
+def enable_2d_related_stuff(choice):
+    return gr.update(visible=True) if choice == '2D' else gr.update(visible=False)
+    
+def disable_by_interpolation(choice):
+    return gr.update(visible=False) if choice in ['Interpolation'] else gr.update(visible=True)
+    
+def disable_by_video_input(choice):
+    return gr.update(visible=False) if choice in ['Video Input'] else gr.update(visible=True)
+    
+def change_comp_mask_x_visibility(choice):
+    return gr.update(visible=choice != "None")
+    
+def change_gif_button_visibility(choice):
+    return gr.update(visible=False, value=False) if int(choice) > 30 else gr.update(visible=True)
+    
+def disable_by_hybrid_composite(choice):
+    return gr.update(visible=True) if choice else gr.update(visible=False)
+        
+def disable_by_hybrid_composite_dynamic(choice, comp_mask_type):
+    if choice == True:
+        if comp_mask_type != 'None':
+            return gr.update(visible=True)
+    return gr.update(visible=False)
+    
+def disable_by_comp_mask(choice):
+    return gr.update(visible=False) if choice == 'None' else gr.update(visible=True)
+        
+def disable_by_non_optical_flow(choice):
+    return gr.update(visible=False) if choice != 'Optical Flow' else gr.update(visible=True)
+    
+# Upscaling Gradio UI related funcs
+def vid_upscale_gradio_update_stats(vid_path, upscale_factor):
+    if not vid_path:
+        return '---', '---', '---', '---'
+    factor = extract_number(upscale_factor)
+    fps, fcount, resolution = get_quick_vid_info(vid_path.name)
+    in_res_str = f"{resolution[0]}*{resolution[1]}"
+    out_res_str = f"{resolution[0] * factor}*{resolution[1] * factor}"
+    return fps, fcount, in_res_str, out_res_str
+def update_upscale_out_res(in_res, upscale_factor):
+    if not in_res:
+        return '---'
+    factor = extract_number(upscale_factor)
+    w, h = [int(x) * factor for x in in_res.split('*')]
+    return f"{w}*{h}"
+def update_upscale_out_res_by_model_name(in_res, upscale_model_name):
+    if not upscale_model_name or in_res == '---':
+        return '---'
+    factor = 2 if upscale_model_name == 'realesr-animevideov3' else 4
+    return f"{int(in_res.split('*')[0]) * factor}*{int(in_res.split('*')[1]) * factor}"

extensions/deforum/scripts/deforum_helpers/human_masking.py

@@ -0,0 +1,72 @@
+import os, cv2
+import torch
+from pathlib import Path
+from multiprocessing import freeze_support
+
+def extract_frames(input_video_path, output_imgs_path):
+    # Open the video file
+    vidcap = cv2.VideoCapture(input_video_path)
+    
+    # Get the total number of frames in the video
+    frame_count = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
+    
+    # Create the output directory if it does not exist
+    if not os.path.exists(output_imgs_path):
+        os.makedirs(output_imgs_path)
+        
+    # Extract the frames
+    for i in range(frame_count):
+        success, image = vidcap.read()
+        if success:
+            cv2.imwrite(os.path.join(output_imgs_path, f"frame{i}.png"), image)
+    print(f"{frame_count} frames extracted and saved to {output_imgs_path}")
+    
+    
+def video2humanmasks(input_frames_path, output_folder_path, output_type, fps):
+    # freeze support is needed for video outputting
+    freeze_support()
+    
+    # check if input path exists and is a directory
+    if not os.path.exists(input_frames_path) or not os.path.isdir(input_frames_path):
+        raise ValueError("Invalid input path: {}".format(input_frames_path))
+        
+    # check if output path exists and is a directory
+    if not os.path.exists(output_folder_path) or not os.path.isdir(output_folder_path):
+        raise ValueError("Invalid output path: {}".format(output_folder_path))
+    
+    # check if output_type is valid
+    valid_output_types = ["video", "pngs", "both"]
+    if output_type.lower() not in valid_output_types:
+        raise ValueError("Invalid output type: {}. Must be one of {}".format(output_type, valid_output_types))
+    
+    # try to predict where torch cache lives, so we can try and fetch models from cache in the next step
+    predicted_torch_model_cache_path = os.path.join(Path.home(), ".cache", "torch", "hub", "hithereai_RobustVideoMatting_master")
+    predicted_rvm_cache_testilfe = os.path.join(predicted_torch_model_cache_path, "hubconf.py")
+
+    # try to fetch the models from cache, and only if it can't be find, download from the internet (to enable offline usage)
+    try:
+        # Try to fetch the models from cache
+        convert_video = torch.hub.load(predicted_torch_model_cache_path, "converter", source='local')
+        model = torch.hub.load(predicted_torch_model_cache_path, "mobilenetv3", source='local').cuda()
+    except:
+        # Download from the internet if not found in cache
+        convert_video = torch.hub.load("hithereai/RobustVideoMatting", "converter")
+        model = torch.hub.load("hithereai/RobustVideoMatting", "mobilenetv3").cuda()
+        
+    output_alpha_vid_path = os.path.join(output_folder_path, "human_masked_video.mp4")
+    # extract humans masks from the input folder' imgs.
+    # in this step PNGs will be extracted only if output_type is set to PNGs. Otherwise a video will be made, and in the case of Both, the video will be extracted in the next step to PNGs
+    convert_video(
+    model, 
+    input_source=input_frames_path,  # full path of the folder that contains all of the extracted input imgs
+    output_type='video' if output_type.upper() in ("VIDEO", "BOTH") else 'png_sequence',
+    output_alpha=output_alpha_vid_path if output_type.upper() in ("VIDEO", "BOTH") else output_folder_path,
+    output_video_mbps=4,
+    output_video_fps=fps, 
+    downsample_ratio=None, # None for auto
+    seq_chunk=12, # Process n frames at once for better parallelism
+    progress=True # show extraction progress
+    )
+
+    if output_type.lower() == "both":
+        extract_frames(output_alpha_vid_path, output_folder_path)    

extensions/deforum/scripts/deforum_helpers/hybrid_video.py

@@ -0,0 +1,436 @@
+import cv2
+import os
+import pathlib
+import numpy as np
+import random
+from PIL import Image, ImageChops, ImageOps, ImageEnhance
+from .video_audio_utilities import vid2frames, get_quick_vid_info, get_frame_name, get_next_frame
+from .human_masking import video2humanmasks
+
+def delete_all_imgs_in_folder(folder_path):
+        files = list(pathlib.Path(folder_path).glob('*.jpg'))
+        files.extend(list(pathlib.Path(folder_path).glob('*.png')))
+        for f in files: os.remove(f)
+
+def hybrid_generation(args, anim_args, root):
+    video_in_frame_path = os.path.join(args.outdir, 'inputframes')
+    hybrid_frame_path = os.path.join(args.outdir, 'hybridframes')
+    human_masks_path = os.path.join(args.outdir, 'human_masks')
+
+    if anim_args.hybrid_generate_inputframes:
+        # create folders for the video input frames and optional hybrid frames to live in
+        os.makedirs(video_in_frame_path, exist_ok=True)
+        os.makedirs(hybrid_frame_path, exist_ok=True)
+        
+        # delete frames if overwrite = true
+        if anim_args.overwrite_extracted_frames:
+            delete_all_imgs_in_folder(hybrid_frame_path)
+
+        # save the video frames from input video
+        print(f"Video to extract: {anim_args.video_init_path}")
+        print(f"Extracting video (1 every {anim_args.extract_nth_frame}) frames to {video_in_frame_path}...")
+        video_fps = vid2frames(video_path=anim_args.video_init_path, video_in_frame_path=video_in_frame_path, n=anim_args.extract_nth_frame, overwrite=anim_args.overwrite_extracted_frames, extract_from_frame=anim_args.extract_from_frame, extract_to_frame=anim_args.extract_to_frame)
+    
+    # extract alpha masks of humans from the extracted input video imgs
+    if anim_args.hybrid_generate_human_masks != "None":
+        # create a folder for the human masks imgs to live in
+        print(f"Checking /creating a folder for the human masks")
+        os.makedirs(human_masks_path, exist_ok=True)
+            
+        # delete frames if overwrite = true
+        if anim_args.overwrite_extracted_frames:
+            delete_all_imgs_in_folder(human_masks_path)
+        
+        # in case that generate_input_frames isn't selected, we won't get the video fps rate as vid2frames isn't called, So we'll check the video fps in here instead
+        if not anim_args.hybrid_generate_inputframes:
+            _, video_fps, _ = get_quick_vid_info(anim_args.video_init_path)
+            
+        # calculate the correct fps of the masked video according to the original video fps and 'extract_nth_frame'
+        output_fps = video_fps/anim_args.extract_nth_frame
+        
+        # generate the actual alpha masks from the input imgs
+        print(f"Extracting alpha humans masks from the input frames")
+        video2humanmasks(video_in_frame_path, human_masks_path, anim_args.hybrid_generate_human_masks, output_fps)
+        
+    # determine max frames from length of input frames
+    anim_args.max_frames = len([f for f in pathlib.Path(video_in_frame_path).glob('*.jpg')])
+    print(f"Using {anim_args.max_frames} input frames from {video_in_frame_path}...")
+
+    # get sorted list of inputfiles
+    inputfiles = sorted(pathlib.Path(video_in_frame_path).glob('*.jpg'))
+
+    # use first frame as init
+    if anim_args.hybrid_use_first_frame_as_init_image:
+        for f in inputfiles:
+            args.init_image = str(f)
+            args.use_init = True
+            print(f"Using init_image from video: {args.init_image}")
+            break
+
+    return args, anim_args, inputfiles
+
+def hybrid_composite(args, anim_args, frame_idx, prev_img, depth_model, hybrid_comp_schedules, root):
+    video_frame = os.path.join(args.outdir, 'inputframes', get_frame_name(anim_args.video_init_path) + f"{frame_idx:05}.jpg")
+    video_depth_frame = os.path.join(args.outdir, 'hybridframes', get_frame_name(anim_args.video_init_path) + f"_vid_depth{frame_idx:05}.jpg")
+    depth_frame = os.path.join(args.outdir, f"{args.timestring}_depth_{frame_idx-1:05}.png")
+    mask_frame = os.path.join(args.outdir, 'hybridframes', get_frame_name(anim_args.video_init_path) + f"_mask{frame_idx:05}.jpg")
+    comp_frame = os.path.join(args.outdir, 'hybridframes', get_frame_name(anim_args.video_init_path) + f"_comp{frame_idx:05}.jpg")
+    prev_frame = os.path.join(args.outdir, 'hybridframes', get_frame_name(anim_args.video_init_path) + f"_prev{frame_idx:05}.jpg")
+    prev_img = cv2.cvtColor(prev_img, cv2.COLOR_BGR2RGB)
+    prev_img_hybrid = Image.fromarray(prev_img)
+    video_image = Image.open(video_frame)
+    video_image = video_image.resize((args.W, args.H), Image.Resampling.LANCZOS)
+    hybrid_mask = None
+
+    # composite mask types
+    if anim_args.hybrid_comp_mask_type == 'Depth': # get depth from last generation
+        hybrid_mask = Image.open(depth_frame)
+    elif anim_args.hybrid_comp_mask_type == 'Video Depth': # get video depth
+        video_depth = depth_model.predict(np.array(video_image), anim_args, root.half_precision)
+        depth_model.save(video_depth_frame, video_depth)
+        hybrid_mask = Image.open(video_depth_frame)
+    elif anim_args.hybrid_comp_mask_type == 'Blend': # create blend mask image
+        hybrid_mask = Image.blend(ImageOps.grayscale(prev_img_hybrid), ImageOps.grayscale(video_image), hybrid_comp_schedules['mask_blend_alpha'])
+    elif anim_args.hybrid_comp_mask_type == 'Difference': # create difference mask image
+        hybrid_mask = ImageChops.difference(ImageOps.grayscale(prev_img_hybrid), ImageOps.grayscale(video_image))
+        
+    # optionally invert mask, if mask type is defined
+    if anim_args.hybrid_comp_mask_inverse and anim_args.hybrid_comp_mask_type != "None":
+        hybrid_mask = ImageOps.invert(hybrid_mask)
+
+    # if a mask type is selected, make composition
+    if hybrid_mask == None:
+        hybrid_comp = video_image
+    else:
+        # ensure grayscale
+        hybrid_mask = ImageOps.grayscale(hybrid_mask)
+        # equalization before
+        if anim_args.hybrid_comp_mask_equalize in ['Before', 'Both']:
+            hybrid_mask = ImageOps.equalize(hybrid_mask)        
+        # contrast
+        hybrid_mask = ImageEnhance.Contrast(hybrid_mask).enhance(hybrid_comp_schedules['mask_contrast'])
+        # auto contrast with cutoffs lo/hi
+        if anim_args.hybrid_comp_mask_auto_contrast:
+            hybrid_mask = autocontrast_grayscale(np.array(hybrid_mask), hybrid_comp_schedules['mask_auto_contrast_cutoff_low'], hybrid_comp_schedules['mask_auto_contrast_cutoff_high'])
+            hybrid_mask = Image.fromarray(hybrid_mask)
+            hybrid_mask = ImageOps.grayscale(hybrid_mask)   
+        if anim_args.hybrid_comp_save_extra_frames:
+            hybrid_mask.save(mask_frame)        
+        # equalization after
+        if anim_args.hybrid_comp_mask_equalize in ['After', 'Both']:
+            hybrid_mask = ImageOps.equalize(hybrid_mask)        
+        # do compositing and save
+        hybrid_comp = Image.composite(prev_img_hybrid, video_image, hybrid_mask)            
+        if anim_args.hybrid_comp_save_extra_frames:
+            hybrid_comp.save(comp_frame)
+
+    # final blend of composite with prev_img, or just a blend if no composite is selected
+    hybrid_blend = Image.blend(prev_img_hybrid, hybrid_comp, hybrid_comp_schedules['alpha'])  
+    if anim_args.hybrid_comp_save_extra_frames:
+        hybrid_blend.save(prev_frame)
+
+    prev_img = cv2.cvtColor(np.array(hybrid_blend), cv2.COLOR_RGB2BGR)
+
+    # restore to np array and return
+    return args, prev_img
+
+def get_matrix_for_hybrid_motion(frame_idx, dimensions, inputfiles, hybrid_motion):
+    img1 = cv2.cvtColor(get_resized_image_from_filename(str(inputfiles[frame_idx-1]), dimensions), cv2.COLOR_BGR2GRAY)
+    img2 = cv2.cvtColor(get_resized_image_from_filename(str(inputfiles[frame_idx]), dimensions), cv2.COLOR_BGR2GRAY)
+    matrix = get_transformation_matrix_from_images(img1, img2, hybrid_motion)
+    print(f"Calculating {hybrid_motion} RANSAC matrix for frames {frame_idx} to {frame_idx+1}")
+    return matrix
+
+def get_matrix_for_hybrid_motion_prev(frame_idx, dimensions, inputfiles, prev_img, hybrid_motion):
+    # first handle invalid images from cadence by returning default matrix
+    height, width = prev_img.shape[:2]
+    if height == 0 or width == 0 or prev_img != np.uint8:
+        return get_hybrid_motion_default_matrix(hybrid_motion)
+    else:
+        prev_img_gray = cv2.cvtColor(prev_img, cv2.COLOR_BGR2GRAY)
+        img = cv2.cvtColor(get_resized_image_from_filename(str(inputfiles[frame_idx]), dimensions), cv2.COLOR_BGR2GRAY)
+        matrix = get_transformation_matrix_from_images(prev_img_gray, img, hybrid_motion)
+        print(f"Calculating {hybrid_motion} RANSAC matrix for frames {frame_idx} to {frame_idx+1}")
+        return matrix
+
+def get_flow_for_hybrid_motion(frame_idx, dimensions, inputfiles, hybrid_frame_path, method, do_flow_visualization=False):
+    print(f"Calculating {method} optical flow for frames {frame_idx} to {frame_idx+1}")
+    i1 = get_resized_image_from_filename(str(inputfiles[frame_idx]), dimensions)
+    i2 = get_resized_image_from_filename(str(inputfiles[frame_idx+1]), dimensions)
+    flow = get_flow_from_images(i1, i2, method)
+    if do_flow_visualization:
+        save_flow_visualization(frame_idx, dimensions, flow, inputfiles, hybrid_frame_path)
+    return flow
+
+def get_flow_for_hybrid_motion_prev(frame_idx, dimensions, inputfiles, hybrid_frame_path, prev_img, method, do_flow_visualization=False):
+    print(f"Calculating {method} optical flow for frames {frame_idx} to {frame_idx+1}")
+    # first handle invalid images from cadence by returning default matrix
+    height, width = prev_img.shape[:2]   
+    if height == 0 or width == 0:
+        flow = get_hybrid_motion_default_flow(dimensions)
+    else:
+        i1 = prev_img.astype(np.uint8)
+        i2 = get_resized_image_from_filename(str(inputfiles[frame_idx]), dimensions)
+        flow = get_flow_from_images(i1, i2, method)
+    if do_flow_visualization:
+        save_flow_visualization(frame_idx, dimensions, flow, inputfiles, hybrid_frame_path)
+    return flow
+
+def image_transform_ransac(image_cv2, xform, hybrid_motion, border_mode=cv2.BORDER_REPLICATE):
+    if hybrid_motion == "Perspective":
+        return image_transform_perspective(image_cv2, xform, border_mode=border_mode)
+    else: # Affine
+        return image_transform_affine(image_cv2, xform, border_mode=border_mode)
+
+def image_transform_optical_flow(img, flow, border_mode=cv2.BORDER_REPLICATE, flow_reverse=False):
+    if not flow_reverse:
+        flow = -flow
+    h, w = img.shape[:2]
+    flow[:, :, 0] += np.arange(w)
+    flow[:, :, 1] += np.arange(h)[:,np.newaxis]
+    return remap(img, flow, border_mode)
+
+def image_transform_affine(image_cv2, xform, border_mode=cv2.BORDER_REPLICATE):
+    return cv2.warpAffine(
+        image_cv2,
+        xform,
+        (image_cv2.shape[1],image_cv2.shape[0]),
+        borderMode=border_mode
+    )
+
+def image_transform_perspective(image_cv2, xform, border_mode=cv2.BORDER_REPLICATE):
+    return cv2.warpPerspective(
+        image_cv2,
+        xform,
+        (image_cv2.shape[1], image_cv2.shape[0]),
+        borderMode=border_mode
+    )
+
+def get_hybrid_motion_default_matrix(hybrid_motion):
+    if hybrid_motion == "Perspective":
+        arr = np.array([[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]])
+    else:
+        arr = np.array([[1., 0., 0.], [0., 1., 0.]])
+    return arr
+
+def get_hybrid_motion_default_flow(dimensions):
+    cols, rows = dimensions
+    flow = np.zeros((rows, cols, 2), np.float32)
+    return flow
+
+def get_transformation_matrix_from_images(img1, img2, hybrid_motion, max_corners=200, quality_level=0.01, min_distance=30, block_size=3):
+    # Detect feature points in previous frame
+    prev_pts = cv2.goodFeaturesToTrack(img1,
+                                        maxCorners=max_corners,
+                                        qualityLevel=quality_level,
+                                        minDistance=min_distance,
+                                        blockSize=block_size)
+
+    if prev_pts is None or len(prev_pts) < 8 or img1 is None or img2 is None:
+        return get_hybrid_motion_default_matrix(hybrid_motion)
+
+    # Get optical flow
+    curr_pts, status, err = cv2.calcOpticalFlowPyrLK(img1, img2, prev_pts, None) 
+   
+    # Filter only valid points
+    idx = np.where(status==1)[0]
+    prev_pts = prev_pts[idx]
+    curr_pts = curr_pts[idx]
+
+    if len(prev_pts) < 8 or len(curr_pts) < 8:
+        return get_hybrid_motion_default_matrix(hybrid_motion)
+    
+    if hybrid_motion == "Perspective":  # Perspective - Find the transformation between points
+        transformation_matrix, mask = cv2.findHomography(prev_pts, curr_pts, cv2.RANSAC, 5.0)
+        return transformation_matrix
+    else: # Affine - Compute a rigid transformation (without depth, only scale + rotation + translation)
+        transformation_rigid_matrix, rigid_mask = cv2.estimateAffinePartial2D(prev_pts, curr_pts)
+        return transformation_rigid_matrix
+
+def get_flow_from_images(i1, i2, method):
+    if method =="DIS Medium":
+        r = get_flow_from_images_DIS(i1, i2, cv2.DISOPTICAL_FLOW_PRESET_MEDIUM)
+    elif method =="DIS Fast":
+        r = get_flow_from_images_DIS(i1, i2, cv2.DISOPTICAL_FLOW_PRESET_FAST)
+    elif method =="DIS UltraFast":
+        r = get_flow_from_images_DIS(i1, i2, cv2.DISOPTICAL_FLOW_PRESET_ULTRAFAST)
+    elif method == "DenseRLOF": # requires running opencv-contrib-python (full opencv) INSTEAD of opencv-python
+        r = get_flow_from_images_Dense_RLOF(i1, i2)
+    elif method == "SF": # requires running opencv-contrib-python (full opencv) INSTEAD of opencv-python
+        r = get_flow_from_images_SF(i1, i2)
+    elif method =="Farneback Fine":
+        r = get_flow_from_images_Farneback(i1, i2, 'fine')
+    else: # Farneback Normal:
+        r = get_flow_from_images_Farneback(i1, i2)
+    return r
+
+def get_flow_from_images_DIS(i1, i2, preset):
+    i1 = cv2.cvtColor(i1, cv2.COLOR_BGR2GRAY)
+    i2 = cv2.cvtColor(i2, cv2.COLOR_BGR2GRAY)
+    dis=cv2.DISOpticalFlow_create(preset)
+    return dis.calc(i1, i2, None)
+
+def get_flow_from_images_Dense_RLOF(i1, i2, last_flow=None):
+    return cv2.optflow.calcOpticalFlowDenseRLOF(i1, i2, flow = last_flow)
+
+def get_flow_from_images_SF(i1, i2, last_flow=None, layers = 3, averaging_block_size = 2, max_flow = 4):
+    return cv2.optflow.calcOpticalFlowSF(i1, i2, layers, averaging_block_size, max_flow)
+
+def get_flow_from_images_Farneback(i1, i2, preset="normal", last_flow=None, pyr_scale = 0.5, levels = 3, winsize = 15, iterations = 3, poly_n = 5, poly_sigma = 1.2, flags = 0):
+    flags = cv2.OPTFLOW_FARNEBACK_GAUSSIAN         # Specify the operation flags
+    pyr_scale = 0.5   # The image scale (<1) to build pyramids for each image
+    if preset == "fine":
+        levels = 13       # The number of pyramid layers, including the initial image
+        winsize = 77      # The averaging window size
+        iterations = 13   # The number of iterations at each pyramid level
+        poly_n = 15       # The size of the pixel neighborhood used to find polynomial expansion in each pixel
+        poly_sigma = 0.8  # The standard deviation of the Gaussian used to smooth derivatives used as a basis for the polynomial expansion
+    else: # "normal"
+        levels = 5        # The number of pyramid layers, including the initial image
+        winsize = 21      # The averaging window size
+        iterations = 5    # The number of iterations at each pyramid level
+        poly_n = 7        # The size of the pixel neighborhood used to find polynomial expansion in each pixel
+        poly_sigma = 1.2  # The standard deviation of the Gaussian used to smooth derivatives used as a basis for the polynomial expansion
+    i1 = cv2.cvtColor(i1, cv2.COLOR_BGR2GRAY)
+    i2 = cv2.cvtColor(i2, cv2.COLOR_BGR2GRAY)
+    flags = 0 # flags = cv2.OPTFLOW_USE_INITIAL_FLOW    
+    flow = cv2.calcOpticalFlowFarneback(i1, i2, last_flow, pyr_scale, levels, winsize, iterations, poly_n, poly_sigma, flags)
+    return flow
+
+def save_flow_visualization(frame_idx, dimensions, flow, inputfiles, hybrid_frame_path):
+    flow_img_file = os.path.join(hybrid_frame_path, f"flow{frame_idx:05}.jpg")
+    flow_img = cv2.imread(str(inputfiles[frame_idx]))
+    flow_img = cv2.resize(flow_img, (dimensions[0], dimensions[1]), cv2.INTER_AREA)
+    flow_img = cv2.cvtColor(flow_img, cv2.COLOR_RGB2GRAY)
+    flow_img = cv2.cvtColor(flow_img, cv2.COLOR_GRAY2BGR)
+    flow_img = draw_flow_lines_in_grid_in_color(flow_img, flow)
+    flow_img = cv2.cvtColor(flow_img, cv2.COLOR_BGR2RGB)
+    cv2.imwrite(flow_img_file, flow_img)
+    print(f"Saved optical flow visualization: {flow_img_file}")
+
+def draw_flow_lines_in_grid_in_color(img, flow, step=8, magnitude_multiplier=1, min_magnitude = 1, max_magnitude = 10000):
+    flow = flow * magnitude_multiplier
+    h, w = img.shape[:2]
+    y, x = np.mgrid[step/2:h:step, step/2:w:step].reshape(2,-1).astype(int)
+    fx, fy = flow[y,x].T
+    lines = np.vstack([x, y, x+fx, y+fy]).T.reshape(-1, 2, 2)
+    lines = np.int32(lines + 0.5)
+    vis = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+    vis = cv2.cvtColor(vis, cv2.COLOR_GRAY2BGR)
+
+    mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1])
+    hsv = np.zeros((flow.shape[0], flow.shape[1], 3), dtype=np.uint8)
+    hsv[...,0] = ang*180/np.pi/2
+    hsv[...,1] = 255
+    hsv[...,2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
+    bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
+    vis = cv2.add(vis, bgr)
+
+    # Iterate through the lines
+    for (x1, y1), (x2, y2) in lines:
+        # Calculate the magnitude of the line
+        magnitude = np.sqrt((x2 - x1)**2 + (y2 - y1)**2)
+
+        # Only draw the line if it falls within the magnitude range
+        if min_magnitude <= magnitude <= max_magnitude:
+            b = int(bgr[y1, x1, 0])
+            g = int(bgr[y1, x1, 1])
+            r = int(bgr[y1, x1, 2])
+            color = (b, g, r)
+            cv2.arrowedLine(vis, (x1, y1), (x2, y2), color, thickness=1, tipLength=0.1)    
+    return vis
+
+def draw_flow_lines_in_color(img, flow, threshold=3, magnitude_multiplier=1, min_magnitude = 0, max_magnitude = 10000):
+    # h, w = img.shape[:2]
+    vis = img.copy()  # Create a copy of the input image
+    
+    # Find the locations in the flow field where the magnitude of the flow is greater than the threshold
+    mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1])
+    idx = np.where(mag > threshold)
+
+    # Create HSV image
+    hsv = np.zeros((flow.shape[0], flow.shape[1], 3), dtype=np.uint8)
+    hsv[...,0] = ang*180/np.pi/2
+    hsv[...,1] = 255
+    hsv[...,2] = cv2.normalize(mag, None, 0, 255, cv2.NORM_MINMAX)
+
+    # Convert HSV image to BGR
+    bgr = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
+
+    # Add color from bgr 
+    vis = cv2.add(vis, bgr)
+
+    # Draw an arrow at each of these locations to indicate the direction of the flow
+    for i, (y, x) in enumerate(zip(idx[0], idx[1])):
+        # Calculate the magnitude of the line
+        x2 = x + magnitude_multiplier * int(flow[y, x, 0])
+        y2 = y + magnitude_multiplier * int(flow[y, x, 1])
+        magnitude = np.sqrt((x2 - x)**2 + (y2 - y)**2)
+
+        # Only draw the line if it falls within the magnitude range
+        if min_magnitude <= magnitude <= max_magnitude:
+            if i % random.randint(100, 200) == 0:
+                b = int(bgr[y, x, 0])
+                g = int(bgr[y, x, 1])
+                r = int(bgr[y, x, 2])
+                color = (b, g, r)
+                cv2.arrowedLine(vis, (x, y), (x2, y2), color, thickness=1, tipLength=0.25)
+
+    return vis
+
+def autocontrast_grayscale(image, low_cutoff=0, high_cutoff=100):
+    # Perform autocontrast on a grayscale np array image.
+    # Find the minimum and maximum values in the image
+    min_val = np.percentile(image, low_cutoff)
+    max_val = np.percentile(image, high_cutoff)
+
+    # Scale the image so that the minimum value is 0 and the maximum value is 255
+    image = 255 * (image - min_val) / (max_val - min_val)
+
+    # Clip values that fall outside the range [0, 255]
+    image = np.clip(image, 0, 255)
+
+    return image
+
+def get_resized_image_from_filename(im, dimensions):
+    img = cv2.imread(im)
+    return cv2.resize(img, (dimensions[0], dimensions[1]), cv2.INTER_AREA)
+
+def remap(img, flow, border_mode = cv2.BORDER_REFLECT_101):
+    # copyMakeBorder doesn't support wrap, but supports replicate. Replaces wrap with reflect101.
+    if border_mode == cv2.BORDER_WRAP:
+        border_mode = cv2.BORDER_REFLECT_101
+    h, w = img.shape[:2]
+    displacement = int(h * 0.25), int(w * 0.25)
+    larger_img = cv2.copyMakeBorder(img, displacement[0], displacement[0], displacement[1], displacement[1], border_mode)
+    lh, lw = larger_img.shape[:2]
+    larger_flow = extend_flow(flow, lw, lh)
+    remapped_img = cv2.remap(larger_img, larger_flow, None, cv2.INTER_LINEAR, border_mode)
+    output_img = center_crop_image(remapped_img, w, h)
+    return output_img
+
+def center_crop_image(img, w, h):
+    y, x, _ = img.shape
+    width_indent = int((x - w) / 2)
+    height_indent = int((y - h) / 2)
+    cropped_img = img[height_indent:y-height_indent, width_indent:x-width_indent]
+    return cropped_img
+
+def extend_flow(flow, w, h):
+    # Get the shape of the original flow image
+    flow_h, flow_w = flow.shape[:2]
+    # Calculate the position of the image in the new image
+    x_offset = int((w - flow_w) / 2)
+    y_offset = int((h - flow_h) / 2)
+    # Generate the X and Y grids
+    x_grid, y_grid = np.meshgrid(np.arange(w), np.arange(h))
+    # Create the new flow image and set it to the X and Y grids
+    new_flow = np.dstack((x_grid, y_grid)).astype(np.float32)
+    # Shift the values of the original flow by the size of the border
+    flow[:,:,0] += x_offset
+    flow[:,:,1] += y_offset
+    # Overwrite the middle of the grid with the original flow
+    new_flow[y_offset:y_offset+flow_h, x_offset:x_offset+flow_w, :] = flow
+    # Return the extended image
+    return new_flow
+    

extensions/deforum/scripts/deforum_helpers/image_sharpening.py

@@ -0,0 +1,22 @@
+import cv2
+import numpy as np
+
+def unsharp_mask(img, kernel_size=(5, 5), sigma=1.0, amount=1.0, threshold=0, mask=None):
+    if amount == 0:
+        return img
+    # Return a sharpened version of the image, using an unsharp mask.
+    # If mask is not None, only areas under mask are handled
+    blurred = cv2.GaussianBlur(img, kernel_size, sigma)
+    sharpened = float(amount + 1) * img - float(amount) * blurred
+    sharpened = np.maximum(sharpened, np.zeros(sharpened.shape))
+    sharpened = np.minimum(sharpened, 255 * np.ones(sharpened.shape))
+    sharpened = sharpened.round().astype(np.uint8)
+    if threshold > 0:
+        low_contrast_mask = np.absolute(img - blurred) < threshold
+        np.copyto(sharpened, img, where=low_contrast_mask)
+    if mask is not None:
+        mask = np.array(mask)
+        masked_sharpened = cv2.bitwise_and(sharpened, sharpened, mask=mask)
+        masked_img = cv2.bitwise_and(img, img, mask=255-mask)
+        sharpened = cv2.add(masked_img, masked_sharpened)
+    return sharpened

extensions/deforum/scripts/deforum_helpers/load_images.py

@@ -0,0 +1,102 @@
+import requests
+import os
+from PIL import Image, ImageOps
+import cv2
+import numpy as np
+import socket
+import torchvision.transforms.functional as TF
+
+def load_img(path : str, shape=None, use_alpha_as_mask=False):
+    # use_alpha_as_mask: Read the alpha channel of the image as the mask image
+    image = load_image(path)
+    if use_alpha_as_mask:
+        image = image.convert('RGBA')
+    else:
+        image = image.convert('RGB')
+
+    if shape is not None:
+        image = image.resize(shape, resample=Image.LANCZOS)
+
+    mask_image = None
+    if use_alpha_as_mask:
+        # Split alpha channel into a mask_image
+        red, green, blue, alpha = Image.Image.split(image)
+        mask_image = alpha.convert('L')
+        image = image.convert('RGB')
+        
+        # check using init image alpha as mask if mask is not blank
+        extrema = mask_image.getextrema()
+        if (extrema == (0,0)) or extrema == (255,255):
+            print("use_alpha_as_mask==True: Using the alpha channel from the init image as a mask, but the alpha channel is blank.")
+            print("ignoring alpha as mask.")
+            mask_image = None
+
+    return image, mask_image
+
+def load_image(image_path :str):
+    image = None
+    if image_path.startswith('http://') or image_path.startswith('https://'):
+        try:
+            host = socket.gethostbyname("www.google.com")
+            s = socket.create_connection((host, 80), 2)
+            s.close()
+        except:
+            raise ConnectionError("There is no active internet connection available - please use local masks and init files only.")
+        
+        try:
+            response = requests.get(image_path, stream=True)
+        except requests.exceptions.RequestException as e:
+            raise ConnectionError("Failed to download image due to no internet connection. Error: {}".format(e))
+        if response.status_code == 404 or response.status_code != 200:
+            raise ConnectionError("Init image url or mask image url is not valid")
+        image = Image.open(response.raw).convert('RGB')
+    else:
+        if not os.path.exists(image_path):
+            raise RuntimeError("Init image path or mask image path is not valid")
+        image = Image.open(image_path).convert('RGB')
+        
+    return image
+
+def prepare_mask(mask_input, mask_shape, mask_brightness_adjust=1.0, mask_contrast_adjust=1.0):
+    """
+    prepares mask for use in webui
+    """
+    if isinstance(mask_input, Image.Image):
+        mask = mask_input
+    else :
+        mask = load_image(mask_input)
+    mask = mask.resize(mask_shape, resample=Image.LANCZOS)
+    if mask_brightness_adjust != 1:
+        mask = TF.adjust_brightness(mask, mask_brightness_adjust)
+    if mask_contrast_adjust != 1:
+        mask = TF.adjust_contrast(mask, mask_contrast_adjust)
+    mask = mask.convert('L')
+    return mask
+
+def check_mask_for_errors(mask_input, invert_mask=False):
+    extrema = mask_input.getextrema()
+    if (invert_mask):
+        if extrema == (255,255): 
+            print("after inverting mask will be blank. ignoring mask")  
+            return None
+    elif extrema == (0,0): 
+        print("mask is blank. ignoring mask")  
+        return None
+    else:
+        return mask_input    
+
+def get_mask(args):
+    return check_mask_for_errors(
+         prepare_mask(args.mask_file, (args.W, args.H), args.mask_contrast_adjust, args.mask_brightness_adjust)
+    )
+
+def get_mask_from_file(mask_file, args):
+    return check_mask_for_errors(
+         prepare_mask(mask_file, (args.W, args.H), args.mask_contrast_adjust, args.mask_brightness_adjust)
+    )
+
+def blank_if_none(mask, w, h, mode):
+    return Image.new(mode, (w, h), (0)) if mask is None else mask
+
+def none_if_blank(mask):
+    return None if mask.getextrema() == (0,0) else mask

extensions/deforum/scripts/deforum_helpers/noise.py

@@ -0,0 +1,64 @@
+import torch
+import numpy as np
+from PIL import ImageOps
+import math
+from .animation import sample_to_cv2
+import cv2
+
+deforum_noise_gen = torch.Generator(device='cpu')
+
+# 2D Perlin noise in PyTorch https://gist.github.com/vadimkantorov/ac1b097753f217c5c11bc2ff396e0a57
+def rand_perlin_2d(shape, res, fade = lambda t: 6*t**5 - 15*t**4 + 10*t**3):
+    delta = (res[0] / shape[0], res[1] / shape[1])
+    d = (shape[0] // res[0], shape[1] // res[1])
+    
+    grid = torch.stack(torch.meshgrid(torch.arange(0, res[0], delta[0]), torch.arange(0, res[1], delta[1]), indexing='ij'), dim = -1) % 1
+    angles = 2*math.pi*torch.rand(res[0]+1, res[1]+1, generator=deforum_noise_gen)
+    gradients = torch.stack((torch.cos(angles), torch.sin(angles)), dim = -1)
+    
+    tile_grads = lambda slice1, slice2: gradients[slice1[0]:slice1[1], slice2[0]:slice2[1]].repeat_interleave(d[0], 0).repeat_interleave(d[1], 1)
+    dot = lambda grad, shift: (torch.stack((grid[:shape[0],:shape[1],0] + shift[0], grid[:shape[0],:shape[1], 1] + shift[1]  ), dim = -1) * grad[:shape[0], :shape[1]]).sum(dim = -1)
+    
+    n00 = dot(tile_grads([0, -1], [0, -1]), [0,  0])
+    n10 = dot(tile_grads([1, None], [0, -1]), [-1, 0])
+    n01 = dot(tile_grads([0, -1],[1, None]), [0, -1])
+    n11 = dot(tile_grads([1, None], [1, None]), [-1,-1])
+    t = fade(grid[:shape[0], :shape[1]])
+    return math.sqrt(2) * torch.lerp(torch.lerp(n00, n10, t[..., 0]), torch.lerp(n01, n11, t[..., 0]), t[..., 1])
+
+def rand_perlin_2d_octaves(shape, res, octaves=1, persistence=0.5):
+    noise = torch.zeros(shape)
+    frequency = 1
+    amplitude = 1
+    for _ in range(int(octaves)):
+        noise += amplitude * rand_perlin_2d(shape, (frequency*res[0], frequency*res[1]))
+        frequency *= 2
+        amplitude *= persistence
+    return noise
+
+def condition_noise_mask(noise_mask, invert_mask = False):
+    if invert_mask:
+        noise_mask = ImageOps.invert(noise_mask)
+    noise_mask = np.array(noise_mask.convert("L"))
+    noise_mask = noise_mask.astype(np.float32) / 255.0
+    noise_mask = np.around(noise_mask, decimals=0)
+    noise_mask = torch.from_numpy(noise_mask)
+    #noise_mask = torch.round(noise_mask)
+    return noise_mask
+
+def add_noise(sample, noise_amt: float, seed: int, noise_type: str, noise_args, noise_mask = None, invert_mask = False):
+    deforum_noise_gen.manual_seed(seed) # Reproducibility
+    sample2dshape = (sample.shape[0], sample.shape[1]) #sample is cv2, so height - width
+    noise = torch.randn((sample.shape[2], sample.shape[0], sample.shape[1]), generator=deforum_noise_gen) # White noise
+    if noise_type == 'perlin':
+        # rand_perlin_2d_octaves is between -1 and 1, so we need to shift it to be between 0 and 1
+        # print(sample.shape)
+        noise = noise * ((rand_perlin_2d_octaves(sample2dshape, (int(noise_args[0]), int(noise_args[1])), octaves=noise_args[2], persistence=noise_args[3]) + torch.ones(sample2dshape)) / 2)
+    if noise_mask is not None:
+        noise_mask = condition_noise_mask(noise_mask, invert_mask)
+        noise_to_add = sample_to_cv2(noise * noise_mask)
+    else:
+        noise_to_add = sample_to_cv2(noise)
+    sample = cv2.addWeighted(sample, 1-noise_amt, noise_to_add, noise_amt, 0) 
+    
+    return sample

extensions/deforum/scripts/deforum_helpers/parseq_adapter.py

@@ -0,0 +1,164 @@
+import copy
+import json
+import logging
+import operator
+from operator import itemgetter
+
+import numpy as np
+import pandas as pd
+import requests
+
+from .animation_key_frames import DeformAnimKeys
+
+logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO)
+
+
+class ParseqAnimKeys():
+    def __init__(self, parseq_args, anim_args):
+
+        # Resolve manifest either directly from supplied value
+        # or via supplied URL
+        manifestOrUrl = parseq_args.parseq_manifest.strip()
+        if (manifestOrUrl.startswith('http')):
+            logging.info(f"Loading Parseq manifest from URL: {manifestOrUrl}")
+            try:
+                body = requests.get(manifestOrUrl).text
+                logging.debug(f"Loaded remote manifest: {body}")
+                self.parseq_json = json.loads(body)
+
+                # Add the parseq manifest without the detailed frame data to parseq_args.
+                # This ensures it will be saved in the settings file, so that you can always
+                # see exactly what parseq prompts and keyframes were used, even if what the URL
+                # points to changes.
+                parseq_args.fetched_parseq_manifest_summary = copy.deepcopy(self.parseq_json)
+                if parseq_args.fetched_parseq_manifest_summary['rendered_frames']:
+                    del parseq_args.fetched_parseq_manifest_summary['rendered_frames']
+                if parseq_args.fetched_parseq_manifest_summary['rendered_frames_meta']:
+                    del parseq_args.fetched_parseq_manifest_summary['rendered_frames_meta']
+
+            except Exception as e:
+                logging.error(f"Unable to load Parseq manifest from URL: {manifestOrUrl}")
+                raise e
+        else:
+            self.parseq_json = json.loads(manifestOrUrl)
+
+        self.default_anim_keys = DeformAnimKeys(anim_args)
+        self.rendered_frames = self.parseq_json['rendered_frames']       
+        self.max_frame = self.get_max('frame')
+        count_defined_frames = len(self.rendered_frames)        
+        expected_defined_frames = self.max_frame+1 # frames are 0-indexed
+
+        self.required_frames = anim_args.max_frames
+
+        if (expected_defined_frames != count_defined_frames): 
+            logging.warning(f"There may be duplicated or missing frame data in the Parseq input: expected {expected_defined_frames} frames including frame 0 because the highest frame number is {self.max_frame}, but there are {count_defined_frames} frames defined.")
+
+        if (anim_args.max_frames > count_defined_frames):
+            logging.info(f"Parseq data defines {count_defined_frames} frames, but the requested animation is {anim_args.max_frames} frames. The last Parseq frame definition will be duplicated to match the expected frame count.")
+        if (anim_args.max_frames < count_defined_frames):
+            logging.info(f"Parseq data defines {count_defined_frames} frames, but the requested animation is {anim_args.max_frames} frames. The last Parseq frame definitions will be ignored.")
+        else:
+            logging.info(f"Parseq data defines {count_defined_frames} frames.")
+
+        # Parseq treats input values as absolute values. So if you want to 
+        # progressively rotate 180 degrees over 4 frames, you specify: 45, 90, 135, 180.
+        # However, many animation parameters are relative to the previous frame if there is enough
+        # loopback strength. So if you want to rotate 180 degrees over 5 frames, the animation engine expects:
+        # 45, 45, 45, 45. Therefore, for such parameter, we use the fact that Parseq supplies delta values.
+        optional_delta = '_delta' if parseq_args.parseq_use_deltas else ''
+        self.angle_series = self.parseq_to_anim_series('angle' + optional_delta)
+        self.zoom_series = self.parseq_to_anim_series('zoom' + optional_delta)        
+        self.translation_x_series = self.parseq_to_anim_series('translation_x' + optional_delta)
+        self.translation_y_series = self.parseq_to_anim_series('translation_y' + optional_delta)
+        self.translation_z_series = self.parseq_to_anim_series('translation_z' + optional_delta)
+        self.rotation_3d_x_series = self.parseq_to_anim_series('rotation_3d_x' + optional_delta)
+        self.rotation_3d_y_series = self.parseq_to_anim_series('rotation_3d_y' + optional_delta)
+        self.rotation_3d_z_series = self.parseq_to_anim_series('rotation_3d_z' + optional_delta)
+        self.perspective_flip_theta_series = self.parseq_to_anim_series('perspective_flip_theta' + optional_delta)
+        self.perspective_flip_phi_series = self.parseq_to_anim_series('perspective_flip_phi' + optional_delta)
+        self.perspective_flip_gamma_series = self.parseq_to_anim_series('perspective_flip_gamma' + optional_delta)
+ 
+        # Non-motion animation args
+        self.perspective_flip_fv_series = self.parseq_to_anim_series('perspective_flip_fv')
+        self.noise_schedule_series = self.parseq_to_anim_series('noise')
+        self.strength_schedule_series = self.parseq_to_anim_series('strength')
+        self.sampler_schedule_series = self.parseq_to_anim_series('sampler_schedule')
+        self.contrast_schedule_series = self.parseq_to_anim_series('contrast')
+        self.cfg_scale_schedule_series = self.parseq_to_anim_series('scale')
+        self.steps_schedule_series = self.parseq_to_anim_series("steps_schedule")
+        self.seed_schedule_series = self.parseq_to_anim_series('seed')
+        self.fov_series = self.parseq_to_anim_series('fov')
+        self.near_series = self.parseq_to_anim_series('near')
+        self.far_series = self.parseq_to_anim_series('far')
+        self.prompts = self.parseq_to_anim_series('deforum_prompt') # formatted as "{positive} --neg {negative}"
+        self.subseed_series = self.parseq_to_anim_series('subseed')
+        self.subseed_strength_series = self.parseq_to_anim_series('subseed_strength')
+        self.kernel_schedule_series = self.parseq_to_anim_series('antiblur_kernel')
+        self.sigma_schedule_series = self.parseq_to_anim_series('antiblur_sigma')
+        self.amount_schedule_series = self.parseq_to_anim_series('antiblur_amount')
+        self.threshold_schedule_series = self.parseq_to_anim_series('antiblur_threshold')
+        
+        # Config:
+        # TODO this is currently ignored. User must ensure the output FPS set in parseq
+        # matches the one set in Deforum to avoid unexpected results.
+        self.config_output_fps = self.parseq_json['options']['output_fps']
+
+    def get_max(self, seriesName):
+        return max(self.rendered_frames, key=itemgetter(seriesName))[seriesName]
+
+    def parseq_to_anim_series(self, seriesName):
+        
+        # Check if valus is present in first frame of JSON data. If not, assume it's undefined.
+        # The Parseq contract is that the first frame (at least) must define values for all fields.
+        try:
+            if self.rendered_frames[0][seriesName] is not None:
+                logging.info(f"Found {seriesName} in first frame of Parseq data. Assuming it's defined.")
+        except KeyError:
+            return None
+
+        key_frame_series = pd.Series([np.nan for a in range(self.required_frames)])
+        
+        for frame in self.rendered_frames:
+            frame_idx = frame['frame']
+            if frame_idx < self.required_frames:                
+                if not np.isnan(key_frame_series[frame_idx]):
+                    logging.warning(f"Duplicate frame definition {frame_idx} detected for data {seriesName}. Latest wins.")        
+                key_frame_series[frame_idx] = frame[seriesName]
+
+        # If the animation will have more frames than Parseq defines,
+        # duplicate final value to match the required frame count.
+        while (frame_idx < self.required_frames):
+            key_frame_series[frame_idx] = operator.itemgetter(-1)(self.rendered_frames)[seriesName]
+            frame_idx += 1
+
+        return key_frame_series
+
+    # fallback to anim_args if the series is not defined in the Parseq data
+    def __getattribute__(inst, name):
+        try:
+            definedField = super(ParseqAnimKeys, inst).__getattribute__(name)
+        except AttributeError:
+            # No field with this name has been explicitly extracted from the JSON data.
+            # It must be a new parameter. Let's see if it's in the raw JSON.
+
+            # parseq doesn't use _series, _schedule or _schedule_series suffixes in the
+            # JSON data - remove them.
+            strippableSuffixes = ['_series', '_schedule']
+            parseqName = name
+            while any(parseqName.endswith(suffix) for suffix in strippableSuffixes):
+                for suffix in strippableSuffixes:
+                    if parseqName.endswith(suffix):
+                        parseqName = parseqName[:-len(suffix)]            
+            
+            # returns None if not defined in Parseq JSON data
+            definedField = inst.parseq_to_anim_series(parseqName)
+            if (definedField is not None):
+                # add the field to the instance so we don't compute it again.
+                setattr(inst, name, definedField)
+
+        if (definedField is not None):
+            return definedField
+        else:
+            logging.info(f"Data for {name} not defined in Parseq data (looked for: '{parseqName}'). Falling back to standard Deforum values.")
+            return getattr(inst.default_anim_keys, name)
+

extensions/deforum/scripts/deforum_helpers/prompt.py

@@ -0,0 +1,113 @@
+import re
+
+def check_is_number(value):
+    float_pattern = r'^(?=.)([+-]?([0-9]*)(\.([0-9]+))?)$'
+    return re.match(float_pattern, value)
+
+def parse_weight(match, frame = 0)->float:
+    import numexpr
+    w_raw = match.group("weight")
+    if w_raw == None:
+        return 1
+    if check_is_number(w_raw):
+        return float(w_raw)
+    else:
+        t = frame
+        if len(w_raw) < 3:
+            print('the value inside `-characters cannot represent a math function')
+            return 1
+        return float(numexpr.evaluate(w_raw[1:-1]))
+
+def split_weighted_subprompts(text, frame = 0):
+    """
+    splits the prompt based on deforum webui implementation, moved from generate.py 
+    """
+    math_parser = re.compile("""
+            (?P<weight>(
+            `[\S\s]*?`# a math function wrapped in `-characters
+            ))
+            """, re.VERBOSE)
+    
+    parsed_prompt = re.sub(math_parser, lambda m: str(parse_weight(m, frame)), text)
+
+    negative_prompts = []
+    positive_prompts = []
+
+    prompt_split = parsed_prompt.split("--neg")
+    if len(prompt_split) > 1:
+        positive_prompts, negative_prompts = parsed_prompt.split("--neg") #TODO: add --neg to vanilla Deforum for compat
+    else:
+        positive_prompts = prompt_split[0]
+        negative_prompts = ""
+
+    return positive_prompts, negative_prompts
+
+def interpolate_prompts(animation_prompts, max_frames):
+    import numpy as np
+    import pandas as pd
+    # Get prompts sorted by keyframe 
+    sorted_prompts = sorted(animation_prompts.items(), key=lambda item: int(item[0]))
+
+    # Setup container for interpolated prompts
+    prompt_series = pd.Series([np.nan for a in range(max_frames)])
+
+    # For every keyframe prompt except the last
+    for i in range(0,len(sorted_prompts)-1):
+        
+        # Get current and next keyframe
+        current_frame = int(sorted_prompts[i][0])
+        next_frame = int(sorted_prompts[i+1][0])
+        
+        # Ensure there's no weird ordering issues or duplication in the animation prompts
+        # (unlikely because we sort above, and the json parser will strip dupes)
+        if current_frame>=next_frame:
+            print(f"WARNING: Sequential prompt keyframes {i}:{current_frame} and {i+1}:{next_frame} are not monotonously increasing; skipping interpolation.")
+            continue
+            
+        # Get current and next keyframes' positive and negative prompts (if any)
+        current_prompt = sorted_prompts[i][1]
+        next_prompt = sorted_prompts[i+1][1]
+        current_positive, current_negative, *_ = current_prompt.split("--neg") + [None]
+        next_positive, next_negative, *_ = next_prompt.split("--neg") + [None]
+        
+        # Calculate how much to shift the weight from current to next prompt at each frame
+        weight_step = 1/(next_frame-current_frame)
+        
+        # Apply weighted prompt interpolation for each frame between current and next keyframe
+        # using the syntax:  prompt1 :weight1 AND prompt1 :weight2 --neg nprompt1 :weight1 AND nprompt1 :weight2
+        # (See: https://github.com/AUTOMATIC1111/stable-diffusion-webui/wiki/Features#composable-diffusion )
+        for f in range(current_frame,next_frame):
+            next_weight = weight_step * (f-current_frame)
+            current_weight = 1 - next_weight
+            
+            # We will build the prompt incrementally depending on which prompts are present
+            prompt_series[f] = ''
+
+            # Cater for the case where neither, either or both current & next have positive prompts:
+            if current_positive:
+                prompt_series[f] += f"{current_positive} :{current_weight}"
+            if current_positive and next_positive:
+                prompt_series[f] += f" AND "
+            if next_positive:
+                prompt_series[f] += f"{next_positive} :{next_weight}"
+            
+            # Cater for the case where neither, either or both current & next have negative prompts:
+            if current_negative or next_negative:
+                prompt_series[f] += " --neg "
+                if current_negative:
+                    prompt_series[f] += f" {current_negative} :{current_weight}"
+                if current_negative and next_negative:
+                    prompt_series[f] += f" AND "
+                if next_negative:
+                    prompt_series[f] += f" {next_negative} :{next_weight}"
+    
+    # Set explicitly declared keyframe prompts (overwriting interpolated values at the keyframe idx). This ensures:
+    # - That final prompt is set, and
+    # - Gives us a chance to emit warnings if any keyframe prompts are already using composable diffusion
+    for i, prompt in animation_prompts.items():
+        prompt_series[int(i)] = prompt
+        if ' AND ' in prompt:
+            print(f"WARNING: keyframe {i}'s prompt is using composable diffusion (aka the 'AND' keyword). This will cause unexpected behaviour with interpolation.")
+    
+    # Return the filled series, in case max_frames is greater than the last keyframe or any ranges were skipped.
+    return prompt_series.ffill().bfill()

extensions/deforum/scripts/deforum_helpers/render.py

@@ -0,0 +1,507 @@
+import os
+import json
+import pandas as pd
+import cv2
+import numpy as np
+from PIL import Image, ImageOps
+from .rich import console
+
+from .generate import generate
+from .noise import add_noise
+from .animation import sample_from_cv2, sample_to_cv2, anim_frame_warp
+from .animation_key_frames import DeformAnimKeys, LooperAnimKeys
+from .video_audio_utilities import get_frame_name, get_next_frame
+from .depth import DepthModel
+from .colors import maintain_colors
+from .parseq_adapter import ParseqAnimKeys
+from .seed import next_seed
+from .blank_frame_reroll import blank_frame_reroll
+from .image_sharpening import unsharp_mask
+from .load_images import get_mask, load_img, get_mask_from_file
+from .hybrid_video import hybrid_generation, hybrid_composite
+from .hybrid_video import get_matrix_for_hybrid_motion, get_matrix_for_hybrid_motion_prev, get_flow_for_hybrid_motion, get_flow_for_hybrid_motion_prev, image_transform_ransac, image_transform_optical_flow
+from .save_images import save_image
+from .composable_masks import compose_mask_with_check
+from .settings import get_keys_to_exclude
+from .deforum_controlnet import unpack_controlnet_vids, is_controlnet_enabled
+# Webui
+from modules.shared import opts, cmd_opts, state, sd_model
+from modules import lowvram, devices, sd_hijack
+
+def render_animation(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, animation_prompts, root):
+    # handle hybrid video generation
+    if anim_args.animation_mode in ['2D','3D']:
+        if anim_args.hybrid_composite or anim_args.hybrid_motion in ['Affine', 'Perspective', 'Optical Flow']:
+            args, anim_args, inputfiles = hybrid_generation(args, anim_args, root)
+            # path required by hybrid functions, even if hybrid_comp_save_extra_frames is False
+            hybrid_frame_path = os.path.join(args.outdir, 'hybridframes')
+
+    # handle controlnet video input frames generation
+    if is_controlnet_enabled(controlnet_args):
+        unpack_controlnet_vids(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, animation_prompts, root)
+
+    # use parseq if manifest is provided
+    use_parseq = parseq_args.parseq_manifest != None and parseq_args.parseq_manifest.strip()
+    # expand key frame strings to values
+    keys = DeformAnimKeys(anim_args) if not use_parseq else ParseqAnimKeys(parseq_args, anim_args)
+    loopSchedulesAndData = LooperAnimKeys(loop_args, anim_args)
+    # resume animation
+    start_frame = 0
+    if anim_args.resume_from_timestring:
+        for tmp in os.listdir(args.outdir):
+            if ".txt" in tmp : 
+                pass
+            else:
+                filename = tmp.split("_")
+                # don't use saved depth maps to count number of frames
+                if anim_args.resume_timestring in filename and "depth" not in filename:
+                    start_frame += 1
+        #start_frame = start_frame - 1
+
+    # create output folder for the batch
+    os.makedirs(args.outdir, exist_ok=True)
+    print(f"Saving animation frames to:\n{args.outdir}")
+
+    # save settings for the batch
+    exclude_keys = get_keys_to_exclude('general')
+    settings_filename = os.path.join(args.outdir, f"{args.timestring}_settings.txt")
+    with open(settings_filename, "w+", encoding="utf-8") as f:
+        args.__dict__["prompts"] = animation_prompts
+        s = {}
+        for d in [dict(args.__dict__), dict(anim_args.__dict__), dict(parseq_args.__dict__), dict(loop_args.__dict__)]:
+            for key, value in d.items():
+                if key not in exclude_keys:
+                    s[key] = value
+        json.dump(s, f, ensure_ascii=False, indent=4)
+
+    # resume from timestring
+    if anim_args.resume_from_timestring:
+        args.timestring = anim_args.resume_timestring
+
+    # Always enable pseudo-3d with parseq. No need for an extra toggle:
+    # Whether it's used or not in practice is defined by the schedules
+    if use_parseq:
+        anim_args.flip_2d_perspective = True        
+
+    # expand prompts out to per-frame
+    if use_parseq:
+        prompt_series = keys.prompts
+    else:
+        prompt_series = pd.Series([np.nan for a in range(anim_args.max_frames)])
+        for i, prompt in animation_prompts.items():
+            prompt_series[int(i)] = prompt
+        prompt_series = prompt_series.ffill().bfill()
+
+    # check for video inits
+    using_vid_init = anim_args.animation_mode == 'Video Input'
+
+    # load depth model for 3D
+    predict_depths = (anim_args.animation_mode == '3D' and anim_args.use_depth_warping) or anim_args.save_depth_maps
+    predict_depths = predict_depths or (anim_args.hybrid_composite and anim_args.hybrid_comp_mask_type in ['Depth','Video Depth'])
+    if predict_depths:
+        depth_model = DepthModel('cpu' if cmd_opts.lowvram or cmd_opts.medvram else root.device)
+        depth_model.load_midas(root.models_path, root.half_precision)
+        if anim_args.midas_weight < 1.0:
+            depth_model.load_adabins(root.models_path)
+        # depth-based hybrid composite mask requires saved depth maps
+        if anim_args.hybrid_composite and anim_args.hybrid_comp_mask_type =='Depth':
+            anim_args.save_depth_maps = True
+    else:
+        depth_model = None
+        anim_args.save_depth_maps = False
+
+    # state for interpolating between diffusion steps
+    turbo_steps = 1 if using_vid_init else int(anim_args.diffusion_cadence)
+    turbo_prev_image, turbo_prev_frame_idx = None, 0
+    turbo_next_image, turbo_next_frame_idx = None, 0
+
+    # resume animation
+    prev_img = None
+    color_match_sample = None
+    if anim_args.resume_from_timestring:
+        last_frame = start_frame-1
+        if turbo_steps > 1:
+            last_frame -= last_frame%turbo_steps
+        path = os.path.join(args.outdir,f"{args.timestring}_{last_frame:05}.png")
+        img = cv2.imread(path)
+        #img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) # Changed the colors on resume
+        prev_img = img
+        if anim_args.color_coherence != 'None':
+            color_match_sample = img
+        if turbo_steps > 1:
+            turbo_next_image, turbo_next_frame_idx = prev_img, last_frame
+            turbo_prev_image, turbo_prev_frame_idx = turbo_next_image, turbo_next_frame_idx
+            start_frame = last_frame+turbo_steps
+
+    args.n_samples = 1
+    frame_idx = start_frame
+
+    # reset the mask vals as they are overwritten in the compose_mask algorithm
+    mask_vals = {}
+    noise_mask_vals = {}
+
+    mask_vals['everywhere'] = Image.new('1', (args.W, args.H), 1)
+    noise_mask_vals['everywhere'] = Image.new('1', (args.W, args.H), 1)
+
+    mask_image = None
+    
+    if args.use_init and args.init_image != None and args.init_image != '':
+        _, mask_image = load_img(args.init_image, 
+                                        shape=(args.W, args.H),  
+                                        use_alpha_as_mask=args.use_alpha_as_mask)
+        mask_vals['init_mask'] = mask_image
+        noise_mask_vals['init_mask'] = mask_image
+    
+    # Grab the first frame masks since they wont be provided until next frame
+    if mask_image is None and args.use_mask:
+        mask_vals['init_mask'] = get_mask(args)
+        noise_mask_vals['init_mask'] = get_mask(args) # TODO?: add a different default noise mask
+
+    if anim_args.use_mask_video:
+        mask_vals['video_mask'] = get_mask_from_file(get_next_frame(args.outdir, anim_args.video_mask_path, frame_idx, True), args)
+        noise_mask_vals['video_mask'] = get_mask_from_file(get_next_frame(args.outdir, anim_args.video_mask_path, frame_idx, True), args)
+    else:
+        mask_vals['video_mask'] = None
+        noise_mask_vals['video_mask'] = None
+
+    #Webui
+    state.job_count = anim_args.max_frames
+    
+    while frame_idx < anim_args.max_frames:
+        #Webui
+        state.job = f"frame {frame_idx + 1}/{anim_args.max_frames}"
+        state.job_no = frame_idx + 1
+        if state.interrupted:
+            break
+
+        print(f"\033[36mAnimation frame: \033[0m{frame_idx}/{anim_args.max_frames}  ")
+
+        noise = keys.noise_schedule_series[frame_idx]
+        strength = keys.strength_schedule_series[frame_idx]
+        scale = keys.cfg_scale_schedule_series[frame_idx]
+        contrast = keys.contrast_schedule_series[frame_idx]
+        kernel = int(keys.kernel_schedule_series[frame_idx])
+        sigma = keys.sigma_schedule_series[frame_idx]
+        amount = keys.amount_schedule_series[frame_idx]
+        threshold = keys.threshold_schedule_series[frame_idx]
+        hybrid_comp_schedules = {
+            "alpha": keys.hybrid_comp_alpha_schedule_series[frame_idx],
+            "mask_blend_alpha": keys.hybrid_comp_mask_blend_alpha_schedule_series[frame_idx],
+            "mask_contrast": keys.hybrid_comp_mask_contrast_schedule_series[frame_idx],
+            "mask_auto_contrast_cutoff_low": int(keys.hybrid_comp_mask_auto_contrast_cutoff_low_schedule_series[frame_idx]),
+            "mask_auto_contrast_cutoff_high": int(keys.hybrid_comp_mask_auto_contrast_cutoff_high_schedule_series[frame_idx]),
+        }        
+        scheduled_sampler_name = None
+        scheduled_clipskip = None
+        mask_seq = None
+        noise_mask_seq = None
+        if anim_args.enable_steps_scheduling and keys.steps_schedule_series[frame_idx] is not None:
+            args.steps = int(keys.steps_schedule_series[frame_idx])
+        if anim_args.enable_sampler_scheduling and keys.sampler_schedule_series[frame_idx] is not None:
+            scheduled_sampler_name = keys.sampler_schedule_series[frame_idx].casefold()
+        if anim_args.enable_clipskip_scheduling and keys.clipskip_schedule_series[frame_idx] is not None:
+            scheduled_clipskip = int(keys.clipskip_schedule_series[frame_idx])
+        if args.use_mask and keys.mask_schedule_series[frame_idx] is not None:
+            mask_seq = keys.mask_schedule_series[frame_idx]
+        if anim_args.use_noise_mask and keys.noise_mask_schedule_series[frame_idx] is not None:
+            noise_mask_seq = keys.noise_mask_schedule_series[frame_idx]
+        
+        if args.use_mask and not anim_args.use_noise_mask:
+            noise_mask_seq = mask_seq
+        
+        depth = None
+
+        if anim_args.animation_mode == '3D' and (cmd_opts.lowvram or cmd_opts.medvram):
+            # Unload the main checkpoint and load the depth model
+            lowvram.send_everything_to_cpu()
+            sd_hijack.model_hijack.undo_hijack(sd_model)
+            devices.torch_gc()
+            depth_model.to(root.device)
+        
+        # emit in-between frames
+        if turbo_steps > 1:
+            tween_frame_start_idx = max(0, frame_idx-turbo_steps)
+            for tween_frame_idx in range(tween_frame_start_idx, frame_idx):
+                tween = float(tween_frame_idx - tween_frame_start_idx + 1) / float(frame_idx - tween_frame_start_idx)
+                print(f" Creating in-between frame: {tween_frame_idx}; tween:{tween:0.2f};")
+
+                advance_prev = turbo_prev_image is not None and tween_frame_idx > turbo_prev_frame_idx
+                advance_next = tween_frame_idx > turbo_next_frame_idx
+
+                if depth_model is not None:
+                    assert(turbo_next_image is not None)
+                    depth = depth_model.predict(turbo_next_image, anim_args, root.half_precision)
+                
+                if advance_prev:
+                    turbo_prev_image, _ = anim_frame_warp(turbo_prev_image, args, anim_args, keys, tween_frame_idx, depth_model, depth=depth, device=root.device, half_precision=root.half_precision)
+                if advance_next:
+                    turbo_next_image, _ = anim_frame_warp(turbo_next_image, args, anim_args, keys, tween_frame_idx, depth_model, depth=depth, device=root.device, half_precision=root.half_precision)
+
+                # hybrid video motion - warps turbo_prev_image or turbo_next_image to match motion
+                if tween_frame_idx > 0:
+                    if anim_args.hybrid_motion in ['Affine', 'Perspective']:
+                        if anim_args.hybrid_motion_use_prev_img:
+                            if advance_prev:
+                                matrix = get_matrix_for_hybrid_motion_prev(tween_frame_idx, (args.W, args.H), inputfiles, turbo_prev_image, anim_args.hybrid_motion)
+                                turbo_prev_image = image_transform_ransac(turbo_prev_image, matrix, anim_args.hybrid_motion, cv2.BORDER_WRAP if anim_args.border == 'wrap' else cv2.BORDER_REPLICATE)
+                            if advance_next:
+                                matrix = get_matrix_for_hybrid_motion_prev(tween_frame_idx, (args.W, args.H), inputfiles, turbo_next_image, anim_args.hybrid_motion)
+                                turbo_next_image = image_transform_ransac(turbo_next_image, matrix, anim_args.hybrid_motion, cv2.BORDER_WRAP if anim_args.border == 'wrap' else cv2.BORDER_REPLICATE)
+                        else:
+                            matrix = get_matrix_for_hybrid_motion(tween_frame_idx-1, (args.W, args.H), inputfiles, anim_args.hybrid_motion)
+                            if advance_prev:
+                                turbo_prev_image = image_transform_ransac(turbo_prev_image, matrix, anim_args.hybrid_motion, cv2.BORDER_WRAP if anim_args.border == 'wrap' else cv2.BORDER_REPLICATE)
+                            if advance_next:
+                                turbo_next_image = image_transform_ransac(turbo_next_image, matrix, anim_args.hybrid_motion, cv2.BORDER_WRAP if anim_args.border == 'wrap' else cv2.BORDER_REPLICATE)
+                    if anim_args.hybrid_motion in ['Optical Flow']:
+                        if anim_args.hybrid_motion_use_prev_img:
+                            if advance_prev:
+                                flow = get_flow_for_hybrid_motion_prev(tween_frame_idx-1, (args.W, args.H), inputfiles, hybrid_frame_path, turbo_prev_image, anim_args.hybrid_flow_method, anim_args.hybrid_comp_save_extra_frames)
+                                turbo_prev_image = image_transform_optical_flow(turbo_prev_image, flow, cv2.BORDER_WRAP if anim_args.border == 'wrap' else cv2.BORDER_REPLICATE)
+                            if advance_next:
+                                flow = get_flow_for_hybrid_motion_prev(tween_frame_idx-1, (args.W, args.H), inputfiles, hybrid_frame_path, turbo_next_image, anim_args.hybrid_flow_method, anim_args.hybrid_comp_save_extra_frames)
+                                turbo_next_image = image_transform_optical_flow(turbo_next_image, flow, cv2.BORDER_WRAP if anim_args.border == 'wrap' else cv2.BORDER_REPLICATE)
+                        else:
+                            flow = get_flow_for_hybrid_motion(tween_frame_idx-1, (args.W, args.H), inputfiles, hybrid_frame_path, anim_args.hybrid_flow_method, anim_args.hybrid_comp_save_extra_frames)
+                            if advance_prev:
+                                turbo_prev_image = image_transform_optical_flow(turbo_prev_image, flow, cv2.BORDER_WRAP if anim_args.border == 'wrap' else cv2.BORDER_REPLICATE)
+                            if advance_next:
+                                turbo_next_image = image_transform_optical_flow(turbo_next_image, flow, cv2.BORDER_WRAP if anim_args.border == 'wrap' else cv2.BORDER_REPLICATE)
+                      
+                turbo_prev_frame_idx = turbo_next_frame_idx = tween_frame_idx
+
+                if turbo_prev_image is not None and tween < 1.0:
+                    img = turbo_prev_image*(1.0-tween) + turbo_next_image*tween
+                else:
+                    img = turbo_next_image
+
+                # intercept and override to grayscale
+                if anim_args.color_force_grayscale:
+                    img = cv2.cvtColor(img.astype(np.uint8), cv2.COLOR_BGR2GRAY)
+                    img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
+
+                filename = f"{args.timestring}_{tween_frame_idx:05}.png"
+                cv2.imwrite(os.path.join(args.outdir, filename), img)
+                if anim_args.save_depth_maps:
+                    depth_model.save(os.path.join(args.outdir, f"{args.timestring}_depth_{tween_frame_idx:05}.png"), depth)
+            if turbo_next_image is not None:
+                prev_img = turbo_next_image
+
+        # apply transforms to previous frame
+        if prev_img is not None:
+            prev_img, depth = anim_frame_warp(prev_img, args, anim_args, keys, frame_idx, depth_model, depth=None, device=root.device, half_precision=root.half_precision)
+
+            # hybrid video motion - warps prev_img to match motion, usually to prepare for compositing
+            if frame_idx > 0:
+                if anim_args.hybrid_motion in ['Affine', 'Perspective']:
+                    if anim_args.hybrid_motion_use_prev_img:
+                        matrix = get_matrix_for_hybrid_motion_prev(frame_idx, (args.W, args.H), inputfiles, prev_img, anim_args.hybrid_motion)
+                    else:
+                        matrix = get_matrix_for_hybrid_motion(frame_idx-1, (args.W, args.H), inputfiles, anim_args.hybrid_motion)
+                    prev_img = image_transform_ransac(prev_img, matrix, anim_args.hybrid_motion, cv2.BORDER_WRAP if anim_args.border == 'wrap' else cv2.BORDER_REPLICATE)    
+                if anim_args.hybrid_motion in ['Optical Flow']:
+                    if anim_args.hybrid_motion_use_prev_img:
+                        flow = get_flow_for_hybrid_motion_prev(frame_idx-1, (args.W, args.H), inputfiles, hybrid_frame_path, prev_img, anim_args.hybrid_flow_method, anim_args.hybrid_comp_save_extra_frames)
+                    else:
+                        flow = get_flow_for_hybrid_motion(frame_idx-1, (args.W, args.H), inputfiles, hybrid_frame_path, anim_args.hybrid_flow_method, anim_args.hybrid_comp_save_extra_frames)
+                    prev_img = image_transform_optical_flow(prev_img, flow, cv2.BORDER_WRAP if anim_args.border == 'wrap' else cv2.BORDER_REPLICATE)
+
+            # do hybrid video - composites video frame into prev_img (now warped if using motion)
+            if anim_args.hybrid_composite:
+                args, prev_img = hybrid_composite(args, anim_args, frame_idx, prev_img, depth_model, hybrid_comp_schedules, root)
+
+            # apply color matching
+            if anim_args.color_coherence != 'None':
+                # video color matching
+                hybrid_available = anim_args.hybrid_composite or anim_args.hybrid_motion in ['Optical Flow', 'Affine', 'Perspective']
+                if anim_args.color_coherence == 'Video Input' and hybrid_available:
+                    video_color_coherence_frame = int(frame_idx) % int(anim_args.color_coherence_video_every_N_frames) == 0
+                    if video_color_coherence_frame:
+                        prev_vid_img = Image.open(os.path.join(args.outdir, 'inputframes', get_frame_name(anim_args.video_init_path) + f"{frame_idx:05}.jpg"))
+                        prev_vid_img = prev_vid_img.resize((args.W, args.H), Image.Resampling.LANCZOS)
+                        color_match_sample = np.asarray(prev_vid_img)
+                        color_match_sample = cv2.cvtColor(color_match_sample, cv2.COLOR_RGB2BGR)
+                if color_match_sample is None:
+                    color_match_sample = prev_img.copy()
+                else:
+                    prev_img = maintain_colors(prev_img, color_match_sample, anim_args.color_coherence)
+
+            # intercept and override to grayscale
+            if anim_args.color_force_grayscale:
+                prev_img = cv2.cvtColor(prev_img, cv2.COLOR_BGR2GRAY)
+                prev_img = cv2.cvtColor(prev_img, cv2.COLOR_GRAY2BGR)
+
+            # apply scaling
+            contrast_image = (prev_img * contrast).round().astype(np.uint8)
+            # anti-blur
+            if amount > 0:
+                contrast_image = unsharp_mask(contrast_image, (kernel, kernel), sigma, amount, threshold, mask_image if args.use_mask else None)
+            # apply frame noising
+            if args.use_mask or anim_args.use_noise_mask:
+                args.noise_mask = compose_mask_with_check(root, args, noise_mask_seq, noise_mask_vals, Image.fromarray(cv2.cvtColor(contrast_image, cv2.COLOR_BGR2RGB)))
+            noised_image = add_noise(contrast_image, noise, args.seed, anim_args.noise_type,
+                            (anim_args.perlin_w, anim_args.perlin_h, anim_args.perlin_octaves, anim_args.perlin_persistence),
+                             args.noise_mask, args.invert_mask)
+
+            # use transformed previous frame as init for current
+            args.use_init = True
+            args.init_sample = Image.fromarray(cv2.cvtColor(noised_image, cv2.COLOR_BGR2RGB))
+            args.strength = max(0.0, min(1.0, strength))
+        
+        args.scale = scale
+       
+        # Pix2Pix Image CFG Scale - does *nothing* with non pix2pix checkpoints
+        args.pix2pix_img_cfg_scale = float(keys.pix2pix_img_cfg_scale_series[frame_idx])
+
+        # grab prompt for current frame
+        args.prompt = prompt_series[frame_idx]
+        
+        if args.seed_behavior == 'schedule' or use_parseq:
+            args.seed = int(keys.seed_schedule_series[frame_idx])
+
+        if anim_args.enable_checkpoint_scheduling:
+            args.checkpoint = keys.checkpoint_schedule_series[frame_idx]
+        else:
+            args.checkpoint = None
+            
+        #SubSeed scheduling
+        if anim_args.enable_subseed_scheduling:
+            args.subseed = int(keys.subseed_schedule_series[frame_idx])
+            args.subseed_strength = float(keys.subseed_strength_schedule_series[frame_idx])
+
+        if use_parseq:
+            args.seed_enable_extras = True
+            args.subseed = int(keys.subseed_series[frame_idx])
+            args.subseed_strength = keys.subseed_strength_series[frame_idx]
+            
+        prompt_to_print, *after_neg = args.prompt.strip().split("--neg")
+        prompt_to_print = prompt_to_print.strip()
+        after_neg = "".join(after_neg).strip()
+
+        print(f"\033[32mSeed: \033[0m{args.seed}")
+        print(f"\033[35mPrompt: \033[0m{prompt_to_print}")
+        if after_neg and after_neg.strip():
+            print(f"\033[91mNeg Prompt: \033[0m{after_neg}")
+        if not using_vid_init:
+            # print motion table to cli if anim mode = 2D or 3D
+            if anim_args.animation_mode in ['2D','3D']:
+                print_render_table(anim_args, keys, frame_idx)
+
+        # grab init image for current frame
+        elif using_vid_init:
+            init_frame = get_next_frame(args.outdir, anim_args.video_init_path, frame_idx, False)
+            print(f"Using video init frame {init_frame}")
+            args.init_image = init_frame
+        if anim_args.use_mask_video:
+            mask_vals['video_mask'] = get_mask_from_file(get_next_frame(args.outdir, anim_args.video_mask_path, frame_idx, True), args)
+
+        if args.use_mask:
+            args.mask_image = compose_mask_with_check(root, args, mask_seq, mask_vals, args.init_sample) if args.init_sample is not None else None # we need it only after the first frame anyway
+
+        # setting up some arguments for the looper
+        loop_args.imageStrength = loopSchedulesAndData.image_strength_schedule_series[frame_idx]
+        loop_args.blendFactorMax = loopSchedulesAndData.blendFactorMax_series[frame_idx]
+        loop_args.blendFactorSlope = loopSchedulesAndData.blendFactorSlope_series[frame_idx]
+        loop_args.tweeningFrameSchedule = loopSchedulesAndData.tweening_frames_schedule_series[frame_idx]
+        loop_args.colorCorrectionFactor = loopSchedulesAndData.color_correction_factor_series[frame_idx]
+        loop_args.use_looper = loopSchedulesAndData.use_looper
+        loop_args.imagesToKeyframe = loopSchedulesAndData.imagesToKeyframe
+        
+        if scheduled_clipskip is not None:
+            opts.data["CLIP_stop_at_last_layers"] = scheduled_clipskip
+        
+        if anim_args.animation_mode == '3D' and (cmd_opts.lowvram or cmd_opts.medvram):
+            depth_model.to('cpu')
+            devices.torch_gc()
+            lowvram.setup_for_low_vram(sd_model, cmd_opts.medvram)
+            sd_hijack.model_hijack.hijack(sd_model)
+        
+        # sample the diffusion model
+        image = generate(args, anim_args, loop_args, controlnet_args, root, frame_idx, sampler_name=scheduled_sampler_name)
+        patience = 10
+
+        # intercept and override to grayscale
+        if anim_args.color_force_grayscale:
+            image = ImageOps.grayscale(image)
+            image = ImageOps.colorize(image, black ="black", white ="white")
+
+        # reroll blank frame 
+        if not image.getbbox():
+            print("Blank frame detected! If you don't have the NSFW filter enabled, this may be due to a glitch!")
+            if args.reroll_blank_frames == 'reroll':
+                while not image.getbbox():
+                    print("Rerolling with +1 seed...")
+                    args.seed += 1
+                    image = generate(args, anim_args, loop_args, controlnet_args, root, frame_idx, sampler_name=scheduled_sampler_name)
+                    patience -= 1
+                    if patience == 0:
+                        print("Rerolling with +1 seed failed for 10 iterations! Try setting webui's precision to 'full' and if it fails, please report this to the devs! Interrupting...")
+                        state.interrupted = True
+                        state.current_image = image
+                        return
+            elif args.reroll_blank_frames == 'interrupt':
+                print("Interrupting to save your eyes...")
+                state.interrupted = True
+                state.current_image = image
+            image = blank_frame_reroll(image, args, root, frame_idx)
+            if image == None:
+                return
+
+        opencv_image = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
+        if not using_vid_init:
+            prev_img = opencv_image
+
+        if turbo_steps > 1:
+            turbo_prev_image, turbo_prev_frame_idx = turbo_next_image, turbo_next_frame_idx
+            turbo_next_image, turbo_next_frame_idx = opencv_image, frame_idx
+            frame_idx += turbo_steps
+        else:    
+            filename = f"{args.timestring}_{frame_idx:05}.png"
+            save_image(image, 'PIL', filename, args, video_args, root)
+
+            if anim_args.save_depth_maps:
+                if cmd_opts.lowvram or cmd_opts.medvram:
+                    lowvram.send_everything_to_cpu()
+                    sd_hijack.model_hijack.undo_hijack(sd_model)
+                    devices.torch_gc()
+                    depth_model.to(root.device)
+                depth = depth_model.predict(opencv_image, anim_args, root.half_precision)
+                depth_model.save(os.path.join(args.outdir, f"{args.timestring}_depth_{frame_idx:05}.png"), depth)
+                if cmd_opts.lowvram or cmd_opts.medvram:
+                    depth_model.to('cpu')
+                    devices.torch_gc()
+                    lowvram.setup_for_low_vram(sd_model, cmd_opts.medvram)
+                    sd_hijack.model_hijack.hijack(sd_model)
+            frame_idx += 1
+
+        state.current_image = image
+
+        args.seed = next_seed(args)
+
+def print_render_table(anim_args, keys, frame_idx):
+    from rich.table import Table
+    from rich import box
+    table = Table(padding=0, box=box.ROUNDED)
+    field_names = []
+    if anim_args.animation_mode == '2D':
+        short_zoom = round(keys.zoom_series[frame_idx], 6)
+        field_names += ["Angle", "Zoom"]
+    field_names += ["Tr X", "Tr Y"]
+    if anim_args.animation_mode == '3D':
+        field_names += ["Tr Z", "Ro X", "Ro Y", "Ro Z"]
+    if anim_args.enable_perspective_flip:
+        field_names += ["Pf T", "Pf P", "Pf G", "Pf F"]
+    for field_name in field_names:
+        table.add_column(field_name, justify="center")
+    
+    rows = []
+    if anim_args.animation_mode == '2D':
+        rows += [str(keys.angle_series[frame_idx]),str(short_zoom)]
+    rows += [str(keys.translation_x_series[frame_idx]),str(keys.translation_y_series[frame_idx])]
+    if anim_args.animation_mode == '3D':
+        rows += [str(keys.translation_z_series[frame_idx]),str(keys.rotation_3d_x_series[frame_idx]),str(keys.rotation_3d_y_series[frame_idx]),str(keys.rotation_3d_z_series[frame_idx])]
+    if anim_args.enable_perspective_flip:
+        rows +=[str(keys.perspective_flip_theta_series[frame_idx]), str(keys.perspective_flip_phi_series[frame_idx]), str(keys.perspective_flip_gamma_series[frame_idx]), str(keys.perspective_flip_fv_series[frame_idx])]
+    table.add_row(*rows)
+    
+    console.print(table)

extensions/deforum/scripts/deforum_helpers/render_modes.py

@@ -0,0 +1,154 @@
+import os
+import pathlib
+import json
+from .render import render_animation
+from .seed import next_seed
+from .video_audio_utilities import vid2frames
+from .prompt import interpolate_prompts
+from .generate import generate
+from .animation_key_frames import DeformAnimKeys
+from .parseq_adapter import ParseqAnimKeys
+from .save_images import save_image
+from .settings import get_keys_to_exclude
+
+# Webui
+from modules.shared import opts, cmd_opts, state
+
+def render_input_video(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, animation_prompts, root):
+    # create a folder for the video input frames to live in
+    video_in_frame_path = os.path.join(args.outdir, 'inputframes') 
+    os.makedirs(video_in_frame_path, exist_ok=True)
+
+    # save the video frames from input video
+    print(f"Exporting Video Frames (1 every {anim_args.extract_nth_frame}) frames to {video_in_frame_path}...")
+    vid2frames(video_path = anim_args.video_init_path, video_in_frame_path=video_in_frame_path, n=anim_args.extract_nth_frame, overwrite=anim_args.overwrite_extracted_frames, extract_from_frame=anim_args.extract_from_frame, extract_to_frame=anim_args.extract_to_frame)
+
+    # determine max frames from length of input frames
+    anim_args.max_frames = len([f for f in pathlib.Path(video_in_frame_path).glob('*.jpg')])
+    args.use_init = True
+    print(f"Loading {anim_args.max_frames} input frames from {video_in_frame_path} and saving video frames to {args.outdir}")
+
+    if anim_args.use_mask_video:
+        # create a folder for the mask video input frames to live in
+        mask_in_frame_path = os.path.join(args.outdir, 'maskframes') 
+        os.makedirs(mask_in_frame_path, exist_ok=True)
+
+        # save the video frames from mask video
+        print(f"Exporting Video Frames (1 every {anim_args.extract_nth_frame}) frames to {mask_in_frame_path}...")
+        vid2frames(video_path=anim_args.video_mask_path,video_in_frame_path=mask_in_frame_path, n=anim_args.extract_nth_frame, overwrite=anim_args.overwrite_extracted_frames, extract_from_frame=anim_args.extract_from_frame, extract_to_frame=anim_args.extract_to_frame)
+        max_mask_frames = len([f for f in pathlib.Path(mask_in_frame_path).glob('*.jpg')])
+
+        # limit max frames if there are less frames in the video mask compared to input video
+        if max_mask_frames < anim_args.max_frames :
+            anim_args.max_mask_frames
+            print ("Video mask contains less frames than init video, max frames limited to number of mask frames.")
+        args.use_mask = True
+        args.overlay_mask = True
+
+
+    render_animation(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, animation_prompts, root)
+
+# Modified a copy of the above to allow using masking video with out a init video.
+def render_animation_with_video_mask(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, animation_prompts, root):
+    # create a folder for the video input frames to live in
+    mask_in_frame_path = os.path.join(args.outdir, 'maskframes') 
+    os.makedirs(mask_in_frame_path, exist_ok=True)
+
+    # save the video frames from mask video
+    print(f"Exporting Video Frames (1 every {anim_args.extract_nth_frame}) frames to {mask_in_frame_path}...")
+    vid2frames(video_path=anim_args.video_mask_path, video_in_frame_path=mask_in_frame_path, n=anim_args.extract_nth_frame, overwrite=anim_args.overwrite_extracted_frames, extract_from_frame=anim_args.extract_from_frame, extract_to_frame=anim_args.extract_to_frame)
+    args.use_mask = True
+    #args.overlay_mask = True
+
+    # determine max frames from length of input frames
+    anim_args.max_frames = len([f for f in pathlib.Path(mask_in_frame_path).glob('*.jpg')])
+    #args.use_init = True
+    print(f"Loading {anim_args.max_frames} input frames from {mask_in_frame_path} and saving video frames to {args.outdir}")
+
+    render_animation(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, animation_prompts, root)
+
+
+def render_interpolation(args, anim_args, video_args, parseq_args, loop_args, controlnet_args, animation_prompts, root):
+
+    # use parseq if manifest is provided
+    use_parseq = parseq_args.parseq_manifest != None and parseq_args.parseq_manifest.strip()
+
+    # expand key frame strings to values
+    keys = DeformAnimKeys(anim_args) if not use_parseq else ParseqAnimKeys(parseq_args, anim_args)
+
+    # create output folder for the batch
+    os.makedirs(args.outdir, exist_ok=True)
+    print(f"Saving interpolation animation frames to {args.outdir}")
+
+    # save settings for the batch
+    exclude_keys = get_keys_to_exclude('general')
+    settings_filename = os.path.join(args.outdir, f"{args.timestring}_settings.txt")
+    with open(settings_filename, "w+", encoding="utf-8") as f:
+        s = {}
+        for d in [dict(args.__dict__), dict(anim_args.__dict__), dict(parseq_args.__dict__)]:
+            for key, value in d.items():
+                if key not in exclude_keys:
+                    s[key] = value
+        json.dump(s, f, ensure_ascii=False, indent=4)
+
+    # Compute interpolated prompts
+    if use_parseq:
+        print("Parseq prompts are assumed to already be interpolated - not doing any additional prompt interpolation")
+        prompt_series = keys.prompts
+    else: 
+        print("Generating interpolated prompts for all frames")
+        prompt_series = interpolate_prompts(animation_prompts, anim_args.max_frames)
+    
+    state.job_count = anim_args.max_frames
+    frame_idx = 0
+    # INTERPOLATION MODE
+    while frame_idx < anim_args.max_frames:
+        # print data to cli
+        prompt_to_print = prompt_series[frame_idx].strip()
+        if prompt_to_print.endswith("--neg"):
+            prompt_to_print = prompt_to_print[:-5]
+        print(f"\033[36mInterpolation frame: \033[0m{frame_idx}/{anim_args.max_frames}  ")
+        print(f"\033[32mSeed: \033[0m{args.seed}")
+        print(f"\033[35mPrompt: \033[0m{prompt_to_print}")
+        
+        state.job = f"frame {frame_idx + 1}/{anim_args.max_frames}"
+        state.job_no = frame_idx + 1
+        
+        if state.interrupted:
+                break
+        
+        # grab inputs for current frame generation
+        args.n_samples = 1
+        args.prompt = prompt_series[frame_idx]
+        args.scale = keys.cfg_scale_schedule_series[frame_idx]
+        args.pix2pix_img_cfg_scale = keys.pix2pix_img_cfg_scale_series[frame_idx]
+        
+        if anim_args.enable_checkpoint_scheduling:
+            args.checkpoint = keys.checkpoint_schedule_series[frame_idx]
+            print(f"Checkpoint changed to: {args.checkpoint}")
+        else:
+            args.checkpoint = None
+            
+        if anim_args.enable_subseed_scheduling:
+            args.subseed = keys.subseed_schedule_series[frame_idx]
+            args.subseed_strength = keys.subseed_strength_schedule_series[frame_idx]
+            
+        if use_parseq:
+            anim_args.enable_subseed_scheduling = True
+            args.subseed = int(keys.subseed_series[frame_idx])
+            args.subseed_strength = keys.subseed_strength_series[frame_idx]
+            
+        if args.seed_behavior == 'schedule' or use_parseq:
+            args.seed = int(keys.seed_schedule_series[frame_idx])
+        
+        image = generate(args, anim_args, loop_args, controlnet_args, root, frame_idx)
+        filename = f"{args.timestring}_{frame_idx:05}.png"
+
+        save_image(image, 'PIL', filename, args, video_args, root)
+
+        state.current_image = image
+        
+        if args.seed_behavior != 'schedule':
+            args.seed = next_seed(args)
+
+        frame_idx += 1

extensions/deforum/scripts/deforum_helpers/rich.py

@@ -0,0 +1,2 @@
+from rich.console import Console
+console = Console()

extensions/deforum/scripts/deforum_helpers/save_images.py

@@ -0,0 +1,80 @@
+from typing import List, Tuple
+from einops import rearrange
+import numpy as np, os, torch
+from PIL import Image
+from torchvision.utils import make_grid
+import time
+
+
+def get_output_folder(output_path, batch_folder):
+    out_path = os.path.join(output_path,time.strftime('%Y-%m'))
+    if batch_folder != "":
+        out_path = os.path.join(out_path, batch_folder)
+    os.makedirs(out_path, exist_ok=True)
+    return out_path
+
+
+def save_samples(
+    args, x_samples: torch.Tensor, seed: int, n_rows: int
+) -> Tuple[Image.Image, List[Image.Image]]:
+    """Function to save samples to disk.
+    Args:
+        args: Stable deforum diffusion arguments.
+        x_samples: Samples to save.
+        seed: Seed for the experiment.
+        n_rows: Number of rows in the grid.
+    Returns:
+        A tuple of the grid image and a list of the generated images.
+        ( grid_image, generated_images )
+    """
+
+    # save samples
+    images = []
+    grid_image = None
+    if args.display_samples or args.save_samples:
+        for index, x_sample in enumerate(x_samples):
+            x_sample = 255.0 * rearrange(x_sample.cpu().numpy(), "c h w -> h w c")
+            images.append(Image.fromarray(x_sample.astype(np.uint8)))
+            if args.save_samples:
+                images[-1].save(
+                    os.path.join(
+                        args.outdir, f"{args.timestring}_{index:02}_{seed}.png"
+                    )
+                )
+
+    # save grid
+    if args.display_grid or args.save_grid:
+        grid = torch.stack([x_samples], 0)
+        grid = rearrange(grid, "n b c h w -> (n b) c h w")
+        grid = make_grid(grid, nrow=n_rows, padding=0)
+
+        # to image
+        grid = 255.0 * rearrange(grid, "c h w -> h w c").cpu().numpy()
+        grid_image = Image.fromarray(grid.astype(np.uint8))
+        if args.save_grid:
+            grid_image.save(
+                os.path.join(args.outdir, f"{args.timestring}_{seed}_grid.png")
+            )
+
+    # return grid_image and individual sample images
+    return grid_image, images
+
+def save_image(image, image_type, filename, args, video_args, root):
+    if video_args.store_frames_in_ram:
+        root.frames_cache.append({'path':os.path.join(args.outdir, filename), 'image':image, 'image_type':image_type})
+    else:
+        image.save(os.path.join(args.outdir, filename))
+
+import cv2, gc
+
+def reset_frames_cache(root):
+    root.frames_cache = []
+    gc.collect()
+
+def dump_frames_cache(root):
+    for image_cache in root.frames_cache:
+        if image_cache['image_type'] == 'cv2':
+            cv2.imwrite(image_cache['path'], image_cache['image'])
+        elif image_cache['image_type'] == 'PIL':
+            image_cache['image'].save(image_cache['path'])
+    # do not reset the cache since we're going to add frame erasing later function #TODO 

extensions/deforum/scripts/deforum_helpers/seed.py

@@ -0,0 +1,26 @@
+import random
+
+def next_seed(args):
+    if args.seed_behavior == 'iter':
+        if args.seed_internal % args.seed_iter_N == 0:
+            args.seed += 1
+        args.seed_internal += 1
+    elif args.seed_behavior == 'ladder':
+        if args.seed_internal == 0:
+            args.seed += 2
+            args.seed_internal = 1
+        else:
+            args.seed -= 1
+            args.seed_internal = 0
+    elif args.seed_behavior == 'alternate':
+        if args.seed_internal == 0:
+            args.seed += 1
+            args.seed_internal = 1
+        else:
+            args.seed -= 1
+            args.seed_internal = 0
+    elif args.seed_behavior == 'fixed':
+        pass # always keep seed the same
+    else:
+        args.seed = random.randint(0, 2**32 - 1)
+    return args.seed

extensions/deforum/scripts/deforum_helpers/settings.py

@@ -0,0 +1,272 @@
+from math import ceil
+import os
+import json
+import deforum_helpers.args as deforum_args
+from .args import mask_fill_choices, DeforumArgs, DeforumAnimArgs
+from .deprecation_utils import handle_deprecated_settings
+import logging
+
+def get_keys_to_exclude(setting_type):
+    if setting_type == 'general':
+        return ["n_batch", "restore_faces", "seed_enable_extras", "save_samples", "display_samples", "show_sample_per_step", "filename_format", "from_img2img_instead_of_link", "scale", "subseed", "subseed_strength", "C", "f", "init_latent", "init_sample", "init_c", "noise_mask", "seed_internal"]
+    else: #video
+        return ["mp4_path", "image_path", "output_format","render_steps","path_name_modifier"]
+
+def load_args(args_dict,anim_args_dict, parseq_args_dict, loop_args_dict, controlnet_args_dict, custom_settings_file, root):
+    print(f"reading custom settings from {custom_settings_file}")
+    if not os.path.isfile(custom_settings_file):
+        print('The custom settings file does not exist. The in-notebook settings will be used instead')
+    else:
+        with open(custom_settings_file, "r") as f:
+            jdata = json.loads(f.read())
+            handle_deprecated_settings(jdata)
+            root.animation_prompts = jdata["prompts"]
+            if "animation_prompts_positive" in jdata:
+                root.animation_prompts_positive = jdata["animation_prompts_positive"]
+            if "animation_prompts_negative" in jdata:
+                root.animation_prompts_negative = jdata["animation_prompts_negative"]
+            for i, k in enumerate(args_dict):
+                if k in jdata:
+                    args_dict[k] = jdata[k]
+                else:
+                    print(f"key {k} doesn't exist in the custom settings data! using the default value of {args_dict[k]}")
+            for i, k in enumerate(anim_args_dict):
+                if k in jdata:
+                    anim_args_dict[k] = jdata[k]
+                else:
+                    print(f"key {k} doesn't exist in the custom settings data! using the default value of {anim_args_dict[k]}")
+            for i, k in enumerate(parseq_args_dict):
+                if k in jdata:
+                    parseq_args_dict[k] = jdata[k]
+                else:
+                    print(f"key {k} doesn't exist in the custom settings data! using the default value of {parseq_args_dict[k]}")                    
+            for i, k in enumerate(loop_args_dict):
+                if k in jdata:
+                    loop_args_dict[k] = jdata[k]
+                else:
+                    print(f"key {k} doesn't exist in the custom settings data! using the default value of {loop_args_dict[k]}")                    
+            print(args_dict)
+            print(anim_args_dict)
+            print(parseq_args_dict)
+            print(loop_args_dict)
+
+# In gradio gui settings save/ load funs:
+def save_settings(*args, **kwargs):
+    settings_path = args[0].strip()
+    data = {deforum_args.settings_component_names[i]: args[i+1] for i in range(0, len(deforum_args.settings_component_names))}
+    from deforum_helpers.args import pack_args, pack_anim_args, pack_parseq_args, pack_loop_args, pack_controlnet_args
+    args_dict = pack_args(data)
+    anim_args_dict = pack_anim_args(data)
+    parseq_dict = pack_parseq_args(data)
+    args_dict["prompts"] = json.loads(data['animation_prompts'])
+    args_dict["animation_prompts_positive"] = data['animation_prompts_positive']
+    args_dict["animation_prompts_negative"] = data['animation_prompts_negative']
+    loop_dict = pack_loop_args(data)
+    controlnet_dict = pack_controlnet_args(data)
+    
+    combined = {**args_dict, **anim_args_dict, **parseq_dict, **loop_dict, **controlnet_dict}
+    exclude_keys = get_keys_to_exclude('general') + ['controlnet_input_video_chosen_file', 'controlnet_input_video_mask_chosen_file']
+    filtered_combined = {k: v for k, v in combined.items() if k not in exclude_keys}
+    
+    print(f"saving custom settings to {settings_path}")
+    with open(settings_path, "w") as f:
+        f.write(json.dumps(filtered_combined, ensure_ascii=False, indent=4))
+    
+    return [""]
+
+def save_video_settings(*args, **kwargs):
+    video_settings_path = args[0].strip()
+    data = {deforum_args.video_args_names[i]: args[i+1] for i in range(0, len(deforum_args.video_args_names))}
+    from deforum_helpers.args import pack_video_args
+    video_args_dict = pack_video_args(data)
+    exclude_keys = get_keys_to_exclude('video')
+    filtered_data = video_args_dict if exclude_keys is None else {k: v for k, v in video_args_dict.items() if k not in exclude_keys}
+    print(f"saving video settings to {video_settings_path}")
+    with open(video_settings_path, "w") as f:
+        f.write(json.dumps(filtered_data, ensure_ascii=False, indent=4))
+    return [""]
+
+def load_settings(*args, **kwargs):
+    settings_path = args[0].strip()
+    data = {deforum_args.settings_component_names[i]: args[i+1] for i in range(0, len(deforum_args.settings_component_names))}
+    print(f"reading custom settings from {settings_path}")
+    jdata = {}
+    if not os.path.isfile(settings_path):
+        print('The custom settings file does not exist. The values will be unchanged.')
+        return [data[name] for name in deforum_args.settings_component_names] + [""]
+    else:
+        with open(settings_path, "r") as f:
+            jdata = json.loads(f.read())
+            handle_deprecated_settings(jdata)
+    ret = []
+    if 'animation_prompts' in jdata:
+        jdata['prompts'] = jdata['animation_prompts']#compatibility with old versions
+    if 'animation_prompts_positive' in jdata:
+        data["animation_prompts_positive"] = jdata['animation_prompts_positive']
+    if 'animation_prompts_negative' in jdata:
+        data["animation_prompts_negative"] = jdata['animation_prompts_negative']
+    for key in data:
+        if key == 'sampler':
+            sampler_val = jdata[key]
+            if type(sampler_val) == int:
+                from modules.sd_samplers import samplers_for_img2img
+                ret.append(samplers_for_img2img[sampler_val].name)
+            else:
+                ret.append(sampler_val)
+        
+        elif key == 'fill':
+            if key in jdata:
+                fill_val = jdata[key]
+                if type(fill_val) == int:                    
+                    ret.append(mask_fill_choices[fill_val])
+                else:
+                    ret.append(fill_val)
+            else:
+                fill_default = DeforumArgs()['fill']
+                logging.debug(f"Fill not found in load file, using default value: {fill_default}")
+                ret.append(mask_fill_choices[fill_default])
+        
+        elif key == 'reroll_blank_frames':
+            if key in jdata:
+                reroll_blank_frames_val = jdata[key]
+                ret.append(reroll_blank_frames_val)
+            else:
+                reroll_blank_frames_default = DeforumArgs()['reroll_blank_frames']
+                logging.debug(f"Reroll blank frames not found in load file, using default value: {reroll_blank_frames_default}")
+                ret.append(reroll_blank_frames_default)
+        
+        elif key == 'noise_type':
+            if key in jdata:
+                noise_type_val = jdata[key]
+                ret.append(noise_type_val)
+            else:
+                noise_type_default = DeforumAnimArgs()['noise_type']
+                logging.debug(f"Noise type not found in load file, using default value: {noise_type_default}")
+                ret.append(noise_type_default)
+            
+        elif key in jdata:
+            ret.append(jdata[key])
+        else:
+            if key == 'animation_prompts':
+                ret.append(json.dumps(jdata['prompts'], ensure_ascii=False, indent=4))
+            elif key == 'animation_prompts_positive' and 'animation_prompts_positive' in jdata:
+                ret.append(jdata['animation_prompts_positive'])
+            elif key == 'animation_prompts_negative' and 'animation_prompts_negative' in jdata:
+                ret.append(jdata['animation_prompts_negative'])
+            else:
+                ret.append(data[key])
+
+    #stuff
+    ret.append("")
+
+    return ret
+
+def load_video_settings(*args, **kwargs):
+    video_settings_path = args[0].strip()
+    data = {deforum_args.video_args_names[i]: args[i+1] for i in range(0, len(deforum_args.video_args_names))}
+    print(f"reading custom video settings from {video_settings_path}")
+    jdata = {}
+    if not os.path.isfile(video_settings_path):
+        print('The custom video settings file does not exist. The values will be unchanged.')
+        return [data[name] for name in deforum_args.video_args_names] + [""]
+    else:
+        with open(video_settings_path, "r") as f:
+            jdata = json.loads(f.read())
+            handle_deprecated_settings(jdata)
+    ret = []
+
+    for key in data:
+        if key == 'add_soundtrack':
+            add_soundtrack_val = jdata[key]
+            if type(add_soundtrack_val) == bool:
+                ret.append('File' if add_soundtrack_val else 'None')
+            else:
+                ret.append(add_soundtrack_val)
+        elif key in jdata:
+            ret.append(jdata[key])
+        else:
+            ret.append(data[key])
+    
+    #stuff
+    ret.append("")
+    
+    return ret
+
+import tqdm
+from modules.shared import state, progress_print_out, opts, cmd_opts
+class DeforumTQDM:
+    def __init__(self, args, anim_args, parseq_args):
+        self._tqdm = None
+        self._args = args
+        self._anim_args = anim_args
+        self._parseq_args = parseq_args
+
+    def reset(self):
+        from .animation_key_frames import DeformAnimKeys
+        from .parseq_adapter import ParseqAnimKeys
+        deforum_total = 0
+        # FIXME: get only amount of steps
+        use_parseq = self._parseq_args.parseq_manifest != None and self._parseq_args.parseq_manifest.strip()
+        keys = DeformAnimKeys(self._anim_args) if not use_parseq else ParseqAnimKeys(self._parseq_args, self._anim_args)        
+        
+        start_frame = 0
+        if self._anim_args.resume_from_timestring:
+            for tmp in os.listdir(self._args.outdir):
+                filename = tmp.split("_")
+                # don't use saved depth maps to count number of frames
+                if self._anim_args.resume_timestring in filename and "depth" not in filename:
+                    start_frame += 1
+            start_frame = start_frame - 1
+        using_vid_init = self._anim_args.animation_mode == 'Video Input'
+        turbo_steps = 1 if using_vid_init else int(self._anim_args.diffusion_cadence)
+        if self._anim_args.resume_from_timestring:
+            last_frame = start_frame-1
+            if turbo_steps > 1:
+                last_frame -= last_frame%turbo_steps
+            if turbo_steps > 1:
+                turbo_next_frame_idx = last_frame
+                turbo_prev_frame_idx = turbo_next_frame_idx
+                start_frame = last_frame+turbo_steps
+        frame_idx = start_frame
+        had_first = False
+        while frame_idx < self._anim_args.max_frames:
+            strength = keys.strength_schedule_series[frame_idx]
+            if not had_first and self._args.use_init and self._args.init_image != None and self._args.init_image != '':
+                deforum_total += int(ceil(self._args.steps * (1-strength)))
+                had_first = True
+            elif not had_first:
+                deforum_total += self._args.steps
+                had_first = True
+            else:
+                deforum_total += int(ceil(self._args.steps * (1-strength)))
+
+            if turbo_steps > 1:
+                frame_idx += turbo_steps
+            else:
+                frame_idx += 1
+        
+        self._tqdm = tqdm.tqdm(
+            desc="Deforum progress",
+            total=deforum_total,
+            position=1,
+            file=progress_print_out
+        )
+
+    def update(self):
+        if not opts.multiple_tqdm or cmd_opts.disable_console_progressbars:
+            return
+        if self._tqdm is None:
+            self.reset()
+        self._tqdm.update()
+
+    def updateTotal(self, new_total):
+        if not opts.multiple_tqdm or cmd_opts.disable_console_progressbars:
+            return
+        if self._tqdm is None:
+            self.reset()
+        self._tqdm.total=new_total
+
+    def clear(self):
+        if self._tqdm is not None:
+            self._tqdm.close()
+            self._tqdm = None

extensions/deforum/scripts/deforum_helpers/src/adabins/__init__.py

@@ -0,0 +1 @@
+from .unet_adaptive_bins import UnetAdaptiveBins

extensions/deforum/scripts/deforum_helpers/src/adabins/layers.py

@@ -0,0 +1,36 @@
+import torch
+import torch.nn as nn
+
+
+class PatchTransformerEncoder(nn.Module):
+    def __init__(self, in_channels, patch_size=10, embedding_dim=128, num_heads=4):
+        super(PatchTransformerEncoder, self).__init__()
+        encoder_layers = nn.TransformerEncoderLayer(embedding_dim, num_heads, dim_feedforward=1024)
+        self.transformer_encoder = nn.TransformerEncoder(encoder_layers, num_layers=4)  # takes shape S,N,E
+
+        self.embedding_convPxP = nn.Conv2d(in_channels, embedding_dim,
+                                           kernel_size=patch_size, stride=patch_size, padding=0)
+
+        self.positional_encodings = nn.Parameter(torch.rand(500, embedding_dim), requires_grad=True)
+
+    def forward(self, x):
+        embeddings = self.embedding_convPxP(x).flatten(2)  # .shape = n,c,s = n, embedding_dim, s
+        # embeddings = nn.functional.pad(embeddings, (1,0))  # extra special token at start ?
+        embeddings = embeddings + self.positional_encodings[:embeddings.shape[2], :].T.unsqueeze(0)
+
+        # change to S,N,E format required by transformer
+        embeddings = embeddings.permute(2, 0, 1)
+        x = self.transformer_encoder(embeddings)  # .shape = S, N, E
+        return x
+
+
+class PixelWiseDotProduct(nn.Module):
+    def __init__(self):
+        super(PixelWiseDotProduct, self).__init__()
+
+    def forward(self, x, K):
+        n, c, h, w = x.size()
+        _, cout, ck = K.size()
+        assert c == ck, "Number of channels in x and Embedding dimension (at dim 2) of K matrix must match"
+        y = torch.matmul(x.view(n, c, h * w).permute(0, 2, 1), K.permute(0, 2, 1))  # .shape = n, hw, cout
+        return y.permute(0, 2, 1).view(n, cout, h, w)

extensions/deforum/scripts/deforum_helpers/src/adabins/miniViT.py

@@ -0,0 +1,45 @@
+import torch
+import torch.nn as nn
+
+from .layers import PatchTransformerEncoder, PixelWiseDotProduct
+
+
+class mViT(nn.Module):
+    def __init__(self, in_channels, n_query_channels=128, patch_size=16, dim_out=256,
+                 embedding_dim=128, num_heads=4, norm='linear'):
+        super(mViT, self).__init__()
+        self.norm = norm
+        self.n_query_channels = n_query_channels
+        self.patch_transformer = PatchTransformerEncoder(in_channels, patch_size, embedding_dim, num_heads)
+        self.dot_product_layer = PixelWiseDotProduct()
+
+        self.conv3x3 = nn.Conv2d(in_channels, embedding_dim, kernel_size=3, stride=1, padding=1)
+        self.regressor = nn.Sequential(nn.Linear(embedding_dim, 256),
+                                       nn.LeakyReLU(),
+                                       nn.Linear(256, 256),
+                                       nn.LeakyReLU(),
+                                       nn.Linear(256, dim_out))
+
+    def forward(self, x):
+        # n, c, h, w = x.size()
+        tgt = self.patch_transformer(x.clone())  # .shape = S, N, E
+
+        x = self.conv3x3(x)
+
+        regression_head, queries = tgt[0, ...], tgt[1:self.n_query_channels + 1, ...]
+
+        # Change from S, N, E to N, S, E
+        queries = queries.permute(1, 0, 2)
+        range_attention_maps = self.dot_product_layer(x, queries)  # .shape = n, n_query_channels, h, w
+
+        y = self.regressor(regression_head)  # .shape = N, dim_out
+        if self.norm == 'linear':
+            y = torch.relu(y)
+            eps = 0.1
+            y = y + eps
+        elif self.norm == 'softmax':
+            return torch.softmax(y, dim=1), range_attention_maps
+        else:
+            y = torch.sigmoid(y)
+        y = y / y.sum(dim=1, keepdim=True)
+        return y, range_attention_maps

extensions/deforum/scripts/deforum_helpers/src/adabins/unet_adaptive_bins.py

@@ -0,0 +1,154 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import os
+from pathlib import Path
+
+from .miniViT import mViT
+
+
+class UpSampleBN(nn.Module):
+    def __init__(self, skip_input, output_features):
+        super(UpSampleBN, self).__init__()
+
+        self._net = nn.Sequential(nn.Conv2d(skip_input, output_features, kernel_size=3, stride=1, padding=1),
+                                  nn.BatchNorm2d(output_features),
+                                  nn.LeakyReLU(),
+                                  nn.Conv2d(output_features, output_features, kernel_size=3, stride=1, padding=1),
+                                  nn.BatchNorm2d(output_features),
+                                  nn.LeakyReLU())
+
+    def forward(self, x, concat_with):
+        up_x = F.interpolate(x, size=[concat_with.size(2), concat_with.size(3)], mode='bilinear', align_corners=True)
+        f = torch.cat([up_x, concat_with], dim=1)
+        return self._net(f)
+
+
+class DecoderBN(nn.Module):
+    def __init__(self, num_features=2048, num_classes=1, bottleneck_features=2048):
+        super(DecoderBN, self).__init__()
+        features = int(num_features)
+
+        self.conv2 = nn.Conv2d(bottleneck_features, features, kernel_size=1, stride=1, padding=1)
+
+        self.up1 = UpSampleBN(skip_input=features // 1 + 112 + 64, output_features=features // 2)
+        self.up2 = UpSampleBN(skip_input=features // 2 + 40 + 24, output_features=features // 4)
+        self.up3 = UpSampleBN(skip_input=features // 4 + 24 + 16, output_features=features // 8)
+        self.up4 = UpSampleBN(skip_input=features // 8 + 16 + 8, output_features=features // 16)
+
+        #         self.up5 = UpSample(skip_input=features // 16 + 3, output_features=features//16)
+        self.conv3 = nn.Conv2d(features // 16, num_classes, kernel_size=3, stride=1, padding=1)
+        # self.act_out = nn.Softmax(dim=1) if output_activation == 'softmax' else nn.Identity()
+
+    def forward(self, features):
+        x_block0, x_block1, x_block2, x_block3, x_block4 = features[4], features[5], features[6], features[8], features[
+            11]
+
+        x_d0 = self.conv2(x_block4)
+
+        x_d1 = self.up1(x_d0, x_block3)
+        x_d2 = self.up2(x_d1, x_block2)
+        x_d3 = self.up3(x_d2, x_block1)
+        x_d4 = self.up4(x_d3, x_block0)
+        #         x_d5 = self.up5(x_d4, features[0])
+        out = self.conv3(x_d4)
+        # out = self.act_out(out)
+        # if with_features:
+        #     return out, features[-1]
+        # elif with_intermediate:
+        #     return out, [x_block0, x_block1, x_block2, x_block3, x_block4, x_d1, x_d2, x_d3, x_d4]
+        return out
+
+
+class Encoder(nn.Module):
+    def __init__(self, backend):
+        super(Encoder, self).__init__()
+        self.original_model = backend
+
+    def forward(self, x):
+        features = [x]
+        for k, v in self.original_model._modules.items():
+            if (k == 'blocks'):
+                for ki, vi in v._modules.items():
+                    features.append(vi(features[-1]))
+            else:
+                features.append(v(features[-1]))
+        return features
+
+
+class UnetAdaptiveBins(nn.Module):
+    def __init__(self, backend, n_bins=100, min_val=0.1, max_val=10, norm='linear'):
+        super(UnetAdaptiveBins, self).__init__()
+        self.num_classes = n_bins
+        self.min_val = min_val
+        self.max_val = max_val
+        self.encoder = Encoder(backend)
+        self.adaptive_bins_layer = mViT(128, n_query_channels=128, patch_size=16,
+                                        dim_out=n_bins,
+                                        embedding_dim=128, norm=norm)
+
+        self.decoder = DecoderBN(num_classes=128)
+        self.conv_out = nn.Sequential(nn.Conv2d(128, n_bins, kernel_size=1, stride=1, padding=0),
+                                      nn.Softmax(dim=1))
+
+    def forward(self, x, **kwargs):
+        unet_out = self.decoder(self.encoder(x), **kwargs)
+        bin_widths_normed, range_attention_maps = self.adaptive_bins_layer(unet_out)
+        out = self.conv_out(range_attention_maps)
+
+        # Post process
+        # n, c, h, w = out.shape
+        # hist = torch.sum(out.view(n, c, h * w), dim=2) / (h * w)  # not used for training
+
+        bin_widths = (self.max_val - self.min_val) * bin_widths_normed  # .shape = N, dim_out
+        bin_widths = nn.functional.pad(bin_widths, (1, 0), mode='constant', value=self.min_val)
+        bin_edges = torch.cumsum(bin_widths, dim=1)
+
+        centers = 0.5 * (bin_edges[:, :-1] + bin_edges[:, 1:])
+        n, dout = centers.size()
+        centers = centers.view(n, dout, 1, 1)
+
+        pred = torch.sum(out * centers, dim=1, keepdim=True)
+
+        return bin_edges, pred
+
+    def get_1x_lr_params(self):  # lr/10 learning rate
+        return self.encoder.parameters()
+
+    def get_10x_lr_params(self):  # lr learning rate
+        modules = [self.decoder, self.adaptive_bins_layer, self.conv_out]
+        for m in modules:
+            yield from m.parameters()
+
+    @classmethod
+    def build(cls, n_bins, **kwargs):
+        basemodel_name = 'tf_efficientnet_b5_ap'
+
+        print('Loading base model ()...'.format(basemodel_name), end='')
+        predicted_torch_model_cache_path = str(Path.home()) + '\\.cache\\torch\\hub\\rwightman_gen-efficientnet-pytorch_master' 
+        predicted_gep_cache_testilfe = Path(predicted_torch_model_cache_path + '\\hubconf.py')
+        #print(f"predicted_gep_cache_testilfe:  {predicted_gep_cache_testilfe}")
+        # try to fetch the models from cache, and only if it can't be find, download from the internet (to enable offline usage)
+        if os.path.isfile(predicted_gep_cache_testilfe):
+            basemodel = torch.hub.load(predicted_torch_model_cache_path, basemodel_name, pretrained=True, source = 'local')        
+        else:
+            basemodel = torch.hub.load('rwightman/gen-efficientnet-pytorch', basemodel_name, pretrained=True)
+        print('Done.')
+
+        # Remove last layer
+        print('Removing last two layers (global_pool & classifier).')
+        basemodel.global_pool = nn.Identity()
+        basemodel.classifier = nn.Identity()
+
+        # Building Encoder-Decoder model
+        print('Building Encoder-Decoder model..', end='')
+        m = cls(basemodel, n_bins=n_bins, **kwargs)
+        print('Done.')
+        return m
+
+
+if __name__ == '__main__':
+    model = UnetAdaptiveBins.build(100)
+    x = torch.rand(2, 3, 480, 640)
+    bins, pred = model(x)
+    print(bins.shape, pred.shape)

extensions/deforum/scripts/deforum_helpers/src/clipseg/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+This license does not apply to the model weights.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

extensions/deforum/scripts/deforum_helpers/src/clipseg/Quickstart.ipynb

@@ -0,0 +1,107 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "import requests\n",
+    "\n",
+    "! wget https://owncloud.gwdg.de/index.php/s/ioHbRzFx6th32hn/download -O weights.zip\n",
+    "! unzip -d weights -j weights.zip\n",
+    "from models.clipseg import CLIPDensePredT\n",
+    "from PIL import Image\n",
+    "from torchvision import transforms\n",
+    "from matplotlib import pyplot as plt\n",
+    "\n",
+    "# load model\n",
+    "model = CLIPDensePredT(version='ViT-B/16', reduce_dim=64)\n",
+    "model.eval();\n",
+    "\n",
+    "# non-strict, because we only stored decoder weights (not CLIP weights)\n",
+    "model.load_state_dict(torch.load('weights/rd64-uni.pth', map_location=torch.device('cpu')), strict=False);"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Load and normalize `example_image.jpg`. You can also load through an URL."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# load and normalize image\n",
+    "input_image = Image.open('example_image.jpg')\n",
+    "\n",
+    "# or load from URL...\n",
+    "# image_url = 'https://farm5.staticflickr.com/4141/4856248695_03475782dc_z.jpg'\n",
+    "# input_image = Image.open(requests.get(image_url, stream=True).raw)\n",
+    "\n",
+    "transform = transforms.Compose([\n",
+    "    transforms.ToTensor(),\n",
+    "    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n",
+    "    transforms.Resize((352, 352)),\n",
+    "])\n",
+    "img = transform(input_image).unsqueeze(0)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Predict and visualize (this might take a few seconds if running without GPU support)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompts = ['a glass', 'something to fill', 'wood', 'a jar']\n",
+    "\n",
+    "# predict\n",
+    "with torch.no_grad():\n",
+    "    preds = model(img.repeat(4,1,1,1), prompts)[0]\n",
+    "\n",
+    "# visualize prediction\n",
+    "_, ax = plt.subplots(1, 5, figsize=(15, 4))\n",
+    "[a.axis('off') for a in ax.flatten()]\n",
+    "ax[0].imshow(input_image)\n",
+    "[ax[i+1].imshow(torch.sigmoid(preds[i][0])) for i in range(4)];\n",
+    "[ax[i+1].text(0, -15, prompts[i]) for i in range(4)];"
+   ]
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "800ed241f7db2bd3aa6942aa3be6809cdb30ee6b0a9e773dfecfa9fef1f4c586"
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

extensions/deforum/scripts/deforum_helpers/src/clipseg/Readme.md

@@ -0,0 +1,84 @@
+# Image Segmentation Using Text and Image Prompts
+This repository contains the code used in the paper ["Image Segmentation Using Text and Image Prompts"](https://arxiv.org/abs/2112.10003).
+
+**The Paper has been accepted to CVPR 2022!**
+
+<img src="overview.png" alt="drawing" height="200em"/>
+
+The systems allows to create segmentation models without training based on:
+- An arbitrary text query
+- Or an image with a mask highlighting stuff or an object.
+
+### Quick Start
+
+In the `Quickstart.ipynb` notebook we provide the code for using a pre-trained CLIPSeg model. If you run the notebook locally, make sure you downloaded the `rd64-uni.pth` weights, either manually or via git lfs extension.
+It can also be used interactively using [MyBinder](https://mybinder.org/v2/gh/timojl/clipseg/HEAD?labpath=Quickstart.ipynb)
+(please note that the VM does not use a GPU, thus inference takes a few seconds).
+
+
+### Dependencies
+This code base depends on pytorch, torchvision and clip (`pip install git+https://github.com/openai/CLIP.git`).
+Additional dependencies are hidden for double blind review.
+
+
+### Datasets
+
+* `PhraseCut` and `PhraseCutPlus`: Referring expression dataset
+* `PFEPascalWrapper`: Wrapper class for PFENet's Pascal-5i implementation
+* `PascalZeroShot`: Wrapper class for PascalZeroShot
+* `COCOWrapper`: Wrapper class for COCO.
+
+### Models
+
+* `CLIPDensePredT`: CLIPSeg model with transformer-based decoder.
+* `ViTDensePredT`: CLIPSeg model with transformer-based decoder.
+
+### Third Party Dependencies
+For some of the datasets third party dependencies are required. Run the following commands in the `third_party` folder.  
+```bash
+git clone https://github.com/cvlab-yonsei/JoEm
+git clone https://github.com/Jia-Research-Lab/PFENet.git
+git clone https://github.com/ChenyunWu/PhraseCutDataset.git
+git clone https://github.com/juhongm999/hsnet.git
+```
+
+### Weights
+
+The MIT license does not apply to these weights. 
+
+We provide two model weights, for D=64 (4.1MB) and D=16 (1.1MB).
+```
+wget https://owncloud.gwdg.de/index.php/s/ioHbRzFx6th32hn/download -O weights.zip
+unzip -d weights -j weights.zip
+```
+
+
+### Training and Evaluation
+
+To train use the `training.py` script with experiment file and experiment id parameters. E.g. `python training.py phrasecut.yaml 0` will train the first phrasecut experiment which is defined by the `configuration` and first `individual_configurations` parameters. Model weights will be written in `logs/`.
+
+For evaluation use `score.py`. E.g. `python score.py phrasecut.yaml 0 0` will train the first phrasecut experiment of `test_configuration` and the first configuration in `individual_configurations`.
+
+
+### Usage of PFENet Wrappers
+
+In order to use the dataset and model wrappers for PFENet, the PFENet repository needs to be cloned to the root folder.
+`git clone https://github.com/Jia-Research-Lab/PFENet.git `
+
+
+### License
+
+The source code files in this repository (excluding model weights) are released under MIT license.
+
+### Citation
+```
+@InProceedings{lueddecke22_cvpr,
+    author    = {L\"uddecke, Timo and Ecker, Alexander},
+    title     = {Image Segmentation Using Text and Image Prompts},
+    booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
+    month     = {June},
+    year      = {2022},
+    pages     = {7086-7096}
+}
+
+```

extensions/deforum/scripts/deforum_helpers/src/clipseg/Tables.ipynb

@@ -0,0 +1,349 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2\n",
+    "\n",
+    "import clip\n",
+    "from evaluation_utils import norm, denorm\n",
+    "from general_utils import *\n",
+    "from datasets.lvis_oneshot3 import LVIS_OneShot3, LVIS_OneShot"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# PhraseCut"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pc = experiment('experiments/phrasecut.yaml', nums=':6').dataframe()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tab1 = pc[['name', 'pc_miou_best',  'pc_fgiou_best', 'pc_ap']]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cols = ['pc_miou_0.3',  'pc_fgiou_0.3', 'pc_ap']\n",
+    "tab1 = pc[['name'] + cols]\n",
+    "for k in cols:\n",
+    "    tab1.loc[:, k] = (100 * tab1.loc[:, k]).round(1)\n",
+    "tab1.loc[:, 'name'] = ['CLIPSeg (PC+)', 'CLIPSeg (PC, $D=128$)', 'CLIPSeg (PC)', 'CLIP-Deconv', 'ViTSeg (PC+)', 'ViTSeg (PC)']\n",
+    "tab1.insert(1, 't', [0.3]*tab1.shape[0])\n",
+    "print(tab1.to_latex(header=False, index=False))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "For 0.1 threshold"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cols = ['pc_miou_0.1',  'pc_fgiou_0.1', 'pc_ap']\n",
+    "tab1 = pc[['name'] + cols]\n",
+    "for k in cols:\n",
+    "    tab1.loc[:, k] = (100 * tab1.loc[:, k]).round(1)\n",
+    "tab1.loc[:, 'name'] = ['CLIPSeg (PC+)', 'CLIPSeg (PC, $D=128$)', 'CLIPSeg (PC)', 'CLIP-Deconv', 'ViTSeg (PC+)', 'ViTSeg (PC)']\n",
+    "tab1.insert(1, 't', [0.1]*tab1.shape[0])\n",
+    "print(tab1.to_latex(header=False, index=False))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# One-shot"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Pascal"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pas = experiment('experiments/pascal_1shot.yaml', nums=':19').dataframe()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pas[['name', 'pas_h2_miou_0.3', 'pas_h2_biniou_0.3', 'pas_h2_ap', 'pas_h2_fgiou_ct']]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pas = experiment('experiments/pascal_1shot.yaml', nums=':8').dataframe()\n",
+    "tab1 = pas[['pas_h2_miou_0.3', 'pas_h2_biniou_0.3', 'pas_h2_ap']]\n",
+    "print('CLIPSeg (PC+) & 0.3 & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[0:4].mean(0).values), '\\\\\\\\')\n",
+    "print('CLIPSeg (PC)  & 0.3 & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[4:8].mean(0).values), '\\\\\\\\')\n",
+    "\n",
+    "pas = experiment('experiments/pascal_1shot.yaml', nums='12:16').dataframe()\n",
+    "tab1 = pas[['pas_h2_miou_0.2', 'pas_h2_biniou_0.2', 'pas_h2_ap']]\n",
+    "print('CLIP-Deconv (PC+) & 0.2 & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[0:4].mean(0).values), '\\\\\\\\')\n",
+    "\n",
+    "pas = experiment('experiments/pascal_1shot.yaml', nums='16:20').dataframe()\n",
+    "tab1 = pas[['pas_t_miou_0.2', 'pas_t_biniou_0.2', 'pas_t_ap']]\n",
+    "print('ViTSeg (PC+) & 0.2 & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[0:4].mean(0).values), '\\\\\\\\')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "#### Pascal Zero-shot (in one-shot setting)\n",
+    "\n",
+    "Using the same setting as one-shot (hence different from the other zero-shot benchmark)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pas = experiment('experiments/pascal_1shot.yaml', nums=':8').dataframe()\n",
+    "tab1 = pas[['pas_t_miou_0.3', 'pas_t_biniou_0.3', 'pas_t_ap']]\n",
+    "print('CLIPSeg (PC+) & 0.3 & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[0:4].mean(0).values), '\\\\\\\\')\n",
+    "print('CLIPSeg (PC) & 0.3 & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[4:8].mean(0).values), '\\\\\\\\')\n",
+    "\n",
+    "pas = experiment('experiments/pascal_1shot.yaml', nums='12:16').dataframe()\n",
+    "tab1 = pas[['pas_t_miou_0.3', 'pas_t_biniou_0.3', 'pas_t_ap']]\n",
+    "print('CLIP-Deconv (PC+) & 0.3 & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[0:4].mean(0).values), '\\\\\\\\')\n",
+    "\n",
+    "pas = experiment('experiments/pascal_1shot.yaml', nums='16:20').dataframe()\n",
+    "tab1 = pas[['pas_t_miou_0.2', 'pas_t_biniou_0.2', 'pas_t_ap']]\n",
+    "print('ViTSeg (PC+) & 0.2 & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[0:4].mean(0).values), '\\\\\\\\')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# without fixed thresholds...\n",
+    "\n",
+    "pas = experiment('experiments/pascal_1shot.yaml', nums=':8').dataframe()\n",
+    "tab1 = pas[['pas_t_best_miou', 'pas_t_best_biniou', 'pas_t_ap']]\n",
+    "print('CLIPSeg (PC+) & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[0:4].mean(0).values), '\\\\\\\\')\n",
+    "print('CLIPSeg (PC) & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[4:8].mean(0).values), '\\\\\\\\')\n",
+    "\n",
+    "pas = experiment('experiments/pascal_1shot.yaml', nums='12:16').dataframe()\n",
+    "tab1 = pas[['pas_t_best_miou', 'pas_t_best_biniou', 'pas_t_ap']]\n",
+    "print('CLIP-Deconv (PC+) & CLIP & ' + ' & '.join(f'{x*100:.1f}' for x in tab1[0:4].mean(0).values), '\\\\\\\\')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### COCO"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "coco = experiment('experiments/coco.yaml', nums=':29').dataframe()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tab1 = coco[['coco_h2_miou_0.1', 'coco_h2_biniou_0.1', 'coco_h2_ap']]\n",
+    "tab2 = coco[['coco_h2_miou_0.2', 'coco_h2_biniou_0.2', 'coco_h2_ap']]\n",
+    "tab3 = coco[['coco_h2_miou_best', 'coco_h2_biniou_best', 'coco_h2_ap']]\n",
+    "print('CLIPSeg (COCO) & 0.1 & CLIP &  ' + ' & '.join(f'{x*100:.1f}' for x in tab1[:4].mean(0).values), '\\\\\\\\')\n",
+    "print('CLIPSeg (COCO+N)  & 0.1 & CLIP &  ' + ' & '.join(f'{x*100:.1f}' for x in tab1[4:8].mean(0).values), '\\\\\\\\')\n",
+    "print('CLIP-Deconv (COCO+N)  & 0.1 & CLIP &  ' + ' & '.join(f'{x*100:.1f}' for x in tab1[12:16].mean(0).values), '\\\\\\\\')\n",
+    "print('ViTSeg (COCO)  & 0.1 & CLIP &  ' + ' & '.join(f'{x*100:.1f}' for x in tab1[8:12].mean(0).values), '\\\\\\\\')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Zero-shot"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "zs = experiment('experiments/pascal_0shot.yaml', nums=':11').dataframe()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "tab1 = zs[['pas_zs_seen', 'pas_zs_unseen']]\n",
+    "print('CLIPSeg (PC+) & CLIP &  ' + ' & '.join(f'{x*100:.1f}' for x in tab1[8:9].values[0].tolist() + tab1[10:11].values[0].tolist()), '\\\\\\\\')\n",
+    "print('CLIP-Deconv & CLIP &  ' + ' & '.join(f'{x*100:.1f}' for x in tab1[2:3].values[0].tolist()  + tab1[3:4].values[0].tolist()), '\\\\\\\\')\n",
+    "print('ViTSeg & ImageNet-1K &  ' + ' & '.join(f'{x*100:.1f}' for x in tab1[4:5].values[0].tolist()  + tab1[5:6].values[0].tolist()), '\\\\\\\\')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Ablation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ablation = experiment('experiments/ablation.yaml', nums=':8').dataframe()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tab1 = ablation[['name', 'pc_miou_best', 'pc_ap', 'pc-vis_miou_best', 'pc-vis_ap']]\n",
+    "for k in ['pc_miou_best', 'pc_ap', 'pc-vis_miou_best', 'pc-vis_ap']:\n",
+    "    tab1.loc[:, k] = (100 * tab1.loc[:, k]).round(1)\n",
+    "tab1.loc[:, 'name'] = ['CLIPSeg', 'no CLIP pre-training', 'no-negatives', '50% negatives', 'no visual', '$D=16$', 'only layer 3', 'highlight mask']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(tab1.loc[[0,1,4,5,6,7],:].to_latex(header=False, index=False))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(tab1.loc[[0,1,4,5,6,7],:].to_latex(header=False, index=False))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Generalization"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "generalization = experiment('experiments/generalize.yaml').dataframe()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gen = generalization[['aff_best_fgiou', 'aff_ap', 'ability_best_fgiou', 'ability_ap', 'part_best_fgiou', 'part_ap']].values"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(\n",
+    "    'CLIPSeg (PC+) & ' + ' & '.join(f'{x*100:.1f}' for x in gen[1]) + ' \\\\\\\\ \\n' + \\\n",
+    "    'CLIPSeg (LVIS)  & ' + ' & '.join(f'{x*100:.1f}' for x in gen[0]) + ' \\\\\\\\ \\n' + \\\n",
+    "    'CLIP-Deconv & ' + ' & '.join(f'{x*100:.1f}' for x in gen[2]) + ' \\\\\\\\ \\n' + \\\n",
+    "    'VITSeg & ' + ' & '.join(f'{x*100:.1f}' for x in gen[3]) + ' \\\\\\\\'\n",
+    ")"
+   ]
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "800ed241f7db2bd3aa6942aa3be6809cdb30ee6b0a9e773dfecfa9fef1f4c586"
+  },
+  "kernelspec": {
+   "display_name": "env2",
+   "language": "python",
+   "name": "env2"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}