diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..a8719ae --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,24 @@ +name: Publish to Comfy registry +on: + workflow_dispatch: + push: + branches: + - main + - master + paths: + - "pyproject.toml" + +jobs: + publish-node: + name: Publish Custom Node to registry + runs-on: ubuntu-latest + # if this is a forked repository. Skipping the workflow. + if: github.event.repository.fork == false + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Publish Custom Node + uses: Comfy-Org/publish-node-action@main + with: + ## Add your own personal access token to your Github Repository secrets and reference it here. + personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..341e31e --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__ +.idea \ No newline at end of file diff --git a/LivePortrait/config/inference_config.py b/LivePortrait/config/inference_config.py new file mode 100644 index 0000000..8494c82 --- /dev/null +++ b/LivePortrait/config/inference_config.py @@ -0,0 +1,8 @@ + +import os + +current_file_path = os.path.abspath(__file__) +current_directory = os.path.dirname(current_file_path) +class InferenceConfig: + def __init__(self): + self.flag_use_half_precision: bool = False # whether to use half precision \ No newline at end of file diff --git a/config/models.yaml b/LivePortrait/config/models.yaml similarity index 100% rename from config/models.yaml rename to LivePortrait/config/models.yaml diff --git a/LivePortrait/src/live_portrait_wrapper.py b/LivePortrait/live_portrait_wrapper.py similarity index 55% rename from LivePortrait/src/live_portrait_wrapper.py rename to LivePortrait/live_portrait_wrapper.py index 935b03e..a71acb2 100644 --- a/LivePortrait/src/live_portrait_wrapper.py +++ b/LivePortrait/live_portrait_wrapper.py @@ -1,91 +1,29 @@ -import os.path as osp import numpy as np -import cv2 import torch -import yaml - -from LivePortrait.src.utils.timer import Timer -from LivePortrait.src.utils.helper import load_model, concat_feat -from LivePortrait.src.utils.camera import headpose_pred_to_degree, get_rotation_matrix -from config.inference_config import InferenceConfig -from LivePortrait.src.utils.rprint import rlog as log +from .utils.helper import concat_feat +from .utils.camera import headpose_pred_to_degree, get_rotation_matrix +from .config.inference_config import InferenceConfig class LivePortraitWrapper(object): - def __init__(self, cfg: InferenceConfig): - - model_config = yaml.load(open(cfg.models_config, 'r'), Loader=yaml.SafeLoader) - - # init F - self.appearance_feature_extractor = load_model(cfg.checkpoint_F, model_config, cfg.device_id, 'appearance_feature_extractor') - log(f'Load appearance_feature_extractor done.') - # init M - self.motion_extractor = load_model(cfg.checkpoint_M, model_config, cfg.device_id, 'motion_extractor') - log(f'Load motion_extractor done.') - # init W - self.warping_module = load_model(cfg.checkpoint_W, model_config, cfg.device_id, 'warping_module') - log(f'Load warping_module done.') - # init G - self.spade_generator = load_model(cfg.checkpoint_G, model_config, cfg.device_id, 'spade_generator') - log(f'Load spade_generator done.') - # init S and R - if cfg.checkpoint_S is not None and osp.exists(cfg.checkpoint_S): - self.stitching_retargeting_module = load_model(cfg.checkpoint_S, model_config, cfg.device_id, 'stitching_retargeting_module') - log(f'Load stitching_retargeting_module done.') - else: - self.stitching_retargeting_module = None - - self.cfg = cfg - self.device_id = cfg.device_id - self.timer = Timer() - - def prepare_source(self, img: np.ndarray) -> torch.Tensor: - """ construct the input as standard - img: HxWx3, uint8, 256x256 - """ - h, w = img.shape[:2] - if h != self.cfg.input_shape[0] or w != self.cfg.input_shape[1]: - x = cv2.resize(img, (self.cfg.input_shape[0], self.cfg.input_shape[1])) - else: - x = img.copy() - - if x.ndim == 3: - x = x[np.newaxis].astype(np.float32) / 255. # HxWx3 -> 1xHxWx3, normalized to 0~1 - elif x.ndim == 4: - x = x.astype(np.float32) / 255. # BxHxWx3, normalized to 0~1 - else: - raise ValueError(f'img ndim should be 3 or 4: {x.ndim}') - x = np.clip(x, 0, 1) # clip to 0~1 - x = torch.from_numpy(x).permute(0, 3, 1, 2) # 1xHxWx3 -> 1x3xHxW - x = x.cuda(self.device_id) - return x - - def prepare_driving_videos(self, imgs) -> torch.Tensor: - """ construct the input as standard - imgs: NxBxHxWx3, uint8 - """ - if isinstance(imgs, list): - _imgs = np.array(imgs)[..., np.newaxis] # TxHxWx3x1 - elif isinstance(imgs, np.ndarray): - _imgs = imgs - else: - raise ValueError(f'imgs type error: {type(imgs)}') + def __init__(self, cfg: InferenceConfig, appearance_feature_extractor, motion_extractor, + warping_module, spade_generator, stitching_retargeting_module): - y = _imgs.astype(np.float32) / 255. - y = np.clip(y, 0, 1) # clip to 0~1 - y = torch.from_numpy(y).permute(0, 4, 3, 1, 2) # TxHxWx3x1 -> Tx1x3xHxW - y = y.cuda(self.device_id) + self.appearance_feature_extractor = appearance_feature_extractor + self.motion_extractor = motion_extractor + self.warping_module = warping_module + self.spade_generator = spade_generator + self.stitching_retargeting_module = stitching_retargeting_module - return y + self.cfg = cfg def extract_feature_3d(self, x: torch.Tensor) -> torch.Tensor: """ get the appearance feature of the image by F x: Bx3xHxW, normalized to 0~1 """ with torch.no_grad(): - with torch.autocast(device_type='cuda', dtype=torch.float16, enabled=self.cfg.flag_use_half_precision): - feature_3d = self.appearance_feature_extractor(x) + feature_3d = self.appearance_feature_extractor(x) return feature_3d.float() @@ -96,8 +34,7 @@ def get_kp_info(self, x: torch.Tensor, **kwargs) -> dict: return: A dict contains keys: 'pitch', 'yaw', 'roll', 't', 'exp', 'scale', 'kp' """ with torch.no_grad(): - with torch.autocast(device_type='cuda', dtype=torch.float16, enabled=self.cfg.flag_use_half_precision): - kp_info = self.motion_extractor(x) + kp_info = self.motion_extractor(x) if self.cfg.flag_use_half_precision: # float the dict @@ -189,11 +126,10 @@ def warp_decode(self, feature_3d: torch.Tensor, kp_source: torch.Tensor, kp_driv """ # The line 18 in Algorithm 1: D(W(f_s; x_s, x′_d,i)) with torch.no_grad(): - with torch.autocast(device_type='cuda', dtype=torch.float16, enabled=self.cfg.flag_use_half_precision): - # get decoder input - ret_dct = self.warping_module(feature_3d, kp_source=kp_source, kp_driving=kp_driving) - # decode - ret_dct['out'] = self.spade_generator(feature=ret_dct['out']) + # get decoder input + ret_dct = self.warping_module(feature_3d, kp_source=kp_source, kp_driving=kp_driving) + # decode + ret_dct['out'] = self.spade_generator(feature=ret_dct['out']) # float the dict if self.cfg.flag_use_half_precision: diff --git a/LivePortrait/src/modules/__init__.py b/LivePortrait/modules/__init__.py similarity index 100% rename from LivePortrait/src/modules/__init__.py rename to LivePortrait/modules/__init__.py diff --git a/LivePortrait/src/modules/appearance_feature_extractor.py b/LivePortrait/modules/appearance_feature_extractor.py similarity index 100% rename from LivePortrait/src/modules/appearance_feature_extractor.py rename to LivePortrait/modules/appearance_feature_extractor.py diff --git a/LivePortrait/src/modules/convnextv2.py b/LivePortrait/modules/convnextv2.py similarity index 100% rename from LivePortrait/src/modules/convnextv2.py rename to LivePortrait/modules/convnextv2.py diff --git a/LivePortrait/src/modules/dense_motion.py b/LivePortrait/modules/dense_motion.py similarity index 98% rename from LivePortrait/src/modules/dense_motion.py rename to LivePortrait/modules/dense_motion.py index 0eec0c4..c1a7f9a 100644 --- a/LivePortrait/src/modules/dense_motion.py +++ b/LivePortrait/modules/dense_motion.py @@ -59,7 +59,7 @@ def create_heatmap_representations(self, feature, kp_driving, kp_source): heatmap = gaussian_driving - gaussian_source # (bs, num_kp, d, h, w) # adding background feature - zeros = torch.zeros(heatmap.shape[0], 1, spatial_size[0], spatial_size[1], spatial_size[2]).type(heatmap.type()).to(heatmap.device) + zeros = torch.zeros(heatmap.shape[0], 1, spatial_size[0], spatial_size[1], spatial_size[2]).type(heatmap.dtype).to(heatmap.device) heatmap = torch.cat([zeros, heatmap], dim=1) heatmap = heatmap.unsqueeze(2) # (bs, 1+num_kp, 1, d, h, w) return heatmap diff --git a/LivePortrait/src/modules/motion_extractor.py b/LivePortrait/modules/motion_extractor.py similarity index 100% rename from LivePortrait/src/modules/motion_extractor.py rename to LivePortrait/modules/motion_extractor.py diff --git a/LivePortrait/src/modules/spade_generator.py b/LivePortrait/modules/spade_generator.py similarity index 100% rename from LivePortrait/src/modules/spade_generator.py rename to LivePortrait/modules/spade_generator.py diff --git a/LivePortrait/src/modules/stitching_retargeting_network.py b/LivePortrait/modules/stitching_retargeting_network.py similarity index 100% rename from LivePortrait/src/modules/stitching_retargeting_network.py rename to LivePortrait/modules/stitching_retargeting_network.py diff --git a/LivePortrait/src/modules/util.py b/LivePortrait/modules/util.py similarity index 100% rename from LivePortrait/src/modules/util.py rename to LivePortrait/modules/util.py diff --git a/LivePortrait/src/modules/warping_network.py b/LivePortrait/modules/warping_network.py similarity index 100% rename from LivePortrait/src/modules/warping_network.py rename to LivePortrait/modules/warping_network.py diff --git a/LivePortrait/src/utils/__init__.py b/LivePortrait/utils/__init__.py similarity index 100% rename from LivePortrait/src/utils/__init__.py rename to LivePortrait/utils/__init__.py diff --git a/LivePortrait/src/utils/camera.py b/LivePortrait/utils/camera.py similarity index 100% rename from LivePortrait/src/utils/camera.py rename to LivePortrait/utils/camera.py diff --git a/LivePortrait/src/utils/face_analysis_diy.py b/LivePortrait/utils/face_analysis_diy.py similarity index 100% rename from LivePortrait/src/utils/face_analysis_diy.py rename to LivePortrait/utils/face_analysis_diy.py diff --git a/LivePortrait/src/utils/helper.py b/LivePortrait/utils/helper.py similarity index 55% rename from LivePortrait/src/utils/helper.py rename to LivePortrait/utils/helper.py index 009c3df..277f63d 100644 --- a/LivePortrait/src/utils/helper.py +++ b/LivePortrait/utils/helper.py @@ -11,11 +11,11 @@ from rich.console import Console from collections import OrderedDict -from LivePortrait.src.modules.spade_generator import SPADEDecoder -from LivePortrait.src.modules.warping_network import WarpingNetwork -from LivePortrait.src.modules.motion_extractor import MotionExtractor -from LivePortrait.src.modules.appearance_feature_extractor import AppearanceFeatureExtractor -from LivePortrait.src.modules.stitching_retargeting_network import StitchingRetargetingNetwork +from ..modules.spade_generator import SPADEDecoder +from ..modules.warping_network import WarpingNetwork +from ..modules.motion_extractor import MotionExtractor +from ..modules.appearance_feature_extractor import AppearanceFeatureExtractor +from ..modules.stitching_retargeting_network import StitchingRetargetingNetwork from .rprint import rlog as log @@ -85,57 +85,6 @@ def concat_feat(kp_source: torch.Tensor, kp_driving: torch.Tensor) -> torch.Tens return feat -def remove_ddp_dumplicate_key(state_dict): - state_dict_new = OrderedDict() - for key in state_dict.keys(): - state_dict_new[key.replace('module.', '')] = state_dict[key] - return state_dict_new - - -def load_model(ckpt_path, model_config, device, model_type): - model_params = model_config['model_params'][f'{model_type}_params'] - - if model_type == 'appearance_feature_extractor': - model = AppearanceFeatureExtractor(**model_params).cuda(device) - elif model_type == 'motion_extractor': - model = MotionExtractor(**model_params).cuda(device) - elif model_type == 'warping_module': - model = WarpingNetwork(**model_params).cuda(device) - elif model_type == 'spade_generator': - model = SPADEDecoder(**model_params).cuda(device) - elif model_type == 'stitching_retargeting_module': - # Special handling for stitching and retargeting module - config = model_config['model_params']['stitching_retargeting_module_params'] - checkpoint = torch.load(ckpt_path, map_location=lambda storage, loc: storage) - - stitcher = StitchingRetargetingNetwork(**config.get('stitching')) - stitcher.load_state_dict(remove_ddp_dumplicate_key(checkpoint['retarget_shoulder'])) - stitcher = stitcher.cuda(device) - stitcher.eval() - - retargetor_lip = StitchingRetargetingNetwork(**config.get('lip')) - retargetor_lip.load_state_dict(remove_ddp_dumplicate_key(checkpoint['retarget_mouth'])) - retargetor_lip = retargetor_lip.cuda(device) - retargetor_lip.eval() - - retargetor_eye = StitchingRetargetingNetwork(**config.get('eye')) - retargetor_eye.load_state_dict(remove_ddp_dumplicate_key(checkpoint['retarget_eye'])) - retargetor_eye = retargetor_eye.cuda(device) - retargetor_eye.eval() - - return { - 'stitching': stitcher, - 'lip': retargetor_lip, - 'eye': retargetor_eye - } - else: - raise ValueError(f"Unknown model type: {model_type}") - - model.load_state_dict(torch.load(ckpt_path, map_location=lambda storage, loc: storage)) - model.eval() - return model - - # get coefficients of Eqn. 7 def calculate_transformation(config, s_kp_info, t_0_kp_info, t_i_kp_info, R_s, R_t_0, R_t_i): if config.relative: diff --git a/LivePortrait/src/utils/io.py b/LivePortrait/utils/io.py similarity index 100% rename from LivePortrait/src/utils/io.py rename to LivePortrait/utils/io.py diff --git a/LivePortrait/src/utils/resources/mask_template.png b/LivePortrait/utils/resources/mask_template.png similarity index 100% rename from LivePortrait/src/utils/resources/mask_template.png rename to LivePortrait/utils/resources/mask_template.png diff --git a/LivePortrait/src/utils/rprint.py b/LivePortrait/utils/rprint.py similarity index 100% rename from LivePortrait/src/utils/rprint.py rename to LivePortrait/utils/rprint.py diff --git a/LivePortrait/src/utils/timer.py b/LivePortrait/utils/timer.py similarity index 100% rename from LivePortrait/src/utils/timer.py rename to LivePortrait/utils/timer.py diff --git a/LivePortrait/src/utils/video.py b/LivePortrait/utils/video.py similarity index 100% rename from LivePortrait/src/utils/video.py rename to LivePortrait/utils/video.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..8b8f352 --- /dev/null +++ b/README.md @@ -0,0 +1,67 @@ +# ComfyUI-AdvancedLivePortrait + +## Update +11/07/2024 + +Expressions are feature-reactive (features: audio, MIDI, motion, proximity, and more). + +8/21/2024 + +You can create a video without a video. + +Track the face of the source video. + +The workflow has been updated. + +## Introduction + +AdvancedLivePortrait is faster and has real-time preview + +https://github.com/user-attachments/assets/90b78639-6477-48af-ba49-7945488df581 + +Edit facial expressions in photos. + +Insert facial expressions into videos. + +Create animations using multiple facial expressions. + +Extract facial expressions from sample photos. + +## Installation + +This project has been registered with ComfyUI-Manager. Now you can install it automatically using the manager. + +## Usage + +The workflows and sample datas placed in '\custom_nodes\ComfyUI-AdvancedLivePortrait\sample\' + +----- + +You can add expressions to the video. See 'workflow2_advanced.json'. + +Describes the 'command' in 'workflow2_advanced.json' + +![readme](https://github.com/user-attachments/assets/339568b2-ad52-4aaf-a6ab-fcd877449c56) + + +[Motion index] = [Changing frame length] : [Length of frames waiting for next motion] + +Motion index 0 is the original source image. + +They are numbered in the order they lead to the motion_link. + +Linking the driving video to 'src_images' will add facial expressions to the driving video. + +----- + +You can save and load expressions with the 'Load Exp Data' 'Save Exp Data' nodes. + +\ComfyUI\output\exp_data\ Path to the folder being saved + +----- + +## Thanks + +Original author's link : https://liveportrait.github.io/ + +This project uses a model converted by kijai. link : https://github.com/kijai/ComfyUI-LivePortraitKJ diff --git a/__init__.py b/__init__.py index 476fdb7..4322a4e 100644 --- a/__init__.py +++ b/__init__.py @@ -1,48 +1,3 @@ -import requests -import os, sys -import subprocess -from tqdm import tqdm -from pip._internal import main as pip_main -from pathlib import Path -from folder_paths import models_dir - -def download_model(file_path, model_url): - print('AdvancedLivePortrait: Downloading model...') - response = requests.get(model_url, stream=True) - try: - if response.status_code == 200: - total_size = int(response.headers.get('content-length', 0)) - block_size = 1024 # 1 Kibibyte - - # tqdm will display a progress bar - with open(file_path, 'wb') as file, tqdm( - desc='Downloading', - total=total_size, - unit='iB', - unit_scale=True, - unit_divisor=1024, - ) as bar: - for data in response.iter_content(block_size): - bar.update(len(data)) - file.write(data) - - except requests.exceptions.RequestException as err: - print('AdvancedLivePortrait: Model download failed: {err}') - print(f'AdvancedLivePortrait: Download it manually from: {model_url}') - print(f'AdvancedLivePortrait: And put it in {file_path}') - except Exception as e: - print(f'AdvancedLivePortrait: An unexpected error occurred: {e}') - -save_path = os.path.join(models_dir, "ultralytics") -if not os.path.exists(save_path): - os.makedirs(save_path, exist_ok=True) - file_path = os.path.join(save_path, "face_yolov8n.pt") - if not Path().is_file(): - download_model(file_path, "https://huggingface.co/Bingsu/adetailer/resolve/main/face_yolov8n.pt") - - - - from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS __all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] diff --git a/config/inference_config.py b/config/inference_config.py deleted file mode 100644 index 3a735f6..0000000 --- a/config/inference_config.py +++ /dev/null @@ -1,21 +0,0 @@ - -import os -from typing import Literal, Tuple -import folder_paths - -current_file_path = os.path.abspath(__file__) -current_directory = os.path.dirname(current_file_path) -class InferenceConfig: - def __init__(self): - self.models_config: str = os.path.join(current_directory, "models.yaml") - self.checkpoint_F: str = os.path.join(folder_paths.models_dir, "liveportrait", "base_models", "appearance_feature_extractor.pth") - self.checkpoint_M: str = os.path.join(folder_paths.models_dir, "liveportrait", "base_models", "motion_extractor.pth") - self.checkpoint_G: str = os.path.join(folder_paths.models_dir, "liveportrait", "base_models", "spade_generator.pth") - self.checkpoint_W: str = os.path.join(folder_paths.models_dir, "liveportrait", "base_models", "warping_module.pth") - self.checkpoint_S: str = os.path.join(folder_paths.models_dir, "liveportrait", "retargeting_models", "stitching_retargeting_module.pth") - self.flag_use_half_precision: bool = True # whether to use half precision - - self.input_shape: Tuple[int, int] = (256, 256) # input shape - - self.mask_crop = None - self.device_id: int = 0 \ No newline at end of file diff --git a/nodes.py b/nodes.py index 6482d4a..3c767fe 100644 --- a/nodes.py +++ b/nodes.py @@ -8,21 +8,38 @@ import comfy.utils import time import copy -import math import dill -import torch.nn.functional as torchfn -from torchvision import transforms +import yaml from ultralytics import YOLO -from scipy.spatial.transform import Rotation as R - current_file_path = os.path.abspath(__file__) current_directory = os.path.dirname(current_file_path) -sys.path.append(current_directory) -from .LivePortrait.src.live_portrait_wrapper import LivePortraitWrapper -from .LivePortrait.src.utils.rprint import rlog as log -from .LivePortrait.src.utils.camera import get_rotation_matrix -from .config.inference_config import InferenceConfig + +from .LivePortrait.live_portrait_wrapper import LivePortraitWrapper +from .LivePortrait.utils.camera import get_rotation_matrix +from .LivePortrait.config.inference_config import InferenceConfig + +from .LivePortrait.modules.spade_generator import SPADEDecoder +from .LivePortrait.modules.warping_network import WarpingNetwork +from .LivePortrait.modules.motion_extractor import MotionExtractor +from .LivePortrait.modules.appearance_feature_extractor import AppearanceFeatureExtractor +from .LivePortrait.modules.stitching_retargeting_network import StitchingRetargetingNetwork +from collections import OrderedDict + +cur_device = None +def get_device(): + global cur_device + if cur_device == None: + if torch.cuda.is_available(): + cur_device = torch.device('cuda') + print("Uses CUDA device.") + elif torch.backends.mps.is_available(): + cur_device = torch.device('mps') + print("Uses MPS device.") + else: + cur_device = torch.device('cpu') + print("Uses CPU device.") + return cur_device def tensor2pil(image): return Image.fromarray(np.clip(255. * image.cpu().numpy().squeeze(), 0, 255).astype(np.uint8)) @@ -35,8 +52,8 @@ def rgb_crop_batch(rgbs, region): return rgbs[:, region[1]:region[3], region[0]:region[2]] def get_rgb_size(rgb): return rgb.shape[1], rgb.shape[0] -def create_transform_matrix(x, y, scale=1): - return np.float32([[scale, 0, x], [0, scale, y]]) +def create_transform_matrix(x, y, s_x, s_y): + return np.float32([[s_x, 0, x], [0, s_y, y]]) def get_model_dir(m): try: @@ -73,64 +90,236 @@ def __init__(self, src_rgb, crop_trans_m, x_s_info, f_s_user, x_s_user, mask_ori self.x_s_user = x_s_user self.mask_ori = mask_ori +import requests +from tqdm import tqdm + class LP_Engine: pipeline = None - bbox_model = None + detect_model = None mask_img = None + temp_img_idx = 0 + + def get_temp_img_name(self): + self.temp_img_idx += 1 + return "expression_edit_preview" + str(self.temp_img_idx) + ".png" + + def download_model(_, file_path, model_url): + print('AdvancedLivePortrait: Downloading model...') + response = requests.get(model_url, stream=True) + try: + if response.status_code == 200: + total_size = int(response.headers.get('content-length', 0)) + block_size = 1024 # 1 Kibibyte + + # tqdm will display a progress bar + with open(file_path, 'wb') as file, tqdm( + desc='Downloading', + total=total_size, + unit='iB', + unit_scale=True, + unit_divisor=1024, + ) as bar: + for data in response.iter_content(block_size): + bar.update(len(data)) + file.write(data) + + except requests.exceptions.RequestException as err: + print('AdvancedLivePortrait: Model download failed: {err}') + print(f'AdvancedLivePortrait: Download it manually from: {model_url}') + print(f'AdvancedLivePortrait: And put it in {file_path}') + except Exception as e: + print(f'AdvancedLivePortrait: An unexpected error occurred: {e}') + + def remove_ddp_dumplicate_key(_, state_dict): + state_dict_new = OrderedDict() + for key in state_dict.keys(): + state_dict_new[key.replace('module.', '')] = state_dict[key] + return state_dict_new + + def filter_for_model(_, checkpoint, prefix): + filtered_checkpoint = {key.replace(prefix + "_module.", ""): value for key, value in checkpoint.items() if + key.startswith(prefix)} + return filtered_checkpoint + + def load_model(self, model_config, model_type): + + device = get_device() + + if model_type == 'stitching_retargeting_module': + ckpt_path = os.path.join(get_model_dir("liveportrait"), "retargeting_models", model_type + ".pth") + else: + ckpt_path = os.path.join(get_model_dir("liveportrait"), "base_models", model_type + ".pth") + + is_safetensors = None + if os.path.isfile(ckpt_path) == False: + is_safetensors = True + ckpt_path = os.path.join(get_model_dir("liveportrait"), model_type + ".safetensors") + if os.path.isfile(ckpt_path) == False: + self.download_model(ckpt_path, + "https://huggingface.co/Kijai/LivePortrait_safetensors/resolve/main/" + model_type + ".safetensors") + model_params = model_config['model_params'][f'{model_type}_params'] + if model_type == 'appearance_feature_extractor': + model = AppearanceFeatureExtractor(**model_params).to(device) + elif model_type == 'motion_extractor': + model = MotionExtractor(**model_params).to(device) + elif model_type == 'warping_module': + model = WarpingNetwork(**model_params).to(device) + elif model_type == 'spade_generator': + model = SPADEDecoder(**model_params).to(device) + elif model_type == 'stitching_retargeting_module': + # Special handling for stitching and retargeting module + config = model_config['model_params']['stitching_retargeting_module_params'] + checkpoint = comfy.utils.load_torch_file(ckpt_path) + + stitcher = StitchingRetargetingNetwork(**config.get('stitching')) + if is_safetensors: + stitcher.load_state_dict(self.filter_for_model(checkpoint, 'retarget_shoulder')) + else: + stitcher.load_state_dict(self.remove_ddp_dumplicate_key(checkpoint['retarget_shoulder'])) + stitcher = stitcher.to(device) + stitcher.eval() + + return { + 'stitching': stitcher, + } + else: + raise ValueError(f"Unknown model type: {model_type}") + + + model.load_state_dict(comfy.utils.load_torch_file(ckpt_path)) + model.eval() + return model + + def load_models(self): + model_path = get_model_dir("liveportrait") + if not os.path.exists(model_path): + os.mkdir(model_path) + + model_config_path = os.path.join(current_directory, 'LivePortrait', 'config', 'models.yaml') + model_config = yaml.safe_load(open(model_config_path, 'r')) - def detect_face(self, image_rgb): + appearance_feature_extractor = self.load_model(model_config, 'appearance_feature_extractor') + motion_extractor = self.load_model(model_config, 'motion_extractor') + warping_module = self.load_model(model_config, 'warping_module') + spade_generator = self.load_model(model_config, 'spade_generator') + stitching_retargeting_module = self.load_model(model_config, 'stitching_retargeting_module') - crop_factor = 1.7 - bbox_drop_size = 10 + self.pipeline = LivePortraitWrapper(InferenceConfig(), appearance_feature_extractor, motion_extractor, warping_module, spade_generator, stitching_retargeting_module) - if self.bbox_model == None: - bbox_model_path = os.path.join(get_model_dir("ultralytics"), "face_yolov8n.pt") - self.bbox_model = YOLO(bbox_model_path) + def get_detect_model(self): + if self.detect_model == None: + model_dir = get_model_dir("ultralytics") + if not os.path.exists(model_dir): os.mkdir(model_dir) + model_path = os.path.join(model_dir, "face_yolov8n.pt") + if not os.path.exists(model_path): + self.download_model(model_path, "https://huggingface.co/Bingsu/adetailer/resolve/main/face_yolov8n.pt") + self.detect_model = YOLO(model_path) - pred = self.bbox_model(image_rgb, conf=0.7, device="") - bboxes = pred[0].boxes.xyxy.cpu().numpy() + return self.detect_model + def get_face_bboxes(self, image_rgb): + detect_model = self.get_detect_model() + pred = detect_model(image_rgb, conf=0.7, device="") + return pred[0].boxes.xyxy.cpu().numpy() + + def detect_face(self, image_rgb, crop_factor, sort = True): + bboxes = self.get_face_bboxes(image_rgb) w, h = get_rgb_size(image_rgb) - # for x, label in zip(segmasks, detected_results[0]): + print(f"w, h:{w, h}") + + cx = w / 2 + min_diff = w + best_box = None for x1, y1, x2, y2 in bboxes: bbox_w = x2 - x1 - bbox_h = y2 - y1 + if bbox_w < 30: continue + diff = abs(cx - (x1 + bbox_w / 2)) + if diff < min_diff: + best_box = [x1, y1, x2, y2] + print(f"diff, min_diff, best_box:{diff, min_diff, best_box}") + min_diff = diff - crop_w = bbox_w * crop_factor - crop_h = bbox_h * crop_factor + if best_box == None: + print("Failed to detect face!!") + return [0, 0, w, h] - crop_w = max(crop_h, crop_w) - crop_h = crop_w + x1, y1, x2, y2 = best_box - kernel_x = x1 + bbox_w / 2 - kernel_y = y1 + bbox_h / 2 + #for x1, y1, x2, y2 in bboxes: + bbox_w = x2 - x1 + bbox_h = y2 - y1 - new_x1, new_x2, crop_w = calc_crop_limit(kernel_x, w, crop_w) + crop_w = bbox_w * crop_factor + crop_h = bbox_h * crop_factor - if crop_w < crop_h: - crop_h = crop_w + crop_w = max(crop_h, crop_w) + crop_h = crop_w - new_y1, new_y2, crop_h = calc_crop_limit(kernel_y, h, crop_h) + kernel_x = int(x1 + bbox_w / 2) + kernel_y = int(y1 + bbox_h / 2) - if crop_h < crop_w: - crop_w = crop_h - new_x1, new_x2, crop_w = calc_crop_limit(kernel_x, w, crop_w) + new_x1 = int(kernel_x - crop_w / 2) + new_x2 = int(kernel_x + crop_w / 2) + new_y1 = int(kernel_y - crop_h / 2) + new_y2 = int(kernel_y + crop_h / 2) + if not sort: return [int(new_x1), int(new_y1), int(new_x2), int(new_y2)] - print("Failed to detect face!!") - return [0, 0, w, h] - - def crop_face(self, rgb_img): - region = self.detect_face(rgb_img) - face_image = rgb_crop(rgb_img, region) - return face_image, region + if new_x1 < 0: + new_x2 -= new_x1 + new_x1 = 0 + elif w < new_x2: + new_x1 -= (new_x2 - w) + new_x2 = w + if new_x1 < 0: + new_x2 -= new_x1 + new_x1 = 0 + + if new_y1 < 0: + new_y2 -= new_y1 + new_y1 = 0 + elif h < new_y2: + new_y1 -= (new_y2 - h) + new_y2 = h + if new_y1 < 0: + new_y2 -= new_y1 + new_y1 = 0 + + if w < new_x2 and h < new_y2: + over_x = new_x2 - w + over_y = new_y2 - h + over_min = min(over_x, over_y) + new_x2 -= over_min + new_y2 -= over_min + + return [int(new_x1), int(new_y1), int(new_x2), int(new_y2)] + + + def calc_face_region(self, square, dsize): + region = copy.deepcopy(square) + is_changed = False + if dsize[0] < region[2]: + region[2] = dsize[0] + is_changed = True + if dsize[1] < region[3]: + region[3] = dsize[1] + is_changed = True + + return region, is_changed + + def expand_img(self, rgb_img, square): + #new_img = rgb_crop(rgb_img, face_region) + crop_trans_m = create_transform_matrix(max(-square[0], 0), max(-square[1], 0), 1, 1) + new_img = cv2.warpAffine(rgb_img, crop_trans_m, (square[2] - square[0], square[3] - square[1]), + cv2.INTER_LINEAR) + return new_img def get_pipeline(self): if self.pipeline == None: print("Load pipeline...") - self.pipeline = LivePortraitWrapper(cfg=InferenceConfig()) + self.load_models() return self.pipeline @@ -138,7 +327,9 @@ def prepare_src_image(self, img): h, w = img.shape[:2] input_shape = [256,256] if h != input_shape[0] or w != input_shape[1]: - x = cv2.resize(img, (input_shape[0], input_shape[1]), interpolation = cv2.INTER_LINEAR) + if 256 < h: interpolation = cv2.INTER_AREA + else: interpolation = cv2.INTER_LINEAR + x = cv2.resize(img, (input_shape[0], input_shape[1]), interpolation = interpolation) else: x = img.copy() @@ -150,30 +341,46 @@ def prepare_src_image(self, img): raise ValueError(f'img ndim should be 3 or 4: {x.ndim}') x = np.clip(x, 0, 1) # clip to 0~1 x = torch.from_numpy(x).permute(0, 3, 1, 2) # 1xHxWx3 -> 1x3xHxW - x = x.cuda() + x = x.to(get_device()) return x - def GetMask(self): + def GetMaskImg(self): if self.mask_img is None: - path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "./LivePortrait/src/utils/resources/mask_template.png") + path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "./LivePortrait/utils/resources/mask_template.png") self.mask_img = cv2.imread(path, cv2.IMREAD_COLOR) return self.mask_img - def prepare_source(self, source_image, is_video = False): + def crop_face(self, img_rgb, crop_factor): + crop_region = self.detect_face(img_rgb, crop_factor) + face_region, is_changed = self.calc_face_region(crop_region, get_rgb_size(img_rgb)) + face_img = rgb_crop(img_rgb, face_region) + if is_changed: face_img = self.expand_img(face_img, crop_region) + return face_img + + def prepare_source(self, source_image, crop_factor, is_video = False, tracking = False): print("Prepare source...") engine = self.get_pipeline() source_image_np = (source_image * 255).byte().numpy() img_rgb = source_image_np[0] - face_img, crop_region = self.crop_face(img_rgb) - - scale = face_img.shape[0] / 512. - crop_trans_m = create_transform_matrix(crop_region[0], crop_region[1], scale) - mask_ori = cv2.warpAffine(self.GetMask(), crop_trans_m, get_rgb_size(img_rgb), cv2.INTER_LINEAR) - mask_ori = mask_ori.astype(np.float32) / 255. psi_list = [] for img_rgb in source_image_np: - face_img = rgb_crop(img_rgb, crop_region) + if tracking or len(psi_list) == 0: + crop_region = self.detect_face(img_rgb, crop_factor) + face_region, is_changed = self.calc_face_region(crop_region, get_rgb_size(img_rgb)) + + s_x = (face_region[2] - face_region[0]) / 512. + s_y = (face_region[3] - face_region[1]) / 512. + crop_trans_m = create_transform_matrix(crop_region[0], crop_region[1], s_x, s_y) + mask_ori = cv2.warpAffine(self.GetMaskImg(), crop_trans_m, get_rgb_size(img_rgb), cv2.INTER_LINEAR) + mask_ori = mask_ori.astype(np.float32) / 255. + + if is_changed: + s = (crop_region[2] - crop_region[0]) / 512. + crop_trans_m = create_transform_matrix(crop_region[0], crop_region[1], s, s) + + face_img = rgb_crop(img_rgb, face_region) + if is_changed: face_img = self.expand_img(face_img, crop_region) i_s = self.prepare_src_image(face_img) x_s_info = engine.get_kp_info(i_s) f_s_user = engine.extract_feature_3d(i_s) @@ -192,9 +399,8 @@ def prepare_driving_video(self, face_images): out_list = [] for f_img in f_img_np: - i_d = pipeline.prepare_source(f_img) + i_d = self.prepare_src_image(f_img) d_info = pipeline.get_kp_info(i_d) - #out_list.append((d_info, get_rotation_matrix(d_info['pitch'], d_info['yaw'], d_info['roll']))) out_list.append(d_info) return out_list @@ -249,6 +455,8 @@ def calc_fe(_, x_d_new, eyes, eyebrow, wink, pupil_x, pupil_y, mouth, eee, woo, x_d_new[0, 13, 1] += eyes * 0.0003 x_d_new[0, 15, 1] += eyes * -0.001 x_d_new[0, 16, 1] += eyes * 0.0003 + x_d_new[0, 1, 1] += eyes * -0.00025 + x_d_new[0, 2, 1] += eyes * 0.00025 if 0 < eyebrow: @@ -277,7 +485,7 @@ def __init__(self, erst = None, es = None): self.s = erst[2] self.t = erst[3] else: - self.e = torch.from_numpy(np.zeros((1, 21, 3))).float().to(device='cuda') + self.e = torch.from_numpy(np.zeros((1, 21, 3))).float().to(get_device()) self.r = torch.Tensor([0, 0, 0]) self.s = 0 self.t = 0 @@ -315,7 +523,10 @@ def wrapper_fn(*args, **kwargs): return wrapper_fn -exp_data_dir = os.path.join(current_directory, "exp_data") +#exp_data_dir = os.path.join(current_directory, "exp_data") +exp_data_dir = os.path.join(folder_paths.output_directory, "exp_data") +if os.path.isdir(exp_data_dir) == False: + os.mkdir(exp_data_dir) class SaveExpData: @classmethod def INPUT_TYPES(s): @@ -328,7 +539,7 @@ def INPUT_TYPES(s): RETURN_TYPES = ("STRING",) RETURN_NAMES = ("file_name",) FUNCTION = "run" - CATEGORY = "파워집돌이" + CATEGORY = "AdvancedLivePortrait" OUTPUT_NODE = True def run(self, file_name, save_exp:ExpressionSet=None): @@ -353,7 +564,7 @@ def INPUT_TYPES(s): RETURN_TYPES = ("EXP_DATA",) RETURN_NAMES = ("exp",) FUNCTION = "run" - CATEGORY = "파워집돌이" + CATEGORY = "AdvancedLivePortrait" def run(self, file_name, ratio): # es = ExpressionSet() @@ -384,25 +595,20 @@ def INPUT_TYPES(s): RETURN_TYPES = ("EXP_DATA",) RETURN_NAMES = ("exp",) FUNCTION = "run" - CATEGORY = "파워집돌이" + CATEGORY = "AdvancedLivePortrait" def run(self, code1, value1, code2, value2, code3, value3, code4, value4, code5, value5, add_exp=None): - #print(f"type(None):{type(None)}") - #if type(add_exp) == type(None): if add_exp == None: es = ExpressionSet() - log(f"exp11:{es.exp[0,1,1]}") else: es = ExpressionSet(es = add_exp) - if id(es.exp) == id(add_exp.exp): - log("id(es.exp) == id(add_exp.exp) is True") codes = [code1, code2, code3, code4, code5] values = [value1, value2, value3, value4, value5] for i in range(5): idx = int(codes[i] / 10) r = codes[i] % 10 - es.exp[0, idx, r] += values[i] * 0.001 + es.e[0, idx, r] += values[i] * 0.001 return (es,) @@ -418,7 +624,7 @@ def INPUT_TYPES(s): RETURN_TYPES = ("EXP_DATA",) RETURN_NAMES = ("exp",) FUNCTION = "run" - CATEGORY = "파워집돌이" + CATEGORY = "AdvancedLivePortrait" OUTPUT_NODE = True def run(self, cut_noise, exp = None): @@ -440,11 +646,17 @@ def __init__(self, es, change, keep): self.es:ExpressionSet = es self.change = change self.keep = keep + +crop_factor_default = 1.7 +crop_factor_min = 1.5 +crop_factor_max = 2.5 + class AdvancedLivePortrait: - def __init__(s): - s.src_images = None - s.driving_images = None - s.pbar = comfy.utils.ProgressBar(1) + def __init__(self): + self.src_images = None + self.driving_images = None + self.pbar = comfy.utils.ProgressBar(1) + self.crop_factor = None @classmethod def INPUT_TYPES(s): @@ -453,7 +665,11 @@ def INPUT_TYPES(s): "required": { "retargeting_eyes": ("FLOAT", {"default": 0, "min": 0, "max": 1, "step": 0.01}), "retargeting_mouth": ("FLOAT", {"default": 0, "min": 0, "max": 1, "step": 0.01}), + "crop_factor": ("FLOAT", {"default": crop_factor_default, + "min": crop_factor_min, "max": crop_factor_max, "step": 0.1}), "turn_on": ("BOOLEAN", {"default": True}), + "tracking_src_vid": ("BOOLEAN", {"default": False}), + "animate_without_vid": ("BOOLEAN", {"default": False}), "command": ("STRING", {"multiline": True, "default": ""}), }, "optional": { @@ -467,7 +683,7 @@ def INPUT_TYPES(s): RETURN_NAMES = ("images",) FUNCTION = "run" OUTPUT_NODE = True - CATEGORY = "파워집돌이" + CATEGORY = "AdvancedLivePortrait" # INPUT_IS_LIST = False # OUTPUT_IS_LIST = (False,) @@ -488,7 +704,6 @@ def parsing_command(self, command, motoin_link): if line == '': continue try: cmds = line.split('=') - #assert len(cmds) == 2, f"(파워집돌이) 명령어오류 {i}번줄: {line}: '=' 기호가 1개 들어가야 합니다" idx = int(cmds[0]) if idx == 0: es = ExpressionSet() else: es = ExpressionSet(es = motoin_link[idx]) @@ -496,8 +711,7 @@ def parsing_command(self, command, motoin_link): change = int(cmds[0]) keep = int(cmds[1]) except: - #log(f"(파워집돌이) 명령어오류 {i}번줄: {line}") - assert False, f"(파워집돌이) 명령어오류 {i}번줄: {line}" + assert False, f"(AdvancedLivePortrait) Command Err Line {i}: {line}" return None, None @@ -509,7 +723,7 @@ def parsing_command(self, command, motoin_link): return cmd_list, total_length - def run(self, retargeting_eyes, retargeting_mouth, turn_on, command, + def run(self, retargeting_eyes, retargeting_mouth, turn_on, tracking_src_vid, animate_without_vid, command, crop_factor, src_images=None, driving_images=None, motion_link=None): if turn_on == False: return (None,None) src_length = 1 @@ -521,12 +735,13 @@ def run(self, retargeting_eyes, retargeting_mouth, turn_on, command, if src_images != None: src_length = len(src_images) - if id(src_images) != id(self.src_images): + if id(src_images) != id(self.src_images) or self.crop_factor != crop_factor: + self.crop_factor = crop_factor self.src_images = src_images if 1 < src_length: - self.psi_list = g_engine.prepare_source(src_images, True) + self.psi_list = g_engine.prepare_source(src_images, crop_factor, True, tracking_src_vid) else: - self.psi_list = [g_engine.prepare_source(src_images)] + self.psi_list = [g_engine.prepare_source(src_images, crop_factor)] cmd_list, cmd_length = self.parsing_command(command, motion_link) @@ -540,9 +755,11 @@ def run(self, retargeting_eyes, retargeting_mouth, turn_on, command, self.driving_values = g_engine.prepare_driving_video(driving_images) driving_length = len(self.driving_values) - #total_length = max(driving_length, cmd_length, src_length) total_length = max(driving_length, src_length) + if animate_without_vid: + total_length = max(total_length, cmd_length) + c_i_es = ExpressionSet() c_o_es = ExpressionSet() d_0_es = None @@ -561,7 +778,6 @@ def run(self, retargeting_eyes, retargeting_mouth, turn_on, command, if i < cmd_length: cmd = cmd_list[cmd_idx] - #cmd = Command()#지울거 if 0 < cmd.change: cmd.change -= 1 c_i_es.add(cmd.es) @@ -586,11 +802,10 @@ def run(self, retargeting_eyes, retargeting_mouth, turn_on, command, if d_0_es is None: d_0_es = ExpressionSet(erst = (d_i_info['exp'], d_i_r, d_i_info['scale'], d_i_info['t'])) - #d_i_es = d_0_es + retargeting(s_es.e, d_0_es.e, retargeting_eyes, (11, 13, 15, 16)) retargeting(s_es.e, d_0_es.e, retargeting_mouth, (14, 17, 19, 20)) - #r_new = (r_d_i @ d_0_es.r.permute(0, 2, 1)) @ r_new new_es.e += d_i_info['exp'] - d_0_es.e new_es.r += d_i_r - d_0_es.r new_es.t += d_i_info['t'] - d_0_es.t @@ -619,6 +834,7 @@ class ExpressionEditor: def __init__(self): self.sample_image = None self.src_image = None + self.crop_factor = None @classmethod def INPUT_TYPES(s): @@ -642,7 +858,10 @@ def INPUT_TYPES(s): "smile": ("FLOAT", {"default": 0, "min": -0.3, "max": 1.3, "step": 0.01, "display": display}), "src_ratio": ("FLOAT", {"default": 1, "min": 0, "max": 1, "step": 0.01, "display": display}), - "sample_ratio": ("FLOAT", {"default": 1, "min": 0, "max": 1, "step": 0.01, "display": display}), + "sample_ratio": ("FLOAT", {"default": 1, "min": -0.2, "max": 1.2, "step": 0.01, "display": display}), + "sample_parts": (["OnlyExpression", "OnlyRotation", "OnlyMouth", "OnlyEyes", "All"],), + "crop_factor": ("FLOAT", {"default": crop_factor_default, + "min": crop_factor_min, "max": crop_factor_max, "step": 0.1}), }, "optional": {"src_image": ("IMAGE",), "motion_link": ("EDITOR_LINK",), @@ -657,13 +876,13 @@ def INPUT_TYPES(s): OUTPUT_NODE = True - CATEGORY = "파워집돌이" + CATEGORY = "AdvancedLivePortrait" # INPUT_IS_LIST = False # OUTPUT_IS_LIST = (False,) def run(self, rotate_pitch, rotate_yaw, rotate_roll, blink, eyebrow, wink, pupil_x, pupil_y, aaa, eee, woo, smile, - src_ratio, sample_ratio, src_image=None, sample_image=None, motion_link=None, add_exp=None): + src_ratio, sample_ratio, sample_parts, crop_factor, src_image=None, sample_image=None, motion_link=None, add_exp=None): rotate_yaw = -rotate_yaw new_editor_link = None @@ -671,8 +890,9 @@ def run(self, rotate_pitch, rotate_yaw, rotate_roll, blink, eyebrow, wink, pupil self.psi = motion_link[0] new_editor_link = motion_link.copy() elif src_image != None: - if id(src_image) != id(self.src_image): - self.psi = g_engine.prepare_source(src_image) + if id(src_image) != id(self.src_image) or self.crop_factor != crop_factor: + self.crop_factor = crop_factor + self.psi = g_engine.prepare_source(src_image, crop_factor) self.src_image = src_image new_editor_link = [] new_editor_link.append(self.psi) @@ -694,14 +914,23 @@ def run(self, rotate_pitch, rotate_yaw, rotate_roll, blink, eyebrow, wink, pupil if id(self.sample_image) != id(sample_image): self.sample_image = sample_image d_image_np = (sample_image * 255).byte().numpy() - d_face, _ = g_engine.crop_face(d_image_np[0]) - i_d = pipeline.prepare_source(d_face) + d_face = g_engine.crop_face(d_image_np[0], 1.7) + i_d = g_engine.prepare_src_image(d_face) self.d_info = pipeline.get_kp_info(i_d) self.d_info['exp'][0, 5, 0] = 0 self.d_info['exp'][0, 5, 1] = 0 - # delta_new += s_exp * (1 - sample_ratio) + self.d_info['exp'] * sample_ratio - es.e += self.d_info['exp'] * sample_ratio + # "OnlyExpression", "OnlyRotation", "OnlyMouth", "OnlyEyes", "All" + if sample_parts == "OnlyExpression" or sample_parts == "All": + es.e += self.d_info['exp'] * sample_ratio + if sample_parts == "OnlyRotation" or sample_parts == "All": + rotate_pitch += self.d_info['pitch'] * sample_ratio + rotate_yaw += self.d_info['yaw'] * sample_ratio + rotate_roll += self.d_info['roll'] * sample_ratio + elif sample_parts == "OnlyMouth": + retargeting(es.e, self.d_info['exp'], sample_ratio, (14, 17, 19, 20)) + elif sample_parts == "OnlyEyes": + retargeting(es.e, self.d_info['exp'], sample_ratio, (1, 2, 11, 13, 15, 16)) es.r = g_engine.calc_fe(es.e, blink, eyebrow, wink, pupil_x, pupil_y, aaa, eee, woo, smile, rotate_pitch, rotate_yaw, rotate_roll) @@ -723,7 +952,7 @@ def run(self, rotate_pitch, rotate_yaw, rotate_roll, blink, eyebrow, wink, pupil out_img = pil2tensor(out) - filename = "fe_edit_preview.png" + filename = g_engine.get_temp_img_name() #"fe_edit_preview.png" folder_paths.get_save_image_path(filename, folder_paths.get_temp_directory()) img = Image.fromarray(crop_out) img.save(os.path.join(folder_paths.get_temp_directory(), filename), compress_level=1) @@ -734,41 +963,307 @@ def run(self, rotate_pitch, rotate_yaw, rotate_roll, blink, eyebrow, wink, pupil return {"ui": {"images": results}, "result": (out_img, new_editor_link, es)} -class TestNode: - def __init__(s): - s.pbar = comfy.utils.ProgressBar(1) +class FlexExpressionEditor(ExpressionEditor): + """ + Enhanced version of ExpressionEditor that adds support for feature-based parameter modulation. + Allows for dynamic modification of facial expressions based on input features (e.g. audio, motion, etc.). + + TODO: Consider refactoring these methods into the base ExpressionEditor class: + - generate_preview_image + - process_sample_image + - create_expression_set + + These methods have been implemented here to avoid modifying the original ExpressionEditor + until the author can review the changes. + """ + + @classmethod + def INPUT_TYPES(cls): + base_inputs = super().INPUT_TYPES() + base_inputs["required"].update({ + "constrain_min_max": ("BOOLEAN", {"default": True}) + }) + # Rename motion_link to flex_motion_link in optional inputs + base_inputs["optional"]["flex_motion_link"] = base_inputs["optional"].pop("motion_link") + base_inputs["optional"].update({ + "feature": ("FEATURE",), + "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.01}), + "feature_threshold": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.01}), + "feature_param": (cls.get_modifiable_params(), {"default": cls.get_modifiable_params()[0]}), + "feature_mode": (["relative", "absolute"], {"default": "absolute"}), + + }) + + return base_inputs + + RETURN_TYPES = ("IMAGE", "EDITOR_LINK", "EXP_DATA", "STRING") + RETURN_NAMES = ("image", "flex_motion_link", "save_exp", "command") + FUNCTION = "run" + OUTPUT_NODE = True + CATEGORY = "AdvancedLivePortrait" @classmethod - def INPUT_TYPES(s): - return {"required": { - #"images": ("IMAGE",), - #"test_value": ("INT", {"default": 1, "min": 1}), - "command": ("STRING", {"multiline": True, "default": ""}), - }, + def get_modifiable_params(cls): + return ["rotate_pitch", "rotate_yaw", "rotate_roll", "blink", "eyebrow", "wink", "pupil_x", "pupil_y", + "aaa", "eee", "woo", "smile", "None"] + + def modulate_param(self, base_value, feature_value, strength, mode="relative", param_name=None, constrain_min_max=True): + """ + Helper method to modulate a parameter based on feature value. + + Args: + base_value: Original parameter value + feature_value: Feature value to modulate with + strength: Modulation strength + mode: Either "relative" or "absolute" + param_name: Name of the parameter being modulated (for min/max constraints) + constrain_min_max: Whether to constrain the output to the parameter's min/max values + + Returns: + Modulated parameter value + """ + if mode == "relative": + modulated_value = base_value * (1 + feature_value * strength) + else: # absolute + modulated_value = base_value * feature_value * strength + + if constrain_min_max and param_name: + # Get parameter constraints from INPUT_TYPES + param_info = self.INPUT_TYPES()["required"].get(param_name) + if param_info and isinstance(param_info[1], dict): + param_min = param_info[1].get("min") + param_max = param_info[1].get("max") + if param_min is not None: + modulated_value = max(param_min, modulated_value) + if param_max is not None: + modulated_value = min(param_max, modulated_value) + + return modulated_value + + #TODO: base class + def generate_preview_image(self, psi, preview_es): + """ + Generate preview image from the expression set and PSI data. + + Args: + psi: PSI data containing source image information + preview_es: Expression set to apply for preview + + Returns: + tuple: (output_image_tensor, results_dict) + """ + s_info = psi.x_s_info + pipeline = g_engine.get_pipeline() + + new_rotate = get_rotation_matrix(s_info['pitch'] + preview_es.r[0], + s_info['yaw'] + preview_es.r[1], + s_info['roll'] + preview_es.r[2]) + x_d_new = (s_info['scale'] * (1 + preview_es.s)) * ((s_info['kp'] + preview_es.e) @ new_rotate) + s_info['t'] + + x_d_new = pipeline.stitching(psi.x_s_user, x_d_new) + crop_out = pipeline.warp_decode(psi.f_s_user, psi.x_s_user, x_d_new) + crop_out = pipeline.parse_output(crop_out['out'])[0] + crop_with_fullsize = cv2.warpAffine(crop_out, psi.crop_trans_m, get_rgb_size(psi.src_rgb), cv2.INTER_LINEAR) + out = np.clip(psi.mask_ori * crop_with_fullsize + (1 - psi.mask_ori) * psi.src_rgb, 0, 255).astype(np.uint8) + out_img = pil2tensor(out) + + filename = g_engine.get_temp_img_name() + folder_paths.get_save_image_path(filename, folder_paths.get_temp_directory()) + img = Image.fromarray(crop_out) + img.save(os.path.join(folder_paths.get_temp_directory(), filename), compress_level=1) + results = [{"filename": filename, "type": "temp"}] + + return out_img, results + + #TODO: base class + def initialize_psi(self, src_image, motion_link, crop_factor): + """Initialize PSI data from source image or motion link""" + new_editor_link = [] + + if motion_link is not None: + self.psi = motion_link[0] + new_editor_link.append(self.psi) + elif src_image is not None: + if id(src_image) != id(self.src_image) or self.crop_factor != crop_factor: + self.crop_factor = crop_factor + self.psi = g_engine.prepare_source(src_image, crop_factor) + self.src_image = src_image + new_editor_link.append(self.psi) + else: + return None + + return new_editor_link + + #TODO: base class + def process_sample_image(self, sample_image, sample_parts, sample_ratio, es, rotate_pitch, rotate_yaw, rotate_roll): + """Process sample image and apply transformations""" + pipeline = g_engine.get_pipeline() + + if id(self.sample_image) != id(sample_image): + self.sample_image = sample_image + d_image_np = (sample_image * 255).byte().numpy() + d_face = g_engine.crop_face(d_image_np[0], 1.7) + i_d = g_engine.prepare_src_image(d_face) + self.d_info = pipeline.get_kp_info(i_d) + self.d_info['exp'][0, 5, 0] = 0 + self.d_info['exp'][0, 5, 1] = 0 + + if sample_parts == "OnlyExpression" or sample_parts == "All": + es.e += self.d_info['exp'] * sample_ratio + if sample_parts == "OnlyRotation" or sample_parts == "All": + rotate_pitch += self.d_info['pitch'] * sample_ratio + rotate_yaw += self.d_info['yaw'] * sample_ratio + rotate_roll += self.d_info['roll'] * sample_ratio + elif sample_parts == "OnlyMouth": + retargeting(es.e, self.d_info['exp'], sample_ratio, (14, 17, 19, 20)) + elif sample_parts == "OnlyEyes": + retargeting(es.e, self.d_info['exp'], sample_ratio, (1, 2, 11, 13, 15, 16)) + + return rotate_pitch, rotate_yaw, rotate_roll + + def run(self, rotate_pitch, rotate_yaw, rotate_roll, blink, eyebrow, wink, pupil_x, pupil_y, aaa, eee, woo, smile, + src_ratio, sample_ratio, sample_parts, crop_factor, constrain_min_max, src_image=None, sample_image=None, + flex_motion_link=None, add_exp=None, feature=None, strength=1.0, feature_threshold=0.0, + feature_param="None", feature_mode="relative"): + rotate_yaw = -rotate_yaw + + # Initialize PSI data + new_editor_link = self.initialize_psi(src_image, flex_motion_link, crop_factor) + if new_editor_link is None: + return (None, None, None, None) + + pipeline = g_engine.get_pipeline() + psi = self.psi + s_info = psi.x_s_info + + es = ExpressionSet() + + # Process sample image if provided + if sample_image is not None: + rotate_pitch, rotate_yaw, rotate_roll = self.process_sample_image( + sample_image, sample_parts, sample_ratio, es, rotate_pitch, rotate_yaw, rotate_roll + ) + + # Add any additional expression data + if add_exp is not None: + es.add(add_exp) + + # Prepare parameters dictionary + params = { + 'blink': blink, + 'eyebrow': eyebrow, + 'wink': wink, + 'pupil_x': pupil_x, + 'pupil_y': pupil_y, + 'aaa': aaa, + 'eee': eee, + 'woo': woo, + 'smile': smile, + 'rotate_pitch': rotate_pitch, + 'rotate_yaw': rotate_yaw, + 'rotate_roll': rotate_roll, } - RETURN_TYPES = ("IMAGE",) - RETURN_NAMES = ("images",) - FUNCTION = "run" - OUTPUT_NODE = True + # When calling calc_fe, unpack the parameters in the correct order + def apply_params(es, params): + return g_engine.calc_fe( + es, + params['blink'], + params['eyebrow'], + params['wink'], + params['pupil_x'], + params['pupil_y'], + params['aaa'], + params['eee'], + params['woo'], + params['smile'], + params['rotate_pitch'], + params['rotate_yaw'], + params['rotate_roll'] + ) + + # Check if feature is provided for modulation + if (feature is not None) and feature_param != "None" and feature_param in params: + total_frames = feature.frame_count + feature_values = [feature.get_value_at_frame(i) for i in range(total_frames)] + else: + # No modulation + total_frames = 1 + if flex_motion_link is not None and len(flex_motion_link) > 1: + total_frames = len(flex_motion_link) - 1 # Subtract 1 because first element is psi + feature_values = [1.0] * total_frames + + # Generate 'command' string expected by AdvancedLivePortrait node + command_lines = [] + for idx in range(total_frames): + # Start with base expression set or previous modifications + if flex_motion_link is not None and idx + 1 < len(flex_motion_link): + es_frame = ExpressionSet(es=flex_motion_link[idx + 1]) + if add_exp is not None: + es_frame.add(add_exp) + else: + es_frame = ExpressionSet(es=es) + + # Apply feature modulation if available + frame_params = params.copy() + if (feature is not None) and feature_param != "None" and feature_param in params: + feature_value = feature_values[idx] + if abs(feature_value) >= feature_threshold: + frame_params[feature_param] = self.modulate_param( + params[feature_param], + feature_value, + strength, + feature_mode, + feature_param, + constrain_min_max + ) + + # Calculate the new rotations and add them to existing ones + new_rotations = apply_params(es_frame.e, frame_params) + if flex_motion_link is not None and idx + 1 < len(flex_motion_link): + es_frame.r += new_rotations # Add to existing rotations + else: + es_frame.r = new_rotations # Set new rotations + + new_editor_link.append(es_frame) + + # Create command line + command_lines.append(f"{idx}=1:0") + + command = '\n'.join(command_lines) + + # Apply the expressions for preview image (using first frame) + preview_es = new_editor_link[1] if len(new_editor_link) > 1 else es + out_img, results = self.generate_preview_image(psi, preview_es) + + # Find the frame with maximum feature value and use its expression + if feature is not None and feature_param != "None" and len(new_editor_link) > 1: + max_idx = feature_values.index(max(feature_values)) + save_exp = new_editor_link[max_idx + 1] # +1 because first element is psi + else: + save_exp = es + + return {"ui": {"images": results}, "result": (out_img, new_editor_link, save_exp, command)} + + - def run(self, command): - self.parsing_command(command) - return (None,) NODE_CLASS_MAPPINGS = { "AdvancedLivePortrait": AdvancedLivePortrait, "ExpressionEditor": ExpressionEditor, - "ExpData": ExpData, "LoadExpData": LoadExpData, "SaveExpData": SaveExpData, + "ExpData": ExpData, "PrintExpData:": PrintExpData, - #"TestNode": TestNode, + "FlexExpressionEditor": FlexExpressionEditor, } NODE_DISPLAY_NAME_MAPPINGS = { - "AdvancedLivePortrait": "Advanced Live Portrait (파워집돌이)", - "ExpressionEditor": "Expression Editor (파워집돌이)", + "AdvancedLivePortrait": "Advanced Live Portrait (PHM)", + "ExpressionEditor": "Expression Editor (PHM)", + "LoadExpData": "Load Exp Data (PHM)", + "SaveExpData": "Save Exp Data (PHM)", + "FlexExpressionEditor": "Flex Expression Editor (RyanOnTheInside)", } \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5b3c401 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "comfyui-advancedliveportrait" +description = "AdvancedLivePortrait with Facial expression editor" +version = "1.0.0" +license = {file = "LICENSE"} +dependencies = ["numpy>=1.26.4", "opencv-python-headless", "imageio-ffmpeg>=0.5.1", "lmdb>=1.4.1", "timm>=1.0.7", "rich>=13.7.1", "albumentations>=1.4.10", "ultralytics", "tyro==0.8.5", "dill"] + +[project.urls] +Repository = "https://github.com/PowerHouseMan/ComfyUI-AdvancedLivePortrait" +# Used by Comfy Registry https://comfyregistry.org + +[tool.comfy] +PublisherId = "starmapking" +DisplayName = "ComfyUI-AdvancedLivePortrait" +Icon = "" diff --git a/sample/exp_image.png b/sample/exp_image.png new file mode 100644 index 0000000..b0a719b Binary files /dev/null and b/sample/exp_image.png differ diff --git a/sample/original_sample_asset/driving/d0.mp4 b/sample/original_sample_asset/driving/d0.mp4 new file mode 100644 index 0000000..f8bb7eb Binary files /dev/null and b/sample/original_sample_asset/driving/d0.mp4 differ diff --git a/sample/original_sample_asset/driving/d1.mp4 b/sample/original_sample_asset/driving/d1.mp4 new file mode 100644 index 0000000..e2825c1 Binary files /dev/null and b/sample/original_sample_asset/driving/d1.mp4 differ diff --git a/sample/original_sample_asset/driving/d2.mp4 b/sample/original_sample_asset/driving/d2.mp4 new file mode 100644 index 0000000..a14da2d Binary files /dev/null and b/sample/original_sample_asset/driving/d2.mp4 differ diff --git a/sample/original_sample_asset/driving/d3.mp4 b/sample/original_sample_asset/driving/d3.mp4 new file mode 100644 index 0000000..3a1965e Binary files /dev/null and b/sample/original_sample_asset/driving/d3.mp4 differ diff --git a/sample/original_sample_asset/driving/d5.mp4 b/sample/original_sample_asset/driving/d5.mp4 new file mode 100644 index 0000000..332bc88 Binary files /dev/null and b/sample/original_sample_asset/driving/d5.mp4 differ diff --git a/sample/original_sample_asset/driving/d6.mp4 b/sample/original_sample_asset/driving/d6.mp4 new file mode 100644 index 0000000..df80c60 Binary files /dev/null and b/sample/original_sample_asset/driving/d6.mp4 differ diff --git a/sample/original_sample_asset/driving/d7.mp4 b/sample/original_sample_asset/driving/d7.mp4 new file mode 100644 index 0000000..81b5ae1 Binary files /dev/null and b/sample/original_sample_asset/driving/d7.mp4 differ diff --git a/sample/original_sample_asset/driving/d8.mp4 b/sample/original_sample_asset/driving/d8.mp4 new file mode 100644 index 0000000..7fabdde Binary files /dev/null and b/sample/original_sample_asset/driving/d8.mp4 differ diff --git a/sample/original_sample_asset/driving/d9.mp4 b/sample/original_sample_asset/driving/d9.mp4 new file mode 100644 index 0000000..966aa37 Binary files /dev/null and b/sample/original_sample_asset/driving/d9.mp4 differ diff --git a/sample/original_sample_asset/source/s0.jpg b/sample/original_sample_asset/source/s0.jpg new file mode 100644 index 0000000..ef44c59 Binary files /dev/null and b/sample/original_sample_asset/source/s0.jpg differ diff --git a/sample/original_sample_asset/source/s1.jpg b/sample/original_sample_asset/source/s1.jpg new file mode 100644 index 0000000..ebacda3 Binary files /dev/null and b/sample/original_sample_asset/source/s1.jpg differ diff --git a/sample/original_sample_asset/source/s10.jpg b/sample/original_sample_asset/source/s10.jpg new file mode 100644 index 0000000..ee9616b Binary files /dev/null and b/sample/original_sample_asset/source/s10.jpg differ diff --git a/sample/original_sample_asset/source/s2.jpg b/sample/original_sample_asset/source/s2.jpg new file mode 100644 index 0000000..e851bd2 Binary files /dev/null and b/sample/original_sample_asset/source/s2.jpg differ diff --git a/sample/original_sample_asset/source/s3.jpg b/sample/original_sample_asset/source/s3.jpg new file mode 100644 index 0000000..9f3ba2a Binary files /dev/null and b/sample/original_sample_asset/source/s3.jpg differ diff --git a/sample/original_sample_asset/source/s4.jpg b/sample/original_sample_asset/source/s4.jpg new file mode 100644 index 0000000..17f611b Binary files /dev/null and b/sample/original_sample_asset/source/s4.jpg differ diff --git a/sample/original_sample_asset/source/s5.jpg b/sample/original_sample_asset/source/s5.jpg new file mode 100644 index 0000000..9abad7e Binary files /dev/null and b/sample/original_sample_asset/source/s5.jpg differ diff --git a/sample/original_sample_asset/source/s6.jpg b/sample/original_sample_asset/source/s6.jpg new file mode 100644 index 0000000..91c13d5 Binary files /dev/null and b/sample/original_sample_asset/source/s6.jpg differ diff --git a/sample/original_sample_asset/source/s7.jpg b/sample/original_sample_asset/source/s7.jpg new file mode 100644 index 0000000..cf96f2d Binary files /dev/null and b/sample/original_sample_asset/source/s7.jpg differ diff --git a/sample/original_sample_asset/source/s8.jpg b/sample/original_sample_asset/source/s8.jpg new file mode 100644 index 0000000..b415ed1 Binary files /dev/null and b/sample/original_sample_asset/source/s8.jpg differ diff --git a/sample/original_sample_asset/source/s9.jpg b/sample/original_sample_asset/source/s9.jpg new file mode 100644 index 0000000..3ef7251 Binary files /dev/null and b/sample/original_sample_asset/source/s9.jpg differ diff --git a/sample/royalty_free.mp3 b/sample/royalty_free.mp3 new file mode 100644 index 0000000..8ae727f Binary files /dev/null and b/sample/royalty_free.mp3 differ diff --git "a/exp_data/\354\236\205\354\210\240\353\202\264\353\260\200\352\270\260.exp" b/sample/sample_exp_data.exp similarity index 100% rename from "exp_data/\354\236\205\354\210\240\353\202\264\353\260\200\352\270\260.exp" rename to sample/sample_exp_data.exp diff --git a/sample/workflow_advanced_screenshot.png b/sample/workflow_advanced_screenshot.png new file mode 100644 index 0000000..755b0cb Binary files /dev/null and b/sample/workflow_advanced_screenshot.png differ diff --git a/sample/workflow_basic_screenshot.png b/sample/workflow_basic_screenshot.png new file mode 100644 index 0000000..6894fa2 Binary files /dev/null and b/sample/workflow_basic_screenshot.png differ diff --git a/sample/workflows/Advanced_Animate_without_vid.json b/sample/workflows/Advanced_Animate_without_vid.json new file mode 100644 index 0000000..e0ff6cb --- /dev/null +++ b/sample/workflows/Advanced_Animate_without_vid.json @@ -0,0 +1,359 @@ +{ + "last_node_id": 18, + "last_link_id": 11, + "nodes": [ + { + "id": 3, + "type": "LoadImage", + "pos": [ + 1385, + 278 + ], + "size": [ + 223.20395091844193, + 308.6326788490376 + ], + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 11 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "slot_index": 1, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "source_image.png", + "image" + ] + }, + { + "id": 7, + "type": "AdvancedLivePortrait", + "pos": [ + 1889, + 270 + ], + "size": { + "0": 235.1999969482422, + "1": 523.0364379882812 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "src_images", + "type": "IMAGE", + "link": null + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "link": 10 + }, + { + "name": "driving_images", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 8 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "AdvancedLivePortrait" + }, + "widgets_values": [ + 0, + 0, + 1.7000000000000002, + true, + false, + true, + "1 = 5:5\n0 = 3:5" + ] + }, + { + "id": 10, + "type": "VHS_VideoCombine", + "pos": [ + 2163, + 203 + ], + "size": [ + 261.79998779296875, + 565.7999877929688 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 8 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 15, + "loop_count": 0, + "filename_prefix": "AdvancedLivePortrait", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AdvancedLivePortrait_00021.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 15 + } + } + } + }, + { + "id": 15, + "type": "ExpressionEditor", + "pos": [ + 1643, + 271 + ], + "size": { + "0": 210, + "1": 690 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "src_image", + "type": "IMAGE", + "link": 11 + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "link": null + }, + { + "name": "sample_image", + "type": "IMAGE", + "link": null + }, + { + "name": "add_exp", + "type": "EXP_DATA", + "link": null + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": null, + "shape": 3 + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "links": [ + 10 + ], + "slot_index": 1, + "shape": 3 + }, + { + "name": "save_exp", + "type": "EXP_DATA", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ExpressionEditor" + }, + "widgets_values": [ + 0, + 0, + 0, + 0, + 0, + 25, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + "OnlyExpression", + 1.7 + ] + }, + { + "id": 16, + "type": "Note", + "pos": [ + 1892, + 831 + ], + "size": [ + 483.8180073265835, + 161.2032413725933 + ], + "flags": {}, + "order": 3, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Command Description\n1 = 5:5\n0 = 3:5\n\n[Motion index] = [Changing frame length] : [Length of frames waiting for next motion]\n\nfor example, first line -> 1 = 5:5\nChange over 5 frames with motion 1, and wait for the next motion for 5 frames" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 17, + "type": "Note", + "pos": [ + 1629, + 996 + ], + "size": [ + 227.12395091844178, + 58 + ], + "flags": {}, + "order": 2, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "motion 1" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 18, + "type": "Note", + "pos": [ + 1373, + 625 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "orginal src : motion 0\n" + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 8, + 7, + 0, + 10, + 0, + "IMAGE" + ], + [ + 10, + 15, + 1, + 7, + 1, + "EDITOR_LINK" + ], + [ + 11, + 3, + 0, + 15, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 1.1000000000000005, + "offset": [ + -841.7371241966999, + 104.19308875020204 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/sample/sample2.json b/sample/workflows/Advanced_Inserting_expressions_into_vid.json similarity index 71% rename from sample/sample2.json rename to sample/workflows/Advanced_Inserting_expressions_into_vid.json index 6c1c152..7810b6f 100644 --- a/sample/sample2.json +++ b/sample/workflows/Advanced_Inserting_expressions_into_vid.json @@ -1,12 +1,12 @@ { - "last_node_id": 10, + "last_node_id": 14, "last_link_id": 8, "nodes": [ { "id": 3, "type": "LoadImage", "pos": [ - 465, + 736, 299 ], "size": { @@ -23,15 +23,15 @@ "links": [ 3 ], - "shape": 3, - "slot_index": 0 + "slot_index": 0, + "shape": 3 }, { "name": "MASK", "type": "MASK", "links": null, - "shape": 3, - "slot_index": 1 + "slot_index": 1, + "shape": 3 } ], "properties": { @@ -46,15 +46,15 @@ "id": 6, "type": "ExpressionEditor", "pos": [ - 844, - 301 + 1085, + 294 ], "size": { - "0": 315, - "1": 642 + "0": 210, + "1": 690 }, "flags": {}, - "order": 2, + "order": 6, "mode": 0, "inputs": [ { @@ -91,8 +91,8 @@ "links": [ 5 ], - "shape": 3, - "slot_index": 1 + "slot_index": 1, + "shape": 3 }, { "name": "save_exp", @@ -118,22 +118,79 @@ 0, 0, 1, - 1 + 1, + "OnlyExpression", + 1.7000000000000002 + ] + }, + { + "id": 7, + "type": "AdvancedLivePortrait", + "pos": [ + 1845, + 304 + ], + "size": { + "0": 235.1999969482422, + "1": 523.0364379882812 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "src_images", + "type": "IMAGE", + "link": null + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "link": 6 + }, + { + "name": "driving_images", + "type": "IMAGE", + "link": 7 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 8 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "AdvancedLivePortrait" + }, + "widgets_values": [ + 0, + 0, + 1.7000000000000002, + "1 = 1:10\n2 = 5:10\n0 = 2:50\n1 = 2:0", + "1 = 1:10\n2 = 5:10\n0 = 2:50\n1 = 2:0", + false, + "1 = 1:10\n2 = 5:10\n0 = 2:50\n1 = 2:0" ] }, { "id": 8, "type": "ExpressionEditor", "pos": [ - 1205, - 299 + 1342, + 304 ], "size": { - "0": 315, - "1": 642 + "0": 210, + "1": 690 }, "flags": {}, - "order": 3, + "order": 7, "mode": 0, "inputs": [ { @@ -170,8 +227,8 @@ "links": [ 6 ], - "shape": 3, - "slot_index": 1 + "slot_index": 1, + "shape": 3 }, { "name": "save_exp", @@ -197,14 +254,16 @@ 0, 0, 1, - 1 + 1, + "OnlyExpression", + 1.7 ] }, { "id": 9, "type": "VHS_LoadVideo", "pos": [ - 1551, + 1570, 399 ], "size": [ @@ -233,8 +292,8 @@ "links": [ 7 ], - "shape": 3, - "slot_index": 0 + "slot_index": 0, + "shape": 3 }, { "name": "frame_count", @@ -283,71 +342,19 @@ } } }, - { - "id": 7, - "type": "AdvancedLivePortrait", - "pos": [ - 1823, - 301 - ], - "size": [ - 289.7234384275689, - 513.2487798331999 - ], - "flags": {}, - "order": 4, - "mode": 0, - "inputs": [ - { - "name": "src_images", - "type": "IMAGE", - "link": null - }, - { - "name": "motion_link", - "type": "EDITOR_LINK", - "link": 6 - }, - { - "name": "driving_images", - "type": "IMAGE", - "link": 7 - } - ], - "outputs": [ - { - "name": "images", - "type": "IMAGE", - "links": [ - 8 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "AdvancedLivePortrait" - }, - "widgets_values": [ - 0, - 0, - true, - "1 = 1:10\n2 = 5:10\n0 = 2:50\n1 = 2:0" - ] - }, { "id": 10, "type": "VHS_VideoCombine", "pos": [ - 2150, - 307 + 2117, + 305 ], "size": [ - 261.8000000000002, - 565.8000000000002 + 261.79998779296875, + 565.7999877929688 ], "flags": {}, - "order": 5, + "order": 9, "mode": 0, "inputs": [ { @@ -385,25 +392,119 @@ "widgets_values": { "frame_rate": 15, "loop_count": 0, - "filename_prefix": "AnimateDiff", + "filename_prefix": "AdvancedLivePortrait", "format": "video/h264-mp4", "pix_fmt": "yuv420p", "crf": 19, "save_metadata": true, "pingpong": false, - "save_output": false, + "save_output": true, "videopreview": { "hidden": false, "paused": false, "params": { - "filename": "AnimateDiff_00003.mp4", + "filename": "AdvancedLivePortrait_00023.mp4", "subfolder": "", - "type": "temp", + "type": "output", "format": "video/h264-mp4", "frame_rate": 15 } } } + }, + { + "id": 11, + "type": "Note", + "pos": [ + 1834, + 935 + ], + "size": { + "0": 539.0599975585938, + "1": 184.96444702148438 + }, + "flags": {}, + "order": 5, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Command Description\n1 = 1:10\n2 = 5:10\n0 = 2:50\n1 = 2:0\n\n[Motion index] = [Changing frame length] : [Length of frames waiting for next motion]\n\nfor example, second line -> 2 = 5:10\nChange over 5 frames with motion 2, and wait for the next motion for 10 frames\n\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 12, + "type": "Note", + "pos": [ + 1089, + 1038 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "motion1\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 13, + "type": "Note", + "pos": [ + 1341, + 1041 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": { + "collapsed": false + }, + "order": 3, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "motion2\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 14, + "type": "Note", + "pos": [ + 776, + 684 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 4, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "orginal src : motion0\n" + ], + "color": "#432", + "bgcolor": "#653" } ], "links": [ @@ -452,10 +553,10 @@ "config": {}, "extra": { "ds": { - "scale": 0.620921323059155, + "scale": 1.1000000000000005, "offset": [ - 501.16957927340843, - 325.0434083711576 + -678.8740830330757, + -35.741089089667945 ] } }, diff --git a/sample/workflows/Bacis_Extracting_expression_from_photo.json b/sample/workflows/Bacis_Extracting_expression_from_photo.json new file mode 100644 index 0000000..e22f5c4 --- /dev/null +++ b/sample/workflows/Bacis_Extracting_expression_from_photo.json @@ -0,0 +1,241 @@ +{ + "last_node_id": 33, + "last_link_id": 39, + "nodes": [ + { + "id": 19, + "type": "LoadImage", + "pos": [ + 1022, + 702 + ], + "size": { + "0": 220.3717041015625, + "1": 314 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 25 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "exp_image.png", + "image" + ] + }, + { + "id": 20, + "type": "LoadImage", + "pos": [ + 1027.712857031252, + 322.70146904296865 + ], + "size": { + "0": 210.9286651611328, + "1": 314 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 26 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "source_image.png", + "image" + ] + }, + { + "id": 21, + "type": "ExpressionEditor", + "pos": [ + 1305, + 338 + ], + "size": [ + 252.62867736816406, + 690 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "src_image", + "type": "IMAGE", + "link": 26 + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "link": null + }, + { + "name": "sample_image", + "type": "IMAGE", + "link": 25 + }, + { + "name": "add_exp", + "type": "EXP_DATA", + "link": null + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 39 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "links": null, + "shape": 3 + }, + { + "name": "save_exp", + "type": "EXP_DATA", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ExpressionEditor" + }, + "widgets_values": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + "OnlyExpression", + 1.7 + ] + }, + { + "id": 33, + "type": "SaveImage", + "pos": [ + 1617, + 390 + ], + "size": [ + 263.6363636363635, + 278.18181818181813 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 39 + } + ], + "properties": {}, + "widgets_values": [ + "ExpressionEditing" + ] + } + ], + "links": [ + [ + 25, + 19, + 0, + 21, + 2, + "IMAGE" + ], + [ + 26, + 20, + 0, + 21, + 0, + "IMAGE" + ], + [ + 39, + 21, + 0, + 33, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Extracting facial expressions from photos", + "bounding": [ + 970, + 233, + 939, + 823 + ], + "color": "#a1309b", + "font_size": 30 + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 1, + "offset": [ + -256.9678555487616, + -36.006961063159906 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/sample/sample.json b/sample/workflows/Bacis_Liveportrait_with_expression_editing.json similarity index 80% rename from sample/sample.json rename to sample/workflows/Bacis_Liveportrait_with_expression_editing.json index 8085518..f3eaebf 100644 --- a/sample/sample.json +++ b/sample/workflows/Bacis_Liveportrait_with_expression_editing.json @@ -1,18 +1,140 @@ { - "last_node_id": 9, - "last_link_id": 8, + "last_node_id": 32, + "last_link_id": 38, "nodes": [ { - "id": 5, - "type": "AdvancedLivePortrait", + "id": 26, + "type": "LoadImage", "pos": [ - 1464, - 324 + 574.0879504882818, + 354.13510488281264 ], "size": { - "0": 263.14520263671875, - "1": 417.1697998046875 + "0": 210, + "1": 314 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 30 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "source_image.png", + "image" + ] + }, + { + "id": 27, + "type": "VHS_LoadVideo", + "pos": [ + 1113, + 443 + ], + "size": [ + 235.1999969482422, + 491.1999969482422 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 31 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" }, + "widgets_values": { + "video": "driving_video.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 2, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "driving_video.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 2 + } + } + } + }, + { + "id": 28, + "type": "AdvancedLivePortrait", + "pos": [ + 1386, + 353 + ], + "size": [ + 235.1999969482422, + 674.0390319824219 + ], "flags": {}, "order": 3, "mode": 0, @@ -20,7 +142,7 @@ { "name": "src_images", "type": "IMAGE", - "link": 8 + "link": 38 }, { "name": "motion_link", @@ -30,8 +152,7 @@ { "name": "driving_images", "type": "IMAGE", - "link": 3, - "slot_index": 2 + "link": 31 } ], "outputs": [ @@ -39,10 +160,10 @@ "name": "images", "type": "IMAGE", "links": [ - 5 + 35 ], - "shape": 3, - "slot_index": 0 + "slot_index": 0, + "shape": 3 } ], "properties": { @@ -51,20 +172,23 @@ "widgets_values": [ 0, 0, + 1.5, true, + "", + false, "" ] }, { - "id": 8, + "id": 29, "type": "VHS_VideoCombine", "pos": [ - 1760, - 327 + 1664, + 357 ], "size": [ - 248.96339416503906, - 552.9633941650391 + 272.9300842285156, + 576.9300842285156 ], "flags": {}, "order": 4, @@ -73,7 +197,7 @@ { "name": "images", "type": "IMAGE", - "link": 5 + "link": 35 }, { "name": "audio", @@ -105,7 +229,7 @@ "widgets_values": { "frame_rate": 15, "loop_count": 0, - "filename_prefix": "AnimateDiff", + "filename_prefix": "AdvancedLivePortrait", "format": "video/h264-mp4", "pix_fmt": "yuv420p", "crf": 19, @@ -116,9 +240,9 @@ "hidden": false, "paused": false, "params": { - "filename": "AnimateDiff_00027.mp4", + "filename": "AdvancedLivePortrait_00001.mp4", "subfolder": "", - "type": "output", + "type": "temp", "format": "video/h264-mp4", "frame_rate": 15 } @@ -126,15 +250,15 @@ } }, { - "id": 9, + "id": 30, "type": "ExpressionEditor", "pos": [ - 932, - 333 + 837.0879504882818, + 349.1351048828126 ], "size": { - "0": 245.65225219726562, - "1": 642 + "0": 256.8715515136719, + "1": 690 }, "flags": {}, "order": 2, @@ -143,7 +267,7 @@ { "name": "src_image", "type": "IMAGE", - "link": 7 + "link": 30 }, { "name": "motion_link", @@ -166,10 +290,10 @@ "name": "image", "type": "IMAGE", "links": [ - 8 + 38 ], - "shape": 3, - "slot_index": 0 + "slot_index": 0, + "shape": 3 }, { "name": "motion_link", @@ -201,174 +325,66 @@ 0, 1.3, 1, - 1 - ] - }, - { - "id": 3, - "type": "LoadImage", - "pos": [ - 689, - 328 - ], - "size": { - "0": 210, - "1": 314 - }, - "flags": {}, - "order": 0, - "mode": 0, - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 7 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": null, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "source_image.png", - "image" + 1, + "OnlyExpression", + 1.7000000000000002 ] - }, - { - "id": 6, - "type": "VHS_LoadVideo", - "pos": [ - 1200, - 393 - ], - "size": [ - 235.1999969482422, - 491.1999969482422 - ], - "flags": {}, - "order": 1, - "mode": 0, - "inputs": [ - { - "name": "meta_batch", - "type": "VHS_BatchManager", - "link": null - }, - { - "name": "vae", - "type": "VAE", - "link": null - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 3 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "frame_count", - "type": "INT", - "links": null, - "shape": 3 - }, - { - "name": "audio", - "type": "AUDIO", - "links": null, - "shape": 3 - }, - { - "name": "video_info", - "type": "VHS_VIDEOINFO", - "links": null, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "VHS_LoadVideo" - }, - "widgets_values": { - "video": "driving_video.mp4", - "force_rate": 0, - "force_size": "Disabled", - "custom_width": 512, - "custom_height": 512, - "frame_load_cap": 0, - "skip_first_frames": 0, - "select_every_nth": 2, - "choose video to upload": "image", - "videopreview": { - "hidden": false, - "paused": false, - "params": { - "frame_load_cap": 0, - "skip_first_frames": 0, - "force_rate": 0, - "filename": "driving_video.mp4", - "type": "input", - "format": "video/mp4", - "select_every_nth": 2 - } - } - } } ], "links": [ [ - 3, - 6, + 30, + 26, + 0, + 30, 0, - 5, - 2, "IMAGE" ], [ - 5, - 5, - 0, - 8, + 31, + 27, 0, + 28, + 2, "IMAGE" ], [ - 7, - 3, + 35, + 28, 0, - 9, + 29, 0, "IMAGE" ], [ - 8, - 9, + 38, + 30, 0, - 5, + 28, 0, "IMAGE" ] ], - "groups": [], + "groups": [ + { + "title": "Live Portrait with facial expression editing", + "bounding": [ + 555, + 266, + 1427, + 789 + ], + "color": "#3f789e", + "font_size": 30 + } + ], "config": {}, "extra": { "ds": { "scale": 0.8264462809917354, "offset": [ - 245.85866448431153, - 197.69871165469377 + 185.1062862025528, + 195.77375741392547 ] } }, diff --git a/sample/workflows/Bacis_Simple_expression_editing.json b/sample/workflows/Bacis_Simple_expression_editing.json new file mode 100644 index 0000000..408833d --- /dev/null +++ b/sample/workflows/Bacis_Simple_expression_editing.json @@ -0,0 +1,194 @@ +{ + "last_node_id": 33, + "last_link_id": 40, + "nodes": [ + { + "id": 14, + "type": "ExpressionEditor", + "pos": [ + 1044.0000000000002, + 362.78992797851555 + ], + "size": [ + 260.81048583984375, + 690 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "src_image", + "type": "IMAGE", + "link": 19 + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "link": null + }, + { + "name": "sample_image", + "type": "IMAGE", + "link": null + }, + { + "name": "add_exp", + "type": "EXP_DATA", + "link": null + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 40 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "links": null, + "shape": 3 + }, + { + "name": "save_exp", + "type": "EXP_DATA", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ExpressionEditor" + }, + "widgets_values": [ + 0, + 0, + 0, + 0, + 0, + 23.5, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + "OnlyExpression", + 1.7000000000000002 + ] + }, + { + "id": 15, + "type": "LoadImage", + "pos": [ + 776, + 366.78992797851555 + ], + "size": { + "0": 210, + "1": 314 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 19 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "source_image.png", + "image" + ] + }, + { + "id": 33, + "type": "SaveImage", + "pos": [ + 1351.0000000000005, + 373.78992797851555 + ], + "size": [ + 257.83631037110194, + 270 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 40 + } + ], + "properties": {}, + "widgets_values": [ + "ExpressionEditing" + ] + } + ], + "links": [ + [ + 19, + 15, + 0, + 14, + 0, + "IMAGE" + ], + [ + 40, + 14, + 0, + 33, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Simple facial expression editing", + "bounding": [ + 745, + 261, + 896, + 835 + ], + "color": "#88A", + "font_size": 30 + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7513148009015777, + "offset": [ + 561.7325573974937, + 267.43204960931945 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/sample/workflows/flex/FLEX_Advanced_Audio_Reactive_expression.json b/sample/workflows/flex/FLEX_Advanced_Audio_Reactive_expression.json new file mode 100644 index 0000000..861cabb --- /dev/null +++ b/sample/workflows/flex/FLEX_Advanced_Audio_Reactive_expression.json @@ -0,0 +1,2330 @@ +{ + "last_node_id": 88, + "last_link_id": 76, + "nodes": [ + { + "id": 42, + "type": "BrightnessFeatureNode", + "pos": [ + -1937.0077758142834, + 948.0409576835841 + ], + "size": { + "0": 317.4000244140625, + "1": 102 + }, + "flags": {}, + "order": 0, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "BrightnessFeatureNode" + }, + "widgets_values": [ + "mean_brightness", + 30 + ] + }, + { + "id": 43, + "type": "ColorFeatureNode", + "pos": [ + -1607.0077758142834, + 948.0409576835841 + ], + "size": { + "0": 317.4000244140625, + "1": 102 + }, + "flags": {}, + "order": 1, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ColorFeatureNode" + }, + "widgets_values": [ + "dominant_color", + 30 + ] + }, + { + "id": 47, + "type": "ManualFeatureFromPipe", + "pos": [ + -1237.0077758142834, + 538.0409576835838 + ], + "size": { + "0": 352.79998779296875, + "1": 150 + }, + "flags": {}, + "order": 2, + "mode": 4, + "inputs": [ + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ManualFeatureFromPipe" + }, + "widgets_values": [ + "0,10,20", + "0.0,0.5,1.0", + 1, + "none" + ] + }, + { + "id": 59, + "type": "PreviewFeature", + "pos": [ + 227, + 750 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": { + "collapsed": true + }, + "order": 40, + "mode": 0, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": 47 + } + ], + "outputs": [ + { + "name": "FEATURE_PREVIEW", + "type": "IMAGE", + "links": [ + 46 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PreviewFeature" + }, + "widgets_values": [ + false + ] + }, + { + "id": 40, + "type": "TimeFeatureNode", + "pos": [ + -1597.0077758142834, + 538.0409576835838 + ], + "size": { + "0": 262.2439880371094, + "1": 150 + }, + "flags": {}, + "order": 3, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "TimeFeatureNode" + }, + "widgets_values": [ + "smooth", + 30, + 1, + 0 + ] + }, + { + "id": 48, + "type": "AreaFeatureNode", + "pos": [ + -1207.0077758142834, + 738.0409576835838 + ], + "size": { + "0": 317.4000244140625, + "1": 126 + }, + "flags": {}, + "order": 4, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + }, + { + "name": "masks", + "type": "MASK", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "AreaFeatureNode" + }, + "widgets_values": [ + "total_area", + 30, + 0.5 + ] + }, + { + "id": 46, + "type": "DepthFeatureNode", + "pos": [ + -1267.0077758142834, + 938.0409576835841 + ], + "size": { + "0": 317.4000244140625, + "1": 102 + }, + "flags": {}, + "order": 5, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + }, + { + "name": "depth_maps", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "DepthFeatureNode" + }, + "widgets_values": [ + "mean_depth", + 30 + ] + }, + { + "id": 67, + "type": "PreviewFeature", + "pos": [ + 227, + 420 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": { + "collapsed": true + }, + "order": 34, + "mode": 0, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": 59 + } + ], + "outputs": [ + { + "name": "FEATURE_PREVIEW", + "type": "IMAGE", + "links": [ + 60 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PreviewFeature" + }, + "widgets_values": [ + false + ] + }, + { + "id": 24, + "type": "AdvancedLivePortrait", + "pos": [ + 2680, + 360 + ], + "size": { + "0": 235.1999969482422, + "1": 474.2106018066406 + }, + "flags": {}, + "order": 44, + "mode": 0, + "inputs": [ + { + "name": "src_images", + "type": "IMAGE", + "link": null + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "link": 71 + }, + { + "name": "driving_images", + "type": "IMAGE", + "link": null + }, + { + "name": "command", + "type": "STRING", + "link": 70, + "widget": { + "name": "command" + } + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 16 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "AdvancedLivePortrait" + }, + "widgets_values": [ + 0, + 0, + 1.7000000000000002, + "1 = 1:10\n2 = 5:10\n0 = 2:50\n1 = 2:0", + false, + true, + "1 = 1:10\n2 = 5:10\n0 = 2:50\n1 = 2:0" + ] + }, + { + "id": 44, + "type": "RhythmFeatureExtractor", + "pos": [ + -1610.415877376783, + 744.9609491386618 + ], + "size": { + "0": 352.79998779296875, + "1": 126 + }, + "flags": {}, + "order": 6, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + }, + { + "name": "audio", + "type": "AUDIO", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "RhythmFeatureExtractor" + }, + "widgets_values": [ + "beat_locations", + 30, + 4 + ] + }, + { + "id": 45, + "type": "PitchFeatureExtractor", + "pos": [ + -1917.415877376783, + 1112.9609491386618 + ], + "size": { + "0": 344.3999938964844, + "1": 122 + }, + "flags": {}, + "order": 7, + "mode": 4, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": null + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "link": null + }, + { + "name": "opt_pitch_range_collections", + "type": "PITCH_RANGE_COLLECTION", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "PitchFeatureExtractor" + }, + "widgets_values": [ + "frequency", + "medium" + ] + }, + { + "id": 68, + "type": "PreviewImage", + "pos": [ + 746, + 420 + ], + "size": { + "0": 198.45782470703125, + "1": 246 + }, + "flags": {}, + "order": 37, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 60 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 31, + "type": "DownloadOpenUnmixModel", + "pos": [ + -164, + 619 + ], + "size": { + "0": 361.20001220703125, + "1": 58 + }, + "flags": {}, + "order": 8, + "mode": 0, + "outputs": [ + { + "name": "OPEN_UNMIX_MODEL", + "type": "OPEN_UNMIX_MODEL", + "links": [ + 23 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "DownloadOpenUnmixModel" + }, + "widgets_values": [ + "umxl" + ] + }, + { + "id": 32, + "type": "AudioSeparatorSimple", + "pos": [ + -164, + 729 + ], + "size": { + "0": 336, + "1": 106 + }, + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "OPEN_UNMIX_MODEL", + "link": 23 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 24 + } + ], + "outputs": [ + { + "name": "audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "drums_audio", + "type": "AUDIO", + "links": [ + 54, + 76 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "vocals_audio", + "type": "AUDIO", + "links": [], + "shape": 3, + "slot_index": 2 + }, + { + "name": "bass_audio", + "type": "AUDIO", + "links": [], + "shape": 3, + "slot_index": 3 + }, + { + "name": "other_audio", + "type": "AUDIO", + "links": [ + 51, + 52 + ], + "shape": 3, + "slot_index": 4 + } + ], + "properties": { + "Node name for S&R": "AudioSeparatorSimple" + } + }, + { + "id": 64, + "type": "PreviewAudio", + "pos": [ + -164, + 1088 + ], + "size": { + "0": 315, + "1": 76 + }, + "flags": {}, + "order": 32, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 51 + } + ], + "properties": { + "Node name for S&R": "PreviewAudio" + }, + "widgets_values": [ + null + ] + }, + { + "id": 51, + "type": "PreviewImage", + "pos": [ + 738.1399993896484, + 729 + ], + "size": { + "0": 206.3178253173828, + "1": 246 + }, + "flags": {}, + "order": 42, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 46 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 41, + "type": "ProximityFeatureNode", + "pos": [ + -1973.415877376783, + 781.9609491386618 + ], + "size": { + "0": 336, + "1": 122 + }, + "flags": {}, + "order": 9, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + }, + { + "name": "anchor_locations", + "type": "LOCATION", + "link": null + }, + { + "name": "query_locations", + "type": "LOCATION", + "link": null + } + ], + "outputs": [ + { + "name": "proximity_feature", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ProximityFeatureNode" + }, + "widgets_values": [ + 30, + "frame" + ] + }, + { + "id": 38, + "type": "MIDILoadAndExtract", + "pos": [ + -1928.415877376783, + 144.96094913866202 + ], + "size": { + "0": 1020, + "1": 346 + }, + "flags": {}, + "order": 10, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "MIDI", + "type": "MIDI", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "selectedNotes": [] + }, + "widgets_values": [ + "Velocity", + "prom_single_track.mid", + "all", + 30, + false, + "", + "upload", + "refresh" + ] + }, + { + "id": 58, + "type": "ManualFeatureFromPipe", + "pos": [ + 227, + 729 + ], + "size": { + "0": 352.79998779296875, + "1": 150 + }, + "flags": {}, + "order": 36, + "mode": 0, + "inputs": [ + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "link": 44 + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": [ + 45, + 47 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "ManualFeatureFromPipe" + }, + "widgets_values": [ + "0", + "-1", + 1, + "ease_out" + ] + }, + { + "id": 21, + "type": "AudioFeatureExtractorFirst", + "pos": [ + 227, + 980 + ], + "size": { + "0": 394.79998779296875, + "1": 170 + }, + "flags": {}, + "order": 33, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 52 + } + ], + "outputs": [ + { + "name": "feature", + "type": "FEATURE", + "links": [ + 73 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "links": [ + 44 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3, + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "AudioFeatureExtractorFirst" + }, + "widgets_values": [ + "amplitude_envelope", + 512, + 512, + 15 + ] + }, + { + "id": 73, + "type": "FeatureRenormalize", + "pos": [ + 227, + 1211 + ], + "size": { + "0": 310.79998779296875, + "1": 126 + }, + "flags": {}, + "order": 35, + "mode": 0, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": 73 + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": [ + 74 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": [ + 72 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "FeatureRenormalize" + }, + "widgets_values": [ + -1, + 1, + false + ] + }, + { + "id": 27, + "type": "VHS_VideoCombine", + "pos": [ + 2945, + 280 + ], + "size": [ + 253.4889678955078, + 557.4889678955078 + ], + "flags": {}, + "order": 45, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 16 + }, + { + "name": "audio", + "type": "AUDIO", + "link": 30 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 15, + "loop_count": 0, + "filename_prefix": "AdvancedLivePortrait", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AdvancedLivePortrait_00306-audio.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 15 + } + } + } + }, + { + "id": 53, + "type": "Note", + "pos": [ + 227, + 1387 + ], + "size": { + "0": 234.10646057128906, + "1": 70.32601165771484 + }, + "flags": {}, + "order": 11, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "audio feature from drums\nmanual feature for fun\nanother audio feature for the synth" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 39, + "type": "MotionFeatureNode", + "pos": [ + -1915, + 542 + ], + "size": { + "0": 268.79998779296875, + "1": 174 + }, + "flags": {}, + "order": 12, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "MotionFeatureNode" + }, + "widgets_values": [ + "mean_motion", + 30, + "Farneback", + 0, + 0 + ] + }, + { + "id": 81, + "type": "FeatureMixer", + "pos": [ + -2048.801412019285, + 1448.3726434811094 + ], + "size": { + "0": 367.79998779296875, + "1": 342 + }, + "flags": {}, + "order": 13, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureMixer" + }, + "widgets_values": [ + 1, + 0, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 1, + 0.5, + false + ] + }, + { + "id": 82, + "type": "FeatureCombine", + "pos": [ + -1645.801412019285, + 1451.3726434811094 + ], + "size": { + "0": 380.4000244140625, + "1": 150 + }, + "flags": {}, + "order": 14, + "mode": 4, + "inputs": [ + { + "name": "feature1", + "type": "FEATURE", + "link": null + }, + { + "name": "feature2", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureCombine" + }, + "widgets_values": [ + "add", + 1, + 1, + false + ] + }, + { + "id": 83, + "type": "FeatureOscillator", + "pos": [ + -1232.801412019285, + 1461.3726434811094 + ], + "size": { + "0": 367.79998779296875, + "1": 198 + }, + "flags": {}, + "order": 15, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureOscillator" + }, + "widgets_values": [ + "sine", + 1, + 0.5, + 0, + 0.5, + false + ] + }, + { + "id": 86, + "type": "FeatureMath", + "pos": [ + -1639.801412019285, + 1655.3726434811094 + ], + "size": { + "0": 367.79998779296875, + "1": 126 + }, + "flags": {}, + "order": 16, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureMath" + }, + "widgets_values": [ + 0, + "add", + false + ] + }, + { + "id": 84, + "type": "FeatureScaler", + "pos": [ + -2049.801412019285, + 1850.3726434811097 + ], + "size": { + "0": 367.79998779296875, + "1": 174 + }, + "flags": {}, + "order": 17, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureScaler" + }, + "widgets_values": [ + "linear", + 0, + 1, + 2, + false + ] + }, + { + "id": 85, + "type": "FeatureSmoothing", + "pos": [ + -1633.801412019285, + 1852.3726434811097 + ], + "size": { + "0": 367.79998779296875, + "1": 174 + }, + "flags": {}, + "order": 18, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureSmoothing" + }, + "widgets_values": [ + "moving_average", + 5, + 0.3, + 1, + false + ] + }, + { + "id": 87, + "type": "FeatureAccumulate", + "pos": [ + -1229.801412019285, + 1731.3726434811094 + ], + "size": { + "0": 367.79998779296875, + "1": 222 + }, + "flags": {}, + "order": 19, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureAccumulate" + }, + "widgets_values": [ + 0, + 1, + 0, + false, + false, + 0, + false + ] + }, + { + "id": 88, + "type": "Note", + "pos": [ + -675, + 1660 + ], + "size": { + "0": 343.6423645019531, + "1": 137.95391845703125 + }, + "flags": {}, + "order": 20, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "\n<--------here's some ways to manipulate features" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 63, + "type": "AudioFeatureExtractorFirst", + "pos": [ + 227, + 420 + ], + "size": { + "0": 394.79998779296875, + "1": 170 + }, + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 54 + } + ], + "outputs": [ + { + "name": "feature", + "type": "FEATURE", + "links": [ + 55, + 59 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "links": [], + "shape": 3, + "slot_index": 1 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3, + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "AudioFeatureExtractorFirst" + }, + "widgets_values": [ + "amplitude_envelope", + 512, + 512, + 15 + ] + }, + { + "id": 56, + "type": "PreviewAudio", + "pos": [ + -152, + 897 + ], + "size": { + "0": 315, + "1": 76 + }, + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 76 + } + ], + "properties": { + "Node name for S&R": "PreviewAudio" + }, + "widgets_values": [ + null + ] + }, + { + "id": 69, + "type": "PreviewImage", + "pos": [ + 746.2026214599609, + 1067 + ], + "size": { + "0": 198.2552032470703, + "1": 246 + }, + "flags": {}, + "order": 39, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 72 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 35, + "type": "Note", + "pos": [ + 1500, + 1290 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 21, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "modulate pupils with synth" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 36, + "type": "Note", + "pos": [ + 1870, + 1290 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": { + "collapsed": false + }, + "order": 22, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "modulate roll with manual feature" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 65, + "type": "Note", + "pos": [ + 2250, + 1290 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": { + "collapsed": false + }, + "order": 23, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "modulate pitch with drums" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 61, + "type": "PreviewAudio", + "pos": [ + -163, + 285 + ], + "size": { + "0": 315, + "1": 76 + }, + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 48 + } + ], + "properties": { + "Node name for S&R": "PreviewAudio" + }, + "widgets_values": [ + null + ] + }, + { + "id": 16, + "type": "VHS_LoadAudioUpload", + "pos": [ + -164, + 420 + ], + "size": { + "0": 243.818359375, + "1": 130 + }, + "flags": {}, + "order": 24, + "mode": 0, + "outputs": [ + { + "name": "audio", + "type": "AUDIO", + "links": [ + 24, + 30, + 48 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadAudioUpload" + }, + "widgets_values": { + "audio": "royalty_free.mp3", + "start_time": 16.01, + "duration": 7, + "choose audio to upload": "image" + } + }, + { + "id": 22, + "type": "LoadImage", + "pos": [ + 1079, + 420 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 25, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 17 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "slot_index": 1, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "source_image.png", + "image" + ] + }, + { + "id": 15, + "type": "FlexExpressionEditor", + "pos": [ + 1430, + 420 + ], + "size": { + "0": 338.77764892578125, + "1": 830 + }, + "flags": {}, + "order": 38, + "mode": 0, + "inputs": [ + { + "name": "src_image", + "type": "IMAGE", + "link": 17 + }, + { + "name": "sample_image", + "type": "IMAGE", + "link": null + }, + { + "name": "add_exp", + "type": "EXP_DATA", + "link": null + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "link": null + }, + { + "name": "feature", + "type": "FEATURE", + "link": 74 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": null, + "shape": 3 + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "links": [ + 18 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "save_exp", + "type": "EXP_DATA", + "links": null, + "shape": 3, + "slot_index": 2 + }, + { + "name": "command", + "type": "STRING", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FlexExpressionEditor" + }, + "widgets_values": [ + 0, + 0, + 0, + 0, + 0, + 0, + -14.5, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + "OnlyExpression", + 1.7, + 1, + 1, + 0, + "pupil_x", + "absolute" + ] + }, + { + "id": 28, + "type": "FlexExpressionEditor", + "pos": [ + 1830, + 420 + ], + "size": { + "0": 336, + "1": 830 + }, + "flags": {}, + "order": 41, + "mode": 0, + "inputs": [ + { + "name": "src_image", + "type": "IMAGE", + "link": null + }, + { + "name": "sample_image", + "type": "IMAGE", + "link": null + }, + { + "name": "add_exp", + "type": "EXP_DATA", + "link": null + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "link": 18 + }, + { + "name": "feature", + "type": "FEATURE", + "link": 45 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [], + "shape": 3, + "slot_index": 0 + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "links": [ + 49 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "save_exp", + "type": "EXP_DATA", + "links": null, + "shape": 3 + }, + { + "name": "command", + "type": "STRING", + "links": [ + 70 + ], + "shape": 3, + "slot_index": 3 + } + ], + "properties": { + "Node name for S&R": "FlexExpressionEditor" + }, + "widgets_values": [ + 0, + 0, + 15, + 0, + 15, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + "OnlyExpression", + 1.7, + 1, + 1, + 0, + "rotate_roll", + "absolute" + ] + }, + { + "id": 62, + "type": "FlexExpressionEditor", + "pos": [ + 2210, + 420 + ], + "size": { + "0": 336, + "1": 830 + }, + "flags": {}, + "order": 43, + "mode": 0, + "inputs": [ + { + "name": "src_image", + "type": "IMAGE", + "link": null + }, + { + "name": "sample_image", + "type": "IMAGE", + "link": null + }, + { + "name": "add_exp", + "type": "EXP_DATA", + "link": null + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "link": 49 + }, + { + "name": "feature", + "type": "FEATURE", + "link": 55 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [], + "shape": 3, + "slot_index": 0 + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "links": [ + 71 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "save_exp", + "type": "EXP_DATA", + "links": null, + "shape": 3 + }, + { + "name": "command", + "type": "STRING", + "links": [], + "shape": 3, + "slot_index": 3 + } + ], + "properties": { + "Node name for S&R": "FlexExpressionEditor" + }, + "widgets_values": [ + 9, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + "OnlyExpression", + 1.7, + 1, + 1, + 0, + "rotate_pitch", + "absolute" + ] + }, + { + "id": 79, + "type": "Note", + "pos": [ + 2688, + 885 + ], + "size": { + "0": 486.4787902832031, + "1": 230.9814453125 + }, + "flags": {}, + "order": 26, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Flex Parameters: \n constrain_min_max: ensures modulated value is kept within bounds of the \n min and max of the parameter\n strength: a strength multiplier\n feature_threshold: minimum feature value to consider\n feature_param: the expression parameter to modulate\n feature_mode:\n -relative: modulation occurs relative to the parameter value\n -absolute: modulation occurs relative to 0\n\nNotes:\n The value for a parameter in a previous editor changes the starting position of \n that \n parameter in the next editor.\n\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 60, + "type": "Note", + "pos": [ + -630, + 616 + ], + "size": { + "0": 343.6423645019531, + "1": 137.95391845703125 + }, + "flags": {}, + "order": 27, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Audio is one of many feature sources, and they are interchangeable.\n\n<--------here's some other feature sources. \ntutorial: https://youtu.be/QmWk2xse7pI" + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 16, + 24, + 0, + 27, + 0, + "IMAGE" + ], + [ + 17, + 22, + 0, + 15, + 0, + "IMAGE" + ], + [ + 18, + 15, + 1, + 28, + 3, + "EDITOR_LINK" + ], + [ + 23, + 31, + 0, + 32, + 0, + "OPEN_UNMIX_MODEL" + ], + [ + 24, + 16, + 0, + 32, + 1, + "AUDIO" + ], + [ + 30, + 16, + 0, + 27, + 1, + "AUDIO" + ], + [ + 44, + 21, + 1, + 58, + 0, + "FEATURE_PIPE" + ], + [ + 45, + 58, + 0, + 28, + 4, + "FEATURE" + ], + [ + 46, + 59, + 0, + 51, + 0, + "IMAGE" + ], + [ + 47, + 58, + 0, + 59, + 0, + "FEATURE" + ], + [ + 48, + 16, + 0, + 61, + 0, + "AUDIO" + ], + [ + 49, + 28, + 1, + 62, + 3, + "EDITOR_LINK" + ], + [ + 51, + 32, + 4, + 64, + 0, + "AUDIO" + ], + [ + 52, + 32, + 4, + 21, + 0, + "AUDIO" + ], + [ + 54, + 32, + 1, + 63, + 0, + "AUDIO" + ], + [ + 55, + 63, + 0, + 62, + 4, + "FEATURE" + ], + [ + 59, + 63, + 0, + 67, + 0, + "FEATURE" + ], + [ + 60, + 67, + 0, + 68, + 0, + "IMAGE" + ], + [ + 70, + 28, + 3, + 24, + 3, + "STRING" + ], + [ + 71, + 62, + 1, + 24, + 1, + "EDITOR_LINK" + ], + [ + 72, + 73, + 1, + 69, + 0, + "IMAGE" + ], + [ + 73, + 21, + 0, + 73, + 0, + "FEATURE" + ], + [ + 74, + 73, + 0, + 15, + 4, + "FEATURE" + ], + [ + 76, + 32, + 1, + 56, + 0, + "AUDIO" + ] + ], + "groups": [ + { + "title": ":)", + "bounding": [ + -2768, + -53, + 1989, + 1400 + ], + "color": "#3f789e", + "font_size": 1000 + }, + { + "title": "Feature Manipulation", + "bounding": [ + -2092, + 1352, + 1315, + 699 + ], + "color": "#3f789e", + "font_size": 24 + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 1.2100000000000002, + "offset": [ + 1166.858494860238, + -303.268542001848 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/sample/workflows/flex/FLEX_Advanced_Inserting_expressions_into_vid.json b/sample/workflows/flex/FLEX_Advanced_Inserting_expressions_into_vid.json new file mode 100644 index 0000000..1bababb --- /dev/null +++ b/sample/workflows/flex/FLEX_Advanced_Inserting_expressions_into_vid.json @@ -0,0 +1,1954 @@ +{ + "last_node_id": 61, + "last_link_id": 39, + "nodes": [ + { + "id": 15, + "type": "LoadImage", + "pos": [ + -400, + 110 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 14 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "slot_index": 1, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "source_image.png", + "image" + ] + }, + { + "id": 30, + "type": "PreviewFeature", + "pos": [ + 240, + 200 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": 22 + } + ], + "outputs": [ + { + "name": "FEATURE_PREVIEW", + "type": "IMAGE", + "links": [], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "PreviewFeature" + }, + "widgets_values": [ + false + ] + }, + { + "id": 24, + "type": "Note", + "pos": [ + -350, + 520 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "orginal src : motion0\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 19, + "type": "VHS_LoadVideo", + "pos": [ + -50, + 110 + ], + "size": [ + 250, + 506 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 11, + 17, + 28 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": [ + 26 + ], + "shape": 3, + "slot_index": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "driving_video.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 2, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "driving_video.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 2 + } + } + } + }, + { + "id": 33, + "type": "VHS_VideoInfo", + "pos": [ + 30, + 60 + ], + "size": { + "0": 262, + "1": 206 + }, + "flags": { + "collapsed": true + }, + "order": 26, + "mode": 0, + "inputs": [ + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "link": 26 + } + ], + "outputs": [ + { + "name": "source_fps🟨", + "type": "FLOAT", + "links": null, + "shape": 3 + }, + { + "name": "source_frame_count🟨", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "source_duration🟨", + "type": "FLOAT", + "links": null, + "shape": 3 + }, + { + "name": "source_width🟨", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "source_height🟨", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "loaded_fps🟦", + "type": "FLOAT", + "links": [ + 27, + 29 + ], + "shape": 3, + "slot_index": 5 + }, + { + "name": "loaded_frame_count🟦", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "loaded_duration🟦", + "type": "FLOAT", + "links": null, + "shape": 3 + }, + { + "name": "loaded_width🟦", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "loaded_height🟦", + "type": "INT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoInfo" + }, + "widgets_values": {} + }, + { + "id": 22, + "type": "Note", + "pos": [ + 1120, + 887 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 3, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "use time feature to control aaa" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 37, + "type": "MIDILoadAndExtract", + "pos": [ + -2344.2929440052535, + -243.81497198427647 + ], + "size": { + "0": 1020, + "1": 346 + }, + "flags": {}, + "order": 4, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "MIDI", + "type": "MIDI", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "selectedNotes": [] + }, + "widgets_values": [ + "Velocity", + "prom_single_track.mid", + "all", + 30, + false, + "", + "upload", + "refresh" + ] + }, + { + "id": 38, + "type": "MotionFeatureNode", + "pos": [ + -2330.8770666284704, + 153.22407887706152 + ], + "size": { + "0": 268.79998779296875, + "1": 174 + }, + "flags": {}, + "order": 5, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "MotionFeatureNode" + }, + "widgets_values": [ + "mean_motion", + 30, + "Farneback", + 0, + 0 + ] + }, + { + "id": 39, + "type": "TimeFeatureNode", + "pos": [ + -2012.8848424427538, + 149.26503656064529 + ], + "size": { + "0": 262.2439880371094, + "1": 150 + }, + "flags": {}, + "order": 6, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "TimeFeatureNode" + }, + "widgets_values": [ + "smooth", + 30, + 1, + 0 + ] + }, + { + "id": 40, + "type": "ProximityFeatureNode", + "pos": [ + -2389.2929440052535, + 393.18502801572333 + ], + "size": { + "0": 336, + "1": 122 + }, + "flags": {}, + "order": 7, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + }, + { + "name": "anchor_locations", + "type": "LOCATION", + "link": null + }, + { + "name": "query_locations", + "type": "LOCATION", + "link": null + } + ], + "outputs": [ + { + "name": "proximity_feature", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ProximityFeatureNode" + }, + "widgets_values": [ + 30, + "frame" + ] + }, + { + "id": 41, + "type": "BrightnessFeatureNode", + "pos": [ + -2352.884842442754, + 559.2650365606456 + ], + "size": { + "0": 317.4000244140625, + "1": 102 + }, + "flags": {}, + "order": 8, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "BrightnessFeatureNode" + }, + "widgets_values": [ + "mean_brightness", + 30 + ] + }, + { + "id": 42, + "type": "ColorFeatureNode", + "pos": [ + -2022.8848424427538, + 559.2650365606456 + ], + "size": { + "0": 317.4000244140625, + "1": 102 + }, + "flags": {}, + "order": 9, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ColorFeatureNode" + }, + "widgets_values": [ + "dominant_color", + 30 + ] + }, + { + "id": 43, + "type": "RhythmFeatureExtractor", + "pos": [ + -2026.2929440052535, + 356.18502801572333 + ], + "size": { + "0": 352.79998779296875, + "1": 126 + }, + "flags": {}, + "order": 10, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + }, + { + "name": "audio", + "type": "AUDIO", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "RhythmFeatureExtractor" + }, + "widgets_values": [ + "beat_locations", + 30, + 4 + ] + }, + { + "id": 44, + "type": "PitchFeatureExtractor", + "pos": [ + -2333.2929440052535, + 724.1850280157233 + ], + "size": { + "0": 344.3999938964844, + "1": 122 + }, + "flags": {}, + "order": 11, + "mode": 4, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": null + }, + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "link": null + }, + { + "name": "opt_pitch_range_collections", + "type": "PITCH_RANGE_COLLECTION", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "PitchFeatureExtractor" + }, + "widgets_values": [ + "frequency", + "medium" + ] + }, + { + "id": 45, + "type": "DepthFeatureNode", + "pos": [ + -1682.8848424427538, + 549.2650365606456 + ], + "size": { + "0": 317.4000244140625, + "1": 102 + }, + "flags": {}, + "order": 12, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + }, + { + "name": "depth_maps", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "DepthFeatureNode" + }, + "widgets_values": [ + "mean_depth", + 30 + ] + }, + { + "id": 46, + "type": "ManualFeatureFromPipe", + "pos": [ + -1652.8848424427538, + 149.26503656064529 + ], + "size": { + "0": 352.79998779296875, + "1": 150 + }, + "flags": {}, + "order": 13, + "mode": 4, + "inputs": [ + { + "name": "feature_pipe", + "type": "FEATURE_PIPE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "ManualFeatureFromPipe" + }, + "widgets_values": [ + "0,10,20", + "0.0,0.5,1.0", + 1, + "none" + ] + }, + { + "id": 47, + "type": "AreaFeatureNode", + "pos": [ + -1622.8848424427538, + 349.2650365606453 + ], + "size": { + "0": 317.4000244140625, + "1": 126 + }, + "flags": {}, + "order": 14, + "mode": 4, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": null + }, + { + "name": "masks", + "type": "MASK", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "AreaFeatureNode" + }, + "widgets_values": [ + "total_area", + 30, + 0.5 + ] + }, + { + "id": 50, + "type": "FeatureCombine", + "pos": [ + -1921.771146009376, + 1180 + ], + "size": { + "0": 380.4000244140625, + "1": 150 + }, + "flags": {}, + "order": 15, + "mode": 4, + "inputs": [ + { + "name": "feature1", + "type": "FEATURE", + "link": null + }, + { + "name": "feature2", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureCombine" + }, + "widgets_values": [ + "add", + 1, + 1, + false + ] + }, + { + "id": 51, + "type": "FeatureOscillator", + "pos": [ + -1501.771146009376, + 1190 + ], + "size": { + "0": 367.79998779296875, + "1": 198 + }, + "flags": {}, + "order": 16, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureOscillator" + }, + "widgets_values": [ + "sine", + 1, + 0.5, + 0, + 0.5, + false + ] + }, + { + "id": 52, + "type": "FeatureScaler", + "pos": [ + -2321.771146009376, + 1580 + ], + "size": { + "0": 367.79998779296875, + "1": 174 + }, + "flags": {}, + "order": 17, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureScaler" + }, + "widgets_values": [ + "linear", + 0, + 1, + 2, + false + ] + }, + { + "id": 53, + "type": "FeatureSmoothing", + "pos": [ + -1911.771146009376, + 1580 + ], + "size": { + "0": 367.79998779296875, + "1": 174 + }, + "flags": {}, + "order": 18, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureSmoothing" + }, + "widgets_values": [ + "moving_average", + 5, + 0.3, + 1, + false + ] + }, + { + "id": 54, + "type": "FeatureMath", + "pos": [ + -1911.771146009376, + 1390 + ], + "size": { + "0": 367.79998779296875, + "1": 126 + }, + "flags": {}, + "order": 19, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureMath" + }, + "widgets_values": [ + 0, + "add", + false + ] + }, + { + "id": 55, + "type": "FeatureAccumulate", + "pos": [ + -1501.771146009376, + 1460 + ], + "size": { + "0": 367.79998779296875, + "1": 222 + }, + "flags": {}, + "order": 20, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureAccumulate" + }, + "widgets_values": [ + 0, + 1, + 0, + false, + false, + 0, + false + ] + }, + { + "id": 49, + "type": "FeatureMixer", + "pos": [ + -2309.771146009376, + 1186 + ], + "size": { + "0": 367.79998779296875, + "1": 342 + }, + "flags": {}, + "order": 21, + "mode": 4, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": null + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": null, + "shape": 3 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FeatureMixer" + }, + "widgets_values": [ + 1, + 0, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 1, + 0.5, + false + ] + }, + { + "id": 56, + "type": "Note", + "pos": [ + -1027, + 1248 + ], + "size": { + "0": 343.6423645019531, + "1": 137.95391845703125 + }, + "flags": {}, + "order": 22, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "\n<--------here's some ways to manipulate features" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 34, + "type": "MotionFeatureNode", + "pos": [ + 243, + 380 + ], + "size": { + "0": 317.4000244140625, + "1": 174 + }, + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": 28 + }, + { + "name": "frame_rate", + "type": "FLOAT", + "link": 29, + "widget": { + "name": "frame_rate" + } + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": [ + 33 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "MotionFeatureNode" + }, + "widgets_values": [ + "mean_motion", + 30, + "Farneback", + 0, + 0 + ] + }, + { + "id": 57, + "type": "FeatureSmoothing", + "pos": [ + 242, + 604 + ], + "size": { + "0": 319.20001220703125, + "1": 174.9882354736328 + }, + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "name": "feature", + "type": "FEATURE", + "link": 33 + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": [ + 35 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "FEATURE_VISUALIZATION", + "type": "IMAGE", + "links": [ + 34 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "FeatureSmoothing" + }, + "widgets_values": [ + "moving_average", + 9, + 0.3, + 1, + false + ] + }, + { + "id": 27, + "type": "TimeFeatureNode", + "pos": [ + 250, + 130 + ], + "size": { + "0": 317.4000244140625, + "1": 150 + }, + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "name": "video_frames", + "type": "IMAGE", + "link": 17 + }, + { + "name": "frame_rate", + "type": "FLOAT", + "link": 27, + "widget": { + "name": "frame_rate" + } + } + ], + "outputs": [ + { + "name": "FEATURE", + "type": "FEATURE", + "links": [ + 22, + 39 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "FEATURE_PIPE", + "type": "FEATURE_PIPE", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "TimeFeatureNode" + }, + "widgets_values": [ + "pulse", + 30, + 1, + 0 + ] + }, + { + "id": 20, + "type": "VHS_VideoCombine", + "pos": [ + 2143, + -9 + ], + "size": [ + 260, + 564 + ], + "flags": {}, + "order": 35, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 13 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 15, + "loop_count": 0, + "filename_prefix": "AdvancedLivePortrait", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AdvancedLivePortrait_00352.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 15 + } + } + } + }, + { + "id": 17, + "type": "AdvancedLivePortrait", + "pos": [ + 1843, + 26 + ], + "size": { + "0": 235.1999969482422, + "1": 523.0364379882812 + }, + "flags": {}, + "order": 34, + "mode": 0, + "inputs": [ + { + "name": "src_images", + "type": "IMAGE", + "link": null + }, + { + "name": "motion_link", + "type": "EDITOR_LINK", + "link": 16 + }, + { + "name": "driving_images", + "type": "IMAGE", + "link": 11 + }, + { + "name": "command", + "type": "STRING", + "link": 25, + "widget": { + "name": "command" + } + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 13 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "AdvancedLivePortrait" + }, + "widgets_values": [ + 0, + 0, + 1.7000000000000002, + "1 = 1:10\n2 = 5:10\n0 = 2:50\n1 = 2:0", + "1 = 1:10\n2 = 5:10\n0 = 2:50\n1 = 2:0", + false, + "1 = 1:10\n2 = 5:10\n0 = 2:50\n1 = 2:0" + ] + }, + { + "id": 21, + "type": "Note", + "pos": [ + 1809, + 612 + ], + "size": { + "0": 638.537841796875, + "1": 195.73060607910156 + }, + "flags": {}, + "order": 23, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Flex Parameters: \n constrain_min_max: ensures modulated value is kept within bounds of the min and max of the parameter\n strength: a strength multiplier\n feature_threshold: minimum feature value to consider\n feature_param: the expression parameter to modulate\n feature_mode:\n -relative: modulation occurs relative to the parameter value\n -absolute: modulation occurs relative to 0\n\nNotes:\n-the value for a parameter in a previous editor changes the starting position of that parameter in the next editor. this means a parameter can be modulated multiple times for interesting effects\n\n" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 25, + "type": "FlexExpressionEditor", + "pos": [ + 1052, + 18 + ], + "size": { + "0": 336, + "1": 830 + }, + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [ + { + "name": "src_image", + "type": "IMAGE", + "link": 14 + }, + { + "name": "sample_image", + "type": "IMAGE", + "link": null + }, + { + "name": "add_exp", + "type": "EXP_DATA", + "link": null + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "link": null + }, + { + "name": "feature", + "type": "FEATURE", + "link": 39 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": null, + "shape": 3 + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "links": [ + 15 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "save_exp", + "type": "EXP_DATA", + "links": null, + "shape": 3 + }, + { + "name": "command", + "type": "STRING", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "FlexExpressionEditor" + }, + "widgets_values": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 120, + 0, + 0, + 0, + 1, + 1, + "OnlyExpression", + 1.7, + true, + 1, + 0, + "aaa", + "absolute" + ] + }, + { + "id": 32, + "type": "PreviewImage", + "pos": [ + 624, + 627 + ], + "size": { + "0": 210, + "1": 246 + }, + "flags": {}, + "order": 33, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 34 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 48, + "type": "Note", + "pos": [ + -1045.8770666284704, + 227.22407887706152 + ], + "size": { + "0": 343.6423645019531, + "1": 137.95391845703125 + }, + "flags": {}, + "order": 24, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Time and motion are two out of many feature sources, and they are interchangeable.\n\n<--------here's some other feature sources.\ntutorial https://youtu.be/QmWk2xse7pI" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 26, + "type": "FlexExpressionEditor", + "pos": [ + 1426, + 7 + ], + "size": { + "0": 336, + "1": 830 + }, + "flags": {}, + "order": 32, + "mode": 0, + "inputs": [ + { + "name": "src_image", + "type": "IMAGE", + "link": null + }, + { + "name": "sample_image", + "type": "IMAGE", + "link": null + }, + { + "name": "add_exp", + "type": "EXP_DATA", + "link": null + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "link": 15 + }, + { + "name": "feature", + "type": "FEATURE", + "link": 35 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": null, + "shape": 3 + }, + { + "name": "flex_motion_link", + "type": "EDITOR_LINK", + "links": [ + 16 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "save_exp", + "type": "EXP_DATA", + "links": null, + "shape": 3 + }, + { + "name": "command", + "type": "STRING", + "links": [ + 25 + ], + "shape": 3, + "slot_index": 3 + } + ], + "properties": { + "Node name for S&R": "FlexExpressionEditor" + }, + "widgets_values": [ + 0, + 20, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + 1, + "OnlyExpression", + 1.7, + 1, + 1, + 0, + "rotate_yaw", + "absolute" + ] + }, + { + "id": 23, + "type": "Note", + "pos": [ + 1490, + 889 + ], + "size": { + "0": 245.08152770996094, + "1": 58 + }, + "flags": { + "collapsed": false + }, + "order": 25, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "use motion of driving video to control yaw\n" + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 11, + 19, + 0, + 17, + 2, + "IMAGE" + ], + [ + 13, + 17, + 0, + 20, + 0, + "IMAGE" + ], + [ + 14, + 15, + 0, + 25, + 0, + "IMAGE" + ], + [ + 15, + 25, + 1, + 26, + 3, + "EDITOR_LINK" + ], + [ + 16, + 26, + 1, + 17, + 1, + "EDITOR_LINK" + ], + [ + 17, + 19, + 0, + 27, + 0, + "IMAGE" + ], + [ + 22, + 27, + 0, + 30, + 0, + "FEATURE" + ], + [ + 25, + 26, + 3, + 17, + 3, + "STRING" + ], + [ + 26, + 19, + 3, + 33, + 0, + "VHS_VIDEOINFO" + ], + [ + 27, + 33, + 5, + 27, + 1, + "FLOAT" + ], + [ + 28, + 19, + 0, + 34, + 0, + "IMAGE" + ], + [ + 29, + 33, + 5, + 34, + 1, + "FLOAT" + ], + [ + 33, + 34, + 0, + 57, + 0, + "FEATURE" + ], + [ + 34, + 57, + 1, + 32, + 0, + "IMAGE" + ], + [ + 35, + 57, + 0, + 26, + 4, + "FEATURE" + ], + [ + 39, + 27, + 0, + 25, + 4, + "FEATURE" + ] + ], + "groups": [ + { + "title": ":)", + "bounding": [ + -3194, + -326, + 2124, + 1397 + ], + "color": "#3f789e", + "font_size": 1000 + }, + { + "title": "Feature manipulation", + "bounding": [ + -2335, + 1095, + 1261, + 752 + ], + "color": "#3f789e", + "font_size": 24 + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650705, + "offset": [ + 45.9245970577685, + 204.65633914907028 + ] + } + }, + "version": 0.4 +} \ No newline at end of file