$ cat node-template.py
Video Creation Legacy
// Creates videos from images. Supports two modes: (1) Image to Video — generates a video from a starting image guided by a text prompt, (2) Audio-Driven Animation — animates a starting image driven by an audio track. Outputs a video file.
Process
Video
template.py
1import os2import sys3import json4import time5import random6import traceback78try:9 import requests10except ImportError:11 import subprocess12 subprocess.check_call([sys.executable, "-m", "pip", "install", "requests"])13 import requests1415COMFYUI_API_URL = os.getenv("COMFYUI_API_URL", "http://192.168.1.39:8188")16INPUT_DIR = "/data/input"17OUTPUT_DIR = "/data/output"1819# ---------- Workflows ----------20WORKFLOWS = {21 "audio_to_video_sonic": {22 "54": {23 "inputs": {"ckpt_name": "svd_xt_1_1.safetensors"},24 "class_type": "ImageOnlyCheckpointLoader",25 "_meta": {"title": "Image Only Checkpoint Loader (img2vid model)"},26 },27 "67": {28 "inputs": {29 "seed": 1351426313,30 "inference_steps": 25,31 "dynamic_scale": 1,32 "fps": 25,33 "model": ["68", 0],34 "data_dict": ["69", 0],35 },36 "class_type": "SONICSampler",37 "_meta": {"title": "SONICSampler"},38 },39 "68": {40 "inputs": {41 "sonic_unet": "unet-001.pth",42 "ip_audio_scale": 1,43 "use_interframe": True,44 "dtype": "fp16",45 "model": ["54", 0],46 },47 "class_type": "SONICTLoader",48 "_meta": {"title": "SONICTLoader"},49 },50 "69": {51 "inputs": {52 "min_resolution": 512,53 "duration": ["118", 1],54 "expand_ratio": 0.5,55 "clip_vision": ["54", 1],56 "vae": ["54", 2],57 "audio": ["118", 0],58 "image": ["83", 0],59 "weight_dtype": ["68", 1],60 },61 "class_type": "SONIC_PreData",62 "_meta": {"title": "SONIC_PreData"},63 },64 "77": {65 "inputs": {"model_name": "RealESRGAN_x2.pth"},66 "class_type": "UpscaleModelLoader",67 "_meta": {"title": "Load Upscale Model"},68 },69 "78": {70 "inputs": {"upscale_model": ["77", 0]},71 "class_type": "ImageUpscaleWithModel",72 "_meta": {"title": "Upscale Image (using Model)"},73 },74 "83": {75 "inputs": {76 "width": 768,77 "height": 512,78 "upscale_method": "nearest-exact",79 "keep_proportion": "crop",80 "pad_color": "0, 0, 0",81 "crop_position": "center",82 "divisible_by": 2,83 "device": "cpu",84 "image": ["113", 0],85 },86 "class_type": "ImageResizeKJv2",87 "_meta": {"title": "Resize Image v2"},88 },89 "93": {90 "inputs": {91 "fps": ["67", 1],92 "images": ["67", 0],93 "audio": ["118", 0],94 },95 "class_type": "CreateVideo",96 "_meta": {"title": "Create Video"},97 },98 "94": {99 "inputs": {100 "filename_prefix": "video/emblema-video",101 "format": "auto",102 "codec": "auto",103 "video": ["93", 0],104 },105 "class_type": "SaveVideo",106 "_meta": {"title": "Save Video"},107 },108 "113": {109 "inputs": {110 "image": "",111 },112 "class_type": "LoadImageFromPath",113 "_meta": {"title": "Load Image From Path"},114 },115 "118": {116 "inputs": {117 "audio": "",118 "start_time": 0,119 "duration": 0,120 },121 "class_type": "VHS_LoadAudioUpload",122 "_meta": {"title": "Load Audio (Upload)"},123 },124 },125 "image_to_video_wan": {126 "84": {127 "inputs": {128 "clip_name": "umt5_xxl_fp8_e4m3fn_scaled.safetensors",129 "type": "wan",130 "device": "default",131 },132 "class_type": "CLIPLoader",133 "_meta": {"title": "Load CLIP"},134 },135 "85": {136 "inputs": {137 "add_noise": "disable",138 "noise_seed": 0,139 "steps": 4,140 "cfg": 1,141 "sampler_name": "euler",142 "scheduler": "simple",143 "start_at_step": 2,144 "end_at_step": 4,145 "return_with_leftover_noise": "disable",146 "model": ["103", 0],147 "positive": ["98", 0],148 "negative": ["98", 1],149 "latent_image": ["86", 0],150 },151 "class_type": "KSamplerAdvanced",152 "_meta": {"title": "KSampler (Advanced)"},153 },154 "86": {155 "inputs": {156 "add_noise": "enable",157 "noise_seed": 304925679570493,158 "steps": 4,159 "cfg": 1,160 "sampler_name": "euler",161 "scheduler": "simple",162 "start_at_step": 0,163 "end_at_step": 2,164 "return_with_leftover_noise": "enable",165 "model": ["104", 0],166 "positive": ["98", 0],167 "negative": ["98", 1],168 "latent_image": ["98", 2],169 },170 "class_type": "KSamplerAdvanced",171 "_meta": {"title": "KSampler (Advanced)"},172 },173 "87": {174 "inputs": {175 "samples": ["85", 0],176 "vae": ["90", 0],177 },178 "class_type": "VAEDecode",179 "_meta": {"title": "VAE Decode"},180 },181 "89": {182 "inputs": {183 "text": "\u8272\u8c03\u8273\u4e3d\uff0c\u8fc7\u66dd\uff0c\u9759\u6001\uff0c\u7ec6\u8282\u6a21\u7cca\u4e0d\u6e05\uff0c\u5b57\u5e55\uff0c\u98ce\u683c\uff0c\u4f5c\u54c1\uff0c\u753b\u4f5c\uff0c\u753b\u9762\uff0c\u9759\u6b62\uff0c\u6574\u4f53\u53d1\u7070\uff0c\u6700\u5dee\u8d28\u91cf\uff0c\u4f4e\u8d28\u91cf\uff0cJPEG\u538b\u7f29\u6b8b\u7559\uff0c\u4e11\u964b\u7684\uff0c\u6b8b\u7f3a\u7684\uff0c\u591a\u4f59\u7684\u624b\u6307\uff0c\u753b\u5f97\u4e0d\u597d\u7684\u624b\u90e8\uff0c\u753b\u5f97\u4e0d\u597d\u7684\u8138\u90e8\uff0c\u7578\u5f62\u7684\uff0c\u6bc1\u5bb9\u7684\uff0c\u5f62\u6001\u7578\u5f62\u7684\u80a2\u4f53\uff0c\u624b\u6307\u878d\u5408\uff0c\u9759\u6b62\u4e0d\u52a8\u7684\u753b\u9762\uff0c\u6742\u4e71\u7684\u80cc\u666f\uff0c\u4e09\u6761\u817f\uff0c\u80cc\u666f\u4eba\u5f88\u591a\uff0c\u5012\u7740\u8d70",184 "clip": ["84", 0],185 },186 "class_type": "CLIPTextEncode",187 "_meta": {"title": "CLIP Text Encode (Negative Prompt)"},188 },189 "90": {190 "inputs": {"vae_name": "wan_2.1_vae.safetensors"},191 "class_type": "VAELoader",192 "_meta": {"title": "Load VAE"},193 },194 "93": {195 "inputs": {196 "text": "",197 "clip": ["84", 0],198 },199 "class_type": "CLIPTextEncode",200 "_meta": {"title": "CLIP Text Encode (Positive Prompt)"},201 },202 "95": {203 "inputs": {204 "unet_name": "wan2.2_i2v_high_noise_14B_fp8_scaled.safetensors",205 "weight_dtype": "default",206 },207 "class_type": "UNETLoader",208 "_meta": {"title": "Load Diffusion Model"},209 },210 "96": {211 "inputs": {212 "unet_name": "wan2.2_i2v_low_noise_14B_fp8_scaled.safetensors",213 "weight_dtype": "default",214 },215 "class_type": "UNETLoader",216 "_meta": {"title": "Load Diffusion Model"},217 },218 "98": {219 "inputs": {220 "width": 1280,221 "height": 720,222 "length": 129,223 "batch_size": 1,224 "positive": ["93", 0],225 "negative": ["89", 0],226 "vae": ["90", 0],227 "start_image": ["116", 0],228 },229 "class_type": "WanImageToVideo",230 "_meta": {"title": "WanImageToVideo"},231 },232 "101": {233 "inputs": {234 "lora_name": "wan2.2_i2v_lightx2v_4steps_lora_v1_high_noise.safetensors",235 "strength_model": 1,236 "model": ["95", 0],237 },238 "class_type": "LoraLoaderModelOnly",239 "_meta": {"title": "Load LoRA"},240 },241 "102": {242 "inputs": {243 "lora_name": "wan2.2_i2v_lightx2v_4steps_lora_v1_low_noise.safetensors",244 "strength_model": 1,245 "model": ["96", 0],246 },247 "class_type": "LoraLoaderModelOnly",248 "_meta": {"title": "Load LoRA"},249 },250 "103": {251 "inputs": {252 "shift": 5,253 "model": ["102", 0],254 },255 "class_type": "ModelSamplingSD3",256 "_meta": {"title": "ModelSamplingSD3"},257 },258 "104": {259 "inputs": {260 "shift": 5,261 "model": ["101", 0],262 },263 "class_type": "ModelSamplingSD3",264 "_meta": {"title": "ModelSamplingSD3"},265 },266 "108": {267 "inputs": {268 "frame_rate": 16,269 "loop_count": 0,270 "filename_prefix": "video/emblema-video",271 "format": "video/h264-mp4",272 "pix_fmt": "yuv420p",273 "crf": 19,274 "save_metadata": True,275 "pingpong": False,276 "save_output": True,277 "images": ["87", 0],278 "audio": ["117", 0],279 },280 "class_type": "VHS_VideoCombine",281 "_meta": {"title": "Video Combine 🎥🅥🅗🅢"},282 },283 "116": {284 "inputs": {285 "image": "",286 },287 "class_type": "LoadImageFromPath",288 "_meta": {"title": "Load Image From Path"},289 },290 "117": {291 "inputs": {292 "audio": "",293 "start_time": 0,294 "duration": 0,295 },296 "class_type": "VHS_LoadAudioUpload",297 "_meta": {"title": "Load Audio (Upload)"},298 },299 },300}301302303def parse_resolution(res_str: str) -> tuple:304 """Parse 'WxH' string to (width, height) ints."""305 w, h = res_str.split("x")306 return int(w), int(h)307308309def upload_file_to_comfyui(local_path: str, content_type: str) -> str:310 """Upload a local file to ComfyUI and return the uploaded filename."""311 with open(local_path, "rb") as f:312 resp = requests.post(313 f"{COMFYUI_API_URL}/upload/image",314 files={"image": (os.path.basename(local_path), f, content_type)},315 timeout=30,316 )317 resp.raise_for_status()318 data = resp.json()319 return data["name"]320321322def detect_audio_mime(filename: str) -> str:323 """Detect MIME type from audio file extension."""324 ext = os.path.splitext(filename)[1].lower()325 mime_map = {326 ".mp3": "audio/mpeg",327 ".wav": "audio/wav",328 ".ogg": "audio/ogg",329 ".flac": "audio/flac",330 ".aac": "audio/aac",331 ".m4a": "audio/mp4",332 }333 return mime_map.get(ext, "application/octet-stream")334335336def build_workflow(workflow_name: str, image_name: str, width: int, height: int, text: str = "", audio_filename: str = "", min_resolution: int = 512) -> dict:337 """Build a video generation workflow with the given parameters."""338 import copy339340 if workflow_name not in WORKFLOWS:341 raise ValueError(f"Unknown workflow: {workflow_name}. Available: {list(WORKFLOWS.keys())}")342343 wf = copy.deepcopy(WORKFLOWS[workflow_name])344345 if workflow_name == "audio_to_video_sonic":346 # Image (node 113): swap LoadImageFromPath -> LoadImage with uploaded name347 wf["113"] = {348 "inputs": {"image": image_name},349 "class_type": "LoadImage",350 "_meta": {"title": "Load Image"},351 }352353 # Audio (node 118): set uploaded audio filename354 wf["118"]["inputs"]["audio"] = audio_filename355356 # Resolution (node 83): set width/height357 wf["83"]["inputs"]["width"] = width358 wf["83"]["inputs"]["height"] = height359360 # Min resolution (node 69): SONIC_PreData361 wf["69"]["inputs"]["min_resolution"] = min_resolution362363 # Seed (node 67): randomize364 wf["67"]["inputs"]["seed"] = random.randint(0, 2**31 - 1)365366 # Output prefix (node 94)367 wf["94"]["inputs"]["filename_prefix"] = "video/emblema-video"368369 elif workflow_name == "image_to_video_wan":370 # Image (node 116): swap LoadImageFromPath -> LoadImage with uploaded name371 wf["116"] = {372 "inputs": {"image": image_name},373 "class_type": "LoadImage",374 "_meta": {"title": "Load Image"},375 }376377 # Text prompt (node 93): set positive CLIP text378 wf["93"]["inputs"]["text"] = text379380 # Resolution (node 98): set width/height on WanImageToVideo381 wf["98"]["inputs"]["width"] = width382 wf["98"]["inputs"]["height"] = height383384 # Seed (node 86): randomize noise_seed385 wf["86"]["inputs"]["noise_seed"] = random.randint(0, 2**31 - 1)386387 # Audio (node 117): optional — wire or remove388 if audio_filename:389 wf["117"]["inputs"]["audio"] = audio_filename390 else:391 del wf["117"]392 wf["108"]["inputs"].pop("audio", None)393394 return wf395396397def submit_prompt(workflow: dict) -> str:398 """Submit workflow to ComfyUI and return prompt_id."""399 resp = requests.post(400 f"{COMFYUI_API_URL}/prompt",401 json={"prompt": workflow},402 timeout=30,403 )404 if resp.status_code != 200:405 try:406 error_detail = resp.json()407 except Exception:408 error_detail = resp.text409 raise RuntimeError(410 f"ComfyUI /prompt returned {resp.status_code}: {json.dumps(error_detail, indent=2) if isinstance(error_detail, dict) else error_detail}"411 )412 data = resp.json()413414 # ComfyUI returns 200 even when nodes have validation errors415 node_errors = data.get("node_errors", {})416 if node_errors:417 raise RuntimeError(418 f"ComfyUI workflow has node errors: {json.dumps(node_errors, indent=2)}"419 )420421 return data["prompt_id"]422423424def wait_for_result(prompt_id: str, timeout: int = 1800, poll_interval: int = 3) -> dict:425 """Poll ComfyUI history until the prompt completes with outputs."""426 deadline = time.time() + timeout427 empty_complete_retries = 0428 max_empty_retries = 3 # grace period for output serialization lag429430 while time.time() < deadline:431 resp = requests.get(432 f"{COMFYUI_API_URL}/history/{prompt_id}",433 timeout=10,434 )435 resp.raise_for_status()436 history = resp.json()437438 if prompt_id in history:439 prompt_data = history[prompt_id]440 status = prompt_data.get("status", {})441442 if status.get("status_str") == "error":443 messages = status.get("messages", [])444 raise RuntimeError(445 f"ComfyUI prompt failed: {json.dumps(messages, indent=2)}"446 )447448 if status.get("completed", False):449 if prompt_data.get("outputs"):450 return prompt_data451452 # Completed but no outputs — retry briefly for race condition453 empty_complete_retries += 1454 if empty_complete_retries >= max_empty_retries:455 raise RuntimeError(456 f"ComfyUI prompt completed but produced no outputs. "457 f"This usually means a node failed silently (missing custom node or model). "458 f"Status: {json.dumps(status, indent=2)}"459 )460461 time.sleep(poll_interval)462463 raise TimeoutError(f"ComfyUI prompt {prompt_id} did not complete within {timeout}s")464465466def download_output_video(prompt_data: dict, output_dir: str) -> str:467 """Download the generated video from ComfyUI."""468 outputs = prompt_data.get("outputs", {})469 for node_id, node_output in outputs.items():470 # VHS/SaveVideo convention: check 'gifs' key first, then 'videos' fallback471 video_list = node_output.get("gifs") or node_output.get("videos") or node_output.get("images") or []472 if video_list:473 vid_info = video_list[0]474 filename = vid_info["filename"]475 subfolder = vid_info.get("subfolder", "")476 vid_type = vid_info.get("type", "output")477478 resp = requests.get(479 f"{COMFYUI_API_URL}/view",480 params={481 "filename": filename,482 "subfolder": subfolder,483 "type": vid_type,484 },485 timeout=120,486 )487 resp.raise_for_status()488489 out_filename = f"generated_{filename}"490 out_path = os.path.join(output_dir, out_filename)491 with open(out_path, "wb") as f:492 f.write(resp.content)493494 return out_filename495496 raise RuntimeError(497 f"No output video found in ComfyUI response. Available outputs: {json.dumps(outputs, indent=2)}"498 )499500501def main():502 try:503 input_json = sys.stdin.read()504 execution_input = json.loads(input_json)505 inputs = execution_input.get("inputs", {})506507 workflow_name = inputs.get("workflow", "image_to_video_wan")508 image = inputs.get("image", "")509 text = inputs.get("text", "")510 audio = inputs.get("audio", "")511 resolution = inputs.get("resolution", "1280x720")512 min_resolution = int(inputs.get("min_resolution", 512))513514 # Parse resolution515 width, height = parse_resolution(resolution)516517 # Common validation518 if not image:519 raise ValueError("Image input is required")520521 # Workflow-specific validation522 if workflow_name == "audio_to_video_sonic":523 if not audio:524 raise ValueError("Audio input is required for Audio-Driven Animation workflow")525 elif workflow_name == "image_to_video_wan":526 if not text:527 raise ValueError("Text prompt is required for Image to Video workflow")528 else:529 raise ValueError(f"Unknown workflow: {workflow_name}")530531 os.makedirs(OUTPUT_DIR, exist_ok=True)532533 # Upload image to ComfyUI534 image_path = os.path.join(INPUT_DIR, image)535 if not os.path.exists(image_path):536 raise FileNotFoundError(f"Input image not found: {image_path}")537 comfyui_image_name = upload_file_to_comfyui(image_path, "image/png")538539 # Upload audio to ComfyUI (required for SONIC, optional for WAN)540 comfyui_audio_name = ""541 if audio:542 audio_path = os.path.join(INPUT_DIR, audio)543 if not os.path.exists(audio_path):544 raise FileNotFoundError(f"Input audio not found: {audio_path}")545 audio_mime = detect_audio_mime(audio)546 comfyui_audio_name = upload_file_to_comfyui(audio_path, audio_mime)547548 # Build workflow, submit, wait, download549 workflow = build_workflow(550 workflow_name, comfyui_image_name, width, height,551 text=text, audio_filename=comfyui_audio_name, min_resolution=min_resolution,552 )553 prompt_id = submit_prompt(workflow)554 prompt_data = wait_for_result(prompt_id)555 out_filename = download_output_video(prompt_data, OUTPUT_DIR)556557 # Log metadata to stderr558 print(f"prompt_id={prompt_id}, workflow={workflow_name}, resolution={resolution}", file=sys.stderr)559560 # Flat output — keys match OUTPUT_SCHEMA561 output = {562 "video": out_filename,563 }564 print(json.dumps(output, indent=2))565566 except Exception as e:567 error_output = {568 "error": str(e),569 "errorType": type(e).__name__,570 "traceback": traceback.format_exc(),571 }572 print(json.dumps(error_output), file=sys.stderr)573 sys.exit(1)574575576if __name__ == "__main__":577 main()