Video Lipsync Legacy

1import os2import sys3import json4import time5import random6import copy7import traceback89try:10    import requests11except ImportError:12    import subprocess13    subprocess.check_call([sys.executable, "-m", "pip", "install", "requests"])14    import requests1516COMFYUI_API_URL = os.getenv("COMFYUI_API_URL", "http://192.168.1.39:8188")17INPUT_DIR = "/data/input"18OUTPUT_DIR = "/data/output"1920# ---------- LatentSync 1.6 workflow (EM_audio_to_video_laten_v1.1) ----------21WORKFLOW = {22    "54": {23        "inputs": {24            "seed": 266,25            "lips_expression": 1.5,26            "inference_steps": 20,27            "images": ["55", 0],28            "audio": ["55", 1],29        },30        "class_type": "LatentSyncNode",31        "_meta": {"title": "LatentSync1.6 Node"},32    },33    "55": {34        "inputs": {35            "mode": "pingpong",36            "fps": 25,37            "silent_padding_sec": 0.5,38            "images": ["60", 0],39            "audio": ["61", 0],40        },41        "class_type": "VideoLengthAdjuster",42        "_meta": {"title": "Video Length Adjuster"},43    },44    "58": {45        "inputs": {46            "frame_rate": 25,47            "loop_count": 0,48            "filename_prefix": "video/ComfyUI",49            "format": "video/h264-mp4",50            "pix_fmt": "yuv420p",51            "crf": 19,52            "save_metadata": True,53            "pingpong": False,54            "save_output": True,55            "images": ["54", 0],56            "audio": ["54", 1],57        },58        "class_type": "VHS_VideoCombine",59        "_meta": {"title": "Video Combine (Upload) \U0001f3a5\U0001d5e5\U0001d5db\U0001d5e6"},60    },61    "60": {62        "inputs": {63            "video": "",64            "force_rate": 0,65            "custom_width": 0,66            "custom_height": 0,67            "frame_load_cap": 0,68            "skip_first_frames": 0,69            "select_every_nth": 1,70            "format": "AnimateDiff",71        },72        "class_type": "VHS_LoadVideo",73        "_meta": {"title": "Load Video (Upload) \U0001f3a5\U0001d5e5\U0001d5db\U0001d5e6"},74    },75    "61": {76        "inputs": {77            "audio": "",78            "start_time": 0,79            "duration": 0,80        },81        "class_type": "VHS_LoadAudioUpload",82        "_meta": {"title": "Load Audio (Upload)\U0001f3a5\U0001d5e5\U0001d5db\U0001d5e6"},83    },84}858687def upload_file_to_comfyui(local_path: str, content_type: str) -> str:88    """Upload a local file to ComfyUI and return the uploaded filename."""89    with open(local_path, "rb") as f:90        resp = requests.post(91            f"{COMFYUI_API_URL}/upload/image",92            files={"image": (os.path.basename(local_path), f, content_type)},93            timeout=30,94        )95    resp.raise_for_status()96    data = resp.json()97    return data["name"]9899100def detect_video_mime(filename: str) -> str:101    """Detect MIME type from video file extension."""102    ext = os.path.splitext(filename)[1].lower()103    mime_map = {104        ".mp4": "video/mp4",105        ".avi": "video/x-msvideo",106        ".mov": "video/quicktime",107        ".mkv": "video/x-matroska",108        ".webm": "video/webm",109        ".wmv": "video/x-ms-wmv",110    }111    return mime_map.get(ext, "application/octet-stream")112113114def detect_audio_mime(filename: str) -> str:115    """Detect MIME type from audio file extension."""116    ext = os.path.splitext(filename)[1].lower()117    mime_map = {118        ".mp3": "audio/mpeg",119        ".wav": "audio/wav",120        ".ogg": "audio/ogg",121        ".flac": "audio/flac",122        ".aac": "audio/aac",123        ".m4a": "audio/mp4",124    }125    return mime_map.get(ext, "application/octet-stream")126127128def build_workflow(video_name: str, audio_name: str) -> dict:129    """Build the lipsync workflow with the given uploaded filenames."""130    wf = copy.deepcopy(WORKFLOW)131132    # Video (node 60): inject uploaded filename and prevent resizing133    wf["60"]["inputs"]["video"] = video_name134    wf["60"]["inputs"]["force_size"] = "Disabled"135136    # Audio (node 61): inject uploaded filename137    wf["61"]["inputs"]["audio"] = audio_name138139    # Seed (node 54): randomize140    wf["54"]["inputs"]["seed"] = random.randint(0, 2**31 - 1)141142    return wf143144145def submit_prompt(workflow: dict) -> str:146    """Submit workflow to ComfyUI and return prompt_id."""147    resp = requests.post(148        f"{COMFYUI_API_URL}/prompt",149        json={"prompt": workflow},150        timeout=30,151    )152    if resp.status_code != 200:153        try:154            error_detail = resp.json()155        except Exception:156            error_detail = resp.text157        raise RuntimeError(158            f"ComfyUI /prompt returned {resp.status_code}: {json.dumps(error_detail, indent=2) if isinstance(error_detail, dict) else error_detail}"159        )160    data = resp.json()161162    # ComfyUI returns 200 even when nodes have validation errors163    node_errors = data.get("node_errors", {})164    if node_errors:165        raise RuntimeError(166            f"ComfyUI workflow has node errors: {json.dumps(node_errors, indent=2)}"167        )168169    return data["prompt_id"]170171172def wait_for_result(prompt_id: str, timeout: int = 900, poll_interval: int = 3) -> dict:173    """Poll ComfyUI history until the prompt completes with outputs."""174    deadline = time.time() + timeout175    empty_complete_retries = 0176    max_empty_retries = 3  # grace period for output serialization lag177178    while time.time() < deadline:179        resp = requests.get(180            f"{COMFYUI_API_URL}/history/{prompt_id}",181            timeout=10,182        )183        resp.raise_for_status()184        history = resp.json()185186        if prompt_id in history:187            prompt_data = history[prompt_id]188            status = prompt_data.get("status", {})189190            if status.get("status_str") == "error":191                messages = status.get("messages", [])192                raise RuntimeError(193                    f"ComfyUI prompt failed: {json.dumps(messages, indent=2)}"194                )195196            if status.get("completed", False):197                if prompt_data.get("outputs"):198                    return prompt_data199200                # Completed but no outputs — retry briefly for race condition201                empty_complete_retries += 1202                if empty_complete_retries >= max_empty_retries:203                    raise RuntimeError(204                        f"ComfyUI prompt completed but produced no outputs. "205                        f"This usually means a node failed silently (missing custom node or model). "206                        f"Status: {json.dumps(status, indent=2)}"207                    )208209        time.sleep(poll_interval)210211    raise TimeoutError(f"ComfyUI prompt {prompt_id} did not complete within {timeout}s")212213214def download_output_video(prompt_data: dict, output_dir: str) -> str:215    """Download the generated video from ComfyUI."""216    outputs = prompt_data.get("outputs", {})217    for node_id, node_output in outputs.items():218        # VHS/SaveVideo convention: check 'gifs' key first, then 'videos' fallback219        video_list = node_output.get("gifs") or node_output.get("videos") or node_output.get("images") or []220        if video_list:221            vid_info = video_list[0]222            filename = vid_info["filename"]223            subfolder = vid_info.get("subfolder", "")224            vid_type = vid_info.get("type", "output")225226            resp = requests.get(227                f"{COMFYUI_API_URL}/view",228                params={229                    "filename": filename,230                    "subfolder": subfolder,231                    "type": vid_type,232                },233                timeout=120,234            )235            resp.raise_for_status()236237            out_filename = f"generated_{filename}"238            out_path = os.path.join(output_dir, out_filename)239            with open(out_path, "wb") as f:240                f.write(resp.content)241242            return out_filename243244    raise RuntimeError(245        f"No output video found in ComfyUI response. Available outputs: {json.dumps(outputs, indent=2)}"246    )247248249def main():250    try:251        input_json = sys.stdin.read()252        execution_input = json.loads(input_json)253        inputs = execution_input.get("inputs", {})254255        video = inputs.get("video", "")256        audio = inputs.get("audio", "")257258        if not video:259            raise ValueError("Video input is required")260        if not audio:261            raise ValueError("Audio input is required")262263        os.makedirs(OUTPUT_DIR, exist_ok=True)264265        # Upload video to ComfyUI266        video_path = os.path.join(INPUT_DIR, video)267        if not os.path.exists(video_path):268            raise FileNotFoundError(f"Input video not found: {video_path}")269        video_mime = detect_video_mime(video)270        comfyui_video_name = upload_file_to_comfyui(video_path, video_mime)271272        # Upload audio to ComfyUI273        audio_path = os.path.join(INPUT_DIR, audio)274        if not os.path.exists(audio_path):275            raise FileNotFoundError(f"Input audio not found: {audio_path}")276        audio_mime = detect_audio_mime(audio)277        comfyui_audio_name = upload_file_to_comfyui(audio_path, audio_mime)278279        # Build workflow, submit, wait, download280        workflow = build_workflow(comfyui_video_name, comfyui_audio_name)281        prompt_id = submit_prompt(workflow)282        prompt_data = wait_for_result(prompt_id)283        out_filename = download_output_video(prompt_data, OUTPUT_DIR)284285        # Log metadata to stderr286        print(f"prompt_id={prompt_id}", file=sys.stderr)287288        # Flat output — keys match OUTPUT_SCHEMA289        output = {290            "video": out_filename,291        }292        print(json.dumps(output, indent=2))293294    except Exception as e:295        error_output = {296            "error": str(e),297            "errorType": type(e).__name__,298            "traceback": traceback.format_exc(),299        }300        print(json.dumps(error_output), file=sys.stderr)301        sys.exit(1)302303304if __name__ == "__main__":305    main()