""" Seedance 多模态参考生视频节点 """ import io as py_io import json import tempfile import aiohttp import torch from ..clients.seedance_client import SeedanceClient from ..clients.gemini_client import GeminiAPIClient from ..utils.image_utils import tensor_to_pil, pil_to_tensor from ..utils.r2_uploader import upload_image, upload_video, upload_audio from ..utils.config import get_base_url_by_route from .seedance_autopass import ( SeedanceAutoPass, _BASE_MODELS as _LATEST_BASE_MODELS, _MODEL_CAPABILITIES as _LATEST_MODEL_CAPABILITIES, _MODEL_ROUTES as _LATEST_MODEL_ROUTES, _RATIOS as _LATEST_RATIOS, _resolve_model_matrix as _resolve_latest_model_matrix, ) from ..utils.video_task import format_seedance_generation_error from comfy_api.latest import InputImpl, io # ── 模型列表 ────────────────────────────────────────────────────────────────── _MM_MODEL_LABEL_TO_ID = { "seedance 2.0 海外版(高并发)": "dreamina-seedance-2-0-hc", "seedance 2.0 fast 海外版(高并发)": "dreamina-seedance-2-0-fast-hc", "seedance 2.0 mini 海外版(高并发)": "dreamina-seedance-2-0-mini-hc", "seedance 2.0 海外版": "seedance-2-0-260128-d", "seedance 2.0 海外版(破限)": "seedance-2-0-260128-d-ep", "seedance 2.0 fast 海外版": "seedance-2-0-fast-260128-d", "seedance 2.0 fast 海外版(破限)": "seedance-2-0-fast-d-ep", "seedance 2.0 mini 海外版": "seedance-2-0-mini-260615-d", "seedance 2.0 mini 海外版(破限)": "seedance-2-0-mini-260615-d-ep", } _MM_MODELS = list(_MM_MODEL_LABEL_TO_ID.keys()) # 多模态节点矩阵式模型配置(主模型 × 模型线路 → 实际模型ID) _MM_BASE_MODELS = list(_LATEST_BASE_MODELS) _MM_ROUTES = list(dict.fromkeys([*_LATEST_MODEL_ROUTES, "国内"])) _MM_MODEL_MATRIX = { ("seedance 2.0", "国内"): "doubao-seedance-2-0-260128-max", ("seedance 2.0 fast", "国内"): "doubao-seedance-2-0-fast-260128-max", ("seedance 2.0 mini", "国内"): "doubao-seedance-2-0-mini-260615-max", ("seedance 2.5", "国内"): "doubao-seedance-2-5-260628-max", } _MM_RATIOS = list(_LATEST_RATIOS) _MM_MAX_CAPABILITIES = _LATEST_MODEL_CAPABILITIES["seedance 2.5"] _MM_IMAGE_LIMIT = _MM_MAX_CAPABILITIES["images"] _MM_VIDEO_LIMIT = _MM_MAX_CAPABILITIES["videos"] _MM_AUDIO_LIMIT = _MM_MAX_CAPABILITIES["audios"] def _resolve_model_matrix(base_model: str, route: str) -> str: """矩阵式解析:主模型 + 模型线路 → 实际模型ID""" model = _MM_MODEL_MATRIX.get((base_model, route)) if model is not None: return model return _resolve_latest_model_matrix(base_model, route) def _resolve_model(model: str) -> str: """兼容批量节点保存的模型展示名;真实模型 ID 原样返回。""" return _MM_MODEL_LABEL_TO_ID.get(model, model) # 多模态参考生视频支持 4k _MM_RESOLUTIONS = ["720p", "1080p", "4k", "480p"] # 受限模型仅支持 480p / 720p(前端做选择时报错提示) _LIMITED_RESOLUTION_MODELS = { "doubao-seedance-2-0-fast-260128", "doubao-seedance-2-0-fast-260128-max", "doubao-seedance-2-0-mini-260615-max", "dreamina-seedance-2-0-fast-hc", "seedance-2-0-fast-260128-d", "seedance-2-0-fast-d-ep", "dreamina-seedance-2-0-mini-hc", } _FAST_UNSUPPORTED_RESOLUTIONS = ["1080p", "4k"] _MAX_IMAGE_BYTES = 30 * 1024 * 1024 _MAX_REQUEST_BODY_BYTES = 64 * 1024 * 1024 # ── 模型能力判断 ────────────────────────────────────────────────────────────── def _check_fast_resolution(model: str, resolution: str): """受限模型不支持 1080p / 4k,提交前拦截(保留旧函数名兼容调用方)。""" if model in _LIMITED_RESOLUTION_MODELS and resolution in _FAST_UNSUPPORTED_RESOLUTIONS: raise ValueError( f"{model} 不支持 {resolution} 分辨率," f"请改用 {_FAST_UNSUPPORTED_RESOLUTIONS} 以外的分辨率(如 720p)。" ) def _is_new_format_model(model: str) -> bool: """判断是否使用新请求体格式的模型(顶层 content,role 用 subject) 端点与老格式模型相同(均为 /v1/video/generations), 仅请求体结构不同,由此函数控制节点侧如何拼装 body。 """ return model in [ "dreamina-seedance-2-0-hc", "dreamina-seedance-2-0-fast-hc", "dreamina-seedance-2-0-mini-hc", "dreamina-seedance-2-5-hc", "seedance-2-0-260128-d", "seedance-2-0-260128-d-ep", "seedance-2-0-fast-260128-d", "seedance-2-0-fast-d-ep", "seedance-2-0-mini-260615-d", "seedance-2-0-mini-260615-d-ep", "doubao-seedance-2-0-260128-max", "doubao-seedance-2-0-fast-260128-max", "doubao-seedance-2-0-mini-260615-max", "doubao-seedance-2-5-260628-max", ] # ── 工具函数 ────────────────────────────────────────────────────────────────── def _format_mb(size_bytes: int) -> str: return f"{size_bytes / 1024 / 1024:.2f}MB" def _tensor_to_png(tensor, label: str = "图片"): """ComfyUI IMAGE tensor → (PIL.Image RGB, png_bytes),并做单图大小校验。""" pil_images = tensor_to_pil(tensor) image = pil_images[0] if image.mode == "RGBA": image = image.convert("RGB") buffered = py_io.BytesIO() image.save(buffered, format="PNG") image_bytes = buffered.getvalue() image_size = len(image_bytes) if image_size > _MAX_IMAGE_BYTES: raise ValueError( f"Seedance {label}大小 {_format_mb(image_size)} 超过单张图片 " f"{_format_mb(_MAX_IMAGE_BYTES)} 限制,请先压缩或缩小图片。" ) return image, image_bytes async def _tensor_to_uploaded_url(tensor, base_url: str, label: str = "图片") -> str: """ComfyUI IMAGE tensor → 上传到 R2 并返回公网 URL(保留单图大小校验)。 与参考视频/音频一致,图片改走上传 URL 而非内联 base64, 以显著缩小请求体、避免触碰 64MB 请求体上限。 """ image, _ = _tensor_to_png(tensor, label) return await upload_image(image, base_url=base_url) def _validate_request_body_size(body: dict, tag: str): body_size = len(json.dumps(body, ensure_ascii=False).encode("utf-8")) if body_size > _MAX_REQUEST_BODY_BYTES: raise ValueError( f"{tag} 请求体大小 {_format_mb(body_size)} 超过 " f"{_format_mb(_MAX_REQUEST_BODY_BYTES)} 限制,请减少参考图片数量或降低图片尺寸。" ) print( f"[{tag}] 请求体大小: {_format_mb(body_size)} " f"(限制 {_format_mb(_MAX_REQUEST_BODY_BYTES)})" ) _REQUEST_LOG_SECRET_FIELDS = { "authorization", "api_key", "apikey", "api-key", "access_token", "token", } _REQUEST_LOG_BASE64_FIELDS = {"data", "b64_json", "base64", "image_base64"} _REQUEST_LOG_URL_FIELDS = {"url", "image", "image_url", "video_url", "audio_url"} def _sanitize_request_body_for_log(value, field_name: str = ""): """Copy a request body for logging without exposing credentials or media URLs.""" normalized_field = field_name.lower() if normalized_field in _REQUEST_LOG_SECRET_FIELDS: return "" if isinstance(value, dict): return { key: _sanitize_request_body_for_log(item, str(key)) for key, item in value.items() } if isinstance(value, list): return [_sanitize_request_body_for_log(item, field_name) for item in value] if isinstance(value, (bytes, bytearray)): return f"" if not isinstance(value, str): return value if normalized_field in _REQUEST_LOG_BASE64_FIELDS: return f"" header, separator, data = value.partition(",") if separator and header.lower().startswith("data:") and ";base64" in header.lower(): return f"{header}," if normalized_field in _REQUEST_LOG_URL_FIELDS and value.lower().startswith(("http://", "https://")): return "" return value def _log_original_request_body(body: dict): safe_body = _sanitize_request_body_for_log(body) print( "[SeedanceMultiModal] 原始请求体(临时 URL 与媒体数据已折叠):\n" f"{json.dumps(safe_body, ensure_ascii=False, indent=2)}" ) async def _url_to_tensor(url: str) -> torch.Tensor: """从 URL 下载图片并转为 ComfyUI IMAGE tensor,失败时返回 None""" try: from PIL import Image async with aiohttp.ClientSession() as session: async with session.get(url, allow_redirects=True) as resp: if resp.status != 200: return None data = await resp.read() img = Image.open(py_io.BytesIO(data)).convert("RGB") return pil_to_tensor([img]) except Exception as e: print(f"[Seedance] 末帧图片下载失败: {e}") return None def _show_balance(): """完成后打印余额(静默失败)""" try: client = GeminiAPIClient() data = client.query_balance_sync() print(f"Seedance: {client.format_balance_info(data)}") except Exception: pass def _make_pbar(): try: from comfy.utils import ProgressBar return ProgressBar(100) except Exception: return None def _make_callbacks(tag: str, pbar): def on_stage(stage: str): if stage == "submitting": print(f"[{tag}] 提交中...") if pbar: pbar.update_absolute(0, 100) elif stage.startswith("submitted:"): print(f"[{tag}] 已提交 → {stage.split(':', 1)[1]}") if pbar: pbar.update_absolute(5, 100) elif stage == "downloading": print(f"[{tag}] 下载视频中...") if pbar: pbar.update_absolute(99, 100) elif stage == "done": print(f"[{tag}] 完成") if pbar: pbar.update_absolute(100, 100) def on_progress(pct: int): if pbar: pbar.update_absolute(5 + int(pct * 0.94), 100) return on_stage, on_progress # ── 多模态参考生视频节点 ────────────────────────────────────────────────────── class SeedanceMultiModal(io.ComfyNode): """Seedance 2.0 / 2.5 多模态参考生视频。""" @classmethod def define_schema(cls): return io.Schema( node_id="SeedanceMultiModal", display_name="Seedance 多模态参考生视频", description="支持动态增加参考图片、视频、音频,以及渐进填写素材 ID。", category="comfyui_o1key/Seedance", inputs=[ io.String.Input("提示词", multiline=True, default=""), io.Combo.Input("主模型", options=_MM_BASE_MODELS, default="seedance 2.0"), io.Combo.Input("模型线路", options=_MM_ROUTES, default="国内"), io.Combo.Input("分辨率", options=_MM_RESOLUTIONS, default="720p"), io.Combo.Input( "宽高比", options=_MM_RATIOS, default="智能", ), io.Combo.Input( "时长", options=["自动"] + [f"{i}秒" for i in range(4, 31)], default="5秒", ), io.Combo.Input("生成音频", options=["关闭", "打开"], default="关闭"), io.Combo.Input("联网搜索", options=["关闭", "打开"], default="关闭"), io.Combo.Input("返回末帧图片", options=["关闭", "打开"], default="关闭"), io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff), io.Autogrow.Input( "参考图片", optional=True, template=io.Autogrow.TemplateNames( input=io.Image.Input("参考图片"), names=[f"参考图片{i}" for i in range(1, _MM_IMAGE_LIMIT + 1)], min=0, ), ), io.Autogrow.Input( "参考视频", optional=True, template=io.Autogrow.TemplateNames( input=io.Video.Input("参考视频"), names=[f"参考视频{i}" for i in range(1, _MM_VIDEO_LIMIT + 1)], min=0, ), ), io.Autogrow.Input( "参考音频", optional=True, template=io.Autogrow.TemplateNames( input=io.Audio.Input("参考音频"), names=[f"参考音频{i}" for i in range(1, _MM_AUDIO_LIMIT + 1)], min=0, ), ), # 保留已发布的 9/3/3 widget 顺序;只更新图片素材的显示名称, # 2.5 的新增素材 ID 仍仅追加,避免 widgets_values 发生位置漂移。 *[io.String.Input(f"图片素材ID{i}", default="") for i in range(1, 10)], *[io.String.Input(f"视频素材ID{i}", default="") for i in range(1, 4)], *[io.String.Input(f"音频素材ID{i}", default="") for i in range(1, 4)], *[io.String.Input(f"图片素材ID{i}", default="") for i in range(10, _MM_IMAGE_LIMIT + 1)], *[io.String.Input(f"视频素材ID{i}", default="") for i in range(4, _MM_VIDEO_LIMIT + 1)], *[io.String.Input(f"音频素材ID{i}", default="") for i in range(4, _MM_AUDIO_LIMIT + 1)], ], outputs=[ io.Video.Output(display_name="视频"), io.Image.Output(display_name="末帧图片"), ], ) @staticmethod def _autogrow_values(kwargs, group_name, legacy_prefix, legacy_max): """读取 V3 动态输入,同时兼容旧工作流的编号端口直接调用。""" group = kwargs.get(group_name) if isinstance(group, dict): return [value for value in group.values() if value is not None] def _first(value): if isinstance(value, list): return value[0] if value else None return value return [ value for index in range(1, legacy_max + 1) if (value := _first(kwargs.get(f"{legacy_prefix}{index}"))) is not None ] @classmethod async def execute(cls, **kwargs): return await cls.generate(**kwargs) @classmethod async def generate(cls, **kwargs): # V3 传入标量;保留列表兼容旧版 INPUT_IS_LIST 的直接调用。 def _first(v, default=None): if isinstance(v, list): return v[0] if v else default return v if v is not None else default prompt = _first(kwargs.get("提示词"), "").strip() base_model = _first(kwargs.get("主模型"), "seedance 2.0") route = _first(kwargs.get("模型线路"), "国内") model = _resolve_model_matrix(base_model, route) resolution = _first(kwargs.get("分辨率")) ratio = _first(kwargs.get("宽高比")) duration_str = _first(kwargs.get("时长"), "5秒") # 解析时长:自动 → -1,其他提取数字 if duration_str == "自动": duration = -1 else: duration = int(duration_str.replace("秒", "")) gen_audio = _first(kwargs.get("生成音频"), "关闭") == "打开" web_search = _first(kwargs.get("联网搜索"), "关闭") == "打开" return_last = _first(kwargs.get("返回末帧图片"), "关闭") == "打开" seed = _first(kwargs.get("seed"), 0) ref_images = cls._autogrow_values(kwargs, "参考图片", "参考图片", _MM_IMAGE_LIMIT) ref_videos = cls._autogrow_values(kwargs, "参考视频", "参考视频", _MM_VIDEO_LIMIT) ref_audios = cls._autogrow_values(kwargs, "参考音频", "参考音频", _MM_AUDIO_LIMIT) # 新名称优先;旧名称兼容未经过浏览器迁移的 API 工作流。 element_ids = [] for i in range(1, _MM_IMAGE_LIMIT + 1): current_id = _first(kwargs.get(f"图片素材ID{i}"), "").strip() legacy_id = _first(kwargs.get(f"真人素材ID{i}"), "").strip() element_ids.append(current_id or legacy_id) video_ids = [_first(kwargs.get(f"视频素材ID{i}"), "").strip() for i in range(1, _MM_VIDEO_LIMIT + 1)] audio_ids = [_first(kwargs.get(f"音频素材ID{i}"), "").strip() for i in range(1, _MM_AUDIO_LIMIT + 1)] element_ids = [eid for eid in element_ids if eid] video_ids = [vid for vid in video_ids if vid] audio_ids = [aid for aid in audio_ids if aid] # ── 校验 ────────────────────────────────────────────────────────── has_image = bool(ref_images) has_video = len(ref_videos) > 0 has_audio = len(ref_audios) > 0 has_element = len(element_ids) > 0 has_video_id = len(video_ids) > 0 has_audio_id = len(audio_ids) > 0 if not has_image and not has_video and not has_audio and not has_element and not has_video_id and not has_audio_id and not prompt: raise ValueError("至少需要提供参考图片、参考视频、图片素材ID、视频素材ID或提示词之一。") if base_model != "seedance 2.5" and (has_audio or has_audio_id) and not has_image and not has_video and not has_element and not has_video_id: raise ValueError("不可单独输入音频,请至少连接一张参考图片、一个参考视频或提供图片/视频素材ID。") # 国内线路仅替换实际模型 ID,能力限制与对应主模型保持一致。 SeedanceAutoPass._validate_dynamic_parameters( base_model, route, duration_str, [*ref_images, *element_ids], [*ref_videos, *video_ids], [*ref_audios, *audio_ids], ) _check_fast_resolution(model, resolution) # 判断是否使用新格式 use_new_format = _is_new_format_model(model) # ── 构建 content 列表 ───────────────────────────────────────────── content = [] base_url = get_base_url_by_route() # 真人素材 ID(与参考图片共用当前模型的图片额度,优先级最高) if has_element: for idx, eid in enumerate(element_ids, start=1): # 确保 element_id 格式正确 asset_url = eid if eid.startswith("asset://") else f"asset://{eid}" # 第一个素材ID作为主体(subject),其余作为参考图片(reference_image) if idx == 1: role = "subject" if use_new_format else "reference_image" else: role = "reference_image" content.append({ "type": "image_url", "image_url": {"url": asset_url}, "role": role, }) role_label = "主体" if role == "subject" else "参考" print(f"[SeedanceMultiModal] 使用图片素材ID{idx}({role_label}) → {asset_url}") # 参考图片(2.5 最多 30 个独立槽位,用户自行选择连接哪几个) if has_image: for idx, img_tensor in enumerate(ref_images, start=1): # 每个 tensor 可能是 [1,H,W,C] 或 [H,W,C],统一确保有 batch 维 if img_tensor.dim() == 3: img_tensor = img_tensor.unsqueeze(0) url = await _tensor_to_uploaded_url(img_tensor, base_url, f"参考图片{idx}") content.append({ "type": "image_url", "image_url": {"url": url}, "role": "reference_image", }) # 参考视频(2.5 最多 10 个) for v in ref_videos: url = await upload_video(v, base_url=base_url) content.append({ "type": "video_url", "video_url": {"url": url}, "role": "reference_video", }) # 视频素材 ID(与参考视频共用当前模型的视频额度) for idx, vid in enumerate(video_ids, start=1): asset_url = vid if vid.startswith("asset://") else f"asset://{vid}" content.append({ "type": "video_url", "video_url": {"url": asset_url}, "role": "reference_video", }) print(f"[SeedanceMultiModal] 使用视频素材ID{idx} → {asset_url}") # 参考音频(2.5 最多 10 段) for a in ref_audios: url = await upload_audio(a, base_url=base_url) content.append({ "type": "audio_url", "audio_url": {"url": url}, "role": "reference_audio", }) # 音频素材 ID(与参考音频共用当前模型的音频额度) for idx, aid in enumerate(audio_ids, start=1): asset_url = aid if aid.startswith("asset://") else f"asset://{aid}" content.append({ "type": "audio_url", "audio_url": {"url": asset_url}, "role": "reference_audio", }) print(f"[SeedanceMultiModal] 使用音频素材ID{idx} → {asset_url}") # 文本提示词(放最后) if prompt: content.append({"type": "text", "text": prompt}) if not content: raise ValueError("content 为空,请至少提供参考图片、参考视频或提示词。") # ── 构建请求体 ────────────────────────────────────────────────────── if use_new_format: # 新格式:顶层 content # 注意:文本提示词应该放在最前面 ordered_content = [] # 先添加文本 text_items = [item for item in content if item.get("type") == "text"] ordered_content.extend(text_items) # 再添加其他内容(图片、视频、音频) non_text_items = [item for item in content if item.get("type") != "text"] ordered_content.extend(non_text_items) body = { "model": model, "content": ordered_content, "duration": duration if duration != -1 else 5, "resolution": resolution, "ratio": ratio if ratio not in ("智能", "adaptive") else "16:9", # adaptive 为旧工作流兼容 "generate_audio": gen_audio, "watermark": False, "return_last_frame": return_last, } if seed != 0: body["seed"] = seed print(f"[SeedanceMultiModal] 新格式请求体: model={model}, content_len={len(ordered_content)}, duration={body['duration']}, resolution={body['resolution']}") else: # 老格式:metadata.content metadata: dict = { "resolution": resolution, "watermark": False, "content": content, } if ratio not in ("智能", "adaptive"): # adaptive 为旧工作流兼容 metadata["ratio"] = ratio if duration != -1: metadata["duration"] = duration if gen_audio: metadata["generate_audio"] = True if return_last: metadata["return_last_frame"] = True if seed != 0: metadata["seed"] = seed if web_search: metadata["tools"] = [{"type": "web_search"}] # 顶层 image:取第一张图的 URL(优先真人素材,其次参考图的上传 URL) first_image_url = next( (item["image_url"]["url"] for item in content if item["type"] == "image_url"), None, ) body = { "model": model, "prompt": prompt if prompt else " ", "metadata": metadata, } if first_image_url: body["image"] = first_image_url _validate_request_body_size(body, "Seedance多模态") _log_original_request_body(body) # ── 保存路径(临时文件,避免与下游保存节点重复落盘)────────────────── _, save_path = tempfile.mkstemp(suffix=".mp4", prefix="seedance_mm_") client = SeedanceClient() client.base_url = base_url pbar = _make_pbar() on_stage, on_prog = _make_callbacks("Seedance多模态", pbar) try: try: result_path, last_frame_url = await client.generate_async( body=body, save_path=save_path, on_stage=on_stage, on_progress=on_prog, use_new_format=use_new_format, ) except Exception as exc: message = format_seedance_generation_error(exc) if message == str(exc): raise raise RuntimeError(message) from None last_frame_tensor = None if return_last and last_frame_url: last_frame_tensor = await _url_to_tensor(last_frame_url) return io.NodeOutput(InputImpl.VideoFromFile(result_path), last_frame_tensor) finally: _show_balance() # ── 节点注册 ────────────────────────────────────────────────────────────────── NODE_CLASS_MAPPINGS = { "SeedanceMultiModal": SeedanceMultiModal, } NODE_DISPLAY_NAME_MAPPINGS = { "SeedanceMultiModal": "Seedance 多模态参考生视频", }