Jony
ba920f2b66
Replace the prior release tree with the current plugin, frontend, tests, and documentation. Document retired node IDs and the public Gitea update source.
630 lines
27 KiB
Python
630 lines
27 KiB
Python
"""
|
||
Seedance 多模态参考生视频节点
|
||
"""
|
||
|
||
import io as py_io
|
||
import json
|
||
import tempfile
|
||
|
||
import aiohttp
|
||
import torch
|
||
|
||
from ..clients.seedance_client import SeedanceClient
|
||
from ..clients.gemini_client import GeminiAPIClient
|
||
from ..utils.image_utils import tensor_to_pil, pil_to_tensor
|
||
from ..utils.r2_uploader import upload_image, upload_video, upload_audio
|
||
from ..utils.config import get_base_url_by_route
|
||
from .seedance_autopass import (
|
||
SeedanceAutoPass,
|
||
_BASE_MODELS as _LATEST_BASE_MODELS,
|
||
_MODEL_CAPABILITIES as _LATEST_MODEL_CAPABILITIES,
|
||
_MODEL_ROUTES as _LATEST_MODEL_ROUTES,
|
||
_RATIOS as _LATEST_RATIOS,
|
||
_resolve_model_matrix as _resolve_latest_model_matrix,
|
||
)
|
||
from ..utils.video_task import format_seedance_generation_error
|
||
|
||
from comfy_api.latest import InputImpl, io
|
||
|
||
|
||
# ── 模型列表 ──────────────────────────────────────────────────────────────────
|
||
|
||
_MM_MODEL_LABEL_TO_ID = {
|
||
"seedance 2.0 海外版（高并发）": "dreamina-seedance-2-0-hc",
|
||
"seedance 2.0 fast 海外版（高并发）": "dreamina-seedance-2-0-fast-hc",
|
||
"seedance 2.0 mini 海外版（高并发）": "dreamina-seedance-2-0-mini-hc",
|
||
"seedance 2.0 海外版": "seedance-2-0-260128-d",
|
||
"seedance 2.0 海外版（破限）": "seedance-2-0-260128-d-ep",
|
||
"seedance 2.0 fast 海外版": "seedance-2-0-fast-260128-d",
|
||
"seedance 2.0 fast 海外版（破限）": "seedance-2-0-fast-d-ep",
|
||
"seedance 2.0 mini 海外版": "seedance-2-0-mini-260615-d",
|
||
"seedance 2.0 mini 海外版（破限）": "seedance-2-0-mini-260615-d-ep",
|
||
}
|
||
_MM_MODELS = list(_MM_MODEL_LABEL_TO_ID.keys())
|
||
|
||
# 多模态节点矩阵式模型配置（主模型 × 模型线路 → 实际模型ID）
|
||
_MM_BASE_MODELS = list(_LATEST_BASE_MODELS)
|
||
_MM_ROUTES = list(dict.fromkeys([*_LATEST_MODEL_ROUTES, "国内"]))
|
||
_MM_MODEL_MATRIX = {
|
||
("seedance 2.0", "国内"): "doubao-seedance-2-0-260128-max",
|
||
("seedance 2.0 fast", "国内"): "doubao-seedance-2-0-fast-260128-max",
|
||
("seedance 2.0 mini", "国内"): "doubao-seedance-2-0-mini-260615-max",
|
||
("seedance 2.5", "国内"): "doubao-seedance-2-5-260628-max",
|
||
}
|
||
_MM_RATIOS = list(_LATEST_RATIOS)
|
||
_MM_MAX_CAPABILITIES = _LATEST_MODEL_CAPABILITIES["seedance 2.5"]
|
||
_MM_IMAGE_LIMIT = _MM_MAX_CAPABILITIES["images"]
|
||
_MM_VIDEO_LIMIT = _MM_MAX_CAPABILITIES["videos"]
|
||
_MM_AUDIO_LIMIT = _MM_MAX_CAPABILITIES["audios"]
|
||
|
||
|
||
def _resolve_model_matrix(base_model: str, route: str) -> str:
|
||
"""矩阵式解析：主模型 + 模型线路 → 实际模型ID"""
|
||
model = _MM_MODEL_MATRIX.get((base_model, route))
|
||
if model is not None:
|
||
return model
|
||
return _resolve_latest_model_matrix(base_model, route)
|
||
|
||
|
||
def _resolve_model(model: str) -> str:
|
||
"""兼容批量节点保存的模型展示名；真实模型 ID 原样返回。"""
|
||
return _MM_MODEL_LABEL_TO_ID.get(model, model)
|
||
|
||
|
||
# 多模态参考生视频支持 4k
|
||
_MM_RESOLUTIONS = ["720p", "1080p", "4k", "480p"]
|
||
# 受限模型仅支持 480p / 720p（前端做选择时报错提示）
|
||
_LIMITED_RESOLUTION_MODELS = {
|
||
"doubao-seedance-2-0-fast-260128",
|
||
"doubao-seedance-2-0-fast-260128-max",
|
||
"doubao-seedance-2-0-mini-260615-max",
|
||
"dreamina-seedance-2-0-fast-hc",
|
||
"seedance-2-0-fast-260128-d",
|
||
"seedance-2-0-fast-d-ep",
|
||
"dreamina-seedance-2-0-mini-hc",
|
||
}
|
||
_FAST_UNSUPPORTED_RESOLUTIONS = ["1080p", "4k"]
|
||
|
||
_MAX_IMAGE_BYTES = 30 * 1024 * 1024
|
||
_MAX_REQUEST_BODY_BYTES = 64 * 1024 * 1024
|
||
|
||
|
||
# ── 模型能力判断 ──────────────────────────────────────────────────────────────
|
||
|
||
def _check_fast_resolution(model: str, resolution: str):
|
||
"""受限模型不支持 1080p / 4k，提交前拦截（保留旧函数名兼容调用方）。"""
|
||
if model in _LIMITED_RESOLUTION_MODELS and resolution in _FAST_UNSUPPORTED_RESOLUTIONS:
|
||
raise ValueError(
|
||
f"{model} 不支持 {resolution} 分辨率，"
|
||
f"请改用 {_FAST_UNSUPPORTED_RESOLUTIONS} 以外的分辨率（如 720p）。"
|
||
)
|
||
|
||
|
||
def _is_new_format_model(model: str) -> bool:
|
||
"""判断是否使用新请求体格式的模型（顶层 content，role 用 subject）
|
||
|
||
端点与老格式模型相同（均为 /v1/video/generations），
|
||
仅请求体结构不同，由此函数控制节点侧如何拼装 body。
|
||
"""
|
||
return model in [
|
||
"dreamina-seedance-2-0-hc",
|
||
"dreamina-seedance-2-0-fast-hc",
|
||
"dreamina-seedance-2-0-mini-hc",
|
||
"dreamina-seedance-2-5-hc",
|
||
"seedance-2-0-260128-d",
|
||
"seedance-2-0-260128-d-ep",
|
||
"seedance-2-0-fast-260128-d",
|
||
"seedance-2-0-fast-d-ep",
|
||
"seedance-2-0-mini-260615-d",
|
||
"seedance-2-0-mini-260615-d-ep",
|
||
"doubao-seedance-2-0-260128-max",
|
||
"doubao-seedance-2-0-fast-260128-max",
|
||
"doubao-seedance-2-0-mini-260615-max",
|
||
"doubao-seedance-2-5-260628-max",
|
||
]
|
||
|
||
|
||
# ── 工具函数 ──────────────────────────────────────────────────────────────────
|
||
|
||
def _format_mb(size_bytes: int) -> str:
|
||
return f"{size_bytes / 1024 / 1024:.2f}MB"
|
||
|
||
|
||
def _tensor_to_png(tensor, label: str = "图片"):
|
||
"""ComfyUI IMAGE tensor → (PIL.Image RGB, png_bytes)，并做单图大小校验。"""
|
||
pil_images = tensor_to_pil(tensor)
|
||
image = pil_images[0]
|
||
if image.mode == "RGBA":
|
||
image = image.convert("RGB")
|
||
|
||
buffered = py_io.BytesIO()
|
||
image.save(buffered, format="PNG")
|
||
image_bytes = buffered.getvalue()
|
||
image_size = len(image_bytes)
|
||
|
||
if image_size > _MAX_IMAGE_BYTES:
|
||
raise ValueError(
|
||
f"Seedance {label}大小 {_format_mb(image_size)} 超过单张图片 "
|
||
f"{_format_mb(_MAX_IMAGE_BYTES)} 限制，请先压缩或缩小图片。"
|
||
)
|
||
|
||
return image, image_bytes
|
||
|
||
|
||
async def _tensor_to_uploaded_url(tensor, base_url: str, label: str = "图片") -> str:
|
||
"""ComfyUI IMAGE tensor → 上传到 R2 并返回公网 URL（保留单图大小校验）。
|
||
|
||
与参考视频/音频一致，图片改走上传 URL 而非内联 base64，
|
||
以显著缩小请求体、避免触碰 64MB 请求体上限。
|
||
"""
|
||
image, _ = _tensor_to_png(tensor, label)
|
||
return await upload_image(image, base_url=base_url)
|
||
|
||
|
||
def _validate_request_body_size(body: dict, tag: str):
|
||
body_size = len(json.dumps(body, ensure_ascii=False).encode("utf-8"))
|
||
if body_size > _MAX_REQUEST_BODY_BYTES:
|
||
raise ValueError(
|
||
f"{tag} 请求体大小 {_format_mb(body_size)} 超过 "
|
||
f"{_format_mb(_MAX_REQUEST_BODY_BYTES)} 限制，请减少参考图片数量或降低图片尺寸。"
|
||
)
|
||
print(
|
||
f"[{tag}] 请求体大小: {_format_mb(body_size)} "
|
||
f"(限制 {_format_mb(_MAX_REQUEST_BODY_BYTES)})"
|
||
)
|
||
|
||
|
||
_REQUEST_LOG_SECRET_FIELDS = {
|
||
"authorization",
|
||
"api_key",
|
||
"apikey",
|
||
"api-key",
|
||
"access_token",
|
||
"token",
|
||
}
|
||
_REQUEST_LOG_BASE64_FIELDS = {"data", "b64_json", "base64", "image_base64"}
|
||
_REQUEST_LOG_URL_FIELDS = {"url", "image", "image_url", "video_url", "audio_url"}
|
||
|
||
|
||
def _sanitize_request_body_for_log(value, field_name: str = ""):
|
||
"""Copy a request body for logging without exposing credentials or media URLs."""
|
||
normalized_field = field_name.lower()
|
||
if normalized_field in _REQUEST_LOG_SECRET_FIELDS:
|
||
return "<redacted>"
|
||
|
||
if isinstance(value, dict):
|
||
return {
|
||
key: _sanitize_request_body_for_log(item, str(key))
|
||
for key, item in value.items()
|
||
}
|
||
if isinstance(value, list):
|
||
return [_sanitize_request_body_for_log(item, field_name) for item in value]
|
||
if isinstance(value, (bytes, bytearray)):
|
||
return f"<binary data, {len(value)} bytes>"
|
||
if not isinstance(value, str):
|
||
return value
|
||
|
||
if normalized_field in _REQUEST_LOG_BASE64_FIELDS:
|
||
return f"<base64 data, {len(value)} chars>"
|
||
|
||
header, separator, data = value.partition(",")
|
||
if separator and header.lower().startswith("data:") and ";base64" in header.lower():
|
||
return f"{header},<base64 data, {len(data)} chars>"
|
||
|
||
if normalized_field in _REQUEST_LOG_URL_FIELDS and value.lower().startswith(("http://", "https://")):
|
||
return "<temporary URL omitted>"
|
||
|
||
return value
|
||
|
||
|
||
def _log_original_request_body(body: dict):
|
||
safe_body = _sanitize_request_body_for_log(body)
|
||
print(
|
||
"[SeedanceMultiModal] 原始请求体（临时 URL 与媒体数据已折叠）：\n"
|
||
f"{json.dumps(safe_body, ensure_ascii=False, indent=2)}"
|
||
)
|
||
|
||
|
||
|
||
async def _url_to_tensor(url: str) -> torch.Tensor:
|
||
"""从 URL 下载图片并转为 ComfyUI IMAGE tensor，失败时返回 None"""
|
||
try:
|
||
from PIL import Image
|
||
async with aiohttp.ClientSession() as session:
|
||
async with session.get(url, allow_redirects=True) as resp:
|
||
if resp.status != 200:
|
||
return None
|
||
data = await resp.read()
|
||
img = Image.open(py_io.BytesIO(data)).convert("RGB")
|
||
return pil_to_tensor([img])
|
||
except Exception as e:
|
||
print(f"[Seedance] 末帧图片下载失败: {e}")
|
||
return None
|
||
|
||
|
||
def _show_balance():
|
||
"""完成后打印余额（静默失败）"""
|
||
try:
|
||
client = GeminiAPIClient()
|
||
data = client.query_balance_sync()
|
||
print(f"Seedance: {client.format_balance_info(data)}")
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def _make_pbar():
|
||
try:
|
||
from comfy.utils import ProgressBar
|
||
return ProgressBar(100)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def _make_callbacks(tag: str, pbar):
|
||
def on_stage(stage: str):
|
||
if stage == "submitting":
|
||
print(f"[{tag}] 提交中...")
|
||
if pbar: pbar.update_absolute(0, 100)
|
||
elif stage.startswith("submitted:"):
|
||
print(f"[{tag}] 已提交 → {stage.split(':', 1)[1]}")
|
||
if pbar: pbar.update_absolute(5, 100)
|
||
elif stage == "downloading":
|
||
print(f"[{tag}] 下载视频中...")
|
||
if pbar: pbar.update_absolute(99, 100)
|
||
elif stage == "done":
|
||
print(f"[{tag}] 完成")
|
||
if pbar: pbar.update_absolute(100, 100)
|
||
|
||
def on_progress(pct: int):
|
||
if pbar: pbar.update_absolute(5 + int(pct * 0.94), 100)
|
||
|
||
return on_stage, on_progress
|
||
|
||
|
||
|
||
# ── 多模态参考生视频节点 ──────────────────────────────────────────────────────
|
||
|
||
class SeedanceMultiModal(io.ComfyNode):
|
||
"""Seedance 2.0 / 2.5 多模态参考生视频。"""
|
||
|
||
@classmethod
|
||
def define_schema(cls):
|
||
return io.Schema(
|
||
node_id="SeedanceMultiModal",
|
||
display_name="Seedance 多模态参考生视频",
|
||
description="支持动态增加参考图片、视频、音频，以及渐进填写素材 ID。",
|
||
category="comfyui_o1key/Seedance",
|
||
inputs=[
|
||
io.String.Input("提示词", multiline=True, default=""),
|
||
io.Combo.Input("主模型", options=_MM_BASE_MODELS, default="seedance 2.0"),
|
||
io.Combo.Input("模型线路", options=_MM_ROUTES, default="国内"),
|
||
io.Combo.Input("分辨率", options=_MM_RESOLUTIONS, default="720p"),
|
||
io.Combo.Input(
|
||
"宽高比",
|
||
options=_MM_RATIOS,
|
||
default="智能",
|
||
),
|
||
io.Combo.Input(
|
||
"时长",
|
||
options=["自动"] + [f"{i}秒" for i in range(4, 31)],
|
||
default="5秒",
|
||
),
|
||
io.Combo.Input("生成音频", options=["关闭", "打开"], default="关闭"),
|
||
io.Combo.Input("联网搜索", options=["关闭", "打开"], default="关闭"),
|
||
io.Combo.Input("返回末帧图片", options=["关闭", "打开"], default="关闭"),
|
||
io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff),
|
||
io.Autogrow.Input(
|
||
"参考图片",
|
||
optional=True,
|
||
template=io.Autogrow.TemplateNames(
|
||
input=io.Image.Input("参考图片"),
|
||
names=[f"参考图片{i}" for i in range(1, _MM_IMAGE_LIMIT + 1)],
|
||
min=0,
|
||
),
|
||
),
|
||
io.Autogrow.Input(
|
||
"参考视频",
|
||
optional=True,
|
||
template=io.Autogrow.TemplateNames(
|
||
input=io.Video.Input("参考视频"),
|
||
names=[f"参考视频{i}" for i in range(1, _MM_VIDEO_LIMIT + 1)],
|
||
min=0,
|
||
),
|
||
),
|
||
io.Autogrow.Input(
|
||
"参考音频",
|
||
optional=True,
|
||
template=io.Autogrow.TemplateNames(
|
||
input=io.Audio.Input("参考音频"),
|
||
names=[f"参考音频{i}" for i in range(1, _MM_AUDIO_LIMIT + 1)],
|
||
min=0,
|
||
),
|
||
),
|
||
# 保留已发布的 9/3/3 widget 顺序；只更新图片素材的显示名称，
|
||
# 2.5 的新增素材 ID 仍仅追加，避免 widgets_values 发生位置漂移。
|
||
*[io.String.Input(f"图片素材ID{i}", default="") for i in range(1, 10)],
|
||
*[io.String.Input(f"视频素材ID{i}", default="") for i in range(1, 4)],
|
||
*[io.String.Input(f"音频素材ID{i}", default="") for i in range(1, 4)],
|
||
*[io.String.Input(f"图片素材ID{i}", default="") for i in range(10, _MM_IMAGE_LIMIT + 1)],
|
||
*[io.String.Input(f"视频素材ID{i}", default="") for i in range(4, _MM_VIDEO_LIMIT + 1)],
|
||
*[io.String.Input(f"音频素材ID{i}", default="") for i in range(4, _MM_AUDIO_LIMIT + 1)],
|
||
],
|
||
outputs=[
|
||
io.Video.Output(display_name="视频"),
|
||
io.Image.Output(display_name="末帧图片"),
|
||
],
|
||
)
|
||
|
||
@staticmethod
|
||
def _autogrow_values(kwargs, group_name, legacy_prefix, legacy_max):
|
||
"""读取 V3 动态输入，同时兼容旧工作流的编号端口直接调用。"""
|
||
group = kwargs.get(group_name)
|
||
if isinstance(group, dict):
|
||
return [value for value in group.values() if value is not None]
|
||
|
||
def _first(value):
|
||
if isinstance(value, list):
|
||
return value[0] if value else None
|
||
return value
|
||
|
||
return [
|
||
value
|
||
for index in range(1, legacy_max + 1)
|
||
if (value := _first(kwargs.get(f"{legacy_prefix}{index}"))) is not None
|
||
]
|
||
|
||
@classmethod
|
||
async def execute(cls, **kwargs):
|
||
return await cls.generate(**kwargs)
|
||
|
||
@classmethod
|
||
async def generate(cls, **kwargs):
|
||
# V3 传入标量；保留列表兼容旧版 INPUT_IS_LIST 的直接调用。
|
||
def _first(v, default=None):
|
||
if isinstance(v, list):
|
||
return v[0] if v else default
|
||
return v if v is not None else default
|
||
|
||
prompt = _first(kwargs.get("提示词"), "").strip()
|
||
base_model = _first(kwargs.get("主模型"), "seedance 2.0")
|
||
route = _first(kwargs.get("模型线路"), "国内")
|
||
model = _resolve_model_matrix(base_model, route)
|
||
resolution = _first(kwargs.get("分辨率"))
|
||
ratio = _first(kwargs.get("宽高比"))
|
||
duration_str = _first(kwargs.get("时长"), "5秒")
|
||
# 解析时长：自动 → -1，其他提取数字
|
||
if duration_str == "自动":
|
||
duration = -1
|
||
else:
|
||
duration = int(duration_str.replace("秒", ""))
|
||
gen_audio = _first(kwargs.get("生成音频"), "关闭") == "打开"
|
||
web_search = _first(kwargs.get("联网搜索"), "关闭") == "打开"
|
||
return_last = _first(kwargs.get("返回末帧图片"), "关闭") == "打开"
|
||
seed = _first(kwargs.get("seed"), 0)
|
||
|
||
ref_images = cls._autogrow_values(kwargs, "参考图片", "参考图片", _MM_IMAGE_LIMIT)
|
||
ref_videos = cls._autogrow_values(kwargs, "参考视频", "参考视频", _MM_VIDEO_LIMIT)
|
||
ref_audios = cls._autogrow_values(kwargs, "参考音频", "参考音频", _MM_AUDIO_LIMIT)
|
||
# 新名称优先；旧名称兼容未经过浏览器迁移的 API 工作流。
|
||
element_ids = []
|
||
for i in range(1, _MM_IMAGE_LIMIT + 1):
|
||
current_id = _first(kwargs.get(f"图片素材ID{i}"), "").strip()
|
||
legacy_id = _first(kwargs.get(f"真人素材ID{i}"), "").strip()
|
||
element_ids.append(current_id or legacy_id)
|
||
video_ids = [_first(kwargs.get(f"视频素材ID{i}"), "").strip() for i in range(1, _MM_VIDEO_LIMIT + 1)]
|
||
audio_ids = [_first(kwargs.get(f"音频素材ID{i}"), "").strip() for i in range(1, _MM_AUDIO_LIMIT + 1)]
|
||
|
||
element_ids = [eid for eid in element_ids if eid]
|
||
video_ids = [vid for vid in video_ids if vid]
|
||
audio_ids = [aid for aid in audio_ids if aid]
|
||
|
||
# ── 校验 ──────────────────────────────────────────────────────────
|
||
has_image = bool(ref_images)
|
||
has_video = len(ref_videos) > 0
|
||
has_audio = len(ref_audios) > 0
|
||
has_element = len(element_ids) > 0
|
||
has_video_id = len(video_ids) > 0
|
||
has_audio_id = len(audio_ids) > 0
|
||
|
||
if not has_image and not has_video and not has_audio and not has_element and not has_video_id and not has_audio_id and not prompt:
|
||
raise ValueError("至少需要提供参考图片、参考视频、图片素材ID、视频素材ID或提示词之一。")
|
||
if base_model != "seedance 2.5" and (has_audio or has_audio_id) and not has_image and not has_video and not has_element and not has_video_id:
|
||
raise ValueError("不可单独输入音频，请至少连接一张参考图片、一个参考视频或提供图片/视频素材ID。")
|
||
# 国内线路仅替换实际模型 ID，能力限制与对应主模型保持一致。
|
||
SeedanceAutoPass._validate_dynamic_parameters(
|
||
base_model,
|
||
route,
|
||
duration_str,
|
||
[*ref_images, *element_ids],
|
||
[*ref_videos, *video_ids],
|
||
[*ref_audios, *audio_ids],
|
||
)
|
||
_check_fast_resolution(model, resolution)
|
||
|
||
# 判断是否使用新格式
|
||
use_new_format = _is_new_format_model(model)
|
||
|
||
# ── 构建 content 列表 ─────────────────────────────────────────────
|
||
content = []
|
||
base_url = get_base_url_by_route()
|
||
|
||
# 真人素材 ID（与参考图片共用当前模型的图片额度，优先级最高）
|
||
if has_element:
|
||
for idx, eid in enumerate(element_ids, start=1):
|
||
# 确保 element_id 格式正确
|
||
asset_url = eid if eid.startswith("asset://") else f"asset://{eid}"
|
||
# 第一个素材ID作为主体（subject），其余作为参考图片（reference_image）
|
||
if idx == 1:
|
||
role = "subject" if use_new_format else "reference_image"
|
||
else:
|
||
role = "reference_image"
|
||
content.append({
|
||
"type": "image_url",
|
||
"image_url": {"url": asset_url},
|
||
"role": role,
|
||
})
|
||
role_label = "主体" if role == "subject" else "参考"
|
||
print(f"[SeedanceMultiModal] 使用图片素材ID{idx}（{role_label}） → {asset_url}")
|
||
|
||
# 参考图片（2.5 最多 30 个独立槽位，用户自行选择连接哪几个）
|
||
if has_image:
|
||
for idx, img_tensor in enumerate(ref_images, start=1):
|
||
# 每个 tensor 可能是 [1,H,W,C] 或 [H,W,C]，统一确保有 batch 维
|
||
if img_tensor.dim() == 3:
|
||
img_tensor = img_tensor.unsqueeze(0)
|
||
url = await _tensor_to_uploaded_url(img_tensor, base_url, f"参考图片{idx}")
|
||
content.append({
|
||
"type": "image_url",
|
||
"image_url": {"url": url},
|
||
"role": "reference_image",
|
||
})
|
||
|
||
# 参考视频（2.5 最多 10 个）
|
||
for v in ref_videos:
|
||
url = await upload_video(v, base_url=base_url)
|
||
content.append({
|
||
"type": "video_url",
|
||
"video_url": {"url": url},
|
||
"role": "reference_video",
|
||
})
|
||
|
||
# 视频素材 ID（与参考视频共用当前模型的视频额度）
|
||
for idx, vid in enumerate(video_ids, start=1):
|
||
asset_url = vid if vid.startswith("asset://") else f"asset://{vid}"
|
||
content.append({
|
||
"type": "video_url",
|
||
"video_url": {"url": asset_url},
|
||
"role": "reference_video",
|
||
})
|
||
print(f"[SeedanceMultiModal] 使用视频素材ID{idx} → {asset_url}")
|
||
|
||
# 参考音频（2.5 最多 10 段）
|
||
for a in ref_audios:
|
||
url = await upload_audio(a, base_url=base_url)
|
||
content.append({
|
||
"type": "audio_url",
|
||
"audio_url": {"url": url},
|
||
"role": "reference_audio",
|
||
})
|
||
|
||
# 音频素材 ID（与参考音频共用当前模型的音频额度）
|
||
for idx, aid in enumerate(audio_ids, start=1):
|
||
asset_url = aid if aid.startswith("asset://") else f"asset://{aid}"
|
||
content.append({
|
||
"type": "audio_url",
|
||
"audio_url": {"url": asset_url},
|
||
"role": "reference_audio",
|
||
})
|
||
print(f"[SeedanceMultiModal] 使用音频素材ID{idx} → {asset_url}")
|
||
|
||
# 文本提示词（放最后）
|
||
if prompt:
|
||
content.append({"type": "text", "text": prompt})
|
||
|
||
if not content:
|
||
raise ValueError("content 为空，请至少提供参考图片、参考视频或提示词。")
|
||
|
||
# ── 构建请求体 ──────────────────────────────────────────────────────
|
||
if use_new_format:
|
||
# 新格式：顶层 content
|
||
# 注意：文本提示词应该放在最前面
|
||
ordered_content = []
|
||
|
||
# 先添加文本
|
||
text_items = [item for item in content if item.get("type") == "text"]
|
||
ordered_content.extend(text_items)
|
||
|
||
# 再添加其他内容（图片、视频、音频）
|
||
non_text_items = [item for item in content if item.get("type") != "text"]
|
||
ordered_content.extend(non_text_items)
|
||
|
||
body = {
|
||
"model": model,
|
||
"content": ordered_content,
|
||
"duration": duration if duration != -1 else 5,
|
||
"resolution": resolution,
|
||
"ratio": ratio if ratio not in ("智能", "adaptive") else "16:9", # adaptive 为旧工作流兼容
|
||
"generate_audio": gen_audio,
|
||
"watermark": False,
|
||
"return_last_frame": return_last,
|
||
}
|
||
if seed != 0:
|
||
body["seed"] = seed
|
||
|
||
print(f"[SeedanceMultiModal] 新格式请求体: model={model}, content_len={len(ordered_content)}, duration={body['duration']}, resolution={body['resolution']}")
|
||
else:
|
||
# 老格式：metadata.content
|
||
metadata: dict = {
|
||
"resolution": resolution,
|
||
"watermark": False,
|
||
"content": content,
|
||
}
|
||
|
||
if ratio not in ("智能", "adaptive"): # adaptive 为旧工作流兼容
|
||
metadata["ratio"] = ratio
|
||
if duration != -1:
|
||
metadata["duration"] = duration
|
||
if gen_audio:
|
||
metadata["generate_audio"] = True
|
||
if return_last:
|
||
metadata["return_last_frame"] = True
|
||
if seed != 0:
|
||
metadata["seed"] = seed
|
||
if web_search:
|
||
metadata["tools"] = [{"type": "web_search"}]
|
||
|
||
# 顶层 image：取第一张图的 URL（优先真人素材，其次参考图的上传 URL）
|
||
first_image_url = next(
|
||
(item["image_url"]["url"] for item in content if item["type"] == "image_url"),
|
||
None,
|
||
)
|
||
|
||
body = {
|
||
"model": model,
|
||
"prompt": prompt if prompt else " ",
|
||
"metadata": metadata,
|
||
}
|
||
if first_image_url:
|
||
body["image"] = first_image_url
|
||
|
||
_validate_request_body_size(body, "Seedance多模态")
|
||
_log_original_request_body(body)
|
||
|
||
# ── 保存路径（临时文件，避免与下游保存节点重复落盘）──────────────────
|
||
_, save_path = tempfile.mkstemp(suffix=".mp4", prefix="seedance_mm_")
|
||
|
||
client = SeedanceClient()
|
||
client.base_url = base_url
|
||
pbar = _make_pbar()
|
||
on_stage, on_prog = _make_callbacks("Seedance多模态", pbar)
|
||
|
||
try:
|
||
try:
|
||
result_path, last_frame_url = await client.generate_async(
|
||
body=body, save_path=save_path,
|
||
on_stage=on_stage, on_progress=on_prog,
|
||
use_new_format=use_new_format,
|
||
)
|
||
except Exception as exc:
|
||
message = format_seedance_generation_error(exc)
|
||
if message == str(exc):
|
||
raise
|
||
raise RuntimeError(message) from None
|
||
last_frame_tensor = None
|
||
if return_last and last_frame_url:
|
||
last_frame_tensor = await _url_to_tensor(last_frame_url)
|
||
return io.NodeOutput(InputImpl.VideoFromFile(result_path), last_frame_tensor)
|
||
finally:
|
||
_show_balance()
|
||
|
||
|
||
# ── 节点注册 ──────────────────────────────────────────────────────────────────
|
||
|
||
NODE_CLASS_MAPPINGS = {
|
||
"SeedanceMultiModal": SeedanceMultiModal,
|
||
}
|
||
|
||
NODE_DISPLAY_NAME_MAPPINGS = {
|
||
"SeedanceMultiModal": "Seedance 多模态参考生视频",
|
||
}
|