diff --git a/src/routes/other/testAI.ts b/src/routes/other/testAI.ts index b64db1d..4becf64 100644 --- a/src/routes/other/testAI.ts +++ b/src/routes/other/testAI.ts @@ -51,10 +51,9 @@ export default router.post( console.log("%c Line:52 🍐 reply", "background:#ffdd4d", reply); res.status(200).send(success(reply)); } catch (err) { - console.log(err); - if (typeof err === "string") return res.status(500).send(error(err)); - const msg = err instanceof Error ? err.message : (err as any)?.error?.message; - return res.status(500).send(error(msg || "未知错误")); + const msg = u.error(err).message; + console.error(msg); + res.status(500).send(error(msg)); } }, ); diff --git a/src/routes/other/testImage.ts b/src/routes/other/testImage.ts index 4d42b2b..0ee30cf 100644 --- a/src/routes/other/testImage.ts +++ b/src/routes/other/testImage.ts @@ -18,14 +18,17 @@ export default router.post( const { modelName, apiKey, baseURL, manufacturer } = req.body; try { const image = await u.ai.image({ - prompt: "生成16:9 四宫格图片,第一宫格是一只猫,第二宫格是一只狗, 第三宫格是一只老虎,第四宫格是猪。保证宫格图片标准等分", + prompt: + "一张16:9比例的图片,完美等分为2x2四宫格布局,各区域无缝衔接:\n左上宫格:一只可爱的猫,毛发蓬松,眼睛明亮,姿态俏皮\n右上宫格:一只友善的狗,金毛犬,表情愉悦,摇着尾巴\n左下宫格:一头健壮的牛,田园背景,目光温和,皮毛光泽\n右下宫格:一匹骏马,姿态优雅,鬃毛飘逸,肌肉健美\n风格要求:四个宫格风格统一,色彩鲜艳饱和,高清画质,细节清晰锐利,专业插画风格,线条干净,统一的左上方光源,柔和阴影,和谐配色,卡通/半写实风格,宫格间用白色或浅灰细线分隔", imageBase64: [], aspectRatio: "16:9", size: "1K", }); res.status(200).send(success(image)); - } catch (e: any) { - return res.status(500).send(error(e?.response?.data ?? e?.message ?? "生成失败")); + } catch (err) { + const msg = u.error(err).message; + console.error(msg); + res.status(500).send(error(msg)); } // try { diff --git a/src/routes/other/testVideo.ts b/src/routes/other/testVideo.ts index c9a3ab0..e699541 100644 --- a/src/routes/other/testVideo.ts +++ b/src/routes/other/testVideo.ts @@ -20,20 +20,21 @@ export default router.post( async (req, res) => { const { modelName, apiKey, baseURL, manufacturer } = req.body; try { - const videoPath = await u.ai.generateVideo( - { - imageBase64: [], - savePath: "", - prompt: "stickman Dances", - duration: 10 as any, - aspectRatio: "16:9" as any, - }, - manufacturer, - ); + const videoPath = await u.ai.video({ + imageBase64: [], + savePath: "test.mp4", + prompt: "stickman Dances", + duration: 4, + resolution: "480p", + aspectRatio: "16:9", + audio: false, + }); const url = await u.oss.getFileUrl(videoPath); res.status(200).send(success(url)); } catch (err: any) { - res.status(500).send(error(err.error.message || "模型调用失败")); + const msg = u.error(err).message; + console.error(msg); + res.status(500).send(error(msg)); } }, ); diff --git a/src/utils.ts b/src/utils.ts index 4022fd3..a08d48b 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -11,6 +11,7 @@ import * as imageTools from "@/utils/imageTools"; import AIText from "@/utils/ai/text/index"; import AIImage from "@/utils/ai/image/index"; +import AIVideo from "@/utils/ai/video/index"; export default { db, @@ -18,6 +19,7 @@ export default { ai: { text: AIText, image: AIImage, + video: AIVideo, }, editImage, number2Chinese, diff --git a/src/utils/ai/image/index.ts b/src/utils/ai/image/index.ts index 4eac7ca..6c10375 100644 --- a/src/utils/ai/image/index.ts +++ b/src/utils/ai/image/index.ts @@ -5,14 +5,9 @@ import axios from "axios"; import volcengine from "./owned/volcengine"; import kling from "./owned/kling"; -import gemini from "./owned/gemini"; import vidu from "./owned/vidu"; import runninghub from "./owned/runninghub"; -interface AIConfig { - model?: string; - apiKey?: string; - baseURL?: string; -} +import gemini from "./owned/gemini"; const urlToBase64 = async (url: string): Promise => { const res = await axios.get(url, { responseType: "arraybuffer" }); @@ -39,8 +34,7 @@ export default async (input: ImageConfig, config?: AIConfig) => { if (!owned) throw new Error("不支持的模型"); let imageUrl = await manufacturerFn(input, { model, apiKey, baseURL }); - console.log("%c Line:41 🍅 imageUrl", "background:#ed9ec7", imageUrl); if (!input.resType) input.resType = "b64"; if (input.resType === "b64" && imageUrl.startsWith("http")) imageUrl = await urlToBase64(imageUrl); - return imageUrl; + return input; }; diff --git a/src/utils/ai/utils.ts b/src/utils/ai/utils.ts index 9bcc2a9..8e38e1c 100644 --- a/src/utils/ai/utils.ts +++ b/src/utils/ai/utils.ts @@ -1,3 +1,66 @@ +import modelList from "./video/modelList"; + +interface ValidateResult { + owned: (typeof modelList)[number]; + images: string[]; + hasStartEndType: boolean; +} + +/** + * 校验视频生成配置与模型是否匹配 + * @param input 视频配置 + * @param config AI配置 + * @param customOwned 自定义模型配置(如果传入则跳过模型查找) + */ +export const validateVideoConfig = (input: VideoConfig, config: AIConfig, customOwned?: (typeof modelList)[number]): ValidateResult => { + if (!config.model) throw new Error("缺少Model名称"); + + const owned = customOwned ?? modelList.find((m) => m.model === config.model); + if (!owned) throw new Error(`不支持的模型: ${config.model}`); + + const images = input.imageBase64 ?? []; + + // 校验图片数量与模型类型是否匹配 + const hasTextType = owned.type.includes("text"); + const hasSingleImageType = owned.type.includes("singleImage"); + const hasStartEndType = owned.type.some((t) => ["startEndRequired", "endFrameOptional", "startFrameOptional"].includes(t)); + const hasMultiImageType = owned.type.includes("multiImage"); + const hasReferenceType = owned.type.includes("reference"); + + if (images.length === 0 && !hasTextType) { + throw new Error(`模型 ${config.model} 不支持纯文本生成,需要提供图片`); + } + if (images.length === 1 && !hasSingleImageType && !hasStartEndType && !hasReferenceType) { + throw new Error(`模型 ${config.model} 不支持单图模式`); + } + if (images.length === 2 && !hasStartEndType) { + throw new Error(`模型 ${config.model} 不支持首尾帧模式`); + } + if (images.length > 2 && !hasMultiImageType) { + throw new Error(`模型 ${config.model} 不支持多图模式`); + } + + // 校验duration和resolution是否在支持范围内 + const validDurationResolution = owned.durationResolutionMap.some( + (map) => map.duration.includes(input.duration) && map.resolution.includes(input.resolution), + ); + if (!validDurationResolution) { + const supportedDurations = [...new Set(owned.durationResolutionMap.flatMap((m) => m.duration))].sort((a, b) => a - b); + const supportedResolutions = [...new Set(owned.durationResolutionMap.flatMap((m) => m.resolution))]; + throw new Error( + `不支持的duration(${input.duration})或resolution(${input.resolution})组合。` + + `支持的duration: ${supportedDurations.join(", ")},支持的resolution: ${supportedResolutions.join(", ")}`, + ); + } + + // 校验音频设置 + if (input.audio && !owned.audio) { + throw new Error(`模型 ${config.model} 不支持生成音频`); + } + + return { owned, images, hasStartEndType }; +}; + export const pollTask = async ( queryFn: () => Promise<{ completed: boolean; imageUrl?: string; error?: string }>, maxAttempts = 500, @@ -10,4 +73,4 @@ export const pollTask = async ( if (completed && imageUrl) return imageUrl; } throw new Error(`任务轮询超时,已尝试 ${maxAttempts} 次`); -}; \ No newline at end of file +}; diff --git a/src/utils/ai/video/index.ts b/src/utils/ai/video/index.ts new file mode 100644 index 0000000..74628b4 --- /dev/null +++ b/src/utils/ai/video/index.ts @@ -0,0 +1,57 @@ +import "./type"; +import u from "@/utils"; +import modelList from "./modelList"; +import axios from "axios"; + +import volcengine from "./owned/volcengine"; +import kling from "./owned/kling"; +import vidu from "./owned/vidu"; + +const modelInstance = { + volcengine: volcengine, + kling: kling, + vidu: vidu, + runninghub: null, + apimart: null, +} as const; + +export default async (input: VideoConfig, config?: AIConfig) => { + const sqlTextModelConfig = await u.getConfig("video"); + const { model, apiKey, baseURL, manufacturer } = { ...sqlTextModelConfig, ...config }; + const manufacturerFn = modelInstance[manufacturer as keyof typeof modelInstance]; + if (!manufacturerFn) if (!manufacturerFn) throw new Error("不支持的视频厂商"); + const owned = modelList.find((m) => m.model === model); + if (!owned) throw new Error("不支持的模型"); + + // 补充图片的 base64 内容类型字符串 + if (input.imageBase64 && input.imageBase64.length > 0) { + input.imageBase64 = input.imageBase64.map((img) => { + if (img.startsWith("data:image/")) { + return img; + } + // 根据 base64 头部判断图片类型 + if (img.startsWith("/9j/")) { + return `data:image/jpeg;base64,${img}`; + } + if (img.startsWith("iVBORw")) { + return `data:image/png;base64,${img}`; + } + if (img.startsWith("R0lGOD")) { + return `data:image/gif;base64,${img}`; + } + if (img.startsWith("UklGR")) { + return `data:image/webp;base64,${img}`; + } + // 默认使用 png + return `data:image/png;base64,${img}`; + }); + } + + let videoUrl = await manufacturerFn(input, { model, apiKey, baseURL }); + if (videoUrl) { + const response = await axios.get(videoUrl, { responseType: "stream" }); + await u.oss.writeFile(input.savePath, response.data); + return input.savePath; + } + return videoUrl; +}; diff --git a/src/utils/ai/video/modelList.ts b/src/utils/ai/video/modelList.ts new file mode 100644 index 0000000..5d4a8dc --- /dev/null +++ b/src/utils/ai/video/modelList.ts @@ -0,0 +1,294 @@ +type VideoGenerationType = + | "singleImage" // 单图 + | "startEndRequired" // 首尾帧(两张都得有) + | "endFrameOptional" // 首尾帧(尾帧可选) + | "startFrameOptional" // 首尾帧(首帧可选) + | "multiImage" // 多图模式 + | "reference" // 参考图模式 + | "text"; // 文本生视频 + +interface DurationResolutionMap { + duration: number[]; + resolution: `${number}p`[]; +} +interface Owned { + manufacturer: string; + model: string; + durationResolutionMap: DurationResolutionMap[]; + aspectRatio: `${number}:${number}`[]; + type: VideoGenerationType[]; + audio: boolean; +} + +const modelList: Owned[] = [ + // ================== 火山引擎/豆包系列 ================== + // doubao-seedance-1-5-pro 文生视频 + { + manufacturer: "volcengine", + model: "doubao-seedance-1-5-pro-251215", + durationResolutionMap: [{ duration: [4, 5, 6, 7, 8, 9, 10, 11, 12], resolution: ["480p", "720p", "1080p"] }], + aspectRatio: ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9"], + type: ["text"], + audio: true, + }, + // doubao-seedance-1-5-pro 图生视频 + { + manufacturer: "volcengine", + model: "doubao-seedance-1-5-pro-251215", + durationResolutionMap: [{ duration: [4, 5, 6, 7, 8, 9, 10, 11, 12], resolution: ["480p", "720p", "1080p"] }], + aspectRatio: [], + type: ["endFrameOptional"], + audio: true, + }, + // doubao-seedance-1-0-pro 文生视频 + { + manufacturer: "volcengine", + model: "doubao-seedance-1-0-pro-250528", + durationResolutionMap: [{ duration: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], resolution: ["480p", "720p", "1080p"] }], + aspectRatio: ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9"], + type: ["text"], + audio: false, + }, + // doubao-seedance-1-0-pro 图生视频 + { + manufacturer: "volcengine", + model: "doubao-seedance-1-0-pro-250528", + durationResolutionMap: [{ duration: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], resolution: ["480p", "720p", "1080p"] }], + aspectRatio: [], + type: ["endFrameOptional"], + audio: false, + }, + // doubao-seedance-1-0-pro-fast 文生视频 + { + manufacturer: "volcengine", + model: "doubao-seedance-1-0-pro-fast-251015", + durationResolutionMap: [{ duration: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], resolution: ["480p", "720p", "1080p"] }], + aspectRatio: ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9"], + type: ["text"], + audio: false, + }, + // doubao-seedance-1-0-pro-fast 图生视频 + { + manufacturer: "volcengine", + model: "doubao-seedance-1-0-pro-fast-251015", + durationResolutionMap: [{ duration: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], resolution: ["480p", "720p", "1080p"] }], + aspectRatio: [], + type: ["singleImage"], + audio: false, + }, + // doubao-seedance-1-0-lite-i2v 图生视频(仅支持图片模式) + { + manufacturer: "volcengine", + model: "doubao-seedance-1-0-lite-i2v-250428", + durationResolutionMap: [{ duration: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], resolution: ["480p", "720p", "1080p"] }], + aspectRatio: [], + type: ["endFrameOptional", "reference"], + audio: false, + }, + // doubao-seedance-1-0-lite-t2v 文生视频(仅支持文本模式) + { + manufacturer: "volcengine", + model: "doubao-seedance-1-0-lite-t2v-250428", + durationResolutionMap: [{ duration: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], resolution: ["480p", "720p", "1080p"] }], + aspectRatio: ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9"], + type: ["text"], + audio: false, + }, + // ================== 可灵系列 ================== + // kling-v1(STD) 文生视频 + { + manufacturer: "kling", + model: "kling-v1(STD)", + durationResolutionMap: [{ duration: [5, 10], resolution: ["720p"] }], + aspectRatio: ["16:9", "1:1", "9:16"], + type: ["text"], + audio: false, + }, + // kling-v1(STD) 图生视频 + { + manufacturer: "kling", + model: "kling-v1(STD)", + durationResolutionMap: [{ duration: [5, 10], resolution: ["720p"] }], + aspectRatio: [], + type: ["startEndRequired"], + audio: false, + }, + // kling-v1(PRO) 文生视频 + { + manufacturer: "kling", + model: "kling-v1(PRO)", + durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }], + aspectRatio: ["16:9", "1:1", "9:16"], + type: ["text"], + audio: false, + }, + // kling-v1(PRO) 图生视频 + { + manufacturer: "kling", + model: "kling-v1(PRO)", + durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }], + aspectRatio: [], + type: ["startEndRequired"], + audio: false, + }, + // kling-v1-6(PRO) 文生视频 + { + manufacturer: "kling", + model: "kling-v1-6(PRO)", + durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }], + aspectRatio: ["16:9", "1:1", "9:16"], + type: ["text"], + audio: false, + }, + // kling-v1-6(PRO) 图生视频 + { + manufacturer: "kling", + model: "kling-v1-6(PRO)", + durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }], + aspectRatio: [], + type: ["startEndRequired"], + audio: false, + }, + // kling-v2-5-turbo(PRO) 文生视频 + { + manufacturer: "kling", + model: "kling-v2-5-turbo(PRO)", + durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }], + aspectRatio: ["16:9", "1:1", "9:16"], + type: ["text"], + audio: false, + }, + // kling-v2-5-turbo(PRO) 图生视频 + { + manufacturer: "kling", + model: "kling-v2-5-turbo(PRO)", + durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }], + aspectRatio: [], + type: ["startEndRequired"], + audio: false, + }, + // kling-v2-6(PRO) 文生视频 + { + manufacturer: "kling", + model: "kling-v2-6(PRO)", + durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }], + aspectRatio: ["16:9", "1:1", "9:16"], + type: ["text"], + audio: false, + }, + // kling-v2-6(PRO) 图生视频 + { + manufacturer: "kling", + model: "kling-v2-6(PRO)", + durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }], + aspectRatio: [], + type: ["startEndRequired"], + audio: false, + }, + // ================== ViduQ3系列 ================== + // viduq3-pro 文生视频 + { + manufacturer: "vidu", + model: "viduq3-pro", + durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], resolution: ["540p", "720p", "1080p"] }], + aspectRatio: ["16:9", "9:16", "3:4", "4:3", "1:1"], + type: ["text"], + audio: true, + }, + // viduq3-pro 图生视频 + { + manufacturer: "vidu", + model: "viduq3-pro", + durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], resolution: ["540p", "720p", "1080p"] }], + aspectRatio: [], + type: ["singleImage"], + audio: true, + }, + // viduq2-pro-fast 图生视频 + { + manufacturer: "vidu", + model: "viduq2-pro-fast", + durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], resolution: ["720p", "1080p"] }], + aspectRatio: [], + type: ["singleImage", "startEndRequired"], + audio: false, + }, + // viduq2-pro 文生视频 + { + manufacturer: "vidu", + model: "viduq2-pro", + durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], resolution: ["540p", "720p", "1080p"] }], + aspectRatio: ["16:9", "9:16", "3:4", "4:3", "1:1"], + type: ["text"], + audio: false, + }, + // viduq2-pro 图生视频 + { + manufacturer: "vidu", + model: "viduq2-pro", + durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], resolution: ["540p", "720p", "1080p"] }], + aspectRatio: [], + type: ["singleImage", "reference", "startEndRequired"], + audio: false, + }, + // viduq2-turbo 文生视频 + { + manufacturer: "vidu", + model: "viduq2-turbo", + durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], resolution: ["540p", "720p", "1080p"] }], + aspectRatio: ["16:9", "9:16", "3:4", "4:3", "1:1"], + type: ["text"], + audio: false, + }, + // viduq2-turbo 图生视频 + { + manufacturer: "vidu", + model: "viduq2-turbo", + durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], resolution: ["540p", "720p", "1080p"] }], + aspectRatio: [], + type: ["singleImage", "reference", "startEndRequired"], + audio: false, + }, + // viduq1 文生视频 + { + manufacturer: "vidu", + model: "viduq1", + durationResolutionMap: [{ duration: [5], resolution: ["1080p"] }], + aspectRatio: ["16:9", "9:16", "1:1"], + type: ["text"], + audio: false, + }, + // viduq1 图生视频 + { + manufacturer: "vidu", + model: "viduq1", + durationResolutionMap: [{ duration: [5], resolution: ["1080p"] }], + aspectRatio: [], + type: ["singleImage", "reference", "startEndRequired"], + audio: false, + }, + // viduq1-classic 图生视频 + { + manufacturer: "vidu", + model: "viduq1-classic", + durationResolutionMap: [{ duration: [5], resolution: ["1080p"] }], + aspectRatio: [], + type: ["singleImage", "startEndRequired"], + audio: false, + }, + // vidu2.0 图生视频 + { + manufacturer: "vidu", + model: "vidu2.0", + durationResolutionMap: [ + { duration: [4], resolution: ["360p", "720p", "1080p"] }, + { duration: [8], resolution: ["720p"] }, + ], + aspectRatio: [], + type: ["singleImage", "reference", "startEndRequired"], + audio: false, + }, + // ================== sora系列 ================== +]; + +export default modelList; diff --git a/src/utils/ai/video/owned/kling.ts b/src/utils/ai/video/owned/kling.ts new file mode 100644 index 0000000..26e3a93 --- /dev/null +++ b/src/utils/ai/video/owned/kling.ts @@ -0,0 +1,93 @@ +import "../type"; +import axios from "axios"; +import { pollTask, validateVideoConfig } from "@/utils/ai/utils"; + +export default async (input: VideoConfig, config: AIConfig) => { + if (!config.apiKey) throw new Error("缺少API Key"); + if (!config.baseURL) throw new Error("缺少baseURL配置"); + + const { images } = validateVideoConfig(input, config); + + // 解析URL配置:图生视频|文生视频|查询地址 + const baseUrl = "https://api-beijing.klingai.com"; + const [ + image2videoUrl = baseUrl + "/v1/videos/image2video", + text2videoUrl = baseUrl + "/v1/videos/text2video", + queryUrl = baseUrl + "/v1/videos/text2video/{id}", + ] = config.baseURL.split("|"); + + const headers = { + Authorization: `Bearer ${config.apiKey}`, + "Content-Type": "application/json", + }; + + // 解析模型名称和模式,例如 "kling-v2-6(PRO)" => modelName: "kling-v2-6", mode: "pro" + const modelMatch = config.model!.match(/^(.+)\((STD|PRO)\)$/i); + const modelName = modelMatch ? modelMatch[1] : config.model; + const mode = modelMatch ? (modelMatch[2].toLowerCase() as "std" | "pro") : "std"; + + // 判断是图生视频还是文生视频 + const hasImage = images.length > 0; + const createUrl = hasImage ? image2videoUrl : text2videoUrl; + + // 去除图片的内容类型前缀(kling要求纯base64) + const stripDataUrl = (str: string) => str.replace(/^data:image\/[^;]+;base64,/, ""); + + // 构建请求体 + const body: Record = { + model_name: modelName, + mode, + duration: String(input.duration), + prompt: input.prompt, + aspect_ratio: input.aspectRatio, + }; + + if (hasImage) { + // 图生视频:首帧和尾帧 + body.image = stripDataUrl(images[0]); + if (images.length > 1) { + body.image_tail = stripDataUrl(images[1]); + } + } + + // 创建任务 + const createResponse = await axios.post(createUrl, body, { headers }); + const createData = createResponse.data; + if (createData.code !== 0) { + throw new Error(`创建任务失败: ${createData.message || "未知错误"}`); + } + + const taskId = createData.data?.task_id; + if (!taskId) { + throw new Error("创建任务失败: 未返回任务ID"); + } + + // 轮询任务状态 + return await pollTask(async () => { + const queryResponse = await axios.get(`${queryUrl.replace("{id}", taskId)}`, { headers }); + const queryData = queryResponse.data; + if (queryData.code !== 0) { + return { completed: false, error: `查询失败: ${queryData.message || "未知错误"}` }; + } + + const task = queryData.data; + const taskStatus = task?.task_status; + + switch (taskStatus) { + case "succeed": { + const videoUrl = task?.task_result?.videos?.[0]?.url; + if (!videoUrl) { + return { completed: false, error: "任务成功但未返回视频URL" }; + } + return { completed: true, imageUrl: videoUrl }; + } + case "failed": + return { completed: false, error: `任务失败: ${task?.task_status_msg || "未知原因"}` }; + case "submitted": + case "processing": + return { completed: false }; + default: + return { completed: false, error: `未知状态: ${taskStatus}` }; + } + }); +}; diff --git a/src/utils/ai/video/owned/vidu.ts b/src/utils/ai/video/owned/vidu.ts new file mode 100644 index 0000000..0142fc3 --- /dev/null +++ b/src/utils/ai/video/owned/vidu.ts @@ -0,0 +1,128 @@ +import "../type"; +import axios from "axios"; +import { pollTask, validateVideoConfig } from "@/utils/ai/utils"; +import modelList from "../modelList"; + +export default async (input: VideoConfig, config: AIConfig) => { + if (!config.model) throw new Error("缺少Model名称"); + if (!config.apiKey) throw new Error("缺少API Key"); + if (!input.prompt && (!input.imageBase64 || input.imageBase64.length === 0)) { + throw new Error("至少需要提供prompt或图片"); + } + + const baseUrl = config.baseURL || "https://api.vidu.cn/ent/v2"; + const authorization = `Token ${config.apiKey}`; + const hasImages = input.imageBase64 && input.imageBase64.length > 0; + + // 根据是否有图片,查找匹配的模型配置 + const customOwned = modelList.find((m) => { + if (m.manufacturer !== "vidu") return false; + if (m.model !== config.model) return false; + if (hasImages) { + return m.type.some((t) => t !== "text"); + } else { + return m.type.includes("text"); + } + }); + + if (!customOwned) { + throw new Error(`未找到匹配的模型配置: ${config.model}`); + } + + // 使用统一校验函数 + const { owned, images } = validateVideoConfig(input, config, customOwned); + + // 判断生成类型 + const genType: "text" | "image" = images.length === 0 ? "text" : "image"; + + // 校验宽高比(仅文生视频需要) + if (genType === "text" && owned.aspectRatio.length > 0 && !owned.aspectRatio.includes(input.aspectRatio as `${number}:${number}`)) { + throw new Error(`模型 ${owned.model} 不支持宽高比 ${input.aspectRatio},支持的宽高比:${owned.aspectRatio.join("、")}`); + } + + // 创建任务 + let taskId: string; + + if (genType === "text") { + // 文生视频 + const requestBody: Record = { + model: owned.model, + prompt: input.prompt, + duration: input.duration, + resolution: input.resolution, + aspect_ratio: input.aspectRatio, + }; + if (owned.audio && input.audio !== undefined) { + requestBody.audio = input.audio; + } + + const response = await axios.post(`${baseUrl}/text2video`, requestBody, { + headers: { + "Content-Type": "application/json", + Authorization: authorization, + }, + }); + taskId = response.data.task_id; + } else { + // 图生视频 + const requestBody: Record = { + model: owned.model, + images: images, + duration: input.duration, + resolution: input.resolution, + }; + if (input.prompt) { + requestBody.prompt = input.prompt; + } + if (owned.audio && input.audio !== undefined) { + requestBody.audio = input.audio; + } + + const response = await axios.post(`${baseUrl}/img2video`, requestBody, { + headers: { + "Content-Type": "application/json", + Authorization: authorization, + }, + }); + taskId = response.data.task_id; + } + + // 轮询任务状态 + return await pollTask(async () => { + const response = await axios.get(`${baseUrl}/tasks`, { + headers: { + "Content-Type": "application/json", + Authorization: authorization, + }, + params: { + task_ids: [taskId], + }, + }); + + const tasks = response.data.tasks; + if (!tasks || tasks.length === 0) { + return { completed: false, error: "任务不存在" }; + } + + const task = tasks[0]; + + switch (task.state) { + case "success": { + const creation = task.creations?.[0]; + return { + completed: true, + videoUrl: creation?.url, + coverUrl: creation?.cover_url, + }; + } + case "failed": + return { completed: false, error: "任务生成失败" }; + case "created": + case "queueing": + case "processing": + return { completed: false }; + default: + return { completed: false, error: `未知状态: ${task.state}` }; + } + }); +}; diff --git a/src/utils/ai/video/owned/volcengine.ts b/src/utils/ai/video/owned/volcengine.ts index 9f9aa1d..dc07fe8 100644 --- a/src/utils/ai/video/owned/volcengine.ts +++ b/src/utils/ai/video/owned/volcengine.ts @@ -1,56 +1,74 @@ import "../type"; import axios from "axios"; -import { pollTask } from "@/utils/ai/utils"; +import { pollTask, validateVideoConfig } from "@/utils/ai/utils"; -interface DoubaoVideoConfig { - prompt: string; - savePath: string; - imageBase64?: string[]; // 单张参考图片 base64 - duration: 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12; // 支持 2~12 秒 - aspectRatio: "16:9" | "9:16" | "1:1" | "4:3" | "3:4" | "21:9" | "adaptive"; - audio?: boolean; -} - -export default async (input: ImageConfig, config: AIConfig) => { - console.log("%c Line:5 🍓 input", "background:#7f2b82", input); - console.log("%c Line:5 🍎 config", "background:#93c0a4", config); - if (!config.model) throw new Error("缺少Model名称"); +export default async (input: VideoConfig, config: AIConfig) => { if (!config.apiKey) throw new Error("缺少API Key"); - const key = "Bearer " + config.apiKey.replaceAll("Bearer ", "").trim(); + const { owned, images, hasStartEndType } = validateVideoConfig(input, config); - const doubaoConfig = config as DoubaoVideoConfig; - const createRes = await axios.post( - config.baseURL ?? "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks", - { - model: "doubao-seedance-1-5-pro-251215", - content: [ - { type: "text", text: input.prompt }, - ...(doubaoConfig.imageBase64 - ? doubaoConfig.imageBase64.map((base64, i) => ({ - type: "image_url", - image_url: { url: base64 }, - role: i === 0 ? "first_frame" : "last_frame", - })) - : []), - ], - generate_audio: doubaoConfig.audio ?? false, - duration: doubaoConfig.duration, - resolution: doubaoConfig.aspectRatio, - watermark: false, + const authorization = "Bearer " + config.apiKey.replace(/^Bearer\s*/i, "").trim(); + const baseUrl = config.baseURL ?? "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks"; + + // 判断是否为首尾帧模式(需要两张图且类型支持首尾帧) + const isStartEndMode = images.length === 2 && hasStartEndType; + + // 构建图片内容 + const imageContent = images.map((base64, index) => { + const item: Record = { + type: "image_url", + image_url: { url: base64 }, + }; + if (isStartEndMode) { + item.role = index === 0 ? "first_frame" : "last_frame"; + } + return item; + }); + + // 构建请求体 + const requestBody: Record = { + model: config.model, + content: [{ type: "text", text: input.prompt }, ...imageContent], + duration: input.duration, + resolution: input.resolution, + watermark: false, + }; + + // 仅当模型支持音频时才添加 generate_audio 字段 + if (owned.audio) { + requestBody.generate_audio = input.audio ?? false; + } + // 创建视频生成任务 + const createResponse = await axios.post(baseUrl, requestBody, { + headers: { + "Content-Type": "application/json", + Authorization: authorization, }, - { headers: { "Content-Type": "application/json", Authorization: key } }, - ); - const taskId = createRes.data.id; + }); + + const taskId = createResponse.data.id; if (!taskId) throw new Error("视频任务创建失败"); + + // 轮询任务状态 return await pollTask(async () => { - const res = await axios.get(`${config.baseURL ?? "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks"}/${taskId}`, { - headers: { Authorization: key }, - }); - const { status, content } = res.data; - if (status === "succeeded") return { completed: true, imageUrl: content?.video_url }; - if (["failed", "cancelled", "expired"].includes(status)) return { completed: false, error: `任务${status}` }; - if (["queued", "running"].includes(status)) return { completed: false }; - return { completed: false, error: `未知状态: ${status}` }; + const { status, content } = ( + await axios.get(`${baseUrl}/${taskId}`, { + headers: { Authorization: authorization }, + }) + ).data; + + switch (status) { + case "succeeded": + return { completed: true, imageUrl: content?.video_url }; + case "failed": + case "cancelled": + case "expired": + return { completed: false, error: `任务${status}` }; + case "queued": + case "running": + return { completed: false }; + default: + return { completed: false, error: `未知状态: ${status}` }; + } }); }; diff --git a/src/utils/ai/video/type.ts b/src/utils/ai/video/type.ts new file mode 100644 index 0000000..0706d2d --- /dev/null +++ b/src/utils/ai/video/type.ts @@ -0,0 +1,15 @@ +interface VideoConfig { + duration: 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12; + resolution: "480p" | "720p" | "1080p"; + aspectRatio: "16:9" | "9:16"; + prompt: string; + savePath: string; + imageBase64?: string[]; + audio?: boolean; +} + +interface AIConfig { + model?: string; + apiKey?: string; + baseURL?: string; +}