diff --git a/src/routes/other/testAI.ts b/src/routes/other/testAI.ts index b64db1d..4becf64 100644 --- a/src/routes/other/testAI.ts +++ b/src/routes/other/testAI.ts @@ -51,10 +51,9 @@ export default router.post( console.log("%c Line:52 🍐 reply", "background:#ffdd4d", reply); res.status(200).send(success(reply)); } catch (err) { - console.log(err); - if (typeof err === "string") return res.status(500).send(error(err)); - const msg = err instanceof Error ? err.message : (err as any)?.error?.message; - return res.status(500).send(error(msg || "未知错误")); + const msg = u.error(err).message; + console.error(msg); + res.status(500).send(error(msg)); } }, ); diff --git a/src/routes/other/testImage.ts b/src/routes/other/testImage.ts index 0d5bbb6..c4aa78f 100644 --- a/src/routes/other/testImage.ts +++ b/src/routes/other/testImage.ts @@ -18,36 +18,17 @@ export default router.post( const { modelName, apiKey, baseURL, manufacturer } = req.body; try { const image = await u.ai.image({ - prompt: "生成16:9 四宫格图片,第一宫格是一只猫,第二宫格是一只狗, 第三宫格是一只老虎,第四宫格是猪。保证四宫格图片标准四等分", + prompt: + "一张16:9比例的图片,完美等分为2x2四宫格布局,各区域无缝衔接:\n左上宫格:一只可爱的猫,毛发蓬松,眼睛明亮,姿态俏皮\n右上宫格:一只友善的狗,金毛犬,表情愉悦,摇着尾巴\n左下宫格:一头健壮的牛,田园背景,目光温和,皮毛光泽\n右下宫格:一匹骏马,姿态优雅,鬃毛飘逸,肌肉健美\n风格要求:四个宫格风格统一,色彩鲜艳饱和,高清画质,细节清晰锐利,专业插画风格,线条干净,统一的左上方光源,柔和阴影,和谐配色,卡通/半写实风格,宫格间用白色或浅灰细线分隔", imageBase64: [], aspectRatio: "16:9", size: "1K", }); res.status(200).send(success(image)); - } catch (e: any) { - console.log("%c Line:28 🥒 e", "background:#fca650", e); - return res.status(500).send(error(e?.response?.data ?? e?.message ?? "生成失败")); + } catch (err) { + const msg = u.error(err).message; + console.error(msg); + res.status(500).send(error(msg)); } - - // try { - // const contentStr = await u.ai.generateImage( - // { - // prompt: "2D cat", - // imageBase64: [], - // aspectRatio: "16:9", - // size: "1K", - // }, - // { - // model: modelName, - // apiKey, - // baseURL, - // manufacturer, - // }, - // ); - // res.status(200).send(success(contentStr)); - // } catch (err: any) { - // const message = err?.response?.data?.error?.message || err?.error?.message || "模型调用失败"; - // res.status(500).send(error(message)); - // } }, ); diff --git a/src/routes/other/testVideo.ts b/src/routes/other/testVideo.ts index c9a3ab0..79ebc6d 100644 --- a/src/routes/other/testVideo.ts +++ b/src/routes/other/testVideo.ts @@ -20,20 +20,21 @@ export default router.post( async (req, res) => { const { modelName, apiKey, baseURL, manufacturer } = req.body; try { - const videoPath = await u.ai.generateVideo( - { - imageBase64: [], - savePath: "", - prompt: "stickman Dances", - duration: 10 as any, - aspectRatio: "16:9" as any, - }, - manufacturer, - ); + const videoPath = await u.ai.video({ + imageBase64: [], + savePath: "test.mp4", + prompt: "stickman Dances", + duration: 4, + resolution: "720p", + aspectRatio: "16:9", + audio: false, + }); const url = await u.oss.getFileUrl(videoPath); res.status(200).send(success(url)); } catch (err: any) { - res.status(500).send(error(err.error.message || "模型调用失败")); + const msg = u.error(err).message; + console.error(msg); + res.status(500).send(error(msg)); } }, ); diff --git a/src/utils.ts b/src/utils.ts index 4022fd3..a08d48b 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -11,6 +11,7 @@ import * as imageTools from "@/utils/imageTools"; import AIText from "@/utils/ai/text/index"; import AIImage from "@/utils/ai/image/index"; +import AIVideo from "@/utils/ai/video/index"; export default { db, @@ -18,6 +19,7 @@ export default { ai: { text: AIText, image: AIImage, + video: AIVideo, }, editImage, number2Chinese, diff --git a/src/utils/ai.ts b/src/utils/ai.ts deleted file mode 100644 index c72198a..0000000 --- a/src/utils/ai.ts +++ /dev/null @@ -1,561 +0,0 @@ -import axios from "axios"; -import u from "@/utils"; -import FormData from "form-data"; -import axiosRetry from "axios-retry"; -import { OpenAIChatModel, type OpenAIChatModelOptions } from "@aigne/openai"; -import sharp from "sharp"; - -axiosRetry(axios, { retries: 3, retryDelay: () => 200 }); - -export const text = async (config: OpenAIChatModelOptions = {}) => { - const { model, apiKey, baseURL } = await u.getConfig("text"); - return new OpenAIChatModel({ - apiKey: apiKey ?? "", - baseURL: baseURL ?? "", - model: model ?? "gpt-4.1", - modelOptions: { temperature: 0.7 }, - ...config, - }); -}; - -interface ImageConfig { - systemPrompt?: string; - prompt: string; - imageBase64: string[]; - size: "1K" | "2K" | "4K"; - aspectRatio: string; - resType?: "url" | "b64"; -} - -const urlToBase64 = async (url: string): Promise => { - const res = await axios.get(url, { responseType: "arraybuffer" }); - const base64 = Buffer.from(res.data).toString("base64"); - const mimeType = res.headers["content-type"] || "image/png"; - return `data:${mimeType};base64,${base64}`; -}; - -const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); - -const pollTask = async ( - queryFn: () => Promise<{ completed: boolean; imageUrl?: string; error?: string }>, - maxAttempts = 500, - interval = 2000, -): Promise => { - for (let i = 0; i < maxAttempts; i++) { - await sleep(interval); - const { completed, imageUrl, error } = await queryFn(); - if (error) throw new Error(error); - if (completed && imageUrl) return imageUrl; - } - throw new Error(`任务轮询超时,已尝试 ${maxAttempts} 次`); -}; - -// 上传 base64 图片到 runninghub -const uploadBase64ToRunninghub = async (base64Image: string, apiKey: string, baseURL: string): Promise => { - try { - apiKey = apiKey.replace("Bearer ", ""); - // 移除 base64 前缀 - const base64Data = base64Image.replace(/^data:image\/\w+;base64,/, ""); - let buffer = Buffer.from(base64Data, "base64"); - - // 压缩图片到 5MB 以下 - const MAX_SIZE = 5 * 1024 * 1024; // 5MB - if (buffer.length > MAX_SIZE) { - let quality = 90; - - while (buffer.length > MAX_SIZE && quality > 10) { - const compressed = await sharp(buffer).jpeg({ quality, mozjpeg: true }).toBuffer(); - buffer = Buffer.from(compressed); - quality -= 10; - } - - // 如果仍然超过限制,进一步调整尺寸 - if (buffer.length > MAX_SIZE) { - const metadata = await sharp(buffer).metadata(); - const scale = Math.sqrt(MAX_SIZE / buffer.length); - - const resized = await sharp(buffer) - .resize({ - width: Math.floor((metadata.width || 1920) * scale), - height: Math.floor((metadata.height || 1080) * scale), - fit: "inside", - }) - .jpeg({ quality: 80, mozjpeg: true }) - .toBuffer(); - - buffer = Buffer.from(resized); - } - } - - // 创建 FormData - const formData = new FormData(); - formData.append("file", buffer, { - filename: "image.jpg", - contentType: "image/jpeg", - }); - - // 上传图片 - const uploadRes = await axios.post(`https://www.runninghub.cn/openapi/v2/media/upload/binary`, formData, { - headers: { Authorization: `Bearer ${apiKey}` }, - }); - - if (uploadRes.data.code !== 0 || !uploadRes.data.data?.download_url) { - throw new Error(`图片上传失败: ${JSON.stringify(uploadRes.data)}`); - } - - return uploadRes.data.data.download_url; - } catch (error) { - console.error("上传图片时发生错误:", error); - throw error; - } -}; - -const generators = { - volcengine: async (config: ImageConfig, apiKey: string, baseURL: string, model: string) => { - if (config.size == "1K") config.size = "2K"; - apiKey = apiKey.replace("Bearer ", ""); - const body: Record = { - model, - prompt: config.prompt, - size: config.size, - response_format: "url", - sequential_image_generation: "disabled", - stream: false, - watermark: false, - }; - // 图生图:存在图片时添加 image 字段 - if (config.imageBase64) { - body.image = config.imageBase64; - } - const res = await axios.post(`https://ark.cn-beijing.volces.com/api/v3/images/generations`, body, { - headers: { Authorization: `Bearer ${apiKey}` }, - }); - return res.data.data[0].url; - }, - - gemini: async (config: ImageConfig, apiKey: string, baseURL: string, model: string) => { - apiKey = apiKey.replace("Bearer ", ""); - const messages = [ - ...(config.systemPrompt ? [{ role: "system", content: config.systemPrompt }] : []), - { role: "user", content: config.prompt }, - ...config.imageBase64.map((img) => ({ role: "user", content: { image: img } })), - ]; - const res = await axios.post( - `${baseURL}/chat/completions`, - { model, stream: false, messages, extra_body: { google: { image_config: { aspect_ratio: config.aspectRatio, image_size: config.size } } } }, - { headers: { Authorization: "Bearer " + apiKey } }, - ); - - return res.data.choices[0].message.content; - }, - - runninghub: async (config: ImageConfig, apiKey: string, baseURL: string) => { - apiKey = apiKey.replace("Bearer ", ""); - const imageUrls = await Promise.all(config.imageBase64.map((base64Image) => uploadBase64ToRunninghub(base64Image, apiKey, baseURL))); - - const endpoint = config.imageBase64.length === 0 ? "/openapi/v2/rhart-image-n-pro/text-to-image" : "/openapi/v2/rhart-image-n-pro/edit"; - const taskRes = await axios.post( - `https://www.runninghub.cn${endpoint}`, - { prompt: config.prompt, resolution: config.size, aspectRatio: config.aspectRatio, ...(imageUrls.length > 0 && { imageUrls }) }, - { headers: { Authorization: "Bearer " + apiKey } }, - ); - const taskId = taskRes.data.taskId; - if (!taskId) throw new Error(`任务创建失败,${JSON.stringify(taskRes.data)}`); - - return pollTask(async () => { - const res = await axios.post(`https://www.runninghub.cn/task/openapi/outputs`, { taskId, apiKey: apiKey }); - const { code, msg, data } = res.data; - if (code === 0 && msg === "success") return { completed: true, imageUrl: data?.[0]?.fileUrl }; - if (code === 804 || code === 813) return { completed: false }; - if (code === 805) return { completed: false, error: `任务失败: ${data?.[0]?.failedReason?.exception_message || "未知原因"}` }; - return { completed: false, error: `未知状态: code=${code}, msg=${msg}` }; - }); - }, - - apimart: async (config: ImageConfig, apiKey: string, baseURL: string, model: string) => { - apiKey = apiKey.replace("Bearer ", ""); - const taskRes = await axios.post( - `https://api.apimart.ai/v1/images/generations`, - { model: "gemini-3-pro-image-preview", prompt: config.prompt, size: config.aspectRatio, n: 1, resolution: config.size }, - { headers: { Authorization: apiKey } }, - ); - - if (taskRes.data.code !== 200 || !taskRes.data.data?.[0]?.task_id) throw new Error("任务创建失败: " + JSON.stringify(taskRes.data)); - - const taskId = taskRes.data.data[0].task_id; - return pollTask(async () => { - const res = await axios.get(`https://api.apimart.ai/v1/tasks/${taskId}`, { headers: { Authorization: apiKey }, params: { language: "en" } }); - if (res.data.code !== 200) return { completed: false, error: `查询失败: ${JSON.stringify(res.data)}` }; - const { status, result } = res.data.data; - if (status === "completed") return { completed: true, imageUrl: result?.images?.[0]?.url?.[0] }; - if (status === "failed" || status === "cancelled") return { completed: false, error: `任务${status}` }; - return { completed: false }; - }); - }, -}; - -export const generateImage = async (config: ImageConfig, replaceConfig?: Awaited>>): Promise => { - let { model, apiKey, baseURL, manufacturer } = await u.getConfig("image"); - if (replaceConfig) { - model = replaceConfig.model || model; - apiKey = replaceConfig.apiKey || apiKey; - baseURL = replaceConfig.baseURL || baseURL; - manufacturer = replaceConfig.manufacturer || manufacturer; - } - const generator = generators[manufacturer as keyof typeof generators]; - if (!generator) throw new Error(`不支持的厂商: ${manufacturer}`); - - let imageUrl = await generator(config, apiKey ?? "", baseURL ?? "", model ?? ""); - if (!config.resType) config.resType = "b64"; - if (config.resType === "b64" && imageUrl.startsWith("http")) imageUrl = await urlToBase64(imageUrl); - return imageUrl; -}; - -type VideoAspectRatio = "16:9" | "9:16" | "1:1" | "4:3" | "3:4" | "21:9" | "adaptive"; -interface BaseVideoConfig { - prompt: string; - savePath: string; - imageBase64?: string[]; // 单张参考图片 base64 -} -interface DoubaoVideoConfig extends BaseVideoConfig { - duration: 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12; // 支持 2~12 秒 - aspectRatio: VideoAspectRatio; - audio?: boolean; -} -interface RunninghubVideoConfig extends BaseVideoConfig { - duration: 10 | 15; // 仅支持 10 或 15 秒 - aspectRatio: "16:9" | "9:16" | "1:1"; // 仅支持这三种比例 -} -interface OpenAIVideoConfig extends BaseVideoConfig { - duration: 10 | 15; // 仅支持 10 或 15 秒 - aspectRatio: Exclude; // 不支持 adaptive -} -type VideoConfig = DoubaoVideoConfig | RunninghubVideoConfig | OpenAIVideoConfig; -const generateVideoWithConfig = async (config: VideoConfig, configItem: { model: string; apiKey: string; baseURL: string; manufacturer: string }) => { - const { apiKey, baseURL, manufacturer, model } = configItem; - const imageArrPath = []; - for (const imageVal of config?.imageBase64!) { - // 判断是否为base64串 - const isBase64 = typeof imageVal === "string" && /^data:image\/[a-zA-Z0-9\+\-\.]+;base64,[\s\S]+$/.test(imageVal.trim()); - if (isBase64) { - imageArrPath.push(imageVal); - } else { - const base64 = await urlToBase64(imageVal); - imageArrPath.push(base64); - } - } - config.imageBase64 = imageArrPath; - let videoUrl: string | null = null; - if (manufacturer === "volcengine") { - const doubaoConfig = config as DoubaoVideoConfig; - const createRes = await axios.post( - baseURL ?? "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks", - { - model: "doubao-seedance-1-5-pro-251215", - content: [ - { type: "text", text: config.prompt }, - ...(doubaoConfig.imageBase64 - ? doubaoConfig.imageBase64.map((base64, i) => ({ - type: "image_url", - image_url: { url: base64 }, - role: i === 0 ? "first_frame" : "last_frame", - })) - : []), - ], - generate_audio: doubaoConfig.audio ?? false, - duration: doubaoConfig.duration, - resolution: doubaoConfig.aspectRatio, - watermark: false, - }, - { headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` } }, - ); - const taskId = createRes.data.id; - if (!taskId) throw new Error("视频任务创建失败"); - videoUrl = await pollTask(async () => { - const res = await axios.get(`${baseURL ?? "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks"}/${taskId}`, { - headers: { Authorization: `Bearer ${apiKey}` }, - }); - const { status, content } = res.data; - if (status === "succeeded") return { completed: true, imageUrl: content?.video_url }; - if (["failed", "cancelled", "expired"].includes(status)) return { completed: false, error: `任务${status}` }; - if (["queued", "running"].includes(status)) return { completed: false }; - return { completed: false, error: `未知状态: ${status}` }; - }); - } else if (manufacturer === "runninghub") { - const runninghubConfig = config as RunninghubVideoConfig; - // 如果有图片,先上传 - let uploadedImageUrl: string | undefined; - if (runninghubConfig.imageBase64 && runninghubConfig.imageBase64.length > 0) { - uploadedImageUrl = await uploadBase64ToRunninghub(runninghubConfig.imageBase64[0]!, apiKey ?? "", "https://www.runninghub.cn"); - } - - const endpoint = uploadedImageUrl ? "/openapi/v2/rhart-video-s/image-to-video" : "/openapi/v2/rhart-video-s/text-to-video"; - const requestBody = uploadedImageUrl - ? { - prompt: config.prompt, - imageUrl: uploadedImageUrl, - duration: String(runninghubConfig.duration) as "10" | "15", - aspectRatio: runninghubConfig.aspectRatio, - } - : { prompt: config.prompt, model }; - const createRes = await axios.post(`https://www.runninghub.cn${endpoint}`, requestBody, { - headers: { Authorization: "Bearer " + apiKey, "Content-Type": "application/json" }, - }); - - const { taskId, status: initialStatus, errorMessage } = createRes.data; - if (!taskId) throw new Error(`视频任务创建失败: ${errorMessage || "未知错误"}`); - if (initialStatus === "FAILED") throw new Error(`任务创建失败: ${errorMessage}`); - videoUrl = await pollTask(async () => { - const res = await axios.post( - `https://www.runninghub.cn/task/openapi/outputs`, - { apiKey: apiKey?.replace("Bearer ", ""), taskId }, - { headers: { Authorization: "Bearer " + apiKey } }, - ); - const { code, msg, data } = res.data; - - // 成功完成 - if (code === 0 && msg === "success" && data?.[0]?.fileUrl) { - return { completed: true, imageUrl: data[0].fileUrl }; - } - - // 进行中 - if (code === 804 || code === 813) { - return { completed: false }; - } - - // 失败 - if (code === 805) { - const failedReason = data?.[0]?.failedReason; - let errorMsg = "未知原因"; - - if (failedReason) { - // 尝试多种可能的错误信息字段 - errorMsg = - failedReason.exception_message || - failedReason.exceptionMessage || - failedReason.message || - failedReason.reason || - JSON.stringify(failedReason); - } - - return { - completed: false, - error: `任务失败: ${errorMsg}`, - }; - } - - // 其他未知状态 - return { - completed: false, - error: `未知状态: code=${code}, msg=${msg}, data=${JSON.stringify(data)}`, - }; - }); - } else if (manufacturer === "openAi") { - const openaiConfig = config as OpenAIVideoConfig; - // 如果有图片,先上传 - let uploadedImageUrl: string | undefined; - if (openaiConfig.imageBase64 && openaiConfig.imageBase64.length) { - const base64Data = openaiConfig.imageBase64[0]!.replace(/^data:image\/\w+;base64,/, ""); - const buffer = Buffer.from(base64Data, "base64"); - const formData = new FormData(); - formData.append("file", buffer, { filename: "image.jpg", contentType: "image/jpeg" }); - const uploadRes = await axios.post(`${baseURL}/videos`, formData, { - headers: { - Authorization: `Bearer ${apiKey}`, - ...formData.getHeaders(), - }, - }); - uploadedImageUrl = uploadRes.data?.id || uploadRes.data?.url; - } - - // 创建视频生成任务 - const formData = new FormData(); - formData.append("model", model); - formData.append("prompt", config.prompt); - formData.append("seconds", String(openaiConfig.duration)); - - // 根据 aspectRatio 设置 size - const sizeMap: Record = { - "16:9": "1920x1080", - "9:16": "1080x1920", - "1:1": "1080x1080", - "4:3": "1440x1080", - "3:4": "1080x1440", - "21:9": "2560x1080", - }; - formData.append("size", sizeMap[openaiConfig.aspectRatio] || "1920x1080"); - if (uploadedImageUrl) { - formData.append("input_reference", uploadedImageUrl); - } - const createRes = await axios.post(`${baseURL}/videos`, formData, { - headers: { - Authorization: `Bearer ${apiKey}`, - ...formData.getHeaders(), - }, - }); - - const taskId = createRes.data?.id; - - if (!taskId) throw new Error("视频任务创建失败"); - // 轮询任务状态 - videoUrl = await pollTask(async () => { - const res = await axios.get(`${baseURL}/videos/${taskId}`, { - headers: { - Authorization: `Bearer ${apiKey}`, - "Content-Type": "application/json", - }, - }); - const { status, imageUrl, failReason } = res.data; - if (status === "SUCCESS") return { completed: true, imageUrl }; - if (status === "FAILURE" || status === "CANCEL") { - return { completed: false, error: `任务${status}: ${failReason || "未知原因"}` }; - } - if (["NOT_START", "SUBMITTED", "IN_PROGRESS", "MODAL"].includes(status)) { - return { completed: false }; - } - return { completed: false, error: `未知状态: ${status}` }; - }); - } else if (manufacturer === "apimart") { - // apimart 视频生成 - const apimartConfig = config as OpenAIVideoConfig; - const apimartBaseURL = "https://api.apimart.ai"; - - // 上传图片到 apimart 图床 - let imageUrls: string[] = []; - if (apimartConfig.imageBase64 && apimartConfig.imageBase64.length > 0) { - for (const base64Image of apimartConfig.imageBase64) { - // 如果已经是 URL,直接使用 - if (base64Image.startsWith("http")) { - imageUrls.push(base64Image); - continue; - } - - // 获取预签名 URL - const presignRes = await axios.post( - "https://apimart.ai/api/upload/presign", - { contentType: "image/jpeg", fileExtension: "jpeg", permanent: false }, - { headers: { "Content-Type": "application/json" } }, - ); - - if (!presignRes.data.success || !presignRes.data.presignedUrl || !presignRes.data.cdnUrl) { - throw new Error(`获取预签名 URL 失败: ${JSON.stringify(presignRes.data)}`); - } - - const { presignedUrl, cdnUrl } = presignRes.data; - - // 移除 base64 前缀并转为 buffer - const base64Data = base64Image.replace(/^data:image\/\w+;base64,/, ""); - const buffer = Buffer.from(base64Data, "base64"); - - // 上传图片到预签名 URL - await axios.put(presignedUrl, buffer, { - headers: { "Content-Type": "image/jpeg" }, - }); - - imageUrls.push(cdnUrl); - } - } - - // 创建视频生成任务 - const requestBody: { - model: string; - prompt: string; - duration: number; - aspect_ratio: string; - image_urls?: string[]; - } = { - model: model || "sora-2", - prompt: config.prompt, - duration: apimartConfig.duration, - aspect_ratio: apimartConfig.aspectRatio, - }; - - if (imageUrls.length > 0) { - requestBody.image_urls = imageUrls; - } - - const createRes = await axios.post(`${apimartBaseURL}/v1/videos/generations`, requestBody, { - headers: { - Authorization: `Bearer ${apiKey}`, - "Content-Type": "application/json", - }, - }); - - if (createRes.data.code !== 200 || !createRes.data.data?.[0]?.task_id) { - const errorMsg = createRes.data.error?.message || JSON.stringify(createRes.data); - throw new Error(`视频任务创建失败: ${errorMsg}`); - } - - const taskId = createRes.data.data[0].task_id; - - // 轮询任务状态 - videoUrl = await pollTask(async () => { - const res = await axios.get(`${apimartBaseURL}/v1/tasks/${taskId}`, { - headers: { Authorization: `Bearer ${apiKey}` }, - params: { language: "en" }, - }); - - // 检查是否有错误 - if (res.data.error) { - return { - completed: false, - error: `查询失败: ${res.data.error.message || JSON.stringify(res.data.error)}`, - }; - } - - if (res.data.code !== 200) { - return { completed: false, error: `查询失败: ${JSON.stringify(res.data)}` }; - } - - const { status, result } = res.data.data; - - if (status === "completed") { - // 获取视频 URL - const videoUrlResult = result?.videos?.[0]?.url?.[0]; - return { completed: true, imageUrl: videoUrlResult }; - } - - if (status === "failed" || status === "cancelled") { - return { completed: false, error: `任务${status}` }; - } - - // 其他状态(submitted, processing 等)继续轮询 - return { completed: false }; - }); - } else { - throw new Error(`不支持的厂商: ${manufacturer}`); - } - return videoUrl; -}; -export const generateVideo = async (config: VideoConfig, manufacturer: string) => { - if (!config.imageBase64 || config.imageBase64.length <= 0) throw new Error("未传图片"); - const configList = await u.getConfig("video", manufacturer); - console.log("%c Line:533 🥔 configList", "background:#ea7e5c", configList); - if (!configList || configList.length === 0) { - throw new Error("未找到任何视频配置"); - } - let lastError: Error | null = null; - for (const configItem of configList) { - // 每个配置项重试1次,共2次尝试 - for (let attempt = 0; attempt < 2; attempt++) { - try { - const videoUrl = await generateVideoWithConfig(config, configItem); - if (videoUrl) { - const response = await axios.get(videoUrl, { responseType: "stream" }); - await u.oss.writeFile(config.savePath, response.data); - return config.savePath; - } - return videoUrl; - } catch (error: any) { - lastError = error as Error; - console.warn(`配置 ${configItem.model} 第 ${attempt + 1} 次尝试失败:`, error?.response?.data || error.message); - // 如果是第一次尝试失败,继续重试 - if (attempt === 0) continue; - // 第二次也失败了,跳到下一个配置项 - break; - } - } - } - // 所有配置都失败了 - throw new Error(`所有视频配置都失败了。最后一次错误: ${lastError?.message || "未知错误"}`); -}; diff --git a/src/utils/ai/image/index.ts b/src/utils/ai/image/index.ts index 5f3560b..77076ec 100644 --- a/src/utils/ai/image/index.ts +++ b/src/utils/ai/image/index.ts @@ -5,16 +5,11 @@ import axios from "axios"; import volcengine from "./owned/volcengine"; import kling from "./owned/kling"; -import gemini from "./owned/gemini"; import vidu from "./owned/vidu"; import runninghub from "./owned/runninghub"; import apimart from "./owned/apimart"; import other from "./owned/other"; -interface AIConfig { - model?: string; - apiKey?: string; - baseURL?: string; -} +import gemini from "./owned/gemini"; const urlToBase64 = async (url: string): Promise => { const res = await axios.get(url, { responseType: "arraybuffer" }); @@ -29,8 +24,8 @@ const modelInstance = { kling: kling, vidu: vidu, runninghub: runninghub, - apimart: apimart, - other + // apimart: apimart, + other, } as const; export default async (input: ImageConfig, config?: AIConfig) => { @@ -38,11 +33,35 @@ export default async (input: ImageConfig, config?: AIConfig) => { const { model, apiKey, baseURL, manufacturer } = { ...sqlTextModelConfig, ...config }; const manufacturerFn = modelInstance[manufacturer as keyof typeof modelInstance]; if (!manufacturerFn) if (!manufacturerFn) throw new Error("不支持的图片厂商"); - // const owned = modelList.find((m) => m.model === model); - // if (!owned) throw new Error("不支持的模型"); + const owned = modelList.find((m) => m.model === model); + if (!owned) throw new Error("不支持的模型"); + + // 补充图片的 base64 内容类型字符串 + if (input.imageBase64 && input.imageBase64.length > 0) { + input.imageBase64 = input.imageBase64.map((img) => { + if (img.startsWith("data:image/")) { + return img; + } + // 根据 base64 头部判断图片类型 + if (img.startsWith("/9j/")) { + return `data:image/jpeg;base64,${img}`; + } + if (img.startsWith("iVBORw")) { + return `data:image/png;base64,${img}`; + } + if (img.startsWith("R0lGOD")) { + return `data:image/gif;base64,${img}`; + } + if (img.startsWith("UklGR")) { + return `data:image/webp;base64,${img}`; + } + // 默认使用 png + return `data:image/png;base64,${img}`; + }); + } let imageUrl = await manufacturerFn(input, { model, apiKey, baseURL }); if (!input.resType) input.resType = "b64"; if (input.resType === "b64" && imageUrl.startsWith("http")) imageUrl = await urlToBase64(imageUrl); - return imageUrl; + return input; }; diff --git a/src/utils/ai/image/modelList.ts b/src/utils/ai/image/modelList.ts index d5cad4b..1cce6d9 100644 --- a/src/utils/ai/image/modelList.ts +++ b/src/utils/ai/image/modelList.ts @@ -40,6 +40,12 @@ const modelList: Owned[] = [ type: "ti2i", }, //Vidu + { + manufacturer: "vidu", + model: "viduq1", + grid: false, + type: "i2i", + }, { manufacturer: "vidu", model: "viduq2", diff --git a/src/utils/ai/image/owned/apimart.ts b/src/utils/ai/image/owned/apimart.ts index f4ca7cc..9ed4b80 100644 --- a/src/utils/ai/image/owned/apimart.ts +++ b/src/utils/ai/image/owned/apimart.ts @@ -24,7 +24,7 @@ export default async (input: ImageConfig, config: AIConfig): Promise => const res = await axios.get(`https://api.apimart.ai/v1/tasks/${taskId}`, { headers: { Authorization: apiKey }, params: { language: "en" } }); if (res.data.code !== 200) return { completed: false, error: `查询失败: ${JSON.stringify(res.data)}` }; const { status, result } = res.data.data; - if (status === "completed") return { completed: true, imageUrl: result?.images?.[0]?.url?.[0] }; + if (status === "completed") return { completed: true, url: result?.images?.[0]?.url?.[0] }; if (status === "failed" || status === "cancelled") return { completed: false, error: `任务${status}` }; return { completed: false }; }); diff --git a/src/utils/ai/image/owned/kling.ts b/src/utils/ai/image/owned/kling.ts index fa32aa4..1f930f4 100644 --- a/src/utils/ai/image/owned/kling.ts +++ b/src/utils/ai/image/owned/kling.ts @@ -96,7 +96,7 @@ export default async (input: ImageConfig, config: AIConfig): Promise => } if (task_status === "succeed") { - return { completed: true, imageUrl: task_result?.images?.[0]?.url }; + return { completed: true, url: task_result?.images?.[0]?.url }; } return { completed: false }; diff --git a/src/utils/ai/image/owned/other.ts b/src/utils/ai/image/owned/other.ts index 80c7d35..6e72ef5 100644 --- a/src/utils/ai/image/owned/other.ts +++ b/src/utils/ai/image/owned/other.ts @@ -1,5 +1,4 @@ import "../type"; -import { createGoogleGenerativeAI } from "@ai-sdk/google"; import { generateImage, generateText } from "ai"; import { createOpenAICompatible } from "@ai-sdk/openai-compatible"; diff --git a/src/utils/ai/image/owned/runninghub.ts b/src/utils/ai/image/owned/runninghub.ts index d2bb758..7a598c2 100644 --- a/src/utils/ai/image/owned/runninghub.ts +++ b/src/utils/ai/image/owned/runninghub.ts @@ -85,7 +85,7 @@ export default async (input: ImageConfig, config: AIConfig): Promise => return pollTask(async () => { const res = await axios.post(`https://www.runninghub.cn/task/openapi/outputs`, { taskId, apiKey: apiKey }); const { code, msg, data } = res.data; - if (code === 0 && msg === "success") return { completed: true, imageUrl: data?.[0]?.fileUrl }; + if (code === 0 && msg === "success") return { completed: true, url: data?.[0]?.fileUrl }; if (code === 804 || code === 813) return { completed: false }; if (code === 805) return { completed: false, error: `任务失败: ${data?.[0]?.failedReason?.exception_message || "未知原因"}` }; return { completed: false, error: `未知状态: code=${code}, msg=${msg}` }; diff --git a/src/utils/ai/image/owned/vidu.ts b/src/utils/ai/image/owned/vidu.ts index 6ceef26..99a8fee 100644 --- a/src/utils/ai/image/owned/vidu.ts +++ b/src/utils/ai/image/owned/vidu.ts @@ -80,7 +80,7 @@ export default async (input: ImageConfig, config: AIConfig): Promise => } if (state === "succeed") { - return { completed: true, imageUrl: creations?.[0]?.url }; + return { completed: true, url: creations?.[0]?.url }; } return { completed: false }; diff --git a/src/utils/ai/utils.ts b/src/utils/ai/utils.ts index 9bcc2a9..55aab17 100644 --- a/src/utils/ai/utils.ts +++ b/src/utils/ai/utils.ts @@ -1,13 +1,76 @@ +import modelList from "./video/modelList"; + +interface ValidateResult { + owned: (typeof modelList)[number]; + images: string[]; + hasStartEndType: boolean; + hasTextType: boolean; +} + +/** + * 校验视频生成配置与模型是否匹配 + * @param input 视频配置 + * @param config AI配置 + * @param customOwned 自定义模型配置(如果传入则跳过模型查找) + */ +export const validateVideoConfig = (input: VideoConfig, config: AIConfig, customOwned?: (typeof modelList)[number]): ValidateResult => { + if (!config.model) throw new Error("缺少Model名称"); + const owned = customOwned ?? modelList.find((m) => m.model === config.model); + if (!owned) throw new Error(`不支持的模型: ${config.model}`); + const images = input.imageBase64 ?? []; + // 校验图片数量与模型类型是否匹配 + const hasTextType = owned.type.includes("text"); + const hasSingleImageType = owned.type.includes("singleImage"); + const hasStartEndType = owned.type.some((t) => ["startEndRequired", "endFrameOptional", "startFrameOptional"].includes(t)); + const hasMultiImageType = owned.type.includes("multiImage"); + const hasReferenceType = owned.type.includes("reference"); + if (images.length === 0 && !hasTextType) { + throw new Error(`模型 ${config.model} 不支持纯文本生成,需要提供图片`); + } + if (images.length === 1 && !hasSingleImageType && !hasStartEndType && !hasReferenceType) { + throw new Error(`模型 ${config.model} 不支持单图模式`); + } + if (images.length === 2 && !hasStartEndType) { + throw new Error(`模型 ${config.model} 不支持首尾帧模式`); + } + if (images.length > 2 && !hasMultiImageType) { + throw new Error(`模型 ${config.model} 不支持多图模式`); + } + // 校验duration和resolution是否在支持范围内 + const validDurationResolution = owned.durationResolutionMap.some( + (map) => map.duration.includes(input.duration) && map.resolution.includes(input.resolution as typeof map.resolution[number]), + ); + if (!validDurationResolution) { + const supportedDurations = [...new Set(owned.durationResolutionMap.flatMap((m) => m.duration))].sort((a, b) => a - b); + const supportedResolutions = [...new Set(owned.durationResolutionMap.flatMap((m) => m.resolution))]; + throw new Error( + `不支持的duration(${input.duration})或resolution(${input.resolution})组合。` + + `支持的duration: ${supportedDurations.join(", ")},支持的resolution: ${supportedResolutions.join(", ")}`, + ); + } + // 校验音频设置 + if (input.audio && !owned.audio) { + throw new Error(`模型 ${config.model} 不支持生成音频`); + } + // 校验宽高比(仅文本生视频需要) + if (hasTextType && images.length === 0 && owned.aspectRatio.length > 0) { + if (!owned.aspectRatio.includes(input.aspectRatio as `${number}:${number}`)) { + throw new Error(`模型 ${config.model} 不支持宽高比 ${input.aspectRatio},支持的宽高比: ${owned.aspectRatio.join(", ")}`); + } + } + return { owned, images, hasStartEndType, hasTextType }; +}; + export const pollTask = async ( - queryFn: () => Promise<{ completed: boolean; imageUrl?: string; error?: string }>, + queryFn: () => Promise<{ completed: boolean; url?: string; error?: string }>, maxAttempts = 500, interval = 2000, ): Promise => { for (let i = 0; i < maxAttempts; i++) { await new Promise((resolve) => setTimeout(resolve, interval)); - const { completed, imageUrl, error } = await queryFn(); + const { completed, url, error } = await queryFn(); if (error) throw new Error(error); - if (completed && imageUrl) return imageUrl; + if (completed && url) return url; } throw new Error(`任务轮询超时,已尝试 ${maxAttempts} 次`); -}; \ No newline at end of file +}; diff --git a/src/utils/ai/video/index.ts b/src/utils/ai/video/index.ts new file mode 100644 index 0000000..41da25c --- /dev/null +++ b/src/utils/ai/video/index.ts @@ -0,0 +1,63 @@ +import "./type"; +import u from "@/utils"; +import modelList from "./modelList"; +import axios from "axios"; + +import volcengine from "./owned/volcengine"; +import kling from "./owned/kling"; +import vidu from "./owned/vidu"; +import wan from "./owned/wan"; +import runninghub from "./owned/runninghub"; +import gemini from "./owned/gemini"; +import apimart from "./owned/apimart"; + +const modelInstance = { + volcengine: volcengine, + kling: kling, + vidu: vidu, + wan: wan, + gemini: gemini, + runninghub: runninghub, + apimart: apimart, +} as const; + +export default async (input: VideoConfig, config?: AIConfig) => { + const sqlTextModelConfig = await u.getConfig("video"); + const { model, apiKey, baseURL, manufacturer } = { ...sqlTextModelConfig, ...config }; + const manufacturerFn = modelInstance[manufacturer as keyof typeof modelInstance]; + if (!manufacturerFn) if (!manufacturerFn) throw new Error("不支持的视频厂商"); + const owned = modelList.find((m) => m.model === model); + if (!owned) throw new Error("不支持的模型"); + + // 补充图片的 base64 内容类型字符串 + if (input.imageBase64 && input.imageBase64.length > 0) { + input.imageBase64 = input.imageBase64.map((img) => { + if (img.startsWith("data:image/")) { + return img; + } + // 根据 base64 头部判断图片类型 + if (img.startsWith("/9j/")) { + return `data:image/jpeg;base64,${img}`; + } + if (img.startsWith("iVBORw")) { + return `data:image/png;base64,${img}`; + } + if (img.startsWith("R0lGOD")) { + return `data:image/gif;base64,${img}`; + } + if (img.startsWith("UklGR")) { + return `data:image/webp;base64,${img}`; + } + // 默认使用 png + return `data:image/png;base64,${img}`; + }); + } + + let videoUrl = await manufacturerFn(input, { model, apiKey, baseURL }); + if (videoUrl) { + const response = await axios.get(videoUrl, { responseType: "stream" }); + await u.oss.writeFile(input.savePath, response.data); + return input.savePath; + } + return videoUrl; +}; diff --git a/src/utils/ai/video/modelList.ts b/src/utils/ai/video/modelList.ts new file mode 100644 index 0000000..14c65aa --- /dev/null +++ b/src/utils/ai/video/modelList.ts @@ -0,0 +1,489 @@ +type VideoGenerationType = + | "singleImage" // 单图 + | "startEndRequired" // 首尾帧(两张都得有) + | "endFrameOptional" // 首尾帧(尾帧可选) + | "startFrameOptional" // 首尾帧(首帧可选) + | "multiImage" // 多图模式 + | "reference" // 参考图模式 + | "text"; // 文本生视频 + +interface DurationResolutionMap { + duration: number[]; + resolution: (`${number}p` | `${number}k`)[]; +} +interface Owned { + manufacturer: string; + model: string; + durationResolutionMap: DurationResolutionMap[]; + aspectRatio: `${number}:${number}`[]; + type: VideoGenerationType[]; + audio: boolean; +} + +const modelList: Owned[] = [ + // ================== 火山引擎/豆包系列 ================== + // doubao-seedance-1-5-pro 文生视频/图生视频 + { + manufacturer: "volcengine", + model: "doubao-seedance-1-5-pro-251215", + durationResolutionMap: [{ duration: [4, 5, 6, 7, 8, 9, 10, 11, 12], resolution: ["480p", "720p", "1080p"] }], + aspectRatio: ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9"], + type: ["text", "endFrameOptional"], + audio: true, + }, + // doubao-seedance-1-0-pro 文生视频/图生视频 + { + manufacturer: "volcengine", + model: "doubao-seedance-1-0-pro-250528", + durationResolutionMap: [{ duration: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], resolution: ["480p", "720p", "1080p"] }], + aspectRatio: ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9"], + type: ["text", "endFrameOptional"], + audio: false, + }, + // doubao-seedance-1-0-pro-fast 文生视频/图生视频 + { + manufacturer: "volcengine", + model: "doubao-seedance-1-0-pro-fast-251015", + durationResolutionMap: [{ duration: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], resolution: ["480p", "720p", "1080p"] }], + aspectRatio: ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9"], + type: ["text", "singleImage"], + audio: false, + }, + // doubao-seedance-1-0-lite-i2v 图生视频(仅支持图片模式) + { + manufacturer: "volcengine", + model: "doubao-seedance-1-0-lite-i2v-250428", + durationResolutionMap: [{ duration: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], resolution: ["480p", "720p", "1080p"] }], + aspectRatio: [], + type: ["endFrameOptional", "reference"], + audio: false, + }, + // doubao-seedance-1-0-lite-t2v 文生视频(仅支持文本模式) + { + manufacturer: "volcengine", + model: "doubao-seedance-1-0-lite-t2v-250428", + durationResolutionMap: [{ duration: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], resolution: ["480p", "720p", "1080p"] }], + aspectRatio: ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9"], + type: ["text"], + audio: false, + }, + // ================== 可灵系列 ================== + // kling-v1(STD) 文生视频 + { + manufacturer: "kling", + model: "kling-v1(STD)", + durationResolutionMap: [{ duration: [5, 10], resolution: ["720p"] }], + aspectRatio: ["16:9", "1:1", "9:16"], + type: ["text"], + audio: false, + }, + // kling-v1(STD) 图生视频 + { + manufacturer: "kling", + model: "kling-v1(STD)", + durationResolutionMap: [{ duration: [5, 10], resolution: ["720p"] }], + aspectRatio: [], + type: ["startEndRequired"], + audio: false, + }, + // kling-v1(PRO) 文生视频 + { + manufacturer: "kling", + model: "kling-v1(PRO)", + durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }], + aspectRatio: ["16:9", "1:1", "9:16"], + type: ["text"], + audio: false, + }, + // kling-v1(PRO) 图生视频 + { + manufacturer: "kling", + model: "kling-v1(PRO)", + durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }], + aspectRatio: [], + type: ["startEndRequired"], + audio: false, + }, + // kling-v1-6(PRO) 文生视频 + { + manufacturer: "kling", + model: "kling-v1-6(PRO)", + durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }], + aspectRatio: ["16:9", "1:1", "9:16"], + type: ["text"], + audio: false, + }, + // kling-v1-6(PRO) 图生视频 + { + manufacturer: "kling", + model: "kling-v1-6(PRO)", + durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }], + aspectRatio: [], + type: ["startEndRequired"], + audio: false, + }, + // kling-v2-5-turbo(PRO) 文生视频 + { + manufacturer: "kling", + model: "kling-v2-5-turbo(PRO)", + durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }], + aspectRatio: ["16:9", "1:1", "9:16"], + type: ["text"], + audio: false, + }, + // kling-v2-5-turbo(PRO) 图生视频 + { + manufacturer: "kling", + model: "kling-v2-5-turbo(PRO)", + durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }], + aspectRatio: [], + type: ["startEndRequired"], + audio: false, + }, + // kling-v2-6(PRO) 文生视频 + { + manufacturer: "kling", + model: "kling-v2-6(PRO)", + durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }], + aspectRatio: ["16:9", "1:1", "9:16"], + type: ["text"], + audio: false, + }, + // kling-v2-6(PRO) 图生视频 + { + manufacturer: "kling", + model: "kling-v2-6(PRO)", + durationResolutionMap: [{ duration: [5, 10], resolution: ["1080p"] }], + aspectRatio: [], + type: ["startEndRequired"], + audio: false, + }, + // ================== ViduQ3系列 ================== + // viduq3-pro 文生视频 + { + manufacturer: "vidu", + model: "viduq3-pro", + durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], resolution: ["540p", "720p", "1080p"] }], + aspectRatio: ["16:9", "9:16", "3:4", "4:3", "1:1"], + type: ["text"], + audio: true, + }, + // viduq3-pro 图生视频 + { + manufacturer: "vidu", + model: "viduq3-pro", + durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], resolution: ["540p", "720p", "1080p"] }], + aspectRatio: [], + type: ["singleImage"], + audio: true, + }, + // viduq2-pro-fast 图生视频 + { + manufacturer: "vidu", + model: "viduq2-pro-fast", + durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], resolution: ["720p", "1080p"] }], + aspectRatio: [], + type: ["singleImage", "startEndRequired"], + audio: false, + }, + // viduq2-pro 文生视频 + { + manufacturer: "vidu", + model: "viduq2-pro", + durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], resolution: ["540p", "720p", "1080p"] }], + aspectRatio: ["16:9", "9:16", "3:4", "4:3", "1:1"], + type: ["text"], + audio: false, + }, + // viduq2-pro 图生视频 + { + manufacturer: "vidu", + model: "viduq2-pro", + durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], resolution: ["540p", "720p", "1080p"] }], + aspectRatio: [], + type: ["singleImage", "reference", "startEndRequired"], + audio: false, + }, + // viduq2-turbo 文生视频 + { + manufacturer: "vidu", + model: "viduq2-turbo", + durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], resolution: ["540p", "720p", "1080p"] }], + aspectRatio: ["16:9", "9:16", "3:4", "4:3", "1:1"], + type: ["text"], + audio: false, + }, + // viduq2-turbo 图生视频 + { + manufacturer: "vidu", + model: "viduq2-turbo", + durationResolutionMap: [{ duration: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], resolution: ["540p", "720p", "1080p"] }], + aspectRatio: [], + type: ["singleImage", "reference", "startEndRequired"], + audio: false, + }, + // viduq1 文生视频 + { + manufacturer: "vidu", + model: "viduq1", + durationResolutionMap: [{ duration: [5], resolution: ["1080p"] }], + aspectRatio: ["16:9", "9:16", "1:1"], + type: ["text"], + audio: false, + }, + // viduq1 图生视频 + { + manufacturer: "vidu", + model: "viduq1", + durationResolutionMap: [{ duration: [5], resolution: ["1080p"] }], + aspectRatio: [], + type: ["singleImage", "reference", "startEndRequired"], + audio: false, + }, + // viduq1-classic 图生视频 + { + manufacturer: "vidu", + model: "viduq1-classic", + durationResolutionMap: [{ duration: [5], resolution: ["1080p"] }], + aspectRatio: [], + type: ["singleImage", "startEndRequired"], + audio: false, + }, + // vidu2.0 图生视频 + { + manufacturer: "vidu", + model: "vidu2.0", + durationResolutionMap: [ + { duration: [4], resolution: ["360p", "720p", "1080p"] }, + { duration: [8], resolution: ["720p"] }, + ], + aspectRatio: [], + type: ["singleImage", "reference", "startEndRequired"], + audio: false, + }, + // ================== 万象系列 ================== + // wan2.6-t2v 文生视频(有声视频) + { + manufacturer: "wan", + model: "wan2.6-t2v", + durationResolutionMap: [{ duration: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], resolution: ["720p", "1080p"] }], + aspectRatio: ["16:9", "9:16", "1:1", "4:3", "3:4"], + type: ["text"], + audio: true, + }, + // wan2.5-t2v-preview 文生视频(有声视频) + { + manufacturer: "wan", + model: "wan2.5-t2v-preview", + durationResolutionMap: [{ duration: [5, 10], resolution: ["480p", "720p", "1080p"] }], + aspectRatio: ["16:9", "9:16", "1:1", "4:3", "3:4"], + type: ["text"], + audio: true, + }, + // wan2.2-t2v-plus 文生视频(无声视频) + { + manufacturer: "wan", + model: "wan2.2-t2v-plus", + durationResolutionMap: [{ duration: [5], resolution: ["480p", "1080p"] }], + aspectRatio: ["16:9", "9:16", "1:1", "4:3", "3:4"], + type: ["text"], + audio: false, + }, + // wanx2.1-t2v-turbo 文生视频(无声视频) + { + manufacturer: "wan", + model: "wanx2.1-t2v-turbo", + durationResolutionMap: [{ duration: [5], resolution: ["480p", "720p"] }], + aspectRatio: ["16:9", "9:16", "1:1", "4:3", "3:4"], + type: ["text"], + audio: false, + }, + // wanx2.1-t2v-plus 文生视频(无声视频) + { + manufacturer: "wan", + model: "wanx2.1-t2v-plus", + durationResolutionMap: [{ duration: [5], resolution: ["720p"] }], + aspectRatio: ["16:9", "9:16", "1:1", "4:3", "3:4"], + type: ["text"], + audio: false, + }, + // wan2.6-i2v-flash 图生视频(有声视频&无声视频) + { + manufacturer: "wan", + model: "wan2.6-i2v-flash", + durationResolutionMap: [{ duration: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], resolution: ["720p", "1080p"] }], + aspectRatio: [], + type: ["singleImage"], + audio: true, + }, + // wan2.6-i2v 图生视频(有声视频) + { + manufacturer: "wan", + model: "wan2.6-i2v", + durationResolutionMap: [{ duration: [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], resolution: ["720p", "1080p"] }], + aspectRatio: [], + type: ["singleImage"], + audio: true, + }, + // wan2.5-i2v-preview 图生视频(有声视频) + { + manufacturer: "wan", + model: "wan2.5-i2v-preview", + durationResolutionMap: [{ duration: [5, 10], resolution: ["480p", "720p", "1080p"] }], + aspectRatio: [], + type: ["singleImage"], + audio: true, + }, + // wan2.2-i2v-flash 图生视频(无声视频) + { + manufacturer: "wan", + model: "wan2.2-i2v-flash", + durationResolutionMap: [{ duration: [5], resolution: ["480p", "720p", "1080p"] }], + aspectRatio: [], + type: ["singleImage"], + audio: false, + }, + // wan2.2-i2v-plus 图生视频(无声视频) + { + manufacturer: "wan", + model: "wan2.2-i2v-plus", + durationResolutionMap: [{ duration: [5], resolution: ["480p", "1080p"] }], + aspectRatio: [], + type: ["singleImage"], + audio: false, + }, + // wanx2.1-i2v-plus 图生视频(无声视频) + { + manufacturer: "wan", + model: "wanx2.1-i2v-plus", + durationResolutionMap: [{ duration: [5], resolution: ["720p"] }], + aspectRatio: [], + type: ["singleImage"], + audio: false, + }, + // wanx2.1-i2v-turbo 图生视频(无声视频) + { + manufacturer: "wan", + model: "wanx2.1-i2v-turbo", + durationResolutionMap: [{ duration: [3, 4, 5], resolution: ["480p", "720p"] }], + aspectRatio: [], + type: ["singleImage"], + audio: false, + }, + // wan2.2-kf2v-flash 首尾帧生视频(无声视频) + { + manufacturer: "wan", + model: "wan2.2-kf2v-flash", + durationResolutionMap: [{ duration: [5], resolution: ["480p", "720p", "1080p"] }], + aspectRatio: [], + type: ["startEndRequired"], + audio: false, + }, + // wanx2.1-kf2v-plus 首尾帧生视频(无声视频) + { + manufacturer: "wan", + model: "wanx2.1-kf2v-plus", + durationResolutionMap: [{ duration: [5], resolution: ["720p"] }], + aspectRatio: [], + type: ["startEndRequired"], + audio: false, + }, + // ================== Gemini Veo 系列 ================== + // Veo 3.1 预览版(支持音频) + { + manufacturer: "gemini", + model: "veo-3.1-generate-preview", + durationResolutionMap: [ + { duration: [4, 6], resolution: ["720p"] }, + { duration: [8], resolution: ["720p", "1080p"] }, + ], + aspectRatio: ["16:9", "9:16"], + type: ["text", "singleImage", "startEndRequired", "endFrameOptional", "reference"], + audio: true, + }, + // Veo 3.1 Fast 预览版(支持音频) + { + manufacturer: "gemini", + model: "veo-3.1-fast-generate-preview", + durationResolutionMap: [ + { duration: [4, 6], resolution: ["720p"] }, + { duration: [8], resolution: ["720p", "1080p"] }, + ], + aspectRatio: ["16:9", "9:16"], + type: ["text", "singleImage", "startEndRequired", "endFrameOptional", "reference"], + audio: true, + }, + // Veo 3 稳定版(支持音频) + { + manufacturer: "gemini", + model: "veo-3.0-generate-preview", + durationResolutionMap: [ + { duration: [4, 6], resolution: ["720p"] }, + { duration: [8], resolution: ["720p", "1080p"] }, + ], + aspectRatio: ["16:9", "9:16"], + type: ["text", "singleImage"], + audio: true, + }, + // Veo 3 Fast 稳定版(支持音频) + { + manufacturer: "gemini", + model: "veo-3.0-fast-generate-preview", + durationResolutionMap: [ + { duration: [4, 6], resolution: ["720p"] }, + { duration: [8], resolution: ["720p", "1080p"] }, + ], + aspectRatio: ["16:9", "9:16"], + type: ["text", "singleImage"], + audio: true, + }, + // Veo 2 稳定版(无音频) + { + manufacturer: "gemini", + model: "veo-2.0-generate-001", + durationResolutionMap: [{ duration: [5, 6, 7, 8], resolution: ["720p"] }], + aspectRatio: ["16:9", "9:16"], + type: ["text", "singleImage"], + audio: false, + }, + // ================== RunningHub 系列 ================== + // sora + { + manufacturer: "runninghub", + model: "sora-2", + durationResolutionMap: [{ duration: [10, 15], resolution: [] }], + aspectRatio: ["16:9", "9:16"], + type: ["singleImage", "text"], + audio: false, + }, + // sora 2 + { + manufacturer: "runninghub", + model: "sora-2-pro", + durationResolutionMap: [{ duration: [15, 25], resolution: [] }], + aspectRatio: ["16:9", "9:16"], + type: ["singleImage", "text"], + audio: false, + }, + // ================== Apimart 系列 ================== + // sora + { + manufacturer: "apimart", + model: "sora-2", + durationResolutionMap: [{ duration: [10, 15], resolution: [] }], + aspectRatio: ["16:9", "9:16"], + type: ["singleImage", "text"], + audio: false, + }, + // sora 2 + { + manufacturer: "apimart", + model: "sora-2-pro", + durationResolutionMap: [{ duration: [15, 25], resolution: [] }], + aspectRatio: ["16:9", "9:16"], + type: ["singleImage", "text"], + audio: false, + }, +]; + +export default modelList; diff --git a/src/utils/ai/video/owned/apimart.ts b/src/utils/ai/video/owned/apimart.ts new file mode 100644 index 0000000..143859a --- /dev/null +++ b/src/utils/ai/video/owned/apimart.ts @@ -0,0 +1,115 @@ +import "../type"; +import axios from "axios"; +import { pollTask } from "@/utils/ai/utils"; +import modelList from "../modelList"; + +// 上传图片到 apimart 图床 +async function uploadImageToApimart(base64Image: string): Promise { + if (base64Image.startsWith("http")) { + return base64Image; + } + + const presignRes = await axios.post( + "https://apimart.ai/api/upload/presign", + { contentType: "image/jpeg", fileExtension: "jpeg", permanent: false }, + { headers: { "Content-Type": "application/json" } }, + ); + + if (!presignRes.data.success || !presignRes.data.presignedUrl || !presignRes.data.cdnUrl) { + throw new Error(`获取预签名 URL 失败: ${JSON.stringify(presignRes.data)}`); + } + + const { presignedUrl, cdnUrl } = presignRes.data; + + const base64Data = base64Image.replace(/^data:image\/\w+;base64,/, ""); + const buffer = Buffer.from(base64Data, "base64"); + + await axios.put(presignedUrl, buffer, { + headers: { "Content-Type": "image/jpeg" }, + }); + + return cdnUrl; +} + +export default async (input: VideoConfig, config: AIConfig) => { + if (!config.model) throw new Error("缺少 Model 名称"); + if (!config.apiKey) throw new Error("缺少 API Key"); + + const owned = modelList.find((m) => m.model === config.model); + if (!owned) throw new Error(`未找到模型: ${config.model}`); + + // 默认 baseURL 配置 + const defaultBaseUrl = "https://api.apimart.ai/v1/videos/generations|https://api.apimart.ai/v1/tasks/{taskId}"; + const [generateUrl, queryUrl] = (config.baseURL || defaultBaseUrl).split("|"); + + const authorization = `Bearer ${config.apiKey}`; + + // 上传图片到图床 + let imageUrls: string[] = []; + if (input.imageBase64 && input.imageBase64.length > 0) { + for (const base64Image of input.imageBase64) { + const imageUrl = await uploadImageToApimart(base64Image); + imageUrls.push(imageUrl); + } + } + + // 构建请求体 + const requestBody: Record = { + model: config.model, + prompt: input.prompt, + duration: input.duration, + aspect_ratio: input.aspectRatio, + }; + + if (imageUrls.length > 0) { + requestBody.image_urls = imageUrls; + } + + // 创建任务 + const createRes = await axios.post(generateUrl, requestBody, { + headers: { + Authorization: authorization, + "Content-Type": "application/json", + }, + }); + + if (createRes.data.code !== 200 || !createRes.data.data?.[0]?.task_id) { + throw new Error(`创建任务失败: ${JSON.stringify(createRes.data)}`); + } + + const taskId = createRes.data.data[0].task_id; + const actualQueryUrl = queryUrl.replace("{taskId}", taskId); + + // 轮询任务状态 + return await pollTask(async () => { + const queryRes = await axios.get(actualQueryUrl, { + headers: { Authorization: authorization }, + }); + + const { code, data } = queryRes.data; + + if (code !== 200 || !data) { + return { completed: false, error: `查询失败: ${JSON.stringify(queryRes.data)}` }; + } + + const { status, result, error } = data; + + switch (status) { + case "completed": + const videoUrl = result?.videos?.[0]?.url?.[0]; + if (!videoUrl) { + return { completed: false, error: "未获取到视频 URL" }; + } + return { completed: true, url: videoUrl }; + case "failed": + return { completed: false, error: error?.message || "任务失败" }; + case "cancelled": + return { completed: false, error: "任务已取消" }; + case "pending": + case "processing": + return { completed: false }; + default: + return { completed: false, error: `未知状态: ${status}` }; + } + }); +}; diff --git a/src/utils/ai/video/owned/gemini.ts b/src/utils/ai/video/owned/gemini.ts new file mode 100644 index 0000000..948c1a0 --- /dev/null +++ b/src/utils/ai/video/owned/gemini.ts @@ -0,0 +1,68 @@ +import "../type"; +import fs from "fs"; +import path from "path"; +import axios from "axios"; +import { pollTask, validateVideoConfig } from "@/utils/ai/utils"; + +const buildInlineImage = (data: string) => ({ inlineData: { mimeType: "image/png", data } }); + +export default async (input: VideoConfig, config: AIConfig) => { + if (!config.model) throw new Error("缺少Model名称"); + if (!config.apiKey) throw new Error("缺少API Key"); + + const { owned, images, hasStartEndType } = validateVideoConfig(input, config); + + const defaultBaseUrl = [ + "https://generativelanguage.googleapis.com/v1beta/models/{model}:predictLongRunning", + "https://generativelanguage.googleapis.com/v1beta/{name}", + ].join("|"); + + const [submitUrl, queryUrl] = (config.baseURL || defaultBaseUrl).split("|"); + const headers = { "x-goog-api-key": config.apiKey }; + + const instance: Record = { prompt: input.prompt }; + const parameters: Record = { + aspectRatio: input.aspectRatio, + durationSeconds: String(input.duration), + ...(input.resolution !== "720p" && { resolution: input.resolution }), + }; + + // 根据图片数量和模型能力决定图片用法 + const len = images.length; + const hasRef = owned.type.includes("reference"); + const hasSingle = owned.type.includes("singleImage"); + + if (len === 2 && hasStartEndType) { + instance.image = buildInlineImage(images[0]); + parameters.lastFrame = buildInlineImage(images[1]); + } else if (len === 1 && (hasSingle || hasStartEndType)) { + instance.image = buildInlineImage(images[0]); + } else if (len >= 1 && len <= 3 && hasRef) { + parameters.referenceImages = images.map((img) => ({ image: buildInlineImage(img), referenceType: "asset" })); + } + + const { data } = await axios.post( + submitUrl.replace("{model}", config.model), + { instances: [instance], parameters }, + { headers: { ...headers, "Content-Type": "application/json" } }, + ); + + if (!data.name) throw new Error("未获取到操作名称"); + + return pollTask(async () => { + const { data: status } = await axios.get(queryUrl.replace("{name}", data.name), { headers }); + const { done, response, error } = status; + + if (!done) return { completed: false }; + if (error) return { completed: false, error: `任务失败: ${error.message || JSON.stringify(error)}` }; + + const videoUri = response?.generateVideoResponse?.generatedSamples?.[0]?.video?.uri; + if (!videoUri) return { completed: false, error: "未获取到视频下载地址" }; + + const videoRes = await axios.get(videoUri, { headers, responseType: "arraybuffer", maxRedirects: 5 }); + const savePath = input.savePath.endsWith(".mp4") ? input.savePath : path.join(input.savePath, `gemini_${Date.now()}.mp4`); + fs.writeFileSync(savePath, Buffer.from(videoRes.data)); + + return { completed: true, url: savePath }; + }); +}; \ No newline at end of file diff --git a/src/utils/ai/video/owned/kling.ts b/src/utils/ai/video/owned/kling.ts new file mode 100644 index 0000000..bb3c539 --- /dev/null +++ b/src/utils/ai/video/owned/kling.ts @@ -0,0 +1,90 @@ +import "../type"; +import axios from "axios"; +import { pollTask, validateVideoConfig } from "@/utils/ai/utils"; + +export default async (input: VideoConfig, config: AIConfig) => { + if (!config.apiKey) throw new Error("缺少API Key"); + if (!config.baseURL) throw new Error("缺少baseURL配置"); + + const { images } = validateVideoConfig(input, config); + + // 解析URL配置:图生视频|文生视频|查询地址 + const defaultBaseUrl = + "https://api-beijing.klingai.com/v1/videos/image2video|https://api-beijing.klingai.com/v1/videos/text2video|https://api-beijing.klingai.com/v1/videos/text2video/{taskId}"; + const [image2videoUrl, text2videoUrl, queryUrl] = (config.baseURL || defaultBaseUrl).split("|"); + + const headers = { + Authorization: `Bearer ${config.apiKey}`, + "Content-Type": "application/json", + }; + + // 解析模型名称和模式,例如 "kling-v2-6(PRO)" => modelName: "kling-v2-6", mode: "pro" + const modelMatch = config.model!.match(/^(.+)\((STD|PRO)\)$/i); + const modelName = modelMatch ? modelMatch[1] : config.model; + const mode = modelMatch ? (modelMatch[2].toLowerCase() as "std" | "pro") : "std"; + + // 判断是图生视频还是文生视频 + const hasImage = images.length > 0; + const createUrl = hasImage ? image2videoUrl : text2videoUrl; + + // 去除图片的内容类型前缀(kling要求纯base64) + const stripDataUrl = (str: string) => str.replace(/^data:image\/[^;]+;base64,/, ""); + + // 构建请求体 + const body: Record = { + model_name: modelName, + mode, + duration: String(input.duration), + prompt: input.prompt, + aspect_ratio: input.aspectRatio, + }; + + if (hasImage) { + // 图生视频:首帧和尾帧 + body.image = stripDataUrl(images[0]); + if (images.length > 1) { + body.image_tail = stripDataUrl(images[1]); + } + } + + // 创建任务 + const createResponse = await axios.post(createUrl, body, { headers }); + const createData = createResponse.data; + if (createData.code !== 0) { + throw new Error(`创建任务失败: ${createData.message || "未知错误"}`); + } + + const taskId = createData.data?.task_id; + if (!taskId) { + throw new Error("创建任务失败: 未返回任务ID"); + } + + // 轮询任务状态 + return await pollTask(async () => { + const queryResponse = await axios.get(`${queryUrl.replace("{taskId}", taskId)}`, { headers }); + const queryData = queryResponse.data; + if (queryData.code !== 0) { + return { completed: false, error: `查询失败: ${queryData.message || "未知错误"}` }; + } + + const task = queryData.data; + const taskStatus = task?.task_status; + + switch (taskStatus) { + case "succeed": { + const videoUrl = task?.task_result?.videos?.[0]?.url; + if (!videoUrl) { + return { completed: false, error: "任务成功但未返回视频URL" }; + } + return { completed: true, url: videoUrl }; + } + case "failed": + return { completed: false, error: `任务失败: ${task?.task_status_msg || "未知原因"}` }; + case "submitted": + case "processing": + return { completed: false }; + default: + return { completed: false, error: `未知状态: ${taskStatus}` }; + } + }); +}; diff --git a/src/utils/ai/video/owned/runninghub.ts b/src/utils/ai/video/owned/runninghub.ts new file mode 100644 index 0000000..df756d2 --- /dev/null +++ b/src/utils/ai/video/owned/runninghub.ts @@ -0,0 +1,91 @@ +import "../type"; +import axios from "axios"; +import sharp from "sharp"; +import FormData from "form-data"; +import { pollTask, validateVideoConfig } from "@/utils/ai/utils"; + +export default async (input: VideoConfig, config: AIConfig) => { + if (!config.apiKey) throw new Error("缺少API Key"); + + const { owned, images, hasTextType } = validateVideoConfig(input, config); + + const defaultBaseUrl = [ + "https://www.runninghub.cn/openapi/v2/rhart-video-s/image-to-video", + "https://www.runninghub.cn/openapi/v2/rhart-video-s/image-to-video-pro", + "https://www.runninghub.cn/openapi/v2/rhart-video-s/text-to-video", + "https://www.runninghub.cn/openapi/v2/rhart-video-s/text-to-video-pro", + "https://www.runninghub.cn/openapi/v2/rhart-video-s/{taskId}", + "https://www.runninghub.cn/openapi/v2/media/upload/binary", + ].join("|"); + + const [image2videoUrl, image2videoProUrl, text2videoUrl, text2videoProUrl, queryUrl, uploadUrl] = (config.baseURL || defaultBaseUrl).split("|"); + + const isPro = owned.model === "sora-2-pro"; + const authorization = `Bearer ${config.apiKey}`; + + // 上传 base64 图片 + const uploadImage = async (base64Image: string): Promise => { + const base64Data = base64Image.replace(/^data:image\/\w+;base64,/, ""); + let buffer: Buffer = Buffer.from(base64Data, "base64"); + const MAX_SIZE = 5 * 1024 * 1024; + + if (buffer.length > MAX_SIZE) { + for (let quality = 90; buffer.length > MAX_SIZE && quality > 10; quality -= 10) { + buffer = await sharp(buffer).jpeg({ quality, mozjpeg: true }).toBuffer(); + } + if (buffer.length > MAX_SIZE) { + const { width = 1920, height = 1080 } = await sharp(buffer).metadata(); + const scale = Math.sqrt(MAX_SIZE / buffer.length); + buffer = await sharp(buffer) + .resize({ width: Math.floor(width * scale), height: Math.floor(height * scale), fit: "inside" }) + .jpeg({ quality: 80, mozjpeg: true }) + .toBuffer(); + } + } + + const formData = new FormData(); + formData.append("file", buffer, { filename: "image.jpg", contentType: "image/jpeg" }); + + const { data } = await axios.post(uploadUrl, formData, { + headers: { Authorization: authorization }, + }); + + if (data.code !== 0 || !data.data?.download_url) { + throw new Error(`图片上传失败: ${JSON.stringify(data)}`); + } + return data.data.download_url; + }; + + // 提交任务 + const submitTask = async (url: string, body: Record) => { + const { data } = await axios.post(url, body, { + headers: { "Content-Type": "application/json", Authorization: authorization }, + }); + if (data.status === "FAILED") throw new Error(`任务提交失败: ${data.errorMessage || "未知错误"}`); + return { taskId: data.taskId, status: data.status, url: data.results?.[0]?.url }; + }; + + const isTextToVideo = images.length === 0 && hasTextType; + const submitUrl = isTextToVideo ? (isPro ? text2videoProUrl : text2videoUrl) : isPro ? image2videoProUrl : image2videoUrl; + + const requestBody: Record = { + prompt: input.prompt, + duration: String(input.duration), + aspectRatio: input.aspectRatio, + ...(isTextToVideo ? {} : { imageUrl: await uploadImage(images[0]) }), + }; + + const { taskId } = await submitTask(submitUrl, requestBody); + + return await pollTask(async () => { + const { data } = await axios.get(queryUrl.replace("{taskId}", taskId), { + headers: { Authorization: authorization }, + }); + if (data.status === "SUCCESS") { + return data.results?.length ? { completed: true, url: data.results[0].url } : { completed: false, error: "任务成功但未返回视频链接" }; + } + if (data.status === "FAILED") return { completed: false, error: `任务失败: ${data.errorMessage || "未知错误"}` }; + if (data.status === "QUEUED" || data.status === "RUNNING") return { completed: false }; + return { completed: false, error: `未知状态: ${data.status}` }; + }); +}; diff --git a/src/utils/ai/video/owned/vidu.ts b/src/utils/ai/video/owned/vidu.ts new file mode 100644 index 0000000..86cac52 --- /dev/null +++ b/src/utils/ai/video/owned/vidu.ts @@ -0,0 +1,132 @@ +import "../type"; +import axios from "axios"; +import { pollTask, validateVideoConfig } from "@/utils/ai/utils"; +import modelList from "../modelList"; + +export default async (input: VideoConfig, config: AIConfig) => { + if (!config.model) throw new Error("缺少Model名称"); + if (!config.apiKey) throw new Error("缺少API Key"); + if (!input.prompt && (!input.imageBase64 || input.imageBase64.length === 0)) { + throw new Error("至少需要提供prompt或图片"); + } + + const defaultBaseUrl = ["https://api.vidu.cn/ent/v2/text2video", "https://api.vidu.cn/ent/v2/img2video", "https://api.vidu.cn/ent/v2/tasks"].join( + "|", + ); + + const [text2videoUrl, image2videoUrl, queryUrl] = (config.baseURL || defaultBaseUrl).split("|"); + + const authorization = `Token ${config.apiKey}`; + const hasImages = input.imageBase64 && input.imageBase64.length > 0; + + // 根据是否有图片,查找匹配的模型配置 + const customOwned = modelList.find((m) => { + if (m.manufacturer !== "vidu") return false; + if (m.model !== config.model) return false; + if (hasImages) { + return m.type.some((t) => t !== "text"); + } else { + return m.type.includes("text"); + } + }); + + if (!customOwned) { + throw new Error(`未找到匹配的模型配置: ${config.model}`); + } + + // 使用统一校验函数 + const { owned, images } = validateVideoConfig(input, config, customOwned); + + // 判断生成类型 + const genType: "text" | "image" = images.length === 0 ? "text" : "image"; + + // 校验宽高比(仅文生视频需要) + if (genType === "text" && owned.aspectRatio.length > 0 && !owned.aspectRatio.includes(input.aspectRatio as `${number}:${number}`)) { + throw new Error(`模型 ${owned.model} 不支持宽高比 ${input.aspectRatio},支持的宽高比:${owned.aspectRatio.join("、")}`); + } + + // 创建任务 + let taskId: string; + + if (genType === "text") { + // 文生视频 + const requestBody: Record = { + model: owned.model, + prompt: input.prompt, + duration: input.duration, + resolution: input.resolution, + aspect_ratio: input.aspectRatio, + }; + if (owned.audio && input.audio !== undefined) { + requestBody.audio = input.audio; + } + + const response = await axios.post(text2videoUrl, requestBody, { + headers: { + "Content-Type": "application/json", + Authorization: authorization, + }, + }); + taskId = response.data.task_id; + } else { + // 图生视频 + const requestBody: Record = { + model: owned.model, + images: images, + duration: input.duration, + resolution: input.resolution, + }; + if (input.prompt) { + requestBody.prompt = input.prompt; + } + if (owned.audio && input.audio !== undefined) { + requestBody.audio = input.audio; + } + + const response = await axios.post(image2videoUrl, requestBody, { + headers: { + "Content-Type": "application/json", + Authorization: authorization, + }, + }); + taskId = response.data.task_id; + } + + // 轮询任务状态 + return await pollTask(async () => { + const response = await axios.get(queryUrl, { + headers: { + "Content-Type": "application/json", + Authorization: authorization, + }, + params: { + task_ids: [taskId], + }, + }); + + const tasks = response.data.tasks; + if (!tasks || tasks.length === 0) { + return { completed: false, error: "任务不存在" }; + } + + const task = tasks[0]; + + switch (task.state) { + case "success": { + const creation = task.creations?.[0]; + return { + completed: true, + url: creation?.url, + }; + } + case "failed": + return { completed: false, error: "任务生成失败" }; + case "created": + case "queueing": + case "processing": + return { completed: false }; + default: + return { completed: false, error: `未知状态: ${task.state}` }; + } + }); +}; diff --git a/src/utils/ai/video/owned/volcengine.ts b/src/utils/ai/video/owned/volcengine.ts index 9f9aa1d..25e1ec4 100644 --- a/src/utils/ai/video/owned/volcengine.ts +++ b/src/utils/ai/video/owned/volcengine.ts @@ -1,56 +1,74 @@ import "../type"; import axios from "axios"; -import { pollTask } from "@/utils/ai/utils"; +import { pollTask, validateVideoConfig } from "@/utils/ai/utils"; -interface DoubaoVideoConfig { - prompt: string; - savePath: string; - imageBase64?: string[]; // 单张参考图片 base64 - duration: 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12; // 支持 2~12 秒 - aspectRatio: "16:9" | "9:16" | "1:1" | "4:3" | "3:4" | "21:9" | "adaptive"; - audio?: boolean; -} - -export default async (input: ImageConfig, config: AIConfig) => { - console.log("%c Line:5 🍓 input", "background:#7f2b82", input); - console.log("%c Line:5 🍎 config", "background:#93c0a4", config); - if (!config.model) throw new Error("缺少Model名称"); +export default async (input: VideoConfig, config: AIConfig) => { if (!config.apiKey) throw new Error("缺少API Key"); - const key = "Bearer " + config.apiKey.replaceAll("Bearer ", "").trim(); + const { owned, images, hasStartEndType } = validateVideoConfig(input, config); - const doubaoConfig = config as DoubaoVideoConfig; - const createRes = await axios.post( - config.baseURL ?? "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks", - { - model: "doubao-seedance-1-5-pro-251215", - content: [ - { type: "text", text: input.prompt }, - ...(doubaoConfig.imageBase64 - ? doubaoConfig.imageBase64.map((base64, i) => ({ - type: "image_url", - image_url: { url: base64 }, - role: i === 0 ? "first_frame" : "last_frame", - })) - : []), - ], - generate_audio: doubaoConfig.audio ?? false, - duration: doubaoConfig.duration, - resolution: doubaoConfig.aspectRatio, - watermark: false, + const authorization = "Bearer " + config.apiKey.replace(/^Bearer\s*/i, "").trim(); + const baseUrl = config.baseURL ?? "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks"; + + // 判断是否为首尾帧模式(需要两张图且类型支持首尾帧) + const isStartEndMode = images.length === 2 && hasStartEndType; + + // 构建图片内容 + const imageContent = images.map((base64, index) => { + const item: Record = { + type: "image_url", + image_url: { url: base64 }, + }; + if (isStartEndMode) { + item.role = index === 0 ? "first_frame" : "last_frame"; + } + return item; + }); + + // 构建请求体 + const requestBody: Record = { + model: config.model, + content: [{ type: "text", text: input.prompt }, ...imageContent], + duration: input.duration, + resolution: input.resolution, + watermark: false, + }; + + // 仅当模型支持音频时才添加 generate_audio 字段 + if (owned.audio) { + requestBody.generate_audio = input.audio ?? false; + } + // 创建视频生成任务 + const createResponse = await axios.post(baseUrl, requestBody, { + headers: { + "Content-Type": "application/json", + Authorization: authorization, }, - { headers: { "Content-Type": "application/json", Authorization: key } }, - ); - const taskId = createRes.data.id; + }); + + const taskId = createResponse.data.id; if (!taskId) throw new Error("视频任务创建失败"); + + // 轮询任务状态 return await pollTask(async () => { - const res = await axios.get(`${config.baseURL ?? "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks"}/${taskId}`, { - headers: { Authorization: key }, - }); - const { status, content } = res.data; - if (status === "succeeded") return { completed: true, imageUrl: content?.video_url }; - if (["failed", "cancelled", "expired"].includes(status)) return { completed: false, error: `任务${status}` }; - if (["queued", "running"].includes(status)) return { completed: false }; - return { completed: false, error: `未知状态: ${status}` }; + const { status, content } = ( + await axios.get(`${baseUrl}/${taskId}`, { + headers: { Authorization: authorization }, + }) + ).data; + + switch (status) { + case "succeeded": + return { completed: true, url: content?.video_url }; + case "failed": + case "cancelled": + case "expired": + return { completed: false, error: `任务${status}` }; + case "queued": + case "running": + return { completed: false }; + default: + return { completed: false, error: `未知状态: ${status}` }; + } }); }; diff --git a/src/utils/ai/video/owned/wan.ts b/src/utils/ai/video/owned/wan.ts new file mode 100644 index 0000000..e319ad6 --- /dev/null +++ b/src/utils/ai/video/owned/wan.ts @@ -0,0 +1,168 @@ +import "../type"; +import axios from "axios"; +import { pollTask, validateVideoConfig } from "@/utils/ai/utils"; + +// 根据分辨率档位和宽高比计算具体尺寸 +const getSizeFromConfig = (resolution: string, aspectRatio: string): string => { + const sizeMap: Record> = { + "480p": { + "16:9": "832*480", + "9:16": "480*832", + "1:1": "624*624", + }, + "720p": { + "16:9": "1280*720", + "9:16": "720*1280", + "1:1": "960*960", + "4:3": "1088*832", + "3:4": "832*1088", + }, + "1080p": { + "16:9": "1920*1080", + "9:16": "1080*1920", + "1:1": "1440*1440", + "4:3": "1632*1248", + "3:4": "1248*1632", + }, + }; + + const resolutionKey = resolution.toLowerCase(); + const size = sizeMap[resolutionKey]?.[aspectRatio]; + + if (!size) { + throw new Error(`不支持的分辨率(${resolution})和宽高比(${aspectRatio})组合`); + } + + return size; +}; + +export default async (input: VideoConfig, config: AIConfig) => { + if (!config.apiKey) throw new Error("缺少API Key"); + + const { owned, images, hasStartEndType, hasTextType } = validateVideoConfig(input, config); + + const defaultBaseUrl = [ + "https://dashscope.aliyuncs.com/api/v1/services/aigc/video-generation/video-synthesis", + "https://dashscope.aliyuncs.com/api/v1/services/aigc/image2video/video-synthesis", + "https://dashscope.aliyuncs.com/api/v1/tasks/{taskId}", + ].join("|"); + + const [i2vUrl, kf2vUrl, queryUrl] = (config.baseURL || defaultBaseUrl).split("|"); + + const types = owned.type; + const authorization = `Bearer ${config.apiKey}`; + + // 确定端点和构建请求体 + let submitUrl: string; + let body: Record; + + if (hasTextType && images.length === 0) { + // 文本生视频 + submitUrl = i2vUrl; + body = { + model: config.model, + input: { + prompt: input.prompt, + }, + parameters: { + size: getSizeFromConfig(input.resolution, input.aspectRatio), + duration: input.duration, + }, + }; + } else if (types.includes("singleImage")) { + // 图生视频 + submitUrl = i2vUrl; + body = { + model: config.model, + input: { + prompt: input.prompt, + img_url: images[0], + }, + parameters: { + resolution: input.resolution.toUpperCase(), + duration: input.duration, + }, + }; + // audio参数仅部分模型支持 + if (owned.audio && input.audio !== undefined) { + body.parameters.audio = input.audio; + } + } else if (hasStartEndType) { + // 首尾帧 + submitUrl = kf2vUrl; + const inputObj: Record = { + prompt: input.prompt, + first_frame_url: images[0], + }; + // 尾帧处理 + if (types.includes("startEndRequired")) { + inputObj.last_frame_url = images[1]; + } else if ((types.includes("endFrameOptional") || types.includes("startFrameOptional")) && images.length >= 2) { + inputObj.last_frame_url = images[1]; + } + body = { + model: config.model, + input: inputObj, + parameters: { + resolution: input.resolution.toUpperCase(), + duration: input.duration, + }, + }; + } else { + throw new Error(`不支持的视频生成类型: ${types.join(", ")}`); + } + + // 提交任务 + const submitResponse = await axios.post(submitUrl, body, { + headers: { + "Content-Type": "application/json", + Authorization: authorization, + "X-DashScope-Async": "enable", + }, + }); + + const submitData = submitResponse.data; + if (submitData.code) { + throw new Error(`任务提交失败: [${submitData.code}] ${submitData.message}`); + } + + const taskId = submitData.output?.task_id; + if (!taskId) { + throw new Error("任务提交失败: 未返回task_id"); + } + + // 轮询任务状态 + return await pollTask(async () => { + const response = await axios.get(queryUrl.replace("{taskId}", taskId), { + headers: { Authorization: authorization }, + }); + + const data = response.data; + + // 顶层错误 + if (data.code) { + return { completed: false, error: `[${data.code}] ${data.message}` }; + } + + const taskStatus = data.output?.task_status; + + switch (taskStatus) { + case "SUCCEEDED": + return { completed: true, url: data.output?.video_url }; + case "FAILED": + return { + completed: false, + error: `任务失败: [${data.output?.code || "UNKNOWN"}] ${data.output?.message || "未知错误"}`, + }; + case "CANCELED": + return { completed: false, error: "任务已取消" }; + case "UNKNOWN": + return { completed: false, error: "任务不存在或状态未知" }; + case "PENDING": + case "RUNNING": + return { completed: false }; + default: + return { completed: false, error: `未知状态: ${taskStatus}` }; + } + }); +}; diff --git a/src/utils/ai/video/type.ts b/src/utils/ai/video/type.ts new file mode 100644 index 0000000..1687c0b --- /dev/null +++ b/src/utils/ai/video/type.ts @@ -0,0 +1,15 @@ +interface VideoConfig { + duration: 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12; + resolution: "480p" | "720p" | "1080p" | "2K" | "4K"; + aspectRatio: "16:9" | "9:16"; + prompt: string; + savePath: string; + imageBase64?: string[]; + audio?: boolean; +} + +interface AIConfig { + model?: string; + apiKey?: string; + baseURL?: string; +}