diff --git a/src/agents/storyboard/generateImagePromptsTool.ts b/src/agents/storyboard/generateImagePromptsTool.ts index 4057221..cbff68c 100644 --- a/src/agents/storyboard/generateImagePromptsTool.ts +++ b/src/agents/storyboard/generateImagePromptsTool.ts @@ -104,9 +104,7 @@ async function generateGridPrompt(options: GridPromptOptions): Promise `- ${r.name}：${r.intro}`).join("\n")} 请仅选择在分镜中明确出现或被提及的角色、场景、道具。不要选择与分镜内容无关的资产。`, }, ], - responseFormat: { - type: "json_schema", - jsonSchema: { - name: "filteredAssets", - strict: true, - schema: z.toJSONSchema(filteredAssetsSchema), - }, + output: { + relevantAssets: z + .array( + z.object({ + name: z.string().describe("资产名称"), + reason: z.string().describe("选择该资产的原因"), + }), + ) + .describe("与分镜内容相关的资产列表"), }, }); + // const result = await chatModel!.invoke({ + // messages: [ + // { + // role: "user", + // content: `请分析以下分镜描述，从可用资产中筛选出与分镜内容直接相关的资产。 - const data = result?.json as z.infer; + // 分镜描述： + // ${prompts.map((p, i) => `${i + 1}. ${p}`).join("\n")} - if (!data?.relevantAssets || data.relevantAssets.length === 0) { + // 可用资产列表： + // ${availableResources.map((r) => `- ${r.name}：${r.intro}`).join("\n")} + + // 请仅选择在分镜中明确出现或被提及的角色、场景、道具。不要选择与分镜内容无关的资产。`, + // }, + // ], + // responseFormat: { + // type: "json_schema", + // jsonSchema: { + // name: "filteredAssets", + // strict: true, + // schema: z.toJSONSchema(filteredAssetsSchema), + // }, + // }, + // }); + + // const data = result?.json as z.infer; + + if (!relevantAssets || relevantAssets.length === 0) { return availableImages; } - const relevantNames = new Set(data.relevantAssets.map((a) => a.name)); + const relevantNames = new Set(relevantAssets.map((a) => a.name)); const filteredImages = availableImages.filter((img) => relevantNames.has(img.name)); return filteredImages.length > 0 ? filteredImages : availableImages; diff --git a/src/routes/assets/polishPrompt.ts b/src/routes/assets/polishPrompt.ts index 062e6df..0c6402b 100644 --- a/src/routes/assets/polishPrompt.ts +++ b/src/routes/assets/polishPrompt.ts @@ -1,7 +1,7 @@ import express from "express"; import u from "@/utils"; import * as zod from "zod"; -import { success } from "@/lib/responseFormat"; +import { error, success } from "@/lib/responseFormat"; import { validateFields } from "@/middleware/middleware"; const router = express.Router(); const jsonSchema = zod.object({ @@ -188,8 +188,7 @@ export default router.post( `; } async function generatePrompt() { - const model = await u.ai.text(); - const result = await model.invoke({ + const { prompt } = await u.ai.text.invoke({ messages: [ { role: "system", @@ -200,21 +199,41 @@ export default router.post( content: userPrompt, }, ], - responseFormat: { - type: "json_schema", - jsonSchema: { - name: "json", - strict: true, - schema: zod.toJSONSchema(jsonSchema), - }, + output: { + prompt: zod.string().describe("提示词"), }, }); - return result.json; + + // const result = await model.invoke({ + // messages: [ + // { + // role: "system", + // content: systemPrompt, + // }, + // { + // role: "user", + // content: userPrompt, + // }, + // ], + // responseFormat: { + // type: "json_schema", + // jsonSchema: { + // name: "json", + // strict: true, + // schema: zod.toJSONSchema(jsonSchema), + // }, + // }, + // }); + return prompt; } - const data = (await generatePrompt()) as any; + try { + const prompt = (await generatePrompt()) as any; + if (!prompt) return res.status(500).send("失败"); - if (!data.prompt) return res.status(500).send("失败"); - - res.status(200).send(success({ prompt: data.prompt, assetsId })); + res.status(200).send(success({ prompt: prompt, assetsId })); + } catch (e: any) { + console.log("%c Line:235 🥚 e", "background:#33a5ff", e); + return res.status(500).send(error(e?.data?.error?.message ?? e?.message ?? "生成失败")); + } }, ); diff --git a/src/routes/setting/getSetting.ts b/src/routes/setting/getSetting.ts index 7110d63..9763f58 100644 --- a/src/routes/setting/getSetting.ts +++ b/src/routes/setting/getSetting.ts @@ -15,27 +15,15 @@ export default router.post( const settingData = await u.db("t_setting").select("*"); - const configData = await u.db("t_config").where("userId", userId).select("*") ; + const configData = await u.db("t_config").where("userId", userId).select("*"); const parsedData = settingData.map((item) => ({ ...item, - imageModel: (() => { - try { - return JSON.parse(item.imageModel ?? "{}"); - } catch { - return null; - } - })(), - languageModel: (() => { - try { - return JSON.parse(item.languageModel ?? "{}"); - } catch { - return null; - } - })(), - videoModel: configData, + imageModel: configData.find((i) => i.type == "image"), + languageModel: configData.find((i) => i.type == "text"), + videoModel: configData.filter((i) => i.type == "video").filter(Boolean), })); res.status(200).send(success(parsedData)); - } + }, ); diff --git a/src/routes/setting/updateSetting.ts b/src/routes/setting/updateSetting.ts index 96f934b..e39e0e1 100644 --- a/src/routes/setting/updateSetting.ts +++ b/src/routes/setting/updateSetting.ts @@ -33,23 +33,47 @@ export default router.post( for (const item of videoModel) { await u.db("t_config").insert({ type: "video", - name: item.name, + name: item.model, model: item.model, apiKey: item.apiKey, baseUrl: item.baseUrl, - index: item.index, createTime: Date.now(), userId, manufacturer: item.manufacturer, }); } } - + if (languageModel) { + await u.db("t_config").where("type", "text").delete(); + await u.db("t_config").insert({ + type: "text", + name: languageModel.model, + model: languageModel.model, + apiKey: languageModel.apiKey, + baseUrl: languageModel.baseUrl, + createTime: Date.now(), + userId, + manufacturer: languageModel.manufacturer, + }); + } + if (imageModel) { + await u.db("t_config").where("type", "image").delete(); + await u.db("t_config").insert({ + type: "image", + name: imageModel.model, + model: imageModel.model, + apiKey: imageModel.apiKey, + baseUrl: imageModel.baseUrl, + createTime: Date.now(), + userId, + manufacturer: imageModel.manufacturer, + }); + } await u.db("t_user").where("id", userId).update({ name, password, }); res.status(200).send(success({ message: "修改全局配置成功" })); - } + }, ); diff --git a/src/routes/storyboard/generateVideoPrompt.ts b/src/routes/storyboard/generateVideoPrompt.ts index ff93ee4..0fa239a 100644 --- a/src/routes/storyboard/generateVideoPrompt.ts +++ b/src/routes/storyboard/generateVideoPrompt.ts @@ -127,12 +127,6 @@ async function generateSingleVideoPrompt({ if (ossPath.includes("http")) { imagePath = new URL(ossPath).pathname; } - - const model = await u.ai.text({}); - if (!model) { - throw new Error("无法获取语言模型，请检查语言模型配置"); - } - const messages: any[] = [ { role: "system", @@ -154,30 +148,27 @@ async function generateSingleVideoPrompt({ ]; try { - const result = await model.invoke({ + const result = await u.ai.text.invoke({ messages, - responseFormat: { - type: "json_schema", - jsonSchema: { - name: "json", - strict: true, - schema: z.toJSONSchema(cellsResultSchema), - }, + output: { + time: z.number().describe("时长,镜头时长 1-15"), + content: z.string().describe("提示词内容"), + name: z.string().describe("分镜名称"), }, }); + console.log("%c Line:156 🍩 result", "background:#33a5ff", result); - if (!result || !result.json) { + if (!result) { console.error("AI 返回结果为空:", result); throw new Error("AI 返回结果为空"); } - const json = result.json as { content: string; time: number; name: string }; - if (!json.content || json.time === undefined || !json.name) { - console.error("AI 返回格式错误:", result.json); + if (!result.content || result.time === undefined || !result.name) { + console.error("AI 返回格式错误:", result); throw new Error("AI 返回格式错误"); } - return json; + return result; } catch (err: any) { console.error("generateSingleVideoPrompt 调用失败:", err?.message || err); throw new Error(`生成视频提示词失败: ${err?.message || "未知错误"}`); diff --git a/src/routes/video/generatePrompt.ts b/src/routes/video/generatePrompt.ts index 0fc0a04..6dc9725 100644 --- a/src/routes/video/generatePrompt.ts +++ b/src/routes/video/generatePrompt.ts @@ -55,14 +55,12 @@ export default router.post( const { prompt, images, duration, type = "single" } = req.body; const mode = type as GenerateMode; - const model = await u.ai.text({}); - const imagePrompts = images.map((i: { filePath: string; prompt: string }, index: number) => `Image ${index + 1}: ${i.prompt}`).join("\n"); const shotCount = images.length; const avgDuration = (parseFloat(duration) / shotCount).toFixed(1); - const result = await model!.invoke({ + const result = await u.ai.text.invoke({ messages: [ { role: "system", @@ -87,6 +85,7 @@ Generate storyboard prompts:`, }, ], }); + console.log("%c Line:64 🥕 result", "background:#7f2b82", result.text); res.status(200).send(success(result.text)); }, diff --git a/src/utils.ts b/src/utils.ts index e9f14c6..cb43ce2 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -8,12 +8,15 @@ import getConfig from "./utils/getConfig"; import { v4 as uuid } from "uuid"; import AIText from "@/utils/ai/text"; - +import generateVideo from "@/utils/ai/generateVideo"; +import generateImage from "@/utils/ai/generateImage"; export default { db, oss, ai: { text: AIText, + generateVideo, + generateImage, }, editImage, number2Chinese, diff --git a/src/utils/ai.ts b/src/utils/ai.ts index a0488aa..c72198a 100644 --- a/src/utils/ai.ts +++ b/src/utils/ai.ts @@ -8,7 +8,7 @@ import sharp from "sharp"; axiosRetry(axios, { retries: 3, retryDelay: () => 200 }); export const text = async (config: OpenAIChatModelOptions = {}) => { - const { model, apiKey, baseURL } = await u.getConfig("language"); + const { model, apiKey, baseURL } = await u.getConfig("text"); return new OpenAIChatModel({ apiKey: apiKey ?? "", baseURL: baseURL ?? "", @@ -530,6 +530,7 @@ const generateVideoWithConfig = async (config: VideoConfig, configItem: { model: export const generateVideo = async (config: VideoConfig, manufacturer: string) => { if (!config.imageBase64 || config.imageBase64.length <= 0) throw new Error("未传图片"); const configList = await u.getConfig("video", manufacturer); + console.log("%c Line:533 🥔 configList", "background:#ea7e5c", configList); if (!configList || configList.length === 0) { throw new Error("未找到任何视频配置"); } diff --git a/src/utils/ai/generateImage.ts b/src/utils/ai/generateImage.ts new file mode 100644 index 0000000..16eb989 --- /dev/null +++ b/src/utils/ai/generateImage.ts @@ -0,0 +1,201 @@ +import axios from "axios"; +import u from "@/utils"; +import FormData from "form-data"; +import axiosRetry from "axios-retry"; +import sharp from "sharp"; + +interface ImageConfig { + systemPrompt?: string; + prompt: string; + imageBase64: string[]; + size: "1K" | "2K" | "4K"; + aspectRatio: string; + resType?: "url" | "b64"; +} + +interface ImageModelConfig { + model?: string; + apiKey?: string; + baseURL?: string; + manufacturer?: "openAi" | "gemini" | "volcengine" | "runninghub" | "apimart"; +} +// 上传 base64 图片到 runninghub +const uploadBase64ToRunninghub = async (base64Image: string, apiKey: string, baseURL: string): Promise => { + try { + apiKey = apiKey.replace("Bearer ", ""); + // 移除 base64 前缀 + const base64Data = base64Image.replace(/^data:image\/\w+;base64,/, ""); + let buffer = Buffer.from(base64Data, "base64"); + + // 压缩图片到 5MB 以下 + const MAX_SIZE = 5 * 1024 * 1024; // 5MB + if (buffer.length > MAX_SIZE) { + let quality = 90; + + while (buffer.length > MAX_SIZE && quality > 10) { + const compressed = await sharp(buffer).jpeg({ quality, mozjpeg: true }).toBuffer(); + buffer = Buffer.from(compressed); + quality -= 10; + } + + // 如果仍然超过限制，进一步调整尺寸 + if (buffer.length > MAX_SIZE) { + const metadata = await sharp(buffer).metadata(); + const scale = Math.sqrt(MAX_SIZE / buffer.length); + + const resized = await sharp(buffer) + .resize({ + width: Math.floor((metadata.width || 1920) * scale), + height: Math.floor((metadata.height || 1080) * scale), + fit: "inside", + }) + .jpeg({ quality: 80, mozjpeg: true }) + .toBuffer(); + + buffer = Buffer.from(resized); + } + } + + // 创建 FormData + const formData = new FormData(); + formData.append("file", buffer, { + filename: "image.jpg", + contentType: "image/jpeg", + }); + + // 上传图片 + const uploadRes = await axios.post(`https://www.runninghub.cn/openapi/v2/media/upload/binary`, formData, { + headers: { Authorization: `Bearer ${apiKey}` }, + }); + + if (uploadRes.data.code !== 0 || !uploadRes.data.data?.download_url) { + throw new Error(`图片上传失败: ${JSON.stringify(uploadRes.data)}`); + } + + return uploadRes.data.data.download_url; + } catch (error) { + console.error("上传图片时发生错误:", error); + throw error; + } +}; +const urlToBase64 = async (url: string): Promise => { + const res = await axios.get(url, { responseType: "arraybuffer" }); + const base64 = Buffer.from(res.data).toString("base64"); + const mimeType = res.headers["content-type"] || "image/png"; + return `data:${mimeType};base64,${base64}`; +}; +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const pollTask = async ( + queryFn: () => Promise<{ completed: boolean; imageUrl?: string; error?: string }>, + maxAttempts = 500, + interval = 2000, +): Promise => { + for (let i = 0; i < maxAttempts; i++) { + await sleep(interval); + const { completed, imageUrl, error } = await queryFn(); + if (error) throw new Error(error); + if (completed && imageUrl) return imageUrl; + } + throw new Error(`任务轮询超时，已尝试 ${maxAttempts} 次`); +}; + +const generators = { + volcengine: async (config: ImageConfig, apiKey: string, baseURL: string, model: string) => { + if (config.size == "1K") config.size = "2K"; + apiKey = apiKey.replace("Bearer ", ""); + const body: Record = { + model, + prompt: config.prompt, + size: config.size, + response_format: "url", + sequential_image_generation: "disabled", + stream: false, + watermark: false, + }; + // 图生图：存在图片时添加 image 字段 + if (config.imageBase64) { + body.image = config.imageBase64; + } + const res = await axios.post(`https://ark.cn-beijing.volces.com/api/v3/images/generations`, body, { + headers: { Authorization: `Bearer ${apiKey}` }, + }); + return res.data.data[0].url; + }, + + gemini: async (config: ImageConfig, apiKey: string, baseURL: string, model: string) => { + apiKey = apiKey.replace("Bearer ", ""); + const messages = [ + ...(config.systemPrompt ? [{ role: "system", content: config.systemPrompt }] : []), + { role: "user", content: config.prompt }, + ...config.imageBase64.map((img) => ({ role: "user", content: { image: img } })), + ]; + const res = await axios.post( + `${baseURL}/chat/completions`, + { model, stream: false, messages, extra_body: { google: { image_config: { aspect_ratio: config.aspectRatio, image_size: config.size } } } }, + { headers: { Authorization: "Bearer " + apiKey } }, + ); + + return res.data.choices[0].message.content; + }, + + runninghub: async (config: ImageConfig, apiKey: string, baseURL: string) => { + apiKey = apiKey.replace("Bearer ", ""); + const imageUrls = await Promise.all(config.imageBase64.map((base64Image) => uploadBase64ToRunninghub(base64Image, apiKey, baseURL))); + + const endpoint = config.imageBase64.length === 0 ? "/openapi/v2/rhart-image-n-pro/text-to-image" : "/openapi/v2/rhart-image-n-pro/edit"; + const taskRes = await axios.post( + `https://www.runninghub.cn${endpoint}`, + { prompt: config.prompt, resolution: config.size, aspectRatio: config.aspectRatio, ...(imageUrls.length > 0 && { imageUrls }) }, + { headers: { Authorization: "Bearer " + apiKey } }, + ); + const taskId = taskRes.data.taskId; + if (!taskId) throw new Error(`任务创建失败，${JSON.stringify(taskRes.data)}`); + + return pollTask(async () => { + const res = await axios.post(`https://www.runninghub.cn/task/openapi/outputs`, { taskId, apiKey: apiKey }); + const { code, msg, data } = res.data; + if (code === 0 && msg === "success") return { completed: true, imageUrl: data?.[0]?.fileUrl }; + if (code === 804 || code === 813) return { completed: false }; + if (code === 805) return { completed: false, error: `任务失败: ${data?.[0]?.failedReason?.exception_message || "未知原因"}` }; + return { completed: false, error: `未知状态: code=${code}, msg=${msg}` }; + }); + }, + + apimart: async (config: ImageConfig, apiKey: string, baseURL: string, model: string) => { + apiKey = apiKey.replace("Bearer ", ""); + const taskRes = await axios.post( + `https://api.apimart.ai/v1/images/generations`, + { model: "gemini-3-pro-image-preview", prompt: config.prompt, size: config.aspectRatio, n: 1, resolution: config.size }, + { headers: { Authorization: apiKey } }, + ); + + if (taskRes.data.code !== 200 || !taskRes.data.data?.[0]?.task_id) throw new Error("任务创建失败: " + JSON.stringify(taskRes.data)); + + const taskId = taskRes.data.data[0].task_id; + return pollTask(async () => { + const res = await axios.get(`https://api.apimart.ai/v1/tasks/${taskId}`, { headers: { Authorization: apiKey }, params: { language: "en" } }); + if (res.data.code !== 200) return { completed: false, error: `查询失败: ${JSON.stringify(res.data)}` }; + const { status, result } = res.data.data; + if (status === "completed") return { completed: true, imageUrl: result?.images?.[0]?.url?.[0] }; + if (status === "failed" || status === "cancelled") return { completed: false, error: `任务${status}` }; + return { completed: false }; + }); + }, +}; +export default async (config: ImageConfig, replaceConfig?: ImageModelConfig) => { + let { model, apiKey, baseURL, manufacturer } = await u.getConfig("image"); + if (replaceConfig) { + model = replaceConfig.model || model; + apiKey = replaceConfig.apiKey || apiKey; + baseURL = replaceConfig.baseURL || baseURL; + manufacturer = replaceConfig.manufacturer || manufacturer; + } + const generator = generators[manufacturer as keyof typeof generators]; + if (!generator) throw new Error(`不支持的厂商: ${manufacturer}`); + + let imageUrl = await generator(config, apiKey ?? "", baseURL ?? "", model ?? ""); + if (!config.resType) config.resType = "b64"; + if (config.resType === "b64" && imageUrl.startsWith("http")) imageUrl = await urlToBase64(imageUrl); + return imageUrl; +}; diff --git a/src/utils/ai/generateVideo.ts b/src/utils/ai/generateVideo.ts new file mode 100644 index 0000000..d82e6b1 --- /dev/null +++ b/src/utils/ai/generateVideo.ts @@ -0,0 +1,438 @@ +import axios from "axios"; +import u from "@/utils"; +import FormData from "form-data"; +import axiosRetry from "axios-retry"; +import sharp from "sharp"; + +type VideoAspectRatio = "16:9" | "9:16" | "1:1" | "4:3" | "3:4" | "21:9" | "adaptive"; +interface BaseVideoConfig { + prompt: string; + savePath: string; + imageBase64?: string[]; // 单张参考图片 base64 +} +interface DoubaoVideoConfig extends BaseVideoConfig { + duration: 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12; // 支持 2~12 秒 + aspectRatio: VideoAspectRatio; + audio?: boolean; +} +interface RunninghubVideoConfig extends BaseVideoConfig { + duration: 10 | 15; // 仅支持 10 或 15 秒 + aspectRatio: "16:9" | "9:16" | "1:1"; // 仅支持这三种比例 +} +interface OpenAIVideoConfig extends BaseVideoConfig { + duration: 10 | 15; // 仅支持 10 或 15 秒 + aspectRatio: Exclude; // 不支持 adaptive +} +type VideoConfig = DoubaoVideoConfig | RunninghubVideoConfig | OpenAIVideoConfig; + +const urlToBase64 = async (url: string): Promise => { + const res = await axios.get(url, { responseType: "arraybuffer" }); + const base64 = Buffer.from(res.data).toString("base64"); + const mimeType = res.headers["content-type"] || "image/png"; + return `data:${mimeType};base64,${base64}`; +}; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const pollTask = async ( + queryFn: () => Promise<{ completed: boolean; imageUrl?: string; error?: string }>, + maxAttempts = 500, + interval = 2000, +): Promise => { + for (let i = 0; i < maxAttempts; i++) { + await sleep(interval); + const { completed, imageUrl, error } = await queryFn(); + if (error) throw new Error(error); + if (completed && imageUrl) return imageUrl; + } + throw new Error(`任务轮询超时，已尝试 ${maxAttempts} 次`); +}; + +// 上传 base64 图片到 runninghub +const uploadBase64ToRunninghub = async (base64Image: string, apiKey: string, baseURL: string): Promise => { + try { + apiKey = apiKey.replace("Bearer ", ""); + // 移除 base64 前缀 + const base64Data = base64Image.replace(/^data:image\/\w+;base64,/, ""); + let buffer = Buffer.from(base64Data, "base64"); + + // 压缩图片到 5MB 以下 + const MAX_SIZE = 5 * 1024 * 1024; // 5MB + if (buffer.length > MAX_SIZE) { + let quality = 90; + + while (buffer.length > MAX_SIZE && quality > 10) { + const compressed = await sharp(buffer).jpeg({ quality, mozjpeg: true }).toBuffer(); + buffer = Buffer.from(compressed); + quality -= 10; + } + + // 如果仍然超过限制，进一步调整尺寸 + if (buffer.length > MAX_SIZE) { + const metadata = await sharp(buffer).metadata(); + const scale = Math.sqrt(MAX_SIZE / buffer.length); + + const resized = await sharp(buffer) + .resize({ + width: Math.floor((metadata.width || 1920) * scale), + height: Math.floor((metadata.height || 1080) * scale), + fit: "inside", + }) + .jpeg({ quality: 80, mozjpeg: true }) + .toBuffer(); + + buffer = Buffer.from(resized); + } + } + + // 创建 FormData + const formData = new FormData(); + formData.append("file", buffer, { + filename: "image.jpg", + contentType: "image/jpeg", + }); + + // 上传图片 + const uploadRes = await axios.post(`https://www.runninghub.cn/openapi/v2/media/upload/binary`, formData, { + headers: { Authorization: `Bearer ${apiKey}` }, + }); + + if (uploadRes.data.code !== 0 || !uploadRes.data.data?.download_url) { + throw new Error(`图片上传失败: ${JSON.stringify(uploadRes.data)}`); + } + + return uploadRes.data.data.download_url; + } catch (error) { + console.error("上传图片时发生错误:", error); + throw error; + } +}; + +const generateVideoWithConfig = async (config: VideoConfig, configItem: { model: string; apiKey: string; baseURL: string; manufacturer: string }) => { + const { apiKey, baseURL, manufacturer, model } = configItem; + const imageArrPath = []; + for (const imageVal of config?.imageBase64!) { + // 判断是否为base64串 + const isBase64 = typeof imageVal === "string" && /^data:image\/[a-zA-Z0-9\+\-\.]+;base64,[\s\S]+$/.test(imageVal.trim()); + if (isBase64) { + imageArrPath.push(imageVal); + } else { + const base64 = await urlToBase64(imageVal); + imageArrPath.push(base64); + } + } + config.imageBase64 = imageArrPath; + let videoUrl: string | null = null; + if (manufacturer === "volcengine") { + const doubaoConfig = config as DoubaoVideoConfig; + const createRes = await axios.post( + baseURL ?? "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks", + { + model: "doubao-seedance-1-5-pro-251215", + content: [ + { type: "text", text: config.prompt }, + ...(doubaoConfig.imageBase64 + ? doubaoConfig.imageBase64.map((base64, i) => ({ + type: "image_url", + image_url: { url: base64 }, + role: i === 0 ? "first_frame" : "last_frame", + })) + : []), + ], + generate_audio: doubaoConfig.audio ?? false, + duration: doubaoConfig.duration, + resolution: doubaoConfig.aspectRatio, + watermark: false, + }, + { headers: { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}` } }, + ); + const taskId = createRes.data.id; + if (!taskId) throw new Error("视频任务创建失败"); + videoUrl = await pollTask(async () => { + const res = await axios.get(`${baseURL ?? "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks"}/${taskId}`, { + headers: { Authorization: `Bearer ${apiKey}` }, + }); + const { status, content } = res.data; + if (status === "succeeded") return { completed: true, imageUrl: content?.video_url }; + if (["failed", "cancelled", "expired"].includes(status)) return { completed: false, error: `任务${status}` }; + if (["queued", "running"].includes(status)) return { completed: false }; + return { completed: false, error: `未知状态: ${status}` }; + }); + } else if (manufacturer === "runninghub") { + const runninghubConfig = config as RunninghubVideoConfig; + // 如果有图片，先上传 + let uploadedImageUrl: string | undefined; + if (runninghubConfig.imageBase64 && runninghubConfig.imageBase64.length > 0) { + uploadedImageUrl = await uploadBase64ToRunninghub(runninghubConfig.imageBase64[0]!, apiKey ?? "", "https://www.runninghub.cn"); + } + + const endpoint = uploadedImageUrl ? "/openapi/v2/rhart-video-s/image-to-video" : "/openapi/v2/rhart-video-s/text-to-video"; + const requestBody = uploadedImageUrl + ? { + prompt: config.prompt, + imageUrl: uploadedImageUrl, + duration: String(runninghubConfig.duration) as "10" | "15", + aspectRatio: runninghubConfig.aspectRatio, + } + : { prompt: config.prompt, model }; + const createRes = await axios.post(`https://www.runninghub.cn${endpoint}`, requestBody, { + headers: { Authorization: "Bearer " + apiKey, "Content-Type": "application/json" }, + }); + + const { taskId, status: initialStatus, errorMessage } = createRes.data; + if (!taskId) throw new Error(`视频任务创建失败: ${errorMessage || "未知错误"}`); + if (initialStatus === "FAILED") throw new Error(`任务创建失败: ${errorMessage}`); + videoUrl = await pollTask(async () => { + const res = await axios.post( + `https://www.runninghub.cn/task/openapi/outputs`, + { apiKey: apiKey?.replace("Bearer ", ""), taskId }, + { headers: { Authorization: "Bearer " + apiKey } }, + ); + const { code, msg, data } = res.data; + + // 成功完成 + if (code === 0 && msg === "success" && data?.[0]?.fileUrl) { + return { completed: true, imageUrl: data[0].fileUrl }; + } + + // 进行中 + if (code === 804 || code === 813) { + return { completed: false }; + } + + // 失败 + if (code === 805) { + const failedReason = data?.[0]?.failedReason; + let errorMsg = "未知原因"; + + if (failedReason) { + // 尝试多种可能的错误信息字段 + errorMsg = + failedReason.exception_message || + failedReason.exceptionMessage || + failedReason.message || + failedReason.reason || + JSON.stringify(failedReason); + } + + return { + completed: false, + error: `任务失败: ${errorMsg}`, + }; + } + + // 其他未知状态 + return { + completed: false, + error: `未知状态: code=${code}, msg=${msg}, data=${JSON.stringify(data)}`, + }; + }); + } else if (manufacturer === "openAi") { + const openaiConfig = config as OpenAIVideoConfig; + // 如果有图片，先上传 + let uploadedImageUrl: string | undefined; + if (openaiConfig.imageBase64 && openaiConfig.imageBase64.length) { + const base64Data = openaiConfig.imageBase64[0]!.replace(/^data:image\/\w+;base64,/, ""); + const buffer = Buffer.from(base64Data, "base64"); + const formData = new FormData(); + formData.append("file", buffer, { filename: "image.jpg", contentType: "image/jpeg" }); + const uploadRes = await axios.post(`${baseURL}/videos`, formData, { + headers: { + Authorization: `Bearer ${apiKey}`, + ...formData.getHeaders(), + }, + }); + uploadedImageUrl = uploadRes.data?.id || uploadRes.data?.url; + } + + // 创建视频生成任务 + const formData = new FormData(); + formData.append("model", model); + formData.append("prompt", config.prompt); + formData.append("seconds", String(openaiConfig.duration)); + + // 根据 aspectRatio 设置 size + const sizeMap: Record = { + "16:9": "1920x1080", + "9:16": "1080x1920", + "1:1": "1080x1080", + "4:3": "1440x1080", + "3:4": "1080x1440", + "21:9": "2560x1080", + }; + formData.append("size", sizeMap[openaiConfig.aspectRatio] || "1920x1080"); + if (uploadedImageUrl) { + formData.append("input_reference", uploadedImageUrl); + } + const createRes = await axios.post(`${baseURL}/videos`, formData, { + headers: { + Authorization: `Bearer ${apiKey}`, + ...formData.getHeaders(), + }, + }); + + const taskId = createRes.data?.id; + + if (!taskId) throw new Error("视频任务创建失败"); + // 轮询任务状态 + videoUrl = await pollTask(async () => { + const res = await axios.get(`${baseURL}/videos/${taskId}`, { + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + }); + const { status, imageUrl, failReason } = res.data; + if (status === "SUCCESS") return { completed: true, imageUrl }; + if (status === "FAILURE" || status === "CANCEL") { + return { completed: false, error: `任务${status}: ${failReason || "未知原因"}` }; + } + if (["NOT_START", "SUBMITTED", "IN_PROGRESS", "MODAL"].includes(status)) { + return { completed: false }; + } + return { completed: false, error: `未知状态: ${status}` }; + }); + } else if (manufacturer === "apimart") { + // apimart 视频生成 + const apimartConfig = config as OpenAIVideoConfig; + const apimartBaseURL = "https://api.apimart.ai"; + + // 上传图片到 apimart 图床 + let imageUrls: string[] = []; + if (apimartConfig.imageBase64 && apimartConfig.imageBase64.length > 0) { + for (const base64Image of apimartConfig.imageBase64) { + // 如果已经是 URL，直接使用 + if (base64Image.startsWith("http")) { + imageUrls.push(base64Image); + continue; + } + + // 获取预签名 URL + const presignRes = await axios.post( + "https://apimart.ai/api/upload/presign", + { contentType: "image/jpeg", fileExtension: "jpeg", permanent: false }, + { headers: { "Content-Type": "application/json" } }, + ); + + if (!presignRes.data.success || !presignRes.data.presignedUrl || !presignRes.data.cdnUrl) { + throw new Error(`获取预签名 URL 失败: ${JSON.stringify(presignRes.data)}`); + } + + const { presignedUrl, cdnUrl } = presignRes.data; + + // 移除 base64 前缀并转为 buffer + const base64Data = base64Image.replace(/^data:image\/\w+;base64,/, ""); + const buffer = Buffer.from(base64Data, "base64"); + + // 上传图片到预签名 URL + await axios.put(presignedUrl, buffer, { + headers: { "Content-Type": "image/jpeg" }, + }); + + imageUrls.push(cdnUrl); + } + } + + // 创建视频生成任务 + const requestBody: { + model: string; + prompt: string; + duration: number; + aspect_ratio: string; + image_urls?: string[]; + } = { + model: model || "sora-2", + prompt: config.prompt, + duration: apimartConfig.duration, + aspect_ratio: apimartConfig.aspectRatio, + }; + + if (imageUrls.length > 0) { + requestBody.image_urls = imageUrls; + } + + const createRes = await axios.post(`${apimartBaseURL}/v1/videos/generations`, requestBody, { + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + }); + + if (createRes.data.code !== 200 || !createRes.data.data?.[0]?.task_id) { + const errorMsg = createRes.data.error?.message || JSON.stringify(createRes.data); + throw new Error(`视频任务创建失败: ${errorMsg}`); + } + + const taskId = createRes.data.data[0].task_id; + + // 轮询任务状态 + videoUrl = await pollTask(async () => { + const res = await axios.get(`${apimartBaseURL}/v1/tasks/${taskId}`, { + headers: { Authorization: `Bearer ${apiKey}` }, + params: { language: "en" }, + }); + + // 检查是否有错误 + if (res.data.error) { + return { + completed: false, + error: `查询失败: ${res.data.error.message || JSON.stringify(res.data.error)}`, + }; + } + + if (res.data.code !== 200) { + return { completed: false, error: `查询失败: ${JSON.stringify(res.data)}` }; + } + + const { status, result } = res.data.data; + + if (status === "completed") { + // 获取视频 URL + const videoUrlResult = result?.videos?.[0]?.url?.[0]; + return { completed: true, imageUrl: videoUrlResult }; + } + + if (status === "failed" || status === "cancelled") { + return { completed: false, error: `任务${status}` }; + } + + // 其他状态（submitted, processing 等）继续轮询 + return { completed: false }; + }); + } else { + throw new Error(`不支持的厂商: ${manufacturer}`); + } + return videoUrl; +}; + +export default async (config: VideoConfig, manufacturer: string) => { + if (!config.imageBase64 || config.imageBase64.length <= 0) throw new Error("未传图片"); + const configItem = await u.getConfig("video", manufacturer); + if (!configItem) { + throw new Error("未找到任何视频配置"); + } + let lastError: Error | null = null; + // for (const configItem of configList) { + // 每个配置项重试1次，共2次尝试 + for (let attempt = 0; attempt < 2; attempt++) { + try { + const videoUrl = await generateVideoWithConfig(config, configItem); + if (videoUrl) { + const response = await axios.get(videoUrl, { responseType: "stream" }); + await u.oss.writeFile(config.savePath, response.data); + return config.savePath; + } + return videoUrl; + } catch (error: any) { + lastError = error as Error; + console.warn(`配置 ${configItem.model} 第 ${attempt + 1} 次尝试失败:`, error?.response?.data || error.message); + // 如果是第一次尝试失败，继续重试 + if (attempt === 0) continue; + // 第二次也失败了,跳到下一个配置项 + break; + } + } + // } + // 所有配置都失败了 + throw new Error(`所有视频配置都失败了。最后一次错误: ${lastError?.message || "未知错误"}`); +}; diff --git a/src/utils/ai/text.ts b/src/utils/ai/text.ts index 29d4ead..f96d75b 100644 --- a/src/utils/ai/text.ts +++ b/src/utils/ai/text.ts @@ -20,7 +20,8 @@ interface AIConfig { } const buildOptions = async (input: AIInput, config: AIConfig) => { - const sqlTextModelConfig = await u.getConfig("text"); + let sqlTextModelConfig = {}; + if (!config || !config?.model || !config?.apiKey || !config?.baseURL) sqlTextModelConfig = await u.getConfig("text"); const { model, apiKey, baseURL } = { ...sqlTextModelConfig, ...config }; const owned = modelList.find((m) => m.model === model); @@ -42,11 +43,11 @@ const buildOptions = async (input: AIInput, config: AIConfig) => { }, }; - const output = input.output ? outputBuilders[owned.responseFormat]?.(input.output) ?? null : null; + const output = input.output ? (outputBuilders[owned.responseFormat]?.(input.output) ?? null) : null; return { config: { - model: modelInstance(model) as LanguageModel, + model: modelInstance(model!) as LanguageModel, ...(input.system && { system: input.system }), ...(input.prompt ? { prompt: input.prompt } : { messages: input.messages! }), ...(input.tools && owned.tool && { tools: input.tools }), diff --git a/src/utils/generateScript.ts b/src/utils/generateScript.ts index 1323102..ae40ab7 100644 --- a/src/utils/generateScript.ts +++ b/src/utils/generateScript.ts @@ -130,8 +130,7 @@ ${novelData}`; const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出AI配置异常"; - const model = await u.ai.text(); - const result = await model.invoke({ + const result = await u.ai.text.invoke({ messages: [ { role: "system", content: mainPrompts }, { role: "user", content: userPrompt }, diff --git a/src/utils/getConfig.ts b/src/utils/getConfig.ts index 3bf3859..8825ea6 100644 --- a/src/utils/getConfig.ts +++ b/src/utils/getConfig.ts @@ -14,6 +14,7 @@ interface TextResData extends BaseConfig { } interface ImageResData extends BaseConfig { + baseURL: string; manufacturer: "openAi" | "gemini" | "volcengine" | "runninghub" | "apimart"; } @@ -34,10 +35,18 @@ const errorMessages: Record = { video: "视频模型配置不存在", }; -const needBaseURL: AIType[] = ["text", "video"]; +const needBaseURL: AIType[] = ["text", "video", "image"]; -export default async function getConfig(aiType: T): Promise { - const config = await u.db("t_config").where("type", aiType).first(); +export default async function getConfig(aiType: T, manufacturer?: string): Promise { + const config = await u + .db("t_config") + .where("type", aiType) + .modify((qb) => { + if (manufacturer) { + qb.where("manufacturer", manufacturer); + } + }) + .first(); if (!config) throw new Error(errorMessages[aiType]);