diff --git a/backup/agents/storyboard/generateImagePromptsTool.ts b/backup/agents/storyboard/generateImagePromptsTool.ts index 4057221..7c99dfa 100644 --- a/backup/agents/storyboard/generateImagePromptsTool.ts +++ b/backup/agents/storyboard/generateImagePromptsTool.ts @@ -104,9 +104,7 @@ async function generateGridPrompt(options: GridPromptOptions): Promise `- ${r.name}：${r.intro}`).join("\n")} 请仅选择在分镜中明确出现或被提及的角色、场景、道具。不要选择与分镜内容无关的资产。`, }, ], - responseFormat: { - type: "json_schema", - jsonSchema: { - name: "filteredAssets", - strict: true, - schema: z.toJSONSchema(filteredAssetsSchema), - }, + output: { + relevantAssets: z + .array( + z.object({ + name: z.string().describe("资产名称"), + reason: z.string().describe("选择该资产的原因"), + }), + ) + .describe("与分镜内容相关的资产列表"), }, }); - const data = result?.json as z.infer; - if (!data?.relevantAssets || data.relevantAssets.length === 0) { + if (!result?.relevantAssets || result.relevantAssets.length === 0) { return availableImages; } - const relevantNames = new Set(data.relevantAssets.map((a) => a.name)); + const relevantNames = new Set(result.relevantAssets.map((a) => a.name)); const filteredImages = availableImages.filter((img) => relevantNames.has(img.name)); return filteredImages.length > 0 ? filteredImages : availableImages; @@ -318,7 +318,7 @@ export default async (cells: { prompt: string }[], scriptId: number, projectId: const processedImages = await processImages(filteredImages); - const contentStr = await u.ai.generateImage({ + const contentStr = await u.ai.image({ systemPrompt: resourcesMapPrompts, prompt: prompts, size: "4K", diff --git a/src/agents/storyboard/generateImageTool.ts b/src/agents/storyboard/generateImageTool.ts index e43e20a..8cf9f79 100644 --- a/src/agents/storyboard/generateImageTool.ts +++ b/src/agents/storyboard/generateImageTool.ts @@ -343,7 +343,7 @@ export default async (cells: { prompt: string }[], scriptId: number, projectId: const processedImages = await processImages(filteredImages); - const contentStr = await u.ai.generateImage({ + const contentStr = await u.ai.image({ systemPrompt: resourcesMapPrompts, prompt: prompts, size: "4K", diff --git a/src/routes/assets/generateAssets.ts b/src/routes/assets/generateAssets.ts index 2243b4f..53a3310 100644 --- a/src/routes/assets/generateAssets.ts +++ b/src/routes/assets/generateAssets.ts @@ -124,7 +124,7 @@ export default router.post( assetsId: id, }); - const contentStr = await u.ai.generateImage({ + const contentStr = await u.ai.image({ systemPrompt, prompt: userPrompt, imageBase64: base64 ? [base64] : [], diff --git a/src/routes/storyboard/batchSuperScoreImage.ts b/src/routes/storyboard/batchSuperScoreImage.ts index 846974b..f10f97d 100644 --- a/src/routes/storyboard/batchSuperScoreImage.ts +++ b/src/routes/storyboard/batchSuperScoreImage.ts @@ -22,7 +22,7 @@ async function superResolutionAndSave( projectId: number, videoRatio: string, ): Promise<{ ossPath: string; base64: string }> { - const contentStr = await u.ai.generateImage({ + const contentStr = await u.ai.image({ aspectRatio: videoRatio, size: "1K", resType: "b64", diff --git a/src/routes/storyboard/chatStoryboard.ts b/src/routes/storyboard/chatStoryboard.ts index 048563f..e2854bc 100644 --- a/src/routes/storyboard/chatStoryboard.ts +++ b/src/routes/storyboard/chatStoryboard.ts @@ -8,7 +8,6 @@ expressWs(router as unknown as Application); router.ws("/", async (ws, req) => { let agent: Storyboard; - const config = await u.getConfig("language"); const projectId = req.query.projectId; const scriptId = req.query.scriptId; @@ -20,10 +19,6 @@ router.ws("/", async (ws, req) => { agent = new Storyboard(Number(projectId), Number(scriptId)); - agent.modelName = config.model ?? ""; - agent.baseURL = config.baseURL ?? ""; - agent.apiKey = config.apiKey ?? ""; - const existing = await u .db("t_chatHistory") .where({ projectId: Number(projectId) }) diff --git a/src/routes/video/addVideoConfig.ts b/src/routes/video/addVideoConfig.ts index 7722637..e055144 100644 --- a/src/routes/video/addVideoConfig.ts +++ b/src/routes/video/addVideoConfig.ts @@ -6,11 +6,13 @@ import { z } from "zod"; const router = express.Router(); // 图片项schema -const imageItemSchema = z.object({ - id: z.number(), - filePath: z.string(), - prompt: z.string().optional(), -}).nullable(); +const imageItemSchema = z + .object({ + id: z.number(), + filePath: z.string(), + prompt: z.string().optional(), + }) + .nullable(); // 新增视频配置 export default router.post( @@ -22,31 +24,24 @@ export default router.post( mode: z.enum(["startEnd", "multi", "single"]), startFrame: imageItemSchema.optional(), endFrame: imageItemSchema.optional(), - images: z.array(z.object({ - id: z.number(), - filePath: z.string(), - prompt: z.string().optional(), - })).optional(), + images: z + .array( + z.object({ + id: z.number(), + filePath: z.string(), + prompt: z.string().optional(), + }), + ) + .optional(), resolution: z.string(), duration: z.number(), prompt: z.string().optional(), }), async (req, res) => { - const { - scriptId, - projectId, - manufacturer, - mode, - startFrame, - endFrame, - images, - resolution, - duration, - prompt - } = req.body; + const { scriptId, projectId, manufacturer, mode, startFrame, endFrame, images, resolution, duration, prompt } = req.body; // 生成新ID - const maxIdResult = await u.db("t_videoConfig").max("id as maxId").first(); + const maxIdResult: any = await u.db("t_videoConfig").max("id as maxId").first(); const newId = (maxIdResult?.maxId || 0) + 1; const now = Date.now(); @@ -68,23 +63,25 @@ export default router.post( updateTime: now, }); - res.status(200).send(success({ - message: "新增视频配置成功", - data: { - id: newId, - scriptId, - projectId, - manufacturer, - mode, - startFrame, - endFrame, - images: images || [], - resolution, - duration, - prompt: prompt || "", - selectedResultId: null, - createdAt: new Date(now).toISOString(), - } - })); + res.status(200).send( + success({ + message: "新增视频配置成功", + data: { + id: newId, + scriptId, + projectId, + manufacturer, + mode, + startFrame, + endFrame, + images: images || [], + resolution, + duration, + prompt: prompt || "", + selectedResultId: null, + createdAt: new Date(now).toISOString(), + }, + }), + ); }, ); diff --git a/src/routes/video/generateVideo.ts b/src/routes/video/generateVideo.ts index a2fdd75..ae18410 100644 --- a/src/routes/video/generateVideo.ts +++ b/src/routes/video/generateVideo.ts @@ -44,7 +44,7 @@ export default router.post( // 过滤掉空值 let fileUrl = filePath.filter((p: string) => p && p.trim() !== ""); - + if (fileUrl.length === 0) { return res.status(400).send(error("请至少选择一张图片")); } @@ -149,17 +149,14 @@ ${prompt} 3. 关键人物在画面中全部清晰显示，不得被遮挡、缺失或省略 4. 画面真实、细致，无畸形、无模糊、无杂物、无多余人物、无文字、水印、logo `; - - const videoPath = await u.ai.generateVideo( - { - imageBase64, - savePath, - prompt: inputPrompt, - duration: duration as any, - aspectRatio: resolution as any, - }, - type!, - ); + const videoPath = await u.ai.video({ + imageBase64, + savePath, + prompt: inputPrompt, + duration: duration as any, + aspectRatio: resolution as any, + resolution: resolution as any, + }); if (videoPath) { // 生成成功，更新状态为 1 diff --git a/src/routes/video/upDateVideoConfig.ts b/src/routes/video/upDateVideoConfig.ts index b690d1d..ea58c6f 100644 --- a/src/routes/video/upDateVideoConfig.ts +++ b/src/routes/video/upDateVideoConfig.ts @@ -47,24 +47,29 @@ export default router.post( // 获取更新后的数据 const updatedConfig = await u.db("t_videoConfig").where({ id }).first(); - - res.status(200).send(success({ - message: "更新视频配置成功", - data: { - id: updatedConfig.id, - scriptId: updatedConfig.scriptId, - projectId: updatedConfig.projectId, - manufacturer: updatedConfig.manufacturer, - mode: updatedConfig.mode, - startFrame: updatedConfig.startFrame ? JSON.parse(updatedConfig.startFrame) : null, - endFrame: updatedConfig.endFrame ? JSON.parse(updatedConfig.endFrame) : null, - images: updatedConfig.images ? JSON.parse(updatedConfig.images) : [], - resolution: updatedConfig.resolution, - duration: updatedConfig.duration, - prompt: updatedConfig.prompt, - selectedResultId: updatedConfig.selectedResultId, - createdAt: new Date(updatedConfig.createTime).toISOString(), - } - })); + if (updatedConfig) { + res.status(200).send( + success({ + message: "更新视频配置成功", + data: { + id: updatedConfig.id, + scriptId: updatedConfig.scriptId, + projectId: updatedConfig.projectId, + manufacturer: updatedConfig.manufacturer, + mode: updatedConfig.mode, + startFrame: updatedConfig.startFrame ? JSON.parse(updatedConfig.startFrame) : null, + endFrame: updatedConfig.endFrame ? JSON.parse(updatedConfig.endFrame) : null, + images: updatedConfig.images ? JSON.parse(updatedConfig.images) : [], + resolution: updatedConfig.resolution, + duration: updatedConfig.duration, + prompt: updatedConfig.prompt, + selectedResultId: updatedConfig.selectedResultId, + createdAt: new Date(updatedConfig.createTime!).toISOString(), + }, + }), + ); + } else { + res.status(200).send(error("更新配置失败")); + } }, ); diff --git a/src/utils/ai/image/index.ts b/src/utils/ai/image/index.ts index 77076ec..281742a 100644 --- a/src/utils/ai/image/index.ts +++ b/src/utils/ai/image/index.ts @@ -63,5 +63,5 @@ export default async (input: ImageConfig, config?: AIConfig) => { let imageUrl = await manufacturerFn(input, { model, apiKey, baseURL }); if (!input.resType) input.resType = "b64"; if (input.resType === "b64" && imageUrl.startsWith("http")) imageUrl = await urlToBase64(imageUrl); - return input; + return imageUrl; }; diff --git a/src/utils/ai/text/modelList.ts b/src/utils/ai/text/modelList.ts index d56b7c5..a37ea8a 100644 --- a/src/utils/ai/text/modelList.ts +++ b/src/utils/ai/text/modelList.ts @@ -4,6 +4,7 @@ import { createZhipu } from "zhipu-ai-provider"; import { createQwen } from "qwen-ai-provider"; import { createGoogleGenerativeAI } from "@ai-sdk/google"; import { createAnthropic } from "@ai-sdk/anthropic"; +import { createOpenAICompatible } from "@ai-sdk/openai-compatible"; interface Owned { manufacturer: string; @@ -18,7 +19,8 @@ interface Owned { | typeof createZhipu | typeof createQwen | typeof createGoogleGenerativeAI - | typeof createAnthropic; + | typeof createAnthropic + | typeof createOpenAICompatible; } const modelList: Owned[] = [ @@ -409,6 +411,15 @@ const modelList: Owned[] = [ instance: createAnthropic, tool: true, }, + { + manufacturer: "other", + model: "gpt-4.1", + responseFormat: "schema", + image: true, + think: false, + instance: createOpenAICompatible, + tool: true, + }, ]; export default modelList; diff --git a/src/utils/editImage.ts b/src/utils/editImage.ts index af058a4..14aaec9 100644 --- a/src/utils/editImage.ts +++ b/src/utils/editImage.ts @@ -79,7 +79,7 @@ async function convertDirectiveAndImages(images: Record, directi */ export default async (images: Record, directive: string, projectId: number) => { const { prompt, images: base64Images } = await convertDirectiveAndImages(images, directive); - const contentStr = await u.ai.generateImage({ + const contentStr = await u.ai.image({ systemPrompt: "根据用户提供的具体修改指令，对上传的图片进行智能编辑。", prompt: prompt, imageBase64: base64Images,