diff --git a/src/agents/productionAgent/index.ts b/src/agents/productionAgent/index.ts index 4676dc1..1ddeb38 100644 --- a/src/agents/productionAgent/index.ts +++ b/src/agents/productionAgent/index.ts @@ -129,11 +129,11 @@ function createSubAgent(parentCtx: AgentContext) { "\n" + [ "你可以使用如下XML格式写入工作区：\n```", - "剧本：", "拍摄计划：内容", "分镜表：内容", "```", ].join("\n"); + // "剧本：", return runAgent({ prompt, diff --git a/src/agents/productionAgent/tools.ts b/src/agents/productionAgent/tools.ts index ada1e8d..386378d 100644 --- a/src/agents/productionAgent/tools.ts +++ b/src/agents/productionAgent/tools.ts @@ -59,12 +59,6 @@ const flowDataSchema = z.object({ assets: z.array(assetItemSchema).describe("衍生资产"), storyboardTable: z.string().describe("分镜表"), storyboard: z.array(storyboardSchema).describe("分镜面板"), - workbench: workbenchDataSchema.describe("工作台配置"), - poster: z - .object({ - items: z.array(posterItemSchema).describe("海报项目列表"), - }) - .describe("海报配置"), }); export type FlowData = z.infer; @@ -154,10 +148,46 @@ export default (toolCpnfig: ToolConfig) => { return res ?? "删除成功"; }, }), + + add_storyboard: tool({ + description: "新增或更新分镜面板", + inputSchema: z.object({ + id: z.number().nullable().describe("分镜面板ID,如果新增则为空"), + title: z.string().describe("分镜面板名称"), + desc: z.string().describe("分镜面板描述"), + group: z.number().describe("分镜面板分组,根据这个字段 对分镜图片，进行同时生成视频，例如 同一分组的两张图片会被用于首尾帧生成视频"), + }), + execute: async (storyboard) => { + const thinking = msg.thinking("正在操作资产..."); + const { projectId, scriptId } = resTool.data; + const createTime = Date.now(); + console.log("%c Line:161 🍤 storyboard", "background:#e41a6a", storyboard); + + const data = { + id: storyboard.id ?? undefined, + title: storyboard.title, + description: storyboard.desc, + createTime, + scriptId, + }; + if (storyboard.id) { + await u.db("o_storyboard").where("id", storyboard.id).update(data); + thinking.appendText(`已更新分镜面板，ID: ${storyboard.id}\n`); + } else { + const [insertedId] = await u.db("o_storyboard").insert(data); + data.id = insertedId; + thinking.appendText(`已新增分镜面板，ID: ${insertedId}\n`); + } + const res = await new Promise((resolve) => socket.emit("addStoryboard", data, (res: any) => resolve(res))); + thinking.updateTitle("分镜面板操作完成"); + thinking.complete(); + return res ?? "操作成功"; + }, + }), generate_deriveAsset: tool({ description: "生成衍生资产", inputSchema: z.object({ - id: z.number().describe("衍生资产ID"), + id: z.array(z.number()).describe("需要生成的 衍生资产ID"), }), execute: async ({ id }) => { const thinking = msg.thinking("正在生成衍生资产..."); @@ -169,11 +199,13 @@ export default (toolCpnfig: ToolConfig) => { }, }), generate_storyboard: tool({ - description: "生成分镜", - inputSchema: z.object({}), - execute: async ({ script }) => { + description: "生成分镜图片", + inputSchema: z.object({ + storyboardIds: z.array(z.number()).describe("分镜ID列表"), + }), + execute: async ({ storyboardIds }) => { const thinking = msg.thinking("正在生成分镜..."); - const res = await new Promise((resolve) => socket.emit("generateStoryboard", { script }, (res: any) => resolve(res))); + const res = await new Promise((resolve) => socket.emit("generateStoryboard", { storyboardIds }, (res: any) => resolve(res))); thinking.appendText("生成的分镜数据:\n" + JSON.stringify(res, null, 2)); thinking.updateTitle("分镜生成完成"); thinking.complete(); diff --git a/src/router.ts b/src/router.ts index 53dc195..5182519 100644 --- a/src/router.ts +++ b/src/router.ts @@ -1,4 +1,4 @@ -// @routes-hash c1392b39a921d712296ccb6b4aea3507 +// @routes-hash 845d6aff66aab1f458a9f08f4f2eed34 import { Express } from "express"; import route1 from "./routes/agents/clearMemory"; @@ -45,77 +45,80 @@ import route41 from "./routes/novel/getNovelIndex"; import route42 from "./routes/novel/updateNovel"; import route43 from "./routes/other/deleteAllData"; import route44 from "./routes/other/getVersion"; -import route45 from "./routes/production/assets/getAssetsData"; -import route46 from "./routes/production/editImage/generateFlowImage"; -import route47 from "./routes/production/editImage/getImageFlow"; -import route48 from "./routes/production/editImage/saveImageFlow"; -import route49 from "./routes/production/editImage/updateImageFlow"; -import route50 from "./routes/production/exportImage"; -import route51 from "./routes/production/getFlowData"; -import route52 from "./routes/production/getProductionData"; -import route53 from "./routes/production/getStoryboardData"; -import route54 from "./routes/production/saveFlowData"; -import route55 from "./routes/production/storyboard/batchGenerateImage"; -import route56 from "./routes/production/storyboard/downPreviewImage"; -import route57 from "./routes/production/storyboard/getStoryboardData"; -import route58 from "./routes/production/storyboard/previewImage"; -import route59 from "./routes/production/workbench/confirmSelection"; -import route60 from "./routes/production/workbench/delVideo"; -import route61 from "./routes/production/workbench/generateVideo"; -import route62 from "./routes/production/workbench/generateVideoPrompt"; -import route63 from "./routes/production/workbench/getChatLines"; -import route64 from "./routes/production/workbench/getVideoModelDetail"; -import route65 from "./routes/production/workbench/videoPolling"; -import route66 from "./routes/project/addProject"; -import route67 from "./routes/project/addVisual"; -import route68 from "./routes/project/addVisualManual"; -import route69 from "./routes/project/deleteVisualManual"; -import route70 from "./routes/project/delProject"; -import route71 from "./routes/project/editProject"; -import route72 from "./routes/project/editVisualManual"; -import route73 from "./routes/project/getProject"; -import route74 from "./routes/project/getVisualManual"; -import route75 from "./routes/project/visualManual"; -import route76 from "./routes/script/addScript"; -import route77 from "./routes/script/delScript"; -import route78 from "./routes/script/exportScript"; -import route79 from "./routes/script/extractAssets"; -import route80 from "./routes/script/getScrptApi"; -import route81 from "./routes/script/pollScriptAssets"; -import route82 from "./routes/script/updateScript"; -import route83 from "./routes/scriptAgent/getPlanData"; -import route84 from "./routes/scriptAgent/setPlanData"; -import route85 from "./routes/setting/about/checkUpdate"; -import route86 from "./routes/setting/about/downloadApp"; -import route87 from "./routes/setting/agentDeploy/agentSetKey"; -import route88 from "./routes/setting/agentDeploy/deployAgentModel"; -import route89 from "./routes/setting/agentDeploy/getAgentDeploy"; -import route90 from "./routes/setting/dbConfig/clearData"; -import route91 from "./routes/setting/dev/getSwitchAiDevTool"; -import route92 from "./routes/setting/dev/updateSwitchAiDevTool"; -import route93 from "./routes/setting/fileManagement/openFolder"; -import route94 from "./routes/setting/getTextModel"; -import route95 from "./routes/setting/loginConfig/getUser"; -import route96 from "./routes/setting/loginConfig/updateUserPwd"; -import route97 from "./routes/setting/memoryConfig/delAllMemory"; -import route98 from "./routes/setting/memoryConfig/getMemory"; -import route99 from "./routes/setting/memoryConfig/sureMemory"; -import route100 from "./routes/setting/promptManage/getPrompt"; -import route101 from "./routes/setting/promptManage/updatePrompt"; -import route102 from "./routes/setting/skillManagement/getSkillContent"; -import route103 from "./routes/setting/skillManagement/getSkillList"; -import route104 from "./routes/setting/skillManagement/saveSkillContent"; -import route105 from "./routes/setting/vendorConfig/addVendor"; -import route106 from "./routes/setting/vendorConfig/deleteVendor"; -import route107 from "./routes/setting/vendorConfig/getVendorList"; -import route108 from "./routes/setting/vendorConfig/modelTest"; -import route109 from "./routes/setting/vendorConfig/updateCode"; -import route110 from "./routes/setting/vendorConfig/updateVendor"; -import route111 from "./routes/task/getProject"; -import route112 from "./routes/task/getTaskApi"; -import route113 from "./routes/task/getTaskCategories"; -import route114 from "./routes/task/taskDetails"; -import route115 from "./routes/test/test"; +import route45 from "./routes/production/assets/batchGenerateAssetsImage"; +import route46 from "./routes/production/assets/getAssetsData"; +import route47 from "./routes/production/assets/pollingImage"; +import route48 from "./routes/production/editImage/generateFlowImage"; +import route49 from "./routes/production/editImage/getImageFlow"; +import route50 from "./routes/production/editImage/saveImageFlow"; +import route51 from "./routes/production/editImage/updateImageFlow"; +import route52 from "./routes/production/exportImage"; +import route53 from "./routes/production/getFlowData"; +import route54 from "./routes/production/getProductionData"; +import route55 from "./routes/production/getStoryboardData"; +import route56 from "./routes/production/saveFlowData"; +import route57 from "./routes/production/storyboard/batchGenerateImage"; +import route58 from "./routes/production/storyboard/downPreviewImage"; +import route59 from "./routes/production/storyboard/getStoryboardData"; +import route60 from "./routes/production/storyboard/pollingImage"; +import route61 from "./routes/production/storyboard/previewImage"; +import route62 from "./routes/production/workbench/confirmSelection"; +import route63 from "./routes/production/workbench/delVideo"; +import route64 from "./routes/production/workbench/generateVideo"; +import route65 from "./routes/production/workbench/generateVideoPrompt"; +import route66 from "./routes/production/workbench/getChatLines"; +import route67 from "./routes/production/workbench/getVideoModelDetail"; +import route68 from "./routes/production/workbench/videoPolling"; +import route69 from "./routes/project/addProject"; +import route70 from "./routes/project/addVisual"; +import route71 from "./routes/project/addVisualManual"; +import route72 from "./routes/project/deleteVisualManual"; +import route73 from "./routes/project/delProject"; +import route74 from "./routes/project/editProject"; +import route75 from "./routes/project/editVisualManual"; +import route76 from "./routes/project/getProject"; +import route77 from "./routes/project/getVisualManual"; +import route78 from "./routes/project/visualManual"; +import route79 from "./routes/script/addScript"; +import route80 from "./routes/script/delScript"; +import route81 from "./routes/script/exportScript"; +import route82 from "./routes/script/extractAssets"; +import route83 from "./routes/script/getScrptApi"; +import route84 from "./routes/script/pollScriptAssets"; +import route85 from "./routes/script/updateScript"; +import route86 from "./routes/scriptAgent/getPlanData"; +import route87 from "./routes/scriptAgent/setPlanData"; +import route88 from "./routes/setting/about/checkUpdate"; +import route89 from "./routes/setting/about/downloadApp"; +import route90 from "./routes/setting/agentDeploy/agentSetKey"; +import route91 from "./routes/setting/agentDeploy/deployAgentModel"; +import route92 from "./routes/setting/agentDeploy/getAgentDeploy"; +import route93 from "./routes/setting/dbConfig/clearData"; +import route94 from "./routes/setting/dev/getSwitchAiDevTool"; +import route95 from "./routes/setting/dev/updateSwitchAiDevTool"; +import route96 from "./routes/setting/fileManagement/openFolder"; +import route97 from "./routes/setting/getTextModel"; +import route98 from "./routes/setting/loginConfig/getUser"; +import route99 from "./routes/setting/loginConfig/updateUserPwd"; +import route100 from "./routes/setting/memoryConfig/delAllMemory"; +import route101 from "./routes/setting/memoryConfig/getMemory"; +import route102 from "./routes/setting/memoryConfig/sureMemory"; +import route103 from "./routes/setting/promptManage/getPrompt"; +import route104 from "./routes/setting/promptManage/updatePrompt"; +import route105 from "./routes/setting/skillManagement/getSkillContent"; +import route106 from "./routes/setting/skillManagement/getSkillList"; +import route107 from "./routes/setting/skillManagement/saveSkillContent"; +import route108 from "./routes/setting/vendorConfig/addVendor"; +import route109 from "./routes/setting/vendorConfig/deleteVendor"; +import route110 from "./routes/setting/vendorConfig/getVendorList"; +import route111 from "./routes/setting/vendorConfig/modelTest"; +import route112 from "./routes/setting/vendorConfig/updateCode"; +import route113 from "./routes/setting/vendorConfig/updateVendor"; +import route114 from "./routes/task/getProject"; +import route115 from "./routes/task/getTaskApi"; +import route116 from "./routes/task/getTaskCategories"; +import route117 from "./routes/task/taskDetails"; +import route118 from "./routes/test/test"; export default async (app: Express) => { app.use("/api/agents/clearMemory", route1); @@ -162,75 +165,78 @@ export default async (app: Express) => { app.use("/api/novel/updateNovel", route42); app.use("/api/other/deleteAllData", route43); app.use("/api/other/getVersion", route44); - app.use("/api/production/assets/getAssetsData", route45); - app.use("/api/production/editImage/generateFlowImage", route46); - app.use("/api/production/editImage/getImageFlow", route47); - app.use("/api/production/editImage/saveImageFlow", route48); - app.use("/api/production/editImage/updateImageFlow", route49); - app.use("/api/production/exportImage", route50); - app.use("/api/production/getFlowData", route51); - app.use("/api/production/getProductionData", route52); - app.use("/api/production/getStoryboardData", route53); - app.use("/api/production/saveFlowData", route54); - app.use("/api/production/storyboard/batchGenerateImage", route55); - app.use("/api/production/storyboard/downPreviewImage", route56); - app.use("/api/production/storyboard/getStoryboardData", route57); - app.use("/api/production/storyboard/previewImage", route58); - app.use("/api/production/workbench/confirmSelection", route59); - app.use("/api/production/workbench/delVideo", route60); - app.use("/api/production/workbench/generateVideo", route61); - app.use("/api/production/workbench/generateVideoPrompt", route62); - app.use("/api/production/workbench/getChatLines", route63); - app.use("/api/production/workbench/getVideoModelDetail", route64); - app.use("/api/production/workbench/videoPolling", route65); - app.use("/api/project/addProject", route66); - app.use("/api/project/addVisual", route67); - app.use("/api/project/addVisualManual", route68); - app.use("/api/project/deleteVisualManual", route69); - app.use("/api/project/delProject", route70); - app.use("/api/project/editProject", route71); - app.use("/api/project/editVisualManual", route72); - app.use("/api/project/getProject", route73); - app.use("/api/project/getVisualManual", route74); - app.use("/api/project/visualManual", route75); - app.use("/api/script/addScript", route76); - app.use("/api/script/delScript", route77); - app.use("/api/script/exportScript", route78); - app.use("/api/script/extractAssets", route79); - app.use("/api/script/getScrptApi", route80); - app.use("/api/script/pollScriptAssets", route81); - app.use("/api/script/updateScript", route82); - app.use("/api/scriptAgent/getPlanData", route83); - app.use("/api/scriptAgent/setPlanData", route84); - app.use("/api/setting/about/checkUpdate", route85); - app.use("/api/setting/about/downloadApp", route86); - app.use("/api/setting/agentDeploy/agentSetKey", route87); - app.use("/api/setting/agentDeploy/deployAgentModel", route88); - app.use("/api/setting/agentDeploy/getAgentDeploy", route89); - app.use("/api/setting/dbConfig/clearData", route90); - app.use("/api/setting/dev/getSwitchAiDevTool", route91); - app.use("/api/setting/dev/updateSwitchAiDevTool", route92); - app.use("/api/setting/fileManagement/openFolder", route93); - app.use("/api/setting/getTextModel", route94); - app.use("/api/setting/loginConfig/getUser", route95); - app.use("/api/setting/loginConfig/updateUserPwd", route96); - app.use("/api/setting/memoryConfig/delAllMemory", route97); - app.use("/api/setting/memoryConfig/getMemory", route98); - app.use("/api/setting/memoryConfig/sureMemory", route99); - app.use("/api/setting/promptManage/getPrompt", route100); - app.use("/api/setting/promptManage/updatePrompt", route101); - app.use("/api/setting/skillManagement/getSkillContent", route102); - app.use("/api/setting/skillManagement/getSkillList", route103); - app.use("/api/setting/skillManagement/saveSkillContent", route104); - app.use("/api/setting/vendorConfig/addVendor", route105); - app.use("/api/setting/vendorConfig/deleteVendor", route106); - app.use("/api/setting/vendorConfig/getVendorList", route107); - app.use("/api/setting/vendorConfig/modelTest", route108); - app.use("/api/setting/vendorConfig/updateCode", route109); - app.use("/api/setting/vendorConfig/updateVendor", route110); - app.use("/api/task/getProject", route111); - app.use("/api/task/getTaskApi", route112); - app.use("/api/task/getTaskCategories", route113); - app.use("/api/task/taskDetails", route114); - app.use("/api/test/test", route115); + app.use("/api/production/assets/batchGenerateAssetsImage", route45); + app.use("/api/production/assets/getAssetsData", route46); + app.use("/api/production/assets/pollingImage", route47); + app.use("/api/production/editImage/generateFlowImage", route48); + app.use("/api/production/editImage/getImageFlow", route49); + app.use("/api/production/editImage/saveImageFlow", route50); + app.use("/api/production/editImage/updateImageFlow", route51); + app.use("/api/production/exportImage", route52); + app.use("/api/production/getFlowData", route53); + app.use("/api/production/getProductionData", route54); + app.use("/api/production/getStoryboardData", route55); + app.use("/api/production/saveFlowData", route56); + app.use("/api/production/storyboard/batchGenerateImage", route57); + app.use("/api/production/storyboard/downPreviewImage", route58); + app.use("/api/production/storyboard/getStoryboardData", route59); + app.use("/api/production/storyboard/pollingImage", route60); + app.use("/api/production/storyboard/previewImage", route61); + app.use("/api/production/workbench/confirmSelection", route62); + app.use("/api/production/workbench/delVideo", route63); + app.use("/api/production/workbench/generateVideo", route64); + app.use("/api/production/workbench/generateVideoPrompt", route65); + app.use("/api/production/workbench/getChatLines", route66); + app.use("/api/production/workbench/getVideoModelDetail", route67); + app.use("/api/production/workbench/videoPolling", route68); + app.use("/api/project/addProject", route69); + app.use("/api/project/addVisual", route70); + app.use("/api/project/addVisualManual", route71); + app.use("/api/project/deleteVisualManual", route72); + app.use("/api/project/delProject", route73); + app.use("/api/project/editProject", route74); + app.use("/api/project/editVisualManual", route75); + app.use("/api/project/getProject", route76); + app.use("/api/project/getVisualManual", route77); + app.use("/api/project/visualManual", route78); + app.use("/api/script/addScript", route79); + app.use("/api/script/delScript", route80); + app.use("/api/script/exportScript", route81); + app.use("/api/script/extractAssets", route82); + app.use("/api/script/getScrptApi", route83); + app.use("/api/script/pollScriptAssets", route84); + app.use("/api/script/updateScript", route85); + app.use("/api/scriptAgent/getPlanData", route86); + app.use("/api/scriptAgent/setPlanData", route87); + app.use("/api/setting/about/checkUpdate", route88); + app.use("/api/setting/about/downloadApp", route89); + app.use("/api/setting/agentDeploy/agentSetKey", route90); + app.use("/api/setting/agentDeploy/deployAgentModel", route91); + app.use("/api/setting/agentDeploy/getAgentDeploy", route92); + app.use("/api/setting/dbConfig/clearData", route93); + app.use("/api/setting/dev/getSwitchAiDevTool", route94); + app.use("/api/setting/dev/updateSwitchAiDevTool", route95); + app.use("/api/setting/fileManagement/openFolder", route96); + app.use("/api/setting/getTextModel", route97); + app.use("/api/setting/loginConfig/getUser", route98); + app.use("/api/setting/loginConfig/updateUserPwd", route99); + app.use("/api/setting/memoryConfig/delAllMemory", route100); + app.use("/api/setting/memoryConfig/getMemory", route101); + app.use("/api/setting/memoryConfig/sureMemory", route102); + app.use("/api/setting/promptManage/getPrompt", route103); + app.use("/api/setting/promptManage/updatePrompt", route104); + app.use("/api/setting/skillManagement/getSkillContent", route105); + app.use("/api/setting/skillManagement/getSkillList", route106); + app.use("/api/setting/skillManagement/saveSkillContent", route107); + app.use("/api/setting/vendorConfig/addVendor", route108); + app.use("/api/setting/vendorConfig/deleteVendor", route109); + app.use("/api/setting/vendorConfig/getVendorList", route110); + app.use("/api/setting/vendorConfig/modelTest", route111); + app.use("/api/setting/vendorConfig/updateCode", route112); + app.use("/api/setting/vendorConfig/updateVendor", route113); + app.use("/api/task/getProject", route114); + app.use("/api/task/getTaskApi", route115); + app.use("/api/task/getTaskCategories", route116); + app.use("/api/task/taskDetails", route117); + app.use("/api/test/test", route118); } diff --git a/src/routes/production/assets/batchGenerateAssetsImage.ts b/src/routes/production/assets/batchGenerateAssetsImage.ts new file mode 100644 index 0000000..79d5b0f --- /dev/null +++ b/src/routes/production/assets/batchGenerateAssetsImage.ts @@ -0,0 +1,80 @@ +import express from "express"; +import u from "@/utils"; +import { z } from "zod"; +import sharp from "sharp"; +import { success } from "@/lib/responseFormat"; +import { validateFields } from "@/middleware/middleware"; +import { Output } from "ai"; +const router = express.Router(); + +export default router.post( + "/", + validateFields({ + assetIds: z.array(z.number()), + projectId: z.number(), + scriptId: z.number(), + }), + async (req, res) => { + const { assetIds, projectId, scriptId } = req.body; + + const projectSettingData = await u.db("o_project").where("id", projectId).select("imageModel", "imageQuality", "artStyle").first(); + + const assetsDataArr = await u.db("o_assets").whereIn("id", assetIds).select("id", "describe", "name", "type"); + const rolePrompt = u.getArtPrompt(projectSettingData!.artStyle!, "art_character_derivative"); + const toolPrompt = u.getArtPrompt(projectSettingData!.artStyle!, "art_prop_derivative"); + const scenePrompt = u.getArtPrompt(projectSettingData!.artStyle!, "art_scene_derivative"); + const promptRecord = { + role: rolePrompt, + tool: toolPrompt, + scene: scenePrompt, + }; + + for (const item of assetsDataArr) { + const { text } = await u.Ai.Text("universalAi").invoke({ + system: ` + 你需要根据用户提供的资产的标题与描述，结合当前项目的美术风格，为我优化提示词以便生成更符合项目美术风格的图片。直接输出提示词，不需要做任何解释说明。 + 美术风格：${promptRecord[item.type! as keyof typeof promptRecord]}`, + messages: [ + { + role: "user", + content: `资产名称:${item.name},资产描述:${item.describe}`, + }, + ], + }); + console.log("%c Line:35 🎂 text", "background:#3f7cff", text); + + const repeloadObj = { + prompt: text, + size: projectSettingData?.imageQuality as "1K" | "2K" | "4K", + aspectRatio: "16:9", + }; + const [imageId] = await u.db("o_image").insert({ + assetsId: item.id, + type: item.type, + state: "生成中", + resolution: projectSettingData?.imageQuality, + model: projectSettingData?.imageModel, + }); + u.Ai.Image(projectSettingData?.imageModel as `${string}:${string}`) + .run({ + prompt: text, + imageBase64: [], + size: projectSettingData?.imageQuality as "1K" | "2K" | "4K", + aspectRatio: "16:9", + taskClass: "生成图片", + describe: "资产图片生成", + relatedObjects: JSON.stringify(repeloadObj), + projectId: projectId, + }) + .then(async (imageCls) => { + const savePath = `/${projectId}/assets/${scriptId}/${u.uuid()}.jpg`; + await imageCls.save(savePath); + // 更新对应数据库 + await u.db("o_assets").where("id", item.id).update({ imageId: imageId }); + await u.db("o_image").where({ id: imageId }).update({ state: "已完成", filePath: savePath }); + }); + } + + return res.status(200).send(success()); + }, +); diff --git a/src/routes/production/assets/pollingImage.ts b/src/routes/production/assets/pollingImage.ts new file mode 100644 index 0000000..7cc5886 --- /dev/null +++ b/src/routes/production/assets/pollingImage.ts @@ -0,0 +1,29 @@ +import express from "express"; +import u from "@/utils"; +import { z } from "zod"; +import { success } from "@/lib/responseFormat"; +import { validateFields } from "@/middleware/middleware"; +const router = express.Router(); + +export default router.post( + "/", + validateFields({ + ids: z.array(z.number()), + }), + async (req, res) => { + const { ids } = req.body; + const data = await u + .db("o_assets") + .leftJoin("o_image", "o_assets.imageId", "o_image.id") + .whereIn("o_assets.id", ids) + .whereNot("o_image.state", "生成中") + .select("o_image.state", "o_assets.id", "o_image.filePath"); + const result = await Promise.all( + data.map(async (item: any) => ({ + ...item, + src: item.filePath ? await u.oss.getFileUrl(item.filePath) : null, + })), + ); + res.status(200).send(success(result)); + }, +); diff --git a/src/routes/production/storyboard/batchGenerateImage.ts b/src/routes/production/storyboard/batchGenerateImage.ts index 446f75e..bf433ea 100644 --- a/src/routes/production/storyboard/batchGenerateImage.ts +++ b/src/routes/production/storyboard/batchGenerateImage.ts @@ -12,61 +12,65 @@ export default router.post( validateFields({ storyboardIds: z.array(z.number()), projectId: z.number(), + scriptId: z.number(), }), async (req, res) => { - const { storyboardIds, projectId } = req.body; + const { storyboardIds, projectId, scriptId } = req.body; const projectSettingData = await u.db("o_project").where("id", projectId).select("imageModel", "imageQuality", "artStyle").first(); const sceneArkPrompt = u.getArtPrompt(projectSettingData?.artStyle || "", "art_storyboard"); const storyboardData = await u.db("o_storyboard").whereIn("id", storyboardIds).select("id", "description", "title"); - const { text } = await u.Ai.Text("universalAi").invoke({ - system: ` + + for (const item of storyboardData) { + const { text } = await u.Ai.Text("universalAi").invoke({ + system: ` 你需要根据用户提供的分镜的标题与描述，结合当前项目的美术风格，为我优化提示词以便生成更符合项目美术风格的分镜图片。请你只优化提示词，不要添加任何额外的描述性文字，请以JSON格式输出: [{id:"对应分镜ID",prompt:"分镜提示词"}]。 美术风格：${sceneArkPrompt}`, - messages: [ - { - role: "user", - content: `一下是我的分镜内容\n ${storyboardData.map((s) => `分镜ID:${s.id},分镜描述:${s.description},分镜标题:${s.title}`).join("\n")}`, - }, - ], - output: Output.object({ - schema: z.array( - z.object({ - prompt: z.string().describe("优化后的提示词"), - }), - ), - }), - }); - for (const item of storyboardData) { + messages: [ + { + role: "user", + content: `分镜描述:${item.description}`, + }, + ], + }); const repeloadObj = { prompt: text, size: projectSettingData?.imageQuality as "1K" | "2K" | "4K", aspectRatio: "16:9", }; - u.Ai.Image(projectSettingData?.imageModel as `${string}:${string}`).run({ + await u.db("o_storyboard").where("id", item.id).update({ prompt: text, - imageBase64: [], - size: projectSettingData?.imageQuality as "1K" | "2K" | "4K", - aspectRatio: "16:9", - taskClass: "生成图片", - describe: "资产图片生成", - relatedObjects: JSON.stringify(repeloadObj), - projectId: projectId, + state: "生成中", }); - // .then(async (imageCls) => { - // const savePath = `/${resTool.data.projectId}/assets/${resTool.data.scriptId}/${u.uuid()}.jpg`; - // await imageCls.save(savePath); - // const obj = { - // ...item, - // id: item.assetId, - // src: await u.oss.getFileUrl(savePath), - // state: "已完成", - // }; - //更新对应数据库 - // await u.db("o_assets").where("id", item.assetId).update({ imageId: imageId }); - // await u.db("o_image").where({ id: imageId }).update({ state: "已完成", filePath: savePath }); - // }); + u.Ai.Image(projectSettingData?.imageModel as `${string}:${string}`) + .run({ + prompt: text, + imageBase64: [], + size: projectSettingData?.imageQuality as "1K" | "2K" | "4K", + aspectRatio: "16:9", + taskClass: "生成图片", + describe: "资产图片生成", + relatedObjects: JSON.stringify(repeloadObj), + projectId: projectId, + }) + .then(async (imageCls) => { + const savePath = `/${projectId}/assets/${scriptId}/${u.uuid()}.jpg`; + await imageCls.save(savePath); + await u.db("o_storyboard").where("id", item.id).update({ + filePath: savePath, + state: "已完成", + }); + }) + .catch(async (e) => { + await u + .db("o_storyboard") + .where("id", item.id) + .update({ + reason: u.error(e).message, + state: "生成失败", + }); + }); } return res.status(200).send(success()); diff --git a/src/routes/production/storyboard/pollingImage.ts b/src/routes/production/storyboard/pollingImage.ts new file mode 100644 index 0000000..0850501 --- /dev/null +++ b/src/routes/production/storyboard/pollingImage.ts @@ -0,0 +1,24 @@ +import express from "express"; +import u from "@/utils"; +import { z } from "zod"; +import { success } from "@/lib/responseFormat"; +import { validateFields } from "@/middleware/middleware"; +const router = express.Router(); + +export default router.post( + "/", + validateFields({ + ids: z.array(z.number()), + }), + async (req, res) => { + const { ids } = req.body; + const data = await u.db("o_storyboard").whereIn("id", ids).whereNot("state", "生成中").select("id", "state", "reason", "filePath", "prompt"); + const result = await Promise.all( + data.map(async (item: any) => ({ + ...item, + src: item.filePath ? await u.oss.getFileUrl(item.filePath) : null, + })), + ); + res.status(200).send(success(result)); + }, +); diff --git a/src/routes/script/extractAssets.ts b/src/routes/script/extractAssets.ts index 873252d..16868a7 100644 --- a/src/routes/script/extractAssets.ts +++ b/src/routes/script/extractAssets.ts @@ -8,6 +8,22 @@ import { tool } from "ai"; import { o_script } from "@/types/database"; const router = express.Router(); + +/** 新资产：AI 首次识别到的资产，需要完整信息 */ +const NewAssetSchema = z.object({ + prompt: z.string().describe("生成提示词"), + name: z.string().describe("资产名称,仅为名称不做其他任何表述"), + desc: z.string().describe("资产描述"), + type: z.enum(["role", "tool", "scene"]).describe("资产类型"), + scriptIds: z.array(z.number()).describe("使用该资产的剧本id数组"), +}); + +/** 已有资产：数据库中已存在的资产，只需给出名称和关联的剧本 */ +const ExistingAssetRefSchema = z.object({ + name: z.string().describe("已有资产的名称,必须与已有资产列表中的名称完全一致"), + scriptIds: z.array(z.number()).describe("使用该资产的剧本id数组"), +}); + export const AssetSchema = z.object({ prompt: z.string().describe("生成提示词"), name: z.string().describe("资产名称,仅为名称不做其他任何表述"), @@ -15,23 +31,24 @@ export const AssetSchema = z.object({ type: z.enum(["role", "tool", "scene"]).describe("资产类型"), }); +type NewAsset = z.infer; +type ExistingAssetRef = z.infer; type Asset = z.infer; -/** 按批次并发执行，每批 batchSize 个同时跑，批次完成后调用 onBatchDone */ -async function pMapBatch( - items: T[], - fn: (item: T) => Promise, - batchSize: number, - onBatchDone?: (batchResults: R[]) => Promise, -): Promise { - const allResults: R[] = []; - for (let i = 0; i < items.length; i += batchSize) { - const batch = items.slice(i, i + batchSize); - const batchResults = await Promise.all(batch.map(fn)); - allResults.push(...batchResults); - if (onBatchDone) await onBatchDone(batchResults); +/** 每批 AI 调用的结果 */ +type GroupResult = { + batchScriptIds: number[]; + newAssets: NewAsset[]; + existingRefs: ExistingAssetRef[]; +} | null; + +/** 将 scriptIds 数组按 groupSize 分组 */ +function chunkArray(arr: T[], groupSize: number): T[][] { + const chunks: T[][] = []; + for (let i = 0; i < arr.length; i += groupSize) { + chunks.push(arr.slice(i, i + groupSize)); } - return allResults; + return chunks; } export default router.post( @@ -39,10 +56,10 @@ export default router.post( validateFields({ scriptIds: z.array(z.number()), projectId: z.number(), - concurrency: z.number().min(1).max(20).optional(), + groupSize: z.number().min(1).max(10).optional(), }), async (req, res) => { - const { scriptIds, projectId, concurrency = 3 } = req.body; + const { scriptIds, projectId, groupSize = 5 } = req.body; if (!scriptIds.length) return res.status(400).send(error("请先选择剧本")); const scripts = await u.db("o_script").whereIn("id", scriptIds); const intansce = u.Ai.Text("universalAi"); @@ -57,34 +74,21 @@ export default router.post( const errors: { scriptId: number; error: string }[] = []; let successCount = 0; - // 每批提取结果：scriptId -> 资产列表 - type BatchResult = { scriptId: number; assets: Asset[] } | null; + // 将 scriptIds 按 groupSize（默认5）分组，每组一起发给 AI + const scriptGroups = chunkArray(scriptIds, groupSize); - /** 一批剧本提取完成后统一入库并建立关联 */ - async function persistBatch(batchResults: BatchResult[]) { - const validResults = batchResults.filter((r): r is { scriptId: number; assets: Asset[] } => r !== null && r.assets.length > 0); - if (!validResults.length) return; + /** 一组剧本提取完成后统一入库并建立关联 */ + async function persistGroupResult(result: GroupResult) { + if (!result) return; + const { batchScriptIds, newAssets, existingRefs } = result; + if (!newAssets.length && !existingRefs.length) return; - // 合并本批所有资产，同名去重 - const mergedAssetsMap = new Map(); - const assetScriptIds = new Map(); - for (const { scriptId, assets } of validResults) { - for (const asset of assets) { - if (!mergedAssetsMap.has(asset.name)) { - mergedAssetsMap.set(asset.name, asset); - } - const ids = assetScriptIds.get(asset.name) || []; - ids.push(scriptId); - assetScriptIds.set(asset.name, ids); - } - } - - // 查询已有资产，避免重复插入 + // 查询已有资产 const existingAssets = await u.db("o_assets").where("projectId", projectId).select("id", "name"); const existingMap = new Map(existingAssets.map((a) => [a.name!, a.id!])); - // 插入不存在的资产 - const toInsert = [...mergedAssetsMap.values()].filter((asset) => !existingMap.has(asset.name)); + // 插入新资产（不在已有列表中的） + const toInsert = newAssets.filter((asset) => !existingMap.has(asset.name)); if (toInsert.length) { await u.db("o_assets").insert( toInsert.map((asset) => ({ @@ -102,13 +106,24 @@ export default router.post( const allAssets = await u.db("o_assets").where("projectId", projectId).select("id", "name"); const nameToId = new Map(allAssets.map((a) => [a.name, a.id])); - // 建立本批各 scriptId 与资产的关联 - const batchScriptIds = validResults.map((r) => r.scriptId); + // 收集所有资产与剧本的关联关系 const scriptAssetRows: { scriptId: number; assetId: number }[] = []; - for (const [name, sIds] of assetScriptIds) { - const assetId = nameToId.get(name); + + // 新资产的关联 + for (const asset of newAssets) { + const assetId = nameToId.get(asset.name); if (assetId) { - for (const sid of sIds) { + for (const sid of asset.scriptIds) { + scriptAssetRows.push({ scriptId: sid, assetId }); + } + } + } + + // 已有资产的关联 + for (const ref of existingRefs) { + const assetId = nameToId.get(ref.name); + if (assetId) { + for (const sid of ref.scriptIds) { scriptAssetRows.push({ scriptId: sid, assetId }); } } @@ -127,74 +142,111 @@ export default router.post( }); } - // 按批次并发提取剧本资产，每批完成后统一入库 - await pMapBatch( - scriptIds, - async (scriptId: number) => { + // 逐组处理（每组最多 groupSize 集剧本一起发给 AI） + for (const group of scriptGroups) { + // 过滤有效剧本 + const validScripts: { id: number; script: o_script }[] = []; + for (const scriptId of group as number[]) { const script = scriptMap.get(scriptId); if (!script) { errors.push({ scriptId, error: "未找到对应剧本" }); await u.db("o_script").where("id", scriptId).update({ extractState: -1, errorReason: "未找到对应剧本" }); - return null; + } else { + validScripts.push({ id: scriptId, script }); } + } + if (!validScripts.length) continue; - // 用闭包收集当前 scriptId 的资产 - let collected: Asset[] = []; + // 查询当前项目已有的资产列表，提供给 AI 参考 + const existingAssets = await u.db("o_assets").where("projectId", projectId).select("name", "type"); + console.log("%c Line:162 🍔 existingAssets", "background:#ea7e5c", existingAssets); + const existingAssetsList = existingAssets.map((a) => `${a.name}(${a.type})`).join("、"); + console.log("%c Line:164 🍫 existingAssetsList", "background:#33a5ff", existingAssetsList); - const resultTool = tool({ - description: "返回结果时必须调用这个工具,", - inputSchema: z.object({ - assetsList: z.array(AssetSchema).describe("剧本所使用资产列表,注意不要包含剧本内容,仅为所使用到的 道具、人物、场景、素材"), - }), - execute: async ({ assetsList }) => { - console.log("[tools] set_flowData script", assetsList); - if (assetsList && assetsList.length) { - collected = assetsList; - } - return true; - }, + // 拼接多集剧本内容，每集用分隔标记 + const scriptsContent = validScripts + .map(({ id, script }) => `===== 【剧本ID: ${id}】${script.name || ""} =====\n${script.content}`) + .join("\n\n"); + + const validScriptIds = validScripts.map((v) => v.id); + + // 用闭包收集 AI 返回的资产 + let collectedNew: NewAsset[] = []; + let collectedExisting: ExistingAssetRef[] = []; + + const resultTool = tool({ + description: "返回结果时必须调用这个工具", + inputSchema: z.object({ + newAssets: z + .array(NewAssetSchema) + .describe("新发现的资产列表（不在已有资产列表中的），需要完整的 prompt、name、desc、type 和使用该资产的 scriptIds"), + existingAssetRefs: z + .array(ExistingAssetRefSchema) + .describe("已有资产的引用列表（在已有资产列表中已存在的），只需给出资产名称和使用该资产的 scriptIds"), + }), + execute: async ({ newAssets, existingAssetRefs }) => { + console.log("[tools] extractAssets result", { newAssets, existingAssetRefs }); + if (newAssets?.length) collectedNew = newAssets; + if (existingAssetRefs?.length) collectedExisting = existingAssetRefs; + return "无需回复用户任何内容"; + }, + }); + + try { + const data = await u.db("o_prompt").where("type", "scriptAssetExtraction").first("data"); + const existingHint = existingAssetsList + ? `\n\n【已有资产列表】：${existingAssetsList}\n对于已有资产，如果在剧本中出现，只需在 existingAssetRefs 中给出资产名称和对应的 scriptIds 数组即可，无需重复生成 prompt/desc/type。对于新发现的资产（不在已有列表中），请在 newAssets 中给出完整信息。` + : ""; + + const output = await intansce.invoke({ + messages: [ + { + role: "system", + content: + data?.data + + "\n\n提取剧本中涉及的资产（角色、场景、道具），参考技能 script_assets_extract 规范，结果必须通过 resultTool 工具返回。" + + "\n\n注意：本次会同时提供多集剧本，每集剧本以 ===== 【剧本ID: xxx】 ===== 分隔。你需要分析每集剧本使用了哪些资产，并在输出中用 scriptIds 数组标明每个资产在哪些剧本中出现。" + + existingHint, + }, + { + role: "user", + content: `请根据以下${validScripts.length}集剧本提取对应的剧本资产（角色、场景、道具）:\n\n${scriptsContent}`, + }, + ], + tools: { resultTool }, }); - - try { - const data = await u.db("o_prompt").where("type", "scriptAssetExtraction").first("data"); - await intansce.invoke({ - messages: [ - { - role: "system", - content: - data?.data + - "\n\n提取剧本中涉及的资产（角色、场景、道具），参考技能 script_assets_extract 规范，结果必须通过 resultTool 工具返回。", - }, - { - role: "user", - content: `请根据以下剧本提取对应的剧本资产（角色、场景、道具、素材片段）:\n\n${script.content}`, - }, - ], - tools: { resultTool }, - }); - } catch (e: any) { - const msg = e?.message || String(e); - console.error(`[extractAssets] scriptId=${scriptId} name=${script.name} 提取失败:`, msg); - errors.push({ scriptId, error: script.name + ":" + u.error(e).message }); + console.log("%c Line:extractAssets 🍧 output", "background:#f5ce50", output.text); + } catch (e: any) { + const msg = e?.message || String(e); + const scriptNames = validScripts.map((v) => v.script.name).join(", "); + console.error(`[extractAssets] group=[${validScriptIds.join(",")}] 提取失败:`, msg); + for (const { id, script } of validScripts) { + errors.push({ scriptId: id, error: (script.name || "") + ":" + u.error(e).message }); await u .db("o_script") - .where("id", scriptId) + .where("id", id) .update({ extractState: -1, errorReason: u.error(e).message }); - return null; } + continue; + } - if (!collected.length) { - errors.push({ scriptId, error: "AI 未返回任何资产" }); - await u.db("o_script").where("id", scriptId).update({ extractState: -1, errorReason: "AI 未返回任何资产" }); - return null; + if (!collectedNew.length && !collectedExisting.length) { + for (const { id } of validScripts) { + errors.push({ scriptId: id, error: "AI 未返回任何资产" }); + await u.db("o_script").where("id", id).update({ extractState: -1, errorReason: "AI 未返回任何资产" }); } + continue; + } - successCount++; - return { scriptId, assets: collected }; - }, - concurrency, - persistBatch, - ); + successCount += validScripts.length; + + // 入库 + await persistGroupResult({ + batchScriptIds: validScriptIds, + newAssets: collectedNew, + existingRefs: collectedExisting, + }); + } return res.send(success("开始提取资产")); }, diff --git a/src/types/database.d.ts b/src/types/database.d.ts index 99c6f71..4544078 100644 --- a/src/types/database.d.ts +++ b/src/types/database.d.ts @@ -1,8 +1,3 @@ -<<<<<<< HEAD -// @db-hash 93b2462070c45c2b449e9a18c4e88763 -//该文件由脚本自动生成，请勿手动修改 - -======= // @db-hash f7bc2fdb80756d5536929eb47155578b //该文件由脚本自动生成，请勿手动修改 @@ -13,7 +8,6 @@ export interface _o_script_old_20260327 { 'name'?: string | null; 'projectId'?: number | null; } ->>>>>>> 9da2610cdedc1e293b351ed3ab67fbc6fcd989f1 export interface memories { 'content': string; 'createTime': number; @@ -34,7 +28,7 @@ export interface o_agentDeploy { 'model'?: string | null; 'modelName'?: string | null; 'name'?: string | null; - 'vendorId'?: string | null; + 'vendorId'?: number | null; } export interface o_agentWorkData { 'createTime'?: number | null; @@ -244,10 +238,7 @@ export interface o_videoConfig { } export interface DB { -<<<<<<< HEAD -======= "_o_script_old_20260327": _o_script_old_20260327; ->>>>>>> 9da2610cdedc1e293b351ed3ab67fbc6fcd989f1 "memories": memories; "o_agentDeploy": o_agentDeploy; "o_agentWorkData": o_agentWorkData;