From 3f597232cd9fd6bf0b6db2f1ad6b7466fd19a37b Mon Sep 17 00:00:00 2001 From: zhishi <1951671751@qq.com> Date: Mon, 30 Mar 2026 21:50:24 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E4=BF=AE=E6=94=B9=20=E7=94=9F=E6=88=90?= =?UTF-8?q?=E5=88=86=E9=95=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/agents/productionAgent/tools.ts | 84 +++--- src/lib/initDB.ts | 1 + src/router.ts | 244 +++++++++--------- src/routes/assets/batchGenerationData.ts | 64 ++--- .../assets/batchGenerateAssetsImage.ts | 61 ++++- .../storyboard/batchGenerateImage.ts | 150 ++++++++++- src/routes/script/extractAssets.ts | 2 - src/types/database.d.ts | 15 +- yarn.lock | 68 ++--- 9 files changed, 425 insertions(+), 264 deletions(-) diff --git a/src/agents/productionAgent/tools.ts b/src/agents/productionAgent/tools.ts index 386378d..41f954c 100644 --- a/src/agents/productionAgent/tools.ts +++ b/src/agents/productionAgent/tools.ts @@ -14,7 +14,7 @@ const deriveAssetSchema = z.object({ state: z.enum(["未生成", "生成中", "已完成", "生成失败"]).describe("衍生资产生成状态"), type: z.enum(["role", "tool", "scene", "clip"]).describe("衍生资产类型"), }); -const assetItemSchema = z.object({ +export const assetItemSchema = z.object({ id: z.number().describe("资产唯一标识"), name: z.string().describe("资产名称"), type: z.enum(["role", "tool", "scene", "clip"]).describe("资产类型"), @@ -100,18 +100,20 @@ export default (toolCpnfig: ToolConfig) => { id: z.number().nullable().describe("衍生资产ID,如果新增则为空"), name: z.string().describe("衍生资产名称"), desc: z.string().describe("衍生资产描述"), - type: z.enum(["role", "tool", "scene", "clip"]).describe("衍生资产类型"), }), execute: async (deriveAsset) => { const thinking = msg.thinking("正在操作资产..."); const { projectId, scriptId } = resTool.data; const startTime = Date.now(); + const parentAssets = await u.db("o_assets").where("id", deriveAsset.assetsId).select("id", "type").first(); + if (!parentAssets) return "关联的资产不存在"; + const data = { id: deriveAsset.id ?? undefined, assetsId: deriveAsset.assetsId, projectId, name: deriveAsset.name, - type: deriveAsset.type, + type: parentAssets.type, describe: deriveAsset.desc, startTime, }; @@ -149,41 +151,6 @@ export default (toolCpnfig: ToolConfig) => { }, }), - add_storyboard: tool({ - description: "新增或更新分镜面板", - inputSchema: z.object({ - id: z.number().nullable().describe("分镜面板ID,如果新增则为空"), - title: z.string().describe("分镜面板名称"), - desc: z.string().describe("分镜面板描述"), - group: z.number().describe("分镜面板分组,根据这个字段 对分镜图片，进行同时生成视频，例如 同一分组的两张图片会被用于首尾帧生成视频"), - }), - execute: async (storyboard) => { - const thinking = msg.thinking("正在操作资产..."); - const { projectId, scriptId } = resTool.data; - const createTime = Date.now(); - console.log("%c Line:161 🍤 storyboard", "background:#e41a6a", storyboard); - - const data = { - id: storyboard.id ?? undefined, - title: storyboard.title, - description: storyboard.desc, - createTime, - scriptId, - }; - if (storyboard.id) { - await u.db("o_storyboard").where("id", storyboard.id).update(data); - thinking.appendText(`已更新分镜面板，ID: ${storyboard.id}\n`); - } else { - const [insertedId] = await u.db("o_storyboard").insert(data); - data.id = insertedId; - thinking.appendText(`已新增分镜面板，ID: ${insertedId}\n`); - } - const res = await new Promise((resolve) => socket.emit("addStoryboard", data, (res: any) => resolve(res))); - thinking.updateTitle("分镜面板操作完成"); - thinking.complete(); - return res ?? "操作成功"; - }, - }), generate_deriveAsset: tool({ description: "生成衍生资产", inputSchema: z.object({ @@ -191,25 +158,42 @@ export default (toolCpnfig: ToolConfig) => { }), execute: async ({ id }) => { const thinking = msg.thinking("正在生成衍生资产..."); - const res = await new Promise((resolve) => socket.emit("generateDeriveAsset", { id }, (res: any) => resolve(res))); - thinking.appendText(`已生成衍生资产，ID: ${id}\n`); - thinking.updateTitle("衍生资产生成完成"); - thinking.complete(); - return res ?? "生成失败"; + new Promise((resolve) => socket.emit("generateDeriveAsset", { id }, (res: any) => resolve(res))) + .then((res) => { + thinking.appendText(`已生成衍生资产，ID: ${JSON.stringify(res, null, 2)}\n`); + thinking.updateTitle("衍生资产开始完成"); + thinking.complete(); + }) + .catch((e) => { + thinking.appendText("衍生资产生成失败:\n" + u.error(e).message); + thinking.updateTitle("衍生资产生成失败"); + thinking.complete(); + }); + + return "开始生成衍生资产"; }, }), generate_storyboard: tool({ description: "生成分镜图片", inputSchema: z.object({ - storyboardIds: z.array(z.number()).describe("分镜ID列表"), + ids: z.array(z.number()).describe("分镜面板中需要更新的分镜 ID 列表,传入id仅作对应分镜更新用，不传入则全部生成"), }), - execute: async ({ storyboardIds }) => { + execute: async ({ ids }) => { + console.log("%c Line:176 🍒 ids", "background:#ea7e5c", ids); const thinking = msg.thinking("正在生成分镜..."); - const res = await new Promise((resolve) => socket.emit("generateStoryboard", { storyboardIds }, (res: any) => resolve(res))); - thinking.appendText("生成的分镜数据:\n" + JSON.stringify(res, null, 2)); - thinking.updateTitle("分镜生成完成"); - thinking.complete(); - return res; + new Promise((resolve) => socket.emit("generateStoryboard", { ids }, (res: any) => resolve(res))) + .then((res) => { + thinking.appendText("生成的分镜数据:\n" + JSON.stringify(res, null, 2)); + thinking.updateTitle("分镜生成完成"); + thinking.complete(); + }) + .catch((e) => { + thinking.appendText("分镜生成失败:\n" + u.error(e).message); + thinking.updateTitle("分镜生成失败"); + thinking.complete(); + }); + + return "开始生成分镜"; }, }), }; diff --git a/src/lib/initDB.ts b/src/lib/initDB.ts index f207906..2e2bf72 100644 --- a/src/lib/initDB.ts +++ b/src/lib/initDB.ts @@ -469,6 +469,7 @@ export default async (knex: Knex, forceInit: boolean = false): Promise => table.text("model"); table.text("resolution"); table.text("state"); + table.text("reason"); table.primary(["id"]); table.unique(["id"]); }, diff --git a/src/router.ts b/src/router.ts index 5182519..f9af96f 100644 --- a/src/router.ts +++ b/src/router.ts @@ -1,4 +1,4 @@ -// @routes-hash 845d6aff66aab1f458a9f08f4f2eed34 +// @routes-hash 8d0bd75a9e06280f64490cd32d7a5b2e import { Express } from "express"; import route1 from "./routes/agents/clearMemory"; @@ -59,66 +59,67 @@ import route55 from "./routes/production/getStoryboardData"; import route56 from "./routes/production/saveFlowData"; import route57 from "./routes/production/storyboard/batchGenerateImage"; import route58 from "./routes/production/storyboard/downPreviewImage"; -import route59 from "./routes/production/storyboard/getStoryboardData"; -import route60 from "./routes/production/storyboard/pollingImage"; -import route61 from "./routes/production/storyboard/previewImage"; -import route62 from "./routes/production/workbench/confirmSelection"; -import route63 from "./routes/production/workbench/delVideo"; -import route64 from "./routes/production/workbench/generateVideo"; -import route65 from "./routes/production/workbench/generateVideoPrompt"; -import route66 from "./routes/production/workbench/getChatLines"; -import route67 from "./routes/production/workbench/getVideoModelDetail"; -import route68 from "./routes/production/workbench/videoPolling"; -import route69 from "./routes/project/addProject"; -import route70 from "./routes/project/addVisual"; -import route71 from "./routes/project/addVisualManual"; -import route72 from "./routes/project/deleteVisualManual"; -import route73 from "./routes/project/delProject"; -import route74 from "./routes/project/editProject"; -import route75 from "./routes/project/editVisualManual"; -import route76 from "./routes/project/getProject"; -import route77 from "./routes/project/getVisualManual"; -import route78 from "./routes/project/visualManual"; -import route79 from "./routes/script/addScript"; -import route80 from "./routes/script/delScript"; -import route81 from "./routes/script/exportScript"; -import route82 from "./routes/script/extractAssets"; -import route83 from "./routes/script/getScrptApi"; -import route84 from "./routes/script/pollScriptAssets"; -import route85 from "./routes/script/updateScript"; -import route86 from "./routes/scriptAgent/getPlanData"; -import route87 from "./routes/scriptAgent/setPlanData"; -import route88 from "./routes/setting/about/checkUpdate"; -import route89 from "./routes/setting/about/downloadApp"; -import route90 from "./routes/setting/agentDeploy/agentSetKey"; -import route91 from "./routes/setting/agentDeploy/deployAgentModel"; -import route92 from "./routes/setting/agentDeploy/getAgentDeploy"; -import route93 from "./routes/setting/dbConfig/clearData"; -import route94 from "./routes/setting/dev/getSwitchAiDevTool"; -import route95 from "./routes/setting/dev/updateSwitchAiDevTool"; -import route96 from "./routes/setting/fileManagement/openFolder"; -import route97 from "./routes/setting/getTextModel"; -import route98 from "./routes/setting/loginConfig/getUser"; -import route99 from "./routes/setting/loginConfig/updateUserPwd"; -import route100 from "./routes/setting/memoryConfig/delAllMemory"; -import route101 from "./routes/setting/memoryConfig/getMemory"; -import route102 from "./routes/setting/memoryConfig/sureMemory"; -import route103 from "./routes/setting/promptManage/getPrompt"; -import route104 from "./routes/setting/promptManage/updatePrompt"; -import route105 from "./routes/setting/skillManagement/getSkillContent"; -import route106 from "./routes/setting/skillManagement/getSkillList"; -import route107 from "./routes/setting/skillManagement/saveSkillContent"; -import route108 from "./routes/setting/vendorConfig/addVendor"; -import route109 from "./routes/setting/vendorConfig/deleteVendor"; -import route110 from "./routes/setting/vendorConfig/getVendorList"; -import route111 from "./routes/setting/vendorConfig/modelTest"; -import route112 from "./routes/setting/vendorConfig/updateCode"; -import route113 from "./routes/setting/vendorConfig/updateVendor"; -import route114 from "./routes/task/getProject"; -import route115 from "./routes/task/getTaskApi"; -import route116 from "./routes/task/getTaskCategories"; -import route117 from "./routes/task/taskDetails"; -import route118 from "./routes/test/test"; +import route59 from "./routes/production/storyboard/generatestory2Image"; +import route60 from "./routes/production/storyboard/getStoryboardData"; +import route61 from "./routes/production/storyboard/pollingImage"; +import route62 from "./routes/production/storyboard/previewImage"; +import route63 from "./routes/production/workbench/confirmSelection"; +import route64 from "./routes/production/workbench/delVideo"; +import route65 from "./routes/production/workbench/generateVideo"; +import route66 from "./routes/production/workbench/generateVideoPrompt"; +import route67 from "./routes/production/workbench/getChatLines"; +import route68 from "./routes/production/workbench/getVideoModelDetail"; +import route69 from "./routes/production/workbench/videoPolling"; +import route70 from "./routes/project/addProject"; +import route71 from "./routes/project/addVisual"; +import route72 from "./routes/project/addVisualManual"; +import route73 from "./routes/project/deleteVisualManual"; +import route74 from "./routes/project/delProject"; +import route75 from "./routes/project/editProject"; +import route76 from "./routes/project/editVisualManual"; +import route77 from "./routes/project/getProject"; +import route78 from "./routes/project/getVisualManual"; +import route79 from "./routes/project/visualManual"; +import route80 from "./routes/script/addScript"; +import route81 from "./routes/script/delScript"; +import route82 from "./routes/script/exportScript"; +import route83 from "./routes/script/extractAssets"; +import route84 from "./routes/script/getScrptApi"; +import route85 from "./routes/script/pollScriptAssets"; +import route86 from "./routes/script/updateScript"; +import route87 from "./routes/scriptAgent/getPlanData"; +import route88 from "./routes/scriptAgent/setPlanData"; +import route89 from "./routes/setting/about/checkUpdate"; +import route90 from "./routes/setting/about/downloadApp"; +import route91 from "./routes/setting/agentDeploy/agentSetKey"; +import route92 from "./routes/setting/agentDeploy/deployAgentModel"; +import route93 from "./routes/setting/agentDeploy/getAgentDeploy"; +import route94 from "./routes/setting/dbConfig/clearData"; +import route95 from "./routes/setting/dev/getSwitchAiDevTool"; +import route96 from "./routes/setting/dev/updateSwitchAiDevTool"; +import route97 from "./routes/setting/fileManagement/openFolder"; +import route98 from "./routes/setting/getTextModel"; +import route99 from "./routes/setting/loginConfig/getUser"; +import route100 from "./routes/setting/loginConfig/updateUserPwd"; +import route101 from "./routes/setting/memoryConfig/delAllMemory"; +import route102 from "./routes/setting/memoryConfig/getMemory"; +import route103 from "./routes/setting/memoryConfig/sureMemory"; +import route104 from "./routes/setting/promptManage/getPrompt"; +import route105 from "./routes/setting/promptManage/updatePrompt"; +import route106 from "./routes/setting/skillManagement/getSkillContent"; +import route107 from "./routes/setting/skillManagement/getSkillList"; +import route108 from "./routes/setting/skillManagement/saveSkillContent"; +import route109 from "./routes/setting/vendorConfig/addVendor"; +import route110 from "./routes/setting/vendorConfig/deleteVendor"; +import route111 from "./routes/setting/vendorConfig/getVendorList"; +import route112 from "./routes/setting/vendorConfig/modelTest"; +import route113 from "./routes/setting/vendorConfig/updateCode"; +import route114 from "./routes/setting/vendorConfig/updateVendor"; +import route115 from "./routes/task/getProject"; +import route116 from "./routes/task/getTaskApi"; +import route117 from "./routes/task/getTaskCategories"; +import route118 from "./routes/task/taskDetails"; +import route119 from "./routes/test/test"; export default async (app: Express) => { app.use("/api/agents/clearMemory", route1); @@ -179,64 +180,65 @@ export default async (app: Express) => { app.use("/api/production/saveFlowData", route56); app.use("/api/production/storyboard/batchGenerateImage", route57); app.use("/api/production/storyboard/downPreviewImage", route58); - app.use("/api/production/storyboard/getStoryboardData", route59); - app.use("/api/production/storyboard/pollingImage", route60); - app.use("/api/production/storyboard/previewImage", route61); - app.use("/api/production/workbench/confirmSelection", route62); - app.use("/api/production/workbench/delVideo", route63); - app.use("/api/production/workbench/generateVideo", route64); - app.use("/api/production/workbench/generateVideoPrompt", route65); - app.use("/api/production/workbench/getChatLines", route66); - app.use("/api/production/workbench/getVideoModelDetail", route67); - app.use("/api/production/workbench/videoPolling", route68); - app.use("/api/project/addProject", route69); - app.use("/api/project/addVisual", route70); - app.use("/api/project/addVisualManual", route71); - app.use("/api/project/deleteVisualManual", route72); - app.use("/api/project/delProject", route73); - app.use("/api/project/editProject", route74); - app.use("/api/project/editVisualManual", route75); - app.use("/api/project/getProject", route76); - app.use("/api/project/getVisualManual", route77); - app.use("/api/project/visualManual", route78); - app.use("/api/script/addScript", route79); - app.use("/api/script/delScript", route80); - app.use("/api/script/exportScript", route81); - app.use("/api/script/extractAssets", route82); - app.use("/api/script/getScrptApi", route83); - app.use("/api/script/pollScriptAssets", route84); - app.use("/api/script/updateScript", route85); - app.use("/api/scriptAgent/getPlanData", route86); - app.use("/api/scriptAgent/setPlanData", route87); - app.use("/api/setting/about/checkUpdate", route88); - app.use("/api/setting/about/downloadApp", route89); - app.use("/api/setting/agentDeploy/agentSetKey", route90); - app.use("/api/setting/agentDeploy/deployAgentModel", route91); - app.use("/api/setting/agentDeploy/getAgentDeploy", route92); - app.use("/api/setting/dbConfig/clearData", route93); - app.use("/api/setting/dev/getSwitchAiDevTool", route94); - app.use("/api/setting/dev/updateSwitchAiDevTool", route95); - app.use("/api/setting/fileManagement/openFolder", route96); - app.use("/api/setting/getTextModel", route97); - app.use("/api/setting/loginConfig/getUser", route98); - app.use("/api/setting/loginConfig/updateUserPwd", route99); - app.use("/api/setting/memoryConfig/delAllMemory", route100); - app.use("/api/setting/memoryConfig/getMemory", route101); - app.use("/api/setting/memoryConfig/sureMemory", route102); - app.use("/api/setting/promptManage/getPrompt", route103); - app.use("/api/setting/promptManage/updatePrompt", route104); - app.use("/api/setting/skillManagement/getSkillContent", route105); - app.use("/api/setting/skillManagement/getSkillList", route106); - app.use("/api/setting/skillManagement/saveSkillContent", route107); - app.use("/api/setting/vendorConfig/addVendor", route108); - app.use("/api/setting/vendorConfig/deleteVendor", route109); - app.use("/api/setting/vendorConfig/getVendorList", route110); - app.use("/api/setting/vendorConfig/modelTest", route111); - app.use("/api/setting/vendorConfig/updateCode", route112); - app.use("/api/setting/vendorConfig/updateVendor", route113); - app.use("/api/task/getProject", route114); - app.use("/api/task/getTaskApi", route115); - app.use("/api/task/getTaskCategories", route116); - app.use("/api/task/taskDetails", route117); - app.use("/api/test/test", route118); + app.use("/api/production/storyboard/generatestory2Image", route59); + app.use("/api/production/storyboard/getStoryboardData", route60); + app.use("/api/production/storyboard/pollingImage", route61); + app.use("/api/production/storyboard/previewImage", route62); + app.use("/api/production/workbench/confirmSelection", route63); + app.use("/api/production/workbench/delVideo", route64); + app.use("/api/production/workbench/generateVideo", route65); + app.use("/api/production/workbench/generateVideoPrompt", route66); + app.use("/api/production/workbench/getChatLines", route67); + app.use("/api/production/workbench/getVideoModelDetail", route68); + app.use("/api/production/workbench/videoPolling", route69); + app.use("/api/project/addProject", route70); + app.use("/api/project/addVisual", route71); + app.use("/api/project/addVisualManual", route72); + app.use("/api/project/deleteVisualManual", route73); + app.use("/api/project/delProject", route74); + app.use("/api/project/editProject", route75); + app.use("/api/project/editVisualManual", route76); + app.use("/api/project/getProject", route77); + app.use("/api/project/getVisualManual", route78); + app.use("/api/project/visualManual", route79); + app.use("/api/script/addScript", route80); + app.use("/api/script/delScript", route81); + app.use("/api/script/exportScript", route82); + app.use("/api/script/extractAssets", route83); + app.use("/api/script/getScrptApi", route84); + app.use("/api/script/pollScriptAssets", route85); + app.use("/api/script/updateScript", route86); + app.use("/api/scriptAgent/getPlanData", route87); + app.use("/api/scriptAgent/setPlanData", route88); + app.use("/api/setting/about/checkUpdate", route89); + app.use("/api/setting/about/downloadApp", route90); + app.use("/api/setting/agentDeploy/agentSetKey", route91); + app.use("/api/setting/agentDeploy/deployAgentModel", route92); + app.use("/api/setting/agentDeploy/getAgentDeploy", route93); + app.use("/api/setting/dbConfig/clearData", route94); + app.use("/api/setting/dev/getSwitchAiDevTool", route95); + app.use("/api/setting/dev/updateSwitchAiDevTool", route96); + app.use("/api/setting/fileManagement/openFolder", route97); + app.use("/api/setting/getTextModel", route98); + app.use("/api/setting/loginConfig/getUser", route99); + app.use("/api/setting/loginConfig/updateUserPwd", route100); + app.use("/api/setting/memoryConfig/delAllMemory", route101); + app.use("/api/setting/memoryConfig/getMemory", route102); + app.use("/api/setting/memoryConfig/sureMemory", route103); + app.use("/api/setting/promptManage/getPrompt", route104); + app.use("/api/setting/promptManage/updatePrompt", route105); + app.use("/api/setting/skillManagement/getSkillContent", route106); + app.use("/api/setting/skillManagement/getSkillList", route107); + app.use("/api/setting/skillManagement/saveSkillContent", route108); + app.use("/api/setting/vendorConfig/addVendor", route109); + app.use("/api/setting/vendorConfig/deleteVendor", route110); + app.use("/api/setting/vendorConfig/getVendorList", route111); + app.use("/api/setting/vendorConfig/modelTest", route112); + app.use("/api/setting/vendorConfig/updateCode", route113); + app.use("/api/setting/vendorConfig/updateVendor", route114); + app.use("/api/task/getProject", route115); + app.use("/api/task/getTaskApi", route116); + app.use("/api/task/getTaskCategories", route117); + app.use("/api/task/taskDetails", route118); + app.use("/api/test/test", route119); } diff --git a/src/routes/assets/batchGenerationData.ts b/src/routes/assets/batchGenerationData.ts index 47a624c..76a5bbf 100644 --- a/src/routes/assets/batchGenerationData.ts +++ b/src/routes/assets/batchGenerationData.ts @@ -6,35 +6,37 @@ import { validateFields } from "@/middleware/middleware"; const router = express.Router(); // 获取资产 -export default router.post("/", - validateFields({ - projectId: z.number(), - type: z.string(), - name: z.string().optional(), - page: z.number(), - limit: z.number(), - }), - async (req, res) => { - const { projectId, type, name, page = 1, limit = 10, } = req.body; - const offset = (page - 1) * limit; - let query = u.db("o_assets").select("*").where("projectId", projectId).andWhere("type", type); - if (name) { - query = query.andWhere("name", "like", `%${name}%`); - } - // 分页查询 - const parentAssets = await query.offset(offset).limit(limit); +export default router.post( + "/", + validateFields({ + projectId: z.number(), + type: z.string(), + name: z.string().optional(), + page: z.number(), + limit: z.number(), + }), + async (req, res) => { + const { projectId, type, name, page = 1, limit = 10 } = req.body; + const offset = (page - 1) * limit; + let query = u.db("o_assets").select("*").where("projectId", projectId).andWhere("type", type); + if (name) { + query = query.andWhere("name", "like", `%${name}%`); + } + // 分页查询 + const parentAssets = await query.offset(offset).limit(limit); - // 统计总数 - const totalQuery = (await u - .db("o_assets") - .where("projectId", projectId) - .andWhere("type", type) - .andWhere((qb) => { - if (name) { - qb.andWhere("name", "like", `%${name}%`); - } - }) - .count("* as total") - .first()) as any; - res.status(200).send(success({ data: parentAssets, total: totalQuery?.total })); - }); + // 统计总数 + const totalQuery = (await u + .db("o_assets") + .where("projectId", projectId) + .andWhere("type", type) + .andWhere((qb) => { + if (name) { + qb.andWhere("name", "like", `%${name}%`); + } + }) + .count("* as total") + .first()) as any; + res.status(200).send(success({ data: parentAssets, total: totalQuery?.total })); + }, +); diff --git a/src/routes/production/assets/batchGenerateAssetsImage.ts b/src/routes/production/assets/batchGenerateAssetsImage.ts index 79d5b0f..7d4e840 100644 --- a/src/routes/production/assets/batchGenerateAssetsImage.ts +++ b/src/routes/production/assets/batchGenerateAssetsImage.ts @@ -5,6 +5,7 @@ import sharp from "sharp"; import { success } from "@/lib/responseFormat"; import { validateFields } from "@/middleware/middleware"; import { Output } from "ai"; +import { urlToBase64 } from "@/utils/vm"; const router = express.Router(); export default router.post( @@ -19,7 +20,25 @@ export default router.post( const projectSettingData = await u.db("o_project").where("id", projectId).select("imageModel", "imageQuality", "artStyle").first(); - const assetsDataArr = await u.db("o_assets").whereIn("id", assetIds).select("id", "describe", "name", "type"); + const assetsDataArr = await u.db("o_assets").whereIn("id", assetIds).select("id", "describe", "name", "type", "assetsId"); + const parentIds = assetsDataArr.map((item) => item.assetsId).filter((id) => id !== null); + const parentAssetsData = await u + .db("o_assets") + .leftJoin("o_image", "o_assets.imageId", "o_image.id") + .whereIn("o_assets.id", parentIds as number[]) + .select("o_assets.id", "o_image.filePath"); + const assetsSrcArr = await Promise.all( + parentAssetsData.map(async (item) => { + return { + src: await u.oss.getFileUrl(item.filePath), + id: item.id, + }; + }), + ); + const imageUrlRecord: Record = {}; + assetsSrcArr.forEach((item) => { + imageUrlRecord[item.id] = item.src; + }); const rolePrompt = u.getArtPrompt(projectSettingData!.artStyle!, "art_character_derivative"); const toolPrompt = u.getArtPrompt(projectSettingData!.artStyle!, "art_prop_derivative"); const scenePrompt = u.getArtPrompt(projectSettingData!.artStyle!, "art_scene_derivative"); @@ -28,7 +47,7 @@ export default router.post( tool: toolPrompt, scene: scenePrompt, }; - + const imageData = []; for (const item of assetsDataArr) { const { text } = await u.Ai.Text("universalAi").invoke({ system: ` @@ -55,26 +74,42 @@ export default router.post( resolution: projectSettingData?.imageQuality, model: projectSettingData?.imageModel, }); - u.Ai.Image(projectSettingData?.imageModel as `${string}:${string}`) - .run({ + const imageBase64 = imageUrlRecord[item.assetsId!] ? await urlToBase64(imageUrlRecord[item.assetsId!]) : null; + try { + const imageCls = await u.Ai.Image(projectSettingData?.imageModel as `${string}:${string}`).run({ prompt: text, - imageBase64: [], + imageBase64: imageBase64 ? [imageBase64] : [], size: projectSettingData?.imageQuality as "1K" | "2K" | "4K", aspectRatio: "16:9", taskClass: "生成图片", describe: "资产图片生成", relatedObjects: JSON.stringify(repeloadObj), projectId: projectId, - }) - .then(async (imageCls) => { - const savePath = `/${projectId}/assets/${scriptId}/${u.uuid()}.jpg`; - await imageCls.save(savePath); - // 更新对应数据库 - await u.db("o_assets").where("id", item.id).update({ imageId: imageId }); - await u.db("o_image").where({ id: imageId }).update({ state: "已完成", filePath: savePath }); }); + const savePath = `/${projectId}/assets/${scriptId}/${u.uuid()}.jpg`; + await imageCls.save(savePath); + // 更新对应数据库 + await u.db("o_assets").where("id", item.id).update({ imageId: imageId }); + await u.db("o_image").where({ id: imageId }).update({ state: "已完成", filePath: savePath }); + imageData.push({ + id: item.id, + state: "已完成", + src: await u.oss.getFileUrl(savePath), + }); + } catch (e) { + console.log("%c Line:95 🥛 e", "background:#fca650", e); + await u + .db("o_image") + .where({ id: imageId }) + .update({ state: "生成失败", reason: u.error(e).message }); + imageData.push({ + id: item.id, + state: "生成失败", + src: "", + }); + } } - return res.status(200).send(success()); + return res.status(200).send(success(imageData)); }, ); diff --git a/src/routes/production/storyboard/batchGenerateImage.ts b/src/routes/production/storyboard/batchGenerateImage.ts index bf433ea..5308af2 100644 --- a/src/routes/production/storyboard/batchGenerateImage.ts +++ b/src/routes/production/storyboard/batchGenerateImage.ts @@ -4,28 +4,137 @@ import { z } from "zod"; import sharp from "sharp"; import { success } from "@/lib/responseFormat"; import { validateFields } from "@/middleware/middleware"; -import { Output } from "ai"; +import { Output, tool } from "ai"; +import { urlToBase64 } from "@/utils/vm"; +import { assetItemSchema } from "@/agents/productionAgent/tools"; const router = express.Router(); +export type AssetData = z.infer; export default router.post( "/", validateFields({ - storyboardIds: z.array(z.number()), + storyboardIds: z.array(z.number()).optional(), projectId: z.number(), scriptId: z.number(), + script: z.string(), + scriptPlan: z.string(), + storyboardTable: z.string(), + assets: z.array(assetItemSchema), }), async (req, res) => { - const { storyboardIds, projectId, scriptId } = req.body; + const { + storyboardIds, + projectId, + scriptId, + script, + scriptPlan, + storyboardTable, + assets, + }: { + storyboardIds: number[]; + projectId: number; + scriptId: number; + script: string; + scriptPlan: string; + storyboardTable: string; + assets: AssetData[]; + } = req.body; + // 当没有 storyboardIds 时，通过 AI 生成新的分镜面板数据 + let finalStoryboardIds: number[] = storyboardIds || []; + if (!storyboardIds || storyboardIds.length === 0) { + const createdIds: number[] = []; + const resultTools = tool({ + description: "结果输出工具（必须调用）", + inputSchema: z.object({ + items: z.array( + z.object({ + title: z.string().describe("分镜名称"), + description: z.string().describe("分镜详细描述"), + relatedAssets: z.array(z.number()).describe("关联衍生资产id数组"), + }), + ), + }), + execute: async (resData) => { + console.log("%c Line:46 🌰 resData", "background:#93c0a4", resData.items); + for (const item of resData.items) { + const [id] = await u.db("o_storyboard").insert({ + title: item.title, + description: item.description, + scriptId: scriptId, + }); + createdIds.push(id); + if (item.relatedAssets.length === 0) continue; + await u.db("o_assets2Storyboard").insert(item.relatedAssets.map((i) => ({ storyboardId: id, assetId: i }))); + console.log("%c Line:68 🍷 createdIds", "background:#33a5ff", createdIds); + } + return true; + }, + }); + const { text } = await u.Ai.Text("universalAi").invoke({ + system: ` + 你需要根据用户提供的剧本、分镜表、拍摄计划和资产列表，来生成一个分镜面板，内容结构为 [{title:"分镜名称",description:"分镜详细描述",relatedAssets:关联衍生资产id}]。 + 你必须调用 resultTools 来输出结果，传入的参数需要包含 items 字段，items 是一个数组，每个元素包含 title（分镜名称）,description（分镜详细描述）,relatedAssets（关联衍生资产id数组）。请直接输出调用工具的代码，不要做任何多余的描述性文字，必须等待工具调用完成。调用工具后你本身的回复 请保持空白，不要添加任何内容。`, + messages: [ + { + role: "user", + content: ` + ====== 剧本 ====== + ${script} + ====== 分镜表 ====== + ${storyboardTable} + ====== 拍摄计划 ====== + ${scriptPlan} + ====== 资产列表 ====== + ${assets.map((i) => i.derive.map((t) => `衍生资产名称:${t.name},衍生资产类型:${t.type},关联资产ID:${t.assetsId}`).join("\n")).join("\n")} + `, + }, + ], + tools: { resultTools }, + }); + console.log("%c Line:52 🍢 text", "background:#93c0a4", text); + finalStoryboardIds = createdIds; + } + await u.db("o_storyboard").whereIn("id", finalStoryboardIds).where("scriptId", scriptId).update({ state: "生成中" }); + console.log("%c Line:98 🍯 finalStoryboardIds", "background:#3f7cff", finalStoryboardIds); + + if (finalStoryboardIds.length === 0) { + res.status(200).send(success()); + return; + } const projectSettingData = await u.db("o_project").where("id", projectId).select("imageModel", "imageQuality", "artStyle").first(); const sceneArkPrompt = u.getArtPrompt(projectSettingData?.artStyle || "", "art_storyboard"); - const storyboardData = await u.db("o_storyboard").whereIn("id", storyboardIds).select("id", "description", "title"); - + const storyboardData = await u.db("o_storyboard").where("scriptId", scriptId).whereIn("id", finalStoryboardIds); + const assetData = await u + .db("o_assets") + .leftJoin("o_assets2Storyboard", "o_assets.id", "o_assets2Storyboard.assetId") + .whereIn("o_assets2Storyboard.storyboardId", finalStoryboardIds) + .select("o_assets2Storyboard.storyboardId", "o_assets.imageId"); + const assetRecord: Record = {}; + assetData.forEach((item: any) => { + if (!assetRecord[item.storyboardId]) { + assetRecord[item.storyboardId] = []; + } + assetRecord[item.storyboardId].push(item.imageId); + }); + res.status(200).send( + success( + storyboardData.map((i) => ({ + id: i.id, + title: i.title, + description: i.description, + prompt: "", + associateAssetsIds: assetRecord[i.id!], + src: null, + state: i.state, + })), + ), + ); for (const item of storyboardData) { const { text } = await u.Ai.Text("universalAi").invoke({ system: ` - 你需要根据用户提供的分镜的标题与描述，结合当前项目的美术风格，为我优化提示词以便生成更符合项目美术风格的分镜图片。请你只优化提示词，不要添加任何额外的描述性文字，请以JSON格式输出: [{id:"对应分镜ID",prompt:"分镜提示词"}]。 + 你需要根据用户提供的分镜的标题与描述，结合当前项目的美术风格，为我生成一段提示词以便生成更符合项目美术风格的分镜图片。直接输出提示词，不做任何解释说明。 美术风格：${sceneArkPrompt}`, messages: [ { @@ -34,6 +143,8 @@ export default router.post( }, ], }); + console.log("%c Line:27 🍫 text", "background:#ffdd4d", text); + const repeloadObj = { prompt: text, size: projectSettingData?.imageQuality as "1K" | "2K" | "4K", @@ -46,7 +157,7 @@ export default router.post( u.Ai.Image(projectSettingData?.imageModel as `${string}:${string}`) .run({ prompt: text, - imageBase64: [], + imageBase64: await getAssetsImageBase64(assetRecord[item.id!] || []), size: projectSettingData?.imageQuality as "1K" | "2K" | "4K", aspectRatio: "16:9", taskClass: "生成图片", @@ -72,7 +183,28 @@ export default router.post( }); }); } - - return res.status(200).send(success()); }, ); +async function getAssetsImageBase64(imageIds: number[]) { + if (imageIds.length === 0) return []; + const imagePaths = await u + .db("o_assets") + .leftJoin("o_image", "o_assets.imageId", "o_image.id") + .whereIn("o_assets.id", imageIds) + .select("o_assets.id", "o_image.filePath"); + if (!imagePaths.length) return []; + const imageUrls = await Promise.all( + imagePaths.map(async (i) => { + if (i.filePath) { + try { + return await urlToBase64(await u.oss.getFileUrl(i.filePath)); + } catch { + return null; + } + } else { + return null; + } + }), + ); + return imageUrls.filter(Boolean) as string[]; +} diff --git a/src/routes/script/extractAssets.ts b/src/routes/script/extractAssets.ts index 16868a7..4acfc05 100644 --- a/src/routes/script/extractAssets.ts +++ b/src/routes/script/extractAssets.ts @@ -63,8 +63,6 @@ export default router.post( if (!scriptIds.length) return res.status(400).send(error("请先选择剧本")); const scripts = await u.db("o_script").whereIn("id", scriptIds); const intansce = u.Ai.Text("universalAi"); - const novelData = await u.db("o_novel").where("projectId", projectId).select("chapterData"); - if (!novelData || novelData.length === 0) return res.status(400).send(error("请先上传小说")); await u.db("o_script").whereIn("id", scriptIds).update({ extractState: 0, }); diff --git a/src/types/database.d.ts b/src/types/database.d.ts index 2a3bc57..4544078 100644 --- a/src/types/database.d.ts +++ b/src/types/database.d.ts @@ -1,6 +1,13 @@ -// @db-hash 93b2462070c45c2b449e9a18c4e88763 +// @db-hash f7bc2fdb80756d5536929eb47155578b //该文件由脚本自动生成，请勿手动修改 +export interface _o_script_old_20260327 { + 'content'?: string | null; + 'createTime'?: number | null; + 'id'?: number; + 'name'?: string | null; + 'projectId'?: number | null; +} export interface memories { 'content': string; 'createTime': number; @@ -21,7 +28,7 @@ export interface o_agentDeploy { 'model'?: string | null; 'modelName'?: string | null; 'name'?: string | null; - 'vendorId'?: string | null; + 'vendorId'?: number | null; } export interface o_agentWorkData { 'createTime'?: number | null; @@ -47,7 +54,6 @@ export interface o_assets { 'name'?: string | null; 'projectId'?: number | null; 'prompt'?: string | null; - 'promptState'?: string | null; 'remark'?: string | null; 'scriptId'?: number | null; 'startTime'?: number | null; @@ -167,7 +173,7 @@ export interface o_storyboard { 'filePath'?: string | null; 'frameMode'?: string | null; 'id'?: number; - 'index'?: number | null; + 'index'?: string | null; 'lines'?: string | null; 'mode'?: string | null; 'model'?: string | null; @@ -232,6 +238,7 @@ export interface o_videoConfig { } export interface DB { + "_o_script_old_20260327": _o_script_old_20260327; "memories": memories; "o_agentDeploy": o_agentDeploy; "o_agentWorkData": o_agentWorkData; diff --git a/yarn.lock b/yarn.lock index ea73d45..038e4b1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -40,10 +40,10 @@ "@hono/node-server" "^1.13.7" hono "^4.6.14" -"@ai-sdk/gateway@3.0.82": - version "3.0.82" - resolved "https://registry.npmmirror.com/@ai-sdk/gateway/-/gateway-3.0.82.tgz#904f150486ad3f9456c71ba6732c3f4b518c76d1" - integrity sha512-ddB9FrkHZank1zyx13vypU0RrPjsXWj3NvlsrJ4yFQnrpR+xh48W4wO9ijndUueBsaADjlvMz2Ghv8yq5tZGCQ== +"@ai-sdk/gateway@3.0.83": + version "3.0.83" + resolved "https://registry.npmmirror.com/@ai-sdk/gateway/-/gateway-3.0.83.tgz#214812d3a2f15447adf4ee81d3ed77dd137b41b3" + integrity sha512-LvlWujbSdEkTBXBLFtF7GS6riXdHhH0O+DpDrCaNQvXeHmSF2jKsOg7JWXiCgygAHM5cWFAO3JYmZp83DjiuBQ== dependencies: "@ai-sdk/provider" "3.0.8" "@ai-sdk/provider-utils" "4.0.21" @@ -423,9 +423,9 @@ integrity sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw== "@hono/node-server@^1.13.7": - version "1.19.11" - resolved "https://registry.npmmirror.com/@hono/node-server/-/node-server-1.19.11.tgz#dc419f0826dd2504e9fc86ad289d5636a0444e2f" - integrity sha512-dr8/3zEaB+p0D2n/IUrlPF1HZm586qgJNXK1a9fhg/PzdtkK7Ksd5l312tJX2yBuALqDYBlG20QEbayqPyxn+g== + version "1.19.12" + resolved "https://registry.npmmirror.com/@hono/node-server/-/node-server-1.19.12.tgz#dae075247959b6d7d2dba4c8bdc8c452ca0c7b40" + integrity sha512-txsUW4SQ1iilgE0l9/e9VQWmELXifEFvmdA1j6WFh/aFPj99hIntrSsq/if0UWyGVkmrRPKA1wCeP+UCr1B9Uw== "@huggingface/jinja@^0.5.3": version "0.5.6" @@ -1042,9 +1042,9 @@ integrity sha512-Fw28YZpRnA3cAHHDlkt7xQHiJ0fcL+NRcIqsocZQUSmbzeIKRpwttJjik5ZGanXP+vlA4SbTg+AbA3bP363l+w== "@xmldom/xmldom@^0.8.8": - version "0.8.11" - resolved "https://registry.npmmirror.com/@xmldom/xmldom/-/xmldom-0.8.11.tgz#b79de2d67389734c57c52595f7a7305e30c2d608" - integrity sha512-cQzWCtO6C8TQiYl1ruKNn2U6Ao4o4WBBcbL61yJl84x+j5sOWWFU9X7DpND8XZG3daDppSsigMdfAIl2upQBRw== + version "0.8.12" + resolved "https://registry.npmmirror.com/@xmldom/xmldom/-/xmldom-0.8.12.tgz#cf488a5435fa06c7374ad1449c69cea0f823624b" + integrity sha512-9k/gHF6n/pAi/9tqr3m3aqkuiNosYTurLLUtc7xQ9sxB/wm7WPygCv8GYa6mS0fLJEHhqMC1ATYhz++U/lRHqg== abbrev@1, abbrev@^1.0.0: version "1.1.1" @@ -1117,11 +1117,11 @@ aggregate-error@^3.0.0: indent-string "^4.0.0" ai@^6.0.67: - version "6.0.140" - resolved "https://registry.npmmirror.com/ai/-/ai-6.0.140.tgz#1135f3fa05c760e346984f02f55de8e2b9800691" - integrity sha512-+jf6fQDPZe+gQlzPoP5mzy+DdfYOpE0cgUm99U8OxVTIPv19gCDuNzlKTZSntcQzLbo6LFXyNhJdV7XTWQ+5vA== + version "6.0.141" + resolved "https://registry.npmmirror.com/ai/-/ai-6.0.141.tgz#c476280ae69b13ad11f2671e49a24ac0412a0b54" + integrity sha512-+GomGQWaId3xN0wcugUW/H7xMMaFkID2PiS7K/Wugj45G3efv0BXhQ3psRZoQVoRbOpdNoUqcK/KTB+FR4h6qg== dependencies: - "@ai-sdk/gateway" "3.0.82" + "@ai-sdk/gateway" "3.0.83" "@ai-sdk/provider" "3.0.8" "@ai-sdk/provider-utils" "4.0.21" "@opentelemetry/api" "1.9.0" @@ -1303,13 +1303,13 @@ axios-retry@^4.5.0: is-retry-allowed "^2.2.0" axios@^1.13.2: - version "1.13.6" - resolved "https://registry.npmmirror.com/axios/-/axios-1.13.6.tgz#c3f92da917dc209a15dd29936d20d5089b6b6c98" - integrity sha512-ChTCHMouEe2kn713WHbQGcuYrr6fXTBiu460OTwWrWob16g1bXn4vtz07Ope7ewMozJAnEquLk5lWQWtBig9DQ== + version "1.14.0" + resolved "https://registry.npmmirror.com/axios/-/axios-1.14.0.tgz#7c29f4cf2ea91ef05018d5aa5399bf23ed3120eb" + integrity sha512-3Y8yrqLSwjuzpXuZ0oIYZ/XGgLwUIBU3uLvbcpb0pidD9ctpShJd43KSlEEkVQg6DS0G9NKyzOvBfUtDKEyHvQ== dependencies: follow-redirects "^1.15.11" form-data "^4.0.5" - proxy-from-env "^1.1.0" + proxy-from-env "^2.1.0" balanced-match@^1.0.0: version "1.0.2" @@ -1396,17 +1396,17 @@ boolean@^3.0.1: integrity sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw== brace-expansion@^1.1.7: - version "1.1.12" - resolved "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.12.tgz#ab9b454466e5a8cc3a187beaad580412a9c5b843" - integrity sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg== + version "1.1.13" + resolved "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.13.tgz#d37875c01dc9eff988dd49d112a57cb67b54efe6" + integrity sha512-9ZLprWS6EENmhEOpjCYW2c8VkmOvckIJZfkr7rBW6dObmfgJ/L1GpSYW5Hpo9lDz4D1+n0Ckz8rU7FwHDQiG/w== dependencies: balanced-match "^1.0.0" concat-map "0.0.1" brace-expansion@^2.0.1, brace-expansion@^2.0.2: - version "2.0.2" - resolved "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-2.0.2.tgz#54fc53237a613d854c7bd37463aad17df87214e7" - integrity sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ== + version "2.0.3" + resolved "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-2.0.3.tgz#0493338bdd58e319b1039c67cf7ee439892c01d9" + integrity sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA== dependencies: balanced-match "^1.0.0" @@ -1870,9 +1870,9 @@ cross-spawn@^7.0.1, cross-spawn@^7.0.6: which "^2.0.1" custom-electron-titlebar@^4.2.8: - version "4.4.0" - resolved "https://registry.npmmirror.com/custom-electron-titlebar/-/custom-electron-titlebar-4.4.0.tgz#0b6aff12db97babf237d052e0d2ea24623de4783" - integrity sha512-bvYxxDQiG/23eMPDAdhjjOXviEXnShHRo8VCDCouVfVDdG3Xs8SkXISYDZK3pyhINsrrJ21s9tSU7h59WDr5oA== + version "4.4.1" + resolved "https://registry.npmmirror.com/custom-electron-titlebar/-/custom-electron-titlebar-4.4.1.tgz#aea64f009697c9771cb2a67d2eb5ac8059696906" + integrity sha512-I+sOGBdslrGpuCWlhda8P0vtRAZK+W2NzjHLsxTiE2bNmhAIs9YLDe6iRBExwU1xVZt+J1hSXzUT67BlAuMWLA== debug@2.6.9: version "2.6.9" @@ -3212,9 +3212,9 @@ keyv@^4.0.0: json-buffer "3.0.1" knex@^3.1.0, knex@^3.2.5: - version "3.2.6" - resolved "https://registry.npmmirror.com/knex/-/knex-3.2.6.tgz#d6653ebf7961a83ae0014db1a7c56c75ce1ffa00" - integrity sha512-26I1Tvx0D9SOsFBF9jRWT/JdE3FPI52jAwYpXfREEtnoqgjGvAi8/ux8ktjaTC9IQcAVTCrREkOpa7lrjeAcow== + version "3.2.7" + resolved "https://registry.npmmirror.com/knex/-/knex-3.2.7.tgz#53bc16470217f12fef516a7a649794a405d3473d" + integrity sha512-VxdDE72x7Tc08E5yCu8HqYoeOm0HOjAraOtYiGSAUJTYkydwfSGBOpQqYHrzM5vjLNzw2JDL2vDH8m7DjIjtgA== dependencies: colorette "2.0.19" commander "^10.0.0" @@ -4230,10 +4230,10 @@ proxy-addr@^2.0.7: forwarded "0.2.0" ipaddr.js "1.9.1" -proxy-from-env@^1.1.0: - version "1.1.0" - resolved "https://registry.npmmirror.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" - integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== +proxy-from-env@^2.1.0: + version "2.1.0" + resolved "https://registry.npmmirror.com/proxy-from-env/-/proxy-from-env-2.1.0.tgz#a7487568adad577cfaaa7e88c49cab3ab3081aba" + integrity sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA== pstree.remy@^1.1.8: version "1.1.8" From 3998d441d44c0cefeaa80019880830263eb2f79f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=B8=85?= <2944435683> Date: Mon, 30 Mar 2026 21:55:12 +0800 Subject: [PATCH 2/2] =?UTF-8?q?ai=E7=94=9F=E6=88=90=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=E5=8D=95=E7=8B=AC=E6=8B=BF=E5=87=BA=E6=9D=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/lib/initDB.ts | 179 +++++++++--------- .../batchGenerateImageAssets.ts | 24 ++- src/routes/assetsGenerate/generateAssets.ts | 24 ++- .../production/workbench/generateVideo.ts | 30 +-- src/types/database.d.ts | 5 +- src/utils/ai.ts | 38 ++-- 6 files changed, 147 insertions(+), 153 deletions(-) diff --git a/src/lib/initDB.ts b/src/lib/initDB.ts index f207906..fd0bea9 100644 --- a/src/lib/initDB.ts +++ b/src/lib/initDB.ts @@ -263,96 +263,95 @@ export default async (knex: Knex, forceInit: boolean = false): Promise => { name: "剧本资产提取", type: "scriptAssetExtraction", - data: ` - --- - name: universal_agent - description: 专注于从剧本内容中提取所使用的资产（角色、场景、道具）并生成结构化资产列表的助手。 - --- - - # Script Assets Extract - - 你是一个专业的剧本内容分析助手，专注于从剧本文本中识别和提取所有涉及的资产（角色、场景、道具），并为每项资产生成可供下游制作流程使用的结构化描述和提示词。 - - ## 何时使用 - - 用户提供剧本内容，你需要逐段阅读并提取其中涉及的所有资产（人物角色、场景地点、道具物件），输出为结构化的资产列表。产出的资产描述将用于后续 AI 图片生成和制作流程。 - - ## 与系统的对应关系 - - - 资产类型： - - \`role\` — 角色（对应 \`o_assets.type = "role"\`） - - \`scene\` — 场景（对应 \`o_assets.type = "scene"\`） - - \`tool\` — 道具（对应 \`o_assets.type = "tool"\`） - - 下游用途：资产提示词生成 → AI 资产图生成 → 分镜制作 - - ## 输出要求 - - **必须通过调用 \`resultTool\` 工具返回结果**，禁止以纯文本、Markdown 表格或 JSON 代码块等形式直接输出资产列表。 - \`resultTool\` 的 schema 会对字段类型和枚举值做强校验，调用时请严格按照下方字段定义填写，确保数据结构正确、字段完整、类型匹配。 - - 每个资产对象包含以下字段： - - | 字段 | 类型 | 必填 | 说明 | - | ---- | ---- | ---- | ---- | - | \`name\` | string | 是 | 资产名称，使用剧本中的原始称呼,不做其他多余描述 | - | \`desc\` | string | 是 | 资产描述，30-80 字的视觉化描述 | - | \`prompt\` | string | 是 | 生成提示词，英文，用于 AI 图片生成 | - | \`type\` | enum | 是 | 资产类型：\`role\` / \`scene\` / \`tool\` | - - ## 提取规则 - - ### 角色（role） - - - 提取剧本中出现的所有有名字的角色 - - \`desc\`：包含外貌特征、服饰风格、体态气质等视觉要素 - - \`prompt\`：英文提示词，描述角色的外观特征，适用于 AI 角色图生成 - - 同一角色有多个称呼时，取最常用的作为 \`name\` - - 无名龙套（如"路人甲"、"士兵"）可跳过，除非其造型对剧情有重要视觉意义 - - ### 场景（scene） - - - 提取剧本中出现的所有场景/地点 - - \`desc\`：包含空间结构、光照氛围、关键陈设、色调基调等视觉要素 - - \`prompt\`：英文提示词，描述场景的整体视觉风格，适用于 AI 场景图生成 - - 同一场景的不同状态（如白天/夜晚）不重复提取，在 \`desc\` 中注明即可 - - ### 道具（tool） - - - 提取剧本中出现的重要道具/物品 - - \`desc\`：包含外观形状、颜色材质、尺寸参考、特殊效果等视觉要素 - - \`prompt\`：英文提示词，描述道具的外观细节，适用于 AI 道具图生成 - - 仅提取有独立视觉意义或剧情功能的道具，通用物品可跳过 - - - ## 提示词（prompt）生成规范 - - - 采用逗号分隔的关键词/短语格式 - - 优先描述**视觉特征**，避免抽象概念 - - 包含风格关键词（如 anime style, manga style 等，根据项目风格决定） - - 角色 prompt 示例：\`a young man, sharp eyebrows, black hair, pale skin, wearing a gray Taoist robe, slender build, cold expression\` - - 场景 prompt 示例：\`dark cave interior, glowing crystals on walls, misty atmosphere, dim blue lighting, stone altar in center\` - - 道具 prompt 示例：\`ancient jade pendant, oval shape, translucent green, carved dragon pattern, glowing faintly\` - - ## 提取流程 - - 1. 通读剧本全文，识别所有出现的角色、场景、道具 - 2. 对每个资产生成结构化的 \`name\`、\`desc\`、\`prompt\`、\`type\` - 3. 去重：同一资产不重复提取 - 4. **必须通过调用 \`resultTool\` 工具输出完整资产列表**，不要分多次调用，一次性将所有资产放入 \`assetsList\` 数组中提交 - - ## 提取原则 - - 1. **忠于剧本**：所有提取基于剧本中的实际内容，不臆造未出现的资产 - 2. **视觉优先**：描述和提示词聚焦视觉特征，便于 AI 图片生成 - 3. **精简实用**：只提取对制作有实际意义的资产，避免过度提取 - 4. **分类准确**：严格按照 role/scene/tool 分类，不混淆 - 5. **提示词质量**：英文提示词应具体、可执行，能直接用于 AI 图片生成 - - ## 注意事项 - - - 资产列表中**不要包含剧本内容本身**，仅提取所使用到的资产 - - 角色的随身物品如果有独立剧情功能，应单独作为道具提取 - - 场景中的固定陈设不需要单独提取为道具，除非该物件有独立剧情作用 + data: `--- +name: universal_agent +description: 专注于从剧本内容中提取所使用的资产（角色、场景、道具）并生成结构化资产列表的助手。 +--- + +# Script Assets Extract + +你是一个专业的剧本内容分析助手，专注于从剧本文本中识别和提取所有涉及的资产（角色、场景、道具），并为每项资产生成可供下游制作流程使用的结构化描述和提示词。 + +## 何时使用 + +用户提供剧本内容，你需要逐段阅读并提取其中涉及的所有资产（人物角色、场景地点、道具物件），输出为结构化的资产列表。产出的资产描述将用于后续 AI 图片生成和制作流程。 + +## 与系统的对应关系 + +- 资产类型： + - \`role\` — 角色（对应 \`o_assets.type = "role"\`） + - \`scene\` — 场景（对应 \`o_assets.type = "scene"\`） + - \`tool\` — 道具（对应 \`o_assets.type = "tool"\`） +- 下游用途：资产提示词生成 → AI 资产图生成 → 分镜制作 + +## 输出要求 + +**必须通过调用 \`resultTool\` 工具返回结果**，禁止以纯文本、Markdown 表格或 JSON 代码块等形式直接输出资产列表。 +\`resultTool\` 的 schema 会对字段类型和枚举值做强校验，调用时请严格按照下方字段定义填写，确保数据结构正确、字段完整、类型匹配。 + +每个资产对象包含以下字段： + +| 字段 | 类型 | 必填 | 说明 | +| ---- | ---- | ---- | ---- | +| \`name\` | string | 是 | 资产名称，使用剧本中的原始称呼,不做其他多余描述 | +| \`desc\` | string | 是 | 资产描述，30-80 字的视觉化描述 | +| \`prompt\` | string | 是 | 生成提示词，英文，用于 AI 图片生成 | +| \`type\` | enum | 是 | 资产类型：\`role\` / \`scene\` / \`tool\` | + +## 提取规则 + +### 角色（role） + +- 提取剧本中出现的所有有名字的角色 +- \`desc\`：包含外貌特征、服饰风格、体态气质等视觉要素 +- \`prompt\`：英文提示词，描述角色的外观特征，适用于 AI 角色图生成 +- 同一角色有多个称呼时，取最常用的作为 \`name\` +- 无名龙套（如"路人甲"、"士兵"）可跳过，除非其造型对剧情有重要视觉意义 + +### 场景（scene） + +- 提取剧本中出现的所有场景/地点 +- \`desc\`：包含空间结构、光照氛围、关键陈设、色调基调等视觉要素 +- \`prompt\`：英文提示词，描述场景的整体视觉风格，适用于 AI 场景图生成 +- 同一场景的不同状态（如白天/夜晚）不重复提取，在 \`desc\` 中注明即可 + +### 道具（tool） + +- 提取剧本中出现的重要道具/物品 +- \`desc\`：包含外观形状、颜色材质、尺寸参考、特殊效果等视觉要素 +- \`prompt\`：英文提示词，描述道具的外观细节，适用于 AI 道具图生成 +- 仅提取有独立视觉意义或剧情功能的道具，通用物品可跳过 + + +## 提示词（prompt）生成规范 + +- 采用逗号分隔的关键词/短语格式 +- 优先描述**视觉特征**，避免抽象概念 +- 包含风格关键词（如 anime style, manga style 等，根据项目风格决定） +- 角色 prompt 示例：\`a young man, sharp eyebrows, black hair, pale skin, wearing a gray Taoist robe, slender build, cold expression\` +- 场景 prompt 示例：\`dark cave interior, glowing crystals on walls, misty atmosphere, dim blue lighting, stone altar in center\` +- 道具 prompt 示例：\`ancient jade pendant, oval shape, translucent green, carved dragon pattern, glowing faintly\` + +## 提取流程 + +1. 通读剧本全文，识别所有出现的角色、场景、道具 +2. 对每个资产生成结构化的 \`name\`、\`desc\`、\`prompt\`、\`type\` +3. 去重：同一资产不重复提取 +4. **必须通过调用 \`resultTool\` 工具输出完整资产列表**，不要分多次调用，一次性将所有资产放入 \`assetsList\` 数组中提交 + +## 提取原则 + +1. **忠于剧本**：所有提取基于剧本中的实际内容，不臆造未出现的资产 +2. **视觉优先**：描述和提示词聚焦视觉特征，便于 AI 图片生成 +3. **精简实用**：只提取对制作有实际意义的资产，避免过度提取 +4. **分类准确**：严格按照 role/scene/tool 分类，不混淆 +5. **提示词质量**：英文提示词应具体、可执行，能直接用于 AI 图片生成 + +## 注意事项 + +- 资产列表中**不要包含剧本内容本身**，仅提取所使用到的资产 +- 角色的随身物品如果有独立剧情功能，应单独作为道具提取 +- 场景中的固定陈设不需要单独提取为道具，除非该物件有独立剧情作用 `, }, ]); diff --git a/src/routes/assetsGenerate/batchGenerateImageAssets.ts b/src/routes/assetsGenerate/batchGenerateImageAssets.ts index 1fce062..55e4e30 100644 --- a/src/routes/assetsGenerate/batchGenerateImageAssets.ts +++ b/src/routes/assetsGenerate/batchGenerateImageAssets.ts @@ -110,16 +110,20 @@ export default router.post("/", validateFields(requestSchema), async (req, res) try { const aiImage = u.Ai.Image(model); - await aiImage.run({ - prompt: userPrompt, - imageBase64: item.base64 ? [item.base64] : [], - size: resolution, - aspectRatio: "16:9", - taskClass: cfg.taskClass, - describe, - projectId, - relatedObjects: JSON.stringify(relatedObjects), - }); + await aiImage.run( + { + prompt: userPrompt, + imageBase64: item.base64 ? [item.base64] : [], + size: resolution, + aspectRatio: "16:9", + }, + { + taskClass: cfg.taskClass, + describe, + projectId, + relatedObjects: JSON.stringify(relatedObjects), + }, + ); aiImage.save(imagePath); const imageData = await u.db("o_image").where("id", imageId).select("*").first(); diff --git a/src/routes/assetsGenerate/generateAssets.ts b/src/routes/assetsGenerate/generateAssets.ts index 4314a4b..830c536 100644 --- a/src/routes/assetsGenerate/generateAssets.ts +++ b/src/routes/assetsGenerate/generateAssets.ts @@ -98,16 +98,20 @@ export default router.post("/", validateFields(requestSchema), async (req, res) try { // 4. 调用 AI 生成图片 const aiImage = u.Ai.Image(model); - await aiImage.run({ - prompt: userPrompt, - imageBase64: base64 ? [base64] : [], - size: resolution, - aspectRatio: "16:9", - taskClass: cfg.taskClass, - describe, - projectId, - relatedObjects: JSON.stringify(relatedObjects), - }); + await aiImage.run( + { + prompt: userPrompt, + imageBase64: base64 ? [base64] : [], + size: resolution, + aspectRatio: "16:9", + }, + { + taskClass: cfg.taskClass, + describe, + projectId, + relatedObjects: JSON.stringify(relatedObjects), + }, + ); aiImage.save(imagePath); // 5. 更新记录 & 返回结果 diff --git a/src/routes/production/workbench/generateVideo.ts b/src/routes/production/workbench/generateVideo.ts index cb38602..eb7582b 100644 --- a/src/routes/production/workbench/generateVideo.ts +++ b/src/routes/production/workbench/generateVideo.ts @@ -105,19 +105,23 @@ export default router.post( console.log("%c Line:110 🍑 prompt", "background:#b03734", prompt); const aiVideo = u.Ai.Video(model); - await aiVideo.run({ - projectId, - prompt, - imageBase64: base64.filter((item) => item !== null) as string[], - mode, - duration, - aspectRatio: (ratio?.videoRatio as `${number}:${number}`) || "16:9", - resolution, - audio, - taskClass: "视频生成", - describe: "根据提示词生成视频", - relatedObjects: JSON.stringify(relatedObjects), - }); + await aiVideo.run( + { + prompt, + imageBase64: base64.filter((item) => item !== null) as string[], + mode, + duration, + aspectRatio: (ratio?.videoRatio as `${number}:${number}`) || "16:9", + resolution, + audio, + }, + { + projectId, + taskClass: "视频生成", + describe: "根据提示词生成视频", + relatedObjects: JSON.stringify(relatedObjects), + }, + ); await aiVideo.save(videoPath); await u.db("o_video").where("id", videoId).update({ state: "生成成功" }); // await u.db("o_videoConfig").where("storyboardId", storyboardId).update({ videoId, updateTime: Date.now() }); diff --git a/src/types/database.d.ts b/src/types/database.d.ts index 9b42b68..ab3e7f9 100644 --- a/src/types/database.d.ts +++ b/src/types/database.d.ts @@ -1,8 +1,4 @@ -<<<<<<< HEAD -// @db-hash 93b2462070c45c2b449e9a18c4e88763 -======= // @db-hash 24748d4ef971381a79c720c846f83847 ->>>>>>> 796947cef173e7fe2f96e21fa8aeae23ff0fdf4a //该文件由脚本自动生成，请勿手动修改 export interface memories { @@ -182,6 +178,7 @@ export interface o_storyboard { 'sound'?: string | null; 'state'?: string | null; 'title'?: string | null; + 'videoPrompt'?: string | null; } export interface o_tasks { 'describe'?: string | null; diff --git a/src/utils/ai.ts b/src/utils/ai.ts index 9c6df1e..4f3548d 100644 --- a/src/utils/ai.ts +++ b/src/utils/ai.ts @@ -25,23 +25,10 @@ async function getVendorTemplateFn(fnName: FnName, modelName: `${string}:${strin const selectedModel = modelList.find((i: any) => i.modelName == name); if (!selectedModel) throw new Error(`未找到模型 ${name} id=${id}`); const jsCode = transform(vendorConfigData.code!, { transforms: ["typescript"] }).code; - const running = u.vm(jsCode); - Object.assign(running.vendor.inputValues, JSON.parse(vendorConfigData.inputValues ?? "{}")); - running.vendor.models = modelList; - const fn = running[fnName]; + const fn = u.vm(jsCode)[fnName]; if (!fn) throw new Error(`未找到供应商配置中的函数 ${fnName} id=${id}`); - if (fnName == "textRequest") { - const model = fn(selectedModel); - if (!model) throw new Error(`供应商 textRequest 返回无效模型 id=${id}`); - return model; - } - return async (input: T) => { - const result = await fn(input, selectedModel); - if (result === undefined || result === null) { - throw new Error(`供应商函数 ${fnName} 未返回有效结果 id=${id}`); - } - return result; - }; + if (fnName == "textRequest") return fn(selectedModel); + else return (input: T) => fn(input, selectedModel); } async function withTaskRecord( @@ -113,6 +100,9 @@ interface ImageConfig { imageBase64: string[]; //输入的图片提示词 size: "1K" | "2K" | "4K"; // 图片尺寸 aspectRatio: `${number}:${number}`; // 长宽比 +} + +interface TaskRecord { taskClass: string; // 任务分类 describe: string; // 任务描述 relatedObjects: string; // 相关对象信息，便于后续分析和追踪 @@ -125,8 +115,8 @@ class AiImage { constructor(key: `${string}:${string}`) { this.key = key; } - async run(input: ImageConfig) { - return withTaskRecord(this.key, input.taskClass, input.describe, input.relatedObjects, input.projectId, async (modelName) => { + async run(input: ImageConfig, taskRecord: TaskRecord) { + return withTaskRecord(this.key, taskRecord.taskClass, taskRecord.describe, taskRecord.relatedObjects, taskRecord.projectId, async (modelName) => { const fn = await getVendorTemplateFn("imageRequest", modelName); this.result = await fn(input); if (this.result.startsWith("http")) this.result = await urlToBase64(this.result); @@ -139,7 +129,6 @@ class AiImage { } } interface VideoConfig { - projectId: number; // 项目ID prompt: string; //视频提示词 imageBase64: string[]; //输入的图片提示词 aspectRatio: `${number}:${number}`; // 长宽比 @@ -147,9 +136,6 @@ interface VideoConfig { duration: number; // 视频时长，单位秒 resolution: string; // 视频分辨率 audio: boolean; // 是否需要配音 - taskClass: string; // 任务分类 - describe: string; // 任务描述 - relatedObjects: string; // 相关对象信息，便于后续分析和追踪 } class AiVideo { @@ -158,8 +144,8 @@ class AiVideo { constructor(key: `${string}:${string}`) { this.key = key; } - async run(input: VideoConfig) { - return withTaskRecord(this.key, input.taskClass, input.describe, input.relatedObjects, input.projectId, async (modelName) => { + async run(input: VideoConfig, taskRecord: TaskRecord) { + return withTaskRecord(this.key, taskRecord.taskClass, taskRecord.describe, taskRecord.relatedObjects, taskRecord.projectId, async (modelName) => { const fn = await getVendorTemplateFn("videoRequest", modelName); this.result = await fn(input); if (this.result.startsWith("http")) this.result = await urlToBase64(this.result); @@ -177,8 +163,8 @@ class AiAudio { constructor(key: `${string}:${string}`) { this.key = key; } - async run(input: VideoConfig) { - return withTaskRecord(this.key, input.taskClass, input.describe, input.relatedObjects, input.projectId, async (modelName) => { + async run(input: VideoConfig, taskRecord: TaskRecord) { + return withTaskRecord(this.key, taskRecord.taskClass, taskRecord.describe, taskRecord.relatedObjects, taskRecord.projectId, async (modelName) => { const fn = await getVendorTemplateFn("ttsRequest", modelName); this.result = await fn(input); if (this.result.startsWith("http")) this.result = await urlToBase64(this.result);