From 9dcc54a8fa31ff07b83ad5724f4568d5f8aec86c Mon Sep 17 00:00:00 2001 From: zhishi <1951671751@qq.com> Date: Tue, 24 Mar 2026 18:43:13 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E5=AE=8C=E5=96=84=E5=88=86=E9=95=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/agents/productionAgent/tools.ts | 170 +++++++++++++++++- src/lib/initDB.ts | 2 + src/routes/production/getFlowData.ts | 10 +- .../production/workbench/getChatLines.ts | 1 + src/socket/routes/productionAgent.ts | 4 +- src/types/database.d.ts | 40 ++++- 6 files changed, 214 insertions(+), 13 deletions(-) diff --git a/src/agents/productionAgent/tools.ts b/src/agents/productionAgent/tools.ts index 97684c6..635e1b7 100644 --- a/src/agents/productionAgent/tools.ts +++ b/src/agents/productionAgent/tools.ts @@ -2,10 +2,12 @@ import { tool, Tool } from "ai"; import { z } from "zod"; import _ from "lodash"; import ResTool from "@/socket/resTool"; - +import u from "@/utils"; +import { useSkill } from "@/utils/agent/skillsTools"; +import { urlToBase64 } from "@/utils/vm"; export const deriveAssetSchema = z.object({ id: z.number().describe("衍生资产ID,如果新增则为空").optional(), - assetsId: z.string().describe("关联的资产ID"), + assetsId: z.number().describe("关联的资产ID"), prompt: z.string().describe("生成提示词"), name: z.string().describe("衍生资产名称"), desc: z.string().describe("衍生资产描述"), @@ -14,14 +16,15 @@ export const deriveAssetSchema = z.object({ type: z.enum(["role", "tool", "scene", "clip"]).describe("衍生资产类型"), }); export const assetItemSchema = z.object({ - assetsId: z.string().describe("资产唯一标识"), + id: z.number().describe("资产唯一标识"), name: z.string().describe("资产名称"), + type: z.enum(["role", "tool", "scene", "clip"]).describe("资产类型"), + prompt: z.string().describe("生成提示词"), desc: z.string().describe("资产描述"), - src: z.string().describe("资产资源路径"), derive: z.array(deriveAssetSchema).describe("衍生资产列表"), }); export const storyboardSchema = z.object({ - id: z.number().describe("分镜ID"), + id: z.number().optional().describe("分镜ID,未从工作区获得的分镜列表视为需要新增;如需新增则为空"), title: z.string().describe("分镜标题"), description: z.string().describe("分镜描述"), camera: z.string().describe("镜头信息"), @@ -107,6 +110,35 @@ export default (resTool: ResTool, toolsNames?: string[]) => { execute: async ({ value }) => { console.log("[tools] set_flowData assets", value); resTool.systemMessage("正在保存 衍生资产 数据"); + if (value && Array.isArray(value) && value.length) { + for (const i of value) { + if (!i?.id) { + const [insertedId] = await u.db("o_assets").insert({ + assetsId: null, + name: i.name, + type: i.type, + prompt: i.prompt, + describe: i.desc, + startTime: Date.now(), + }); + i.id = insertedId; + } + if (i.derive && Array.isArray(i.derive) && i.derive.length) { + for (const sub of i.derive) { + if (sub.id) continue; + const [insertedId] = await u.db("o_assets").insert({ + assetsId: +i.id || null, + name: sub.name, + type: sub.type, + prompt: sub.prompt, + describe: sub.desc, + startTime: Date.now(), + }); + sub.id = insertedId; + } + } + } + } socket.emit("setFlowData", { key: "assets", value }); return true; }, @@ -127,6 +159,28 @@ export default (resTool: ResTool, toolsNames?: string[]) => { execute: async ({ value }) => { console.log("[tools] set_flowData storyboard", value); resTool.systemMessage("正在保存 分镜列表 数据..."); + for (const item of value) { + if (!item.id) { + const [insertedId] = await u.db("o_storyboard").insert({ + title: item.title, + prompt: item.prompt, + description: item.description, + filePath: item.src, + frameMode: item.frameMode, + duration: String(item.duration), + camera: item.camera, + sound: item.sound, + lines: item.lines, + state: "未生成", + }); + if (item.associateAssetsIds.length) { + await u.db("o_assets2Storyboard").insert(item.associateAssetsIds.map((i) => ({ storyboardId: insertedId, assetId: i }))); + } + item.id = insertedId; + } + } + console.log("%c Line:181 🍏 value", "background:#93c0a4", value); + socket.emit("setFlowData", { key: "storyboard", value }); return true; }, @@ -175,7 +229,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => { inputSchema: z.object({ images: z.array( z.object({ - id: z.string().describe("图片唯一标识符"), + id: z.number().describe("从工作区获取到的分镜id"), prompt: z.string().describe("图片生成提示词"), referenceIds: z.array(z.string()).describe("依赖的参考图id数组，无依赖填空数组[]"), assetIds: z.array(z.number()).optional().describe("参考的资产图"), @@ -184,18 +238,116 @@ export default (resTool: ResTool, toolsNames?: string[]) => { }), execute: async ({ images }) => { console.log("[tools] generated_assets", images); - return new Promise((resolve) => socket.emit("generatedAssets", { images }, (res: any) => resolve(res))); + + const skill = await useSkill("universal-agent"); + for (const item of images) { + resTool.systemMessage(`生在生成分镜 id:${item.id} 图片`); + await u.db("o_storyboard").where("id", item.id).update({ state: "生成中" }); + u.Ai.Image("1:doubao-seedream-4-5-251128") + .run({ + systemPrompt: skill.prompt, + prompt: item.prompt, + imageBase64: await getAssetsImageBase64(item.assetIds ?? []), + size: "2K", + aspectRatio: "16:9", + taskClass: "资产图片生成", + describe: "生成图片", + relatedObjects: "hhhh", + projectId: resTool.data.projectId, + }) + .then(async (imageCls) => { + const savePath = `/${resTool.data.projectId}/storyboard/${u.uuid()}.jpg`; + await imageCls.save(savePath); + const obj = { + ...item, + id: item.id, + src: await u.oss.getFileUrl(savePath), + state: "已完成", + }; + await u.db("o_storyboard").where("id", item.id).update({ state: "已完成", filePath: savePath }); + resTool.systemMessage(`分镜 id:${item.id} 图片生成完成`); + socket.emit("setFlowData", { key: "setStoryboardImage", value: obj }); + }); + + socket.emit("setFlowData", { key: "setStoryboardImage", value: { ...item, id: item.id, src: "", state: "生成中" } }); + } + return "分镜图片生成中"; }, }), generate_assets_images: tool({ - description: "生成分镜图", + description: ` + 生成 资产图片 不区分原资产于衍生资产 + 参数说明： + - images: 图片任务数组 + - assetId: 资产id + - prompt: 图片生成提示词 + 示例： + images:[ + {assetId: 1, prompt: "一张猫的图片"} + ] + `, inputSchema: z.object({ images: z.array(z.object({ assetId: z.number(), prompt: z.string() })) }), execute: async ({ images }) => { + const skill = await useSkill("universal-agent"); + for (const item of images) { + const [imageId] = await u.db("o_image").insert({ + assetsId: item.assetId, + model: "1:doubao-seedream-4-5-251128", + state: "生成中", + resolution: "2K", + }); + u.Ai.Image("1:doubao-seedream-4-5-251128") + .run({ + systemPrompt: skill.prompt, + prompt: item.prompt, + imageBase64: [], + size: "2K", + aspectRatio: "16:9", + taskClass: "资产图片生成", + describe: "生成图片", + relatedObjects: "hhhh", + projectId: resTool.data.projectId, + }) + .then(async (imageCls) => { + const savePath = `/${resTool.data.projectId}/assets/${u.uuid()}.jpg`; + await imageCls.save(savePath); + const obj = { + ...item, + id: item.assetId, + src: await u.oss.getFileUrl(savePath), + state: "生成成功", + }; + await u.db("o_image").where({ id: imageId }).update({ state: "已完成", filePath: savePath }); + socket.emit("setFlowData", { key: "setAssetsImage", value: obj }); + }); + + socket.emit("setFlowData", { key: "setAssetsImage", value: { ...item, id: item.assetId, src: "", state: "生成中" } }); + } console.log("[tools] generate_assets_images", images); - return new Promise((resolve) => socket.emit("generateAssetsImages", { images }, (res: any) => resolve(res))); + return "资产生成中"; }, }), }; return toolsNames ? Object.fromEntries(Object.entries(tools).filter(([n]) => toolsNames.includes(n))) : tools; }; + +async function getAssetsImageBase64(imageIds: number[]) { + if (imageIds.length === 0) return []; + const imagePaths = await u + .db("o_assets") + .leftJoin("o_image", "o_assets.imageId", "o_image.id") + .whereIn("o_assets.id", imageIds) + .select("o_assets.id", "o_image.filePath"); + if (!imagePaths.length) return []; + const imageUrls = await Promise.all( + imagePaths.map(async (i) => { + if (i.filePath) { + return await urlToBase64(await u.oss.getFileUrl(i.filePath)); + } else { + return null; + } + }), + ); + return imageUrls.filter(Boolean) as string[]; +} diff --git a/src/lib/initDB.ts b/src/lib/initDB.ts index 84576b5..c4e47e5 100644 --- a/src/lib/initDB.ts +++ b/src/lib/initDB.ts @@ -303,6 +303,8 @@ export default async (knex: Knex, forceInit: boolean = false): Promise => table.text("frameMode"); table.text("camera"); table.text("sound"); + table.text("state"); + table.text("reason"); table.integer("createTime"); table.primary(["id"]); table.unique(["id"]); diff --git a/src/routes/production/getFlowData.ts b/src/routes/production/getFlowData.ts index 3212f8f..2f507e0 100644 --- a/src/routes/production/getFlowData.ts +++ b/src/routes/production/getFlowData.ts @@ -21,17 +21,21 @@ export default router.post( .select("data") .first(); - const scriptData = await u.db("o_script").where("projectId", projectId).first(); - + const scriptData = await u.db("o_script").where("projectId", projectId).where("id", episodesId).first(); + const scriptAssets = await u.db("o_scriptAssets").where("scriptId", episodesId); + const assetIds = scriptAssets.map((i) => i.assetId); const assetsData = await u .db("o_assets") .leftJoin("o_image", "o_assets.imageId", "o_image.id") .select("o_assets.*", "o_image.filePath") + .where("o_assets.id", "in", assetIds) + .whereNull("o_assets.assetsId") .where("o_assets.projectId", projectId); let childAssetsData = await u .db("o_assets") .leftJoin("o_image", "o_assets.imageId", "o_image.id") .select("o_assets.*", "o_image.filePath") + .where("o_assets.id", "in", assetIds) .where("o_assets.projectId", projectId) .whereNotNull("o_assets.assetsId"); @@ -43,6 +47,8 @@ export default router.post( assetsData.map(async (item) => ({ assetsId: item.id, name: item.name ?? "", + type: item.type ?? "", + prompt: item.prompt ?? "", desc: item.describe ?? "", src: item.filePath && (await u.oss.getFileUrl(item.filePath!)), derive: await Promise.all( diff --git a/src/routes/production/workbench/getChatLines.ts b/src/routes/production/workbench/getChatLines.ts index 0f21412..29bc9a9 100644 --- a/src/routes/production/workbench/getChatLines.ts +++ b/src/routes/production/workbench/getChatLines.ts @@ -35,6 +35,7 @@ async function getLines(prompt: string) { const resText = await u.Ai.Text("universalAgent").invoke({ system: skill.prompt, messages: [{ role: "user", content: prompt }], + tools: skill.tools, output: Output.array({ element: z.object({ lines: z.string().describe("台词内容"), diff --git a/src/socket/routes/productionAgent.ts b/src/socket/routes/productionAgent.ts index a7248a6..45b3e2a 100644 --- a/src/socket/routes/productionAgent.ts +++ b/src/socket/routes/productionAgent.ts @@ -35,7 +35,9 @@ export default (nsp: Namespace) => { console.log("[productionAgent] 已连接:", socket.id); - const resTool = new ResTool(socket); + const resTool = new ResTool(socket, { + projectId: socket.handshake.auth.projectId, + }); let abortController: AbortController | null = null; socket.on("message", async (text: string) => { diff --git a/src/types/database.d.ts b/src/types/database.d.ts index 97e7555..efdd9da 100644 --- a/src/types/database.d.ts +++ b/src/types/database.d.ts @@ -1,6 +1,39 @@ -// @db-hash 83c8dadf13c2aee689597b709a690870 +// @db-hash 08cfec7bed045942e10f27f6d3f34623 //该文件由脚本自动生成，请勿手动修改 +export interface _o_storyboard_old_20260324 { + 'camera'?: string | null; + 'createTime'?: number | null; + 'description'?: string | null; + 'duration'?: string | null; + 'filePath'?: string | null; + 'frameMode'?: string | null; + 'id'?: number; + 'mode'?: string | null; + 'model'?: string | null; + 'prompt'?: string | null; + 'resolution'?: string | null; + 'scriptId'?: number | null; + 'sound'?: string | null; + 'title'?: string | null; +} +export interface _o_storyboard_old_20260324_1 { + 'camera'?: string | null; + 'createTime'?: number | null; + 'description'?: string | null; + 'duration'?: string | null; + 'filePath'?: string | null; + 'frameMode'?: string | null; + 'id'?: number; + 'lines'?: string | null; + 'mode'?: string | null; + 'model'?: string | null; + 'prompt'?: string | null; + 'resolution'?: string | null; + 'scriptId'?: number | null; + 'sound'?: string | null; + 'title'?: string | null; +} export interface memories { 'content': string; 'createTime': number; @@ -130,12 +163,15 @@ export interface o_storyboard { 'filePath'?: string | null; 'frameMode'?: string | null; 'id'?: number; + 'lines'?: string | null; 'mode'?: string | null; 'model'?: string | null; 'prompt'?: string | null; + 'reason'?: string | null; 'resolution'?: string | null; 'scriptId'?: number | null; 'sound'?: string | null; + 'state'?: string | null; 'title'?: string | null; } export interface o_storyboardFlow { @@ -195,6 +231,8 @@ export interface o_videoConfig { } export interface DB { + "_o_storyboard_old_20260324": _o_storyboard_old_20260324; + "_o_storyboard_old_20260324_1": _o_storyboard_old_20260324_1; "memories": memories; "o_agentDeploy": o_agentDeploy; "o_agentWorkData": o_agentWorkData; From 80d0c8143f633d87d673e57d873201816fc472ca Mon Sep 17 00:00:00 2001 From: zhishi <1951671751@qq.com> Date: Tue, 24 Mar 2026 21:38:22 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E5=AE=8C=E5=96=84=E5=88=86=E9=95=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../skills/production-agent/decision/SKILL.md | 2 +- .../references/storyboard-generation.md | 12 +-- src/agents/productionAgent/tools.ts | 93 +++++++++++++++---- src/agents/scriptAgent/tools.ts | 13 +-- src/lib/initDB.ts | 1 + src/routes/migrate/migrateData.ts | 2 +- src/routes/production/getFlowData.ts | 31 ++++++- src/socket/routes/productionAgent.ts | 12 ++- src/types/database.d.ts | 24 ++++- 9 files changed, 149 insertions(+), 41 deletions(-) diff --git a/data/skills/production-agent/decision/SKILL.md b/data/skills/production-agent/decision/SKILL.md index fe3fb98..714eee2 100644 --- a/data/skills/production-agent/decision/SKILL.md +++ b/data/skills/production-agent/decision/SKILL.md @@ -53,4 +53,4 @@ description: 短剧漫剧制作决策层。负责分析用户需求、制定执 - **用户目标优先**：默认直接响应并推进用户当前任务，不要为了流程完整性而强制先生成计划 - **计划按需维护**：仅当用户明确要求新增/修改拍摄计划时，才更新拍摄计划，且每次改动都调用 `set_plane` 同步到前端 - **提取衍生资产后**：计划中必须包含"询问用户是否生成资产图片"步骤。若用户确认，执行层将调用相应工具批量生成衍生资产图片 -- **生成分镜表后**：计划中必须包含"询问用户是否生成分镜图片"步骤。若用户确认，执行层将调用相应工具生成分镜图 +- **生成分镜面板后**：计划中必须包含"询问用户是否生成分镜图片"步骤。若用户确认，执行层将调用相应工具生成分镜图 diff --git a/data/skills/production-agent/execution/references/storyboard-generation.md b/data/skills/production-agent/execution/references/storyboard-generation.md index 8f82f2b..4d4a1c1 100644 --- a/data/skills/production-agent/execution/references/storyboard-generation.md +++ b/data/skills/production-agent/execution/references/storyboard-generation.md @@ -1,9 +1,9 @@ -# 分镜表生成（从剧本 + 资产 → storyboardTable） +# 分镜面板生成（从剧本 + 资产 → storyboardTable） 本指南只做一件事： -根据剧本内容和已有资产，将剧本拆分为一系列分镜，生成结构化的分镜表。 +根据剧本内容和已有资产，将剧本拆分为一系列分镜，生成结构化的分镜面板。 -> **核心概念**：分镜表是将剧本转化为视觉画面的中间产物。每条分镜对应一个独立的画面/镜头，包含画面描述、镜头语言、台词、音效和关联资产等信息，用于后续图片生成。 +> **核心概念**：分镜面板是将剧本转化为视觉画面的中间产物。每条分镜对应一个独立的画面/镜头，包含画面描述、镜头语言、台词、音效和关联资产等信息，用于后续图片生成。 ## 1. 输入与输出 @@ -14,7 +14,7 @@ ### 输出 -调用 `set_flowData` 将分镜表写入工作区： +调用 `set_flowData` 将分镜面板写入工作区： ```ts set_flowData({ @@ -232,8 +232,8 @@ set_flowData({ 1. `get_flowData("script")` — 获取剧本内容 2. `get_flowData("assets")` — 获取已有资产列表 3. 分析剧本，按照拆分原则划分分镜，并为每条分镜填写所有字段 -4. 调用 `set_flowData({ key: "storyboardTable", value: 分镜数组 })` 一次性保存完整分镜表 -5. 向用户汇报分镜表概要（总共多少条分镜，覆盖的场景概括） +4. 调用 `set_flowData({ key: "storyboardTable", value: 分镜数组 })` 一次性保存完整分镜面板 +5. 向用户汇报分镜面板概要（总共多少条分镜，覆盖的场景概括） 6. **询问用户是否需要生成分镜图片**： - 如果用户确认，调用 `generate_storyboard_images({ script: 剧本文本 })` 生成分镜图 - 如果用户拒绝，跳过此步骤，流程结束 diff --git a/src/agents/productionAgent/tools.ts b/src/agents/productionAgent/tools.ts index 635e1b7..d73986d 100644 --- a/src/agents/productionAgent/tools.ts +++ b/src/agents/productionAgent/tools.ts @@ -52,7 +52,7 @@ export const flowDataSchema = z.object({ script: z.string().describe("剧本内容"), scriptPlan: z.string().describe("拍摄计划"), assets: z.array(assetItemSchema).describe("衍生资产"), - storyboardTable: z.string().describe("分镜表"), + storyboardTable: z.string().describe("分镜面板"), storyboard: z.array(storyboardSchema).describe("分镜列表"), workbench: workbenchDataSchema.describe("工作台配置"), poster: z @@ -104,6 +104,39 @@ export default (resTool: ResTool, toolsNames?: string[]) => { return true; }, }), + // add_flowData_assets: tool({ + // description: "新增对应衍生资产列表到工作区，严禁包含 不需要新增的数据", + // inputSchema: z.object({ value: z.array(deriveAssetSchema).describe("需要新增的资产列表") }), + // execute: async ({ value }) => { + // console.log("[tools] set_flowData add_flowData_assets", value); + // resTool.systemMessage("正在保存 衍生资产 数据"); + // const addAssetsData = []; + // if (value && Array.isArray(value) && value.length) { + // for (const i of value) { + // const [insertedId] = await u.db("o_assets").insert({ + // assetsId: +i.assetsId || null, + // projectId: resTool.data.projectId, + // name: i.name, + // type: i.type, + // prompt: i.prompt, + // describe: i.desc, + // startTime: Date.now(), + // }); + // console.log("%c Line:141 🍑 resTool.data.scriptId", "background:#ea7e5c", resTool.data.scriptId); + // await u.db("o_scriptAssets").insert({ + // scriptId: resTool.data.scriptId, + // assetId: insertedId, + // }); + // addAssetsData.push({ + // ...i, + // id: insertedId, + // }); + // } + // } + // socket.emit("setFlowData", { key: "addAssets", value: addAssetsData }); + // return true; + // }, + // }), set_flowData_assets: tool({ description: "保存衍生资产列表到工作区", inputSchema: z.object({ value: flowDataSchema.shape.assets }), @@ -128,12 +161,18 @@ export default (resTool: ResTool, toolsNames?: string[]) => { if (sub.id) continue; const [insertedId] = await u.db("o_assets").insert({ assetsId: +i.id || null, + projectId: resTool.data.projectId, name: sub.name, type: sub.type, prompt: sub.prompt, describe: sub.desc, startTime: Date.now(), }); + console.log("%c Line:141 🍑 resTool.data.scriptId", "background:#ea7e5c", resTool.data.scriptId); + await u.db("o_scriptAssets").insert({ + scriptId: resTool.data.scriptId, + assetId: insertedId, + }); sub.id = insertedId; } } @@ -144,11 +183,11 @@ export default (resTool: ResTool, toolsNames?: string[]) => { }, }), set_flowData_storyboardTable: tool({ - description: "保存分镜表到工作区", + description: "保存分镜模板到工作区", inputSchema: z.object({ value: flowDataSchema.shape.storyboardTable }), execute: async ({ value }) => { console.log("[tools] set_flowData storyboardTable", value); - resTool.systemMessage("正在保存 分镜表 数据..."); + resTool.systemMessage("正在保存 分镜面板 数据..."); socket.emit("setFlowData", { key: "storyboardTable", value }); return true; }, @@ -179,8 +218,6 @@ export default (resTool: ResTool, toolsNames?: string[]) => { item.id = insertedId; } } - console.log("%c Line:181 🍏 value", "background:#93c0a4", value); - socket.emit("setFlowData", { key: "storyboard", value }); return true; }, @@ -206,6 +243,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => { }, }), + //todo referenceIds 图片未使用 提示词待调 generate_storyboard_images: tool({ description: `生成一组图片任务，支持图片间的依赖关系（以图生图）。 @@ -242,16 +280,20 @@ export default (resTool: ResTool, toolsNames?: string[]) => { const skill = await useSkill("universal-agent"); for (const item of images) { resTool.systemMessage(`生在生成分镜 id:${item.id} 图片`); + //更新对应分镜状态 await u.db("o_storyboard").where("id", item.id).update({ state: "生成中" }); - u.Ai.Image("1:doubao-seedream-4-5-251128") + // 异步生成 + const imageModel = resTool.data.imageModel; + + u.Ai.Image(imageModel?.modelId) .run({ systemPrompt: skill.prompt, prompt: item.prompt, imageBase64: await getAssetsImageBase64(item.assetIds ?? []), - size: "2K", - aspectRatio: "16:9", - taskClass: "资产图片生成", - describe: "生成图片", + size: imageModel?.quality, + aspectRatio: imageModel?.ratio, + taskClass: "生成图片", + describe: "分镜图片生成", relatedObjects: "hhhh", projectId: resTool.data.projectId, }) @@ -264,16 +306,21 @@ export default (resTool: ResTool, toolsNames?: string[]) => { src: await u.oss.getFileUrl(savePath), state: "已完成", }; + // 更新对应分镜状态 await u.db("o_storyboard").where("id", item.id).update({ state: "已完成", filePath: savePath }); + // 前端对话框提示 resTool.systemMessage(`分镜 id:${item.id} 图片生成完成`); + // 更新前端界面展示 socket.emit("setFlowData", { key: "setStoryboardImage", value: obj }); }); - + //更新前端为生成中 socket.emit("setFlowData", { key: "setStoryboardImage", value: { ...item, id: item.id, src: "", state: "生成中" } }); } return "分镜图片生成中"; }, }), + + //todo 图片是否需要参考 原资产 提示词待调 generate_assets_images: tool({ description: ` 生成 资产图片 不区分原资产于衍生资产 @@ -289,22 +336,25 @@ export default (resTool: ResTool, toolsNames?: string[]) => { inputSchema: z.object({ images: z.array(z.object({ assetId: z.number(), prompt: z.string() })) }), execute: async ({ images }) => { const skill = await useSkill("universal-agent"); + //获取所设置模型 + const imageModel = resTool.data.imageModel; for (const item of images) { const [imageId] = await u.db("o_image").insert({ + // 数据库插入图片记录 assetsId: item.assetId, - model: "1:doubao-seedream-4-5-251128", + model: imageModel?.modelId, state: "生成中", - resolution: "2K", + resolution: imageModel?.quality, }); - u.Ai.Image("1:doubao-seedream-4-5-251128") + u.Ai.Image(imageModel?.modelId) .run({ systemPrompt: skill.prompt, prompt: item.prompt, imageBase64: [], - size: "2K", - aspectRatio: "16:9", - taskClass: "资产图片生成", - describe: "生成图片", + size: imageModel?.quality, + aspectRatio: imageModel?.ratio, + taskClass: "生成图片", + describe: "资产图片生成", relatedObjects: "hhhh", projectId: resTool.data.projectId, }) @@ -315,12 +365,15 @@ export default (resTool: ResTool, toolsNames?: string[]) => { ...item, id: item.assetId, src: await u.oss.getFileUrl(savePath), - state: "生成成功", + state: "已完成", }; + //更新对应数据库 + await u.db("o_assets").where("id", item.assetId).update({ imageId: imageId }); await u.db("o_image").where({ id: imageId }).update({ state: "已完成", filePath: savePath }); + //通知前端更新 socket.emit("setFlowData", { key: "setAssetsImage", value: obj }); }); - + //通知前端更新状态 socket.emit("setFlowData", { key: "setAssetsImage", value: { ...item, id: item.assetId, src: "", state: "生成中" } }); } console.log("[tools] generate_assets_images", images); diff --git a/src/agents/scriptAgent/tools.ts b/src/agents/scriptAgent/tools.ts index 16538e3..c945515 100644 --- a/src/agents/scriptAgent/tools.ts +++ b/src/agents/scriptAgent/tools.ts @@ -5,14 +5,11 @@ import _ from "lodash"; import ResTool from "@/socket/resTool"; export const AssetSchema = z.object({ - id: z.number().describe("衍生资产ID,如果新增则为空").optional(), - assetsId: z.string().describe("关联的资产ID"), + id: z.number().describe("资产ID,如果新增则为空").optional(), prompt: z.string().describe("生成提示词"), - name: z.string().describe("衍生资产名称"), - desc: z.string().describe("衍生资产描述"), - src: z.string().describe("衍生资产资源路径").optional(), - state: z.enum(["未生成", "生成中", "已完成", "生成失败"]).describe("衍生资产生成状态，新增默认未生成"), - type: z.enum(["role", "tool", "scene", "clip"]).describe("衍生资产类型"), + name: z.string().describe("资产名称"), + desc: z.string().describe("资产描述"), + type: z.enum(["role", "tool", "scene", "clip"]).describe("资产类型"), }); export const ScriptSchema = z.object({ id: z.number().describe("剧本ID,如果新增则为空").optional(), @@ -97,7 +94,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => { description: "将剧本内容插入sqlite数据库，供后续业务使用", inputSchema: z.object({ script: ScriptSchema, - assetsList: z.array(AssetSchema).describe("剧本所使用资产列表"), + assetsList: z.array(AssetSchema).describe("剧本所使用资产列表,注意不要包含剧本内容,仅为所使用到的 道具、人物、场景、素材"), }), execute: async ({ assetsList, script }) => { console.log("%c Line:103 🍷 script", "background:#42b983", script); diff --git a/src/lib/initDB.ts b/src/lib/initDB.ts index c873c91..f1d2754 100644 --- a/src/lib/initDB.ts +++ b/src/lib/initDB.ts @@ -305,6 +305,7 @@ export default async (knex: Knex, forceInit: boolean = false): Promise => table.text("frameMode"); table.text("camera"); table.text("sound"); + table.text("lines"); table.text("state"); table.text("reason"); table.integer("createTime"); diff --git a/src/routes/migrate/migrateData.ts b/src/routes/migrate/migrateData.ts index f26247e..8053de7 100644 --- a/src/routes/migrate/migrateData.ts +++ b/src/routes/migrate/migrateData.ts @@ -109,7 +109,7 @@ export default router.post( if (tableName === 't_outline') { } //迁移脚本表 if (tableName === 't_script') { } - //迁移分镜表 + //迁移分镜面板 if (tableName === 't_storyboard') { } //迁移视频表 if (tableName === 't_video') { } diff --git a/src/routes/production/getFlowData.ts b/src/routes/production/getFlowData.ts index 2f507e0..00f23e3 100644 --- a/src/routes/production/getFlowData.ts +++ b/src/routes/production/getFlowData.ts @@ -35,9 +35,10 @@ export default router.post( .db("o_assets") .leftJoin("o_image", "o_assets.imageId", "o_image.id") .select("o_assets.*", "o_image.filePath") - .where("o_assets.id", "in", assetIds) .where("o_assets.projectId", projectId) + .where("o_assets.id", "in", assetIds) .whereNotNull("o_assets.assetsId"); + console.log("%c Line:35 🥚 childAssetsData", "background:#f5ce50", childAssetsData); if (!sqlData) { const flowData: FlowData = { @@ -45,7 +46,7 @@ export default router.post( scriptPlan: "", assets: await Promise.all( assetsData.map(async (item) => ({ - assetsId: item.id, + id: item.id, name: item.name ?? "", type: item.type ?? "", prompt: item.prompt ?? "", @@ -58,6 +59,8 @@ export default router.post( id: child.id, assetsId: item.id, name: child.name ?? "", + type: child.type, + prompt: child.prompt, desc: child.describe ?? "", src: child.filePath && (await u.oss.getFileUrl(child.filePath!)), state: child.state ?? "未生成", //todo：矫正状态值 @@ -84,6 +87,30 @@ export default router.post( } else { try { const flowData = JSON.parse(sqlData!.data ?? "{}"); + flowData.assets = await Promise.all( + assetsData.map(async (item) => ({ + id: item.id, + name: item.name ?? "", + type: item.type ?? "", + prompt: item.prompt ?? "", + desc: item.describe ?? "", + src: item.filePath && (await u.oss.getFileUrl(item.filePath!)), + derive: await Promise.all( + childAssetsData + .filter((child) => child.assetsId === item.id) + .map(async (child) => ({ + id: child.id, + assetsId: item.id, + name: child.name ?? "", + prompt: child.prompt, + type: child.type, + desc: child.describe ?? "", + src: child.filePath && (await u.oss.getFileUrl(child.filePath!)), + state: child.state ?? "未生成", //todo：矫正状态值 + })), + ), + })), + ); res.status(200).send(success(flowData)); } catch (err) { res.status(200).send(error()); diff --git a/src/socket/routes/productionAgent.ts b/src/socket/routes/productionAgent.ts index 45b3e2a..b7ef731 100644 --- a/src/socket/routes/productionAgent.ts +++ b/src/socket/routes/productionAgent.ts @@ -26,7 +26,7 @@ export default (nsp: Namespace) => { socket.disconnect(); return; } - const isolationKey = socket.handshake.auth.isolationKey; + let isolationKey = socket.handshake.auth.isolationKey; if (!isolationKey) { console.log("[productionAgent] 连接失败，缺少 isolationKey"); socket.disconnect(); @@ -37,6 +37,7 @@ export default (nsp: Namespace) => { const resTool = new ResTool(socket, { projectId: socket.handshake.auth.projectId, + scriptId: socket.handshake.auth.scriptId, }); let abortController: AbortController | null = null; @@ -44,6 +45,7 @@ export default (nsp: Namespace) => { abortController?.abort(); abortController = new AbortController(); const currentController = abortController; + console.log("%c Line:30 🍑 isolationKey", "background:#e41a6a", isolationKey); const textStream = await agent.decisionAI({ socket, isolationKey, text, abortSignal: currentController.signal, resTool }); @@ -62,7 +64,13 @@ export default (nsp: Namespace) => { } } }); - + socket.on("setModelData", async (data: any) => { + resTool.data.imageModel = data; + }); + socket.on("setKeyScript", async (data: any) => { + isolationKey = data.key; + resTool.data.scriptId = data.scriptId; + }); socket.on("stop", () => { abortController?.abort(); abortController = null; diff --git a/src/types/database.d.ts b/src/types/database.d.ts index 0527267..a501920 100644 --- a/src/types/database.d.ts +++ b/src/types/database.d.ts @@ -1,6 +1,24 @@ -// @db-hash 47c0e014bdbd44b60c4ebc95f4d99e0e +// @db-hash 1ce1a8f10cb90caac306536b78942cb3 //该文件由脚本自动生成，请勿手动修改 +export interface _o_storyboard_old_20260324 { + 'camera'?: string | null; + 'createTime'?: number | null; + 'description'?: string | null; + 'duration'?: string | null; + 'filePath'?: string | null; + 'frameMode'?: string | null; + 'id'?: number; + 'mode'?: string | null; + 'model'?: string | null; + 'prompt'?: string | null; + 'reason'?: string | null; + 'resolution'?: string | null; + 'scriptId'?: number | null; + 'sound'?: string | null; + 'state'?: string | null; + 'title'?: string | null; +} export interface memories { 'content': string; 'createTime': number; @@ -132,12 +150,15 @@ export interface o_storyboard { 'filePath'?: string | null; 'frameMode'?: string | null; 'id'?: number; + 'lines'?: string | null; 'mode'?: string | null; 'model'?: string | null; 'prompt'?: string | null; + 'reason'?: string | null; 'resolution'?: string | null; 'scriptId'?: number | null; 'sound'?: string | null; + 'state'?: string | null; 'title'?: string | null; } export interface o_storyboardFlow { @@ -197,6 +218,7 @@ export interface o_videoConfig { } export interface DB { + "_o_storyboard_old_20260324": _o_storyboard_old_20260324; "memories": memories; "o_agentDeploy": o_agentDeploy; "o_agentWorkData": o_agentWorkData;