diff --git a/env/.env.dev b/env/.env.dev index 584000d..5b6dc86 100644 --- a/env/.env.dev +++ b/env/.env.dev @@ -1,4 +1,4 @@ NODE_ENV=dev PORT=10588 -OSSURL=http://127.0.0.1:10588/ +OSSURL=http://192.168.0.74:10588/ diff --git a/src/agents/productionAgent/tools.ts b/src/agents/productionAgent/tools.ts index e1782eb..a4f57f6 100644 --- a/src/agents/productionAgent/tools.ts +++ b/src/agents/productionAgent/tools.ts @@ -350,7 +350,6 @@ export default (resTool: ResTool, toolsNames?: string[]) => { await u.db("o_storyboardFlow").whereIn("storyboardId", ids).delete(); const flowData: FlowData = await new Promise((resolve) => socket.emit("getFlowData", { key: "storyboard" }, (res: any) => resolve(res))); const storyboardData = flowData["storyboard"].filter((item) => !ids.includes(item.id)); - socket.emit("setFlowData", { key: "storyboard", value: storyboardData }); return true; }, @@ -408,27 +407,26 @@ export default (resTool: ResTool, toolsNames?: string[]) => { return true; }, }), - - //todo referenceIds 图片未使用 提示词待调 + // todo 提示词待调 generate_storyboard_images: tool({ - description: `生成一组图片任务,支持图片间的依赖关系(以图生图)。 + description: `生成一组图片任务,支持图片间的依赖关系(以图生图),基于有向无环图(DAG)拓扑排序执行。 参数说明: - images: 图片任务数组 - - id: 图片唯一标识符 + - id: 图片唯一标识符(分镜id) - prompt: 图片生成提示词 - referenceIds: 依赖的参考图id数组,无依赖填空数组[] - assetIds: 参考的资产图id数组(可选) - 依赖规则: + 依赖规则: 1. referenceIds中的id必须存在于images数组中 2. 禁止循环依赖(如A依赖B,B依赖A) 3. 被依赖的图片会先生成,其结果作为参考图传入 示例:生成猫图,再以猫图为参考生成狗图 images: [ - {id: "cat", prompt: "一只橘猫", referenceIds: [], assetIds: []}, - {id: "dog", prompt: "风格相同的金毛犬", referenceIds: ["cat"], assetIds: []} + {id: 1, prompt: "一只橘猫", referenceIds: [], assetIds: []}, + {id: 2, prompt: "风格相同的金毛犬", referenceIds: [1], assetIds: []} ]`, inputSchema: z.object({ images: z.array( @@ -441,53 +439,141 @@ export default (resTool: ResTool, toolsNames?: string[]) => { ), }), execute: async ({ images }) => { - console.log("[tools] generated_assets", images); + console.log("[tools] generate_storyboard_images", images); + // --- 构建任务id集合 --- + const taskIds = new Set(images.map((item) => item.id)); + const imageMap = new Map(images.map((item) => [item.id, item])); + + // --- 检测循环依赖 (Kahn算法拓扑排序) --- + // 将 referenceIds 分为:本批次内依赖 vs 外部已有依赖 + // 只有本批次内的依赖才参与 DAG 调度,外部依赖直接从数据库获取 + const inDegree = new Map(); + // adjacency: 被依赖者 -> 依赖它的节点列表 + const adjacency = new Map(); + + for (const item of images) { + // 只统计本批次内的依赖作为入度 + const internalDeps = item.referenceIds.filter((refId) => taskIds.has(refId)); + inDegree.set(item.id, internalDeps.length); + for (const depId of internalDeps) { + if (!adjacency.has(depId)) adjacency.set(depId, []); + adjacency.get(depId)!.push(item.id); + } + } + + // 拓扑排序,按层级分组(同层可并行) + const levels: number[][] = []; + let queue = images.filter((item) => (inDegree.get(item.id) ?? 0) === 0).map((item) => item.id); + + const visited = new Set(); + while (queue.length > 0) { + levels.push([...queue]); + const nextQueue: number[] = []; + for (const nodeId of queue) { + visited.add(nodeId); + for (const childId of adjacency.get(nodeId) ?? []) { + inDegree.set(childId, (inDegree.get(childId) ?? 1) - 1); + if (inDegree.get(childId) === 0) { + nextQueue.push(childId); + } + } + } + queue = nextQueue; + } + // 循环依赖检测 + if (visited.size !== images.length) { + const cyclicIds = images.filter((item) => !visited.has(item.id)).map((item) => item.id); + resTool.systemMessage(`检测到循环依赖,涉及分镜id: ${cyclicIds.join(", ")},请修正后重试`); + return `错误:检测到循环依赖,涉及分镜id: ${cyclicIds.join(", ")}`; + } + + console.log("%c Line:496 🌶", "background:#ea7e5c"); + resTool.systemMessage(`图片生成调度计划:共 ${levels.length} 层,${images.length} 张图片`); + + // --- 准备公共数据 --- const skill = await useSkill("universal-agent"); const projectData = await u.db("o_project").where("id", resTool.data.projectId).select("videoRatio").first(); - for (const item of images) { - resTool.systemMessage(`生在生成分镜 id:${item.id} 图片`); - //更新对应分镜状态 - await u.db("o_storyboard").where("id", item.id).update({ state: "生成中" }); - // 异步生成 - const imageModel = resTool.data.imageModel; + const imageModel = resTool.data.imageModel; - u.Ai.Image(imageModel?.modelId) - .run({ - systemPrompt: skill.prompt, - prompt: item.prompt, - imageBase64: [...(await getAssetsImageBase64(item.assetIds ?? [])), ...(await getStoryboardImageBase64(item.referenceIds))], - size: imageModel?.quality, - aspectRatio: (projectData?.videoRatio as `${number}:${number}`) ?? "16:9", - taskClass: "生成图片", - describe: "分镜图片生成", - relatedObjects: "hhhh", - projectId: resTool.data.projectId, - }) - .then(async (imageCls) => { - const savePath = `/${resTool.data.projectId}/storyboard/${u.uuid()}.jpg`; - await imageCls.save(savePath); - const obj = { - ...item, - id: item.id, - src: await u.oss.getFileUrl(savePath), - state: "已完成", - }; - // 更新对应分镜状态 - await u.db("o_storyboard").where("id", item.id).update({ state: "已完成", filePath: savePath }); - // 前端对话框提示 - resTool.systemMessage(`分镜 id:${item.id} 图片生成完成`); - // 更新前端界面展示 - socket.emit("setFlowData", { key: "setStoryboardImage", value: obj }); - }); - //更新前端为生成中 - socket.emit("setFlowData", { key: "setStoryboardImage", value: { ...item, id: item.id, src: "", state: "生成中" } }); + // 生成单张图片的函数 + const generateOneImage = async (item: (typeof images)[0]) => { + resTool.systemMessage(`正在生成分镜 id:${item.id} 图片`); + // 更新数据库状态为生成中 + await u.db("o_storyboard").where("id", item.id).update({ state: "生成中" }); + // 更新前端为生成中 + socket.emit("setFlowData", { + key: "setStoryboardImage", + value: { ...item, id: item.id, src: "", state: "生成中", referenceIds: item.referenceIds }, + }); + + // 获取参考图base64(包括资产图和已生成的分镜参考图) + const [assetsBase64, referenceBase64] = await Promise.all([ + getAssetsImageBase64(item.assetIds ?? []), + getStoryboardImageBase64(item.referenceIds), + ]); + + const imageCls = await u.Ai.Image(imageModel?.modelId).run({ + systemPrompt: skill.prompt, + prompt: item.prompt, + imageBase64: [...assetsBase64, ...referenceBase64], + size: imageModel?.quality, + aspectRatio: (projectData?.videoRatio as `${number}:${number}`) ?? "16:9", + taskClass: "生成图片", + describe: "分镜图片生成", + relatedObjects: "hhhh", + projectId: resTool.data.projectId, + }); + + const savePath = `/${resTool.data.projectId}/storyboard/${u.uuid()}.jpg`; + await imageCls.save(savePath); + + // 更新数据库状态为已完成 + await u.db("o_storyboard").where("id", item.id).update({ state: "已完成", filePath: savePath }); + + const obj = { + ...item, + id: item.id, + src: await u.oss.getFileUrl(savePath), + state: "已完成", + referenceIds: item.referenceIds, + }; + // 前端对话框提示 + resTool.systemMessage(`分镜 id:${item.id} 图片生成完成`); + // 更新前端界面展示 + socket.emit("setFlowData", { key: "setStoryboardImage", value: obj }); + }; + + // --- 按层级顺序执行:同层并行,层间串行 --- + for (let levelIndex = 0; levelIndex < levels.length; levelIndex++) { + const levelIds = levels[levelIndex]; + const levelItems = levelIds.map((id) => imageMap.get(id)!); + resTool.systemMessage(`开始生成第 ${levelIndex + 1}/${levels.length} 层,共 ${levelItems.length} 张图片 (ids: ${levelIds.join(", ")})`); + + // 同层内所有图片并行生成,使用 allSettled 确保不会因单张失败中断整层 + const results = await Promise.allSettled(levelItems.map((item) => generateOneImage(item))); + + // 处理失败的任务 + for (let i = 0; i < results.length; i++) { + if (results[i].status === "rejected") { + const failedId = levelIds[i]; + const reason = (results[i] as PromiseRejectedResult).reason; + console.error(`[tools] 分镜 id:${failedId} 图片生成失败`, reason); + resTool.systemMessage(`分镜 id:${failedId} 图片生成失败: ${reason?.message || reason}`); + await u.db("o_storyboard").where("id", failedId).update({ state: "生成失败" }); + socket.emit("setFlowData", { + key: "setStoryboardImage", + value: { id: failedId, src: "", state: "生成失败" }, + }); + } + } } - return "分镜图片生成中"; + + return "分镜图片生成完成"; }, }), - //todo 图片是否需要参考 原资产 提示词待调 + //todo 提示词待调 generate_assets_images: tool({ description: ` 生成 资产图片 不区分原资产于衍生资产 @@ -505,15 +591,29 @@ export default (resTool: ResTool, toolsNames?: string[]) => { z.object({ assetId: z.number().describe("衍生资产id"), prompt: z.string().describe("提示词"), - refenceAssetsId: z.array(z.number()).describe("参考[资产]id,注意:资产和衍生资产为两种类型,衍生资产归类在资产下面"), }), ), }), execute: async ({ images }) => { const skill = await useSkill("universal-agent"); + console.log("[tools] generate_assets_images", images); + //先获取到前端资产数据 + const flowData: FlowData = await new Promise((resolve) => socket.emit("getFlowData", { key: "assets" }, (res: any) => resolve(res))); + const assetsData = flowData["assets"]; + const assetsImage: { assetId: number; prompt: string; id?: number }[] = [...images]; + //获取对应的 原资产id + assetsImage.forEach((item) => { + for (const i of assetsData) { + const findData = i.derive.find((m) => m.id == item.assetId); + if (findData) { + item.id = findData.id; + break; + } + } + }); //获取所设置模型 const imageModel = resTool.data.imageModel; - for (const item of images) { + for (const item of assetsImage) { const [imageId] = await u.db("o_image").insert({ // 数据库插入图片记录 assetsId: item.assetId, @@ -525,7 +625,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => { .run({ // systemPrompt: skill.prompt, prompt: item.prompt, - imageBase64: await getAssetsImageBase64(item.refenceAssetsId ?? []), + imageBase64: await getAssetsImageBase64(item.id ? [item.id] : []), size: imageModel?.quality, aspectRatio: "16:9", taskClass: "生成图片", @@ -551,7 +651,6 @@ export default (resTool: ResTool, toolsNames?: string[]) => { //通知前端更新状态 socket.emit("setFlowData", { key: "setAssetsImage", value: { ...item, id: item.assetId, src: "", state: "生成中" } }); } - console.log("[tools] generate_assets_images", images); return "资产生成中"; }, }), diff --git a/src/agents/scriptAgent/tools.ts b/src/agents/scriptAgent/tools.ts index c945515..b4abe1b 100644 --- a/src/agents/scriptAgent/tools.ts +++ b/src/agents/scriptAgent/tools.ts @@ -108,6 +108,10 @@ export default (resTool: ResTool, toolsNames?: string[]) => { if (assetsList && assetsList.length) { const assetId = []; for (const i of assetsList) { + if (i.id) { + assetId.push(i.id); + continue; + } const [id] = await u.db("o_assets").insert({ name: i.name, prompt: i.prompt, diff --git a/src/routes/production/editImage/saveImageFlow.ts b/src/routes/production/editImage/saveImageFlow.ts index adadbd7..73e959b 100644 --- a/src/routes/production/editImage/saveImageFlow.ts +++ b/src/routes/production/editImage/saveImageFlow.ts @@ -13,9 +13,10 @@ export default router.post( imageUrl: z.string(), id: z.number().nullable().optional(), type: z.enum(["role", "scene", "storyboard", "clip", "tool"]), + episodesId: z.number(), }), async (req, res) => { - const { edges, nodes, imageUrl, id, type } = req.body; + const { edges, nodes, imageUrl, id, type, episodesId } = req.body; let imagePath = ""; try { imagePath = new URL(imageUrl).pathname; @@ -56,6 +57,7 @@ export default router.post( } else { const [storyboardId] = await u.db("o_storyboard").insert({ filePath: imagePath, + scriptId: episodesId, createTime: Date.now(), }); insertFlowId = storyboardId; diff --git a/src/routes/production/editImage/updateImageFlow.ts b/src/routes/production/editImage/updateImageFlow.ts index 4c03c56..2953c65 100644 --- a/src/routes/production/editImage/updateImageFlow.ts +++ b/src/routes/production/editImage/updateImageFlow.ts @@ -14,9 +14,10 @@ export default router.post( imageUrl: z.string(), type: z.enum(["role", "scene", "storyboard", "clip", "tool"]), flowId: z.number(), + episodesId: z.number(), }), async (req, res) => { - const { edges, nodes, id, imageUrl, flowId, type } = req.body; + const { edges, nodes, id, imageUrl, flowId, type, episodesId } = req.body; nodes.forEach((node: any) => { if (node.type == "upload") { node.data.image = node.data.image ? new URL(node.data.image).pathname : ""; @@ -30,7 +31,6 @@ export default router.post( imagePath = new URL(imageUrl).pathname; } catch (e) {} if (imagePath) { - console.log("%c Line:34 🍰", "background:#33a5ff"); if (type == "storyboard") { await u.db("o_storyboard").where("id", id).update({ filePath: imagePath, diff --git a/src/routes/production/getFlowData.ts b/src/routes/production/getFlowData.ts index 00f23e3..bd8733b 100644 --- a/src/routes/production/getFlowData.ts +++ b/src/routes/production/getFlowData.ts @@ -86,34 +86,102 @@ export default router.post( return res.status(200).send(success(flowData)); } else { try { - const flowData = JSON.parse(sqlData!.data ?? "{}"); - flowData.assets = await Promise.all( - assetsData.map(async (item) => ({ - id: item.id, - name: item.name ?? "", - type: item.type ?? "", - prompt: item.prompt ?? "", - desc: item.describe ?? "", - src: item.filePath && (await u.oss.getFileUrl(item.filePath!)), - derive: await Promise.all( - childAssetsData - .filter((child) => child.assetsId === item.id) - .map(async (child) => ({ - id: child.id, - assetsId: item.id, - name: child.name ?? "", - prompt: child.prompt, - type: child.type, - desc: child.describe ?? "", - src: child.filePath && (await u.oss.getFileUrl(child.filePath!)), - state: child.state ?? "未生成", //todo:矫正状态值 - })), - ), - })), + const storyboardData = await u.db("o_storyboard").where("scriptId", episodesId); + console.log("%c Line:90 🍡 storyboardData", "background:#ed9ec7", storyboardData.length); + await Promise.all( + storyboardData.map(async (i) => { + if (i.filePath) { + try { + i.filePath = await u.oss.getFileUrl(i.filePath); + } catch { + i.filePath = ""; + } + } else { + i.filePath = ""; + } + }), ); + const storyboardIds = storyboardData.map((i) => i.id); + const assetsIds = await u.db("o_assets2Storyboard").whereIn("storyboardId", storyboardIds); + const assets2StoryboardMap: Record = {}; + assetsIds.forEach((i) => { + if (!assets2StoryboardMap[i.storyboardId!]) { + assets2StoryboardMap[i.storyboardId!] = []; + } + assets2StoryboardMap[i.storyboardId!].push(i.assetId!); + }); + const flowData = JSON.parse(sqlData!.data ?? "{}"); + // 将原有 flowData.assets 按 id 建立索引,以便后续合并保留旧字段 + const existingAssetsMap: Record = {}; + if (Array.isArray(flowData.assets)) { + flowData.assets.forEach((a: any) => { + existingAssetsMap[a.id] = a; + }); + } + flowData.assets = await Promise.all( + assetsData.map(async (item) => { + const existing = existingAssetsMap[item.id] ?? {}; + // 将原有 derive 按 id 建立索引 + const existingDeriveMap: Record = {}; + if (Array.isArray(existing.derive)) { + existing.derive.forEach((d: any) => { + existingDeriveMap[d.id] = d; + }); + } + return { + ...existing, + id: item.id, + name: item.name ?? "", + type: item.type ?? "", + prompt: item.prompt ?? "", + desc: item.describe ?? "", + src: item.filePath && (await u.oss.getFileUrl(item.filePath!)), + derive: await Promise.all( + childAssetsData + .filter((child) => child.assetsId === item.id) + .map(async (child) => ({ + ...(existingDeriveMap[child.id] ?? {}), + id: child.id, + assetsId: item.id, + name: child.name ?? "", + prompt: child.prompt, + type: child.type, + desc: child.describe ?? "", + src: child.filePath && (await u.oss.getFileUrl(child.filePath!)), + state: child.state ?? "未生成", //todo:矫正状态值 + })), + ), + }; + }), + ); + // 将原有 flowData.storyboard 按 id 建立索引,以便后续合并保留旧字段 + const existingStoryboardMap: Record = {}; + if (Array.isArray(flowData.storyboard)) { + flowData.storyboard.forEach((s: any) => { + existingStoryboardMap[s.id] = s; + }); + } + flowData.storyboard = storyboardData.map((i) => { + const existing = existingStoryboardMap[i.id!] ?? {}; + return { + ...existing, + id: i.id, + title: i.title, + description: i.description, + camera: i.camera, + duration: i.duration ? +i.duration : 0, + frameMode: i.frameMode, + prompt: i.prompt, + lines: i.lines, + sound: i.sound, + associateAssetsIds: assets2StoryboardMap[i.id!] ?? [], + src: i.filePath, + state: i.state, + }; + }); res.status(200).send(success(flowData)); } catch (err) { - res.status(200).send(error()); + res.status(400).send(error()); } } }, diff --git a/src/routes/production/workbench/getChatLines.ts b/src/routes/production/workbench/getChatLines.ts index 29bc9a9..98802f7 100644 --- a/src/routes/production/workbench/getChatLines.ts +++ b/src/routes/production/workbench/getChatLines.ts @@ -42,6 +42,8 @@ async function getLines(prompt: string) { }), }), }); + console.log("%c Line:36 🍉 resText", "background:#e41a6a", resText); + const parseLines = JSON.parse(resText.text); const chatLines = parseLines.elements.map((i: any) => i.lines); return chatLines; diff --git a/src/types/database.d.ts b/src/types/database.d.ts index a4b9c30..6aef2aa 100644 --- a/src/types/database.d.ts +++ b/src/types/database.d.ts @@ -1,4 +1,4 @@ -// @db-hash 62a748aea9d1ecee865c4cf05add24fc +// @db-hash a3673cf3a1d1c9cbf22ae3cfff196a71 //该文件由脚本自动生成,请勿手动修改 export interface memories { @@ -53,7 +53,7 @@ export interface o_assets { } export interface o_assets2Storyboard { 'assetId'?: number; - 'storyboardId': number; + 'storyboardId'?: number; } export interface o_event { 'createTime'?: number | null; @@ -149,11 +149,6 @@ export interface o_storyboard { 'state'?: string | null; 'title'?: string | null; } -export interface o_storyboardFlow { - 'flowData': string; - 'id'?: number; - 'storyboardId': number; -} export interface o_tasks { 'describe'?: string | null; 'id'?: number; @@ -224,7 +219,6 @@ export interface DB { "o_scriptAssets": o_scriptAssets; "o_setting": o_setting; "o_storyboard": o_storyboard; - "o_storyboardFlow": o_storyboardFlow; "o_tasks": o_tasks; "o_user": o_user; "o_vendorConfig": o_vendorConfig;