diff --git a/src/agents/productionAgent/index.ts b/src/agents/productionAgent/index.ts index 84a1757..702d95c 100644 --- a/src/agents/productionAgent/index.ts +++ b/src/agents/productionAgent/index.ts @@ -11,8 +11,10 @@ export interface AgentContext { socket: Socket; isolationKey: string; text: string; + userMessageTime?: number; abortSignal?: AbortSignal; resTool: ResTool; + msg: ReturnType; } function buildSystemPrompt(skillPrompt: string, mem: Awaited>): string { @@ -52,7 +54,7 @@ export async function decisionAI(ctx: AgentContext) { ...skill.tools, ...memory.getTools(), run_sub_agent: runSubAgent(ctx), - ...useTools(ctx.resTool), + ...useTools({ resTool: ctx.resTool, msg: ctx.msg }), }, onFinish: async (completion) => { await memory.add("assistant:decision", completion.text); @@ -65,78 +67,80 @@ export async function decisionAI(ctx: AgentContext) { //====================== 执行层 ====================== export async function executionAI(ctx: AgentContext) { - const { isolationKey, text, abortSignal, resTool } = ctx; + const { text, abortSignal } = ctx; - resTool.systemMessage("执行层AI 接管聊天"); + const skill = await useSkill("production_agent_execution.md"); - const memory = new Memory("productionAgent", isolationKey); - const [skill, mem] = await Promise.all([useSkill("production_agent_execution.md"), memory.get(text)]); - - const systemPrompt = buildSystemPrompt(skill.prompt, mem); + const subMsg = ctx.resTool.newMessage("assistant", "执行导演"); const { textStream } = await u.Ai.Text("productionAgent").stream({ - system: systemPrompt, + system: skill.prompt, messages: [{ role: "user", content: text }], abortSignal, tools: { ...skill.tools, - ...memory.getTools(), - ...useTools(ctx.resTool), - }, - onFinish: async (completion) => { - await memory.add("assistant:execution", completion.text); + ...useTools({ resTool: ctx.resTool, msg: subMsg }), }, }); - return textStream; + return { textStream, subMsg }; } export async function supervisionAI(ctx: AgentContext) { - const { isolationKey, text, abortSignal } = ctx; - const memory = new Memory("productionAgent", isolationKey); - const [skill, mem] = await Promise.all([useSkill("production_agent_supervision.md"), memory.get(text)]); + const { text, abortSignal } = ctx; - const systemPrompt = buildSystemPrompt(skill.prompt, mem); + const skill = await useSkill("production_agent_supervision.md"); + const subMsg = ctx.resTool.newMessage("assistant", "编辑"); - const { textStream } = await u.Ai.Text("productionAgent").stream({ - system: systemPrompt, + const { textStream } = await u.Ai.Text("scriptAgent").stream({ + system: skill.prompt, messages: [{ role: "user", content: text }], abortSignal, tools: { ...skill.tools, - ...useTools(ctx.resTool), - }, - onFinish: async (completion) => { - await memory.add("assistant:supervision", completion.text); + ...useTools({ + resTool: ctx.resTool, + msg: subMsg, + }), }, }); - return textStream; + return { textStream, subMsg }; } //工具函数 function runSubAgent(parentCtx: AgentContext) { + const memory = new Memory("scriptAgent", parentCtx.isolationKey); return tool({ description: "启动子Agent执行独立任务。可用子Agent:executionAI, decisionAI, supervisionAI", inputSchema: z.object({ agent: z.enum(["executionAI", "supervisionAI"]).describe("子Agent名称"), - prompt: z.string().max(100).describe("交给子Agent的任务简约描述"), + prompt: z.string().describe("交给子Agent的任务简约描述,100字以内"), }), execute: async ({ agent, prompt }) => { - //todo 传入md有问题 const fn = [executionAI, supervisionAI][subAgentList.indexOf(agent)]; - //运行子Agent - const subTextStream = await fn({ ...parentCtx, text: prompt }); - let msg: ReturnType; + // 先完成主Agent当前的消息 + parentCtx.msg.complete(); + // 子Agent用新消息回复 + const { textStream: subTextStream, subMsg } = await fn({ ...parentCtx, text: prompt }); + let text = subMsg.text(); let fullResponse = ""; - for await (const chunk of subTextStream) { - if (!msg!) msg = parentCtx.resTool.textMessage(); - msg.send(chunk); + text.append(chunk); fullResponse += chunk; } - msg!.end(); + text.complete(); + subMsg.complete(); + if (fullResponse.trim()) { + await memory.add(`assistant:${agent === "executionAI" ? "execution" : "supervision"}`, fullResponse, { + name: agent === "executionAI" ? "编剧" : "编辑", + createTime: new Date(subMsg.datetime).getTime(), + }); + } + + // 为主Agent后续输出创建新消息 + parentCtx.msg = parentCtx.resTool.newMessage("assistant", "统筹"); return fullResponse; }, diff --git a/src/agents/productionAgent/tools.ts b/src/agents/productionAgent/tools.ts index 3d6c64a..ba7c1bf 100644 --- a/src/agents/productionAgent/tools.ts +++ b/src/agents/productionAgent/tools.ts @@ -3,7 +3,6 @@ import { z } from "zod"; import _ from "lodash"; import ResTool from "@/socket/resTool"; import u from "@/utils"; -import { useSkill } from "@/utils/agent/skillsTools"; import { urlToBase64 } from "@/utils/vm"; export const deriveAssetSchema = z.object({ id: z.number().describe("衍生资产ID,如果新增则为空"), @@ -75,7 +74,14 @@ const flowDataKeyLabels = Object.fromEntries( Object.entries(flowDataSchema.shape).map(([key, schema]) => [key, (schema as z.ZodTypeAny).description ?? key]), ) as Record; -export default (resTool: ResTool, toolsNames?: string[]) => { +interface ToolConfig { + resTool: ResTool; + toolsNames?: string[]; + msg: ReturnType; +} + +export default (toolCpnfig: ToolConfig) => { + const { resTool, toolsNames, msg } = toolCpnfig; const { socket } = resTool; const tools: Record = { get_flowData: tool({ @@ -84,9 +90,12 @@ export default (resTool: ResTool, toolsNames?: string[]) => { key: keySchema.describe("数据key"), }), execute: async ({ key }) => { - resTool.systemMessage(`正在阅读 ${flowDataKeyLabels[key]} 数据...`); + const thinking = msg.thinking(`正在获取${flowDataKeyLabels[key]}工作区数据...`); console.log("[tools] get_flowData", key); const flowData: FlowData = await new Promise((resolve) => socket.emit("getFlowData", { key }, (res: any) => resolve(res))); + thinking.appendText(`获取到${flowDataKeyLabels[key]}:\n` + flowData[key]); + thinking.updateTitle(`获取${flowDataKeyLabels[key]}完成`); + thinking.complete(); return flowData[key]; }, }), @@ -95,8 +104,10 @@ export default (resTool: ResTool, toolsNames?: string[]) => { inputSchema: z.object({ value: flowDataSchema.shape.script }), execute: async ({ value }) => { console.log("[tools] set_flowData script", value); - resTool.systemMessage("正在保存 剧本 数据"); + const thinking = msg.thinking("正在保存 剧本 数据"); socket.emit("setFlowData", { key: "script", value }); + thinking.updateTitle("保存 剧本 数据完成"); + thinking.complete(); return true; }, }), @@ -105,8 +116,10 @@ export default (resTool: ResTool, toolsNames?: string[]) => { inputSchema: z.object({ value: flowDataSchema.shape.scriptPlan }), execute: async ({ value }) => { console.log("[tools] set_flowData scriptPlan", value); - resTool.systemMessage("正在保存 拍摄计划 数据"); + const thinking = msg.thinking("正在保存 拍摄计划 数据"); socket.emit("setFlowData", { key: "scriptPlan", value }); + thinking.updateTitle("保存 拍摄计划 数据完成"); + thinking.complete(); return true; }, }), @@ -115,7 +128,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => { inputSchema: z.object({ value: z.array(deriveAssetSchema.omit({ id: true })).describe("需要新增的衍生资产列表") }), execute: async ({ value }) => { console.log("[tools] set_flowData add_flowData_assets", value); - resTool.systemMessage("正在保存 衍生资产 数据"); + const thinking = msg.thinking("正在保存 衍生资产 数据"); const setData = [...value] as z.infer[]; const { projectId, scriptId } = resTool.data; const startTime = Date.now(); @@ -154,6 +167,9 @@ export default (resTool: ResTool, toolsNames?: string[]) => { i.derive = [...(i.derive || []), ...watiAddAssetsMap[i.id]]; } }); + thinking.updateTitle("保存 衍生资产 数据完成"); + thinking.complete(); + socket.emit("setFlowData", { key: "assets", value: assetsData }); return true; }, @@ -163,7 +179,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => { inputSchema: z.object({ value: z.array(deriveAssetSchema).describe("需要更新的衍生资产列表") }), execute: async ({ value }) => { console.log("[tools] update_flowData update_flowData_assets", value); - resTool.systemMessage("正在保存 衍生资产 数据"); + const thinking = msg.thinking("正在保存 衍生资产 数据"); for (const i of value) { await u .db("o_assets") @@ -195,6 +211,8 @@ export default (resTool: ResTool, toolsNames?: string[]) => { asset.derive = (asset.derive || []).map((d) => updatedMap[d.id] ?? d); } }); + thinking.updateTitle("保存 衍生资产 数据完成"); + thinking.complete(); socket.emit("setFlowData", { key: "assets", value: assetsData }); return true; }, @@ -204,13 +222,15 @@ export default (resTool: ResTool, toolsNames?: string[]) => { inputSchema: z.object({ ids: z.array(z.number()).describe("需要删除的 衍生资产id ") }), execute: async ({ ids }) => { console.log("[tools] delete_flowData delete_flowData_assets", ids); - resTool.systemMessage("正在保存 衍生资产 数据"); + const thinking = msg.thinking("正在删除指定 衍生资产 数据..."); await u.db("o_assets").whereIn("id", ids).delete(); const flowData: FlowData = await new Promise((resolve) => socket.emit("getFlowData", { key: "assets" }, (res: any) => resolve(res))); const assetsData = flowData.assets; assetsData.forEach((i) => { i.derive = (i.derive || []).filter((d) => !ids.includes(d.id)); }); + thinking.updateTitle("删除指定 衍生资产 数据完成"); + thinking.complete(); // 将 derive 中已存在的条目替换为更新后的数据 socket.emit("setFlowData", { key: "assets", value: assetsData }); return true; @@ -265,8 +285,10 @@ export default (resTool: ResTool, toolsNames?: string[]) => { inputSchema: z.object({ value: flowDataSchema.shape.storyboardTable }), execute: async ({ value }) => { console.log("[tools] set_flowData storyboardTable", value); - resTool.systemMessage("正在保存 分镜表 数据..."); + const thinking = msg.thinking("正在保存 分镜表 数据..."); socket.emit("setFlowData", { key: "storyboardTable", value }); + thinking.updateTitle("保存 分镜表 数据完成"); + thinking.complete(); return true; }, }), @@ -275,7 +297,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => { inputSchema: z.object({ value: z.array(storyboardSchema.omit({ id: true })) }), execute: async ({ value }) => { console.log("[tools] add_flowData storyboard", value); - resTool.systemMessage("正在新增 分镜面板 数据..."); + const thinking = msg.thinking("正在保存 分镜面板 数据..."); const setData = [...value] as z.infer[]; for (const item of setData) { item.src = ""; @@ -302,6 +324,9 @@ export default (resTool: ResTool, toolsNames?: string[]) => { const storyboardData = flowData["storyboard"].concat([...setData]); socket.emit("setFlowData", { key: "storyboard", value: storyboardData }); + thinking.updateTitle("保存 分镜面板 数据完成"); + thinking.complete(); + return true; }, }), @@ -310,7 +335,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => { inputSchema: z.object({ value: flowDataSchema.shape.storyboard }), execute: async ({ value }) => { console.log("[tools] update_flowData storyboard", value); - resTool.systemMessage("正在更新 分镜面板 数据..."); + const thinking = msg.thinking("正在保存 分镜面板 数据..."); for (const item of value) { await u .db("o_storyboard") @@ -344,6 +369,8 @@ export default (resTool: ResTool, toolsNames?: string[]) => { }; }); socket.emit("setFlowData", { key: "storyboard", value: storyboardData }); + thinking.updateTitle("保存 分镜面板 数据完成"); + thinking.complete(); return true; }, }), @@ -352,13 +379,15 @@ export default (resTool: ResTool, toolsNames?: string[]) => { inputSchema: z.object({ ids: z.array(z.number()).describe("需要删除的 分镜id ") }), execute: async ({ ids }) => { console.log("[tools] delete_flowData storyboard", ids); - resTool.systemMessage("正在删除指定 分镜面板 数据..."); + const thinking = msg.thinking("正在删除指定 分镜面板 数据..."); await u.db("o_storyboard").whereIn("id", ids).delete(); await u.db("o_assets2Storyboard").whereIn("storyboardId", ids).delete(); await u.db("o_storyboardFlow").whereIn("storyboardId", ids).delete(); const flowData: FlowData = await new Promise((resolve) => socket.emit("getFlowData", { key: "storyboard" }, (res: any) => resolve(res))); const storyboardData = flowData["storyboard"].filter((item) => !ids.includes(item.id)); socket.emit("setFlowData", { key: "storyboard", value: storyboardData }); + thinking.updateTitle("删除指定 分镜面板 数据完成"); + thinking.complete(); return true; }, }), @@ -400,8 +429,10 @@ export default (resTool: ResTool, toolsNames?: string[]) => { inputSchema: z.object({ value: flowDataSchema.shape.workbench }), execute: async ({ value }) => { console.log("[tools] set_flowData workbench", value); - resTool.systemMessage("正在保存 工作台配置 数据..."); + const thinking = msg.thinking("正在保存 工作台配置 数据..."); socket.emit("setFlowData", { key: "workbench", value }); + thinking.updateTitle("保存 工作台配置 数据完成"); + thinking.complete(); return true; }, }), @@ -410,7 +441,9 @@ export default (resTool: ResTool, toolsNames?: string[]) => { inputSchema: z.object({ value: flowDataSchema.shape.poster }), execute: async ({ value }) => { console.log("[tools] set_flowData poster", value); - resTool.systemMessage("正在保存 海报 数据..."); + const thinking = msg.thinking("正在保存 海报配置 数据..."); + thinking.updateTitle("保存 海报配置 数据完成"); + thinking.complete(); socket.emit("setFlowData", { key: "poster", value }); return true; }, @@ -448,7 +481,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => { }), execute: async ({ images }) => { console.log("[tools] generate_storyboard_images", images); - + const thinking = msg.thinking("正在生成 分镜图片 数据..."); // --- 构建任务id集合 --- const taskIds = new Set(images.map((item) => item.id)); const imageMap = new Map(images.map((item) => [item.id, item])); @@ -492,11 +525,13 @@ export default (resTool: ResTool, toolsNames?: string[]) => { // 循环依赖检测 if (visited.size !== images.length) { const cyclicIds = images.filter((item) => !visited.has(item.id)).map((item) => item.id); - resTool.systemMessage(`检测到循环依赖,涉及分镜id: ${cyclicIds.join(", ")},请修正后重试`); + thinking.appendText(`检测到循环依赖,涉及分镜id: ${cyclicIds.join(", ")},请修正后重试`); + thinking.updateTitle("循环依赖错误"); + thinking.error(); return `错误:检测到循环依赖,涉及分镜id: ${cyclicIds.join(", ")}`; } - resTool.systemMessage(`图片生成调度计划:共 ${levels.length} 层,${images.length} 张图片`); + thinking.appendText(`图片生成调度计划:共 ${levels.length} 层,${images.length} 张图片`); // --- 准备公共数据 --- const projectData = await u.db("o_project").where("id", resTool.data.projectId).select("videoRatio").first(); @@ -504,7 +539,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => { // 生成单张图片的函数 const generateOneImage = async (item: (typeof images)[0]) => { - resTool.systemMessage(`正在生成分镜 id:${item.id} 图片`); + const thinking = msg.thinking(`正在生成分镜 id:${item.id} 图片`); // 更新数据库状态为生成中 await u.db("o_storyboard").where("id", item.id).update({ state: "生成中" }); // 更新前端为生成中 @@ -544,7 +579,8 @@ export default (resTool: ResTool, toolsNames?: string[]) => { referenceIds: item.referenceIds, }; // 前端对话框提示 - resTool.systemMessage(`分镜 id:${item.id} 图片生成完成`); + thinking.appendText(`分镜 id:${item.id} 图片生成完成`); + thinking.complete(); // 更新前端界面展示 socket.emit("setFlowData", { key: "setStoryboardImage", value: obj }); }; @@ -553,7 +589,9 @@ export default (resTool: ResTool, toolsNames?: string[]) => { for (let levelIndex = 0; levelIndex < levels.length; levelIndex++) { const levelIds = levels[levelIndex]; const levelItems = levelIds.map((id) => imageMap.get(id)!); - resTool.systemMessage(`开始生成第 ${levelIndex + 1}/${levels.length} 层,共 ${levelItems.length} 张图片 (ids: ${levelIds.join(", ")})`); + const thinking = msg.thinking( + `开始生成第 ${levelIndex + 1}/${levels.length} 层,共 ${levelItems.length} 张图片 (ids: ${levelIds.join(", ")})`, + ); // 同层内所有图片并行生成,使用 allSettled 确保不会因单张失败中断整层 const results = await Promise.allSettled(levelItems.map((item) => generateOneImage(item))); @@ -564,7 +602,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => { const failedId = levelIds[i]; const reason = (results[i] as PromiseRejectedResult).reason; console.error(`[tools] 分镜 id:${failedId} 图片生成失败`, reason); - resTool.systemMessage(`分镜 id:${failedId} 图片生成失败: ${reason?.message || reason}`); + thinking.appendText(`分镜 id:${failedId} 图片生成失败: ${reason?.message || reason}`); await u.db("o_storyboard").where("id", failedId).update({ state: "生成失败" }); socket.emit("setFlowData", { key: "setStoryboardImage", @@ -572,7 +610,12 @@ export default (resTool: ResTool, toolsNames?: string[]) => { }); } } + thinking.appendText(`第 ${levelIndex + 1}/${levels.length} 层图片生成完成`); + thinking.complete(); } + thinking.appendText("所有分镜图片生成完成"); + thinking.updateTitle("分镜图片生成完成"); + thinking.complete(); return "分镜图片生成完成"; }, diff --git a/src/agents/scriptAgent/index.ts b/src/agents/scriptAgent/index.ts index 63f1458..a50938b 100644 --- a/src/agents/scriptAgent/index.ts +++ b/src/agents/scriptAgent/index.ts @@ -69,6 +69,9 @@ export async function decisionAI(ctx: AgentContext) { run_sub_agent: runSubAgent(ctx), ...useTools({ resTool: ctx.resTool, msg: ctx.msg }), }, + onFinish: async (completion) => { + await memory.add("assistant:decision", completion.text); + }, }); return textStream; diff --git a/src/socket/routes/productionAgent.ts b/src/socket/routes/productionAgent.ts index b7ef731..611cd5f 100644 --- a/src/socket/routes/productionAgent.ts +++ b/src/socket/routes/productionAgent.ts @@ -3,6 +3,7 @@ import u from "@/utils"; import { Namespace, Socket } from "socket.io"; import * as agent from "@/agents/productionAgent/index"; import ResTool from "@/socket/resTool"; +import Memory from "@/utils/agent/memory"; async function verifyToken(rawToken: string): Promise { const setting = await u.db("o_setting").where("key", "tokenKey").select("value").first(); @@ -41,29 +42,67 @@ export default (nsp: Namespace) => { }); let abortController: AbortController | null = null; - socket.on("message", async (text: string) => { + socket.on("chat", async (data: { content: string }) => { + const { content } = data; abortController?.abort(); abortController = new AbortController(); const currentController = abortController; - console.log("%c Line:30 🍑 isolationKey", "background:#e41a6a", isolationKey); + const memory = new Memory("scriptAgent", isolationKey); - const textStream = await agent.decisionAI({ socket, isolationKey, text, abortSignal: currentController.signal, resTool }); + const msg = resTool.newMessage("assistant", "统筹"); + const ctx: agent.AgentContext = { + socket, + isolationKey, + text: content, + userMessageTime: new Date(msg.datetime).getTime() - 1, + abortSignal: currentController.signal, + resTool, + msg, + }; - let msg = resTool.textMessage(); + const textStream = await agent.decisionAI(ctx); + + let currentMsg = ctx.msg; + let text = currentMsg.text(); + let currentContent = ""; + + const persistCurrentMessage = async () => { + if (!currentContent.trim()) return; + await memory.add("assistant:decision", currentContent, { + name: "统筹", + createTime: new Date(currentMsg.datetime).getTime(), + }); + currentContent = ""; + }; + + const syncCurrentMessage = async () => { + if (ctx.msg === currentMsg) return; + text.complete(); + currentMsg.complete(); + await persistCurrentMessage(); + currentMsg = ctx.msg; + text = currentMsg.text(); + }; try { for await (const chunk of textStream) { - msg.send(chunk); + await syncCurrentMessage(); + text.append(chunk); + currentContent += chunk; } } catch (err: any) { if (err.name !== "AbortError") throw err; } finally { - msg.end(); + await syncCurrentMessage(); + text.complete(); + currentMsg.complete(); + await persistCurrentMessage(); if (abortController === currentController) { abortController = null; } } }); + socket.on("setModelData", async (data: any) => { resTool.data.imageModel = data; }); @@ -71,6 +110,7 @@ export default (nsp: Namespace) => { isolationKey = data.key; resTool.data.scriptId = data.scriptId; }); + socket.on("stop", () => { abortController?.abort(); abortController = null; diff --git a/src/types/database.d.ts b/src/types/database.d.ts index b49367a..eed488e 100644 --- a/src/types/database.d.ts +++ b/src/types/database.d.ts @@ -1,4 +1,4 @@ -// @db-hash 8aa6e47033e9f59d1f8b797d5b4fccd3 +// @db-hash 8e5f2b7a28d4494b291d802b055b6399 //该文件由脚本自动生成,请勿手动修改 export interface memories { @@ -47,7 +47,6 @@ export interface o_assets { 'name'?: string | null; 'projectId'?: number | null; 'prompt'?: string | null; - 'promptState'?: string | null; 'remark'?: string | null; 'scriptId'?: number | null; 'startTime'?: number | null; @@ -120,16 +119,13 @@ export interface o_project { 'videoRatio'?: string | null; } export interface o_prompt { - 'data'?: string | null; 'id'?: number; 'name'?: string | null; - 'type'?: string | null; + 'rompt'?: string | null; } export interface o_script { 'content'?: string | null; 'createTime'?: number | null; - 'errorReason'?: string | null; - 'extractState'?: number | null; 'id'?: number; 'name'?: string | null; 'projectId'?: number | null; @@ -166,7 +162,7 @@ export interface o_storyboard { 'filePath'?: string | null; 'frameMode'?: string | null; 'id'?: number; - 'index'?: number | null; + 'index'?: string | null; 'lines'?: string | null; 'mode'?: string | null; 'model'?: string | null;