import { Socket } from "socket.io"; import { tool } from "ai"; import { z } from "zod"; import u from "@/utils"; import Memory from "@/utils/agent/memory"; import { createSkillTools, parseFrontmatter, scanSkills, useSkill } from "@/utils/agent/skillsTools"; import useTools from "@/agents/productionAgent/tools"; import ResTool from "@/socket/resTool"; import * as fs from "fs"; import path from "path"; export interface AgentContext { socket: Socket; isolationKey: string; text: string; userMessageTime?: number; abortSignal?: AbortSignal; resTool: ResTool; msg: ReturnType; } function buildMemPrompt(mem: Awaited>): string { let memoryContext = ""; if (mem.rag.length) { memoryContext += `[相关记忆]\n${mem.rag.map((r) => r.content).join("\n")}`; } if (mem.summaries.length) { if (memoryContext) memoryContext += "\n\n"; memoryContext += `[历史摘要]\n${mem.summaries.map((s, i) => `${i + 1}. ${s.content}`).join("\n")}`; } if (mem.shortTerm.length) { if (memoryContext) memoryContext += "\n\n"; memoryContext += `[近期对话]\n${mem.shortTerm.map((m) => `${m.role}: ${m.content}`).join("\n")}`; } return `## Memory\n以下是你对用户的记忆,可作为参考但不要主动提及:\n${memoryContext}`; } export async function decisionAI(ctx: AgentContext) { const { isolationKey, text, abortSignal } = ctx; const memory = new Memory("productionAgent", isolationKey); await memory.add("user", text); const skill = path.join(u.getPath("skills"), "production_agent_decision.md"); const prompt = await fs.promises.readFile(skill, "utf-8"); const projectInfo = await u.db("o_project").where("id", ctx.resTool.data.projectId).first(); if (!projectInfo) throw new Error(`项目不存在,ID: ${ctx.resTool.data.projectId}`); const [_, imageModelName] = projectInfo.imageModel!.split(":"); const [id, videoModelName] = projectInfo.videoModel!.split(":"); const models = await u.vendor.getModelList(id); if(!models.length) throw new Error(`项目使用的模型不存在,ID: ${projectInfo.videoModel}`); const findData = models.find((i: any) => i.modelName == videoModelName); const isRef = findData.mode.every((i: any) => Array.isArray(i)); const modelInfo = `项目使用的模型如下:\n图像模型:${imageModelName}\n视频模型:${videoModelName}\n多参:${isRef ? "是" : "否"}`; const mem = buildMemPrompt(await memory.get(text)); const { textStream } = await u.Ai.Text("productionAgent").stream({ messages: [ { role: "system", content: prompt }, { role: "assistant", content: mem + "\n" + modelInfo }, { role: "user", content: text }, ], abortSignal, tools: { ...memory.getTools(), ...useTools({ resTool: ctx.resTool, msg: ctx.msg }), ...(await createSubAgent(ctx)), }, onFinish: async (completion) => { await memory.add("assistant:decision", removeAllXmlTags(completion.text)); }, }); return textStream; } async function createSubAgent(parentCtx: AgentContext) { const { resTool, abortSignal } = parentCtx; const memory = new Memory("productionAgent", parentCtx.isolationKey); async function runAgent({ prompt, system, name, memoryKey, tools: extraTools, messages, }: { prompt: string; system: string; name: string; memoryKey: string; tools?: Record; messages?: { role: "user" | "assistant" | "system"; content: string }[]; }) { parentCtx.msg.complete(); const subMsg = resTool.newMessage("assistant", name); const text = subMsg.text(); let fullResponse = ""; const { textStream } = await u.Ai.Text("productionAgent").stream({ system, messages: messages ?? [{ role: "user", content: prompt }], abortSignal, tools: { ...extraTools, ...useTools({ resTool, msg: subMsg }) }, }); try { for await (const chunk of textStream) { await new Promise((resolve) => setTimeout(() => resolve(), 1)); text.append(chunk); fullResponse += chunk; } text.complete(); subMsg.complete(); } catch (err: any) { text.complete(); subMsg.stop(); throw err; } if (fullResponse.trim()) { await memory.add(memoryKey, removeAllXmlTags(fullResponse), { name, createTime: new Date(subMsg.datetime).getTime(), }); } parentCtx.msg = resTool.newMessage("assistant", "视频策划"); return fullResponse; } const promptInput = z.object({ prompt: z.string().describe("交给子Agent的任务简约描述,100字以内"), }); const projectInfo = await u.db("o_project").where("id", resTool.data.projectId).first(); if (!projectInfo) throw new Error(`项目不存在,ID: ${resTool.data.projectId}`); const artSkills = await createArtSkills(projectInfo?.artStyle!, projectInfo?.directorManual!); const [_, imageModelName] = projectInfo.imageModel!.split(":"); const [id, videoModelName] = projectInfo.videoModel!.split(":"); const models = await u.vendor.getModelList(id); if(!models.length) throw new Error(`项目使用的模型不存在,ID: ${projectInfo.videoModel}`); const findData = models.find((i: any) => i.modelName == videoModelName); const isRef = findData.mode.every((i: any) => Array.isArray(i)); const modelInfo = `项目使用的模型如下:\n图像模型:${imageModelName}\n视频模型:${videoModelName}\n多参:${isRef ? "是" : "否"}`; // const run_sub_agent_execution = tool({ // description: "执行层子Agent,负责衍生资产、", // inputSchema: promptInput, // execute: async ({ prompt }) => { // const skill = path.join(u.getPath("skills"), "production_agent_execution.md"); // const systemPrompt = await fs.promises.readFile(skill, "utf-8"); // const addPrompt = // "\n" + // [ // "你必须使用如下XML格式写入工作区:\n```", // "拍摄计划:内容", // "分镜表:内容", // "分镜面板:", // "```", // ].join("\n"); // return runAgent({ // prompt, // system: systemPrompt + addPrompt, // name: "执行导演", // memoryKey: "assistant:execution", // messages: [ // { role: "assistant", content: artSkills.prompt + `\n${modelInfo}` }, // { role: "user", content: prompt + addPrompt }, // ], // tools: { ...artSkills.tools }, // }); // }, // }); //衍生资产分析与信息写入 const run_sub_agent_derive_assets = tool({ description: "运行执行subAgent来完成衍生资产分析与信息写入相关任务", inputSchema: promptInput, execute: async ({ prompt }) => { const skill = path.join(u.getPath("skills"), "production_execution_derive_assets.md"); const systemPrompt = await fs.promises.readFile(skill, "utf-8"); return runAgent({ prompt, system: systemPrompt, name: "执行导演", memoryKey: "assistant:execution", messages: [ { role: "assistant", content: artSkills.prompt + `\n${modelInfo}` }, { role: "user", content: prompt }, ], tools: { activate_skill: artSkills.tools.activate_skill }, }); }, }); //衍生资产图片生成 const run_sub_agent_generate_assets = tool({ description: "运行执行subAgent来完成衍生资产图片生成相关任务", inputSchema: promptInput, execute: async ({ prompt }) => { const skill = path.join(u.getPath("skills"), "production_execution_generate_assets.md"); const systemPrompt = await fs.promises.readFile(skill, "utf-8"); return runAgent({ prompt, system: systemPrompt, name: "执行导演", memoryKey: "assistant:execution", messages: [ { role: "assistant", content: artSkills.prompt + `\n${modelInfo}` }, { role: "user", content: prompt }, ], tools: { activate_skill: artSkills.tools.activate_skill }, }); }, }); //拍摄计划 const run_sub_agent_director_plan = tool({ description: "运行执行subAgent来完成导演规划相关任务", inputSchema: promptInput, execute: async ({ prompt }) => { const skill = path.join(u.getPath("skills"), "production_execution_director_plan.md"); const systemPrompt = await fs.promises.readFile(skill, "utf-8"); const addPrompt = "\n你必须使用如下XML格式写入工作区:\n```\n内容\n```"; return runAgent({ prompt, system: systemPrompt + addPrompt, name: "执行导演", memoryKey: "assistant:execution", messages: [ { role: "assistant", content: artSkills.prompt + `\n${modelInfo}` }, { role: "user", content: prompt + addPrompt }, ], tools: { activate_skill: artSkills.tools.activate_skill }, }); }, }); //分镜图生成 const run_sub_agent_storyboard_gen = tool({ description: "运行执行subAgent来完成分镜图生成相关任务", inputSchema: promptInput, execute: async ({ prompt }) => { const skill = path.join(u.getPath("skills"), "production_execution_storyboard_gen.md"); const systemPrompt = await fs.promises.readFile(skill, "utf-8"); return runAgent({ prompt, system: systemPrompt, name: "执行导演", memoryKey: "assistant:execution", messages: [ { role: "assistant", content: artSkills.prompt + `\n${modelInfo}` }, { role: "user", content: prompt }, ], tools: { activate_skill: artSkills.tools.activate_skill }, }); }, }); // const mainSkills: { path: string; name: string; description: string }[] = []; // for (const skill of mainSkill) { // const skillPath = path.join(rootDir, skill + ".md"); // if (!fs.existsSync(skillPath)) throw new Error(`主技能文件不存在: ${skillPath}`); // if (!isPathInside(skillPath, normalizedRootDir)) throw new Error(`技能名称无效:检测到路径穿越。${skillPath}`); // const content = await fs.promises.readFile(skillPath, "utf-8"); // const parsed = parseFrontmatter(content); // mainSkills.push({ path: skillPath, ...parsed }); // } const productionSkills = await useProductionSkills(projectInfo?.artStyle!, projectInfo?.directorManual!); //分镜面板写入 const run_sub_agent_storyboard_panel = tool({ description: "运行执行subAgent来完成分镜面板写入相关任务", inputSchema: promptInput, execute: async ({ prompt }) => { const skill = path.join(u.getPath("skills"), "production_execution_storyboard_panel.md"); const systemPrompt = await fs.promises.readFile(skill, "utf-8"); const addPrompt = "\n你必须使用如下XML格式写入工作区:\n```\n\n```"; return runAgent({ prompt, system: systemPrompt + addPrompt, name: "执行导演", memoryKey: "assistant:execution", messages: [ { role: "assistant", content: productionSkills.prompt + `\n${modelInfo}` }, { role: "user", content: prompt + addPrompt }, ], tools: { activate_skill: productionSkills.tools.activate_skill }, }); }, }); //分镜表写入 const run_sub_agent_storyboard_table = tool({ description: "运行执行subAgent来完成分镜表构建相关任务", inputSchema: promptInput, execute: async ({ prompt }) => { const skill = path.join(u.getPath("skills"), "production_execution_storyboard_table.md"); const systemPrompt = await fs.promises.readFile(skill, "utf-8"); const addPrompt = "\n你必须使用如下XML格式写入工作区:\n```\n内容\n```"; return runAgent({ prompt, system: systemPrompt + addPrompt, name: "执行导演", memoryKey: "assistant:execution", messages: [ { role: "assistant", content: productionSkills.prompt + `\n${modelInfo}` }, { role: "user", content: prompt + addPrompt }, ], tools: { activate_skill: productionSkills.tools.activate_skill }, }); }, }); const run_sub_agent_supervision = tool({ description: "运行监督层subAgent执行独立任务,完成后返回结果", inputSchema: promptInput, execute: async ({ prompt }) => { const skill = path.join(u.getPath("skills"), "production_agent_supervision.md"); const systemPrompt = await fs.promises.readFile(skill, "utf-8"); return runAgent({ prompt, system: systemPrompt, name: "监制", memoryKey: "assistant:supervision", }); }, }); return { run_sub_agent_derive_assets, run_sub_agent_generate_assets, run_sub_agent_director_plan, run_sub_agent_storyboard_gen, run_sub_agent_storyboard_panel, run_sub_agent_storyboard_table, run_sub_agent_supervision, }; } async function createArtSkills(artName: string, storyName: string) { const artWorkerPath = u.getPath(["skills", "art_skills", artName, "driector_skills"]); const storyWorkerPath = u.getPath(["skills", "story_skills", storyName, "driector_skills"]); const skillList = [...(await scanSkills(artWorkerPath + "/*.md")), ...(await scanSkills(storyWorkerPath + "/*.md"))]; const mainSkills: { path: string; name: string; description: string }[] = []; for (const skillPath of skillList) { if (!fs.existsSync(skillPath)) throw new Error(`主技能文件不存在: ${skillPath}`); const content = await fs.promises.readFile(skillPath, "utf-8"); const parsed = parseFrontmatter(content); mainSkills.push({ path: skillPath, ...parsed }); } const res = { prompt: `## Skills 以下技能提供了专业任务的专用指令。 当任务与某个技能的描述匹配时,调用 activate_skill 工具并传入技能名称来加载完整指令。 ${buildSkillPrompt(mainSkills)}`, tools: createSkillTools(mainSkills, { mainSkill: mainSkills, secondarySkills: [], tertiarySkills: [] }), }; return res; } function removeAllXmlTags(text: string): string { text = text.replace(/<([a-zA-Z][\w-]*)(\s+[^>]*)?>([\s\S]*?)<\/\1>/g, ""); text = text.replace(/<([a-zA-Z][\w-]*)(\s+[^>]*)?\/>/g, ""); text = text.replace(/<\/?[a-zA-Z][\w-]*(\s+[^>]*)?>/g, ""); return text.trim(); } export function buildSkillPrompt(skills: { name: string; description: string }[]): string { const skillEntries = skills .map((s) => ` \n ${s.name}\n ${s.description}\n `) .join("\n"); return ` ${skillEntries} `; } async function useProductionSkills(artName: string, storyName: string) { const artWorkerPath = u.getPath(["skills", "art_skills", artName, "driector_skills"]); const storyWorkerPath = u.getPath(["skills", "story_skills", storyName, "driector_skills"]); const productionPath = u.getPath(["skills", "production_skills"]); const skillList = [ ...(await scanSkills(artWorkerPath + "/*.md")), ...(await scanSkills(storyWorkerPath + "/*.md")), ...(await scanSkills(productionPath + "/*.md")), ]; const mainSkills: { path: string; name: string; description: string }[] = []; for (const skillPath of skillList) { if (!fs.existsSync(skillPath)) throw new Error(`主技能文件不存在: ${skillPath}`); const content = await fs.promises.readFile(skillPath, "utf-8"); const parsed = parseFrontmatter(content); mainSkills.push({ path: skillPath, ...parsed }); } const res = { prompt: `## Skills 以下技能提供了专业任务的专用指令。 当任务与某个技能的描述匹配时,调用 activate_skill 工具并传入技能名称来加载完整指令。 ${buildSkillPrompt(mainSkills)}`, tools: createSkillTools(mainSkills, { mainSkill: mainSkills, secondarySkills: [], tertiarySkills: [] }), }; return res; }