zyc/video-flow-toon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch '108' of https://github.com/HBAI-Ltd/Toonflow-app into 108

Browse Source 
# Conflicts:
#	src/types/database.d.ts
This commit is contained in:
小帅 2026-03-30 16:58:28 +08:00
commit 2525c51f38
20 changed files with 1478 additions and 468 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
data/skills/art_prompts/chinese_sweet_romance/driector_skills/director_planning.md
View File

@ -1,3 +1,9 @@
---
name: director_planning
description: 导演规划技法，定义古风甜宠写实超现实主义在主题立意、视觉基调、叙事节奏、场景意图与声音设计上的全局规划方法。
metaData: director_skills
---
# 导演规划 · 古风甜宠写实超现实主义 · 风格技法参考 # 导演规划 · 古风甜宠写实超现实主义 · 风格技法参考
--- ---

6
data/skills/art_prompts/chinese_sweet_romance/driector_skills/director_storyboard_table.md
View File

@ -1,3 +1,9 @@
---
name: director_storyboard_table
description: 分镜表设计技法，规范古风甜宠写实超现实主义在景别、运镜、时长、动作、光影与转场上的镜头语言表达。
metaData: director_skills
---
# 分镜表设计 · 古风甜宠写实超现实主义 · 风格技法参考 # 分镜表设计 · 古风甜宠写实超现实主义 · 风格技法参考
--- ---

1
package.json
View File

@ -70,6 +70,7 @@
"sqlite3": "^6.0.1", "sqlite3": "^6.0.1",
"sucrase": "^3.35.1", "sucrase": "^3.35.1",
"uuid": "^13.0.0", "uuid": "^13.0.0",
"vercel-minimax-ai-provider": "^0.0.2",
"vm2": "^3.10.5", "vm2": "^3.10.5",
"zhipu-ai-provider": "^0.2.2", "zhipu-ai-provider": "^0.2.2",
"zod": "^4.3.5" "zod": "^4.3.5"

118
src/agents/productionAgent/index.ts
View File

@ -3,7 +3,7 @@ import { tool } from "ai";
import { z } from "zod"; import { z } from "zod";
import u from "@/utils"; import u from "@/utils";
import Memory from "@/utils/agent/memory"; import Memory from "@/utils/agent/memory";
import { useSkill } from "@/utils/agent/skillsTools"; import { buildSkillPrompt, createSkillTools, parseFrontmatter, scanSkills, useSkill } from "@/utils/agent/skillsTools";
import useTools from "@/agents/productionAgent/tools"; import useTools from "@/agents/productionAgent/tools";
import ResTool from "@/socket/resTool"; import ResTool from "@/socket/resTool";
import * as fs from "fs"; import * as fs from "fs";
@ -77,12 +77,14 @@ function createSubAgent(parentCtx: AgentContext) {
name, name,
memoryKey, memoryKey,
tools: extraTools, tools: extraTools,
messages,
}: { }: {
prompt: string; prompt: string;
system: string; system: string;
name: string; name: string;
memoryKey: string; memoryKey: string;
tools?: Record<string, any>; tools?: Record<string, any>;
messages?: { role: "user" | "assistant" | "system"; content: string }[];
}) { }) {
parentCtx.msg.complete(); parentCtx.msg.complete();
const subMsg = resTool.newMessage("assistant", name); const subMsg = resTool.newMessage("assistant", name);
@ -91,7 +93,7 @@ function createSubAgent(parentCtx: AgentContext) {
const { textStream } = await u.Ai.Text("scriptAgent").stream({ const { textStream } = await u.Ai.Text("scriptAgent").stream({
system, system,
messages: [{ role: "user", content: prompt }], messages: messages ?? [{ role: "user", content: prompt }],
abortSignal, abortSignal,
tools: { ...extraTools, ...useTools({ resTool, msg: subMsg }) }, tools: { ...extraTools, ...useTools({ resTool, msg: subMsg }) },
}); });
@ -129,17 +131,26 @@ function createSubAgent(parentCtx: AgentContext) {
"\n" + "\n" +
[ [
"你可以使用如下XML格式写入工作区：\n```", "你可以使用如下XML格式写入工作区：\n```",
"剧本：<script>内容</script>",
"拍摄计划：<scriptPlan>内容</scriptPlan>", "拍摄计划：<scriptPlan>内容</scriptPlan>",
"分镜表：<storyboardTable>内容</storyboardTable>", "分镜表：<storyboardTable>内容</storyboardTable>",
"```", "```",
].join("\n"); ].join("\n");
// "剧本：<script>内容</script>",
const projectInfo = await u.db("o_project").where("id", resTool.data.projectId).first();
if (!projectInfo) throw new Error(`项目不存在，ID: ${resTool.data.projectId}`);
const artSkills = await createArtSkills(projectInfo?.artStyle!);
return runAgent({ return runAgent({
prompt, prompt,
system: systemPrompt + addPrompt, system: systemPrompt + addPrompt,
name: "执行导演", name: "执行导演",
memoryKey: "assistant:execution", memoryKey: "assistant:execution",
messages: [
{ role: "assistant", content: artSkills.prompt },
{ role: "user", content: prompt },
],
tools: { ...artSkills.tools },
}); });
}, },
}); });
@ -162,89 +173,18 @@ function createSubAgent(parentCtx: AgentContext) {
return { run_sub_agent_execution, run_sub_agent_supervision }; return { run_sub_agent_execution, run_sub_agent_supervision };
} }
// //====================== 执行层 ====================== async function createArtSkills(artName: string) {
const path = u.getPath(["skills", "art_prompts", artName, "driector_skills"]);
// export async function executionAI(ctx: AgentContext) { const skillList = await scanSkills(path + "/*.md");
// const { text, abortSignal } = ctx; const mainSkills: { path: string; name: string; description: string }[] = [];
for (const skillPath of skillList) {
// const skill = await useSkill({ if (!fs.existsSync(skillPath)) throw new Error(`主技能文件不存在: ${skillPath}`);
// mainSkill: "production_agent_execution", const content = await fs.promises.readFile(skillPath, "utf-8");
// workspace: ["production_agent_skills/execution"], const parsed = parseFrontmatter(content);
// attachedSkills: ["production_agent_skills/execution/driector_art_skills/chinese_sweet_romance/driector_skills"], //todo：后续可以改为动态加载 mainSkills.push({ path: skillPath, ...parsed });
// }); }
return {
// const subMsg = ctx.resTool.newMessage("assistant", "执行导演"); prompt: buildSkillPrompt(mainSkills),
tools: createSkillTools(mainSkills, { mainSkill: mainSkills, secondarySkills: [], tertiarySkills: [] }),
// const { textStream } = await u.Ai.Text("productionAgent").stream({ };
// system: skill.prompt, }
// messages: [{ role: "user", content: text }],
// abortSignal,
// tools: {
// ...skill.tools,
// ...useTools({ resTool: ctx.resTool, msg: subMsg }),
// },
// });
// return { textStream, subMsg };
// }
// export async function supervisionAI(ctx: AgentContext) {
// const { text, abortSignal } = ctx;
// const skill = await useSkill({ mainSkill: "production_agent_supervision", workspace: ["production_agent_skills/supervision"] });
// const subMsg = ctx.resTool.newMessage("assistant", "监制");
// const { textStream } = await u.Ai.Text("productionAgent").stream({
// system: skill.prompt,
// messages: [{ role: "user", content: text }],
// abortSignal,
// tools: {
// ...skill.tools,
// ...useTools({
// resTool: ctx.resTool,
// msg: subMsg,
// }),
// },
// });
// return { textStream, subMsg };
// }
// //工具函数
// function runSubAgent(parentCtx: AgentContext) {
// const memory = new Memory("productionAgent", parentCtx.isolationKey);
// return tool({
// description: "启动子Agent执行独立任务。可用子Agent:executionAI, decisionAI, supervisionAI",
// inputSchema: z.object({
// agent: z.enum(["executionAI", "supervisionAI"]).describe("子Agent名称"),
// prompt: z.string().describe("交给子Agent的任务简约描述，100字以内"),
// }),
// execute: async ({ agent, prompt }) => {
// const fn = [executionAI, supervisionAI][subAgentList.indexOf(agent)];
// // 先完成主Agent当前的消息
// parentCtx.msg.complete();
// // 子Agent用新消息回复
// const { textStream: subTextStream, subMsg } = await fn({ ...parentCtx, text: prompt });
// let text = subMsg.text();
// let fullResponse = "";
// for await (const chunk of subTextStream) {
// text.append(chunk);
// fullResponse += chunk;
// }
// text.complete();
// subMsg.complete();
// if (fullResponse.trim()) {
// await memory.add(`assistant:${agent === "executionAI" ? "execution" : "supervision"}`, fullResponse, {
// name: agent === "executionAI" ? "执行导演" : "监制",
// createTime: new Date(subMsg.datetime).getTime(),
// });
// }
// // 为主Agent后续输出创建新消息
// parentCtx.msg = parentCtx.resTool.newMessage("assistant", "监制");
// return fullResponse;
// },
// });
// }

54
src/agents/productionAgent/tools.ts
View File

@ -59,12 +59,6 @@ const flowDataSchema = z.object({
assets: z.array(assetItemSchema).describe("衍生资产"), assets: z.array(assetItemSchema).describe("衍生资产"),
storyboardTable: z.string().describe("分镜表"), storyboardTable: z.string().describe("分镜表"),
storyboard: z.array(storyboardSchema).describe("分镜面板"), storyboard: z.array(storyboardSchema).describe("分镜面板"),
workbench: workbenchDataSchema.describe("工作台配置"),
poster: z
.object({
items: z.array(posterItemSchema).describe("海报项目列表"),
})
.describe("海报配置"),
}); });
export type FlowData = z.infer<typeof flowDataSchema>; export type FlowData = z.infer<typeof flowDataSchema>;
@ -154,10 +148,46 @@ export default (toolCpnfig: ToolConfig) => {
return res ?? "删除成功"; return res ?? "删除成功";
}, },
}), }),
add_storyboard: tool({
description: "新增或更新分镜面板",
inputSchema: z.object({
id: z.number().nullable().describe("分镜面板ID,如果新增则为空"),
title: z.string().describe("分镜面板名称"),
desc: z.string().describe("分镜面板描述"),
group: z.number().describe("分镜面板分组,根据这个字段 对分镜图片，进行同时生成视频，例如 同一分组的两张图片会被用于首尾帧生成视频"),
}),
execute: async (storyboard) => {
const thinking = msg.thinking("正在操作资产...");
const { projectId, scriptId } = resTool.data;
const createTime = Date.now();
console.log("%c Line:161 🍤 storyboard", "background:#e41a6a", storyboard);
const data = {
id: storyboard.id ?? undefined,
title: storyboard.title,
description: storyboard.desc,
createTime,
scriptId,
};
if (storyboard.id) {
await u.db("o_storyboard").where("id", storyboard.id).update(data);
thinking.appendText(`已更新分镜面板，ID: ${storyboard.id}\n`);
} else {
const [insertedId] = await u.db("o_storyboard").insert(data);
data.id = insertedId;
thinking.appendText(`已新增分镜面板，ID: ${insertedId}\n`);
}
const res = await new Promise((resolve) => socket.emit("addStoryboard", data, (res: any) => resolve(res)));
thinking.updateTitle("分镜面板操作完成");
thinking.complete();
return res ?? "操作成功";
},
}),
generate_deriveAsset: tool({ generate_deriveAsset: tool({
description: "生成衍生资产", description: "生成衍生资产",
inputSchema: z.object({ inputSchema: z.object({
id: z.number().describe("衍生资产ID"), id: z.array(z.number()).describe("需要生成的 衍生资产ID"),
}), }),
execute: async ({ id }) => { execute: async ({ id }) => {
const thinking = msg.thinking("正在生成衍生资产..."); const thinking = msg.thinking("正在生成衍生资产...");
@ -169,11 +199,13 @@ export default (toolCpnfig: ToolConfig) => {
}, },
}), }),
generate_storyboard: tool({ generate_storyboard: tool({
description: "生成分镜", description: "生成分镜图片",
inputSchema: z.object({}), inputSchema: z.object({
execute: async ({ script }) => { storyboardIds: z.array(z.number()).describe("分镜ID列表"),
}),
execute: async ({ storyboardIds }) => {
const thinking = msg.thinking("正在生成分镜..."); const thinking = msg.thinking("正在生成分镜...");
const res = await new Promise((resolve) => socket.emit("generateStoryboard", { script }, (res: any) => resolve(res))); const res = await new Promise((resolve) => socket.emit("generateStoryboard", { storyboardIds }, (res: any) => resolve(res)));
thinking.appendText("生成的分镜数据:\n" + JSON.stringify(res, null, 2)); thinking.appendText("生成的分镜数据:\n" + JSON.stringify(res, null, 2));
thinking.updateTitle("分镜生成完成"); thinking.updateTitle("分镜生成完成");
thinking.complete(); thinking.complete();

292
src/router.ts
View File

@ -1,4 +1,4 @@
// @routes-hash c1392b39a921d712296ccb6b4aea3507 // @routes-hash 845d6aff66aab1f458a9f08f4f2eed34
import { Express } from "express"; import { Express } from "express";
import route1 from "./routes/agents/clearMemory"; import route1 from "./routes/agents/clearMemory";
@ -45,77 +45,80 @@ import route41 from "./routes/novel/getNovelIndex";
import route42 from "./routes/novel/updateNovel"; import route42 from "./routes/novel/updateNovel";
import route43 from "./routes/other/deleteAllData"; import route43 from "./routes/other/deleteAllData";
import route44 from "./routes/other/getVersion"; import route44 from "./routes/other/getVersion";
import route45 from "./routes/production/assets/getAssetsData"; import route45 from "./routes/production/assets/batchGenerateAssetsImage";
import route46 from "./routes/production/editImage/generateFlowImage"; import route46 from "./routes/production/assets/getAssetsData";
import route47 from "./routes/production/editImage/getImageFlow"; import route47 from "./routes/production/assets/pollingImage";
import route48 from "./routes/production/editImage/saveImageFlow"; import route48 from "./routes/production/editImage/generateFlowImage";
import route49 from "./routes/production/editImage/updateImageFlow"; import route49 from "./routes/production/editImage/getImageFlow";
import route50 from "./routes/production/exportImage"; import route50 from "./routes/production/editImage/saveImageFlow";
import route51 from "./routes/production/getFlowData"; import route51 from "./routes/production/editImage/updateImageFlow";
import route52 from "./routes/production/getProductionData"; import route52 from "./routes/production/exportImage";
import route53 from "./routes/production/getStoryboardData"; import route53 from "./routes/production/getFlowData";
import route54 from "./routes/production/saveFlowData"; import route54 from "./routes/production/getProductionData";
import route55 from "./routes/production/storyboard/batchGenerateImage"; import route55 from "./routes/production/getStoryboardData";
import route56 from "./routes/production/storyboard/downPreviewImage"; import route56 from "./routes/production/saveFlowData";
import route57 from "./routes/production/storyboard/getStoryboardData"; import route57 from "./routes/production/storyboard/batchGenerateImage";
import route58 from "./routes/production/storyboard/previewImage"; import route58 from "./routes/production/storyboard/downPreviewImage";
import route59 from "./routes/production/workbench/confirmSelection"; import route59 from "./routes/production/storyboard/getStoryboardData";
import route60 from "./routes/production/workbench/delVideo"; import route60 from "./routes/production/storyboard/pollingImage";
import route61 from "./routes/production/workbench/generateVideo"; import route61 from "./routes/production/storyboard/previewImage";
import route62 from "./routes/production/workbench/generateVideoPrompt"; import route62 from "./routes/production/workbench/confirmSelection";
import route63 from "./routes/production/workbench/getChatLines"; import route63 from "./routes/production/workbench/delVideo";
import route64 from "./routes/production/workbench/getVideoModelDetail"; import route64 from "./routes/production/workbench/generateVideo";
import route65 from "./routes/production/workbench/videoPolling"; import route65 from "./routes/production/workbench/generateVideoPrompt";
import route66 from "./routes/project/addProject"; import route66 from "./routes/production/workbench/getChatLines";
import route67 from "./routes/project/addVisual"; import route67 from "./routes/production/workbench/getVideoModelDetail";
import route68 from "./routes/project/addVisualManual"; import route68 from "./routes/production/workbench/videoPolling";
import route69 from "./routes/project/deleteVisualManual"; import route69 from "./routes/project/addProject";
import route70 from "./routes/project/delProject"; import route70 from "./routes/project/addVisual";
import route71 from "./routes/project/editProject"; import route71 from "./routes/project/addVisualManual";
import route72 from "./routes/project/editVisualManual"; import route72 from "./routes/project/deleteVisualManual";
import route73 from "./routes/project/getProject"; import route73 from "./routes/project/delProject";
import route74 from "./routes/project/getVisualManual"; import route74 from "./routes/project/editProject";
import route75 from "./routes/project/visualManual"; import route75 from "./routes/project/editVisualManual";
import route76 from "./routes/script/addScript"; import route76 from "./routes/project/getProject";
import route77 from "./routes/script/delScript"; import route77 from "./routes/project/getVisualManual";
import route78 from "./routes/script/exportScript"; import route78 from "./routes/project/visualManual";
import route79 from "./routes/script/extractAssets"; import route79 from "./routes/script/addScript";
import route80 from "./routes/script/getScrptApi"; import route80 from "./routes/script/delScript";
import route81 from "./routes/script/pollScriptAssets"; import route81 from "./routes/script/exportScript";
import route82 from "./routes/script/updateScript"; import route82 from "./routes/script/extractAssets";
import route83 from "./routes/scriptAgent/getPlanData"; import route83 from "./routes/script/getScrptApi";
import route84 from "./routes/scriptAgent/setPlanData"; import route84 from "./routes/script/pollScriptAssets";
import route85 from "./routes/setting/about/checkUpdate"; import route85 from "./routes/script/updateScript";
import route86 from "./routes/setting/about/downloadApp"; import route86 from "./routes/scriptAgent/getPlanData";
import route87 from "./routes/setting/agentDeploy/agentSetKey"; import route87 from "./routes/scriptAgent/setPlanData";
import route88 from "./routes/setting/agentDeploy/deployAgentModel"; import route88 from "./routes/setting/about/checkUpdate";
import route89 from "./routes/setting/agentDeploy/getAgentDeploy"; import route89 from "./routes/setting/about/downloadApp";
import route90 from "./routes/setting/dbConfig/clearData"; import route90 from "./routes/setting/agentDeploy/agentSetKey";
import route91 from "./routes/setting/dev/getSwitchAiDevTool"; import route91 from "./routes/setting/agentDeploy/deployAgentModel";
import route92 from "./routes/setting/dev/updateSwitchAiDevTool"; import route92 from "./routes/setting/agentDeploy/getAgentDeploy";
import route93 from "./routes/setting/fileManagement/openFolder"; import route93 from "./routes/setting/dbConfig/clearData";
import route94 from "./routes/setting/getTextModel"; import route94 from "./routes/setting/dev/getSwitchAiDevTool";
import route95 from "./routes/setting/loginConfig/getUser"; import route95 from "./routes/setting/dev/updateSwitchAiDevTool";
import route96 from "./routes/setting/loginConfig/updateUserPwd"; import route96 from "./routes/setting/fileManagement/openFolder";
import route97 from "./routes/setting/memoryConfig/delAllMemory"; import route97 from "./routes/setting/getTextModel";
import route98 from "./routes/setting/memoryConfig/getMemory"; import route98 from "./routes/setting/loginConfig/getUser";
import route99 from "./routes/setting/memoryConfig/sureMemory"; import route99 from "./routes/setting/loginConfig/updateUserPwd";
import route100 from "./routes/setting/promptManage/getPrompt"; import route100 from "./routes/setting/memoryConfig/delAllMemory";
import route101 from "./routes/setting/promptManage/updatePrompt"; import route101 from "./routes/setting/memoryConfig/getMemory";
import route102 from "./routes/setting/skillManagement/getSkillContent"; import route102 from "./routes/setting/memoryConfig/sureMemory";
import route103 from "./routes/setting/skillManagement/getSkillList"; import route103 from "./routes/setting/promptManage/getPrompt";
import route104 from "./routes/setting/skillManagement/saveSkillContent"; import route104 from "./routes/setting/promptManage/updatePrompt";
import route105 from "./routes/setting/vendorConfig/addVendor"; import route105 from "./routes/setting/skillManagement/getSkillContent";
import route106 from "./routes/setting/vendorConfig/deleteVendor"; import route106 from "./routes/setting/skillManagement/getSkillList";
import route107 from "./routes/setting/vendorConfig/getVendorList"; import route107 from "./routes/setting/skillManagement/saveSkillContent";
import route108 from "./routes/setting/vendorConfig/modelTest"; import route108 from "./routes/setting/vendorConfig/addVendor";
import route109 from "./routes/setting/vendorConfig/updateCode"; import route109 from "./routes/setting/vendorConfig/deleteVendor";
import route110 from "./routes/setting/vendorConfig/updateVendor"; import route110 from "./routes/setting/vendorConfig/getVendorList";
import route111 from "./routes/task/getProject"; import route111 from "./routes/setting/vendorConfig/modelTest";
import route112 from "./routes/task/getTaskApi"; import route112 from "./routes/setting/vendorConfig/updateCode";
import route113 from "./routes/task/getTaskCategories"; import route113 from "./routes/setting/vendorConfig/updateVendor";
import route114 from "./routes/task/taskDetails"; import route114 from "./routes/task/getProject";
import route115 from "./routes/test/test"; import route115 from "./routes/task/getTaskApi";
import route116 from "./routes/task/getTaskCategories";
import route117 from "./routes/task/taskDetails";
import route118 from "./routes/test/test";
export default async (app: Express) => { export default async (app: Express) => {
app.use("/api/agents/clearMemory", route1); app.use("/api/agents/clearMemory", route1);
@ -162,75 +165,78 @@ export default async (app: Express) => {
app.use("/api/novel/updateNovel", route42); app.use("/api/novel/updateNovel", route42);
app.use("/api/other/deleteAllData", route43); app.use("/api/other/deleteAllData", route43);
app.use("/api/other/getVersion", route44); app.use("/api/other/getVersion", route44);
app.use("/api/production/assets/getAssetsData", route45); app.use("/api/production/assets/batchGenerateAssetsImage", route45);
app.use("/api/production/editImage/generateFlowImage", route46); app.use("/api/production/assets/getAssetsData", route46);
app.use("/api/production/editImage/getImageFlow", route47); app.use("/api/production/assets/pollingImage", route47);
app.use("/api/production/editImage/saveImageFlow", route48); app.use("/api/production/editImage/generateFlowImage", route48);
app.use("/api/production/editImage/updateImageFlow", route49); app.use("/api/production/editImage/getImageFlow", route49);
app.use("/api/production/exportImage", route50); app.use("/api/production/editImage/saveImageFlow", route50);
app.use("/api/production/getFlowData", route51); app.use("/api/production/editImage/updateImageFlow", route51);
app.use("/api/production/getProductionData", route52); app.use("/api/production/exportImage", route52);
app.use("/api/production/getStoryboardData", route53); app.use("/api/production/getFlowData", route53);
app.use("/api/production/saveFlowData", route54); app.use("/api/production/getProductionData", route54);
app.use("/api/production/storyboard/batchGenerateImage", route55); app.use("/api/production/getStoryboardData", route55);
app.use("/api/production/storyboard/downPreviewImage", route56); app.use("/api/production/saveFlowData", route56);
app.use("/api/production/storyboard/getStoryboardData", route57); app.use("/api/production/storyboard/batchGenerateImage", route57);
app.use("/api/production/storyboard/previewImage", route58); app.use("/api/production/storyboard/downPreviewImage", route58);
app.use("/api/production/workbench/confirmSelection", route59); app.use("/api/production/storyboard/getStoryboardData", route59);
app.use("/api/production/workbench/delVideo", route60); app.use("/api/production/storyboard/pollingImage", route60);
app.use("/api/production/workbench/generateVideo", route61); app.use("/api/production/storyboard/previewImage", route61);
app.use("/api/production/workbench/generateVideoPrompt", route62); app.use("/api/production/workbench/confirmSelection", route62);
app.use("/api/production/workbench/getChatLines", route63); app.use("/api/production/workbench/delVideo", route63);
app.use("/api/production/workbench/getVideoModelDetail", route64); app.use("/api/production/workbench/generateVideo", route64);
app.use("/api/production/workbench/videoPolling", route65); app.use("/api/production/workbench/generateVideoPrompt", route65);
app.use("/api/project/addProject", route66); app.use("/api/production/workbench/getChatLines", route66);
app.use("/api/project/addVisual", route67); app.use("/api/production/workbench/getVideoModelDetail", route67);
app.use("/api/project/addVisualManual", route68); app.use("/api/production/workbench/videoPolling", route68);
app.use("/api/project/deleteVisualManual", route69); app.use("/api/project/addProject", route69);
app.use("/api/project/delProject", route70); app.use("/api/project/addVisual", route70);
app.use("/api/project/editProject", route71); app.use("/api/project/addVisualManual", route71);
app.use("/api/project/editVisualManual", route72); app.use("/api/project/deleteVisualManual", route72);
app.use("/api/project/getProject", route73); app.use("/api/project/delProject", route73);
app.use("/api/project/getVisualManual", route74); app.use("/api/project/editProject", route74);
app.use("/api/project/visualManual", route75); app.use("/api/project/editVisualManual", route75);
app.use("/api/script/addScript", route76); app.use("/api/project/getProject", route76);
app.use("/api/script/delScript", route77); app.use("/api/project/getVisualManual", route77);
app.use("/api/script/exportScript", route78); app.use("/api/project/visualManual", route78);
app.use("/api/script/extractAssets", route79); app.use("/api/script/addScript", route79);
app.use("/api/script/getScrptApi", route80); app.use("/api/script/delScript", route80);
app.use("/api/script/pollScriptAssets", route81); app.use("/api/script/exportScript", route81);
app.use("/api/script/updateScript", route82); app.use("/api/script/extractAssets", route82);
app.use("/api/scriptAgent/getPlanData", route83); app.use("/api/script/getScrptApi", route83);
app.use("/api/scriptAgent/setPlanData", route84); app.use("/api/script/pollScriptAssets", route84);
app.use("/api/setting/about/checkUpdate", route85); app.use("/api/script/updateScript", route85);
app.use("/api/setting/about/downloadApp", route86); app.use("/api/scriptAgent/getPlanData", route86);
app.use("/api/setting/agentDeploy/agentSetKey", route87); app.use("/api/scriptAgent/setPlanData", route87);
app.use("/api/setting/agentDeploy/deployAgentModel", route88); app.use("/api/setting/about/checkUpdate", route88);
app.use("/api/setting/agentDeploy/getAgentDeploy", route89); app.use("/api/setting/about/downloadApp", route89);
app.use("/api/setting/dbConfig/clearData", route90); app.use("/api/setting/agentDeploy/agentSetKey", route90);
app.use("/api/setting/dev/getSwitchAiDevTool", route91); app.use("/api/setting/agentDeploy/deployAgentModel", route91);
app.use("/api/setting/dev/updateSwitchAiDevTool", route92); app.use("/api/setting/agentDeploy/getAgentDeploy", route92);
app.use("/api/setting/fileManagement/openFolder", route93); app.use("/api/setting/dbConfig/clearData", route93);
app.use("/api/setting/getTextModel", route94); app.use("/api/setting/dev/getSwitchAiDevTool", route94);
app.use("/api/setting/loginConfig/getUser", route95); app.use("/api/setting/dev/updateSwitchAiDevTool", route95);
app.use("/api/setting/loginConfig/updateUserPwd", route96); app.use("/api/setting/fileManagement/openFolder", route96);
app.use("/api/setting/memoryConfig/delAllMemory", route97); app.use("/api/setting/getTextModel", route97);
app.use("/api/setting/memoryConfig/getMemory", route98); app.use("/api/setting/loginConfig/getUser", route98);
app.use("/api/setting/memoryConfig/sureMemory", route99); app.use("/api/setting/loginConfig/updateUserPwd", route99);
app.use("/api/setting/promptManage/getPrompt", route100); app.use("/api/setting/memoryConfig/delAllMemory", route100);
app.use("/api/setting/promptManage/updatePrompt", route101); app.use("/api/setting/memoryConfig/getMemory", route101);
app.use("/api/setting/skillManagement/getSkillContent", route102); app.use("/api/setting/memoryConfig/sureMemory", route102);
app.use("/api/setting/skillManagement/getSkillList", route103); app.use("/api/setting/promptManage/getPrompt", route103);
app.use("/api/setting/skillManagement/saveSkillContent", route104); app.use("/api/setting/promptManage/updatePrompt", route104);
app.use("/api/setting/vendorConfig/addVendor", route105); app.use("/api/setting/skillManagement/getSkillContent", route105);
app.use("/api/setting/vendorConfig/deleteVendor", route106); app.use("/api/setting/skillManagement/getSkillList", route106);
app.use("/api/setting/vendorConfig/getVendorList", route107); app.use("/api/setting/skillManagement/saveSkillContent", route107);
app.use("/api/setting/vendorConfig/modelTest", route108); app.use("/api/setting/vendorConfig/addVendor", route108);
app.use("/api/setting/vendorConfig/updateCode", route109); app.use("/api/setting/vendorConfig/deleteVendor", route109);
app.use("/api/setting/vendorConfig/updateVendor", route110); app.use("/api/setting/vendorConfig/getVendorList", route110);
app.use("/api/task/getProject", route111); app.use("/api/setting/vendorConfig/modelTest", route111);
app.use("/api/task/getTaskApi", route112); app.use("/api/setting/vendorConfig/updateCode", route112);
app.use("/api/task/getTaskCategories", route113); app.use("/api/setting/vendorConfig/updateVendor", route113);
app.use("/api/task/taskDetails", route114); app.use("/api/task/getProject", route114);
app.use("/api/test/test", route115); app.use("/api/task/getTaskApi", route115);
app.use("/api/task/getTaskCategories", route116);
app.use("/api/task/taskDetails", route117);
app.use("/api/test/test", route118);
} }

3
src/routes/novel/event/generateEvents.ts
View File

@ -23,9 +23,6 @@ export default router.post(
if (allChapters.length === 0) { if (allChapters.length === 0) {
return res.status(400).send(success("没有对应章节")); return res.status(400).send(success("没有对应章节"));
} }
if (allChapters.filter((item) => item.eventState === 0).length) {
return res.status(400).send(success("存在未完成事件，请先等待事件完成"));
}
await u.db("o_novel").where("projectId", projectId).whereIn("id", novelIds).update({ eventState: 0, event: null }); await u.db("o_novel").where("projectId", projectId).whereIn("id", novelIds).update({ eventState: 0, event: null });
novel.emitter.on("item", async (item) => { novel.emitter.on("item", async (item) => {
await u await u

80
src/routes/production/assets/batchGenerateAssetsImage.ts Normal file
View File

@ -0,0 +1,80 @@
import express from "express";
import u from "@/utils";
import { z } from "zod";
import sharp from "sharp";
import { success } from "@/lib/responseFormat";
import { validateFields } from "@/middleware/middleware";
import { Output } from "ai";
const router = express.Router();
export default router.post(
"/",
validateFields({
assetIds: z.array(z.number()),
projectId: z.number(),
scriptId: z.number(),
}),
async (req, res) => {
const { assetIds, projectId, scriptId } = req.body;
const projectSettingData = await u.db("o_project").where("id", projectId).select("imageModel", "imageQuality", "artStyle").first();
const assetsDataArr = await u.db("o_assets").whereIn("id", assetIds).select("id", "describe", "name", "type");
const rolePrompt = u.getArtPrompt(projectSettingData!.artStyle!, "art_character_derivative");
const toolPrompt = u.getArtPrompt(projectSettingData!.artStyle!, "art_prop_derivative");
const scenePrompt = u.getArtPrompt(projectSettingData!.artStyle!, "art_scene_derivative");
const promptRecord = {
role: rolePrompt,
tool: toolPrompt,
scene: scenePrompt,
};
for (const item of assetsDataArr) {
const { text } = await u.Ai.Text("universalAi").invoke({
system: `
你需要根据用户提供的资产的标题与描述，结合当前项目的美术风格，为我优化提示词以便生成更符合项目美术风格的图片。直接输出提示词，不需要做任何解释说明。
美术风格：${promptRecord[item.type! as keyof typeof promptRecord]}`,
messages: [
{
role: "user",
content: `资产名称:${item.name},资产描述:${item.describe}`,
},
],
});
console.log("%c Line:35 🎂 text", "background:#3f7cff", text);
const repeloadObj = {
prompt: text,
size: projectSettingData?.imageQuality as "1K" | "2K" | "4K",
aspectRatio: "16:9",
};
const [imageId] = await u.db("o_image").insert({
assetsId: item.id,
type: item.type,
state: "生成中",
resolution: projectSettingData?.imageQuality,
model: projectSettingData?.imageModel,
});
u.Ai.Image(projectSettingData?.imageModel as `${string}:${string}`)
.run({
prompt: text,
imageBase64: [],
size: projectSettingData?.imageQuality as "1K" | "2K" | "4K",
aspectRatio: "16:9",
taskClass: "生成图片",
describe: "资产图片生成",
relatedObjects: JSON.stringify(repeloadObj),
projectId: projectId,
})
.then(async (imageCls) => {
const savePath = `/${projectId}/assets/${scriptId}/${u.uuid()}.jpg`;
await imageCls.save(savePath);
// 更新对应数据库
await u.db("o_assets").where("id", item.id).update({ imageId: imageId });
await u.db("o_image").where({ id: imageId }).update({ state: "已完成", filePath: savePath });
});
}
return res.status(200).send(success());
},
);

29
src/routes/production/assets/pollingImage.ts Normal file
View File

@ -0,0 +1,29 @@
import express from "express";
import u from "@/utils";
import { z } from "zod";
import { success } from "@/lib/responseFormat";
import { validateFields } from "@/middleware/middleware";
const router = express.Router();
export default router.post(
"/",
validateFields({
ids: z.array(z.number()),
}),
async (req, res) => {
const { ids } = req.body;
const data = await u
.db("o_assets")
.leftJoin("o_image", "o_assets.imageId", "o_image.id")
.whereIn("o_assets.id", ids)
.whereNot("o_image.state", "生成中")
.select("o_image.state", "o_assets.id", "o_image.filePath");
const result = await Promise.all(
data.map(async (item: any) => ({
...item,
src: item.filePath ? await u.oss.getFileUrl(item.filePath) : null,
})),
);
res.status(200).send(success(result));
},
);

82
src/routes/production/storyboard/batchGenerateImage.ts
View File

@ -12,61 +12,65 @@ export default router.post(
validateFields({ validateFields({
storyboardIds: z.array(z.number()), storyboardIds: z.array(z.number()),
projectId: z.number(), projectId: z.number(),
scriptId: z.number(),
}), }),
async (req, res) => { async (req, res) => {
const { storyboardIds, projectId } = req.body; const { storyboardIds, projectId, scriptId } = req.body;
const projectSettingData = await u.db("o_project").where("id", projectId).select("imageModel", "imageQuality", "artStyle").first(); const projectSettingData = await u.db("o_project").where("id", projectId).select("imageModel", "imageQuality", "artStyle").first();
const sceneArkPrompt = u.getArtPrompt(projectSettingData?.artStyle || "", "art_storyboard"); const sceneArkPrompt = u.getArtPrompt(projectSettingData?.artStyle || "", "art_storyboard");
const storyboardData = await u.db("o_storyboard").whereIn("id", storyboardIds).select("id", "description", "title"); const storyboardData = await u.db("o_storyboard").whereIn("id", storyboardIds).select("id", "description", "title");
const { text } = await u.Ai.Text("universalAi").invoke({
system: ` for (const item of storyboardData) {
const { text } = await u.Ai.Text("universalAi").invoke({
system: `
你需要根据用户提供的分镜的标题与描述，结合当前项目的美术风格，为我优化提示词以便生成更符合项目美术风格的分镜图片。请你只优化提示词，不要添加任何额外的描述性文字，请以JSON格式输出: [{id:"对应分镜ID",prompt:"分镜提示词"}]。 你需要根据用户提供的分镜的标题与描述，结合当前项目的美术风格，为我优化提示词以便生成更符合项目美术风格的分镜图片。请你只优化提示词，不要添加任何额外的描述性文字，请以JSON格式输出: [{id:"对应分镜ID",prompt:"分镜提示词"}]。
美术风格：${sceneArkPrompt}`, 美术风格：${sceneArkPrompt}`,
messages: [ messages: [
{ {
role: "user", role: "user",
content: `一下是我的分镜内容\n ${storyboardData.map((s) => `分镜ID:${s.id},分镜描述:${s.description},分镜标题:${s.title}`).join("\n")}`, content: `分镜描述:${item.description}`,
}, },
], ],
output: Output.object({ });
schema: z.array(
z.object({
prompt: z.string().describe("优化后的提示词"),
}),
),
}),
});
for (const item of storyboardData) {
const repeloadObj = { const repeloadObj = {
prompt: text, prompt: text,
size: projectSettingData?.imageQuality as "1K" | "2K" | "4K", size: projectSettingData?.imageQuality as "1K" | "2K" | "4K",
aspectRatio: "16:9", aspectRatio: "16:9",
}; };
u.Ai.Image(projectSettingData?.imageModel as `${string}:${string}`).run({ await u.db("o_storyboard").where("id", item.id).update({
prompt: text, prompt: text,
imageBase64: [], state: "生成中",
size: projectSettingData?.imageQuality as "1K" | "2K" | "4K",
aspectRatio: "16:9",
taskClass: "生成图片",
describe: "资产图片生成",
relatedObjects: JSON.stringify(repeloadObj),
projectId: projectId,
}); });
// .then(async (imageCls) => { u.Ai.Image(projectSettingData?.imageModel as `${string}:${string}`)
// const savePath = `/${resTool.data.projectId}/assets/${resTool.data.scriptId}/${u.uuid()}.jpg`; .run({
// await imageCls.save(savePath); prompt: text,
// const obj = { imageBase64: [],
// ...item, size: projectSettingData?.imageQuality as "1K" | "2K" | "4K",
// id: item.assetId, aspectRatio: "16:9",
// src: await u.oss.getFileUrl(savePath), taskClass: "生成图片",
// state: "已完成", describe: "资产图片生成",
// }; relatedObjects: JSON.stringify(repeloadObj),
//更新对应数据库 projectId: projectId,
// await u.db("o_assets").where("id", item.assetId).update({ imageId: imageId }); })
// await u.db("o_image").where({ id: imageId }).update({ state: "已完成", filePath: savePath }); .then(async (imageCls) => {
// }); const savePath = `/${projectId}/assets/${scriptId}/${u.uuid()}.jpg`;
await imageCls.save(savePath);
await u.db("o_storyboard").where("id", item.id).update({
filePath: savePath,
state: "已完成",
});
})
.catch(async (e) => {
await u
.db("o_storyboard")
.where("id", item.id)
.update({
reason: u.error(e).message,
state: "生成失败",
});
});
} }
return res.status(200).send(success()); return res.status(200).send(success());

24
src/routes/production/storyboard/pollingImage.ts Normal file
View File

@ -0,0 +1,24 @@
import express from "express";
import u from "@/utils";
import { z } from "zod";
import { success } from "@/lib/responseFormat";
import { validateFields } from "@/middleware/middleware";
const router = express.Router();
export default router.post(
"/",
validateFields({
ids: z.array(z.number()),
}),
async (req, res) => {
const { ids } = req.body;
const data = await u.db("o_storyboard").whereIn("id", ids).whereNot("state", "生成中").select("id", "state", "reason", "filePath", "prompt");
const result = await Promise.all(
data.map(async (item: any) => ({
...item,
src: item.filePath ? await u.oss.getFileUrl(item.filePath) : null,
})),
);
res.status(200).send(success(result));
},
);

246
src/routes/script/extractAssets.ts
View File

@ -8,6 +8,22 @@ import { tool } from "ai";
import { o_script } from "@/types/database"; import { o_script } from "@/types/database";
const router = express.Router(); const router = express.Router();
/** 新资产：AI 首次识别到的资产，需要完整信息 */
const NewAssetSchema = z.object({
prompt: z.string().describe("生成提示词"),
name: z.string().describe("资产名称,仅为名称不做其他任何表述"),
desc: z.string().describe("资产描述"),
type: z.enum(["role", "tool", "scene"]).describe("资产类型"),
scriptIds: z.array(z.number()).describe("使用该资产的剧本id数组"),
});
/** 已有资产：数据库中已存在的资产，只需给出名称和关联的剧本 */
const ExistingAssetRefSchema = z.object({
name: z.string().describe("已有资产的名称,必须与已有资产列表中的名称完全一致"),
scriptIds: z.array(z.number()).describe("使用该资产的剧本id数组"),
});
export const AssetSchema = z.object({ export const AssetSchema = z.object({
prompt: z.string().describe("生成提示词"), prompt: z.string().describe("生成提示词"),
name: z.string().describe("资产名称,仅为名称不做其他任何表述"), name: z.string().describe("资产名称,仅为名称不做其他任何表述"),
@ -15,23 +31,24 @@ export const AssetSchema = z.object({
type: z.enum(["role", "tool", "scene"]).describe("资产类型"), type: z.enum(["role", "tool", "scene"]).describe("资产类型"),
}); });
type NewAsset = z.infer<typeof NewAssetSchema>;
type ExistingAssetRef = z.infer<typeof ExistingAssetRefSchema>;
type Asset = z.infer<typeof AssetSchema>; type Asset = z.infer<typeof AssetSchema>;
/** 按批次并发执行，每批 batchSize 个同时跑，批次完成后调用 onBatchDone */ /** 每批 AI 调用的结果 */
async function pMapBatch<T, R>( type GroupResult = {
items: T[], batchScriptIds: number[];
fn: (item: T) => Promise<R>, newAssets: NewAsset[];
batchSize: number, existingRefs: ExistingAssetRef[];
onBatchDone?: (batchResults: R[]) => Promise<void>, } | null;
): Promise<R[]> {
const allResults: R[] = []; /** 将 scriptIds 数组按 groupSize 分组 */
for (let i = 0; i < items.length; i += batchSize) { function chunkArray<T>(arr: T[], groupSize: number): T[][] {
const batch = items.slice(i, i + batchSize); const chunks: T[][] = [];
const batchResults = await Promise.all(batch.map(fn)); for (let i = 0; i < arr.length; i += groupSize) {
allResults.push(...batchResults); chunks.push(arr.slice(i, i + groupSize));
if (onBatchDone) await onBatchDone(batchResults);
} }
return allResults; return chunks;
} }
export default router.post( export default router.post(
@ -39,10 +56,10 @@ export default router.post(
validateFields({ validateFields({
scriptIds: z.array(z.number()), scriptIds: z.array(z.number()),
projectId: z.number(), projectId: z.number(),
concurrency: z.number().min(1).max(20).optional(), groupSize: z.number().min(1).max(10).optional(),
}), }),
async (req, res) => { async (req, res) => {
const { scriptIds, projectId, concurrency = 3 } = req.body; const { scriptIds, projectId, groupSize = 5 } = req.body;
if (!scriptIds.length) return res.status(400).send(error("请先选择剧本")); if (!scriptIds.length) return res.status(400).send(error("请先选择剧本"));
const scripts = await u.db("o_script").whereIn("id", scriptIds); const scripts = await u.db("o_script").whereIn("id", scriptIds);
const intansce = u.Ai.Text("universalAi"); const intansce = u.Ai.Text("universalAi");
@ -57,34 +74,21 @@ export default router.post(
const errors: { scriptId: number; error: string }[] = []; const errors: { scriptId: number; error: string }[] = [];
let successCount = 0; let successCount = 0;
// 每批提取结果：scriptId -> 资产列表 // 将 scriptIds 按 groupSize（默认5）分组，每组一起发给 AI
type BatchResult = { scriptId: number; assets: Asset[] } | null; const scriptGroups = chunkArray(scriptIds, groupSize);
/** 一批剧本提取完成后统一入库并建立关联 */ /** 一组剧本提取完成后统一入库并建立关联 */
async function persistBatch(batchResults: BatchResult[]) { async function persistGroupResult(result: GroupResult) {
const validResults = batchResults.filter((r): r is { scriptId: number; assets: Asset[] } => r !== null && r.assets.length > 0); if (!result) return;
if (!validResults.length) return; const { batchScriptIds, newAssets, existingRefs } = result;
if (!newAssets.length && !existingRefs.length) return;
// 合并本批所有资产，同名去重 // 查询已有资产
const mergedAssetsMap = new Map<string, Asset>();
const assetScriptIds = new Map<string, number[]>();
for (const { scriptId, assets } of validResults) {
for (const asset of assets) {
if (!mergedAssetsMap.has(asset.name)) {
mergedAssetsMap.set(asset.name, asset);
}
const ids = assetScriptIds.get(asset.name) || [];
ids.push(scriptId);
assetScriptIds.set(asset.name, ids);
}
}
// 查询已有资产，避免重复插入
const existingAssets = await u.db("o_assets").where("projectId", projectId).select("id", "name"); const existingAssets = await u.db("o_assets").where("projectId", projectId).select("id", "name");
const existingMap = new Map(existingAssets.map((a) => [a.name!, a.id!])); const existingMap = new Map(existingAssets.map((a) => [a.name!, a.id!]));
// 插入不存在的资产 // 插入新资产（不在已有列表中的）
const toInsert = [...mergedAssetsMap.values()].filter((asset) => !existingMap.has(asset.name)); const toInsert = newAssets.filter((asset) => !existingMap.has(asset.name));
if (toInsert.length) { if (toInsert.length) {
await u.db("o_assets").insert( await u.db("o_assets").insert(
toInsert.map((asset) => ({ toInsert.map((asset) => ({
@ -102,13 +106,24 @@ export default router.post(
const allAssets = await u.db("o_assets").where("projectId", projectId).select("id", "name"); const allAssets = await u.db("o_assets").where("projectId", projectId).select("id", "name");
const nameToId = new Map(allAssets.map((a) => [a.name, a.id])); const nameToId = new Map(allAssets.map((a) => [a.name, a.id]));
// 建立本批各 scriptId 与资产的关联 // 收集所有资产与剧本的关联关系
const batchScriptIds = validResults.map((r) => r.scriptId);
const scriptAssetRows: { scriptId: number; assetId: number }[] = []; const scriptAssetRows: { scriptId: number; assetId: number }[] = [];
for (const [name, sIds] of assetScriptIds) {
const assetId = nameToId.get(name); // 新资产的关联
for (const asset of newAssets) {
const assetId = nameToId.get(asset.name);
if (assetId) { if (assetId) {
for (const sid of sIds) { for (const sid of asset.scriptIds) {
scriptAssetRows.push({ scriptId: sid, assetId });
}
}
}
// 已有资产的关联
for (const ref of existingRefs) {
const assetId = nameToId.get(ref.name);
if (assetId) {
for (const sid of ref.scriptIds) {
scriptAssetRows.push({ scriptId: sid, assetId }); scriptAssetRows.push({ scriptId: sid, assetId });
} }
} }
@ -127,74 +142,111 @@ export default router.post(
}); });
} }
// 按批次并发提取剧本资产，每批完成后统一入库 // 逐组处理（每组最多 groupSize 集剧本一起发给 AI）
await pMapBatch<number, BatchResult>( for (const group of scriptGroups) {
scriptIds, // 过滤有效剧本
async (scriptId: number) => { const validScripts: { id: number; script: o_script }[] = [];
for (const scriptId of group as number[]) {
const script = scriptMap.get(scriptId); const script = scriptMap.get(scriptId);
if (!script) { if (!script) {
errors.push({ scriptId, error: "未找到对应剧本" }); errors.push({ scriptId, error: "未找到对应剧本" });
await u.db("o_script").where("id", scriptId).update({ extractState: -1, errorReason: "未找到对应剧本" }); await u.db("o_script").where("id", scriptId).update({ extractState: -1, errorReason: "未找到对应剧本" });
return null; } else {
validScripts.push({ id: scriptId, script });
} }
}
if (!validScripts.length) continue;
// 用闭包收集当前 scriptId 的资产 // 查询当前项目已有的资产列表，提供给 AI 参考
let collected: Asset[] = []; const existingAssets = await u.db("o_assets").where("projectId", projectId).select("name", "type");
console.log("%c Line:162 🍔 existingAssets", "background:#ea7e5c", existingAssets);
const existingAssetsList = existingAssets.map((a) => `${a.name}(${a.type})`).join("、");
console.log("%c Line:164 🍫 existingAssetsList", "background:#33a5ff", existingAssetsList);
const resultTool = tool({ // 拼接多集剧本内容，每集用分隔标记
description: "返回结果时必须调用这个工具,", const scriptsContent = validScripts
inputSchema: z.object({ .map(({ id, script }) => `===== 【剧本ID: ${id}】${script.name || ""} =====\n${script.content}`)
assetsList: z.array(AssetSchema).describe("剧本所使用资产列表,注意不要包含剧本内容,仅为所使用到的 道具、人物、场景、素材"), .join("\n\n");
}),
execute: async ({ assetsList }) => { const validScriptIds = validScripts.map((v) => v.id);
console.log("[tools] set_flowData script", assetsList);
if (assetsList && assetsList.length) { // 用闭包收集 AI 返回的资产
collected = assetsList; let collectedNew: NewAsset[] = [];
} let collectedExisting: ExistingAssetRef[] = [];
return true;
}, const resultTool = tool({
description: "返回结果时必须调用这个工具",
inputSchema: z.object({
newAssets: z
.array(NewAssetSchema)
.describe("新发现的资产列表（不在已有资产列表中的），需要完整的 prompt、name、desc、type 和使用该资产的 scriptIds"),
existingAssetRefs: z
.array(ExistingAssetRefSchema)
.describe("已有资产的引用列表（在已有资产列表中已存在的），只需给出资产名称和使用该资产的 scriptIds"),
}),
execute: async ({ newAssets, existingAssetRefs }) => {
console.log("[tools] extractAssets result", { newAssets, existingAssetRefs });
if (newAssets?.length) collectedNew = newAssets;
if (existingAssetRefs?.length) collectedExisting = existingAssetRefs;
return "无需回复用户任何内容";
},
});
try {
const data = await u.db("o_prompt").where("type", "scriptAssetExtraction").first("data");
const existingHint = existingAssetsList
? `\n\n【已有资产列表】：${existingAssetsList}\n对于已有资产，如果在剧本中出现，只需在 existingAssetRefs 中给出资产名称和对应的 scriptIds 数组即可，无需重复生成 prompt/desc/type。对于新发现的资产（不在已有列表中），请在 newAssets 中给出完整信息。`
: "";
const output = await intansce.invoke({
messages: [
{
role: "system",
content:
data?.data +
"\n\n提取剧本中涉及的资产（角色、场景、道具），参考技能 script_assets_extract 规范，结果必须通过 resultTool 工具返回。" +
"\n\n注意：本次会同时提供多集剧本，每集剧本以 ===== 【剧本ID: xxx】 ===== 分隔。你需要分析每集剧本使用了哪些资产，并在输出中用 scriptIds 数组标明每个资产在哪些剧本中出现。" +
existingHint,
},
{
role: "user",
content: `请根据以下${validScripts.length}集剧本提取对应的剧本资产（角色、场景、道具）:\n\n${scriptsContent}`,
},
],
tools: { resultTool },
}); });
console.log("%c Line:extractAssets 🍧 output", "background:#f5ce50", output.text);
try { } catch (e: any) {
const data = await u.db("o_prompt").where("type", "scriptAssetExtraction").first("data"); const msg = e?.message || String(e);
await intansce.invoke({ const scriptNames = validScripts.map((v) => v.script.name).join(", ");
messages: [ console.error(`[extractAssets] group=[${validScriptIds.join(",")}] 提取失败:`, msg);
{ for (const { id, script } of validScripts) {
role: "system", errors.push({ scriptId: id, error: (script.name || "") + ":" + u.error(e).message });
content:
data?.data +
"\n\n提取剧本中涉及的资产（角色、场景、道具），参考技能 script_assets_extract 规范，结果必须通过 resultTool 工具返回。",
},
{
role: "user",
content: `请根据以下剧本提取对应的剧本资产（角色、场景、道具、素材片段）:\n\n${script.content}`,
},
],
tools: { resultTool },
});
} catch (e: any) {
const msg = e?.message || String(e);
console.error(`[extractAssets] scriptId=${scriptId} name=${script.name} 提取失败:`, msg);
errors.push({ scriptId, error: script.name + ":" + u.error(e).message });
await u await u
.db("o_script") .db("o_script")
.where("id", scriptId) .where("id", id)
.update({ extractState: -1, errorReason: u.error(e).message }); .update({ extractState: -1, errorReason: u.error(e).message });
return null;
} }
continue;
}
if (!collected.length) { if (!collectedNew.length && !collectedExisting.length) {
errors.push({ scriptId, error: "AI 未返回任何资产" }); for (const { id } of validScripts) {
await u.db("o_script").where("id", scriptId).update({ extractState: -1, errorReason: "AI 未返回任何资产" }); errors.push({ scriptId: id, error: "AI 未返回任何资产" });
return null; await u.db("o_script").where("id", id).update({ extractState: -1, errorReason: "AI 未返回任何资产" });
} }
continue;
}
successCount++; successCount += validScripts.length;
return { scriptId, assets: collected };
}, // 入库
concurrency, await persistGroupResult({
persistBatch, batchScriptIds: validScriptIds,
); newAssets: collectedNew,
existingRefs: collectedExisting,
});
}
return res.send(success("开始提取资产")); return res.send(success("开始提取资产"));
}, },

663
src/socket/resTool copy.ts Normal file
View File

@ -0,0 +1,663 @@
import u from "@/utils";
import { Socket } from "socket.io";
import type {
ChatMessageStatus,
AIMessageContent,
TextContent,
MarkdownContent,
ImageContent,
ThinkingContent,
SearchContent,
SuggestionContent,
ToolCallContent,
ActivityContent,
ReasoningContent,
} from "./chatMessagesData";
type ContentType = AIMessageContent["type"];
class ResTool {
public socket: Socket;
public data: Record<string, any>;
constructor(socket: Socket, data: Record<string, any> = {}) {
this.socket = socket;
this.data = data;
}
// 创建新消息
newMessage(role: "assistant" | "user" | "system" = "assistant", name?: string) {
const messageId = u.uuid();
const datetime = new Date().toISOString();
this.socket.emit("message", {
id: messageId,
role,
name,
status: "pending" as ChatMessageStatus,
datetime,
content: [],
});
return new MessageBuilder(this.socket, messageId, role, name, datetime);
}
// 发送错误消息
sendError(messageId: string, error: string) {
this.socket.emit("message:update", {
id: messageId,
status: "error" as ChatMessageStatus,
ext: { error },
});
}
// 发送完成状态
sendComplete(messageId: string) {
this.socket.emit("message:update", {
id: messageId,
status: "complete" as ChatMessageStatus,
});
}
}
// 消息构建器
class MessageBuilder {
private socket: Socket;
private messageId: string;
private messageRole: "assistant" | "user" | "system";
private messageName?: string;
private messageDatetime: string;
constructor(socket: Socket, messageId: string, role: "assistant" | "user" | "system", name?: string, datetime?: string) {
this.socket = socket;
this.messageId = messageId;
this.messageRole = role;
this.messageName = name;
this.messageDatetime = datetime ?? new Date().toISOString();
}
get id() {
return this.messageId;
}
get role() {
return this.messageRole;
}
get name() {
return this.messageName;
}
get datetime() {
return this.messageDatetime;
}
// 更新消息状态
updateStatus(status: ChatMessageStatus) {
this.socket.emit("message:update", {
id: this.messageId,
status,
});
return this;
}
// 添加文本内容
text(initialText = "") {
const contentId = u.uuid();
const content: TextContent = {
type: "text",
id: contentId,
data: "",
status: "pending",
};
this.socket.emit("content:add", {
messageId: this.messageId,
content,
});
const stream = new AutoThinkingTextStream(this.socket, this.messageId, contentId, this);
if (initialText) {
stream.append(initialText);
}
return stream;
}
// 添加 Markdown 内容
markdown(initialText = "") {
const contentId = u.uuid();
const content: MarkdownContent = {
type: "markdown",
id: contentId,
data: initialText,
status: "pending",
};
this.socket.emit("content:add", {
messageId: this.messageId,
content,
});
return new ContentStream<string>(this.socket, this.messageId, contentId, "markdown");
}
// 添加思考内容
thinking(title = "思考中...") {
const contentId = u.uuid();
const content: ThinkingContent = {
type: "thinking",
id: contentId,
data: { title, text: "" },
status: "pending",
};
this.socket.emit("content:add", {
messageId: this.messageId,
content,
});
return new ThinkingStream(this.socket, this.messageId, contentId);
}
// 添加搜索内容
search(title = "搜索中...") {
const contentId = u.uuid();
const content: SearchContent = {
type: "search",
id: contentId,
data: { title, references: [] },
status: "pending",
};
this.socket.emit("content:add", {
messageId: this.messageId,
content,
});
return new SearchStream(this.socket, this.messageId, contentId);
}
// 添加图片内容
image(data: ImageContent["data"]) {
const contentId = u.uuid();
const content: ImageContent = {
type: "image",
id: contentId,
data,
status: "complete",
};
this.socket.emit("content:add", {
messageId: this.messageId,
content,
});
return this;
}
// 添加建议内容
suggestion(suggestions: SuggestionContent["data"]) {
const contentId = u.uuid();
const content: SuggestionContent = {
type: "suggestion",
id: contentId,
data: suggestions,
status: "complete",
};
this.socket.emit("content:add", {
messageId: this.messageId,
content,
});
return this;
}
// 添加工具调用内容
toolCall(data: ToolCallContent["data"]) {
const contentId = u.uuid();
const content: ToolCallContent = {
type: "toolcall",
id: contentId,
data: { ...data, parentMessageId: this.messageId },
status: "pending",
};
this.socket.emit("content:add", {
messageId: this.messageId,
content,
});
return new ToolCallStream(this.socket, this.messageId, contentId, data.toolCallId);
}
// 添加活动内容
activity<T = Record<string, any>>(activityType: string, content: T) {
const contentId = u.uuid();
const activityContent: ActivityContent<T> = {
type: "activity",
id: contentId,
data: {
activityType,
messageId: this.messageId,
content,
},
status: "complete",
};
this.socket.emit("content:add", {
messageId: this.messageId,
content: activityContent,
});
return this;
}
// 添加推理内容
reasoning() {
const contentId = u.uuid();
const content: ReasoningContent = {
type: "reasoning",
id: contentId,
data: [],
status: "pending",
};
this.socket.emit("content:add", {
messageId: this.messageId,
content,
});
return new ReasoningBuilder(this.socket, this.messageId, contentId);
}
// 完成消息
complete() {
this.socket.emit("message:update", {
id: this.messageId,
status: "complete" as ChatMessageStatus,
});
}
// 停止消息
stop() {
this.socket.emit("message:update", {
id: this.messageId,
status: "stop" as ChatMessageStatus,
});
}
// 错误
error(errorMsg?: string) {
this.socket.emit("message:update", {
id: this.messageId,
status: "error" as ChatMessageStatus,
ext: errorMsg ? { error: errorMsg } : undefined,
});
}
}
// 内容流基类
class ContentStream<T> {
protected socket: Socket;
protected messageId: string;
protected contentId: string;
protected contentType: ContentType;
constructor(socket: Socket, messageId: string, contentId: string, contentType: ContentType) {
this.socket = socket;
this.messageId = messageId;
this.contentId = contentId;
this.contentType = contentType;
}
get id() {
return this.contentId;
}
// 流式追加数据
append(chunk: string) {
this.socket.emit("content:update", {
messageId: this.messageId,
contentId: this.contentId,
type: this.contentType,
data: chunk,
strategy: "append",
status: "streaming",
});
return this;
}
// 合并/替换数据
merge(data: T) {
this.socket.emit("content:update", {
messageId: this.messageId,
contentId: this.contentId,
type: this.contentType,
data,
strategy: "merge",
status: "streaming",
});
return this;
}
// 完成内容
complete(finalData?: T) {
this.socket.emit("content:update", {
messageId: this.messageId,
contentId: this.contentId,
type: this.contentType,
data: finalData,
status: "complete",
});
return this;
}
// 错误
error() {
this.socket.emit("content:update", {
messageId: this.messageId,
contentId: this.contentId,
status: "error",
});
return this;
}
}
// 思考内容流
class ThinkingStream extends ContentStream<ThinkingContent["data"]> {
constructor(socket: Socket, messageId: string, contentId: string) {
super(socket, messageId, contentId, "thinking");
}
// 追加思考文本
appendText(chunk: string) {
this.socket.emit("content:update", {
messageId: this.messageId,
contentId: this.contentId,
type: "thinking",
data: { text: chunk },
strategy: "append",
status: "streaming",
});
return this;
}
// 更新标题
updateTitle(title: string) {
this.socket.emit("content:update", {
messageId: this.messageId,
contentId: this.contentId,
type: "thinking",
data: { title },
strategy: "merge",
status: "streaming",
});
return this;
}
}
// 文本内容流：自动把 <think>...</think> 转为 thinking 内容
class AutoThinkingTextStream extends ContentStream<string> {
private static readonly OPEN_TAG = "<think>";
private static readonly CLOSE_TAG = "</think>";
private readonly messageBuilder: MessageBuilder;
private pending = "";
private inThinking = false;
private thinkingStream: ThinkingStream | null = null;
constructor(socket: Socket, messageId: string, contentId: string, messageBuilder: MessageBuilder) {
super(socket, messageId, contentId, "text");
this.messageBuilder = messageBuilder;
}
override append(chunk: string) {
if (!chunk) return this;
let rest = this.pending + chunk;
this.pending = "";
while (rest.length > 0) {
if (!this.inThinking) {
const openIndex = rest.indexOf(AutoThinkingTextStream.OPEN_TAG);
if (openIndex < 0) {
const keepLen = AutoThinkingTextStream.OPEN_TAG.length - 1;
const flushLen = Math.max(0, rest.length - keepLen);
if (flushLen > 0) {
this.appendText(rest.slice(0, flushLen));
rest = rest.slice(flushLen);
}
this.pending = rest;
break;
}
this.appendText(rest.slice(0, openIndex));
this.inThinking = true;
this.ensureThinkingStream();
rest = rest.slice(openIndex + AutoThinkingTextStream.OPEN_TAG.length);
continue;
}
const closeIndex = rest.indexOf(AutoThinkingTextStream.CLOSE_TAG);
if (closeIndex < 0) {
const keepLen = AutoThinkingTextStream.CLOSE_TAG.length - 1;
const flushLen = Math.max(0, rest.length - keepLen);
if (flushLen > 0) {
this.appendThinking(rest.slice(0, flushLen));
rest = rest.slice(flushLen);
}
this.pending = rest;
break;
}
this.appendThinking(rest.slice(0, closeIndex));
this.finishThinking();
rest = rest.slice(closeIndex + AutoThinkingTextStream.CLOSE_TAG.length);
}
return this;
}
override complete(finalData?: string) {
if (finalData) {
this.append(finalData);
}
if (this.pending) {
if (this.inThinking) {
this.appendThinking(this.pending);
} else {
this.appendText(this.pending);
}
this.pending = "";
}
this.finishThinking();
super.complete();
return this;
}
override error() {
if (this.thinkingStream) {
this.thinkingStream.error();
this.thinkingStream = null;
}
this.pending = "";
this.inThinking = false;
return super.error();
}
private appendText(text: string) {
if (!text) return;
super.append(text);
}
private appendThinking(text: string) {
if (!text) return;
this.ensureThinkingStream().appendText(text);
}
private ensureThinkingStream() {
if (!this.thinkingStream) {
this.thinkingStream = this.messageBuilder.thinking("思考中...");
}
return this.thinkingStream;
}
private finishThinking() {
if (this.thinkingStream) {
this.thinkingStream.complete();
this.thinkingStream = null;
}
this.inThinking = false;
}
}
// 搜索内容流
class SearchStream extends ContentStream<SearchContent["data"]> {
constructor(socket: Socket, messageId: string, contentId: string) {
super(socket, messageId, contentId, "search");
}
// 添加引用
addReference(ref: Exclude<SearchContent["data"]["references"], undefined>[0]) {
this.socket.emit("content:update", {
messageId: this.messageId,
contentId: this.contentId,
type: "search",
data: { references: [ref] },
strategy: "append",
status: "streaming",
});
return this;
}
// 批量添加引用
addReferences(refs: SearchContent["data"]["references"]) {
this.socket.emit("content:update", {
messageId: this.messageId,
contentId: this.contentId,
type: "search",
data: { references: refs },
strategy: "append",
status: "streaming",
});
return this;
}
// 更新标题
updateTitle(title: string) {
this.socket.emit("content:update", {
messageId: this.messageId,
contentId: this.contentId,
type: "search",
data: { title },
strategy: "merge",
status: "streaming",
});
return this;
}
}
// 工具调用流
class ToolCallStream extends ContentStream<ToolCallContent["data"]> {
private toolCallId: string;
constructor(socket: Socket, messageId: string, contentId: string, toolCallId: string) {
super(socket, messageId, contentId, "toolcall");
this.toolCallId = toolCallId;
}
// 追加参数块
appendArgs(chunk: string) {
this.socket.emit("content:update", {
messageId: this.messageId,
contentId: this.contentId,
type: "toolcall",
data: { toolCallId: this.toolCallId, args: chunk },
strategy: "append",
status: "streaming",
});
return this;
}
// 追加结果块
appendResult(chunk: string) {
this.socket.emit("content:update", {
messageId: this.messageId,
contentId: this.contentId,
type: "toolcall",
data: { toolCallId: this.toolCallId, chunk },
strategy: "append",
status: "streaming",
});
return this;
}
// 设置完整结果
setResult(result: string) {
this.socket.emit("content:update", {
messageId: this.messageId,
contentId: this.contentId,
type: "toolcall",
data: { toolCallId: this.toolCallId, result },
strategy: "merge",
status: "complete",
});
return this;
}
// 更新事件类型
updateEventType(eventType: ToolCallContent["data"]["eventType"]) {
this.socket.emit("content:update", {
messageId: this.messageId,
contentId: this.contentId,
type: "toolcall",
data: { toolCallId: this.toolCallId, eventType },
strategy: "merge",
status: "streaming",
});
return this;
}
}
// 推理构建器
class ReasoningBuilder {
private socket: Socket;
private messageId: string;
private contentId: string;
constructor(socket: Socket, messageId: string, contentId: string) {
this.socket = socket;
this.messageId = messageId;
this.contentId = contentId;
}
// 添加子内容
addContent(content: AIMessageContent) {
this.socket.emit("content:update", {
messageId: this.messageId,
contentId: this.contentId,
type: "reasoning",
data: [content],
strategy: "append",
status: "streaming",
});
return this;
}
// 完成推理
complete() {
this.socket.emit("content:update", {
messageId: this.messageId,
contentId: this.contentId,
type: "reasoning",
status: "complete",
});
return this;
}
}
export default ResTool;
export { MessageBuilder, ContentStream, ThinkingStream, SearchStream, ToolCallStream, ReasoningBuilder };

156
src/socket/resTool.ts
View File

@ -107,7 +107,7 @@ class MessageBuilder {
const content: TextContent = { const content: TextContent = {
type: "text", type: "text",
id: contentId, id: contentId,
data: initialText, data: "",
status: "pending", status: "pending",
}; };
@ -116,7 +116,11 @@ class MessageBuilder {
content, content,
}); });
return new ContentStream<string>(this.socket, this.messageId, contentId, "text"); const stream = new AutoThinkingTextStream(this.socket, this.messageId, contentId, this);
if (initialText) {
stream.append(initialText);
}
return stream;
} }
// 添加 Markdown 内容 // 添加 Markdown 内容
@ -393,6 +397,154 @@ class ThinkingStream extends ContentStream<ThinkingContent["data"]> {
} }
} }
// 文本内容流：自动把 <think>...</think> 转为 thinking 内容
class AutoThinkingTextStream extends ContentStream<string> {
private static readonly OPEN_TAG = "<think>";
private static readonly CLOSE_TAG = "</think>";
private readonly messageBuilder: MessageBuilder;
private pending = "";
private inThinking = false;
private thinkingStream: ThinkingStream | null = null;
private thinkingBuffer = "";
private thinkingStartTime: number = 0;
constructor(socket: Socket, messageId: string, contentId: string, messageBuilder: MessageBuilder) {
super(socket, messageId, contentId, "text");
this.messageBuilder = messageBuilder;
}
/**
* 检查 str 的尾部是否是 tag 的某个非空真前缀。
* 返回需要保留的尾部字符数（0 表示不需要缓冲）。
*/
private static tailPrefixLen(str: string, tag: string): number {
const maxCheck = Math.min(str.length, tag.length - 1);
for (let len = maxCheck; len >= 1; len--) {
if (str.endsWith(tag.slice(0, len))) {
return len;
}
}
return 0;
}
override append(chunk: string) {
if (!chunk) return this;
let rest = this.pending + chunk;
this.pending = "";
while (rest.length > 0) {
if (!this.inThinking) {
// 寻找 <think> 开始标签
const openIndex = rest.indexOf(AutoThinkingTextStream.OPEN_TAG);
if (openIndex >= 0) {
this.flushText(rest.slice(0, openIndex));
this.inThinking = true;
this.thinkingStartTime = Date.now();
this.thinkingBuffer = "";
this.ensureThinkingStream();
rest = rest.slice(openIndex + AutoThinkingTextStream.OPEN_TAG.length);
continue;
}
// 检查尾部是否可能是标签的部分前缀
const keep = AutoThinkingTextStream.tailPrefixLen(rest, AutoThinkingTextStream.OPEN_TAG);
if (keep > 0) {
this.flushText(rest.slice(0, rest.length - keep));
this.pending = rest.slice(rest.length - keep);
} else {
this.flushText(rest);
}
break;
} else {
// 寻找 </think> 结束标签
const closeIndex = rest.indexOf(AutoThinkingTextStream.CLOSE_TAG);
if (closeIndex >= 0) {
this.flushThinking(rest.slice(0, closeIndex));
this.finishThinking();
rest = rest.slice(closeIndex + AutoThinkingTextStream.CLOSE_TAG.length);
continue;
}
// 检查尾部是否可能是标签的部分前缀
const keep = AutoThinkingTextStream.tailPrefixLen(rest, AutoThinkingTextStream.CLOSE_TAG);
if (keep > 0) {
this.flushThinking(rest.slice(0, rest.length - keep));
this.pending = rest.slice(rest.length - keep);
} else {
this.flushThinking(rest);
}
break;
}
}
return this;
}
override complete(finalData?: string) {
if (finalData) {
this.append(finalData);
}
if (this.pending) {
if (this.inThinking) {
this.flushThinking(this.pending);
} else {
this.flushText(this.pending);
}
this.pending = "";
}
this.finishThinking();
super.complete();
return this;
}
override error() {
if (this.thinkingStream) {
this.thinkingStream.error();
this.thinkingStream = null;
}
this.pending = "";
this.thinkingBuffer = "";
this.inThinking = false;
return super.error();
}
/** 输出普通文本 */
private flushText(text: string) {
if (!text) return;
super.append(text);
}
/** 输出思考文本：累积完整内容，用 merge 策略发送，避免前端 append 丢失 */
private flushThinking(text: string) {
if (!text) return;
this.thinkingBuffer += text;
this.ensureThinkingStream().merge({ title: "思考中...", text: this.thinkingBuffer });
}
private ensureThinkingStream() {
if (!this.thinkingStream) {
this.thinkingStartTime = Date.now();
this.thinkingStream = this.messageBuilder.thinking("思考中...");
}
return this.thinkingStream;
}
private finishThinking() {
if (this.thinkingStream) {
const elapsed = ((Date.now() - this.thinkingStartTime) / 1000).toFixed(1);
this.thinkingStream.updateTitle(`思考完毕（${elapsed}秒）`);
this.thinkingStream.complete({ title: `思考完毕（${elapsed}秒）`, text: this.thinkingBuffer });
this.thinkingStream = null;
this.thinkingBuffer = "";
}
this.inThinking = false;
}
}
// 搜索内容流 // 搜索内容流
class SearchStream extends ContentStream<SearchContent["data"]> { class SearchStream extends ContentStream<SearchContent["data"]> {
constructor(socket: Socket, messageId: string, contentId: string) { constructor(socket: Socket, messageId: string, contentId: string) {

62
src/types/database.d.ts vendored
View File

@ -1,62 +1,13 @@
// @db-hash ea0c51ccb8c93a2f019139db9621721e // @db-hash 93b2462070c45c2b449e9a18c4e88763
//该文件由脚本自动生成，请勿手动修改 //该文件由脚本自动生成，请勿手动修改
export interface _o_project_old_20260330 {
'artStyle'?: string | null;
'createTime'?: number | null;
'id'?: number | null;
'intro'?: string | null;
'name'?: string | null;
'projectType'?: string | null;
'type'?: string | null;
'userId'?: number | null;
'videoRatio'?: string | null;
}
export interface _o_storyboard_old_20260325 {
'camera'?: string | null;
'createTime'?: number | null;
'description'?: string | null;
'duration'?: string | null;
'filePath'?: string | null;
'frameMode'?: string | null;
'id'?: number;
'lines'?: string | null;
'mode'?: string | null;
'model'?: string | null;
'prompt'?: string | null;
'reason'?: string | null;
'resolution'?: string | null;
'scriptId'?: number | null;
'sound'?: string | null;
'state'?: string | null;
'title'?: string | null;
}
export interface _o_storyboard_old_20260330 {
'camera'?: string | null;
'createTime'?: number | null;
'description'?: string | null;
'duration'?: string | null;
'filePath'?: string | null;
'frameMode'?: string | null;
'id'?: number;
'index'?: string | null;
'lines'?: string | null;
'mode'?: string | null;
'model'?: string | null;
'prompt'?: string | null;
'reason'?: string | null;
'resolution'?: string | null;
'scriptId'?: number | null;
'sound'?: string | null;
'state'?: string | null;
'title'?: string | null;
}
export interface memories { export interface memories {
'content': string; 'content': string;
'createTime': number; 'createTime': number;
'embedding'?: string | null; 'embedding'?: string | null;
'id'?: string; 'id'?: string;
'isolationKey': string; 'isolationKey': string;
'name'?: string | null;
'relatedMessageIds'?: string | null; 'relatedMessageIds'?: string | null;
'role'?: string | null; 'role'?: string | null;
'summarized'?: number | null; 'summarized'?: number | null;
@ -96,6 +47,7 @@ export interface o_assets {
'name'?: string | null; 'name'?: string | null;
'projectId'?: number | null; 'projectId'?: number | null;
'prompt'?: string | null; 'prompt'?: string | null;
'promptState'?: string | null;
'remark'?: string | null; 'remark'?: string | null;
'scriptId'?: number | null; 'scriptId'?: number | null;
'startTime'?: number | null; 'startTime'?: number | null;
@ -177,6 +129,8 @@ export interface o_prompt {
export interface o_script { export interface o_script {
'content'?: string | null; 'content'?: string | null;
'createTime'?: number | null; 'createTime'?: number | null;
'errorReason'?: string | null;
'extractState'?: number | null;
'id'?: number; 'id'?: number;
'name'?: string | null; 'name'?: string | null;
'projectId'?: number | null; 'projectId'?: number | null;
@ -213,7 +167,7 @@ export interface o_storyboard {
'filePath'?: string | null; 'filePath'?: string | null;
'frameMode'?: string | null; 'frameMode'?: string | null;
'id'?: number; 'id'?: number;
'index'?: string | null; 'index'?: number | null;
'lines'?: string | null; 'lines'?: string | null;
'mode'?: string | null; 'mode'?: string | null;
'model'?: string | null; 'model'?: string | null;
@ -224,7 +178,6 @@ export interface o_storyboard {
'sound'?: string | null; 'sound'?: string | null;
'state'?: string | null; 'state'?: string | null;
'title'?: string | null; 'title'?: string | null;
'videoPrompt'?: string | null;
} }
export interface o_tasks { export interface o_tasks {
'describe'?: string | null; 'describe'?: string | null;
@ -279,9 +232,6 @@ export interface o_videoConfig {
} }
export interface DB { export interface DB {
"_o_project_old_20260330": _o_project_old_20260330;
"_o_storyboard_old_20260325": _o_storyboard_old_20260325;
"_o_storyboard_old_20260330": _o_storyboard_old_20260330;
"memories": memories; "memories": memories;
"o_agentDeploy": o_agentDeploy; "o_agentDeploy": o_agentDeploy;
"o_agentWorkData": o_agentWorkData; "o_agentWorkData": o_agentWorkData;

61
src/utils/agent/skillsTools.ts
View File

@ -4,6 +4,7 @@ import path from "path";
import isPathInside from "is-path-inside"; import isPathInside from "is-path-inside";
import getPath from "@/utils/getPath"; import getPath from "@/utils/getPath";
import * as fs from "fs"; import * as fs from "fs";
import fg from "fast-glob";
type SkillAttribution = type SkillAttribution =
//剧本Agent //剧本Agent
@ -18,13 +19,13 @@ type SkillAttribution =
| "production_agent_supervision"; | "production_agent_supervision";
interface SkillInput { interface SkillInput {
mainSkill: SkillAttribution; mainSkill: SkillAttribution[];
workspace?: string[]; workspace?: string[];
attachedSkills?: string[]; attachedSkills?: string[];
} }
interface SkillPaths { interface SkillPaths {
mainSkill: string; mainSkill: { path: string; name: string; description: string }[];
secondarySkills: string[]; secondarySkills: string[];
tertiarySkills: string[]; tertiarySkills: string[];
} }
@ -40,7 +41,7 @@ function ensureNonEmptyBody(body: string, fallback: string): string {
// ==================== 解析 SKILL.md ==================== // ==================== 解析 SKILL.md ====================
function parseFrontmatter(content: string): { name: string; description: string } { export function parseFrontmatter(content: string): { name: string; description: string } {
const match = content.match(/^\uFEFF?---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/); const match = content.match(/^\uFEFF?---[ \t]*\r?\n([\s\S]*?)\r?\n---[ \t]*(?:\r?\n|$)/);
if (!match?.[1]) { if (!match?.[1]) {
throw new Error(`技能文件缺少有效的 frontmatter，确保以 --- 包裹并包含 name 和 description 字段。${content}`); throw new Error(`技能文件缺少有效的 frontmatter，确保以 --- 包裹并包含 name 和 description 字段。${content}`);
@ -121,9 +122,16 @@ export async function useSkill(input: SkillInput) {
const { mainSkill, workspace = [], attachedSkills = [] } = input; const { mainSkill, workspace = [], attachedSkills = [] } = input;
const rootDir = getPath("skills"); const rootDir = getPath("skills");
const normalizedRootDir = path.resolve(rootDir); const normalizedRootDir = path.resolve(rootDir);
const mainPath = path.join(rootDir, mainSkill + ".md");
if (!fs.existsSync(mainPath)) throw new Error(`主技能文件不存在: ${mainPath}`); const mainSkills: { path: string; name: string; description: string }[] = [];
if (!isPathInside(mainPath, normalizedRootDir)) throw new Error(`技能名称无效：检测到路径穿越。${mainPath}`); for (const skill of mainSkill) {
const skillPath = path.join(rootDir, skill + ".md");
if (!fs.existsSync(skillPath)) throw new Error(`主技能文件不存在: ${skillPath}`);
if (!isPathInside(skillPath, normalizedRootDir)) throw new Error(`技能名称无效：检测到路径穿越。${skillPath}`);
const content = await fs.promises.readFile(skillPath, "utf-8");
const parsed = parseFrontmatter(content);
mainSkills.push({ path: skillPath, ...parsed });
}
const resolveSafeSkillDir = (dir: string): string | null => { const resolveSafeSkillDir = (dir: string): string | null => {
const resolvedDir = path.resolve(normalizedRootDir, dir); const resolvedDir = path.resolve(normalizedRootDir, dir);
@ -147,50 +155,52 @@ export async function useSkill(input: SkillInput) {
}); });
const skillPaths: SkillPaths = { const skillPaths: SkillPaths = {
mainSkill: mainPath, mainSkill: mainSkills,
secondarySkills: collectMdFiles(workspace, false), secondarySkills: collectMdFiles(workspace, false),
tertiarySkills: collectMdFiles(attachedSkills, true), tertiarySkills: collectMdFiles(attachedSkills, true),
}; };
const content = await fs.promises.readFile(mainPath, "utf-8"); return { prompt: buildSkillPrompt(mainSkills), tools: createSkillTools(mainSkills, skillPaths), skillPaths };
const skill = parseFrontmatter(content);
return { prompt: buildPrompt(skill), tools: createSkillTools(skill, skillPaths), skillPaths };
} }
function buildPrompt(skill: { name: string; description: string }): string { export function buildSkillPrompt(skills: { name: string; description: string }[]): string {
const skillEntries = skills
.map((s) => ` <skill>\n <name>${s.name}</name>\n <description>${s.description}</description>\n </skill>`)
.join("\n");
return `## Skills return `## Skills
以下技能提供了专业任务的专用指令。 以下技能提供了专业任务的专用指令。
当任务与某个技能的描述匹配时，调用 activate_skill 工具并传入技能名称来加载完整指令。 当任务与某个技能的描述匹配时，调用 activate_skill 工具并传入技能名称来加载完整指令。
加载后遵循技能指令执行任务，需要时调用 read_skill_file 读取资源文件内容。 加载后遵循技能指令执行任务，需要时调用 read_skill_file 读取资源文件内容。
<available_skills> <available_skills>
<skill> ${skillEntries}
<name>${skill.name}</name>
<description>${skill.description}</description>
</skill>
</available_skills>`; </available_skills>`;
} }
function createSkillTools(skill: { name: string; description: string }, skillPaths: SkillPaths) { export function createSkillTools(skills: { name: string; description: string }[], skillPaths: SkillPaths) {
const activated = new Set<string>(); // 已激活技能集合，防止重复加载 const activated = new Set<string>(); // 已激活技能集合，防止重复加载
const skillsRootDir = path.resolve(getPath("skills")); const skillsRootDir = path.resolve(getPath("skills"));
const skillNames = skills.map((s) => s.name);
const skillMap = new Map(skillPaths.mainSkill.map((s) => [s.name, s]));
return { return {
activate_skill: tool({ activate_skill: tool({
description: `激活一个技能，加载其完整指令和捆绑资源列表到上下文。可用技能：${skill.name}`, description: `激活一个技能，加载其完整指令和捆绑资源列表到上下文。可用技能：${skillNames.join(", ")}`,
inputSchema: z.object({ inputSchema: z.object({
name: z.enum([skill.name] as [string, ...string[]]).describe("要激活的技能名称"), name: z.enum(skillNames as [string, ...string[]]).describe("要激活的技能名称"),
}), }),
execute: async ({ name }) => { execute: async ({ name }) => {
if (activated.has(name)) { if (activated.has(name)) {
console.log(`⚡[主技能] ℹ️ 技能 "${name}" 已激活，跳过重复注入`); console.log(`⚡[主技能] ℹ️ 技能 "${name}" 已激活，跳过重复注入`);
return { alreadyActive: true, message: `技能 "${name}" 已激活，无需重复加载` }; return { alreadyActive: true, message: `技能 "${name}" 已激活，无需重复加载` };
} }
const matched = skillMap.get(name);
if (!matched) return { error: `未找到技能 "${name}"` };
let raw = ""; let raw = "";
try { try {
raw = await fs.promises.readFile(skillPaths.mainSkill, "utf-8"); raw = await fs.promises.readFile(matched.path, "utf-8");
console.log(`⚡[主技能] ✓ 已读取主技能文件： ${skillPaths.mainSkill}（${raw.length} 字符）`); console.log(`⚡[主技能] ✓ 已读取主技能文件： ${matched.path}（${raw.length} 字符）`);
} catch (error) { } catch (error) {
console.log(`⚡[主技能] ✗ 读取失败：未找到文件 "${skillPaths.mainSkill}"`); console.log(`⚡[主技能] ✗ 读取失败：未找到文件 "${matched.path}"`);
} }
activated.add(name); activated.add(name);
console.log(`⚡[主技能] ✓ 技能 "${name}" 已激活`); console.log(`⚡[主技能] ✓ 技能 "${name}" 已激活`);
@ -253,3 +263,12 @@ function createSkillTools(skill: { name: string; description: string }, skillPat
}), }),
}; };
} }
export async function scanSkills(folderPath: string) {
const unixPath = toUnixPath(folderPath);
const entries = await fg(unixPath, {
onlyFiles: true,
absolute: true,
});
return entries;
}

1
src/utils/ai.ts
View File

@ -116,7 +116,6 @@ class AiImage {
return withTaskRecord(this.key, input.taskClass, input.describe, input.relatedObjects, input.projectId, async (modelName) => { return withTaskRecord(this.key, input.taskClass, input.describe, input.relatedObjects, input.projectId, async (modelName) => {
const fn = await getVendorTemplateFn("imageRequest", modelName); const fn = await getVendorTemplateFn("imageRequest", modelName);
this.result = await fn(input); this.result = await fn(input);
console.log("%c Line:119 🌽 this.result", "background:#ed9ec7", this.result);
if (this.result.startsWith("http")) this.result = await urlToBase64(this.result); if (this.result.startsWith("http")) this.result = await urlToBase64(this.result);
return this; return this;
}); });

11
src/utils/cleanNovel.ts
View File

@ -1,7 +1,7 @@
import { EventEmitter } from "events"; import { EventEmitter } from "events";
import { o_novel } from "@/types/database"; import { o_novel } from "@/types/database";
import { useSkill } from "@/utils/agent/skillsTools";
import u from "@/utils"; import u from "@/utils";
import { stripThink } from "@/utils/stripThink";
export interface EventType { export interface EventType {
id: number; id: number;
event: string; event: string;
@ -24,11 +24,11 @@ class CleanNovel {
this.concurrency = concurrency; this.concurrency = concurrency;
} }
private async processChapter(novel: o_novel, intansce: ReturnType<typeof u.Ai.Text>): Promise<EventType | null> { private async processChapter(novel: o_novel): Promise<EventType | null> {
try { try {
const prompt = await u.getPrompts("event"); const prompt = await u.getPrompts("event");
const data = await u.db("o_prompt").where("type", "eventExtraction").first("data"); const data = await u.db("o_prompt").where("type", "eventExtraction").first("data");
const resData = await intansce.invoke({ const resData = await u.Ai.Text("universalAi").invoke({
system: data ? JSON.stringify(data.data) : (prompt as string), system: data ? JSON.stringify(data.data) : (prompt as string),
messages: [ messages: [
{ {
@ -45,7 +45,7 @@ class CleanNovel {
}, },
], ],
}); });
const preData = resData.text; const preData = stripThink(resData.text);
this.emitter.emit("item", { id: novel.id, event: preData }); this.emitter.emit("item", { id: novel.id, event: preData });
return { id: novel.id!, event: preData }; return { id: novel.id!, event: preData };
} catch (e) { } catch (e) {
@ -56,7 +56,6 @@ class CleanNovel {
async start(allChapters: o_novel[], projectId: number): Promise<EventType[]> { async start(allChapters: o_novel[], projectId: number): Promise<EventType[]> {
const totalEvent: EventType[] = []; const totalEvent: EventType[] = [];
const intansce = u.Ai.Text("universalAi");
// 并发控制：通过信号量限制同时执行的任务数 // 并发控制：通过信号量限制同时执行的任务数
let running = 0; let running = 0;
@ -68,7 +67,7 @@ class CleanNovel {
const novel = allChapters[index++]; const novel = allChapters[index++];
running++; running++;
return this.processChapter(novel, intansce).then((result) => { return this.processChapter(novel).then((result) => {
if (result) totalEvent.push(result); if (result) totalEvent.push(result);
running--; running--;
return runNext(); return runNext();

2
src/utils/vm.ts
View File

@ -9,6 +9,7 @@ import { createGoogleGenerativeAI } from "@ai-sdk/google";
import { createAnthropic } from "@ai-sdk/anthropic"; import { createAnthropic } from "@ai-sdk/anthropic";
import { createOpenAICompatible } from "@ai-sdk/openai-compatible"; import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
import { createXai } from "@ai-sdk/xai"; import { createXai } from "@ai-sdk/xai";
import { createMinimax } from "vercel-minimax-ai-provider";
import FormData from "form-data"; import FormData from "form-data";
export default function runCode(code: string) { export default function runCode(code: string) {
@ -24,6 +25,7 @@ export default function runCode(code: string) {
createAnthropic, createAnthropic,
createOpenAICompatible, createOpenAICompatible,
createXai, createXai,
createMinimax,
createGoogleGenerativeAI, createGoogleGenerativeAI,
zipImage, zipImage,
zipImageResolution, zipImageResolution,

49
yarn.lock
View File

@ -7,6 +7,14 @@
resolved "https://registry.npmmirror.com/7zip-bin/-/7zip-bin-5.2.0.tgz#7a03314684dd6572b7dfa89e68ce31d60286854d" resolved "https://registry.npmmirror.com/7zip-bin/-/7zip-bin-5.2.0.tgz#7a03314684dd6572b7dfa89e68ce31d60286854d"
integrity sha512-ukTPVhqG4jNzMro2qA9HSCSSVJN3aN7tlb+hfqYCt3ER0yWroeA2VR38MNrOHLQ/cVj+DaIMad0kFCtWWowh/A== integrity sha512-ukTPVhqG4jNzMro2qA9HSCSSVJN3aN7tlb+hfqYCt3ER0yWroeA2VR38MNrOHLQ/cVj+DaIMad0kFCtWWowh/A==
"@ai-sdk/anthropic@3.0.6":
version "3.0.6"
resolved "https://registry.npmmirror.com/@ai-sdk/anthropic/-/anthropic-3.0.6.tgz#155909d705efe3f82c72153b68291c3a2557397c"
integrity sha512-Ns5OOPHXbODzitvqCySnAFZCAm9ldpx+fdbC0c/f9QwX5b4MQtQJIQ0xZyKm+tB/ynBoeV6zhtyWDXjYeVEWIw==
dependencies:
"@ai-sdk/provider" "3.0.1"
"@ai-sdk/provider-utils" "4.0.3"
"@ai-sdk/anthropic@^3.0.35": "@ai-sdk/anthropic@^3.0.35":
version "3.0.64" version "3.0.64"
resolved "https://registry.npmmirror.com/@ai-sdk/anthropic/-/anthropic-3.0.64.tgz#755e310e74a4ab364108df39e491d7fa9c5f6bd3" resolved "https://registry.npmmirror.com/@ai-sdk/anthropic/-/anthropic-3.0.64.tgz#755e310e74a4ab364108df39e491d7fa9c5f6bd3"
@ -74,6 +82,24 @@
"@standard-schema/spec" "^1.1.0" "@standard-schema/spec" "^1.1.0"
eventsource-parser "^3.0.6" eventsource-parser "^3.0.6"
"@ai-sdk/provider-utils@4.0.3":
version "4.0.3"
resolved "https://registry.npmmirror.com/@ai-sdk/provider-utils/-/provider-utils-4.0.3.tgz#0487848465b016de37e0b216184cbbd161d014e6"
integrity sha512-Vo2p61dDld8Dy/O66zKQpE4nqHojiEEYEjZcSbICjE7h8Z6QmHzBfd+ss/paIDdyXyS0yHmC1GoRYYKo89cqZQ==
dependencies:
"@ai-sdk/provider" "3.0.1"
"@standard-schema/spec" "^1.1.0"
eventsource-parser "^3.0.6"
"@ai-sdk/provider-utils@4.0.4":
version "4.0.4"
resolved "https://registry.npmmirror.com/@ai-sdk/provider-utils/-/provider-utils-4.0.4.tgz#b2f5af446f152be64124725677a900be615c8766"
integrity sha512-VxhX0B/dWGbpNHxrKCWUAJKXIXV015J4e7qYjdIU9lLWeptk0KMLGcqkB4wFxff5Njqur8dt8wRi1MN9lZtDqg==
dependencies:
"@ai-sdk/provider" "3.0.2"
"@standard-schema/spec" "^1.1.0"
eventsource-parser "^3.0.6"
"@ai-sdk/provider-utils@^3.0.0": "@ai-sdk/provider-utils@^3.0.0":
version "3.0.22" version "3.0.22"
resolved "https://registry.npmmirror.com/@ai-sdk/provider-utils/-/provider-utils-3.0.22.tgz#fc9824f5a5c290a95c14888de130b02e52020060" resolved "https://registry.npmmirror.com/@ai-sdk/provider-utils/-/provider-utils-3.0.22.tgz#fc9824f5a5c290a95c14888de130b02e52020060"
@ -90,6 +116,20 @@
dependencies: dependencies:
json-schema "^0.4.0" json-schema "^0.4.0"
"@ai-sdk/provider@3.0.1":
version "3.0.1"
resolved "https://registry.npmmirror.com/@ai-sdk/provider/-/provider-3.0.1.tgz#5bd8809910fc401f024c7784a77eb116171d0296"
integrity sha512-2lR4w7mr9XrydzxBSjir4N6YMGdXD+Np1Sh0RXABh7tWdNFFwIeRI1Q+SaYZMbfL8Pg8RRLcrxQm51yxTLhokg==
dependencies:
json-schema "^0.4.0"
"@ai-sdk/provider@3.0.2":
version "3.0.2"
resolved "https://registry.npmmirror.com/@ai-sdk/provider/-/provider-3.0.2.tgz#d4ee0b53e2c0b2a1b3e36f7356844fda53e63487"
integrity sha512-HrEmNt/BH/hkQ7zpi2o6N3k1ZR1QTb7z85WYhYygiTxOQuaml4CMtHCWRbric5WPU+RNsYI7r1EpyVQMKO1pYw==
dependencies:
json-schema "^0.4.0"
"@ai-sdk/provider@3.0.7": "@ai-sdk/provider@3.0.7":
version "3.0.7" version "3.0.7"
resolved "https://registry.npmmirror.com/@ai-sdk/provider/-/provider-3.0.7.tgz#470bb8f9e46ec9d8d62b07b4c1f5737b991ebe83" resolved "https://registry.npmmirror.com/@ai-sdk/provider/-/provider-3.0.7.tgz#470bb8f9e46ec9d8d62b07b4c1f5737b991ebe83"
@ -5302,6 +5342,15 @@ vary@^1, vary@^1.1.2:
resolved "https://registry.npmmirror.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" resolved "https://registry.npmmirror.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc"
integrity sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg== integrity sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==
vercel-minimax-ai-provider@^0.0.2:
version "0.0.2"
resolved "https://registry.npmmirror.com/vercel-minimax-ai-provider/-/vercel-minimax-ai-provider-0.0.2.tgz#84192a8a86b756b23904ad9c5127c9132817b987"
integrity sha512-h9QzLL7RBmOreqWfr2fcoFVNTJgusENJVagVm8vAi+DBfd+1t+sVJZ/hAhKrtuCKCrm33BlOSWVdJehQFju5jQ==
dependencies:
"@ai-sdk/anthropic" "3.0.6"
"@ai-sdk/provider" "3.0.2"
"@ai-sdk/provider-utils" "4.0.4"
verror@^1.10.0: verror@^1.10.0:
version "1.10.1" version "1.10.1"
resolved "https://registry.npmmirror.com/verror/-/verror-1.10.1.tgz#4bf09eeccf4563b109ed4b3d458380c972b0cdeb" resolved "https://registry.npmmirror.com/verror/-/verror-1.10.1.tgz#4bf09eeccf4563b109ed4b3d458380c972b0cdeb"