zyc/video-flow-toon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch '108' of https://github.com/HBAI-Ltd/Toonflow-app into 108

Browse Source 
# Conflicts:
#	src/types/database.d.ts
This commit is contained in:
ACT丶流星雨 2026-03-24 22:53:10 +08:00
commit ce22f96cb7
10 changed files with 310 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
data/skills/production-agent/decision/SKILL.md
View File

@ -53,4 +53,4 @@ description: 短剧漫剧制作决策层。负责分析用户需求、制定执
- **用户目标优先**：默认直接响应并推进用户当前任务，不要为了流程完整性而强制先生成计划
- **计划按需维护**：仅当用户明确要求新增/修改拍摄计划时，才更新拍摄计划，且每次改动都调用 `set_plane` 同步到前端
- **提取衍生资产后**：计划中必须包含"询问用户是否生成资产图片"步骤。若用户确认，执行层将调用相应工具批量生成衍生资产图片
- **生成分镜表后**：计划中必须包含"询问用户是否生成分镜图片"步骤。若用户确认，执行层将调用相应工具生成分镜图
- **生成分镜面板后**：计划中必须包含"询问用户是否生成分镜图片"步骤。若用户确认，执行层将调用相应工具生成分镜图

12
data/skills/production-agent/execution/references/storyboard-generation.md
View File

@ -1,9 +1,9 @@
# 分镜表生成（从剧本 + 资产 → storyboardTable）
# 分镜面板生成（从剧本 + 资产 → storyboardTable）
本指南只做一件事：
根据剧本内容和已有资产，将剧本拆分为一系列分镜，生成结构化的分镜表。
根据剧本内容和已有资产，将剧本拆分为一系列分镜，生成结构化的分镜面板。
> **核心概念**：分镜表是将剧本转化为视觉画面的中间产物。每条分镜对应一个独立的画面/镜头，包含画面描述、镜头语言、台词、音效和关联资产等信息，用于后续图片生成。
> **核心概念**：分镜面板是将剧本转化为视觉画面的中间产物。每条分镜对应一个独立的画面/镜头，包含画面描述、镜头语言、台词、音效和关联资产等信息，用于后续图片生成。
## 1. 输入与输出
@ -14,7 +14,7 @@
### 输出
调用 `set_flowData` 将分镜表写入工作区：
调用 `set_flowData` 将分镜面板写入工作区：
```ts
set_flowData({
@ -232,8 +232,8 @@ set_flowData({
1. `get_flowData("script")` — 获取剧本内容
2. `get_flowData("assets")` — 获取已有资产列表
3. 分析剧本，按照拆分原则划分分镜，并为每条分镜填写所有字段
4. 调用 `set_flowData({ key: "storyboardTable", value: 分镜数组 })` 一次性保存完整分镜表
5. 向用户汇报分镜表概要（总共多少条分镜，覆盖的场景概括）
4. 调用 `set_flowData({ key: "storyboardTable", value: 分镜数组 })` 一次性保存完整分镜面板
5. 向用户汇报分镜面板概要（总共多少条分镜，覆盖的场景概括）
6. **询问用户是否需要生成分镜图片**：
- 如果用户确认，调用 `generate_storyboard_images({ script: 剧本文本 })` 生成分镜图
- 如果用户拒绝，跳过此步骤，流程结束

229
src/agents/productionAgent/tools.ts
View File

@ -2,10 +2,12 @@ import { tool, Tool } from "ai";
import { z } from "zod";
import _ from "lodash";
import ResTool from "@/socket/resTool";
import u from "@/utils";
import { useSkill } from "@/utils/agent/skillsTools";
import { urlToBase64 } from "@/utils/vm";
export const deriveAssetSchema = z.object({
id: z.number().describe("衍生资产ID,如果新增则为空").optional(),
assetsId: z.string().describe("关联的资产ID"),
assetsId: z.number().describe("关联的资产ID"),
prompt: z.string().describe("生成提示词"),
name: z.string().describe("衍生资产名称"),
desc: z.string().describe("衍生资产描述"),
@ -14,14 +16,15 @@ export const deriveAssetSchema = z.object({
type: z.enum(["role", "tool", "scene", "clip"]).describe("衍生资产类型"),
});
export const assetItemSchema = z.object({
assetsId: z.string().describe("资产唯一标识"),
id: z.number().describe("资产唯一标识"),
name: z.string().describe("资产名称"),
type: z.enum(["role", "tool", "scene", "clip"]).describe("资产类型"),
prompt: z.string().describe("生成提示词"),
desc: z.string().describe("资产描述"),
src: z.string().describe("资产资源路径"),
derive: z.array(deriveAssetSchema).describe("衍生资产列表"),
});
export const storyboardSchema = z.object({
id: z.number().describe("分镜ID"),
id: z.number().optional().describe("分镜ID,未从工作区获得的分镜列表视为需要新增;如需新增则为空"),
title: z.string().describe("分镜标题"),
description: z.string().describe("分镜描述"),
camera: z.string().describe("镜头信息"),
@ -49,7 +52,7 @@ export const flowDataSchema = z.object({
script: z.string().describe("剧本内容"),
scriptPlan: z.string().describe("拍摄计划"),
assets: z.array(assetItemSchema).describe("衍生资产"),
storyboardTable: z.string().describe("分镜表"),
storyboardTable: z.string().describe("分镜面板"),
storyboard: z.array(storyboardSchema).describe("分镜列表"),
workbench: workbenchDataSchema.describe("工作台配置"),
poster: z
@ -101,22 +104,90 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
return true;
},
}),
// add_flowData_assets: tool({
// description: "新增对应衍生资产列表到工作区，严禁包含 不需要新增的数据",
// inputSchema: z.object({ value: z.array(deriveAssetSchema).describe("需要新增的资产列表") }),
// execute: async ({ value }) => {
// console.log("[tools] set_flowData add_flowData_assets", value);
// resTool.systemMessage("正在保存 衍生资产 数据");
// const addAssetsData = [];
// if (value && Array.isArray(value) && value.length) {
// for (const i of value) {
// const [insertedId] = await u.db("o_assets").insert({
// assetsId: +i.assetsId || null,
// projectId: resTool.data.projectId,
// name: i.name,
// type: i.type,
// prompt: i.prompt,
// describe: i.desc,
// startTime: Date.now(),
// });
// console.log("%c Line:141 🍑 resTool.data.scriptId", "background:#ea7e5c", resTool.data.scriptId);
// await u.db("o_scriptAssets").insert({
// scriptId: resTool.data.scriptId,
// assetId: insertedId,
// });
// addAssetsData.push({
// ...i,
// id: insertedId,
// });
// }
// }
// socket.emit("setFlowData", { key: "addAssets", value: addAssetsData });
// return true;
// },
// }),
set_flowData_assets: tool({
description: "保存衍生资产列表到工作区",
inputSchema: z.object({ value: flowDataSchema.shape.assets }),
execute: async ({ value }) => {
console.log("[tools] set_flowData assets", value);
resTool.systemMessage("正在保存 衍生资产 数据");
if (value && Array.isArray(value) && value.length) {
for (const i of value) {
if (!i?.id) {
const [insertedId] = await u.db("o_assets").insert({
assetsId: null,
name: i.name,
type: i.type,
prompt: i.prompt,
describe: i.desc,
startTime: Date.now(),
});
i.id = insertedId;
}
if (i.derive && Array.isArray(i.derive) && i.derive.length) {
for (const sub of i.derive) {
if (sub.id) continue;
const [insertedId] = await u.db("o_assets").insert({
assetsId: +i.id || null,
projectId: resTool.data.projectId,
name: sub.name,
type: sub.type,
prompt: sub.prompt,
describe: sub.desc,
startTime: Date.now(),
});
console.log("%c Line:141 🍑 resTool.data.scriptId", "background:#ea7e5c", resTool.data.scriptId);
await u.db("o_scriptAssets").insert({
scriptId: resTool.data.scriptId,
assetId: insertedId,
});
sub.id = insertedId;
}
}
}
}
socket.emit("setFlowData", { key: "assets", value });
return true;
},
}),
set_flowData_storyboardTable: tool({
description: "保存分镜表到工作区",
description: "保存分镜模板到工作区",
inputSchema: z.object({ value: flowDataSchema.shape.storyboardTable }),
execute: async ({ value }) => {
console.log("[tools] set_flowData storyboardTable", value);
resTool.systemMessage("正在保存 分镜表 数据...");
resTool.systemMessage("正在保存 分镜面板 数据...");
socket.emit("setFlowData", { key: "storyboardTable", value });
return true;
},
@ -127,6 +198,26 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
execute: async ({ value }) => {
console.log("[tools] set_flowData storyboard", value);
resTool.systemMessage("正在保存 分镜列表 数据...");
for (const item of value) {
if (!item.id) {
const [insertedId] = await u.db("o_storyboard").insert({
title: item.title,
prompt: item.prompt,
description: item.description,
filePath: item.src,
frameMode: item.frameMode,
duration: String(item.duration),
camera: item.camera,
sound: item.sound,
lines: item.lines,
state: "未生成",
});
if (item.associateAssetsIds.length) {
await u.db("o_assets2Storyboard").insert(item.associateAssetsIds.map((i) => ({ storyboardId: insertedId, assetId: i })));
}
item.id = insertedId;
}
}
socket.emit("setFlowData", { key: "storyboard", value });
return true;
},
@ -152,6 +243,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
},
}),
//todo referenceIds 图片未使用 提示词待调
generate_storyboard_images: tool({
description: `生成一组图片任务，支持图片间的依赖关系（以图生图）。
@ -175,7 +267,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
inputSchema: z.object({
images: z.array(
z.object({
id: z.string().describe("图片唯一标识符"),
id: z.number().describe("从工作区获取到的分镜id"),
prompt: z.string().describe("图片生成提示词"),
referenceIds: z.array(z.string()).describe("依赖的参考图id数组，无依赖填空数组[]"),
assetIds: z.array(z.number()).optional().describe("参考的资产图"),
@ -184,18 +276,131 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
}),
execute: async ({ images }) => {
console.log("[tools] generated_assets", images);
return new Promise((resolve) => socket.emit("generatedAssets", { images }, (res: any) => resolve(res)));
const skill = await useSkill("universal-agent");
for (const item of images) {
resTool.systemMessage(`生在生成分镜 id:${item.id} 图片`);
//更新对应分镜状态
await u.db("o_storyboard").where("id", item.id).update({ state: "生成中" });
// 异步生成
const imageModel = resTool.data.imageModel;
u.Ai.Image(imageModel?.modelId)
.run({
systemPrompt: skill.prompt,
prompt: item.prompt,
imageBase64: await getAssetsImageBase64(item.assetIds ?? []),
size: imageModel?.quality,
aspectRatio: imageModel?.ratio,
taskClass: "生成图片",
describe: "分镜图片生成",
relatedObjects: "hhhh",
projectId: resTool.data.projectId,
})
.then(async (imageCls) => {
const savePath = `/${resTool.data.projectId}/storyboard/${u.uuid()}.jpg`;
await imageCls.save(savePath);
const obj = {
...item,
id: item.id,
src: await u.oss.getFileUrl(savePath),
state: "已完成",
};
// 更新对应分镜状态
await u.db("o_storyboard").where("id", item.id).update({ state: "已完成", filePath: savePath });
// 前端对话框提示
resTool.systemMessage(`分镜 id:${item.id} 图片生成完成`);
// 更新前端界面展示
socket.emit("setFlowData", { key: "setStoryboardImage", value: obj });
});
//更新前端为生成中
socket.emit("setFlowData", { key: "setStoryboardImage", value: { ...item, id: item.id, src: "", state: "生成中" } });
}
return "分镜图片生成中";
},
}),
//todo 图片是否需要参考 原资产 提示词待调
generate_assets_images: tool({
description: "生成分镜图",
description: `
生成 资产图片 不区分原资产于衍生资产
参数说明：
- images: 图片任务数组
- assetId: 资产id
- prompt: 图片生成提示词
示例：
images:[
{assetId: 1, prompt: "一张猫的图片"}
]
`,
inputSchema: z.object({ images: z.array(z.object({ assetId: z.number(), prompt: z.string() })) }),
execute: async ({ images }) => {
const skill = await useSkill("universal-agent");
//获取所设置模型
const imageModel = resTool.data.imageModel;
for (const item of images) {
const [imageId] = await u.db("o_image").insert({
// 数据库插入图片记录
assetsId: item.assetId,
model: imageModel?.modelId,
state: "生成中",
resolution: imageModel?.quality,
});
u.Ai.Image(imageModel?.modelId)
.run({
systemPrompt: skill.prompt,
prompt: item.prompt,
imageBase64: [],
size: imageModel?.quality,
aspectRatio: imageModel?.ratio,
taskClass: "生成图片",
describe: "资产图片生成",
relatedObjects: "hhhh",
projectId: resTool.data.projectId,
})
.then(async (imageCls) => {
const savePath = `/${resTool.data.projectId}/assets/${u.uuid()}.jpg`;
await imageCls.save(savePath);
const obj = {
...item,
id: item.assetId,
src: await u.oss.getFileUrl(savePath),
state: "已完成",
};
//更新对应数据库
await u.db("o_assets").where("id", item.assetId).update({ imageId: imageId });
await u.db("o_image").where({ id: imageId }).update({ state: "已完成", filePath: savePath });
//通知前端更新
socket.emit("setFlowData", { key: "setAssetsImage", value: obj });
});
//通知前端更新状态
socket.emit("setFlowData", { key: "setAssetsImage", value: { ...item, id: item.assetId, src: "", state: "生成中" } });
}
console.log("[tools] generate_assets_images", images);
return new Promise((resolve) => socket.emit("generateAssetsImages", { images }, (res: any) => resolve(res)));
return "资产生成中";
},
}),
};
return toolsNames ? Object.fromEntries(Object.entries(tools).filter(([n]) => toolsNames.includes(n))) : tools;
};
async function getAssetsImageBase64(imageIds: number[]) {
if (imageIds.length === 0) return [];
const imagePaths = await u
.db("o_assets")
.leftJoin("o_image", "o_assets.imageId", "o_image.id")
.whereIn("o_assets.id", imageIds)
.select("o_assets.id", "o_image.filePath");
if (!imagePaths.length) return [];
const imageUrls = await Promise.all(
imagePaths.map(async (i) => {
if (i.filePath) {
return await urlToBase64(await u.oss.getFileUrl(i.filePath));
} else {
return null;
}
}),
);
return imageUrls.filter(Boolean) as string[];
}

13
src/agents/scriptAgent/tools.ts
View File

@ -5,14 +5,11 @@ import _ from "lodash";
import ResTool from "@/socket/resTool";
export const AssetSchema = z.object({
id: z.number().describe("衍生资产ID,如果新增则为空").optional(),
assetsId: z.string().describe("关联的资产ID"),
id: z.number().describe("资产ID,如果新增则为空").optional(),
prompt: z.string().describe("生成提示词"),
name: z.string().describe("衍生资产名称"),
desc: z.string().describe("衍生资产描述"),
src: z.string().describe("衍生资产资源路径").optional(),
state: z.enum(["未生成", "生成中", "已完成", "生成失败"]).describe("衍生资产生成状态，新增默认未生成"),
type: z.enum(["role", "tool", "scene", "clip"]).describe("衍生资产类型"),
name: z.string().describe("资产名称"),
desc: z.string().describe("资产描述"),
type: z.enum(["role", "tool", "scene", "clip"]).describe("资产类型"),
});
export const ScriptSchema = z.object({
id: z.number().describe("剧本ID,如果新增则为空").optional(),
@ -97,7 +94,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
description: "将剧本内容插入sqlite数据库，供后续业务使用",
inputSchema: z.object({
script: ScriptSchema,
assetsList: z.array(AssetSchema).describe("剧本所使用资产列表"),
assetsList: z.array(AssetSchema).describe("剧本所使用资产列表,注意不要包含剧本内容,仅为所使用到的 道具、人物、场景、素材"),
}),
execute: async ({ assetsList, script }) => {
console.log("%c Line:103 🍷 script", "background:#42b983", script);

3
src/lib/initDB.ts
View File

@ -305,6 +305,9 @@ export default async (knex: Knex, forceInit: boolean = false): Promise<void> =>
table.text("frameMode");
table.text("camera");
table.text("sound");
table.text("lines");
table.text("state");
table.text("reason");
table.integer("createTime");
table.primary(["id"]);
table.unique(["id"]);

2
src/routes/migrate/migrateData.ts
View File

@ -109,7 +109,7 @@ export default router.post(
if (tableName === 't_outline') { }
//迁移脚本表
if (tableName === 't_script') { }
//迁移分镜表
//迁移分镜面板
if (tableName === 't_storyboard') { }
//迁移视频表
if (tableName === 't_video') { }

39
src/routes/production/getFlowData.ts
View File

@ -21,19 +21,24 @@ export default router.post(
.select("data")
.first();
const scriptData = await u.db("o_script").where("projectId", projectId).first();
const scriptData = await u.db("o_script").where("projectId", projectId).where("id", episodesId).first();
const scriptAssets = await u.db("o_scriptAssets").where("scriptId", episodesId);
const assetIds = scriptAssets.map((i) => i.assetId);
const assetsData = await u
.db("o_assets")
.leftJoin("o_image", "o_assets.imageId", "o_image.id")
.select("o_assets.*", "o_image.filePath")
.where("o_assets.id", "in", assetIds)
.whereNull("o_assets.assetsId")
.where("o_assets.projectId", projectId);
let childAssetsData = await u
.db("o_assets")
.leftJoin("o_image", "o_assets.imageId", "o_image.id")
.select("o_assets.*", "o_image.filePath")
.where("o_assets.projectId", projectId)
.where("o_assets.id", "in", assetIds)
.whereNotNull("o_assets.assetsId");
console.log("%c Line:35 🥚 childAssetsData", "background:#f5ce50", childAssetsData);
if (!sqlData) {
const flowData: FlowData = {
@ -41,8 +46,10 @@ export default router.post(
scriptPlan: "",
assets: await Promise.all(
assetsData.map(async (item) => ({
assetsId: item.id,
id: item.id,
name: item.name ?? "",
type: item.type ?? "",
prompt: item.prompt ?? "",
desc: item.describe ?? "",
src: item.filePath && (await u.oss.getFileUrl(item.filePath!)),
derive: await Promise.all(
@ -52,6 +59,8 @@ export default router.post(
id: child.id,
assetsId: item.id,
name: child.name ?? "",
type: child.type,
prompt: child.prompt,
desc: child.describe ?? "",
src: child.filePath && (await u.oss.getFileUrl(child.filePath!)),
state: child.state ?? "未生成", //todo：矫正状态值
@ -78,6 +87,30 @@ export default router.post(
} else {
try {
const flowData = JSON.parse(sqlData!.data ?? "{}");
flowData.assets = await Promise.all(
assetsData.map(async (item) => ({
id: item.id,
name: item.name ?? "",
type: item.type ?? "",
prompt: item.prompt ?? "",
desc: item.describe ?? "",
src: item.filePath && (await u.oss.getFileUrl(item.filePath!)),
derive: await Promise.all(
childAssetsData
.filter((child) => child.assetsId === item.id)
.map(async (child) => ({
id: child.id,
assetsId: item.id,
name: child.name ?? "",
prompt: child.prompt,
type: child.type,
desc: child.describe ?? "",
src: child.filePath && (await u.oss.getFileUrl(child.filePath!)),
state: child.state ?? "未生成", //todo：矫正状态值
})),
),
})),
);
res.status(200).send(success(flowData));
} catch (err) {
res.status(200).send(error());

1
src/routes/production/workbench/getChatLines.ts
View File

@ -35,6 +35,7 @@ async function getLines(prompt: string) {
const resText = await u.Ai.Text("universalAgent").invoke({
system: skill.prompt,
messages: [{ role: "user", content: prompt }],
tools: skill.tools,
output: Output.array({
element: z.object({
lines: z.string().describe("台词内容"),

16
src/socket/routes/productionAgent.ts
View File

@ -26,7 +26,7 @@ export default (nsp: Namespace) => {
socket.disconnect();
return;
}
const isolationKey = socket.handshake.auth.isolationKey;
let isolationKey = socket.handshake.auth.isolationKey;
if (!isolationKey) {
console.log("[productionAgent] 连接失败，缺少 isolationKey");
socket.disconnect();
@ -35,13 +35,17 @@ export default (nsp: Namespace) => {
console.log("[productionAgent] 已连接:", socket.id);
const resTool = new ResTool(socket);
const resTool = new ResTool(socket, {
projectId: socket.handshake.auth.projectId,
scriptId: socket.handshake.auth.scriptId,
});
let abortController: AbortController | null = null;
socket.on("message", async (text: string) => {
abortController?.abort();
abortController = new AbortController();
const currentController = abortController;
console.log("%c Line:30 🍑 isolationKey", "background:#e41a6a", isolationKey);
const textStream = await agent.decisionAI({ socket, isolationKey, text, abortSignal: currentController.signal, resTool });
@ -60,7 +64,13 @@ export default (nsp: Namespace) => {
}
}
});
socket.on("setModelData", async (data: any) => {
resTool.data.imageModel = data;
});
socket.on("setKeyScript", async (data: any) => {
isolationKey = data.key;
resTool.data.scriptId = data.scriptId;
});
socket.on("stop", () => {
abortController?.abort();
abortController = null;

31
src/types/database.d.ts vendored
View File

@ -1,6 +1,24 @@
// @db-hash f6a9a8164252ce954394431079615459
// @db-hash 1ce1a8f10cb90caac306536b78942cb3
//该文件由脚本自动生成，请勿手动修改
export interface _o_storyboard_old_20260324 {
'camera'?: string | null;
'createTime'?: number | null;
'description'?: string | null;
'duration'?: string | null;
'filePath'?: string | null;
'frameMode'?: string | null;
'id'?: number;
'mode'?: string | null;
'model'?: string | null;
'prompt'?: string | null;
'reason'?: string | null;
'resolution'?: string | null;
'scriptId'?: number | null;
'sound'?: string | null;
'state'?: string | null;
'title'?: string | null;
}
export interface memories {
'content': string;
'createTime': number;
@ -32,11 +50,14 @@ export interface o_agentWorkData {
'updateTime'?: number | null;
}
export interface o_artStyle {
'fileUrl'?: string | null;
'id'?: number;
'label'?: string | null;
'name'?: string | null;
'styles'?: string | null;
'prompt'?: string | null;
}
export interface o_assets {
'assetsId'?: number | null;
'describe'?: string | null;
'id'?: number;
'imageId'?: number | null;
@ -45,9 +66,7 @@ export interface o_assets {
'prompt'?: string | null;
'remark'?: string | null;
'scriptId'?: number | null;
'sonId'?: number | null;
'startTime'?: number | null;
'state'?: string | null;
'type'?: string | null;
}
export interface o_assets2Storyboard {
@ -131,12 +150,15 @@ export interface o_storyboard {
'filePath'?: string | null;
'frameMode'?: string | null;
'id'?: number;
'lines'?: string | null;
'mode'?: string | null;
'model'?: string | null;
'prompt'?: string | null;
'reason'?: string | null;
'resolution'?: string | null;
'scriptId'?: number | null;
'sound'?: string | null;
'state'?: string | null;
'title'?: string | null;
}
export interface o_storyboardFlow {
@ -196,6 +218,7 @@ export interface o_videoConfig {
}
export interface DB {
"_o_storyboard_old_20260324": _o_storyboard_old_20260324;
"memories": memories;
"o_agentDeploy": o_agentDeploy;
"o_agentWorkData": o_agentWorkData;