zyc/video-flow-toon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

完善分镜

Browse Source
This commit is contained in:
zhishi 2026-03-24 21:38:22 +08:00
parent fcedbc35c3
commit 80d0c8143f
9 changed files with 149 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
data/skills/production-agent/decision/SKILL.md
View File

@ -53,4 +53,4 @@ description: 短剧漫剧制作决策层。负责分析用户需求、制定执
- **用户目标优先**：默认直接响应并推进用户当前任务，不要为了流程完整性而强制先生成计划
- **计划按需维护**：仅当用户明确要求新增/修改拍摄计划时，才更新拍摄计划，且每次改动都调用 `set_plane` 同步到前端
- **提取衍生资产后**：计划中必须包含"询问用户是否生成资产图片"步骤。若用户确认，执行层将调用相应工具批量生成衍生资产图片
- **生成分镜表后**：计划中必须包含"询问用户是否生成分镜图片"步骤。若用户确认，执行层将调用相应工具生成分镜图
- **生成分镜面板后**：计划中必须包含"询问用户是否生成分镜图片"步骤。若用户确认，执行层将调用相应工具生成分镜图

12
data/skills/production-agent/execution/references/storyboard-generation.md
View File

@ -1,9 +1,9 @@
# 分镜表生成（从剧本 + 资产 → storyboardTable）
# 分镜面板生成（从剧本 + 资产 → storyboardTable）
本指南只做一件事：
根据剧本内容和已有资产，将剧本拆分为一系列分镜，生成结构化的分镜表。
根据剧本内容和已有资产，将剧本拆分为一系列分镜，生成结构化的分镜面板。
> **核心概念**：分镜表是将剧本转化为视觉画面的中间产物。每条分镜对应一个独立的画面/镜头，包含画面描述、镜头语言、台词、音效和关联资产等信息，用于后续图片生成。
> **核心概念**：分镜面板是将剧本转化为视觉画面的中间产物。每条分镜对应一个独立的画面/镜头，包含画面描述、镜头语言、台词、音效和关联资产等信息，用于后续图片生成。
## 1. 输入与输出
@ -14,7 +14,7 @@
### 输出
调用 `set_flowData` 将分镜表写入工作区：
调用 `set_flowData` 将分镜面板写入工作区：
```ts
set_flowData({
@ -232,8 +232,8 @@ set_flowData({
1. `get_flowData("script")` — 获取剧本内容
2. `get_flowData("assets")` — 获取已有资产列表
3. 分析剧本，按照拆分原则划分分镜，并为每条分镜填写所有字段
4. 调用 `set_flowData({ key: "storyboardTable", value: 分镜数组 })` 一次性保存完整分镜表
5. 向用户汇报分镜表概要（总共多少条分镜，覆盖的场景概括）
4. 调用 `set_flowData({ key: "storyboardTable", value: 分镜数组 })` 一次性保存完整分镜面板
5. 向用户汇报分镜面板概要（总共多少条分镜，覆盖的场景概括）
6. **询问用户是否需要生成分镜图片**：
- 如果用户确认，调用 `generate_storyboard_images({ script: 剧本文本 })` 生成分镜图
- 如果用户拒绝，跳过此步骤，流程结束

93
src/agents/productionAgent/tools.ts
View File

@ -52,7 +52,7 @@ export const flowDataSchema = z.object({
script: z.string().describe("剧本内容"),
scriptPlan: z.string().describe("拍摄计划"),
assets: z.array(assetItemSchema).describe("衍生资产"),
storyboardTable: z.string().describe("分镜表"),
storyboardTable: z.string().describe("分镜面板"),
storyboard: z.array(storyboardSchema).describe("分镜列表"),
workbench: workbenchDataSchema.describe("工作台配置"),
poster: z
@ -104,6 +104,39 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
return true;
},
}),
// add_flowData_assets: tool({
// description: "新增对应衍生资产列表到工作区，严禁包含 不需要新增的数据",
// inputSchema: z.object({ value: z.array(deriveAssetSchema).describe("需要新增的资产列表") }),
// execute: async ({ value }) => {
// console.log("[tools] set_flowData add_flowData_assets", value);
// resTool.systemMessage("正在保存 衍生资产 数据");
// const addAssetsData = [];
// if (value && Array.isArray(value) && value.length) {
// for (const i of value) {
// const [insertedId] = await u.db("o_assets").insert({
// assetsId: +i.assetsId || null,
// projectId: resTool.data.projectId,
// name: i.name,
// type: i.type,
// prompt: i.prompt,
// describe: i.desc,
// startTime: Date.now(),
// });
// console.log("%c Line:141 🍑 resTool.data.scriptId", "background:#ea7e5c", resTool.data.scriptId);
// await u.db("o_scriptAssets").insert({
// scriptId: resTool.data.scriptId,
// assetId: insertedId,
// });
// addAssetsData.push({
// ...i,
// id: insertedId,
// });
// }
// }
// socket.emit("setFlowData", { key: "addAssets", value: addAssetsData });
// return true;
// },
// }),
set_flowData_assets: tool({
description: "保存衍生资产列表到工作区",
inputSchema: z.object({ value: flowDataSchema.shape.assets }),
@ -128,12 +161,18 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
if (sub.id) continue;
const [insertedId] = await u.db("o_assets").insert({
assetsId: +i.id || null,
projectId: resTool.data.projectId,
name: sub.name,
type: sub.type,
prompt: sub.prompt,
describe: sub.desc,
startTime: Date.now(),
});
console.log("%c Line:141 🍑 resTool.data.scriptId", "background:#ea7e5c", resTool.data.scriptId);
await u.db("o_scriptAssets").insert({
scriptId: resTool.data.scriptId,
assetId: insertedId,
});
sub.id = insertedId;
}
}
@ -144,11 +183,11 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
},
}),
set_flowData_storyboardTable: tool({
description: "保存分镜表到工作区",
description: "保存分镜模板到工作区",
inputSchema: z.object({ value: flowDataSchema.shape.storyboardTable }),
execute: async ({ value }) => {
console.log("[tools] set_flowData storyboardTable", value);
resTool.systemMessage("正在保存 分镜表 数据...");
resTool.systemMessage("正在保存 分镜面板 数据...");
socket.emit("setFlowData", { key: "storyboardTable", value });
return true;
},
@ -179,8 +218,6 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
item.id = insertedId;
}
}
console.log("%c Line:181 🍏 value", "background:#93c0a4", value);
socket.emit("setFlowData", { key: "storyboard", value });
return true;
},
@ -206,6 +243,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
},
}),
//todo referenceIds 图片未使用 提示词待调
generate_storyboard_images: tool({
description: `生成一组图片任务，支持图片间的依赖关系（以图生图）。
@ -242,16 +280,20 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
const skill = await useSkill("universal-agent");
for (const item of images) {
resTool.systemMessage(`生在生成分镜 id:${item.id} 图片`);
//更新对应分镜状态
await u.db("o_storyboard").where("id", item.id).update({ state: "生成中" });
u.Ai.Image("1:doubao-seedream-4-5-251128")
// 异步生成
const imageModel = resTool.data.imageModel;
u.Ai.Image(imageModel?.modelId)
.run({
systemPrompt: skill.prompt,
prompt: item.prompt,
imageBase64: await getAssetsImageBase64(item.assetIds ?? []),
size: "2K",
aspectRatio: "16:9",
taskClass: "资产图片生成",
describe: "生成图片",
size: imageModel?.quality,
aspectRatio: imageModel?.ratio,
taskClass: "生成图片",
describe: "分镜图片生成",
relatedObjects: "hhhh",
projectId: resTool.data.projectId,
})
@ -264,16 +306,21 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
src: await u.oss.getFileUrl(savePath),
state: "已完成",
};
// 更新对应分镜状态
await u.db("o_storyboard").where("id", item.id).update({ state: "已完成", filePath: savePath });
// 前端对话框提示
resTool.systemMessage(`分镜 id:${item.id} 图片生成完成`);
// 更新前端界面展示
socket.emit("setFlowData", { key: "setStoryboardImage", value: obj });
});
//更新前端为生成中
socket.emit("setFlowData", { key: "setStoryboardImage", value: { ...item, id: item.id, src: "", state: "生成中" } });
}
return "分镜图片生成中";
},
}),
//todo 图片是否需要参考 原资产 提示词待调
generate_assets_images: tool({
description: `
生成 资产图片 不区分原资产于衍生资产
@ -289,22 +336,25 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
inputSchema: z.object({ images: z.array(z.object({ assetId: z.number(), prompt: z.string() })) }),
execute: async ({ images }) => {
const skill = await useSkill("universal-agent");
//获取所设置模型
const imageModel = resTool.data.imageModel;
for (const item of images) {
const [imageId] = await u.db("o_image").insert({
// 数据库插入图片记录
assetsId: item.assetId,
model: "1:doubao-seedream-4-5-251128",
model: imageModel?.modelId,
state: "生成中",
resolution: "2K",
resolution: imageModel?.quality,
});
u.Ai.Image("1:doubao-seedream-4-5-251128")
u.Ai.Image(imageModel?.modelId)
.run({
systemPrompt: skill.prompt,
prompt: item.prompt,
imageBase64: [],
size: "2K",
aspectRatio: "16:9",
taskClass: "资产图片生成",
describe: "生成图片",
size: imageModel?.quality,
aspectRatio: imageModel?.ratio,
taskClass: "生成图片",
describe: "资产图片生成",
relatedObjects: "hhhh",
projectId: resTool.data.projectId,
})
@ -315,12 +365,15 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
...item,
id: item.assetId,
src: await u.oss.getFileUrl(savePath),
state: "生成成功",
state: "已完成",
};
//更新对应数据库
await u.db("o_assets").where("id", item.assetId).update({ imageId: imageId });
await u.db("o_image").where({ id: imageId }).update({ state: "已完成", filePath: savePath });
//通知前端更新
socket.emit("setFlowData", { key: "setAssetsImage", value: obj });
});
//通知前端更新状态
socket.emit("setFlowData", { key: "setAssetsImage", value: { ...item, id: item.assetId, src: "", state: "生成中" } });
}
console.log("[tools] generate_assets_images", images);

13
src/agents/scriptAgent/tools.ts
View File

@ -5,14 +5,11 @@ import _ from "lodash";
import ResTool from "@/socket/resTool";
export const AssetSchema = z.object({
id: z.number().describe("衍生资产ID,如果新增则为空").optional(),
assetsId: z.string().describe("关联的资产ID"),
id: z.number().describe("资产ID,如果新增则为空").optional(),
prompt: z.string().describe("生成提示词"),
name: z.string().describe("衍生资产名称"),
desc: z.string().describe("衍生资产描述"),
src: z.string().describe("衍生资产资源路径").optional(),
state: z.enum(["未生成", "生成中", "已完成", "生成失败"]).describe("衍生资产生成状态，新增默认未生成"),
type: z.enum(["role", "tool", "scene", "clip"]).describe("衍生资产类型"),
name: z.string().describe("资产名称"),
desc: z.string().describe("资产描述"),
type: z.enum(["role", "tool", "scene", "clip"]).describe("资产类型"),
});
export const ScriptSchema = z.object({
id: z.number().describe("剧本ID,如果新增则为空").optional(),
@ -97,7 +94,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
description: "将剧本内容插入sqlite数据库，供后续业务使用",
inputSchema: z.object({
script: ScriptSchema,
assetsList: z.array(AssetSchema).describe("剧本所使用资产列表"),
assetsList: z.array(AssetSchema).describe("剧本所使用资产列表,注意不要包含剧本内容,仅为所使用到的 道具、人物、场景、素材"),
}),
execute: async ({ assetsList, script }) => {
console.log("%c Line:103 🍷 script", "background:#42b983", script);

1
src/lib/initDB.ts
View File

@ -305,6 +305,7 @@ export default async (knex: Knex, forceInit: boolean = false): Promise<void> =>
table.text("frameMode");
table.text("camera");
table.text("sound");
table.text("lines");
table.text("state");
table.text("reason");
table.integer("createTime");

2
src/routes/migrate/migrateData.ts
View File

@ -109,7 +109,7 @@ export default router.post(
if (tableName === 't_outline') { }
//迁移脚本表
if (tableName === 't_script') { }
//迁移分镜表
//迁移分镜面板
if (tableName === 't_storyboard') { }
//迁移视频表
if (tableName === 't_video') { }

31
src/routes/production/getFlowData.ts
View File

@ -35,9 +35,10 @@ export default router.post(
.db("o_assets")
.leftJoin("o_image", "o_assets.imageId", "o_image.id")
.select("o_assets.*", "o_image.filePath")
.where("o_assets.id", "in", assetIds)
.where("o_assets.projectId", projectId)
.where("o_assets.id", "in", assetIds)
.whereNotNull("o_assets.assetsId");
console.log("%c Line:35 🥚 childAssetsData", "background:#f5ce50", childAssetsData);
if (!sqlData) {
const flowData: FlowData = {
@ -45,7 +46,7 @@ export default router.post(
scriptPlan: "",
assets: await Promise.all(
assetsData.map(async (item) => ({
assetsId: item.id,
id: item.id,
name: item.name ?? "",
type: item.type ?? "",
prompt: item.prompt ?? "",
@ -58,6 +59,8 @@ export default router.post(
id: child.id,
assetsId: item.id,
name: child.name ?? "",
type: child.type,
prompt: child.prompt,
desc: child.describe ?? "",
src: child.filePath && (await u.oss.getFileUrl(child.filePath!)),
state: child.state ?? "未生成", //todo：矫正状态值
@ -84,6 +87,30 @@ export default router.post(
} else {
try {
const flowData = JSON.parse(sqlData!.data ?? "{}");
flowData.assets = await Promise.all(
assetsData.map(async (item) => ({
id: item.id,
name: item.name ?? "",
type: item.type ?? "",
prompt: item.prompt ?? "",
desc: item.describe ?? "",
src: item.filePath && (await u.oss.getFileUrl(item.filePath!)),
derive: await Promise.all(
childAssetsData
.filter((child) => child.assetsId === item.id)
.map(async (child) => ({
id: child.id,
assetsId: item.id,
name: child.name ?? "",
prompt: child.prompt,
type: child.type,
desc: child.describe ?? "",
src: child.filePath && (await u.oss.getFileUrl(child.filePath!)),
state: child.state ?? "未生成", //todo：矫正状态值
})),
),
})),
);
res.status(200).send(success(flowData));
} catch (err) {
res.status(200).send(error());

12
src/socket/routes/productionAgent.ts
View File

@ -26,7 +26,7 @@ export default (nsp: Namespace) => {
socket.disconnect();
return;
}
const isolationKey = socket.handshake.auth.isolationKey;
let isolationKey = socket.handshake.auth.isolationKey;
if (!isolationKey) {
console.log("[productionAgent] 连接失败，缺少 isolationKey");
socket.disconnect();
@ -37,6 +37,7 @@ export default (nsp: Namespace) => {
const resTool = new ResTool(socket, {
projectId: socket.handshake.auth.projectId,
scriptId: socket.handshake.auth.scriptId,
});
let abortController: AbortController | null = null;
@ -44,6 +45,7 @@ export default (nsp: Namespace) => {
abortController?.abort();
abortController = new AbortController();
const currentController = abortController;
console.log("%c Line:30 🍑 isolationKey", "background:#e41a6a", isolationKey);
const textStream = await agent.decisionAI({ socket, isolationKey, text, abortSignal: currentController.signal, resTool });
@ -62,7 +64,13 @@ export default (nsp: Namespace) => {
}
}
});
socket.on("setModelData", async (data: any) => {
resTool.data.imageModel = data;
});
socket.on("setKeyScript", async (data: any) => {
isolationKey = data.key;
resTool.data.scriptId = data.scriptId;
});
socket.on("stop", () => {
abortController?.abort();
abortController = null;

24
src/types/database.d.ts vendored
View File

@ -1,6 +1,24 @@
// @db-hash 47c0e014bdbd44b60c4ebc95f4d99e0e
// @db-hash 1ce1a8f10cb90caac306536b78942cb3
//该文件由脚本自动生成，请勿手动修改
export interface _o_storyboard_old_20260324 {
'camera'?: string | null;
'createTime'?: number | null;
'description'?: string | null;
'duration'?: string | null;
'filePath'?: string | null;
'frameMode'?: string | null;
'id'?: number;
'mode'?: string | null;
'model'?: string | null;
'prompt'?: string | null;
'reason'?: string | null;
'resolution'?: string | null;
'scriptId'?: number | null;
'sound'?: string | null;
'state'?: string | null;
'title'?: string | null;
}
export interface memories {
'content': string;
'createTime': number;
@ -132,12 +150,15 @@ export interface o_storyboard {
'filePath'?: string | null;
'frameMode'?: string | null;
'id'?: number;
'lines'?: string | null;
'mode'?: string | null;
'model'?: string | null;
'prompt'?: string | null;
'reason'?: string | null;
'resolution'?: string | null;
'scriptId'?: number | null;
'sound'?: string | null;
'state'?: string | null;
'title'?: string | null;
}
export interface o_storyboardFlow {
@ -197,6 +218,7 @@ export interface o_videoConfig {
}
export interface DB {
"_o_storyboard_old_20260324": _o_storyboard_old_20260324;
"memories": memories;
"o_agentDeploy": o_agentDeploy;
"o_agentWorkData": o_agentWorkData;