zyc/video-flow-toon
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

完善分镜

Browse Source
This commit is contained in:
zhishi 2026-03-24 18:43:13 +08:00
parent 38b8503c64
commit 9dcc54a8fa
6 changed files with 214 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

170
src/agents/productionAgent/tools.ts
View File

@ -2,10 +2,12 @@ import { tool, Tool } from "ai";
import { z } from "zod";
import _ from "lodash";
import ResTool from "@/socket/resTool";
import u from "@/utils";
import { useSkill } from "@/utils/agent/skillsTools";
import { urlToBase64 } from "@/utils/vm";
export const deriveAssetSchema = z.object({
id: z.number().describe("衍生资产ID,如果新增则为空").optional(),
assetsId: z.string().describe("关联的资产ID"),
assetsId: z.number().describe("关联的资产ID"),
prompt: z.string().describe("生成提示词"),
name: z.string().describe("衍生资产名称"),
desc: z.string().describe("衍生资产描述"),
@ -14,14 +16,15 @@ export const deriveAssetSchema = z.object({
type: z.enum(["role", "tool", "scene", "clip"]).describe("衍生资产类型"),
});
export const assetItemSchema = z.object({
assetsId: z.string().describe("资产唯一标识"),
id: z.number().describe("资产唯一标识"),
name: z.string().describe("资产名称"),
type: z.enum(["role", "tool", "scene", "clip"]).describe("资产类型"),
prompt: z.string().describe("生成提示词"),
desc: z.string().describe("资产描述"),
src: z.string().describe("资产资源路径"),
derive: z.array(deriveAssetSchema).describe("衍生资产列表"),
});
export const storyboardSchema = z.object({
id: z.number().describe("分镜ID"),
id: z.number().optional().describe("分镜ID,未从工作区获得的分镜列表视为需要新增;如需新增则为空"),
title: z.string().describe("分镜标题"),
description: z.string().describe("分镜描述"),
camera: z.string().describe("镜头信息"),
@ -107,6 +110,35 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
execute: async ({ value }) => {
console.log("[tools] set_flowData assets", value);
resTool.systemMessage("正在保存 衍生资产 数据");
if (value && Array.isArray(value) && value.length) {
for (const i of value) {
if (!i?.id) {
const [insertedId] = await u.db("o_assets").insert({
assetsId: null,
name: i.name,
type: i.type,
prompt: i.prompt,
describe: i.desc,
startTime: Date.now(),
});
i.id = insertedId;
}
if (i.derive && Array.isArray(i.derive) && i.derive.length) {
for (const sub of i.derive) {
if (sub.id) continue;
const [insertedId] = await u.db("o_assets").insert({
assetsId: +i.id || null,
name: sub.name,
type: sub.type,
prompt: sub.prompt,
describe: sub.desc,
startTime: Date.now(),
});
sub.id = insertedId;
}
}
}
}
socket.emit("setFlowData", { key: "assets", value });
return true;
},
@ -127,6 +159,28 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
execute: async ({ value }) => {
console.log("[tools] set_flowData storyboard", value);
resTool.systemMessage("正在保存 分镜列表 数据...");
for (const item of value) {
if (!item.id) {
const [insertedId] = await u.db("o_storyboard").insert({
title: item.title,
prompt: item.prompt,
description: item.description,
filePath: item.src,
frameMode: item.frameMode,
duration: String(item.duration),
camera: item.camera,
sound: item.sound,
lines: item.lines,
state: "未生成",
});
if (item.associateAssetsIds.length) {
await u.db("o_assets2Storyboard").insert(item.associateAssetsIds.map((i) => ({ storyboardId: insertedId, assetId: i })));
}
item.id = insertedId;
}
}
console.log("%c Line:181 🍏 value", "background:#93c0a4", value);
socket.emit("setFlowData", { key: "storyboard", value });
return true;
},
@ -175,7 +229,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
inputSchema: z.object({
images: z.array(
z.object({
id: z.string().describe("图片唯一标识符"),
id: z.number().describe("从工作区获取到的分镜id"),
prompt: z.string().describe("图片生成提示词"),
referenceIds: z.array(z.string()).describe("依赖的参考图id数组，无依赖填空数组[]"),
assetIds: z.array(z.number()).optional().describe("参考的资产图"),
@ -184,18 +238,116 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
}),
execute: async ({ images }) => {
console.log("[tools] generated_assets", images);
return new Promise((resolve) => socket.emit("generatedAssets", { images }, (res: any) => resolve(res)));
const skill = await useSkill("universal-agent");
for (const item of images) {
resTool.systemMessage(`生在生成分镜 id:${item.id} 图片`);
await u.db("o_storyboard").where("id", item.id).update({ state: "生成中" });
u.Ai.Image("1:doubao-seedream-4-5-251128")
.run({
systemPrompt: skill.prompt,
prompt: item.prompt,
imageBase64: await getAssetsImageBase64(item.assetIds ?? []),
size: "2K",
aspectRatio: "16:9",
taskClass: "资产图片生成",
describe: "生成图片",
relatedObjects: "hhhh",
projectId: resTool.data.projectId,
})
.then(async (imageCls) => {
const savePath = `/${resTool.data.projectId}/storyboard/${u.uuid()}.jpg`;
await imageCls.save(savePath);
const obj = {
...item,
id: item.id,
src: await u.oss.getFileUrl(savePath),
state: "已完成",
};
await u.db("o_storyboard").where("id", item.id).update({ state: "已完成", filePath: savePath });
resTool.systemMessage(`分镜 id:${item.id} 图片生成完成`);
socket.emit("setFlowData", { key: "setStoryboardImage", value: obj });
});
socket.emit("setFlowData", { key: "setStoryboardImage", value: { ...item, id: item.id, src: "", state: "生成中" } });
}
return "分镜图片生成中";
},
}),
generate_assets_images: tool({
description: "生成分镜图",
description: `
生成 资产图片 不区分原资产于衍生资产
参数说明：
- images: 图片任务数组
- assetId: 资产id
- prompt: 图片生成提示词
示例：
images:[
{assetId: 1, prompt: "一张猫的图片"}
]
`,
inputSchema: z.object({ images: z.array(z.object({ assetId: z.number(), prompt: z.string() })) }),
execute: async ({ images }) => {
const skill = await useSkill("universal-agent");
for (const item of images) {
const [imageId] = await u.db("o_image").insert({
assetsId: item.assetId,
model: "1:doubao-seedream-4-5-251128",
state: "生成中",
resolution: "2K",
});
u.Ai.Image("1:doubao-seedream-4-5-251128")
.run({
systemPrompt: skill.prompt,
prompt: item.prompt,
imageBase64: [],
size: "2K",
aspectRatio: "16:9",
taskClass: "资产图片生成",
describe: "生成图片",
relatedObjects: "hhhh",
projectId: resTool.data.projectId,
})
.then(async (imageCls) => {
const savePath = `/${resTool.data.projectId}/assets/${u.uuid()}.jpg`;
await imageCls.save(savePath);
const obj = {
...item,
id: item.assetId,
src: await u.oss.getFileUrl(savePath),
state: "生成成功",
};
await u.db("o_image").where({ id: imageId }).update({ state: "已完成", filePath: savePath });
socket.emit("setFlowData", { key: "setAssetsImage", value: obj });
});
socket.emit("setFlowData", { key: "setAssetsImage", value: { ...item, id: item.assetId, src: "", state: "生成中" } });
}
console.log("[tools] generate_assets_images", images);
return new Promise((resolve) => socket.emit("generateAssetsImages", { images }, (res: any) => resolve(res)));
return "资产生成中";
},
}),
};
return toolsNames ? Object.fromEntries(Object.entries(tools).filter(([n]) => toolsNames.includes(n))) : tools;
};
async function getAssetsImageBase64(imageIds: number[]) {
if (imageIds.length === 0) return [];
const imagePaths = await u
.db("o_assets")
.leftJoin("o_image", "o_assets.imageId", "o_image.id")
.whereIn("o_assets.id", imageIds)
.select("o_assets.id", "o_image.filePath");
if (!imagePaths.length) return [];
const imageUrls = await Promise.all(
imagePaths.map(async (i) => {
if (i.filePath) {
return await urlToBase64(await u.oss.getFileUrl(i.filePath));
} else {
return null;
}
}),
);
return imageUrls.filter(Boolean) as string[];
}

2
src/lib/initDB.ts
View File

@ -303,6 +303,8 @@ export default async (knex: Knex, forceInit: boolean = false): Promise<void> =>
table.text("frameMode");
table.text("camera");
table.text("sound");
table.text("state");
table.text("reason");
table.integer("createTime");
table.primary(["id"]);
table.unique(["id"]);

10
src/routes/production/getFlowData.ts
View File

@ -21,17 +21,21 @@ export default router.post(
.select("data")
.first();
const scriptData = await u.db("o_script").where("projectId", projectId).first();
const scriptData = await u.db("o_script").where("projectId", projectId).where("id", episodesId).first();
const scriptAssets = await u.db("o_scriptAssets").where("scriptId", episodesId);
const assetIds = scriptAssets.map((i) => i.assetId);
const assetsData = await u
.db("o_assets")
.leftJoin("o_image", "o_assets.imageId", "o_image.id")
.select("o_assets.*", "o_image.filePath")
.where("o_assets.id", "in", assetIds)
.whereNull("o_assets.assetsId")
.where("o_assets.projectId", projectId);
let childAssetsData = await u
.db("o_assets")
.leftJoin("o_image", "o_assets.imageId", "o_image.id")
.select("o_assets.*", "o_image.filePath")
.where("o_assets.id", "in", assetIds)
.where("o_assets.projectId", projectId)
.whereNotNull("o_assets.assetsId");
@ -43,6 +47,8 @@ export default router.post(
assetsData.map(async (item) => ({
assetsId: item.id,
name: item.name ?? "",
type: item.type ?? "",
prompt: item.prompt ?? "",
desc: item.describe ?? "",
src: item.filePath && (await u.oss.getFileUrl(item.filePath!)),
derive: await Promise.all(

1
src/routes/production/workbench/getChatLines.ts
View File

@ -35,6 +35,7 @@ async function getLines(prompt: string) {
const resText = await u.Ai.Text("universalAgent").invoke({
system: skill.prompt,
messages: [{ role: "user", content: prompt }],
tools: skill.tools,
output: Output.array({
element: z.object({
lines: z.string().describe("台词内容"),

4
src/socket/routes/productionAgent.ts
View File

@ -35,7 +35,9 @@ export default (nsp: Namespace) => {
console.log("[productionAgent] 已连接:", socket.id);
const resTool = new ResTool(socket);
const resTool = new ResTool(socket, {
projectId: socket.handshake.auth.projectId,
});
let abortController: AbortController | null = null;
socket.on("message", async (text: string) => {

40
src/types/database.d.ts vendored
View File

@ -1,6 +1,39 @@
// @db-hash 83c8dadf13c2aee689597b709a690870
// @db-hash 08cfec7bed045942e10f27f6d3f34623
//该文件由脚本自动生成，请勿手动修改
export interface _o_storyboard_old_20260324 {
'camera'?: string | null;
'createTime'?: number | null;
'description'?: string | null;
'duration'?: string | null;
'filePath'?: string | null;
'frameMode'?: string | null;
'id'?: number;
'mode'?: string | null;
'model'?: string | null;
'prompt'?: string | null;
'resolution'?: string | null;
'scriptId'?: number | null;
'sound'?: string | null;
'title'?: string | null;
}
export interface _o_storyboard_old_20260324_1 {
'camera'?: string | null;
'createTime'?: number | null;
'description'?: string | null;
'duration'?: string | null;
'filePath'?: string | null;
'frameMode'?: string | null;
'id'?: number;
'lines'?: string | null;
'mode'?: string | null;
'model'?: string | null;
'prompt'?: string | null;
'resolution'?: string | null;
'scriptId'?: number | null;
'sound'?: string | null;
'title'?: string | null;
}
export interface memories {
'content': string;
'createTime': number;
@ -130,12 +163,15 @@ export interface o_storyboard {
'filePath'?: string | null;
'frameMode'?: string | null;
'id'?: number;
'lines'?: string | null;
'mode'?: string | null;
'model'?: string | null;
'prompt'?: string | null;
'reason'?: string | null;
'resolution'?: string | null;
'scriptId'?: number | null;
'sound'?: string | null;
'state'?: string | null;
'title'?: string | null;
}
export interface o_storyboardFlow {
@ -195,6 +231,8 @@ export interface o_videoConfig {
}
export interface DB {
"_o_storyboard_old_20260324": _o_storyboard_old_20260324;
"_o_storyboard_old_20260324_1": _o_storyboard_old_20260324_1;
"memories": memories;
"o_agentDeploy": o_agentDeploy;
"o_agentWorkData": o_agentWorkData;