Merge branch '108' of https://github.com/HBAI-Ltd/Toonflow-app into 108
# Conflicts: # src/types/database.d.ts
This commit is contained in:
commit
ce22f96cb7
@ -53,4 +53,4 @@ description: 短剧漫剧制作决策层。负责分析用户需求、制定执
|
|||||||
- **用户目标优先**:默认直接响应并推进用户当前任务,不要为了流程完整性而强制先生成计划
|
- **用户目标优先**:默认直接响应并推进用户当前任务,不要为了流程完整性而强制先生成计划
|
||||||
- **计划按需维护**:仅当用户明确要求新增/修改拍摄计划时,才更新拍摄计划,且每次改动都调用 `set_plane` 同步到前端
|
- **计划按需维护**:仅当用户明确要求新增/修改拍摄计划时,才更新拍摄计划,且每次改动都调用 `set_plane` 同步到前端
|
||||||
- **提取衍生资产后**:计划中必须包含"询问用户是否生成资产图片"步骤。若用户确认,执行层将调用相应工具批量生成衍生资产图片
|
- **提取衍生资产后**:计划中必须包含"询问用户是否生成资产图片"步骤。若用户确认,执行层将调用相应工具批量生成衍生资产图片
|
||||||
- **生成分镜表后**:计划中必须包含"询问用户是否生成分镜图片"步骤。若用户确认,执行层将调用相应工具生成分镜图
|
- **生成分镜面板后**:计划中必须包含"询问用户是否生成分镜图片"步骤。若用户确认,执行层将调用相应工具生成分镜图
|
||||||
|
|||||||
@ -1,9 +1,9 @@
|
|||||||
# 分镜表生成(从剧本 + 资产 → storyboardTable)
|
# 分镜面板生成(从剧本 + 资产 → storyboardTable)
|
||||||
|
|
||||||
本指南只做一件事:
|
本指南只做一件事:
|
||||||
根据剧本内容和已有资产,将剧本拆分为一系列分镜,生成结构化的分镜表。
|
根据剧本内容和已有资产,将剧本拆分为一系列分镜,生成结构化的分镜面板。
|
||||||
|
|
||||||
> **核心概念**:分镜表是将剧本转化为视觉画面的中间产物。每条分镜对应一个独立的画面/镜头,包含画面描述、镜头语言、台词、音效和关联资产等信息,用于后续图片生成。
|
> **核心概念**:分镜面板是将剧本转化为视觉画面的中间产物。每条分镜对应一个独立的画面/镜头,包含画面描述、镜头语言、台词、音效和关联资产等信息,用于后续图片生成。
|
||||||
|
|
||||||
## 1. 输入与输出
|
## 1. 输入与输出
|
||||||
|
|
||||||
@ -14,7 +14,7 @@
|
|||||||
|
|
||||||
### 输出
|
### 输出
|
||||||
|
|
||||||
调用 `set_flowData` 将分镜表写入工作区:
|
调用 `set_flowData` 将分镜面板写入工作区:
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
set_flowData({
|
set_flowData({
|
||||||
@ -232,8 +232,8 @@ set_flowData({
|
|||||||
1. `get_flowData("script")` — 获取剧本内容
|
1. `get_flowData("script")` — 获取剧本内容
|
||||||
2. `get_flowData("assets")` — 获取已有资产列表
|
2. `get_flowData("assets")` — 获取已有资产列表
|
||||||
3. 分析剧本,按照拆分原则划分分镜,并为每条分镜填写所有字段
|
3. 分析剧本,按照拆分原则划分分镜,并为每条分镜填写所有字段
|
||||||
4. 调用 `set_flowData({ key: "storyboardTable", value: 分镜数组 })` 一次性保存完整分镜表
|
4. 调用 `set_flowData({ key: "storyboardTable", value: 分镜数组 })` 一次性保存完整分镜面板
|
||||||
5. 向用户汇报分镜表概要(总共多少条分镜,覆盖的场景概括)
|
5. 向用户汇报分镜面板概要(总共多少条分镜,覆盖的场景概括)
|
||||||
6. **询问用户是否需要生成分镜图片**:
|
6. **询问用户是否需要生成分镜图片**:
|
||||||
- 如果用户确认,调用 `generate_storyboard_images({ script: 剧本文本 })` 生成分镜图
|
- 如果用户确认,调用 `generate_storyboard_images({ script: 剧本文本 })` 生成分镜图
|
||||||
- 如果用户拒绝,跳过此步骤,流程结束
|
- 如果用户拒绝,跳过此步骤,流程结束
|
||||||
|
|||||||
@ -2,10 +2,12 @@ import { tool, Tool } from "ai";
|
|||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
import _ from "lodash";
|
import _ from "lodash";
|
||||||
import ResTool from "@/socket/resTool";
|
import ResTool from "@/socket/resTool";
|
||||||
|
import u from "@/utils";
|
||||||
|
import { useSkill } from "@/utils/agent/skillsTools";
|
||||||
|
import { urlToBase64 } from "@/utils/vm";
|
||||||
export const deriveAssetSchema = z.object({
|
export const deriveAssetSchema = z.object({
|
||||||
id: z.number().describe("衍生资产ID,如果新增则为空").optional(),
|
id: z.number().describe("衍生资产ID,如果新增则为空").optional(),
|
||||||
assetsId: z.string().describe("关联的资产ID"),
|
assetsId: z.number().describe("关联的资产ID"),
|
||||||
prompt: z.string().describe("生成提示词"),
|
prompt: z.string().describe("生成提示词"),
|
||||||
name: z.string().describe("衍生资产名称"),
|
name: z.string().describe("衍生资产名称"),
|
||||||
desc: z.string().describe("衍生资产描述"),
|
desc: z.string().describe("衍生资产描述"),
|
||||||
@ -14,14 +16,15 @@ export const deriveAssetSchema = z.object({
|
|||||||
type: z.enum(["role", "tool", "scene", "clip"]).describe("衍生资产类型"),
|
type: z.enum(["role", "tool", "scene", "clip"]).describe("衍生资产类型"),
|
||||||
});
|
});
|
||||||
export const assetItemSchema = z.object({
|
export const assetItemSchema = z.object({
|
||||||
assetsId: z.string().describe("资产唯一标识"),
|
id: z.number().describe("资产唯一标识"),
|
||||||
name: z.string().describe("资产名称"),
|
name: z.string().describe("资产名称"),
|
||||||
|
type: z.enum(["role", "tool", "scene", "clip"]).describe("资产类型"),
|
||||||
|
prompt: z.string().describe("生成提示词"),
|
||||||
desc: z.string().describe("资产描述"),
|
desc: z.string().describe("资产描述"),
|
||||||
src: z.string().describe("资产资源路径"),
|
|
||||||
derive: z.array(deriveAssetSchema).describe("衍生资产列表"),
|
derive: z.array(deriveAssetSchema).describe("衍生资产列表"),
|
||||||
});
|
});
|
||||||
export const storyboardSchema = z.object({
|
export const storyboardSchema = z.object({
|
||||||
id: z.number().describe("分镜ID"),
|
id: z.number().optional().describe("分镜ID,未从工作区获得的分镜列表视为需要新增;如需新增则为空"),
|
||||||
title: z.string().describe("分镜标题"),
|
title: z.string().describe("分镜标题"),
|
||||||
description: z.string().describe("分镜描述"),
|
description: z.string().describe("分镜描述"),
|
||||||
camera: z.string().describe("镜头信息"),
|
camera: z.string().describe("镜头信息"),
|
||||||
@ -49,7 +52,7 @@ export const flowDataSchema = z.object({
|
|||||||
script: z.string().describe("剧本内容"),
|
script: z.string().describe("剧本内容"),
|
||||||
scriptPlan: z.string().describe("拍摄计划"),
|
scriptPlan: z.string().describe("拍摄计划"),
|
||||||
assets: z.array(assetItemSchema).describe("衍生资产"),
|
assets: z.array(assetItemSchema).describe("衍生资产"),
|
||||||
storyboardTable: z.string().describe("分镜表"),
|
storyboardTable: z.string().describe("分镜面板"),
|
||||||
storyboard: z.array(storyboardSchema).describe("分镜列表"),
|
storyboard: z.array(storyboardSchema).describe("分镜列表"),
|
||||||
workbench: workbenchDataSchema.describe("工作台配置"),
|
workbench: workbenchDataSchema.describe("工作台配置"),
|
||||||
poster: z
|
poster: z
|
||||||
@ -101,22 +104,90 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
|
|||||||
return true;
|
return true;
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
|
// add_flowData_assets: tool({
|
||||||
|
// description: "新增对应衍生资产列表到工作区,严禁包含 不需要新增的数据",
|
||||||
|
// inputSchema: z.object({ value: z.array(deriveAssetSchema).describe("需要新增的资产列表") }),
|
||||||
|
// execute: async ({ value }) => {
|
||||||
|
// console.log("[tools] set_flowData add_flowData_assets", value);
|
||||||
|
// resTool.systemMessage("正在保存 衍生资产 数据");
|
||||||
|
// const addAssetsData = [];
|
||||||
|
// if (value && Array.isArray(value) && value.length) {
|
||||||
|
// for (const i of value) {
|
||||||
|
// const [insertedId] = await u.db("o_assets").insert({
|
||||||
|
// assetsId: +i.assetsId || null,
|
||||||
|
// projectId: resTool.data.projectId,
|
||||||
|
// name: i.name,
|
||||||
|
// type: i.type,
|
||||||
|
// prompt: i.prompt,
|
||||||
|
// describe: i.desc,
|
||||||
|
// startTime: Date.now(),
|
||||||
|
// });
|
||||||
|
// console.log("%c Line:141 🍑 resTool.data.scriptId", "background:#ea7e5c", resTool.data.scriptId);
|
||||||
|
// await u.db("o_scriptAssets").insert({
|
||||||
|
// scriptId: resTool.data.scriptId,
|
||||||
|
// assetId: insertedId,
|
||||||
|
// });
|
||||||
|
// addAssetsData.push({
|
||||||
|
// ...i,
|
||||||
|
// id: insertedId,
|
||||||
|
// });
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// socket.emit("setFlowData", { key: "addAssets", value: addAssetsData });
|
||||||
|
// return true;
|
||||||
|
// },
|
||||||
|
// }),
|
||||||
set_flowData_assets: tool({
|
set_flowData_assets: tool({
|
||||||
description: "保存衍生资产列表到工作区",
|
description: "保存衍生资产列表到工作区",
|
||||||
inputSchema: z.object({ value: flowDataSchema.shape.assets }),
|
inputSchema: z.object({ value: flowDataSchema.shape.assets }),
|
||||||
execute: async ({ value }) => {
|
execute: async ({ value }) => {
|
||||||
console.log("[tools] set_flowData assets", value);
|
console.log("[tools] set_flowData assets", value);
|
||||||
resTool.systemMessage("正在保存 衍生资产 数据");
|
resTool.systemMessage("正在保存 衍生资产 数据");
|
||||||
|
if (value && Array.isArray(value) && value.length) {
|
||||||
|
for (const i of value) {
|
||||||
|
if (!i?.id) {
|
||||||
|
const [insertedId] = await u.db("o_assets").insert({
|
||||||
|
assetsId: null,
|
||||||
|
name: i.name,
|
||||||
|
type: i.type,
|
||||||
|
prompt: i.prompt,
|
||||||
|
describe: i.desc,
|
||||||
|
startTime: Date.now(),
|
||||||
|
});
|
||||||
|
i.id = insertedId;
|
||||||
|
}
|
||||||
|
if (i.derive && Array.isArray(i.derive) && i.derive.length) {
|
||||||
|
for (const sub of i.derive) {
|
||||||
|
if (sub.id) continue;
|
||||||
|
const [insertedId] = await u.db("o_assets").insert({
|
||||||
|
assetsId: +i.id || null,
|
||||||
|
projectId: resTool.data.projectId,
|
||||||
|
name: sub.name,
|
||||||
|
type: sub.type,
|
||||||
|
prompt: sub.prompt,
|
||||||
|
describe: sub.desc,
|
||||||
|
startTime: Date.now(),
|
||||||
|
});
|
||||||
|
console.log("%c Line:141 🍑 resTool.data.scriptId", "background:#ea7e5c", resTool.data.scriptId);
|
||||||
|
await u.db("o_scriptAssets").insert({
|
||||||
|
scriptId: resTool.data.scriptId,
|
||||||
|
assetId: insertedId,
|
||||||
|
});
|
||||||
|
sub.id = insertedId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
socket.emit("setFlowData", { key: "assets", value });
|
socket.emit("setFlowData", { key: "assets", value });
|
||||||
return true;
|
return true;
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
set_flowData_storyboardTable: tool({
|
set_flowData_storyboardTable: tool({
|
||||||
description: "保存分镜表到工作区",
|
description: "保存分镜模板到工作区",
|
||||||
inputSchema: z.object({ value: flowDataSchema.shape.storyboardTable }),
|
inputSchema: z.object({ value: flowDataSchema.shape.storyboardTable }),
|
||||||
execute: async ({ value }) => {
|
execute: async ({ value }) => {
|
||||||
console.log("[tools] set_flowData storyboardTable", value);
|
console.log("[tools] set_flowData storyboardTable", value);
|
||||||
resTool.systemMessage("正在保存 分镜表 数据...");
|
resTool.systemMessage("正在保存 分镜面板 数据...");
|
||||||
socket.emit("setFlowData", { key: "storyboardTable", value });
|
socket.emit("setFlowData", { key: "storyboardTable", value });
|
||||||
return true;
|
return true;
|
||||||
},
|
},
|
||||||
@ -127,6 +198,26 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
|
|||||||
execute: async ({ value }) => {
|
execute: async ({ value }) => {
|
||||||
console.log("[tools] set_flowData storyboard", value);
|
console.log("[tools] set_flowData storyboard", value);
|
||||||
resTool.systemMessage("正在保存 分镜列表 数据...");
|
resTool.systemMessage("正在保存 分镜列表 数据...");
|
||||||
|
for (const item of value) {
|
||||||
|
if (!item.id) {
|
||||||
|
const [insertedId] = await u.db("o_storyboard").insert({
|
||||||
|
title: item.title,
|
||||||
|
prompt: item.prompt,
|
||||||
|
description: item.description,
|
||||||
|
filePath: item.src,
|
||||||
|
frameMode: item.frameMode,
|
||||||
|
duration: String(item.duration),
|
||||||
|
camera: item.camera,
|
||||||
|
sound: item.sound,
|
||||||
|
lines: item.lines,
|
||||||
|
state: "未生成",
|
||||||
|
});
|
||||||
|
if (item.associateAssetsIds.length) {
|
||||||
|
await u.db("o_assets2Storyboard").insert(item.associateAssetsIds.map((i) => ({ storyboardId: insertedId, assetId: i })));
|
||||||
|
}
|
||||||
|
item.id = insertedId;
|
||||||
|
}
|
||||||
|
}
|
||||||
socket.emit("setFlowData", { key: "storyboard", value });
|
socket.emit("setFlowData", { key: "storyboard", value });
|
||||||
return true;
|
return true;
|
||||||
},
|
},
|
||||||
@ -152,6 +243,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
|
|||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
|
|
||||||
|
//todo referenceIds 图片未使用 提示词待调
|
||||||
generate_storyboard_images: tool({
|
generate_storyboard_images: tool({
|
||||||
description: `生成一组图片任务,支持图片间的依赖关系(以图生图)。
|
description: `生成一组图片任务,支持图片间的依赖关系(以图生图)。
|
||||||
|
|
||||||
@ -175,7 +267,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
|
|||||||
inputSchema: z.object({
|
inputSchema: z.object({
|
||||||
images: z.array(
|
images: z.array(
|
||||||
z.object({
|
z.object({
|
||||||
id: z.string().describe("图片唯一标识符"),
|
id: z.number().describe("从工作区获取到的分镜id"),
|
||||||
prompt: z.string().describe("图片生成提示词"),
|
prompt: z.string().describe("图片生成提示词"),
|
||||||
referenceIds: z.array(z.string()).describe("依赖的参考图id数组,无依赖填空数组[]"),
|
referenceIds: z.array(z.string()).describe("依赖的参考图id数组,无依赖填空数组[]"),
|
||||||
assetIds: z.array(z.number()).optional().describe("参考的资产图"),
|
assetIds: z.array(z.number()).optional().describe("参考的资产图"),
|
||||||
@ -184,18 +276,131 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
|
|||||||
}),
|
}),
|
||||||
execute: async ({ images }) => {
|
execute: async ({ images }) => {
|
||||||
console.log("[tools] generated_assets", images);
|
console.log("[tools] generated_assets", images);
|
||||||
return new Promise((resolve) => socket.emit("generatedAssets", { images }, (res: any) => resolve(res)));
|
|
||||||
|
const skill = await useSkill("universal-agent");
|
||||||
|
for (const item of images) {
|
||||||
|
resTool.systemMessage(`生在生成分镜 id:${item.id} 图片`);
|
||||||
|
//更新对应分镜状态
|
||||||
|
await u.db("o_storyboard").where("id", item.id).update({ state: "生成中" });
|
||||||
|
// 异步生成
|
||||||
|
const imageModel = resTool.data.imageModel;
|
||||||
|
|
||||||
|
u.Ai.Image(imageModel?.modelId)
|
||||||
|
.run({
|
||||||
|
systemPrompt: skill.prompt,
|
||||||
|
prompt: item.prompt,
|
||||||
|
imageBase64: await getAssetsImageBase64(item.assetIds ?? []),
|
||||||
|
size: imageModel?.quality,
|
||||||
|
aspectRatio: imageModel?.ratio,
|
||||||
|
taskClass: "生成图片",
|
||||||
|
describe: "分镜图片生成",
|
||||||
|
relatedObjects: "hhhh",
|
||||||
|
projectId: resTool.data.projectId,
|
||||||
|
})
|
||||||
|
.then(async (imageCls) => {
|
||||||
|
const savePath = `/${resTool.data.projectId}/storyboard/${u.uuid()}.jpg`;
|
||||||
|
await imageCls.save(savePath);
|
||||||
|
const obj = {
|
||||||
|
...item,
|
||||||
|
id: item.id,
|
||||||
|
src: await u.oss.getFileUrl(savePath),
|
||||||
|
state: "已完成",
|
||||||
|
};
|
||||||
|
// 更新对应分镜状态
|
||||||
|
await u.db("o_storyboard").where("id", item.id).update({ state: "已完成", filePath: savePath });
|
||||||
|
// 前端对话框提示
|
||||||
|
resTool.systemMessage(`分镜 id:${item.id} 图片生成完成`);
|
||||||
|
// 更新前端界面展示
|
||||||
|
socket.emit("setFlowData", { key: "setStoryboardImage", value: obj });
|
||||||
|
});
|
||||||
|
//更新前端为生成中
|
||||||
|
socket.emit("setFlowData", { key: "setStoryboardImage", value: { ...item, id: item.id, src: "", state: "生成中" } });
|
||||||
|
}
|
||||||
|
return "分镜图片生成中";
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
|
|
||||||
|
//todo 图片是否需要参考 原资产 提示词待调
|
||||||
generate_assets_images: tool({
|
generate_assets_images: tool({
|
||||||
description: "生成分镜图",
|
description: `
|
||||||
|
生成 资产图片 不区分原资产于衍生资产
|
||||||
|
参数说明:
|
||||||
|
- images: 图片任务数组
|
||||||
|
- assetId: 资产id
|
||||||
|
- prompt: 图片生成提示词
|
||||||
|
示例:
|
||||||
|
images:[
|
||||||
|
{assetId: 1, prompt: "一张猫的图片"}
|
||||||
|
]
|
||||||
|
`,
|
||||||
inputSchema: z.object({ images: z.array(z.object({ assetId: z.number(), prompt: z.string() })) }),
|
inputSchema: z.object({ images: z.array(z.object({ assetId: z.number(), prompt: z.string() })) }),
|
||||||
execute: async ({ images }) => {
|
execute: async ({ images }) => {
|
||||||
|
const skill = await useSkill("universal-agent");
|
||||||
|
//获取所设置模型
|
||||||
|
const imageModel = resTool.data.imageModel;
|
||||||
|
for (const item of images) {
|
||||||
|
const [imageId] = await u.db("o_image").insert({
|
||||||
|
// 数据库插入图片记录
|
||||||
|
assetsId: item.assetId,
|
||||||
|
model: imageModel?.modelId,
|
||||||
|
state: "生成中",
|
||||||
|
resolution: imageModel?.quality,
|
||||||
|
});
|
||||||
|
u.Ai.Image(imageModel?.modelId)
|
||||||
|
.run({
|
||||||
|
systemPrompt: skill.prompt,
|
||||||
|
prompt: item.prompt,
|
||||||
|
imageBase64: [],
|
||||||
|
size: imageModel?.quality,
|
||||||
|
aspectRatio: imageModel?.ratio,
|
||||||
|
taskClass: "生成图片",
|
||||||
|
describe: "资产图片生成",
|
||||||
|
relatedObjects: "hhhh",
|
||||||
|
projectId: resTool.data.projectId,
|
||||||
|
})
|
||||||
|
.then(async (imageCls) => {
|
||||||
|
const savePath = `/${resTool.data.projectId}/assets/${u.uuid()}.jpg`;
|
||||||
|
await imageCls.save(savePath);
|
||||||
|
const obj = {
|
||||||
|
...item,
|
||||||
|
id: item.assetId,
|
||||||
|
src: await u.oss.getFileUrl(savePath),
|
||||||
|
state: "已完成",
|
||||||
|
};
|
||||||
|
//更新对应数据库
|
||||||
|
await u.db("o_assets").where("id", item.assetId).update({ imageId: imageId });
|
||||||
|
await u.db("o_image").where({ id: imageId }).update({ state: "已完成", filePath: savePath });
|
||||||
|
//通知前端更新
|
||||||
|
socket.emit("setFlowData", { key: "setAssetsImage", value: obj });
|
||||||
|
});
|
||||||
|
//通知前端更新状态
|
||||||
|
socket.emit("setFlowData", { key: "setAssetsImage", value: { ...item, id: item.assetId, src: "", state: "生成中" } });
|
||||||
|
}
|
||||||
console.log("[tools] generate_assets_images", images);
|
console.log("[tools] generate_assets_images", images);
|
||||||
return new Promise((resolve) => socket.emit("generateAssetsImages", { images }, (res: any) => resolve(res)));
|
return "资产生成中";
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
};
|
};
|
||||||
|
|
||||||
return toolsNames ? Object.fromEntries(Object.entries(tools).filter(([n]) => toolsNames.includes(n))) : tools;
|
return toolsNames ? Object.fromEntries(Object.entries(tools).filter(([n]) => toolsNames.includes(n))) : tools;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
async function getAssetsImageBase64(imageIds: number[]) {
|
||||||
|
if (imageIds.length === 0) return [];
|
||||||
|
const imagePaths = await u
|
||||||
|
.db("o_assets")
|
||||||
|
.leftJoin("o_image", "o_assets.imageId", "o_image.id")
|
||||||
|
.whereIn("o_assets.id", imageIds)
|
||||||
|
.select("o_assets.id", "o_image.filePath");
|
||||||
|
if (!imagePaths.length) return [];
|
||||||
|
const imageUrls = await Promise.all(
|
||||||
|
imagePaths.map(async (i) => {
|
||||||
|
if (i.filePath) {
|
||||||
|
return await urlToBase64(await u.oss.getFileUrl(i.filePath));
|
||||||
|
} else {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return imageUrls.filter(Boolean) as string[];
|
||||||
|
}
|
||||||
|
|||||||
@ -5,14 +5,11 @@ import _ from "lodash";
|
|||||||
import ResTool from "@/socket/resTool";
|
import ResTool from "@/socket/resTool";
|
||||||
|
|
||||||
export const AssetSchema = z.object({
|
export const AssetSchema = z.object({
|
||||||
id: z.number().describe("衍生资产ID,如果新增则为空").optional(),
|
id: z.number().describe("资产ID,如果新增则为空").optional(),
|
||||||
assetsId: z.string().describe("关联的资产ID"),
|
|
||||||
prompt: z.string().describe("生成提示词"),
|
prompt: z.string().describe("生成提示词"),
|
||||||
name: z.string().describe("衍生资产名称"),
|
name: z.string().describe("资产名称"),
|
||||||
desc: z.string().describe("衍生资产描述"),
|
desc: z.string().describe("资产描述"),
|
||||||
src: z.string().describe("衍生资产资源路径").optional(),
|
type: z.enum(["role", "tool", "scene", "clip"]).describe("资产类型"),
|
||||||
state: z.enum(["未生成", "生成中", "已完成", "生成失败"]).describe("衍生资产生成状态,新增默认未生成"),
|
|
||||||
type: z.enum(["role", "tool", "scene", "clip"]).describe("衍生资产类型"),
|
|
||||||
});
|
});
|
||||||
export const ScriptSchema = z.object({
|
export const ScriptSchema = z.object({
|
||||||
id: z.number().describe("剧本ID,如果新增则为空").optional(),
|
id: z.number().describe("剧本ID,如果新增则为空").optional(),
|
||||||
@ -97,7 +94,7 @@ export default (resTool: ResTool, toolsNames?: string[]) => {
|
|||||||
description: "将剧本内容插入sqlite数据库,供后续业务使用",
|
description: "将剧本内容插入sqlite数据库,供后续业务使用",
|
||||||
inputSchema: z.object({
|
inputSchema: z.object({
|
||||||
script: ScriptSchema,
|
script: ScriptSchema,
|
||||||
assetsList: z.array(AssetSchema).describe("剧本所使用资产列表"),
|
assetsList: z.array(AssetSchema).describe("剧本所使用资产列表,注意不要包含剧本内容,仅为所使用到的 道具、人物、场景、素材"),
|
||||||
}),
|
}),
|
||||||
execute: async ({ assetsList, script }) => {
|
execute: async ({ assetsList, script }) => {
|
||||||
console.log("%c Line:103 🍷 script", "background:#42b983", script);
|
console.log("%c Line:103 🍷 script", "background:#42b983", script);
|
||||||
|
|||||||
@ -305,6 +305,9 @@ export default async (knex: Knex, forceInit: boolean = false): Promise<void> =>
|
|||||||
table.text("frameMode");
|
table.text("frameMode");
|
||||||
table.text("camera");
|
table.text("camera");
|
||||||
table.text("sound");
|
table.text("sound");
|
||||||
|
table.text("lines");
|
||||||
|
table.text("state");
|
||||||
|
table.text("reason");
|
||||||
table.integer("createTime");
|
table.integer("createTime");
|
||||||
table.primary(["id"]);
|
table.primary(["id"]);
|
||||||
table.unique(["id"]);
|
table.unique(["id"]);
|
||||||
|
|||||||
@ -109,7 +109,7 @@ export default router.post(
|
|||||||
if (tableName === 't_outline') { }
|
if (tableName === 't_outline') { }
|
||||||
//迁移脚本表
|
//迁移脚本表
|
||||||
if (tableName === 't_script') { }
|
if (tableName === 't_script') { }
|
||||||
//迁移分镜表
|
//迁移分镜面板
|
||||||
if (tableName === 't_storyboard') { }
|
if (tableName === 't_storyboard') { }
|
||||||
//迁移视频表
|
//迁移视频表
|
||||||
if (tableName === 't_video') { }
|
if (tableName === 't_video') { }
|
||||||
|
|||||||
@ -21,19 +21,24 @@ export default router.post(
|
|||||||
.select("data")
|
.select("data")
|
||||||
.first();
|
.first();
|
||||||
|
|
||||||
const scriptData = await u.db("o_script").where("projectId", projectId).first();
|
const scriptData = await u.db("o_script").where("projectId", projectId).where("id", episodesId).first();
|
||||||
|
const scriptAssets = await u.db("o_scriptAssets").where("scriptId", episodesId);
|
||||||
|
const assetIds = scriptAssets.map((i) => i.assetId);
|
||||||
const assetsData = await u
|
const assetsData = await u
|
||||||
.db("o_assets")
|
.db("o_assets")
|
||||||
.leftJoin("o_image", "o_assets.imageId", "o_image.id")
|
.leftJoin("o_image", "o_assets.imageId", "o_image.id")
|
||||||
.select("o_assets.*", "o_image.filePath")
|
.select("o_assets.*", "o_image.filePath")
|
||||||
|
.where("o_assets.id", "in", assetIds)
|
||||||
|
.whereNull("o_assets.assetsId")
|
||||||
.where("o_assets.projectId", projectId);
|
.where("o_assets.projectId", projectId);
|
||||||
let childAssetsData = await u
|
let childAssetsData = await u
|
||||||
.db("o_assets")
|
.db("o_assets")
|
||||||
.leftJoin("o_image", "o_assets.imageId", "o_image.id")
|
.leftJoin("o_image", "o_assets.imageId", "o_image.id")
|
||||||
.select("o_assets.*", "o_image.filePath")
|
.select("o_assets.*", "o_image.filePath")
|
||||||
.where("o_assets.projectId", projectId)
|
.where("o_assets.projectId", projectId)
|
||||||
|
.where("o_assets.id", "in", assetIds)
|
||||||
.whereNotNull("o_assets.assetsId");
|
.whereNotNull("o_assets.assetsId");
|
||||||
|
console.log("%c Line:35 🥚 childAssetsData", "background:#f5ce50", childAssetsData);
|
||||||
|
|
||||||
if (!sqlData) {
|
if (!sqlData) {
|
||||||
const flowData: FlowData = {
|
const flowData: FlowData = {
|
||||||
@ -41,8 +46,10 @@ export default router.post(
|
|||||||
scriptPlan: "",
|
scriptPlan: "",
|
||||||
assets: await Promise.all(
|
assets: await Promise.all(
|
||||||
assetsData.map(async (item) => ({
|
assetsData.map(async (item) => ({
|
||||||
assetsId: item.id,
|
id: item.id,
|
||||||
name: item.name ?? "",
|
name: item.name ?? "",
|
||||||
|
type: item.type ?? "",
|
||||||
|
prompt: item.prompt ?? "",
|
||||||
desc: item.describe ?? "",
|
desc: item.describe ?? "",
|
||||||
src: item.filePath && (await u.oss.getFileUrl(item.filePath!)),
|
src: item.filePath && (await u.oss.getFileUrl(item.filePath!)),
|
||||||
derive: await Promise.all(
|
derive: await Promise.all(
|
||||||
@ -52,6 +59,8 @@ export default router.post(
|
|||||||
id: child.id,
|
id: child.id,
|
||||||
assetsId: item.id,
|
assetsId: item.id,
|
||||||
name: child.name ?? "",
|
name: child.name ?? "",
|
||||||
|
type: child.type,
|
||||||
|
prompt: child.prompt,
|
||||||
desc: child.describe ?? "",
|
desc: child.describe ?? "",
|
||||||
src: child.filePath && (await u.oss.getFileUrl(child.filePath!)),
|
src: child.filePath && (await u.oss.getFileUrl(child.filePath!)),
|
||||||
state: child.state ?? "未生成", //todo:矫正状态值
|
state: child.state ?? "未生成", //todo:矫正状态值
|
||||||
@ -78,6 +87,30 @@ export default router.post(
|
|||||||
} else {
|
} else {
|
||||||
try {
|
try {
|
||||||
const flowData = JSON.parse(sqlData!.data ?? "{}");
|
const flowData = JSON.parse(sqlData!.data ?? "{}");
|
||||||
|
flowData.assets = await Promise.all(
|
||||||
|
assetsData.map(async (item) => ({
|
||||||
|
id: item.id,
|
||||||
|
name: item.name ?? "",
|
||||||
|
type: item.type ?? "",
|
||||||
|
prompt: item.prompt ?? "",
|
||||||
|
desc: item.describe ?? "",
|
||||||
|
src: item.filePath && (await u.oss.getFileUrl(item.filePath!)),
|
||||||
|
derive: await Promise.all(
|
||||||
|
childAssetsData
|
||||||
|
.filter((child) => child.assetsId === item.id)
|
||||||
|
.map(async (child) => ({
|
||||||
|
id: child.id,
|
||||||
|
assetsId: item.id,
|
||||||
|
name: child.name ?? "",
|
||||||
|
prompt: child.prompt,
|
||||||
|
type: child.type,
|
||||||
|
desc: child.describe ?? "",
|
||||||
|
src: child.filePath && (await u.oss.getFileUrl(child.filePath!)),
|
||||||
|
state: child.state ?? "未生成", //todo:矫正状态值
|
||||||
|
})),
|
||||||
|
),
|
||||||
|
})),
|
||||||
|
);
|
||||||
res.status(200).send(success(flowData));
|
res.status(200).send(success(flowData));
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
res.status(200).send(error());
|
res.status(200).send(error());
|
||||||
|
|||||||
@ -35,6 +35,7 @@ async function getLines(prompt: string) {
|
|||||||
const resText = await u.Ai.Text("universalAgent").invoke({
|
const resText = await u.Ai.Text("universalAgent").invoke({
|
||||||
system: skill.prompt,
|
system: skill.prompt,
|
||||||
messages: [{ role: "user", content: prompt }],
|
messages: [{ role: "user", content: prompt }],
|
||||||
|
tools: skill.tools,
|
||||||
output: Output.array({
|
output: Output.array({
|
||||||
element: z.object({
|
element: z.object({
|
||||||
lines: z.string().describe("台词内容"),
|
lines: z.string().describe("台词内容"),
|
||||||
|
|||||||
@ -26,7 +26,7 @@ export default (nsp: Namespace) => {
|
|||||||
socket.disconnect();
|
socket.disconnect();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const isolationKey = socket.handshake.auth.isolationKey;
|
let isolationKey = socket.handshake.auth.isolationKey;
|
||||||
if (!isolationKey) {
|
if (!isolationKey) {
|
||||||
console.log("[productionAgent] 连接失败,缺少 isolationKey");
|
console.log("[productionAgent] 连接失败,缺少 isolationKey");
|
||||||
socket.disconnect();
|
socket.disconnect();
|
||||||
@ -35,13 +35,17 @@ export default (nsp: Namespace) => {
|
|||||||
|
|
||||||
console.log("[productionAgent] 已连接:", socket.id);
|
console.log("[productionAgent] 已连接:", socket.id);
|
||||||
|
|
||||||
const resTool = new ResTool(socket);
|
const resTool = new ResTool(socket, {
|
||||||
|
projectId: socket.handshake.auth.projectId,
|
||||||
|
scriptId: socket.handshake.auth.scriptId,
|
||||||
|
});
|
||||||
let abortController: AbortController | null = null;
|
let abortController: AbortController | null = null;
|
||||||
|
|
||||||
socket.on("message", async (text: string) => {
|
socket.on("message", async (text: string) => {
|
||||||
abortController?.abort();
|
abortController?.abort();
|
||||||
abortController = new AbortController();
|
abortController = new AbortController();
|
||||||
const currentController = abortController;
|
const currentController = abortController;
|
||||||
|
console.log("%c Line:30 🍑 isolationKey", "background:#e41a6a", isolationKey);
|
||||||
|
|
||||||
const textStream = await agent.decisionAI({ socket, isolationKey, text, abortSignal: currentController.signal, resTool });
|
const textStream = await agent.decisionAI({ socket, isolationKey, text, abortSignal: currentController.signal, resTool });
|
||||||
|
|
||||||
@ -60,7 +64,13 @@ export default (nsp: Namespace) => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
socket.on("setModelData", async (data: any) => {
|
||||||
|
resTool.data.imageModel = data;
|
||||||
|
});
|
||||||
|
socket.on("setKeyScript", async (data: any) => {
|
||||||
|
isolationKey = data.key;
|
||||||
|
resTool.data.scriptId = data.scriptId;
|
||||||
|
});
|
||||||
socket.on("stop", () => {
|
socket.on("stop", () => {
|
||||||
abortController?.abort();
|
abortController?.abort();
|
||||||
abortController = null;
|
abortController = null;
|
||||||
|
|||||||
31
src/types/database.d.ts
vendored
31
src/types/database.d.ts
vendored
@ -1,6 +1,24 @@
|
|||||||
// @db-hash f6a9a8164252ce954394431079615459
|
// @db-hash 1ce1a8f10cb90caac306536b78942cb3
|
||||||
//该文件由脚本自动生成,请勿手动修改
|
//该文件由脚本自动生成,请勿手动修改
|
||||||
|
|
||||||
|
export interface _o_storyboard_old_20260324 {
|
||||||
|
'camera'?: string | null;
|
||||||
|
'createTime'?: number | null;
|
||||||
|
'description'?: string | null;
|
||||||
|
'duration'?: string | null;
|
||||||
|
'filePath'?: string | null;
|
||||||
|
'frameMode'?: string | null;
|
||||||
|
'id'?: number;
|
||||||
|
'mode'?: string | null;
|
||||||
|
'model'?: string | null;
|
||||||
|
'prompt'?: string | null;
|
||||||
|
'reason'?: string | null;
|
||||||
|
'resolution'?: string | null;
|
||||||
|
'scriptId'?: number | null;
|
||||||
|
'sound'?: string | null;
|
||||||
|
'state'?: string | null;
|
||||||
|
'title'?: string | null;
|
||||||
|
}
|
||||||
export interface memories {
|
export interface memories {
|
||||||
'content': string;
|
'content': string;
|
||||||
'createTime': number;
|
'createTime': number;
|
||||||
@ -32,11 +50,14 @@ export interface o_agentWorkData {
|
|||||||
'updateTime'?: number | null;
|
'updateTime'?: number | null;
|
||||||
}
|
}
|
||||||
export interface o_artStyle {
|
export interface o_artStyle {
|
||||||
|
'fileUrl'?: string | null;
|
||||||
'id'?: number;
|
'id'?: number;
|
||||||
|
'label'?: string | null;
|
||||||
'name'?: string | null;
|
'name'?: string | null;
|
||||||
'styles'?: string | null;
|
'prompt'?: string | null;
|
||||||
}
|
}
|
||||||
export interface o_assets {
|
export interface o_assets {
|
||||||
|
'assetsId'?: number | null;
|
||||||
'describe'?: string | null;
|
'describe'?: string | null;
|
||||||
'id'?: number;
|
'id'?: number;
|
||||||
'imageId'?: number | null;
|
'imageId'?: number | null;
|
||||||
@ -45,9 +66,7 @@ export interface o_assets {
|
|||||||
'prompt'?: string | null;
|
'prompt'?: string | null;
|
||||||
'remark'?: string | null;
|
'remark'?: string | null;
|
||||||
'scriptId'?: number | null;
|
'scriptId'?: number | null;
|
||||||
'sonId'?: number | null;
|
|
||||||
'startTime'?: number | null;
|
'startTime'?: number | null;
|
||||||
'state'?: string | null;
|
|
||||||
'type'?: string | null;
|
'type'?: string | null;
|
||||||
}
|
}
|
||||||
export interface o_assets2Storyboard {
|
export interface o_assets2Storyboard {
|
||||||
@ -131,12 +150,15 @@ export interface o_storyboard {
|
|||||||
'filePath'?: string | null;
|
'filePath'?: string | null;
|
||||||
'frameMode'?: string | null;
|
'frameMode'?: string | null;
|
||||||
'id'?: number;
|
'id'?: number;
|
||||||
|
'lines'?: string | null;
|
||||||
'mode'?: string | null;
|
'mode'?: string | null;
|
||||||
'model'?: string | null;
|
'model'?: string | null;
|
||||||
'prompt'?: string | null;
|
'prompt'?: string | null;
|
||||||
|
'reason'?: string | null;
|
||||||
'resolution'?: string | null;
|
'resolution'?: string | null;
|
||||||
'scriptId'?: number | null;
|
'scriptId'?: number | null;
|
||||||
'sound'?: string | null;
|
'sound'?: string | null;
|
||||||
|
'state'?: string | null;
|
||||||
'title'?: string | null;
|
'title'?: string | null;
|
||||||
}
|
}
|
||||||
export interface o_storyboardFlow {
|
export interface o_storyboardFlow {
|
||||||
@ -196,6 +218,7 @@ export interface o_videoConfig {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface DB {
|
export interface DB {
|
||||||
|
"_o_storyboard_old_20260324": _o_storyboard_old_20260324;
|
||||||
"memories": memories;
|
"memories": memories;
|
||||||
"o_agentDeploy": o_agentDeploy;
|
"o_agentDeploy": o_agentDeploy;
|
||||||
"o_agentWorkData": o_agentWorkData;
|
"o_agentWorkData": o_agentWorkData;
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user