Merge branch '108' of https://github.com/HBAI-Ltd/Toonflow-app into 108
# Conflicts: # src/types/database.d.ts
This commit is contained in:
commit
be5420a3c3
@ -98,14 +98,19 @@ function createSubAgent(parentCtx: AgentContext) {
|
|||||||
tools: { ...extraTools, ...useTools({ resTool, msg: subMsg }) },
|
tools: { ...extraTools, ...useTools({ resTool, msg: subMsg }) },
|
||||||
});
|
});
|
||||||
|
|
||||||
for await (const chunk of textStream) {
|
try {
|
||||||
text.append(chunk);
|
for await (const chunk of textStream) {
|
||||||
fullResponse += chunk;
|
text.append(chunk);
|
||||||
|
fullResponse += chunk;
|
||||||
|
}
|
||||||
|
text.complete();
|
||||||
|
subMsg.complete();
|
||||||
|
} catch (err: any) {
|
||||||
|
text.complete();
|
||||||
|
subMsg.stop();
|
||||||
|
throw err;
|
||||||
}
|
}
|
||||||
|
|
||||||
text.complete();
|
|
||||||
subMsg.complete();
|
|
||||||
|
|
||||||
if (fullResponse.trim()) {
|
if (fullResponse.trim()) {
|
||||||
await memory.add(memoryKey, fullResponse, {
|
await memory.add(memoryKey, fullResponse, {
|
||||||
name,
|
name,
|
||||||
@ -130,7 +135,7 @@ function createSubAgent(parentCtx: AgentContext) {
|
|||||||
const addPrompt =
|
const addPrompt =
|
||||||
"\n" +
|
"\n" +
|
||||||
[
|
[
|
||||||
"你可以使用如下XML格式写入工作区:\n```",
|
"你必须使用如下XML格式写入工作区:\n```",
|
||||||
"拍摄计划:<scriptPlan>内容</scriptPlan>",
|
"拍摄计划:<scriptPlan>内容</scriptPlan>",
|
||||||
"分镜表:<storyboardTable>内容</storyboardTable>",
|
"分镜表:<storyboardTable>内容</storyboardTable>",
|
||||||
"```",
|
"```",
|
||||||
@ -163,7 +168,7 @@ function createSubAgent(parentCtx: AgentContext) {
|
|||||||
const systemPrompt = await fs.promises.readFile(skill, "utf-8");
|
const systemPrompt = await fs.promises.readFile(skill, "utf-8");
|
||||||
return runAgent({
|
return runAgent({
|
||||||
prompt,
|
prompt,
|
||||||
system: systemPrompt + "你可以使用如下XML格式写入工作区:\n<storySkeleton>故事骨架内容</storySkeleton>",
|
system: systemPrompt + "你必须使用如下XML格式写入工作区:\n<storySkeleton>故事骨架内容</storySkeleton>",
|
||||||
name: "监制",
|
name: "监制",
|
||||||
memoryKey: "assistant:supervision",
|
memoryKey: "assistant:supervision",
|
||||||
});
|
});
|
||||||
|
|||||||
@ -108,14 +108,19 @@ function createSubAgent(parentCtx: AgentContext) {
|
|||||||
tools: { ...extraTools, ...useTools({ resTool, msg: subMsg }) },
|
tools: { ...extraTools, ...useTools({ resTool, msg: subMsg }) },
|
||||||
});
|
});
|
||||||
|
|
||||||
for await (const chunk of textStream) {
|
try {
|
||||||
text.append(chunk);
|
for await (const chunk of textStream) {
|
||||||
fullResponse += chunk;
|
text.append(chunk);
|
||||||
|
fullResponse += chunk;
|
||||||
|
}
|
||||||
|
text.complete();
|
||||||
|
subMsg.complete();
|
||||||
|
} catch (err: any) {
|
||||||
|
text.complete();
|
||||||
|
subMsg.stop();
|
||||||
|
throw err;
|
||||||
}
|
}
|
||||||
|
|
||||||
text.complete();
|
|
||||||
subMsg.complete();
|
|
||||||
|
|
||||||
if (fullResponse.trim()) {
|
if (fullResponse.trim()) {
|
||||||
await memory.add(memoryKey, fullResponse, {
|
await memory.add(memoryKey, fullResponse, {
|
||||||
name,
|
name,
|
||||||
@ -139,7 +144,7 @@ function createSubAgent(parentCtx: AgentContext) {
|
|||||||
const systemPrompt = await fs.promises.readFile(skill, "utf-8");
|
const systemPrompt = await fs.promises.readFile(skill, "utf-8");
|
||||||
return runAgent({
|
return runAgent({
|
||||||
prompt,
|
prompt,
|
||||||
system: systemPrompt + "\n你可以使用如下XML格式写入工作区:\n<storySkeleton>故事骨架内容</storySkeleton>",
|
system: systemPrompt + "\n你必须使用如下XML格式写入工作区:\n<storySkeleton>故事骨架内容</storySkeleton>",
|
||||||
name: "编剧",
|
name: "编剧",
|
||||||
memoryKey: "assistant:execution:storySkeleton",
|
memoryKey: "assistant:execution:storySkeleton",
|
||||||
});
|
});
|
||||||
@ -154,7 +159,7 @@ function createSubAgent(parentCtx: AgentContext) {
|
|||||||
const systemPrompt = await fs.promises.readFile(skill, "utf-8");
|
const systemPrompt = await fs.promises.readFile(skill, "utf-8");
|
||||||
return runAgent({
|
return runAgent({
|
||||||
prompt,
|
prompt,
|
||||||
system: systemPrompt + "\n你可以使用如下XML格式写入工作区:\n<adaptationStrategy>改编策略内容</adaptationStrategy>",
|
system: systemPrompt + "\n你必须使用如下XML格式写入工作区:\n<adaptationStrategy>改编策略内容</adaptationStrategy>",
|
||||||
name: "编剧",
|
name: "编剧",
|
||||||
memoryKey: "assistant:execution:adaptationStrategy",
|
memoryKey: "assistant:execution:adaptationStrategy",
|
||||||
});
|
});
|
||||||
@ -171,7 +176,7 @@ function createSubAgent(parentCtx: AgentContext) {
|
|||||||
prompt,
|
prompt,
|
||||||
system:
|
system:
|
||||||
systemPrompt +
|
systemPrompt +
|
||||||
`\n你可以使用如下XML格式写入工作区:\nXML不得添加任何额外标签<script><item name="剧本名称">剧本内容</item><item name="剧本名称">剧本内容</item><item name="剧本名称">剧本内容</item></script>`,
|
`\n你必须使用如下XML格式写入工作区:\nXML不得添加任何额外标签<script><item name="剧本名称">剧本内容</item><item name="剧本名称">剧本内容</item><item name="剧本名称">剧本内容</item></script>`,
|
||||||
name: "编剧",
|
name: "编剧",
|
||||||
memoryKey: "assistant:execution:script",
|
memoryKey: "assistant:execution:script",
|
||||||
});
|
});
|
||||||
|
|||||||
@ -263,96 +263,95 @@ export default async (knex: Knex, forceInit: boolean = false): Promise<void> =>
|
|||||||
{
|
{
|
||||||
name: "剧本资产提取",
|
name: "剧本资产提取",
|
||||||
type: "scriptAssetExtraction",
|
type: "scriptAssetExtraction",
|
||||||
data: `
|
data: `---
|
||||||
---
|
name: universal_agent
|
||||||
name: universal_agent
|
description: 专注于从剧本内容中提取所使用的资产(角色、场景、道具)并生成结构化资产列表的助手。
|
||||||
description: 专注于从剧本内容中提取所使用的资产(角色、场景、道具)并生成结构化资产列表的助手。
|
---
|
||||||
---
|
|
||||||
|
# Script Assets Extract
|
||||||
# Script Assets Extract
|
|
||||||
|
你是一个专业的剧本内容分析助手,专注于从剧本文本中识别和提取所有涉及的资产(角色、场景、道具),并为每项资产生成可供下游制作流程使用的结构化描述和提示词。
|
||||||
你是一个专业的剧本内容分析助手,专注于从剧本文本中识别和提取所有涉及的资产(角色、场景、道具),并为每项资产生成可供下游制作流程使用的结构化描述和提示词。
|
|
||||||
|
## 何时使用
|
||||||
## 何时使用
|
|
||||||
|
用户提供剧本内容,你需要逐段阅读并提取其中涉及的所有资产(人物角色、场景地点、道具物件),输出为结构化的资产列表。产出的资产描述将用于后续 AI 图片生成和制作流程。
|
||||||
用户提供剧本内容,你需要逐段阅读并提取其中涉及的所有资产(人物角色、场景地点、道具物件),输出为结构化的资产列表。产出的资产描述将用于后续 AI 图片生成和制作流程。
|
|
||||||
|
## 与系统的对应关系
|
||||||
## 与系统的对应关系
|
|
||||||
|
- 资产类型:
|
||||||
- 资产类型:
|
- \`role\` — 角色(对应 \`o_assets.type = "role"\`)
|
||||||
- \`role\` — 角色(对应 \`o_assets.type = "role"\`)
|
- \`scene\` — 场景(对应 \`o_assets.type = "scene"\`)
|
||||||
- \`scene\` — 场景(对应 \`o_assets.type = "scene"\`)
|
- \`tool\` — 道具(对应 \`o_assets.type = "tool"\`)
|
||||||
- \`tool\` — 道具(对应 \`o_assets.type = "tool"\`)
|
- 下游用途:资产提示词生成 → AI 资产图生成 → 分镜制作
|
||||||
- 下游用途:资产提示词生成 → AI 资产图生成 → 分镜制作
|
|
||||||
|
## 输出要求
|
||||||
## 输出要求
|
|
||||||
|
**必须通过调用 \`resultTool\` 工具返回结果**,禁止以纯文本、Markdown 表格或 JSON 代码块等形式直接输出资产列表。
|
||||||
**必须通过调用 \`resultTool\` 工具返回结果**,禁止以纯文本、Markdown 表格或 JSON 代码块等形式直接输出资产列表。
|
\`resultTool\` 的 schema 会对字段类型和枚举值做强校验,调用时请严格按照下方字段定义填写,确保数据结构正确、字段完整、类型匹配。
|
||||||
\`resultTool\` 的 schema 会对字段类型和枚举值做强校验,调用时请严格按照下方字段定义填写,确保数据结构正确、字段完整、类型匹配。
|
|
||||||
|
每个资产对象包含以下字段:
|
||||||
每个资产对象包含以下字段:
|
|
||||||
|
| 字段 | 类型 | 必填 | 说明 |
|
||||||
| 字段 | 类型 | 必填 | 说明 |
|
| ---- | ---- | ---- | ---- |
|
||||||
| ---- | ---- | ---- | ---- |
|
| \`name\` | string | 是 | 资产名称,使用剧本中的原始称呼,不做其他多余描述 |
|
||||||
| \`name\` | string | 是 | 资产名称,使用剧本中的原始称呼,不做其他多余描述 |
|
| \`desc\` | string | 是 | 资产描述,30-80 字的视觉化描述 |
|
||||||
| \`desc\` | string | 是 | 资产描述,30-80 字的视觉化描述 |
|
| \`prompt\` | string | 是 | 生成提示词,英文,用于 AI 图片生成 |
|
||||||
| \`prompt\` | string | 是 | 生成提示词,英文,用于 AI 图片生成 |
|
| \`type\` | enum | 是 | 资产类型:\`role\` / \`scene\` / \`tool\` |
|
||||||
| \`type\` | enum | 是 | 资产类型:\`role\` / \`scene\` / \`tool\` |
|
|
||||||
|
## 提取规则
|
||||||
## 提取规则
|
|
||||||
|
### 角色(role)
|
||||||
### 角色(role)
|
|
||||||
|
- 提取剧本中出现的所有有名字的角色
|
||||||
- 提取剧本中出现的所有有名字的角色
|
- \`desc\`:包含外貌特征、服饰风格、体态气质等视觉要素
|
||||||
- \`desc\`:包含外貌特征、服饰风格、体态气质等视觉要素
|
- \`prompt\`:英文提示词,描述角色的外观特征,适用于 AI 角色图生成
|
||||||
- \`prompt\`:英文提示词,描述角色的外观特征,适用于 AI 角色图生成
|
- 同一角色有多个称呼时,取最常用的作为 \`name\`
|
||||||
- 同一角色有多个称呼时,取最常用的作为 \`name\`
|
- 无名龙套(如"路人甲"、"士兵")可跳过,除非其造型对剧情有重要视觉意义
|
||||||
- 无名龙套(如"路人甲"、"士兵")可跳过,除非其造型对剧情有重要视觉意义
|
|
||||||
|
### 场景(scene)
|
||||||
### 场景(scene)
|
|
||||||
|
- 提取剧本中出现的所有场景/地点
|
||||||
- 提取剧本中出现的所有场景/地点
|
- \`desc\`:包含空间结构、光照氛围、关键陈设、色调基调等视觉要素
|
||||||
- \`desc\`:包含空间结构、光照氛围、关键陈设、色调基调等视觉要素
|
- \`prompt\`:英文提示词,描述场景的整体视觉风格,适用于 AI 场景图生成
|
||||||
- \`prompt\`:英文提示词,描述场景的整体视觉风格,适用于 AI 场景图生成
|
- 同一场景的不同状态(如白天/夜晚)不重复提取,在 \`desc\` 中注明即可
|
||||||
- 同一场景的不同状态(如白天/夜晚)不重复提取,在 \`desc\` 中注明即可
|
|
||||||
|
### 道具(tool)
|
||||||
### 道具(tool)
|
|
||||||
|
- 提取剧本中出现的重要道具/物品
|
||||||
- 提取剧本中出现的重要道具/物品
|
- \`desc\`:包含外观形状、颜色材质、尺寸参考、特殊效果等视觉要素
|
||||||
- \`desc\`:包含外观形状、颜色材质、尺寸参考、特殊效果等视觉要素
|
- \`prompt\`:英文提示词,描述道具的外观细节,适用于 AI 道具图生成
|
||||||
- \`prompt\`:英文提示词,描述道具的外观细节,适用于 AI 道具图生成
|
- 仅提取有独立视觉意义或剧情功能的道具,通用物品可跳过
|
||||||
- 仅提取有独立视觉意义或剧情功能的道具,通用物品可跳过
|
|
||||||
|
|
||||||
|
## 提示词(prompt)生成规范
|
||||||
## 提示词(prompt)生成规范
|
|
||||||
|
- 采用逗号分隔的关键词/短语格式
|
||||||
- 采用逗号分隔的关键词/短语格式
|
- 优先描述**视觉特征**,避免抽象概念
|
||||||
- 优先描述**视觉特征**,避免抽象概念
|
- 包含风格关键词(如 anime style, manga style 等,根据项目风格决定)
|
||||||
- 包含风格关键词(如 anime style, manga style 等,根据项目风格决定)
|
- 角色 prompt 示例:\`a young man, sharp eyebrows, black hair, pale skin, wearing a gray Taoist robe, slender build, cold expression\`
|
||||||
- 角色 prompt 示例:\`a young man, sharp eyebrows, black hair, pale skin, wearing a gray Taoist robe, slender build, cold expression\`
|
- 场景 prompt 示例:\`dark cave interior, glowing crystals on walls, misty atmosphere, dim blue lighting, stone altar in center\`
|
||||||
- 场景 prompt 示例:\`dark cave interior, glowing crystals on walls, misty atmosphere, dim blue lighting, stone altar in center\`
|
- 道具 prompt 示例:\`ancient jade pendant, oval shape, translucent green, carved dragon pattern, glowing faintly\`
|
||||||
- 道具 prompt 示例:\`ancient jade pendant, oval shape, translucent green, carved dragon pattern, glowing faintly\`
|
|
||||||
|
## 提取流程
|
||||||
## 提取流程
|
|
||||||
|
1. 通读剧本全文,识别所有出现的角色、场景、道具
|
||||||
1. 通读剧本全文,识别所有出现的角色、场景、道具
|
2. 对每个资产生成结构化的 \`name\`、\`desc\`、\`prompt\`、\`type\`
|
||||||
2. 对每个资产生成结构化的 \`name\`、\`desc\`、\`prompt\`、\`type\`
|
3. 去重:同一资产不重复提取
|
||||||
3. 去重:同一资产不重复提取
|
4. **必须通过调用 \`resultTool\` 工具输出完整资产列表**,不要分多次调用,一次性将所有资产放入 \`assetsList\` 数组中提交
|
||||||
4. **必须通过调用 \`resultTool\` 工具输出完整资产列表**,不要分多次调用,一次性将所有资产放入 \`assetsList\` 数组中提交
|
|
||||||
|
## 提取原则
|
||||||
## 提取原则
|
|
||||||
|
1. **忠于剧本**:所有提取基于剧本中的实际内容,不臆造未出现的资产
|
||||||
1. **忠于剧本**:所有提取基于剧本中的实际内容,不臆造未出现的资产
|
2. **视觉优先**:描述和提示词聚焦视觉特征,便于 AI 图片生成
|
||||||
2. **视觉优先**:描述和提示词聚焦视觉特征,便于 AI 图片生成
|
3. **精简实用**:只提取对制作有实际意义的资产,避免过度提取
|
||||||
3. **精简实用**:只提取对制作有实际意义的资产,避免过度提取
|
4. **分类准确**:严格按照 role/scene/tool 分类,不混淆
|
||||||
4. **分类准确**:严格按照 role/scene/tool 分类,不混淆
|
5. **提示词质量**:英文提示词应具体、可执行,能直接用于 AI 图片生成
|
||||||
5. **提示词质量**:英文提示词应具体、可执行,能直接用于 AI 图片生成
|
|
||||||
|
## 注意事项
|
||||||
## 注意事项
|
|
||||||
|
- 资产列表中**不要包含剧本内容本身**,仅提取所使用到的资产
|
||||||
- 资产列表中**不要包含剧本内容本身**,仅提取所使用到的资产
|
- 角色的随身物品如果有独立剧情功能,应单独作为道具提取
|
||||||
- 角色的随身物品如果有独立剧情功能,应单独作为道具提取
|
- 场景中的固定陈设不需要单独提取为道具,除非该物件有独立剧情作用
|
||||||
- 场景中的固定陈设不需要单独提取为道具,除非该物件有独立剧情作用
|
|
||||||
`,
|
`,
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
@ -527,6 +526,7 @@ export default async (knex: Knex, forceInit: boolean = false): Promise<void> =>
|
|||||||
table.text("state");
|
table.text("state");
|
||||||
table.integer("scriptId");
|
table.integer("scriptId");
|
||||||
table.integer("storyboardId");
|
table.integer("storyboardId");
|
||||||
|
table.integer("projectId");
|
||||||
table.primary(["id"]);
|
table.primary(["id"]);
|
||||||
table.unique(["id"]);
|
table.unique(["id"]);
|
||||||
},
|
},
|
||||||
|
|||||||
@ -1,32 +1,54 @@
|
|||||||
import express from "express";
|
import express from "express";
|
||||||
import u from "@/utils";
|
import u from "@/utils";
|
||||||
|
import { z } from "zod";
|
||||||
import { success } from "@/lib/responseFormat";
|
import { success } from "@/lib/responseFormat";
|
||||||
|
import { validateFields } from "@/middleware/middleware";
|
||||||
const router = express.Router();
|
const router = express.Router();
|
||||||
|
|
||||||
// 获取生成图片
|
// 获取生成图片
|
||||||
export default router.post("/", async (req, res) => {
|
export default router.post(
|
||||||
const list = await u.db("o_assets").leftJoin("o_image", "o_assets.id", "=", "o_image.assetsId").where("o_assets.type", "clip").select("*");
|
"/",
|
||||||
const data = await Promise.all(
|
validateFields({
|
||||||
list.map(async (item) => ({
|
projectId: z.number(),
|
||||||
...item,
|
}),
|
||||||
filePath: item.filePath ? await u.oss.getFileUrl(item.filePath) : "",
|
async (req, res) => {
|
||||||
})),
|
const { projectId } = req.body;
|
||||||
);
|
const list = await u
|
||||||
// 查询o_videoConfig表,拿到已选中的videoId
|
.db("o_assets")
|
||||||
const configRows = await u.db("o_videoConfig").select("videoId");
|
.leftJoin("o_image", "o_assets.id", "=", "o_image.assetsId")
|
||||||
const selectedIds = new Set(configRows.map((row) => row.videoId));
|
.where("o_assets.type", "clip")
|
||||||
|
.andWhere("projectId", projectId)
|
||||||
|
.select("*");
|
||||||
|
const data = await Promise.all(
|
||||||
|
list.map(async (item) => ({
|
||||||
|
...item,
|
||||||
|
filePath: item.filePath ? await u.oss.getFileUrl(item.filePath) : "",
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
//拿到本地片尾视频并插入到data中
|
||||||
|
const ending = await u.oss.getFileUrl("/ending/1d7a2dfdd0c057823797fdf97677a7a0.mp4");
|
||||||
|
data.push({
|
||||||
|
id: 0,
|
||||||
|
name: "片尾",
|
||||||
|
filePath: ending,
|
||||||
|
type: "clip",
|
||||||
|
});
|
||||||
|
// 查询o_videoConfig表,拿到已选中的videoId
|
||||||
|
const configRows = await u.db("o_videoConfig").select("videoId");
|
||||||
|
const selectedIds = new Set(configRows.map((row) => row.videoId));
|
||||||
|
|
||||||
// 查询o_video表
|
// 查询o_video表
|
||||||
const videoRows = await u.db("o_video").where("state", "生成成功").select("*");
|
const videoRows = await u.db("o_video").where("state", "生成成功").andWhere("projectId", projectId).select("*");
|
||||||
|
// 处理并返回结果
|
||||||
|
const video = await Promise.all(
|
||||||
|
videoRows.map(async (row) => ({
|
||||||
|
id: row.id,
|
||||||
|
filePath: row.filePath ? await u.oss.getFileUrl(row.filePath) : "",
|
||||||
|
selected: selectedIds.has(row.id),
|
||||||
|
storyboard: row.storyboardId,
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
|
||||||
// 处理并返回结果
|
res.status(200).send(success({ data, video }));
|
||||||
const video = await Promise.all(
|
},
|
||||||
videoRows.map(async (row) => ({
|
);
|
||||||
id: row.id,
|
|
||||||
filePath: row.filePath ? await u.oss.getFileUrl(row.filePath) : "",
|
|
||||||
selected: selectedIds.has(row.id),
|
|
||||||
storyboard: row.storyboardId,
|
|
||||||
})),
|
|
||||||
);
|
|
||||||
res.status(200).send(success({ data, video }));
|
|
||||||
});
|
|
||||||
|
|||||||
@ -110,16 +110,20 @@ export default router.post("/", validateFields(requestSchema), async (req, res)
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
const aiImage = u.Ai.Image(model);
|
const aiImage = u.Ai.Image(model);
|
||||||
await aiImage.run({
|
await aiImage.run(
|
||||||
prompt: userPrompt,
|
{
|
||||||
imageBase64: item.base64 ? [item.base64] : [],
|
prompt: userPrompt,
|
||||||
size: resolution,
|
imageBase64: item.base64 ? [item.base64] : [],
|
||||||
aspectRatio: "16:9",
|
size: resolution,
|
||||||
taskClass: cfg.taskClass,
|
aspectRatio: "16:9",
|
||||||
describe,
|
},
|
||||||
projectId,
|
{
|
||||||
relatedObjects: JSON.stringify(relatedObjects),
|
taskClass: cfg.taskClass,
|
||||||
});
|
describe,
|
||||||
|
projectId,
|
||||||
|
relatedObjects: JSON.stringify(relatedObjects),
|
||||||
|
},
|
||||||
|
);
|
||||||
aiImage.save(imagePath);
|
aiImage.save(imagePath);
|
||||||
|
|
||||||
const imageData = await u.db("o_image").where("id", imageId).select("*").first();
|
const imageData = await u.db("o_image").where("id", imageId).select("*").first();
|
||||||
|
|||||||
@ -98,16 +98,20 @@ export default router.post("/", validateFields(requestSchema), async (req, res)
|
|||||||
try {
|
try {
|
||||||
// 4. 调用 AI 生成图片
|
// 4. 调用 AI 生成图片
|
||||||
const aiImage = u.Ai.Image(model);
|
const aiImage = u.Ai.Image(model);
|
||||||
await aiImage.run({
|
await aiImage.run(
|
||||||
prompt: userPrompt,
|
{
|
||||||
imageBase64: base64 ? [base64] : [],
|
prompt: userPrompt,
|
||||||
size: resolution,
|
imageBase64: base64 ? [base64] : [],
|
||||||
aspectRatio: "16:9",
|
size: resolution,
|
||||||
taskClass: cfg.taskClass,
|
aspectRatio: "16:9",
|
||||||
describe,
|
},
|
||||||
projectId,
|
{
|
||||||
relatedObjects: JSON.stringify(relatedObjects),
|
taskClass: cfg.taskClass,
|
||||||
});
|
describe,
|
||||||
|
projectId,
|
||||||
|
relatedObjects: JSON.stringify(relatedObjects),
|
||||||
|
},
|
||||||
|
);
|
||||||
aiImage.save(imagePath);
|
aiImage.save(imagePath);
|
||||||
|
|
||||||
// 5. 更新记录 & 返回结果
|
// 5. 更新记录 & 返回结果
|
||||||
|
|||||||
@ -39,6 +39,7 @@ export default router.post(
|
|||||||
state: "生成中",
|
state: "生成中",
|
||||||
scriptId,
|
scriptId,
|
||||||
storyboardId,
|
storyboardId,
|
||||||
|
projectId,
|
||||||
};
|
};
|
||||||
const [videoId] = await u.db("o_video").insert(videoData);
|
const [videoId] = await u.db("o_video").insert(videoData);
|
||||||
//查询分镜是否已有配置
|
//查询分镜是否已有配置
|
||||||
@ -105,19 +106,23 @@ export default router.post(
|
|||||||
console.log("%c Line:110 🍑 prompt", "background:#b03734", prompt);
|
console.log("%c Line:110 🍑 prompt", "background:#b03734", prompt);
|
||||||
|
|
||||||
const aiVideo = u.Ai.Video(model);
|
const aiVideo = u.Ai.Video(model);
|
||||||
await aiVideo.run({
|
await aiVideo.run(
|
||||||
projectId,
|
{
|
||||||
prompt,
|
prompt,
|
||||||
imageBase64: base64.filter((item) => item !== null) as string[],
|
imageBase64: base64.filter((item) => item !== null) as string[],
|
||||||
mode,
|
mode,
|
||||||
duration,
|
duration,
|
||||||
aspectRatio: (ratio?.videoRatio as `${number}:${number}`) || "16:9",
|
aspectRatio: (ratio?.videoRatio as `${number}:${number}`) || "16:9",
|
||||||
resolution,
|
resolution,
|
||||||
audio,
|
audio,
|
||||||
taskClass: "视频生成",
|
},
|
||||||
describe: "根据提示词生成视频",
|
{
|
||||||
relatedObjects: JSON.stringify(relatedObjects),
|
projectId,
|
||||||
});
|
taskClass: "视频生成",
|
||||||
|
describe: "根据提示词生成视频",
|
||||||
|
relatedObjects: JSON.stringify(relatedObjects),
|
||||||
|
},
|
||||||
|
);
|
||||||
await aiVideo.save(videoPath);
|
await aiVideo.save(videoPath);
|
||||||
await u.db("o_video").where("id", videoId).update({ state: "生成成功" });
|
await u.db("o_video").where("id", videoId).update({ state: "生成成功" });
|
||||||
// await u.db("o_videoConfig").where("storyboardId", storyboardId).update({ videoId, updateTime: Date.now() });
|
// await u.db("o_videoConfig").where("storyboardId", storyboardId).update({ videoId, updateTime: Date.now() });
|
||||||
|
|||||||
@ -2,25 +2,35 @@ import express from "express";
|
|||||||
const router = express.Router();
|
const router = express.Router();
|
||||||
import u from "@/utils";
|
import u from "@/utils";
|
||||||
import fs from "fs";
|
import fs from "fs";
|
||||||
import { useSkill } from "@/utils/agent/skillsTools";
|
import Memory from "@/utils/agent/memory";
|
||||||
|
|
||||||
|
|
||||||
|
function buildMemPrompt(mem: Awaited<ReturnType<Memory["get"]>>): string {
|
||||||
|
let memoryContext = "";
|
||||||
|
if (mem.rag.length) {
|
||||||
|
memoryContext += `[相关记忆]\n${mem.rag.map((r) => r.content).join("\n")}`;
|
||||||
|
}
|
||||||
|
if (mem.summaries.length) {
|
||||||
|
if (memoryContext) memoryContext += "\n\n";
|
||||||
|
memoryContext += `[历史摘要]\n${mem.summaries.map((s, i) => `${i + 1}. ${s.content}`).join("\n")}`;
|
||||||
|
}
|
||||||
|
if (mem.shortTerm.length) {
|
||||||
|
if (memoryContext) memoryContext += "\n\n";
|
||||||
|
memoryContext += `[近期对话]\n${mem.shortTerm.map((m) => `${m.role}: ${m.content}`).join("\n")}`;
|
||||||
|
}
|
||||||
|
return `## Memory\n以下是你对用户的记忆,可作为参考但不要主动提及:\n${memoryContext}`;
|
||||||
|
}
|
||||||
|
|
||||||
export default router.get("/", async (req, res) => {
|
export default router.get("/", async (req, res) => {
|
||||||
const skill = await useSkill(
|
|
||||||
{
|
|
||||||
mainSkill: "production_agent_execution",
|
|
||||||
workspace: ["production_agent_skills/execution"],
|
|
||||||
attachedSkills: ["art_prompts/chinese_sweet_romance/driector_skills"],
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
const test = await u.Ai.Text("scriptAgent").invoke({
|
const isolationKey = "test";
|
||||||
system: skill.prompt,
|
const input = "你好"
|
||||||
messages: [
|
|
||||||
{ role: "user", content: "渐进式激活skill,技能->资源1->资源2...一直渐进到最深处,并输出你的阅读路线,同级目录你只用读取一个无需全部读取" },
|
|
||||||
],
|
|
||||||
tools: skill.tools,
|
|
||||||
});
|
|
||||||
|
|
||||||
console.log("%c Line:21 🌽 text", "background:#ea7e5c", test.text);
|
const memory = new Memory("productionAgent", isolationKey);
|
||||||
res.send(test.text);
|
await memory.add("user", input);
|
||||||
|
|
||||||
|
|
||||||
|
const mem = buildMemPrompt(await memory.get(input));
|
||||||
|
|
||||||
|
res.send(mem);
|
||||||
});
|
});
|
||||||
|
|||||||
@ -36,12 +36,22 @@ export default (nsp: Namespace) => {
|
|||||||
|
|
||||||
console.log("[productionAgent] 已连接:", socket.id);
|
console.log("[productionAgent] 已连接:", socket.id);
|
||||||
|
|
||||||
const resTool = new ResTool(socket, {
|
let resTool = new ResTool(socket, {
|
||||||
projectId: socket.handshake.auth.projectId,
|
projectId: socket.handshake.auth.projectId,
|
||||||
scriptId: socket.handshake.auth.scriptId,
|
scriptId: socket.handshake.auth.scriptId,
|
||||||
});
|
});
|
||||||
let abortController: AbortController | null = null;
|
let abortController: AbortController | null = null;
|
||||||
|
|
||||||
|
socket.on("updateContext", (data: { isolationKey: string; projectId: number; scriptId: number }, callback) => {
|
||||||
|
isolationKey = data.isolationKey;
|
||||||
|
resTool = new ResTool(socket, {
|
||||||
|
projectId: data.projectId,
|
||||||
|
scriptId: data.scriptId,
|
||||||
|
});
|
||||||
|
console.log("[productionAgent] 上下文已更新:", isolationKey);
|
||||||
|
callback?.({ success: true });
|
||||||
|
});
|
||||||
|
|
||||||
socket.on("chat", async (data: { content: string }) => {
|
socket.on("chat", async (data: { content: string }) => {
|
||||||
const { content } = data;
|
const { content } = data;
|
||||||
abortController?.abort();
|
abortController?.abort();
|
||||||
@ -84,6 +94,7 @@ export default (nsp: Namespace) => {
|
|||||||
text = currentMsg.text();
|
text = currentMsg.text();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let aborted = false;
|
||||||
try {
|
try {
|
||||||
for await (const chunk of textStream) {
|
for await (const chunk of textStream) {
|
||||||
await syncCurrentMessage();
|
await syncCurrentMessage();
|
||||||
@ -91,11 +102,21 @@ export default (nsp: Namespace) => {
|
|||||||
currentContent += chunk;
|
currentContent += chunk;
|
||||||
}
|
}
|
||||||
} catch (err: any) {
|
} catch (err: any) {
|
||||||
if (err.name !== "AbortError") throw err;
|
if (err.name === "AbortError" || currentController.signal.aborted) {
|
||||||
|
aborted = true;
|
||||||
|
} else {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
} finally {
|
} finally {
|
||||||
await syncCurrentMessage();
|
await syncCurrentMessage();
|
||||||
text.complete();
|
if (aborted) {
|
||||||
currentMsg.complete();
|
text.append("[已停止]");
|
||||||
|
text.complete();
|
||||||
|
currentMsg.stop();
|
||||||
|
} else {
|
||||||
|
text.complete();
|
||||||
|
currentMsg.complete();
|
||||||
|
}
|
||||||
await persistCurrentMessage();
|
await persistCurrentMessage();
|
||||||
if (abortController === currentController) {
|
if (abortController === currentController) {
|
||||||
abortController = null;
|
abortController = null;
|
||||||
|
|||||||
@ -83,6 +83,7 @@ export default (nsp: Namespace) => {
|
|||||||
text = currentMsg.text();
|
text = currentMsg.text();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let aborted = false;
|
||||||
try {
|
try {
|
||||||
for await (const chunk of textStream) {
|
for await (const chunk of textStream) {
|
||||||
await syncCurrentMessage();
|
await syncCurrentMessage();
|
||||||
@ -90,11 +91,20 @@ export default (nsp: Namespace) => {
|
|||||||
currentContent += chunk;
|
currentContent += chunk;
|
||||||
}
|
}
|
||||||
} catch (err: any) {
|
} catch (err: any) {
|
||||||
if (err.name !== "AbortError") throw err;
|
if (err.name === "AbortError" || currentController.signal.aborted) {
|
||||||
|
aborted = true;
|
||||||
|
} else {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
} finally {
|
} finally {
|
||||||
await syncCurrentMessage();
|
await syncCurrentMessage();
|
||||||
text.complete();
|
if (aborted) {
|
||||||
currentMsg.complete();
|
text.complete();
|
||||||
|
currentMsg.stop();
|
||||||
|
} else {
|
||||||
|
text.complete();
|
||||||
|
currentMsg.complete();
|
||||||
|
}
|
||||||
await persistCurrentMessage();
|
await persistCurrentMessage();
|
||||||
if (abortController === currentController) {
|
if (abortController === currentController) {
|
||||||
abortController = null;
|
abortController = null;
|
||||||
|
|||||||
15
src/types/database.d.ts
vendored
15
src/types/database.d.ts
vendored
@ -1,13 +1,6 @@
|
|||||||
// @db-hash f7bc2fdb80756d5536929eb47155578b
|
// @db-hash 93b2462070c45c2b449e9a18c4e88763
|
||||||
//该文件由脚本自动生成,请勿手动修改
|
//该文件由脚本自动生成,请勿手动修改
|
||||||
|
|
||||||
export interface _o_script_old_20260327 {
|
|
||||||
'content'?: string | null;
|
|
||||||
'createTime'?: number | null;
|
|
||||||
'id'?: number;
|
|
||||||
'name'?: string | null;
|
|
||||||
'projectId'?: number | null;
|
|
||||||
}
|
|
||||||
export interface memories {
|
export interface memories {
|
||||||
'content': string;
|
'content': string;
|
||||||
'createTime': number;
|
'createTime': number;
|
||||||
@ -28,7 +21,7 @@ export interface o_agentDeploy {
|
|||||||
'model'?: string | null;
|
'model'?: string | null;
|
||||||
'modelName'?: string | null;
|
'modelName'?: string | null;
|
||||||
'name'?: string | null;
|
'name'?: string | null;
|
||||||
'vendorId'?: number | null;
|
'vendorId'?: string | null;
|
||||||
}
|
}
|
||||||
export interface o_agentWorkData {
|
export interface o_agentWorkData {
|
||||||
'createTime'?: number | null;
|
'createTime'?: number | null;
|
||||||
@ -54,6 +47,7 @@ export interface o_assets {
|
|||||||
'name'?: string | null;
|
'name'?: string | null;
|
||||||
'projectId'?: number | null;
|
'projectId'?: number | null;
|
||||||
'prompt'?: string | null;
|
'prompt'?: string | null;
|
||||||
|
'promptState'?: string | null;
|
||||||
'remark'?: string | null;
|
'remark'?: string | null;
|
||||||
'scriptId'?: number | null;
|
'scriptId'?: number | null;
|
||||||
'startTime'?: number | null;
|
'startTime'?: number | null;
|
||||||
@ -173,7 +167,7 @@ export interface o_storyboard {
|
|||||||
'filePath'?: string | null;
|
'filePath'?: string | null;
|
||||||
'frameMode'?: string | null;
|
'frameMode'?: string | null;
|
||||||
'id'?: number;
|
'id'?: number;
|
||||||
'index'?: string | null;
|
'index'?: number | null;
|
||||||
'lines'?: string | null;
|
'lines'?: string | null;
|
||||||
'mode'?: string | null;
|
'mode'?: string | null;
|
||||||
'model'?: string | null;
|
'model'?: string | null;
|
||||||
@ -238,7 +232,6 @@ export interface o_videoConfig {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface DB {
|
export interface DB {
|
||||||
"_o_script_old_20260327": _o_script_old_20260327;
|
|
||||||
"memories": memories;
|
"memories": memories;
|
||||||
"o_agentDeploy": o_agentDeploy;
|
"o_agentDeploy": o_agentDeploy;
|
||||||
"o_agentWorkData": o_agentWorkData;
|
"o_agentWorkData": o_agentWorkData;
|
||||||
|
|||||||
@ -25,23 +25,10 @@ async function getVendorTemplateFn(fnName: FnName, modelName: `${string}:${strin
|
|||||||
const selectedModel = modelList.find((i: any) => i.modelName == name);
|
const selectedModel = modelList.find((i: any) => i.modelName == name);
|
||||||
if (!selectedModel) throw new Error(`未找到模型 ${name} id=${id}`);
|
if (!selectedModel) throw new Error(`未找到模型 ${name} id=${id}`);
|
||||||
const jsCode = transform(vendorConfigData.code!, { transforms: ["typescript"] }).code;
|
const jsCode = transform(vendorConfigData.code!, { transforms: ["typescript"] }).code;
|
||||||
const running = u.vm(jsCode);
|
const fn = u.vm(jsCode)[fnName];
|
||||||
Object.assign(running.vendor.inputValues, JSON.parse(vendorConfigData.inputValues ?? "{}"));
|
|
||||||
running.vendor.models = modelList;
|
|
||||||
const fn = running[fnName];
|
|
||||||
if (!fn) throw new Error(`未找到供应商配置中的函数 ${fnName} id=${id}`);
|
if (!fn) throw new Error(`未找到供应商配置中的函数 ${fnName} id=${id}`);
|
||||||
if (fnName == "textRequest") {
|
if (fnName == "textRequest") return fn(selectedModel);
|
||||||
const model = fn(selectedModel);
|
else return <T>(input: T) => fn(input, selectedModel);
|
||||||
if (!model) throw new Error(`供应商 textRequest 返回无效模型 id=${id}`);
|
|
||||||
return model;
|
|
||||||
}
|
|
||||||
return async <T>(input: T) => {
|
|
||||||
const result = await fn(input, selectedModel);
|
|
||||||
if (result === undefined || result === null) {
|
|
||||||
throw new Error(`供应商函数 ${fnName} 未返回有效结果 id=${id}`);
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function withTaskRecord<T>(
|
async function withTaskRecord<T>(
|
||||||
@ -113,6 +100,9 @@ interface ImageConfig {
|
|||||||
imageBase64: string[]; //输入的图片提示词
|
imageBase64: string[]; //输入的图片提示词
|
||||||
size: "1K" | "2K" | "4K"; // 图片尺寸
|
size: "1K" | "2K" | "4K"; // 图片尺寸
|
||||||
aspectRatio: `${number}:${number}`; // 长宽比
|
aspectRatio: `${number}:${number}`; // 长宽比
|
||||||
|
}
|
||||||
|
|
||||||
|
interface TaskRecord {
|
||||||
taskClass: string; // 任务分类
|
taskClass: string; // 任务分类
|
||||||
describe: string; // 任务描述
|
describe: string; // 任务描述
|
||||||
relatedObjects: string; // 相关对象信息,便于后续分析和追踪
|
relatedObjects: string; // 相关对象信息,便于后续分析和追踪
|
||||||
@ -125,8 +115,8 @@ class AiImage {
|
|||||||
constructor(key: `${string}:${string}`) {
|
constructor(key: `${string}:${string}`) {
|
||||||
this.key = key;
|
this.key = key;
|
||||||
}
|
}
|
||||||
async run(input: ImageConfig) {
|
async run(input: ImageConfig, taskRecord: TaskRecord) {
|
||||||
return withTaskRecord(this.key, input.taskClass, input.describe, input.relatedObjects, input.projectId, async (modelName) => {
|
return withTaskRecord(this.key, taskRecord.taskClass, taskRecord.describe, taskRecord.relatedObjects, taskRecord.projectId, async (modelName) => {
|
||||||
const fn = await getVendorTemplateFn("imageRequest", modelName);
|
const fn = await getVendorTemplateFn("imageRequest", modelName);
|
||||||
this.result = await fn(input);
|
this.result = await fn(input);
|
||||||
if (this.result.startsWith("http")) this.result = await urlToBase64(this.result);
|
if (this.result.startsWith("http")) this.result = await urlToBase64(this.result);
|
||||||
@ -139,7 +129,6 @@ class AiImage {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
interface VideoConfig {
|
interface VideoConfig {
|
||||||
projectId: number; // 项目ID
|
|
||||||
prompt: string; //视频提示词
|
prompt: string; //视频提示词
|
||||||
imageBase64: string[]; //输入的图片提示词
|
imageBase64: string[]; //输入的图片提示词
|
||||||
aspectRatio: `${number}:${number}`; // 长宽比
|
aspectRatio: `${number}:${number}`; // 长宽比
|
||||||
@ -147,9 +136,6 @@ interface VideoConfig {
|
|||||||
duration: number; // 视频时长,单位秒
|
duration: number; // 视频时长,单位秒
|
||||||
resolution: string; // 视频分辨率
|
resolution: string; // 视频分辨率
|
||||||
audio: boolean; // 是否需要配音
|
audio: boolean; // 是否需要配音
|
||||||
taskClass: string; // 任务分类
|
|
||||||
describe: string; // 任务描述
|
|
||||||
relatedObjects: string; // 相关对象信息,便于后续分析和追踪
|
|
||||||
}
|
}
|
||||||
|
|
||||||
class AiVideo {
|
class AiVideo {
|
||||||
@ -158,8 +144,8 @@ class AiVideo {
|
|||||||
constructor(key: `${string}:${string}`) {
|
constructor(key: `${string}:${string}`) {
|
||||||
this.key = key;
|
this.key = key;
|
||||||
}
|
}
|
||||||
async run(input: VideoConfig) {
|
async run(input: VideoConfig, taskRecord: TaskRecord) {
|
||||||
return withTaskRecord(this.key, input.taskClass, input.describe, input.relatedObjects, input.projectId, async (modelName) => {
|
return withTaskRecord(this.key, taskRecord.taskClass, taskRecord.describe, taskRecord.relatedObjects, taskRecord.projectId, async (modelName) => {
|
||||||
const fn = await getVendorTemplateFn("videoRequest", modelName);
|
const fn = await getVendorTemplateFn("videoRequest", modelName);
|
||||||
this.result = await fn(input);
|
this.result = await fn(input);
|
||||||
if (this.result.startsWith("http")) this.result = await urlToBase64(this.result);
|
if (this.result.startsWith("http")) this.result = await urlToBase64(this.result);
|
||||||
@ -177,8 +163,8 @@ class AiAudio {
|
|||||||
constructor(key: `${string}:${string}`) {
|
constructor(key: `${string}:${string}`) {
|
||||||
this.key = key;
|
this.key = key;
|
||||||
}
|
}
|
||||||
async run(input: VideoConfig) {
|
async run(input: VideoConfig, taskRecord: TaskRecord) {
|
||||||
return withTaskRecord(this.key, input.taskClass, input.describe, input.relatedObjects, input.projectId, async (modelName) => {
|
return withTaskRecord(this.key, taskRecord.taskClass, taskRecord.describe, taskRecord.relatedObjects, taskRecord.projectId, async (modelName) => {
|
||||||
const fn = await getVendorTemplateFn("ttsRequest", modelName);
|
const fn = await getVendorTemplateFn("ttsRequest", modelName);
|
||||||
this.result = await fn(input);
|
this.result = await fn(input);
|
||||||
if (this.result.startsWith("http")) this.result = await urlToBase64(this.result);
|
if (this.result.startsWith("http")) this.result = await urlToBase64(this.result);
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user