Merge branch 'develop' of https://github.com/HBAI-Ltd/Toonflow-app into develop
# Conflicts: # src/utils.ts # src/utils/getConfig.ts
This commit is contained in:
zhishi
commit
53e547d405
36
backup/agents/models.ts
Normal file
36
backup/agents/models.ts
Normal file
@ -0,0 +1,36 @@
|
||||
import { ChatOpenAI, ChatOpenAIFields } from "@langchain/openai";
|
||||
|
||||
export const openAI = (config: ChatOpenAIFields = {}) => {
|
||||
return new ChatOpenAI({
|
||||
modelName: "gpt-4.1",
|
||||
temperature: 1,
|
||||
configuration: {
|
||||
apiKey: process.env.AI_OPENAI_KEY,
|
||||
baseURL: process.env.AI_OPENAI_URL,
|
||||
},
|
||||
...config,
|
||||
});
|
||||
};
|
||||
|
||||
export const doubao = (config: ChatOpenAIFields = {}) => {
|
||||
return new ChatOpenAI({
|
||||
model: "doubao-seed-1-6-flash-250828",
|
||||
temperature: 1,
|
||||
configuration: {
|
||||
apiKey: process.env.AI_TIKTOK_KEY,
|
||||
baseURL: process.env.AI_TIKTOK_URL,
|
||||
},
|
||||
...config,
|
||||
});
|
||||
};
|
||||
|
||||
export const deepseek = (config: ChatOpenAIFields = {}) =>
|
||||
new ChatOpenAI({
|
||||
model: "DeepSeek-V3.2",
|
||||
temperature: 1,
|
||||
configuration: {
|
||||
apiKey: process.env.AI_DEEPSEEK_KEY,
|
||||
baseURL: process.env.AI_DEEPSEEK_URL,
|
||||
},
|
||||
...config,
|
||||
});
|
||||
769
backup/agents/outlineScript/index.ts
Normal file
769
backup/agents/outlineScript/index.ts
Normal file
@ -0,0 +1,769 @@
|
||||
// @/agents/outlineScript.ts
|
||||
import u from "@/utils";
|
||||
import { createAgent } from "langchain";
|
||||
import { EventEmitter } from "events";
|
||||
import { openAI } from "@/agents/models";
|
||||
import { z } from "zod";
|
||||
import { tool } from "@langchain/core/tools";
|
||||
import type { DB } from "@/types/database";
|
||||
// ==================== 类型定义 ====================
|
||||
|
||||
type AgentType = "AI1" | "AI2" | "director";
|
||||
type AssetType = "角色" | "道具" | "场景";
|
||||
type RefreshEvent = "storyline" | "outline" | "assets";
|
||||
|
||||
interface AssetItem {
|
||||
name: string;
|
||||
description: string;
|
||||
}
|
||||
|
||||
interface EpisodeData {
|
||||
episodeIndex: number;
|
||||
title: string;
|
||||
chapterRange: number[];
|
||||
scenes: AssetItem[]; // 按 outline 出场顺序排列
|
||||
characters: AssetItem[]; // 按 outline 出场顺序排列
|
||||
props: AssetItem[]; // 按 outline 出场顺序排列
|
||||
coreConflict: string;
|
||||
outline: string; // 最高优先级，剧本生成的唯一权威
|
||||
openingHook: string; // outline 第一句话的视觉化，开篇第一个镜头
|
||||
keyEvents: string[]; // 4个元素：[起, 承, 转, 合]，严格按 outline 顺序
|
||||
emotionalCurve: string; // 对应 keyEvents 各阶段
|
||||
visualHighlights: string[]; // 按 outline 顺序排列的标志性镜头
|
||||
endingHook: string; // outline 之后的悬念延伸
|
||||
classicQuotes: string[];
|
||||
}
|
||||
|
||||
// ==================== Schema 定义 ====================
|
||||
|
||||
const sceneItemSchema = z.object({
|
||||
name: z.string().describe("场景名称，如'五星酒店宴会厅'、'老旧出租屋'"),
|
||||
description: z.string().describe("环境描写：空间结构、光线氛围、装饰陈设、环境细节"),
|
||||
});
|
||||
|
||||
const characterItemSchema = z.object({
|
||||
name: z.string().describe("角色姓名（必须是具体人名，禁止'众人'、'群众'等集合描述）"),
|
||||
description: z.string().describe("人设样貌：年龄体态、五官特征、发型妆容、服装配饰、气质神态"),
|
||||
});
|
||||
|
||||
const propItemSchema = z.object({
|
||||
name: z.string().describe("道具名称"),
|
||||
description: z.string().describe("样式描写：材质质感、颜色图案、形状尺寸、磨损痕迹、特殊标记"),
|
||||
});
|
||||
|
||||
const episodeSchema = z.object({
|
||||
episodeIndex: z.number().describe("集数索引，从1开始递增"),
|
||||
title: z.string().describe("8字内标题，疑问/感叹句，含情绪爆点"),
|
||||
chapterRange: z.array(z.number()).describe("关联章节号数组"),
|
||||
scenes: z.array(sceneItemSchema).describe("场景列表，按 outline 出场顺序排列"),
|
||||
characters: z.array(characterItemSchema).describe("角色列表，按 outline 出场顺序排列，必须是独立个体"),
|
||||
props: z.array(propItemSchema).describe("道具列表，按 outline 出场顺序排列，至少3个"),
|
||||
coreConflict: z.string().describe("核心矛盾：A想要X vs B阻碍X"),
|
||||
outline: z.string().describe("100-300字剧情主干，最高优先级，剧本生成的唯一权威，按时间顺序完整叙述"),
|
||||
openingHook: z.string().describe("开场镜头：outline 第一句话的视觉化，必须作为剧本第一个镜头"),
|
||||
keyEvents: z.array(z.string()).length(4).describe("4个元素的数组：[起, 承, 转, 合]，严格按 outline 顺序从中提取"),
|
||||
emotionalCurve: z.string().describe("情绪曲线，如：2(压抑)→5(反抗)→9(爆发)→3(余波)，对应 keyEvents 各阶段"),
|
||||
visualHighlights: z.array(z.string()).describe("3-5个标志性镜头，按 outline 叙事顺序排列"),
|
||||
endingHook: z.string().describe("结尾悬念：outline 之后的延伸，勾引下集"),
|
||||
classicQuotes: z.array(z.string()).describe("1-2句金句，每句≤15字，必须从原文提取"),
|
||||
});
|
||||
|
||||
// ==================== 常量配置 ====================
|
||||
|
||||
// ==================== 主类 ====================
|
||||
|
||||
export default class OutlineScript {
|
||||
private readonly projectId: number;
|
||||
readonly emitter = new EventEmitter();
|
||||
history: Array<[string, string]> = [];
|
||||
novelChapters: DB["t_novel"][] = [];
|
||||
|
||||
modelName = "gpt-4.1";
|
||||
apiKey = "";
|
||||
baseURL = "";
|
||||
|
||||
constructor(projectId: number) {
|
||||
this.projectId = projectId;
|
||||
}
|
||||
|
||||
// ==================== 公共方法 ====================
|
||||
|
||||
get events() {
|
||||
return this.emitter;
|
||||
}
|
||||
|
||||
setNovel(chapters: DB["t_novel"][]) {
|
||||
this.novelChapters = chapters;
|
||||
}
|
||||
|
||||
// ==================== 私有工具方法 ====================
|
||||
|
||||
private emit(event: string, data?: any) {
|
||||
this.emitter.emit(event, data);
|
||||
}
|
||||
|
||||
private refresh(type: RefreshEvent) {
|
||||
this.emit("refresh", type);
|
||||
}
|
||||
|
||||
private log(action: string, detail?: string) {
|
||||
const msg = detail ? `${action}: ${detail}` : action;
|
||||
console.log(`\n[${new Date().toLocaleTimeString()}] ${msg}\n`);
|
||||
}
|
||||
|
||||
private safeParseJson<T>(str: string, fallback: T): T {
|
||||
try {
|
||||
return JSON.parse(str);
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
private uniqueByName<T extends { name: string }>(items: T[]): T[] {
|
||||
return Array.from(new Map(items.map((item) => [item.name, item])).values());
|
||||
}
|
||||
|
||||
// ==================== 数据库操作 ====================
|
||||
|
||||
private async getProjectInfo(): Promise<any> {
|
||||
return u.db("t_project").where({ id: this.projectId }).first();
|
||||
}
|
||||
|
||||
private async getNovelInfo(asString = false): Promise<any> {
|
||||
const info = await this.getProjectInfo();
|
||||
if (!info) return asString ? "未查询到项目信息" : null;
|
||||
|
||||
if (asString) {
|
||||
const fields = [
|
||||
`小说名称: ${info.name}`,
|
||||
`小说简介: ${info.intro}`,
|
||||
`小说类型: ${info.type}`,
|
||||
`目标短剧类型: ${info.artStyle}`,
|
||||
`短剧画幅: ${info.videoRatio}`,
|
||||
];
|
||||
return fields.join("\n");
|
||||
}
|
||||
return info;
|
||||
}
|
||||
|
||||
// ==================== 故事线操作 ====================
|
||||
|
||||
private async findStoryline() {
|
||||
return u.db("t_storyline").where({ projectId: this.projectId }).first();
|
||||
}
|
||||
|
||||
private async upsertStorylineContent(content: string) {
|
||||
const existing = await this.findStoryline();
|
||||
if (existing) {
|
||||
await u.db("t_storyline").where({ projectId: this.projectId }).update({ content });
|
||||
} else {
|
||||
await u.db("t_storyline").insert({ projectId: this.projectId, content });
|
||||
}
|
||||
this.refresh("storyline");
|
||||
}
|
||||
|
||||
private async deleteStorylineContent() {
|
||||
const deleted = await u.db("t_storyline").where({ projectId: this.projectId }).del();
|
||||
this.refresh("storyline");
|
||||
return deleted;
|
||||
}
|
||||
|
||||
// ==================== 大纲操作 ====================
|
||||
|
||||
private async findOutlines() {
|
||||
return u.db("t_outline").where({ projectId: this.projectId }).orderBy("episode", "asc");
|
||||
}
|
||||
|
||||
private async findOutlineById(id: number) {
|
||||
return u.db("t_outline").where({ id, projectId: this.projectId }).first();
|
||||
}
|
||||
|
||||
private async getMaxEpisode(): Promise<number> {
|
||||
const result: any = await u.db("t_outline").where({ projectId: this.projectId }).max("episode as max").first();
|
||||
return result?.max ?? 0;
|
||||
}
|
||||
|
||||
private async clearOutlinesAndScripts() {
|
||||
const outlines = await u.db("t_outline").select("id").where({ projectId: this.projectId });
|
||||
if (outlines.length === 0) return 0;
|
||||
|
||||
const outlineIds = outlines.map((o) => o.id);
|
||||
await u.db("t_script").whereIn("outlineId", outlineIds).del();
|
||||
await u.db("t_outline").where({ projectId: this.projectId }).del();
|
||||
|
||||
return outlines.length;
|
||||
}
|
||||
|
||||
private async insertOutlines(episodes: EpisodeData[], startEpisode: number) {
|
||||
const insertList = episodes.map((ep, idx) => ({
|
||||
projectId: this.projectId,
|
||||
data: JSON.stringify({ ...ep, episodeIndex: startEpisode + idx }),
|
||||
episode: startEpisode + idx,
|
||||
}));
|
||||
|
||||
await u.db("t_outline").insert(insertList);
|
||||
return insertList.length;
|
||||
}
|
||||
|
||||
private async createEmptyScripts(outlineIds: Array<{ id: number; data: string }>) {
|
||||
const scripts = outlineIds.map((item) => {
|
||||
const data = this.safeParseJson<Partial<EpisodeData>>(item.data, {});
|
||||
return {
|
||||
name: `第${data.episodeIndex ?? ""}集`,
|
||||
content: "",
|
||||
projectId: this.projectId,
|
||||
outlineId: item.id,
|
||||
};
|
||||
});
|
||||
|
||||
if (scripts.length > 0) {
|
||||
await u.db("t_script").insert(scripts);
|
||||
}
|
||||
return scripts.length;
|
||||
}
|
||||
|
||||
private async saveOutlineData(episodes: EpisodeData[], overwrite: boolean, startEpisode?: number) {
|
||||
if (overwrite) {
|
||||
const cleared = await this.clearOutlinesAndScripts();
|
||||
if (cleared > 0) {
|
||||
this.log("清理旧数据", `删除了 ${cleared} 条大纲及关联剧本`);
|
||||
}
|
||||
}
|
||||
|
||||
const actualStart = overwrite ? 1 : startEpisode ?? (await this.getMaxEpisode()) + 1;
|
||||
const insertedCount = await this.insertOutlines(episodes, actualStart);
|
||||
|
||||
const newOutlines = await u
|
||||
.db("t_outline")
|
||||
.select("id", "data")
|
||||
.where({ projectId: this.projectId })
|
||||
.orderBy("episode", "desc")
|
||||
.limit(insertedCount);
|
||||
|
||||
const scriptCount = await this.createEmptyScripts(newOutlines as Array<{ id: number; data: string }>);
|
||||
|
||||
this.refresh("outline");
|
||||
return { insertedCount, scriptCount };
|
||||
}
|
||||
|
||||
private async updateOutlineData(id: number, data: EpisodeData) {
|
||||
const existing = await this.findOutlineById(id);
|
||||
if (!existing) return false;
|
||||
|
||||
await u
|
||||
.db("t_outline")
|
||||
.where({ id })
|
||||
.update({ data: JSON.stringify(data) });
|
||||
this.refresh("outline");
|
||||
return true;
|
||||
}
|
||||
|
||||
private async deleteOutlineData(ids: number[]) {
|
||||
const results = await Promise.allSettled(ids.map((id) => u.deleteOutline(id, this.projectId)));
|
||||
this.refresh("outline");
|
||||
return results;
|
||||
}
|
||||
|
||||
private formatOutlineDetail(ep: any): string {
|
||||
const formatList = (items: any[], formatter: (item: any) => string) =>
|
||||
items?.map((item, i) => ` ${i + 1}. ${formatter(item)}`).join("\n") || " 无";
|
||||
|
||||
// keyEvents 按顺序显示：起、承、转、合
|
||||
const keyEventsLabels = ["起", "承", "转", "合"];
|
||||
const formatKeyEvents = (events: string[]) => events?.map((e, i) => ` 【${keyEventsLabels[i] || i + 1}】${e}`).join("\n") || " 无";
|
||||
|
||||
return `
|
||||
大纲ID: ${ep.id}
|
||||
第 ${ep.episodeIndex} 集: ${ep.title || ""}
|
||||
${"=".repeat(50)}
|
||||
章节范围: ${ep.chapterRange?.join(", ") || ""}
|
||||
核心矛盾: ${ep.coreConflict || ""}
|
||||
|
||||
【剧情主干】(最高优先级，剧本生成的唯一权威):
|
||||
${ep.outline || "无"}
|
||||
|
||||
【开场镜头】(必须作为剧本第一个镜头):
|
||||
${ep.openingHook || "无"}
|
||||
|
||||
【剧情节点】(严格按顺序：起→承→转→合):
|
||||
${formatKeyEvents(ep.keyEvents)}
|
||||
|
||||
情绪曲线: ${ep.emotionalCurve || ""}
|
||||
|
||||
【视觉重点】(按剧情主干顺序排列):
|
||||
${formatList(ep.visualHighlights, (v) => v)}
|
||||
|
||||
【结尾悬念】:
|
||||
${ep.endingHook || "无"}
|
||||
|
||||
【经典台词】:
|
||||
${formatList(ep.classicQuotes, (q) => q)}
|
||||
|
||||
角色(按出场顺序): ${ep.characters?.map((c: AssetItem) => `${c.name}(${c.description})`).join("; ") || "无"}
|
||||
场景(按出场顺序): ${ep.scenes?.map((s: AssetItem) => `${s.name}(${s.description})`).join("; ") || "无"}
|
||||
道具(按出场顺序): ${ep.props?.map((p: AssetItem) => `${p.name}(${p.description})`).join("; ") || "无"}`;
|
||||
}
|
||||
|
||||
private async getOutlineText(simplified: boolean): Promise<string> {
|
||||
const records = await this.findOutlines();
|
||||
|
||||
if (!records.length) return "当前项目暂无大纲";
|
||||
|
||||
const episodes = records.map((r) => ({
|
||||
id: r.id,
|
||||
episode: r.episode,
|
||||
...this.safeParseJson<Partial<EpisodeData>>(r.data ?? "{}", {}),
|
||||
}));
|
||||
|
||||
if (simplified) {
|
||||
const list = episodes.map((ep) => `第 ${ep.episodeIndex ?? ep.episode} 集 (id=${ep.id})`).join("\n");
|
||||
return `项目大纲 (共 ${episodes.length} 集):\n${list}`;
|
||||
}
|
||||
|
||||
const details = episodes.map((ep) => this.formatOutlineDetail(ep)).join("\n");
|
||||
return `项目大纲 (共 ${episodes.length} 集)\n\n${details}`;
|
||||
}
|
||||
|
||||
// ==================== 资产操作 ====================
|
||||
|
||||
private async findAssetByTypeAndName(type: AssetType, name: string) {
|
||||
return u.db("t_assets").where({ projectId: this.projectId, type, name }).first();
|
||||
}
|
||||
|
||||
private async upsertAsset(type: AssetType, item: AssetItem): Promise<"inserted" | "updated" | "skipped"> {
|
||||
const existing = await this.findAssetByTypeAndName(type, item.name);
|
||||
|
||||
if (!existing) {
|
||||
await u.db("t_assets").insert({
|
||||
projectId: this.projectId,
|
||||
type,
|
||||
name: item.name,
|
||||
intro: item.description,
|
||||
prompt: item.description,
|
||||
});
|
||||
return "inserted";
|
||||
}
|
||||
|
||||
if (existing.intro !== item.description) {
|
||||
await u.db("t_assets").where({ id: existing.id }).update({
|
||||
intro: item.description,
|
||||
prompt: item.description,
|
||||
});
|
||||
return "updated";
|
||||
}
|
||||
|
||||
return "skipped";
|
||||
}
|
||||
|
||||
private extractAssetsFromOutlines(outlines: Array<{ data?: string | null | undefined }>): {
|
||||
characters: AssetItem[];
|
||||
props: AssetItem[];
|
||||
scenes: AssetItem[];
|
||||
} {
|
||||
const result = { characters: [] as AssetItem[], props: [] as AssetItem[], scenes: [] as AssetItem[] };
|
||||
|
||||
for (const outline of outlines) {
|
||||
const data = this.safeParseJson<Partial<EpisodeData>>(outline.data ?? "{}", {});
|
||||
if (data.characters) result.characters.push(...data.characters);
|
||||
if (data.props) result.props.push(...data.props);
|
||||
if (data.scenes) result.scenes.push(...data.scenes);
|
||||
}
|
||||
|
||||
return {
|
||||
characters: this.uniqueByName(result.characters),
|
||||
props: this.uniqueByName(result.props),
|
||||
scenes: this.uniqueByName(result.scenes),
|
||||
};
|
||||
}
|
||||
|
||||
private async generateAssetsFromOutlines() {
|
||||
const outlines = await u.db("t_outline").select("data").where({ projectId: this.projectId });
|
||||
|
||||
if (!outlines.length) return { inserted: 0, updated: 0, skipped: 0 };
|
||||
|
||||
const { characters, props, scenes } = this.extractAssetsFromOutlines(outlines);
|
||||
|
||||
// 只做新增和更新，不做删除
|
||||
const stats = { inserted: 0, updated: 0, skipped: 0 };
|
||||
|
||||
const processItems = async (items: AssetItem[], type: AssetType) => {
|
||||
for (const item of items) {
|
||||
const result = await this.upsertAsset(type, item);
|
||||
stats[result]++;
|
||||
}
|
||||
};
|
||||
|
||||
await processItems(characters, "角色");
|
||||
await processItems(props, "道具");
|
||||
await processItems(scenes, "场景");
|
||||
|
||||
this.refresh("assets");
|
||||
return { ...stats };
|
||||
}
|
||||
|
||||
// ==================== Tool 定义：故事线 ====================
|
||||
|
||||
getStoryline = tool(
|
||||
async () => {
|
||||
this.log("获取故事线");
|
||||
const storyline = await this.findStoryline();
|
||||
return storyline?.content ?? "当前项目暂无故事线";
|
||||
},
|
||||
{
|
||||
name: "getStoryline",
|
||||
description: "获取当前项目的故事线内容",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
saveStoryline = tool(
|
||||
async ({ content }) => {
|
||||
this.log("保存故事线");
|
||||
await this.upsertStorylineContent(content);
|
||||
return "故事线保存成功";
|
||||
},
|
||||
{
|
||||
name: "saveStoryline",
|
||||
description: "保存或更新当前项目的故事线，会覆盖已有内容",
|
||||
schema: z.object({
|
||||
content: z.string().describe("故事线完整内容"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
deleteStoryline = tool(
|
||||
async () => {
|
||||
this.log("删除故事线");
|
||||
const deleted = await this.deleteStorylineContent();
|
||||
return deleted > 0 ? "故事线删除成功" : "当前项目没有故事线";
|
||||
},
|
||||
{
|
||||
name: "deleteStoryline",
|
||||
description: "删除当前项目的故事线",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
// ==================== Tool 定义：大纲 ====================
|
||||
|
||||
getOutline = tool(
|
||||
async ({ simplified = false }) => {
|
||||
this.log("获取大纲", `简化模式: ${simplified}`);
|
||||
return this.getOutlineText(simplified);
|
||||
},
|
||||
{
|
||||
name: "getOutline",
|
||||
description: "获取项目大纲。simplified=true返回简化列表，false返回完整内容",
|
||||
schema: z.object({
|
||||
simplified: z.boolean().default(false).describe("是否返回简化版本"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
saveOutline = tool(
|
||||
async ({ episodes, overwrite = true, startEpisode }) => {
|
||||
this.log("保存大纲", `覆盖模式: ${overwrite}, 集数: ${episodes.length}`);
|
||||
const { insertedCount, scriptCount } = await this.saveOutlineData(episodes as EpisodeData[], overwrite, startEpisode);
|
||||
return `大纲保存成功：插入 ${insertedCount} 集大纲，创建 ${scriptCount} 个剧本记录`;
|
||||
},
|
||||
{
|
||||
name: "saveOutline",
|
||||
description: "保存大纲数据。overwrite=true会清空现有大纲后写入，false则追加到末尾",
|
||||
schema: z.object({
|
||||
episodes: z.array(episodeSchema).min(1).describe("大纲数据数组"),
|
||||
overwrite: z.boolean().default(true).describe("是否覆盖现有大纲"),
|
||||
startEpisode: z.number().optional().describe("追加模式下的起始集数（不填则自动递增）"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
updateOutline = tool(
|
||||
async ({ id, data }) => {
|
||||
this.log("更新大纲", `ID: ${id}`);
|
||||
const success = await this.updateOutlineData(id, data as EpisodeData);
|
||||
return success ? `大纲ID ${id} 更新成功` : `未找到大纲ID: ${id}`;
|
||||
},
|
||||
{
|
||||
name: "updateOutline",
|
||||
description: "更新指定ID的单集大纲内容",
|
||||
schema: z.object({
|
||||
id: z.number().describe("大纲ID"),
|
||||
data: episodeSchema.describe("更新后的大纲数据"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
deleteOutline = tool(
|
||||
async ({ ids }) => {
|
||||
this.log("删除大纲", `IDs: ${ids.join(", ")}`);
|
||||
const results = await this.deleteOutlineData(ids);
|
||||
const summary = results.map((r, i) => `ID ${ids[i]}: ${r.status === "fulfilled" ? "成功" : "失败"}`).join(", ");
|
||||
return `删除结果: ${summary}`;
|
||||
},
|
||||
{
|
||||
name: "deleteOutline",
|
||||
description: "根据大纲ID删除指定大纲及关联数据",
|
||||
schema: z.object({
|
||||
ids: z.array(z.number()).min(1).describe("要删除的大纲ID数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
// ==================== Tool 定义：章节 ====================
|
||||
|
||||
getChapter = tool(
|
||||
async ({ chapterNumbers }) => {
|
||||
this.log("获取章节", `章节号: ${chapterNumbers.join(", ")}`);
|
||||
|
||||
const results = await Promise.all(
|
||||
chapterNumbers.map(async (num) => {
|
||||
const chapter = await u
|
||||
.db("t_novel")
|
||||
.where({ projectId: this.projectId, chapterIndex: num })
|
||||
.select("chapterData", "chapterIndex", "chapter")
|
||||
.first();
|
||||
|
||||
if (chapter) {
|
||||
return `\n【第${chapter.chapterIndex}章 ${chapter.chapter || ""}】\n${chapter.chapterData}`;
|
||||
}
|
||||
return `\n【第${num}章】未找到`;
|
||||
}),
|
||||
);
|
||||
|
||||
return results.join("\n\n---\n");
|
||||
},
|
||||
{
|
||||
name: "getChapter",
|
||||
description: "根据章节编号获取小说章节的完整原文内容，支持批量获取",
|
||||
schema: z.object({
|
||||
chapterNumbers: z.array(z.number()).min(1).describe("章节编号数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
// ==================== Tool 定义：资产 ====================
|
||||
|
||||
generateAssets = tool(
|
||||
async () => {
|
||||
this.log("生成资产");
|
||||
const stats = await this.generateAssetsFromOutlines();
|
||||
|
||||
if (stats.inserted === 0 && stats.updated === 0 && stats.skipped === 0) {
|
||||
return "当前项目没有大纲数据，无法生成资产";
|
||||
}
|
||||
|
||||
return `资产生成完成：新增 ${stats.inserted}，更新 ${stats.updated}，保持 ${stats.skipped}`;
|
||||
},
|
||||
{
|
||||
name: "generateAssets",
|
||||
description: "从当前项目的所有大纲中提取并生成角色、道具、场景资产，自动去重并清理冗余",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
// ==================== 上下文构建 ====================
|
||||
|
||||
private getChapterContext(): string {
|
||||
if (!this.novelChapters.length) return "无章节数据";
|
||||
return this.novelChapters.map((c) => `章节号:${c.chapterIndex}，分卷:${c.reel}，章节名:${c.chapter}`).join("\n");
|
||||
}
|
||||
|
||||
private async buildEnvironmentContext(): Promise<string> {
|
||||
const [novelInfo, storyline, outlineCount] = await Promise.all([
|
||||
this.getNovelInfo(true),
|
||||
this.findStoryline(),
|
||||
u.db("t_outline").where({ projectId: this.projectId }).count("id as count").first() as any,
|
||||
]);
|
||||
|
||||
return `<环境信息>
|
||||
项目ID: ${this.projectId}
|
||||
系统时间: ${new Date().toLocaleString()}
|
||||
|
||||
${novelInfo}
|
||||
|
||||
已加载章节列表:
|
||||
${this.getChapterContext()}
|
||||
|
||||
故事线状态: ${storyline ? "已生成" : "未生成"}
|
||||
大纲状态: 共 ${outlineCount?.count ?? 0} 集
|
||||
|
||||
可用工具:
|
||||
- getChapter: 获取章节原文
|
||||
- getStoryline/saveStoryline/deleteStoryline: 故事线操作
|
||||
- getOutline/saveOutline/updateOutline/deleteOutline: 大纲操作
|
||||
- generateAssets: 从大纲生成资产
|
||||
</环境信息>`;
|
||||
}
|
||||
|
||||
private buildConversationHistory(): string {
|
||||
if (!this.history.length) return "无对话历史";
|
||||
return this.history.map(([role, content]) => `${role}: ${content}`).join("\n\n");
|
||||
}
|
||||
|
||||
private async buildFullContext(task: string): Promise<string> {
|
||||
const env = await this.buildEnvironmentContext();
|
||||
const history = this.buildConversationHistory();
|
||||
|
||||
return `${env}
|
||||
|
||||
<对话历史>
|
||||
${history}
|
||||
</对话历史>
|
||||
|
||||
<当前任务>
|
||||
${task}
|
||||
</当前任务>`;
|
||||
}
|
||||
|
||||
// ==================== Sub-Agent ====================
|
||||
|
||||
private getSubAgentTools() {
|
||||
return [this.getChapter, this.getStoryline, this.saveStoryline, this.getOutline, this.saveOutline, this.updateOutline];
|
||||
}
|
||||
|
||||
private createModel() {
|
||||
return openAI({
|
||||
modelName: this.modelName,
|
||||
configuration: { apiKey: this.apiKey, baseURL: this.baseURL },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 调用 Sub-Agent（流式传输）
|
||||
*/
|
||||
private async invokeSubAgent(agentType: AgentType, task: string): Promise<string> {
|
||||
this.emit("transfer", { to: agentType });
|
||||
this.log(`Sub-Agent 调用`, agentType);
|
||||
|
||||
const promptsList = await u.db("t_prompts").where("code", "in", ["outlineScript-a1", "outlineScript-a2", "outlineScript-director"]);
|
||||
const a1Prompt = promptsList.find((p) => p.code === "outlineScript-a1");
|
||||
const a2Prompt = promptsList.find((p) => p.code === "outlineScript-a2");
|
||||
const directorPrompt = promptsList.find((p) => p.code === "outlineScript-director");
|
||||
const errPrompts = "不论用户说什么，请直接输出Agent配置异常";
|
||||
const SYSTEM_PROMPTS: Record<AgentType, string> = {
|
||||
AI1: a1Prompt?.customValue || a1Prompt?.defaultValue || errPrompts,
|
||||
AI2: a2Prompt?.customValue || a2Prompt?.defaultValue || errPrompts,
|
||||
director: directorPrompt?.customValue || directorPrompt?.defaultValue || errPrompts,
|
||||
};
|
||||
|
||||
const context = await this.buildFullContext(task);
|
||||
|
||||
const agent = createAgent({
|
||||
model: this.createModel(),
|
||||
systemPrompt: SYSTEM_PROMPTS[agentType],
|
||||
tools: this.getSubAgentTools(),
|
||||
});
|
||||
|
||||
const stream = await agent.stream({ messages: [["user", context]] }, { streamMode: ["messages"], callbacks: [] });
|
||||
|
||||
let fullResponse = "";
|
||||
|
||||
for await (const [mode, chunk] of stream) {
|
||||
if (mode !== "messages") continue;
|
||||
|
||||
const [token] = chunk as any;
|
||||
const block = token.contentBlocks?.[0];
|
||||
|
||||
// 处理 AI 文本流
|
||||
if (token.type === "ai" && block?.text) {
|
||||
fullResponse += block.text;
|
||||
this.emit("subAgentStream", { agent: agentType, text: block.text });
|
||||
}
|
||||
|
||||
// 处理 tool 调用
|
||||
if (token.type === "ai" && token.tool_calls?.length) {
|
||||
for (const toolCall of token.tool_calls) {
|
||||
this.emit("toolCall", { agent: agentType, name: toolCall.name, args: toolCall.args });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.emit("subAgentEnd", { agent: agentType });
|
||||
this.history.push(["ai", fullResponse]);
|
||||
this.log(`Sub-Agent 完成`, agentType);
|
||||
|
||||
return fullResponse ?? `${agentType}已完成任务`;
|
||||
}
|
||||
|
||||
private createSubAgentTool(agentType: AgentType, description: string) {
|
||||
return tool(async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), {
|
||||
name: agentType,
|
||||
description,
|
||||
schema: z.object({
|
||||
taskDescription: z.string().describe("具体的任务描述，包含章节范围、修改要求等详细信息"),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
// ==================== 主入口 ====================
|
||||
|
||||
private getAllTools() {
|
||||
return [
|
||||
this.createSubAgentTool("AI1", "调用故事师。负责分析小说原文并生成故事线，会自行调用 saveStoryline 保存结果。"),
|
||||
this.createSubAgentTool("AI2", "调用大纲师。负责根据故事线生成剧集大纲，会自行调用 saveOutline 保存结果。"),
|
||||
this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲，会自行调用 updateOutline 或 saveStoryline 进行修改。"),
|
||||
this.getChapter,
|
||||
this.getStoryline,
|
||||
this.saveStoryline,
|
||||
this.deleteStoryline,
|
||||
this.getOutline,
|
||||
this.saveOutline,
|
||||
this.updateOutline,
|
||||
this.deleteOutline,
|
||||
this.generateAssets,
|
||||
];
|
||||
}
|
||||
|
||||
async call(msg: string): Promise<string> {
|
||||
this.history.push(["user", msg]);
|
||||
|
||||
const envContext = await this.buildEnvironmentContext();
|
||||
|
||||
const prompts = await u.db("t_prompts").where("code", "outlineScript-main").first();
|
||||
|
||||
const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常";
|
||||
|
||||
const mainAgent = createAgent({
|
||||
model: this.createModel(),
|
||||
tools: this.getAllTools(),
|
||||
systemPrompt: `${envContext}\n${mainPrompts}`,
|
||||
});
|
||||
const stream = await mainAgent.stream({ messages: this.history }, { streamMode: ["messages"], callbacks: [] });
|
||||
|
||||
let fullResponse = "";
|
||||
|
||||
for await (const [mode, chunk] of stream) {
|
||||
if (mode !== "messages") continue;
|
||||
|
||||
const [token] = chunk as any;
|
||||
const block = token.contentBlocks?.[0];
|
||||
|
||||
// 处理 AI 文本流
|
||||
if (token.type === "ai" && block?.text) {
|
||||
fullResponse += block.text;
|
||||
this.emit("data", block.text);
|
||||
}
|
||||
|
||||
// 处理 tool 调用
|
||||
if (token.type === "ai" && token.tool_calls?.length) {
|
||||
for (const toolCall of token.tool_calls) {
|
||||
this.emit("toolCall", { agent: "main", name: toolCall.name, args: toolCall.args });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.history.push(["assistant", fullResponse]);
|
||||
this.emit("response", fullResponse);
|
||||
|
||||
return fullResponse;
|
||||
}
|
||||
}
|
||||
130
backup/agents/storyboard/generateImagePromptsTool.ts
Normal file
130
backup/agents/storyboard/generateImagePromptsTool.ts
Normal file
@ -0,0 +1,130 @@
|
||||
import u from "@/utils";
|
||||
|
||||
type AspectRatio = "16:9" | "9:16" | "21:9" | "1:1" | "4:3" | "3:4" | "3:2" | "2:3";
|
||||
|
||||
interface GridLayoutResult {
|
||||
cols: number;
|
||||
rows: number;
|
||||
totalCells: number;
|
||||
placeholderCount: number;
|
||||
}
|
||||
|
||||
interface GridPromptOptions {
|
||||
prompts: string[];
|
||||
style: string;
|
||||
aspectRatio: AspectRatio;
|
||||
assetsName: { name: string; intro: string }[];
|
||||
}
|
||||
|
||||
interface GridPromptResult {
|
||||
prompt: string;
|
||||
gridLayout: GridLayoutResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据prompts数量计算宫格布局
|
||||
*/
|
||||
function calculateGridLayout(count: number): GridLayoutResult {
|
||||
let cols: number;
|
||||
let rows: number;
|
||||
if (count <= 0) {
|
||||
cols = 1;
|
||||
rows = 1;
|
||||
} else if (count === 1) {
|
||||
cols = 1;
|
||||
rows = 1;
|
||||
} else if (count === 2) {
|
||||
cols = 2;
|
||||
rows = 1;
|
||||
} else if (count === 3) {
|
||||
cols = 3;
|
||||
rows = 1;
|
||||
} else if (count === 4) {
|
||||
cols = 2;
|
||||
rows = 2;
|
||||
} else if (count <= 9) {
|
||||
cols = 3;
|
||||
rows = 3;
|
||||
} else {
|
||||
cols = 3;
|
||||
rows = Math.ceil(count / 3);
|
||||
}
|
||||
const totalCells = cols * rows;
|
||||
const placeholderCount = totalCells - count;
|
||||
return { cols, rows, totalCells, placeholderCount };
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取宽高比描述
|
||||
*/
|
||||
function getAspectRatioDescription(aspectRatio: AspectRatio): string {
|
||||
const descriptions: Record<AspectRatio, string> = {
|
||||
"16:9": "电影宽银幕",
|
||||
"9:16": "竖屏短剧",
|
||||
"21:9": "超宽银幕史诗感",
|
||||
"1:1": "方形构图",
|
||||
"4:3": "经典银幕",
|
||||
"3:4": "竖版经典",
|
||||
"3:2": "摄影标准",
|
||||
"2:3": "竖版摄影",
|
||||
};
|
||||
return descriptions[aspectRatio] || "标准比例";
|
||||
}
|
||||
|
||||
/**
|
||||
* 生成电影级宫格分镜提示词
|
||||
*/
|
||||
async function generateGridPrompt(options: GridPromptOptions): Promise<GridPromptResult> {
|
||||
const { prompts, style, aspectRatio, assetsName } = options;
|
||||
const layout = calculateGridLayout(prompts.length);
|
||||
const aspectRatioDesc = getAspectRatioDescription(aspectRatio);
|
||||
|
||||
// 构建宫格位置描述
|
||||
const gridPositions: string[] = [];
|
||||
for (let i = 0; i < layout.totalCells; i++) {
|
||||
const row = Math.floor(i / layout.cols) + 1;
|
||||
const col = (i % layout.cols) + 1;
|
||||
if (i < prompts.length) {
|
||||
gridPositions.push(`[第${row}行第${col}列]: ${prompts[i]}`);
|
||||
} else {
|
||||
gridPositions.push(`[第${row}行第${col}列]: 纯黑图`);
|
||||
}
|
||||
}
|
||||
|
||||
// 构建资产说明
|
||||
const assetsSection =
|
||||
assetsName.length > 0
|
||||
? `\n【可用资产】\n${assetsName.map((a) => `- ${a.name}：${a.intro}`).join("\n")}\n\n⚠️ 必须使用完整资产名称，禁止简称或代词。`
|
||||
: "";
|
||||
|
||||
const promptsData = await u.db("t_prompts").where("code", "generateImagePrompts").first();
|
||||
|
||||
const mainPrompts = promptsData?.customValue || promptsData?.defaultValue;
|
||||
const errData = `请输出${options.prompts.length}张图片\n提示词如下:\n${options.prompts.map((p, i) => `第${i + 1}格: ${p}`).join("\n")}`;
|
||||
|
||||
if (!mainPrompts) return { prompt: errData, gridLayout: layout };
|
||||
|
||||
const chatModel = await u.ai.text({});
|
||||
|
||||
const result = await chatModel!.invoke({
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: mainPrompts,
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: `请优化以下分镜提示词：\n\n【布局】${layout.cols}列×${layout.rows}行=${
|
||||
layout.totalCells
|
||||
}格\n【比例】${aspectRatio}（${aspectRatioDesc}）\n【风格】${style}\n${assetsSection}\n\n【原始内容】\n${gridPositions.join("\n")}`,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
return {
|
||||
prompt: result?.text ?? errData,
|
||||
gridLayout: layout,
|
||||
};
|
||||
}
|
||||
|
||||
export default generateGridPrompt;
|
||||
334
backup/agents/storyboard/generateImageTool.ts
Normal file
334
backup/agents/storyboard/generateImageTool.ts
Normal file
@ -0,0 +1,334 @@
|
||||
import generateImagePromptsTool from "@/agents/storyboard/generateImagePromptsTool";
|
||||
import u from "@/utils";
|
||||
import sharp from "sharp";
|
||||
import { z } from "zod";
|
||||
|
||||
interface AssetItem {
|
||||
name: string;
|
||||
description: string;
|
||||
}
|
||||
|
||||
interface EpisodeData {
|
||||
episodeIndex: number;
|
||||
title: string;
|
||||
chapterRange: number[];
|
||||
scenes: AssetItem[];
|
||||
characters: AssetItem[];
|
||||
props: AssetItem[];
|
||||
coreConflict: string;
|
||||
openingHook: string;
|
||||
outline: string;
|
||||
keyEvents: string[];
|
||||
emotionalCurve: string;
|
||||
visualHighlights: string[];
|
||||
endingHook: string;
|
||||
classicQuotes: string[];
|
||||
}
|
||||
|
||||
interface ImageInfo {
|
||||
name: string;
|
||||
type: string;
|
||||
filePath: string;
|
||||
}
|
||||
|
||||
interface ResourceItem {
|
||||
name: string;
|
||||
intro: string;
|
||||
}
|
||||
|
||||
// 资产过滤响应的 schema
|
||||
const filteredAssetsSchema = z.object({
|
||||
relevantAssets: z
|
||||
.array(
|
||||
z.object({
|
||||
name: z.string().describe("资产名称"),
|
||||
reason: z.string().describe("选择该资产的原因"),
|
||||
}),
|
||||
)
|
||||
.describe("与分镜内容相关的资产列表"),
|
||||
});
|
||||
|
||||
// 压缩图片直到不超过指定大小
|
||||
async function compressImage(buffer: Buffer, maxSizeBytes: number = 3 * 1024 * 1024): Promise<Buffer> {
|
||||
if (buffer.length <= maxSizeBytes) {
|
||||
return buffer;
|
||||
}
|
||||
let quality = 90;
|
||||
let compressedBuffer = await sharp(buffer).jpeg({ quality }).toBuffer();
|
||||
while (compressedBuffer.length > maxSizeBytes && quality > 10) {
|
||||
quality -= 10;
|
||||
compressedBuffer = await sharp(buffer).jpeg({ quality }).toBuffer();
|
||||
}
|
||||
if (compressedBuffer.length > maxSizeBytes) {
|
||||
const metadata = await sharp(buffer).metadata();
|
||||
let scale = 0.9;
|
||||
while (compressedBuffer.length > maxSizeBytes && scale > 0.1) {
|
||||
const newWidth = Math.round((metadata.width || 1000) * scale);
|
||||
const newHeight = Math.round((metadata.height || 1000) * scale);
|
||||
compressedBuffer = await sharp(buffer)
|
||||
.resize(newWidth, newHeight, { fit: "inside" })
|
||||
.jpeg({ quality: Math.max(quality, 30) })
|
||||
.toBuffer();
|
||||
scale -= 0.1;
|
||||
}
|
||||
}
|
||||
return compressedBuffer;
|
||||
}
|
||||
|
||||
// 拼接多张图片为一张
|
||||
async function mergeImages(imagePaths: string[]): Promise<Buffer> {
|
||||
const imageBuffers = await Promise.all(imagePaths.map((path) => u.oss.getFile(path)));
|
||||
const imageMetadatas = await Promise.all(imageBuffers.map((buffer) => sharp(buffer).metadata()));
|
||||
const maxHeight = Math.max(...imageMetadatas.map((m) => m.height || 0));
|
||||
const resizedImages = await Promise.all(
|
||||
imageBuffers.map(async (buffer, index) => {
|
||||
const metadata = imageMetadatas[index];
|
||||
const aspectRatio = (metadata.width || 1) / (metadata.height || 1);
|
||||
const newWidth = Math.round(maxHeight * aspectRatio);
|
||||
return {
|
||||
buffer: await sharp(buffer).resize(newWidth, maxHeight, { fit: "cover" }).toBuffer(),
|
||||
width: newWidth,
|
||||
};
|
||||
}),
|
||||
);
|
||||
let currentX = 0;
|
||||
const compositeInputs = resizedImages.map(({ buffer, width }) => {
|
||||
const input = {
|
||||
input: buffer,
|
||||
left: currentX,
|
||||
top: 0,
|
||||
};
|
||||
currentX += width;
|
||||
return input;
|
||||
});
|
||||
const mergedImage = await sharp({
|
||||
create: {
|
||||
width: currentX,
|
||||
height: maxHeight,
|
||||
channels: 4,
|
||||
background: { r: 255, g: 255, b: 255, alpha: 1 },
|
||||
},
|
||||
})
|
||||
.composite(compositeInputs)
|
||||
.jpeg({ quality: 90 })
|
||||
.toBuffer();
|
||||
return compressImage(mergedImage);
|
||||
}
|
||||
|
||||
// 进一步压缩单张图片到指定大小
|
||||
async function compressToSize(buffer: Buffer, targetSize: number): Promise<Buffer> {
|
||||
if (buffer.length <= targetSize) {
|
||||
return buffer;
|
||||
}
|
||||
|
||||
const metadata = await sharp(buffer).metadata();
|
||||
let quality = 80;
|
||||
let scale = 1.0;
|
||||
let compressedBuffer = buffer;
|
||||
|
||||
// 先尝试降低质量
|
||||
while (compressedBuffer.length > targetSize && quality > 10) {
|
||||
compressedBuffer = await sharp(buffer).jpeg({ quality }).toBuffer();
|
||||
quality -= 10;
|
||||
}
|
||||
|
||||
// 如果还是太大，缩小尺寸
|
||||
while (compressedBuffer.length > targetSize && scale > 0.2) {
|
||||
scale -= 0.1;
|
||||
const newWidth = Math.round((metadata.width || 1000) * scale);
|
||||
const newHeight = Math.round((metadata.height || 1000) * scale);
|
||||
compressedBuffer = await sharp(buffer)
|
||||
.resize(newWidth, newHeight, { fit: "inside" })
|
||||
.jpeg({ quality: Math.max(quality, 20) })
|
||||
.toBuffer();
|
||||
}
|
||||
|
||||
return compressedBuffer;
|
||||
}
|
||||
|
||||
// 确保图片列表总大小不超过指定限制
|
||||
async function ensureTotalSizeLimit(buffers: Buffer[], maxTotalBytes: number = 10 * 1024 * 1024): Promise<Buffer[]> {
|
||||
let totalSize = buffers.reduce((sum, buf) => sum + buf.length, 0);
|
||||
|
||||
if (totalSize <= maxTotalBytes) {
|
||||
return buffers;
|
||||
}
|
||||
|
||||
// 计算每张图片的平均目标大小
|
||||
const avgTargetSize = Math.floor(maxTotalBytes / buffers.length);
|
||||
|
||||
// 按大小降序排列，优先压缩大图片
|
||||
const indexedBuffers = buffers.map((buf, idx) => ({ buf, idx, size: buf.length }));
|
||||
indexedBuffers.sort((a, b) => b.size - a.size);
|
||||
|
||||
const result = [...buffers];
|
||||
|
||||
for (const item of indexedBuffers) {
|
||||
totalSize = result.reduce((sum, buf) => sum + buf.length, 0);
|
||||
if (totalSize <= maxTotalBytes) {
|
||||
break;
|
||||
}
|
||||
|
||||
// 计算这张图片需要压缩到的目标大小
|
||||
const excessSize = totalSize - maxTotalBytes;
|
||||
const targetSize = Math.max(item.buf.length - excessSize, avgTargetSize, 100 * 1024); // 最小100KB
|
||||
|
||||
if (item.buf.length > targetSize) {
|
||||
result[item.idx] = await compressToSize(item.buf, targetSize);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// 处理图片列表，确保不超过10张且每张不超过3MB，总大小不超过10MB
|
||||
async function processImages(images: ImageInfo[]): Promise<Buffer[]> {
|
||||
const maxImages = 10;
|
||||
let processedBuffers: Buffer[];
|
||||
|
||||
if (images.length <= maxImages) {
|
||||
const buffers = await Promise.all(images.map((img) => u.oss.getFile(img.filePath)));
|
||||
processedBuffers = await Promise.all(buffers.map((buffer) => compressImage(buffer)));
|
||||
} else {
|
||||
const mergeStartIndex = maxImages - 1;
|
||||
const firstBuffers = await Promise.all(images.slice(0, mergeStartIndex).map((img) => u.oss.getFile(img.filePath)));
|
||||
const compressedFirstImages = await Promise.all(firstBuffers.map((buffer) => compressImage(buffer)));
|
||||
const imagesToMergeList = images.slice(mergeStartIndex).map((img) => img.filePath);
|
||||
const mergedImage = await mergeImages(imagesToMergeList);
|
||||
processedBuffers = [...compressedFirstImages, mergedImage];
|
||||
}
|
||||
|
||||
// 确保总大小不超过10MB
|
||||
return ensureTotalSizeLimit(processedBuffers);
|
||||
}
|
||||
|
||||
// 使用 AI 过滤与分镜相关的资产
|
||||
async function filterRelevantAssets(prompts: string[], allResources: ResourceItem[], availableImages: ImageInfo[]): Promise<ImageInfo[]> {
|
||||
if (allResources.length === 0 || availableImages.length === 0) {
|
||||
return availableImages;
|
||||
}
|
||||
|
||||
const availableNames = new Set(availableImages.map((img) => img.name));
|
||||
const availableResources = allResources.filter((r) => availableNames.has(r.name));
|
||||
|
||||
if (availableResources.length === 0) {
|
||||
return availableImages;
|
||||
}
|
||||
|
||||
const chatModel = await u.ai.text({});
|
||||
const result = await chatModel!.invoke({
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: `请分析以下分镜描述，从可用资产中筛选出与分镜内容直接相关的资产。
|
||||
|
||||
分镜描述：
|
||||
${prompts.map((p, i) => `${i + 1}. ${p}`).join("\n")}
|
||||
|
||||
可用资产列表：
|
||||
${availableResources.map((r) => `- ${r.name}：${r.intro}`).join("\n")}
|
||||
|
||||
请仅选择在分镜中明确出现或被提及的角色、场景、道具。不要选择与分镜内容无关的资产。`,
|
||||
},
|
||||
],
|
||||
responseFormat: {
|
||||
type: "json_schema",
|
||||
jsonSchema: {
|
||||
name: "filteredAssets",
|
||||
strict: true,
|
||||
schema: z.toJSONSchema(filteredAssetsSchema),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const data = result?.json as z.infer<typeof filteredAssetsSchema>;
|
||||
|
||||
if (!data?.relevantAssets || data.relevantAssets.length === 0) {
|
||||
return availableImages;
|
||||
}
|
||||
|
||||
const relevantNames = new Set(data.relevantAssets.map((a) => a.name));
|
||||
const filteredImages = availableImages.filter((img) => relevantNames.has(img.name));
|
||||
|
||||
return filteredImages.length > 0 ? filteredImages : availableImages;
|
||||
}
|
||||
|
||||
// 构建资产映射提示词
|
||||
function buildResourcesMapPrompts(images: ImageInfo[]): string {
|
||||
if (images.length === 0) return "";
|
||||
|
||||
const mapping = images.map((item, index) => {
|
||||
if (index < 9) {
|
||||
return `${item.name}=图片${index + 1}`;
|
||||
} else {
|
||||
return `${item.name}=图10-${index - 8}`;
|
||||
}
|
||||
});
|
||||
|
||||
return `其中人物、场景、道具参考对照关系如下：${mapping.join(", ")}。`;
|
||||
}
|
||||
|
||||
export default async (cells: { prompt: string }[], scriptId: number, projectId: number) => {
|
||||
const scriptData = await u.db("t_script").where({ id: scriptId, projectId }).first();
|
||||
const projectInfo = await u.db("t_project").where({ id: projectId }).first();
|
||||
|
||||
const row = await u.db("t_outline").where({ id: scriptData?.outlineId!, projectId }).first();
|
||||
const outline: EpisodeData | null = row?.data ? JSON.parse(row.data) : null;
|
||||
|
||||
const resources: ResourceItem[] = outline
|
||||
? (["characters", "props", "scenes"] as const).flatMap((k) => outline[k]?.map((i) => ({ name: i.name, intro: i.description })) ?? [])
|
||||
: [];
|
||||
|
||||
const resourceNames = resources.map((r) => r.name);
|
||||
const imagesRaw = await u.db("t_assets").whereIn("name", resourceNames).andWhere({ projectId }).select("name", "type", "filePath");
|
||||
|
||||
const allImages = imagesRaw
|
||||
.sort((a, b) => {
|
||||
const order = ["角色", "场景", "道具"];
|
||||
return order.indexOf(a.type!) - order.indexOf(b.type!);
|
||||
})
|
||||
.filter((img) => img.filePath) as ImageInfo[];
|
||||
|
||||
if (allImages.length === 0) {
|
||||
throw new Error("未找到可用的图片资源");
|
||||
}
|
||||
|
||||
const cellPrompts = cells.map((c) => c.prompt);
|
||||
|
||||
// 使用 AI 过滤相关资产
|
||||
const filteredImages = await filterRelevantAssets(cellPrompts, resources, allImages);
|
||||
|
||||
const resourcesMapPrompts = buildResourcesMapPrompts(filteredImages);
|
||||
console.log("====润色前：", cellPrompts);
|
||||
const promptsData = await generateImagePromptsTool({
|
||||
prompts: cellPrompts,
|
||||
style: `类型：${projectInfo?.type!}，风格：${projectInfo?.artStyle!}`,
|
||||
aspectRatio: projectInfo?.videoRatio! as any,
|
||||
assetsName: resources,
|
||||
});
|
||||
|
||||
// const prompts = `请生成${promptsData.gridLayout.totalCells}格,${promptsData.gridLayout.cols}列×${promptsData.gridLayout.rows}行宫格图。
|
||||
|
||||
// ${promptsData.prompt}
|
||||
|
||||
// 注意：请严格按照提示词内容生成图片，确保人物样貌、艺术风格、色调光影一致。
|
||||
// `;
|
||||
const prompts = promptsData.prompt;
|
||||
console.log("====润色后：", prompts);
|
||||
|
||||
const processedImages = await processImages(filteredImages);
|
||||
|
||||
const contentStr = await u.ai.generateImage({
|
||||
systemPrompt: resourcesMapPrompts,
|
||||
prompt: prompts,
|
||||
size: "4K",
|
||||
aspectRatio: projectInfo?.videoRatio ? (projectInfo.videoRatio as any) : "16:9",
|
||||
imageBase64: processedImages.map((buf) => buf.toString("base64")),
|
||||
});
|
||||
|
||||
const match = contentStr.match(/base64,([A-Za-z0-9+/=]+)/);
|
||||
const base64Str = match?.[1] ?? contentStr;
|
||||
const buffer = Buffer.from(base64Str, "base64");
|
||||
|
||||
return buffer;
|
||||
};
|
||||
94
backup/agents/storyboard/imageSplitting.ts
Normal file
94
backup/agents/storyboard/imageSplitting.ts
Normal file
@ -0,0 +1,94 @@
|
||||
import sharp from "sharp";
|
||||
|
||||
interface GridLayoutResult {
|
||||
cols: number;
|
||||
rows: number;
|
||||
totalCells: number;
|
||||
placeholderCount: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算宫格布局
|
||||
* 1张: 1x1
|
||||
* 2张: 2x1
|
||||
* 3张: 3x1
|
||||
* 4张: 2x2
|
||||
* 5-9张: 3x3
|
||||
* 10-12张: 3x4
|
||||
* 13-15张: 3x5
|
||||
* ...以此类推（3列，行数递增）
|
||||
*/
|
||||
function calculateGridLayout(count: number): GridLayoutResult {
|
||||
let cols: number;
|
||||
let rows: number;
|
||||
if (count <= 0) {
|
||||
cols = 1;
|
||||
rows = 1;
|
||||
} else if (count === 1) {
|
||||
cols = 1;
|
||||
rows = 1;
|
||||
} else if (count === 2) {
|
||||
cols = 2;
|
||||
rows = 1;
|
||||
} else if (count === 3) {
|
||||
cols = 3;
|
||||
rows = 1;
|
||||
} else if (count === 4) {
|
||||
cols = 2;
|
||||
rows = 2;
|
||||
} else if (count <= 9) {
|
||||
// 5-9格统一用3x3
|
||||
cols = 3;
|
||||
rows = 3;
|
||||
} else {
|
||||
cols = 3;
|
||||
rows = Math.ceil(count / 3);
|
||||
}
|
||||
const totalCells = cols * rows;
|
||||
const placeholderCount = totalCells - count;
|
||||
return { cols, rows, totalCells, placeholderCount };
|
||||
}
|
||||
|
||||
/**
|
||||
* 分割宫格图片
|
||||
* @param image - 输入的宫格图片 Buffer
|
||||
* @param length - 实际需要的图片数量（不包含占位图）
|
||||
* @returns 分割后的单张图片 Buffer 数组
|
||||
*/
|
||||
export default async (image: Buffer, length: number): Promise<Buffer[]> => {
|
||||
const metadata = await sharp(image).metadata();
|
||||
const { width: totalWidth, height: totalHeight } = metadata;
|
||||
|
||||
if (!totalWidth || !totalHeight) {
|
||||
throw new Error("无法获取图片尺寸");
|
||||
}
|
||||
|
||||
const { cols, rows } = calculateGridLayout(length);
|
||||
|
||||
const cellWidth = Math.floor(totalWidth / cols);
|
||||
const cellHeight = Math.floor(totalHeight / rows);
|
||||
|
||||
const buffers: Buffer[] = [];
|
||||
|
||||
for (let i = 0; i < length; i++) {
|
||||
const row = Math.floor(i / cols);
|
||||
const col = i % cols;
|
||||
|
||||
const left = col * cellWidth;
|
||||
const top = row * cellHeight;
|
||||
|
||||
const cellBuffer = await sharp(image)
|
||||
.extract({
|
||||
left,
|
||||
top,
|
||||
width: cellWidth,
|
||||
height: cellHeight,
|
||||
})
|
||||
.png()
|
||||
.toBuffer();
|
||||
|
||||
buffers.push(cellBuffer);
|
||||
}
|
||||
|
||||
return buffers;
|
||||
};
|
||||
737
backup/agents/storyboard/index.ts
Normal file
737
backup/agents/storyboard/index.ts
Normal file
@ -0,0 +1,737 @@
|
||||
// @/agents/Storyboard.ts
|
||||
import u from "@/utils";
|
||||
import { createAgent } from "langchain";
|
||||
import { EventEmitter } from "events";
|
||||
import { openAI } from "@/agents/models";
|
||||
import { z } from "zod";
|
||||
import { tool } from "@langchain/core/tools";
|
||||
import type { DB } from "@/types/database";
|
||||
import generateImageTool from "./generateImageTool";
|
||||
import imageSplitting from "./imageSplitting";
|
||||
|
||||
// ==================== 类型定义 ====================
|
||||
|
||||
type AgentType = "segmentAgent" | "shotAgent";
|
||||
type RefreshEvent = "storyline" | "outline" | "assets";
|
||||
|
||||
// ==================== 常量配置 ====================
|
||||
|
||||
// const SYSTEM_PROMPTS: Record<AgentType, string> = {
|
||||
// segmentAgent: segmentPrompts,
|
||||
// shotAgent: shotPrompts,
|
||||
// director: directorPrompts,
|
||||
// };
|
||||
|
||||
// ==================== 类型定义：片段和画面 ====================
|
||||
|
||||
interface Segment {
|
||||
index: number;
|
||||
description: string;
|
||||
emotion?: string;
|
||||
action?: string;
|
||||
}
|
||||
|
||||
interface Shot {
|
||||
id: number; // 分镜独立ID
|
||||
segmentId: number; // 所属片段ID
|
||||
title: string;
|
||||
x: number;
|
||||
y: number;
|
||||
cells: Array<{ src?: string; prompt?: string; id?: string }>; // 镜头数组，每个cell是一个镜头
|
||||
}
|
||||
|
||||
// ==================== 主类 ====================
|
||||
|
||||
export default class Storyboard {
|
||||
private readonly projectId: number;
|
||||
private readonly scriptId: number;
|
||||
readonly emitter = new EventEmitter();
|
||||
history: Array<[string, string]> = [];
|
||||
novelChapters: DB["t_novel"][] = [];
|
||||
|
||||
// 存储 segmentAgent 生成的片段结果
|
||||
private segments: Segment[] = [];
|
||||
// 存储 shotAgent 生成的分镜结果
|
||||
private shots: Shot[] = [];
|
||||
// 分镜ID计数器
|
||||
private shotIdCounter: number = 0;
|
||||
// 存储正在生成分镜图的分镜ID
|
||||
private generatingShots: Set<number> = new Set();
|
||||
|
||||
modelName = "gpt-4.1";
|
||||
apiKey = "";
|
||||
baseURL = "";
|
||||
|
||||
constructor(projectId: number, scriptId: number) {
|
||||
this.projectId = projectId;
|
||||
this.scriptId = scriptId;
|
||||
}
|
||||
|
||||
// 更新shopts
|
||||
public updatePreShots(segmentId: number, cellId: number, cell: { src?: string; prompt?: string; id?: string }) {
|
||||
console.log("%c Line:76 🍤 segmentId", "background:#465975", segmentId);
|
||||
console.log("%c Line:76 🍷 cellId", "background:#ffdd4d", cellId);
|
||||
console.log("%c Line:76 🍢 cell", "background:#ffdd4d", cell);
|
||||
const shotIndex = this.shots.findIndex((item) => item.segmentId === segmentId);
|
||||
if (shotIndex === -1) {
|
||||
return `分镜 ${segmentId} 不存在，请检查分镜ID是否正确`;
|
||||
}
|
||||
const cellIndex = this.shots[shotIndex].cells.findIndex((item) => item.id === cellId.toString());
|
||||
if (cellIndex === -1) {
|
||||
return `镜头 ${cellId} 不存在，请检查镜头ID是否正确`;
|
||||
}
|
||||
this.shots[shotIndex].cells[cellIndex] = { ...this.shots[shotIndex].cells[cellIndex], ...cell };
|
||||
}
|
||||
|
||||
// ==================== 公共方法 ====================
|
||||
|
||||
get events() {
|
||||
return this.emitter;
|
||||
}
|
||||
// ==================== 私有工具方法 ====================
|
||||
|
||||
private emit(event: string, data?: any) {
|
||||
this.emitter.emit(event, data);
|
||||
}
|
||||
|
||||
private refresh(type: RefreshEvent) {
|
||||
this.emit("refresh", type);
|
||||
}
|
||||
|
||||
private log(action: string, detail?: string) {
|
||||
const msg = detail ? `${action}: ${detail}` : action;
|
||||
console.log(`\n[${new Date().toLocaleTimeString()}] ${msg}\n`);
|
||||
}
|
||||
|
||||
// ==================== 剧本相关操作 ====================
|
||||
|
||||
getScript = tool(
|
||||
async () => {
|
||||
this.log("获取剧本", `scriptId: ${this.scriptId}`);
|
||||
const script = await u.db("t_script").where({ id: this.scriptId, projectId: this.projectId }).first();
|
||||
if (!script) throw new Error("剧本不存在");
|
||||
return `剧本集：${script.name}\n\n内容：\n\`\`\`${script.content}\`\`\``;
|
||||
},
|
||||
{
|
||||
name: "getScript",
|
||||
description: "获取剧本内容",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
// ==================== 资产相关操作 ====================
|
||||
|
||||
/**
|
||||
* 获取资产列表（供 segmentAgent 和 shotAgent 调用）
|
||||
*/
|
||||
getAssets = tool(
|
||||
async () => {
|
||||
this.log("获取资产列表", `scriptId: ${this.scriptId}`);
|
||||
const scriptData = await u.db("t_script").where({ id: this.scriptId, projectId: this.projectId }).first();
|
||||
const row = await u.db("t_outline").where({ id: scriptData?.outlineId!, projectId: this.projectId }).first();
|
||||
const outline: any | null = row?.data ? JSON.parse(row.data) : null;
|
||||
|
||||
if (!outline) {
|
||||
return "暂无资产数据";
|
||||
}
|
||||
|
||||
// 提取资源名称和描述（与generateImageTool保持一致的字段名）
|
||||
const resources = outline
|
||||
? (["characters", "props", "scenes"] as const).flatMap(
|
||||
(k) => outline[k]?.map((i: any) => ({ name: i.name, description: i.description })) ?? [],
|
||||
)
|
||||
: [];
|
||||
|
||||
if (resources.length === 0) {
|
||||
return "暂无资产数据";
|
||||
}
|
||||
|
||||
// 分类提取资源并格式化
|
||||
const characters = outline?.characters?.map((item: any) => `- ${item.name}${item.description ? `：${item.description}` : ""}`) ?? [];
|
||||
const props = outline?.props?.map((item: any) => `- ${item.name}${item.description ? `：${item.description}` : ""}`) ?? [];
|
||||
const scenes = outline?.scenes?.map((item: any) => `- ${item.name}${item.description ? `：${item.description}` : ""}`) ?? [];
|
||||
|
||||
const sections = [
|
||||
characters.length ? `【角色】\n${characters.join("\n")}` : "",
|
||||
props.length ? `【道具】\n${props.join("\n")}` : "",
|
||||
scenes.length ? `【场景】\n${scenes.join("\n")}` : "",
|
||||
].filter(Boolean);
|
||||
|
||||
if (sections.length === 0) {
|
||||
return "暂无资产数据";
|
||||
}
|
||||
|
||||
return `<资产列表>
|
||||
${sections.join("\n\n")}
|
||||
</资产列表>
|
||||
|
||||
⚠️ 重要规则：
|
||||
1. 必须原封不动地使用上述资产名称，禁止使用近义词、缩写或任何变体
|
||||
2. 禁止在资产名称前后添加修饰词
|
||||
3. 禁止捏造资产列表中不存在的角色、场景、道具`;
|
||||
},
|
||||
{
|
||||
name: "getAssets",
|
||||
description: "获取资产列表（角色、道具、场景），包含名称和详细介绍。生成片段和分镜时必须先调用此工具获取资产信息，确保名称一致性",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
// ==================== 片段和分镜工具 ====================
|
||||
|
||||
/**
|
||||
* 获取当前存储的片段数据（供 shotAgent 调用）
|
||||
*/
|
||||
getSegments = tool(
|
||||
async () => {
|
||||
this.log("获取片段数据", `共 ${this.segments.length} 个片段`);
|
||||
if (this.segments.length === 0) {
|
||||
return "暂无片段数据，请先调用 segmentAgent 生成片段";
|
||||
}
|
||||
return JSON.stringify(this.segments, null, 2);
|
||||
},
|
||||
{
|
||||
name: "getSegments",
|
||||
description: "获取当前已生成的片段数据，用于生成分镜",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
/**
|
||||
* 更新/存储片段数据（供 segmentAgent 调用）
|
||||
*/
|
||||
updateSegments = tool(
|
||||
async ({ segments }: { segments: Segment[] }) => {
|
||||
this.log("更新片段数据", `共 ${segments.length} 个片段`);
|
||||
this.segments = segments;
|
||||
this.emit("segmentsUpdated", this.segments);
|
||||
return `成功存储 ${segments.length} 个片段`;
|
||||
},
|
||||
{
|
||||
name: "updateSegments",
|
||||
description: "存储生成的片段数据，segmentAgent 在生成片段后必须调用此工具保存结果",
|
||||
schema: z.object({
|
||||
segments: z
|
||||
.array(
|
||||
z.object({
|
||||
index: z.number().describe("片段序号"),
|
||||
description: z.string().describe("片段描述"),
|
||||
emotion: z.string().optional().describe("情绪氛围"),
|
||||
action: z.string().optional().describe("主要动作"),
|
||||
}),
|
||||
)
|
||||
.describe("片段数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
/**
|
||||
* 添加分镜（供 shotAgent 调用）
|
||||
*/
|
||||
addShots = tool(
|
||||
async ({ shots }: { shots: Array<{ segmentIndex: number; prompts: string[] }> }) => {
|
||||
const added: { id: number; segmentIndex: number }[] = [];
|
||||
const skipped: number[] = [];
|
||||
|
||||
for (const item of shots) {
|
||||
const exists = this.shots.some((f) => f.segmentId === item.segmentIndex);
|
||||
if (exists) {
|
||||
skipped.push(item.segmentIndex);
|
||||
continue;
|
||||
}
|
||||
// 分配独立的分镜ID
|
||||
this.shotIdCounter++;
|
||||
const shotId = this.shotIdCounter;
|
||||
this.shots.push({
|
||||
id: shotId,
|
||||
segmentId: item.segmentIndex,
|
||||
title: `分镜 ${shotId}`,
|
||||
x: 0,
|
||||
y: 0,
|
||||
cells: item.prompts.map((prompt) => ({ id: u.uuid(), prompt })),
|
||||
});
|
||||
added.push({ id: shotId, segmentIndex: item.segmentIndex });
|
||||
}
|
||||
|
||||
const addedInfo = added.map((a) => `分镜${a.id}(片段${a.segmentIndex})`).join(", ");
|
||||
this.log("添加分镜", `新增: [${addedInfo}], 跳过片段: [${skipped.join(", ")}]`);
|
||||
this.emit("shotsUpdated", this.shots);
|
||||
|
||||
if (skipped.length) {
|
||||
return `已添加${addedInfo}；片段 ${skipped.join(", ")} 已存在分镜被跳过。当前共 ${this.shots.length} 个分镜`;
|
||||
}
|
||||
return `已添加${addedInfo}。当前共 ${this.shots.length} 个分镜`;
|
||||
},
|
||||
{
|
||||
name: "addShots",
|
||||
description: "添加新的分镜。每个分镜有独立ID，包含多个镜头（每个镜头对应一个提示词）。如果片段已存在分镜会跳过",
|
||||
schema: z.object({
|
||||
shots: z
|
||||
.array(
|
||||
z.object({
|
||||
segmentIndex: z.number().describe("对应的片段序号"),
|
||||
prompts: z.array(z.string()).describe("镜头提示词数组，每个提示词对应一个镜头（中文）"),
|
||||
}),
|
||||
)
|
||||
.describe("要添加的分镜数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
/**
|
||||
* 更新指定分镜（供 shotAgent 调用）
|
||||
* 保留原有 cells 的 id 和 src 字段，只更新 prompt
|
||||
*/
|
||||
updateShots = tool(
|
||||
async ({ shotId, prompts }: { shotId: number; prompts: string[] }) => {
|
||||
const existingIndex = this.shots.findIndex((item) => item.id === shotId);
|
||||
|
||||
if (existingIndex === -1) {
|
||||
return `分镜 ${shotId} 不存在，请检查分镜ID是否正确`;
|
||||
}
|
||||
|
||||
const existingCells = this.shots[existingIndex].cells;
|
||||
|
||||
// 更新 cells，保留原有的 id 和 src 字段
|
||||
this.shots[existingIndex].cells = prompts.map((prompt, i) => {
|
||||
const existingCell = existingCells[i];
|
||||
if (existingCell) {
|
||||
// 保留原有 cell 的 id 和 src，只更新 prompt
|
||||
return { ...existingCell, prompt };
|
||||
} else {
|
||||
// 新增的 cell
|
||||
return { id: u.uuid(), prompt };
|
||||
}
|
||||
});
|
||||
|
||||
this.log("更新分镜", `分镜 ${shotId}`);
|
||||
this.emit("shotsUpdated", this.shots);
|
||||
|
||||
return `已更新分镜 ${shotId}`;
|
||||
},
|
||||
{
|
||||
name: "updateShots",
|
||||
description: "更新指定分镜的镜头提示词。通过分镜ID指定要修改的分镜",
|
||||
schema: z.object({
|
||||
shotId: z.number().describe("要更新的分镜ID"),
|
||||
prompts: z.array(z.string()).describe("新的镜头提示词数组，每个提示词对应一个镜头"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
/**
|
||||
* 删除指定分镜（供 shotAgent 调用）
|
||||
*/
|
||||
deleteShots = tool(
|
||||
async ({ shotIds }: { shotIds: number[] }) => {
|
||||
const deleted: number[] = [];
|
||||
const notFound: number[] = [];
|
||||
|
||||
for (const shotId of shotIds) {
|
||||
const idx = this.shots.findIndex((item) => item.id === shotId);
|
||||
if (idx === -1) {
|
||||
notFound.push(shotId);
|
||||
} else {
|
||||
this.shots.splice(idx, 1);
|
||||
deleted.push(shotId);
|
||||
}
|
||||
}
|
||||
|
||||
this.log("删除分镜", `删除: [分镜${deleted.join(", 分镜")}], 未找到: [分镜${notFound.join(", 分镜")}]`);
|
||||
this.emit("shotsUpdated", this.shots);
|
||||
|
||||
if (notFound.length) {
|
||||
return `已删除分镜 ${deleted.join(", ")}；分镜 ${notFound.join(", ")} 不存在。当前共 ${this.shots.length} 个分镜`;
|
||||
}
|
||||
return `已删除分镜 ${deleted.join(", ")}。当前共 ${this.shots.length} 个分镜`;
|
||||
},
|
||||
{
|
||||
name: "deleteShots",
|
||||
description: "删除指定的分镜。通过分镜ID指定要删除的分镜",
|
||||
schema: z.object({
|
||||
shotIds: z.array(z.number()).describe("要删除的分镜ID数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
/**
|
||||
* 生成分镜图（异步执行，使用 nanoBanana）
|
||||
*/
|
||||
generateShotImage = tool(
|
||||
async ({ shotIds }: { shotIds: number[] }) => {
|
||||
const toGenerate: number[] = [];
|
||||
const alreadyGenerating: number[] = [];
|
||||
const notFound: number[] = [];
|
||||
|
||||
for (const shotId of shotIds) {
|
||||
const shot = this.shots.find((f) => f.id === shotId);
|
||||
if (!shot) {
|
||||
notFound.push(shotId);
|
||||
continue;
|
||||
}
|
||||
if (this.generatingShots.has(shotId)) {
|
||||
alreadyGenerating.push(shotId);
|
||||
continue;
|
||||
}
|
||||
toGenerate.push(shotId);
|
||||
}
|
||||
|
||||
if (toGenerate.length === 0) {
|
||||
if (notFound.length) {
|
||||
return `分镜 ${notFound.join(", ")} 不存在，请检查分镜ID是否正确`;
|
||||
}
|
||||
if (alreadyGenerating.length) {
|
||||
return `分镜 ${alreadyGenerating.join(", ")} 正在生成中，请稍候`;
|
||||
}
|
||||
return "没有需要生成的分镜";
|
||||
}
|
||||
|
||||
// 标记为正在生成
|
||||
for (const id of toGenerate) {
|
||||
this.generatingShots.add(id);
|
||||
}
|
||||
|
||||
// 通知前端开始生成
|
||||
this.emit("shotImageGenerateStart", { shotIds: toGenerate });
|
||||
this.log("开始生成分镜图", `分镜: [${toGenerate.join(", ")}]`);
|
||||
|
||||
// 异步执行图片生成（不阻塞 Agent 流程）
|
||||
this.executeShotImageGeneration(toGenerate).catch((err) => {
|
||||
this.log("分镜图生成错误", err.message);
|
||||
this.emit("shotImageGenerateError", { shotIds: toGenerate, error: err.message });
|
||||
});
|
||||
|
||||
let result = `已开始为分镜 ${toGenerate.join(", ")} 生成分镜图，生成过程在后台进行`;
|
||||
if (alreadyGenerating.length) {
|
||||
result += `；分镜 ${alreadyGenerating.join(", ")} 正在生成中`;
|
||||
}
|
||||
if (notFound.length) {
|
||||
result += `；分镜 ${notFound.join(", ")} 不存在`;
|
||||
}
|
||||
return result;
|
||||
},
|
||||
{
|
||||
name: "generateShotImage",
|
||||
description:
|
||||
"为指定分镜生成分镜图。每个分镜会根据其所有提示词生成一张完整宫格图，然后自动分割为单格图片。通过分镜ID指定，不需要指定具体格子，整个分镜是一个完整的生成单元",
|
||||
schema: z.object({
|
||||
shotIds: z.array(z.number()).describe("要生成分镜图的分镜ID数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
|
||||
/**
|
||||
* 执行分镜图生成的具体逻辑（异步并发）
|
||||
* 每个分镜包含多个镜头，所有镜头的提示词合并生成一张宫格图，再分割为单张镜头图片
|
||||
*/
|
||||
async executeShotImageGeneration(shotIds: number[]): Promise<void> {
|
||||
await Promise.all(shotIds.map((shotId) => this.generateSingleShotImage(shotId)));
|
||||
}
|
||||
|
||||
/**
|
||||
* 生成单个分镜的图片
|
||||
*/
|
||||
private async generateSingleShotImage(shotId: number): Promise<void> {
|
||||
try {
|
||||
const shot = this.shots.find((f) => f.id === shotId);
|
||||
if (!shot) return;
|
||||
|
||||
// 提取所有镜头的有效提示词
|
||||
const prompts: string[] = shot.cells.map((c) => c.prompt).filter((p): p is string => Boolean(p));
|
||||
|
||||
if (prompts.length === 0) {
|
||||
this.log("跳过分镜图生成", `分镜 ${shotId} 没有有效的镜头提示词`);
|
||||
this.generatingShots.delete(shotId);
|
||||
return;
|
||||
}
|
||||
|
||||
// 通知前端正在生成该分镜
|
||||
this.emit("shotImageGenerateProgress", { shotId, status: "generating", message: "正在调用 AI 生成宫格图片" });
|
||||
|
||||
// 根据所有镜头提示词生成宫格图片
|
||||
const gridImage = await generateImageTool(
|
||||
prompts.map((p) => ({ prompt: p })),
|
||||
this.scriptId,
|
||||
this.projectId,
|
||||
);
|
||||
|
||||
// 通知前端正在分割图片
|
||||
this.emit("shotImageGenerateProgress", { shotId, status: "splitting", message: "正在分割宫格图片为单张镜头图" });
|
||||
|
||||
// 分割宫格图片为单张镜头图片
|
||||
const imageBuffers = await imageSplitting(gridImage, prompts.length);
|
||||
|
||||
// 通知前端正在保存图片
|
||||
this.emit("shotImageGenerateProgress", { shotId, status: "saving", message: `正在保存 ${imageBuffers.length} 张镜头图片` });
|
||||
|
||||
// 保存分割后的镜头图片到 OSS，并获取文件路径
|
||||
const timestamp = Date.now();
|
||||
const imagePaths: string[] = [];
|
||||
|
||||
for (let i = 0; i < imageBuffers.length; i++) {
|
||||
const fileName = `${this.projectId}/chat/${this.scriptId}/storyboard/shot_${shotId}_take_${i}_${timestamp}.png`;
|
||||
await u.oss.writeFile(fileName, imageBuffers[i]);
|
||||
const imageUrl = await u.oss.getFileUrl(fileName);
|
||||
imagePaths.push(imageUrl);
|
||||
|
||||
// 每保存一张镜头图片通知进度
|
||||
this.emit("shotImageGenerateProgress", {
|
||||
shotId,
|
||||
status: "saving",
|
||||
message: `已保存 ${i + 1}/${imageBuffers.length} 张镜头图片`,
|
||||
progress: Math.round(((i + 1) / imageBuffers.length) * 100),
|
||||
});
|
||||
}
|
||||
|
||||
// 更新每个镜头的 src 字段
|
||||
shot.cells = shot.cells.map((cell, i) => ({
|
||||
id: u.uuid(),
|
||||
...cell,
|
||||
src: imagePaths[i] || cell.src,
|
||||
}));
|
||||
|
||||
// 生成完成后更新状态
|
||||
this.generatingShots.delete(shotId);
|
||||
this.emit("shotImageGenerateComplete", { shotId, shot, imagePaths });
|
||||
this.emit("shotsUpdated", this.shots);
|
||||
this.log("分镜图生成完成", `分镜 ${shotId}，共 ${imagePaths.length} 张镜头图片`);
|
||||
} catch (err: any) {
|
||||
this.generatingShots.delete(shotId);
|
||||
this.emit("shotImageGenerateError", { shotId, error: err.message });
|
||||
this.log("分镜图生成失败", `分镜 ${shotId}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 公共访问器 ====================
|
||||
|
||||
/**
|
||||
* 获取当前片段数据
|
||||
*/
|
||||
getSegmentsData(): Segment[] {
|
||||
return this.segments;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取当前分镜数据
|
||||
*/
|
||||
getShotsData(): Shot[] {
|
||||
return this.shots;
|
||||
}
|
||||
|
||||
// ==================== 上下文构建 ====================
|
||||
|
||||
private async buildEnvironmentContext(): Promise<string> {
|
||||
const projectInfo = await u.db("t_project").where({ id: this.projectId }).first();
|
||||
|
||||
const row = await u.db("t_outline").where({ id: this.scriptId, projectId: this.projectId }).first();
|
||||
const outline: any | null = row?.data ? JSON.parse(row.data) : null;
|
||||
|
||||
// 分类提取资源名称
|
||||
const characters = outline?.characters?.map((i: any) => i.name) ?? [];
|
||||
const props = outline?.props?.map((i: any) => i.name) ?? [];
|
||||
const scenes = outline?.scenes?.map((i: any) => i.name) ?? [];
|
||||
|
||||
const assetList =
|
||||
[
|
||||
characters.length ? `【角色】${characters.join("、")}` : "",
|
||||
props.length ? `【道具】${props.join("、")}` : "",
|
||||
scenes.length ? `【场景】${scenes.join("、")}` : "",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n") || "无";
|
||||
|
||||
return `<环境信息>
|
||||
项目ID: ${this.projectId}
|
||||
系统时间: ${new Date().toLocaleString()}
|
||||
|
||||
项目名称: ${projectInfo?.name || "未知"}
|
||||
项目简介: ${projectInfo?.intro || "无"}
|
||||
类型: ${projectInfo?.type || "未知"}
|
||||
风格: ${projectInfo?.artStyle || "未知"}
|
||||
视频比例: ${projectInfo?.videoRatio || "未知"}
|
||||
|
||||
资产列表:
|
||||
${assetList}
|
||||
|
||||
</环境信息>`;
|
||||
}
|
||||
|
||||
private buildConversationHistory(): string {
|
||||
if (!this.history.length) return "无对话历史";
|
||||
return this.history.map(([role, content]) => `${role}: ${content}`).join("\n\n");
|
||||
}
|
||||
|
||||
private async buildFullContext(task: string): Promise<string> {
|
||||
const env = await this.buildEnvironmentContext();
|
||||
const history = this.buildConversationHistory();
|
||||
|
||||
return `${env}
|
||||
|
||||
<对话历史>
|
||||
${history}
|
||||
</对话历史>
|
||||
|
||||
<当前任务>
|
||||
${task}
|
||||
</当前任务>`;
|
||||
}
|
||||
|
||||
// ==================== Sub-Agent ====================
|
||||
|
||||
private createModel() {
|
||||
return openAI({
|
||||
modelName: this.modelName,
|
||||
configuration: { apiKey: this.apiKey, baseURL: this.baseURL },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取不同 Sub-Agent 可用的工具
|
||||
*/
|
||||
private getSubAgentTools(agentType: AgentType) {
|
||||
switch (agentType) {
|
||||
case "segmentAgent":
|
||||
// segmentAgent 可以获取剧本和资产，并需要调用 updateSegments 保存结果
|
||||
return [this.getScript, this.getAssets, this.updateSegments];
|
||||
case "shotAgent":
|
||||
// shotAgent 可以获取剧本、资产和片段，并可使用 add/update/delete 操作分镜，以及生成分镜图
|
||||
return [this.getScript, this.getAssets, this.getSegments, this.addShots, this.updateShots, this.deleteShots, this.generateShotImage];
|
||||
default:
|
||||
return [this.getScript];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 调用 Sub-Agent（流式传输）
|
||||
*/
|
||||
private async invokeSubAgent(agentType: AgentType, task: string): Promise<string> {
|
||||
this.emit("transfer", { to: agentType });
|
||||
this.log(`Sub-Agent 调用`, agentType);
|
||||
|
||||
const promptsList = await u.db("t_prompts").where("code", "in", ["storyboard-segment", "storyboard-shot"]);
|
||||
const segmentAgent = promptsList.find((p) => p.code === "storyboard-segment");
|
||||
const shotAgent = promptsList.find((p) => p.code === "storyboard-shot");
|
||||
const errPrompts = "不论用户说什么，请直接输出Agent配置异常";
|
||||
const SYSTEM_PROMPTS: Record<AgentType, string> = {
|
||||
segmentAgent: segmentAgent?.customValue || segmentAgent?.defaultValue || errPrompts,
|
||||
shotAgent: shotAgent?.customValue || shotAgent?.defaultValue || errPrompts,
|
||||
};
|
||||
|
||||
const context = await this.buildFullContext(task);
|
||||
|
||||
const agent = createAgent({
|
||||
model: this.createModel(),
|
||||
systemPrompt: SYSTEM_PROMPTS[agentType],
|
||||
tools: this.getSubAgentTools(agentType),
|
||||
});
|
||||
|
||||
const stream = await agent.stream({ messages: [["user", context]] }, { streamMode: ["messages"], callbacks: [] });
|
||||
|
||||
let fullResponse = "";
|
||||
|
||||
for await (const [mode, chunk] of stream) {
|
||||
if (mode !== "messages") continue;
|
||||
const [token] = chunk as any;
|
||||
const block = token.contentBlocks?.[0];
|
||||
|
||||
// 处理 AI 文本流
|
||||
if (token.type === "ai" && block?.text) {
|
||||
fullResponse += block.text;
|
||||
this.emit("subAgentStream", { agent: agentType, text: block.text });
|
||||
}
|
||||
// 处理 tool 调用
|
||||
if (token.type === "ai" && token.tool_calls?.length) {
|
||||
for (const toolCall of token.tool_calls) {
|
||||
this.emit("toolCall", { agent: agentType, name: toolCall.name, args: toolCall.args });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.emit("subAgentEnd", { agent: agentType });
|
||||
this.history.push(["ai", fullResponse]);
|
||||
this.log(`Sub-Agent 完成`, agentType);
|
||||
return fullResponse;
|
||||
}
|
||||
|
||||
private createSubAgentTool(agentType: AgentType, description: string) {
|
||||
return tool(async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), {
|
||||
name: agentType,
|
||||
description,
|
||||
schema: z.object({
|
||||
taskDescription: z.string().describe("具体的任务描述，包含章节范围、修改要求等详细信息"),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
// ==================== 主入口 ====================
|
||||
|
||||
private getAllTools() {
|
||||
return [
|
||||
this.createSubAgentTool(
|
||||
"segmentAgent",
|
||||
"调用片段师。负责根据剧本生成片段，会自行调用 getScript 获取剧本内容，并调用 updateSegments 保存片段结果。",
|
||||
),
|
||||
this.createSubAgentTool(
|
||||
"shotAgent",
|
||||
"调用分镜师。负责根据片段生成分镜提示词，会自行调用 getSegments 获取片段数据，并调用 addShots/updateShots 保存分镜结果。",
|
||||
),
|
||||
// this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲，会自行调用 updateOutline 或 saveStoryline 进行修改。"),
|
||||
this.getScript,
|
||||
this.getSegments,
|
||||
this.generateShotImage,
|
||||
...this.getSubAgentTools("segmentAgent"),
|
||||
...this.getSubAgentTools("shotAgent"),
|
||||
];
|
||||
}
|
||||
|
||||
async call(msg: string): Promise<string> {
|
||||
console.log("模型名称:", this.modelName);
|
||||
this.history.push(["user", msg]);
|
||||
|
||||
const envContext = await this.buildEnvironmentContext();
|
||||
|
||||
const prompts = await u.db("t_prompts").where("code", "storyboard-main").first();
|
||||
|
||||
const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常";
|
||||
|
||||
const mainAgent = createAgent({
|
||||
model: this.createModel(),
|
||||
tools: this.getAllTools(),
|
||||
systemPrompt: `${envContext}\n${mainPrompts}`,
|
||||
});
|
||||
const stream = await mainAgent.stream({ messages: this.history }, { streamMode: ["messages"], callbacks: [] });
|
||||
|
||||
let fullResponse = "";
|
||||
|
||||
for await (const [mode, chunk] of stream) {
|
||||
if (mode !== "messages") continue;
|
||||
const [token] = chunk as any;
|
||||
const block = token.contentBlocks?.[0];
|
||||
// 处理 AI 文本流
|
||||
if (token.type === "ai" && block?.text) {
|
||||
fullResponse += block.text;
|
||||
this.emit("data", block.text);
|
||||
}
|
||||
|
||||
// 处理 tool 调用
|
||||
if (token.type === "ai" && token.tool_calls?.length) {
|
||||
for (const toolCall of token.tool_calls) {
|
||||
this.emit("toolCall", { agent: "main", name: toolCall.name, args: toolCall.args });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.history.push(["assistant", fullResponse]);
|
||||
this.emit("response", fullResponse);
|
||||
|
||||
return fullResponse;
|
||||
}
|
||||
}
|
||||
@ -49,6 +49,7 @@
|
||||
"langchain": "^1.2.10",
|
||||
"morgan": "^1.10.1",
|
||||
"qwen-ai-provider": "^0.1.1",
|
||||
"serialize-error": "^13.0.1",
|
||||
"sharp": "^0.34.5",
|
||||
"sqlite3": "^5.1.7",
|
||||
"zhipu-ai-provider": "^0.2.2",
|
||||
|
||||
@ -1,10 +1,8 @@
|
||||
// @/agents/outlineScript.ts
|
||||
import u from "@/utils";
|
||||
import { createAgent } from "langchain";
|
||||
import { EventEmitter } from "events";
|
||||
import { openAI } from "@/agents/models";
|
||||
import { tool, ModelMessage } from "ai";
|
||||
import { z } from "zod";
|
||||
import { tool } from "@langchain/core/tools";
|
||||
import type { DB } from "@/types/database";
|
||||
// ==================== 类型定义 ====================
|
||||
|
||||
@ -75,13 +73,9 @@ const episodeSchema = z.object({
|
||||
export default class OutlineScript {
|
||||
private readonly projectId: number;
|
||||
readonly emitter = new EventEmitter();
|
||||
history: Array<[string, string]> = [];
|
||||
history: Array<ModelMessage> = [];
|
||||
novelChapters: DB["t_novel"][] = [];
|
||||
|
||||
modelName = "gpt-4.1";
|
||||
apiKey = "";
|
||||
baseURL = "";
|
||||
|
||||
constructor(projectId: number) {
|
||||
this.projectId = projectId;
|
||||
}
|
||||
@ -403,123 +397,107 @@ ${formatList(ep.classicQuotes, (q) => q)}
|
||||
|
||||
// ==================== Tool 定义：故事线 ====================
|
||||
|
||||
getStoryline = tool(
|
||||
async () => {
|
||||
getStoryline = tool({
|
||||
title: "getStoryline",
|
||||
description: "Get the weather in a location",
|
||||
inputSchema: z.object({}),
|
||||
execute: async () => {
|
||||
this.log("获取故事线");
|
||||
const storyline = await this.findStoryline();
|
||||
return storyline?.content ?? "当前项目暂无故事线";
|
||||
},
|
||||
{
|
||||
name: "getStoryline",
|
||||
description: "获取当前项目的故事线内容",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
saveStoryline = tool(
|
||||
async ({ content }) => {
|
||||
saveStoryline = tool({
|
||||
title: "saveStoryline",
|
||||
description: "保存或更新当前项目的故事线，会覆盖已有内容",
|
||||
inputSchema: z.object({
|
||||
content: z.string().describe("故事线完整内容"),
|
||||
}),
|
||||
execute: async ({ content }) => {
|
||||
this.log("保存故事线");
|
||||
await this.upsertStorylineContent(content);
|
||||
return "故事线保存成功";
|
||||
},
|
||||
{
|
||||
name: "saveStoryline",
|
||||
description: "保存或更新当前项目的故事线，会覆盖已有内容",
|
||||
schema: z.object({
|
||||
content: z.string().describe("故事线完整内容"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
deleteStoryline = tool(
|
||||
async () => {
|
||||
deleteStoryline = tool({
|
||||
title: "deleteStoryline",
|
||||
description: "删除当前项目的故事线",
|
||||
inputSchema: z.object({}),
|
||||
execute: async () => {
|
||||
this.log("删除故事线");
|
||||
const deleted = await this.deleteStorylineContent();
|
||||
return deleted > 0 ? "故事线删除成功" : "当前项目没有故事线";
|
||||
},
|
||||
{
|
||||
name: "deleteStoryline",
|
||||
description: "删除当前项目的故事线",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
// ==================== Tool 定义：大纲 ====================
|
||||
|
||||
getOutline = tool(
|
||||
async ({ simplified = false }) => {
|
||||
getOutline = tool({
|
||||
title: "getOutline",
|
||||
description: "获取项目大纲。simplified=true返回简化列表，false返回完整内容",
|
||||
inputSchema: z.object({
|
||||
simplified: z.boolean().default(false).describe("是否返回简化版本"),
|
||||
}),
|
||||
execute: async ({ simplified }) => {
|
||||
this.log("获取大纲", `简化模式: ${simplified}`);
|
||||
return this.getOutlineText(simplified);
|
||||
},
|
||||
{
|
||||
name: "getOutline",
|
||||
description: "获取项目大纲。simplified=true返回简化列表，false返回完整内容",
|
||||
schema: z.object({
|
||||
simplified: z.boolean().default(false).describe("是否返回简化版本"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
saveOutline = tool(
|
||||
async ({ episodes, overwrite = true, startEpisode }) => {
|
||||
this.log("保存大纲", `覆盖模式: ${overwrite}, 集数: ${episodes.length}`);
|
||||
const { insertedCount, scriptCount } = await this.saveOutlineData(episodes as EpisodeData[], overwrite, startEpisode);
|
||||
return `大纲保存成功：插入 ${insertedCount} 集大纲，创建 ${scriptCount} 个剧本记录`;
|
||||
},
|
||||
{
|
||||
name: "saveOutline",
|
||||
saveOutline = tool({
|
||||
title: "saveOutline",
|
||||
description: "保存大纲数据。overwrite=true会清空现有大纲后写入，false则追加到末尾",
|
||||
schema: z.object({
|
||||
inputSchema: z.object({
|
||||
episodes: z.array(episodeSchema).min(1).describe("大纲数据数组"),
|
||||
overwrite: z.boolean().default(true).describe("是否覆盖现有大纲"),
|
||||
startEpisode: z.number().optional().describe("追加模式下的起始集数（不填则自动递增）"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
execute: async ({ episodes, overwrite = true, startEpisode }) => {
|
||||
this.log("保存大纲", `覆盖模式: ${overwrite}, 集数: ${episodes.length}`);
|
||||
const { insertedCount, scriptCount } = await this.saveOutlineData(episodes as EpisodeData[], overwrite, startEpisode);
|
||||
return `大纲保存成功：插入 ${insertedCount} 集大纲，创建 ${scriptCount} 个剧本记录`;
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
updateOutline = tool(
|
||||
async ({ id, data }) => {
|
||||
updateOutline = tool({
|
||||
title: "updateOutline",
|
||||
description: "更新指定ID的单集大纲内容",
|
||||
inputSchema: z.object({
|
||||
id: z.number().describe("大纲ID"),
|
||||
data: episodeSchema.describe("更新后的大纲数据"),
|
||||
}),
|
||||
execute: async ({ id, data }) => {
|
||||
this.log("更新大纲", `ID: ${id}`);
|
||||
const success = await this.updateOutlineData(id, data as EpisodeData);
|
||||
return success ? `大纲ID ${id} 更新成功` : `未找到大纲ID: ${id}`;
|
||||
},
|
||||
{
|
||||
name: "updateOutline",
|
||||
description: "更新指定ID的单集大纲内容",
|
||||
schema: z.object({
|
||||
id: z.number().describe("大纲ID"),
|
||||
data: episodeSchema.describe("更新后的大纲数据"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
deleteOutline = tool(
|
||||
async ({ ids }) => {
|
||||
deleteOutline = tool({
|
||||
title: "deleteOutline",
|
||||
description: "根据大纲ID删除指定大纲及关联数据",
|
||||
inputSchema: z.object({
|
||||
ids: z.array(z.number()).min(1).describe("要删除的大纲ID数组"),
|
||||
}),
|
||||
execute: async ({ ids }) => {
|
||||
this.log("删除大纲", `IDs: ${ids.join(", ")}`);
|
||||
const results = await this.deleteOutlineData(ids);
|
||||
const summary = results.map((r, i) => `ID ${ids[i]}: ${r.status === "fulfilled" ? "成功" : "失败"}`).join(", ");
|
||||
return `删除结果: ${summary}`;
|
||||
},
|
||||
{
|
||||
name: "deleteOutline",
|
||||
description: "根据大纲ID删除指定大纲及关联数据",
|
||||
schema: z.object({
|
||||
ids: z.array(z.number()).min(1).describe("要删除的大纲ID数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
// ==================== Tool 定义：章节 ====================
|
||||
|
||||
getChapter = tool(
|
||||
async ({ chapterNumbers }) => {
|
||||
getChapter = tool({
|
||||
title: "getChapter",
|
||||
description: "根据章节编号获取小说章节的完整原文内容，支持批量获取",
|
||||
inputSchema: z.object({
|
||||
chapterNumbers: z.array(z.number()).min(1).describe("章节编号数组"),
|
||||
}),
|
||||
execute: async ({ chapterNumbers }) => {
|
||||
this.log("获取章节", `章节号: ${chapterNumbers.join(", ")}`);
|
||||
|
||||
const results = await Promise.all(
|
||||
@ -539,36 +517,24 @@ ${formatList(ep.classicQuotes, (q) => q)}
|
||||
|
||||
return results.join("\n\n---\n");
|
||||
},
|
||||
{
|
||||
name: "getChapter",
|
||||
description: "根据章节编号获取小说章节的完整原文内容，支持批量获取",
|
||||
schema: z.object({
|
||||
chapterNumbers: z.array(z.number()).min(1).describe("章节编号数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
// ==================== Tool 定义：资产 ====================
|
||||
|
||||
generateAssets = tool(
|
||||
async () => {
|
||||
generateAssets = tool({
|
||||
title: "generateAssets",
|
||||
description: "从当前项目的所有大纲中提取并生成角色、道具、场景资产，自动去重并清理冗余",
|
||||
inputSchema: z.object({}),
|
||||
execute: async () => {
|
||||
this.log("生成资产");
|
||||
const stats = await this.generateAssetsFromOutlines();
|
||||
|
||||
if (stats.inserted === 0 && stats.updated === 0 && stats.skipped === 0) {
|
||||
return "当前项目没有大纲数据，无法生成资产";
|
||||
}
|
||||
|
||||
return `资产生成完成：新增 ${stats.inserted}，更新 ${stats.updated}，保持 ${stats.skipped}`;
|
||||
},
|
||||
{
|
||||
name: "generateAssets",
|
||||
description: "从当前项目的所有大纲中提取并生成角色、道具、场景资产，自动去重并清理冗余",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
// ==================== 上下文构建 ====================
|
||||
|
||||
@ -606,7 +572,7 @@ ${this.getChapterContext()}
|
||||
|
||||
private buildConversationHistory(): string {
|
||||
if (!this.history.length) return "无对话历史";
|
||||
return this.history.map(([role, content]) => `${role}: ${content}`).join("\n\n");
|
||||
return this.history.map(({ role, content }) => `${role}: ${content}`).join("\n\n");
|
||||
}
|
||||
|
||||
private async buildFullContext(task: string): Promise<string> {
|
||||
@ -627,14 +593,14 @@ ${task}
|
||||
// ==================== Sub-Agent ====================
|
||||
|
||||
private getSubAgentTools() {
|
||||
return [this.getChapter, this.getStoryline, this.saveStoryline, this.getOutline, this.saveOutline, this.updateOutline];
|
||||
}
|
||||
|
||||
private createModel() {
|
||||
return openAI({
|
||||
modelName: this.modelName,
|
||||
configuration: { apiKey: this.apiKey, baseURL: this.baseURL },
|
||||
});
|
||||
return {
|
||||
getChapter: this.getChapter,
|
||||
getStoryline: this.getStoryline,
|
||||
saveStoryline: this.saveStoryline,
|
||||
getOutline: this.getOutline,
|
||||
saveOutline: this.saveOutline,
|
||||
updateOutline: this.updateOutline,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@ -657,74 +623,69 @@ ${task}
|
||||
|
||||
const context = await this.buildFullContext(task);
|
||||
|
||||
const agent = createAgent({
|
||||
model: this.createModel(),
|
||||
systemPrompt: SYSTEM_PROMPTS[agentType],
|
||||
const { fullStream } = await u.ai.text.stream({
|
||||
system: SYSTEM_PROMPTS[agentType],
|
||||
tools: this.getSubAgentTools(),
|
||||
messages: [{ role: "user", content: context }],
|
||||
maxStep: 100,
|
||||
});
|
||||
|
||||
const stream = await agent.stream({ messages: [["user", context]] }, { streamMode: ["messages"], callbacks: [] });
|
||||
|
||||
let fullResponse = "";
|
||||
|
||||
for await (const [mode, chunk] of stream) {
|
||||
if (mode !== "messages") continue;
|
||||
|
||||
const [token] = chunk as any;
|
||||
const block = token.contentBlocks?.[0];
|
||||
|
||||
// 处理 AI 文本流
|
||||
if (token.type === "ai" && block?.text) {
|
||||
fullResponse += block.text;
|
||||
this.emit("subAgentStream", { agent: agentType, text: block.text });
|
||||
}
|
||||
|
||||
// 处理 tool 调用
|
||||
if (token.type === "ai" && token.tool_calls?.length) {
|
||||
for (const toolCall of token.tool_calls) {
|
||||
this.emit("toolCall", { agent: agentType, name: toolCall.name, args: toolCall.args });
|
||||
for await (const item of fullStream) {
|
||||
if (item.type == "tool-call") {
|
||||
this.emit("toolCall", { agent: "main", name: item.title, args: null });
|
||||
}
|
||||
if (item.type == "text-delta") {
|
||||
fullResponse += item.text;
|
||||
this.emit("subAgentStream", { agent: agentType, text: item.text });
|
||||
}
|
||||
}
|
||||
|
||||
this.emit("subAgentEnd", { agent: agentType });
|
||||
this.history.push(["ai", fullResponse]);
|
||||
this.history.push({
|
||||
role: "assistant",
|
||||
content: fullResponse,
|
||||
});
|
||||
this.log(`Sub-Agent 完成`, agentType);
|
||||
|
||||
return fullResponse ?? `${agentType}已完成任务`;
|
||||
}
|
||||
|
||||
private createSubAgentTool(agentType: AgentType, description: string) {
|
||||
return tool(async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), {
|
||||
name: agentType,
|
||||
return tool({
|
||||
title: agentType,
|
||||
description,
|
||||
schema: z.object({
|
||||
inputSchema: z.object({
|
||||
taskDescription: z.string().describe("具体的任务描述，包含章节范围、修改要求等详细信息"),
|
||||
}),
|
||||
execute: async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription),
|
||||
});
|
||||
}
|
||||
|
||||
// ==================== 主入口 ====================
|
||||
|
||||
private getAllTools() {
|
||||
return [
|
||||
this.createSubAgentTool("AI1", "调用故事师。负责分析小说原文并生成故事线，会自行调用 saveStoryline 保存结果。"),
|
||||
this.createSubAgentTool("AI2", "调用大纲师。负责根据故事线生成剧集大纲，会自行调用 saveOutline 保存结果。"),
|
||||
this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲，会自行调用 updateOutline 或 saveStoryline 进行修改。"),
|
||||
this.getChapter,
|
||||
this.getStoryline,
|
||||
this.saveStoryline,
|
||||
this.deleteStoryline,
|
||||
this.getOutline,
|
||||
this.saveOutline,
|
||||
this.updateOutline,
|
||||
this.deleteOutline,
|
||||
this.generateAssets,
|
||||
];
|
||||
return {
|
||||
AI1: this.createSubAgentTool("AI1", "调用故事师。负责分析小说原文并生成故事线，会自行调用 saveStoryline 保存结果。"),
|
||||
AI2: this.createSubAgentTool("AI2", "调用大纲师。负责根据故事线生成剧集大纲，会自行调用 saveOutline 保存结果。"),
|
||||
director: this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲，会自行调用 updateOutline 或 saveStoryline 进行修改。"),
|
||||
getChapter: this.getChapter,
|
||||
getStoryline: this.getStoryline,
|
||||
saveStoryline: this.saveStoryline,
|
||||
deleteStoryline: this.deleteStoryline,
|
||||
getOutline: this.getOutline,
|
||||
saveOutline: this.saveOutline,
|
||||
updateOutline: this.updateOutline,
|
||||
deleteOutline: this.deleteOutline,
|
||||
generateAssets: this.generateAssets,
|
||||
};
|
||||
}
|
||||
|
||||
async call(msg: string): Promise<string> {
|
||||
this.history.push(["user", msg]);
|
||||
this.history.push({
|
||||
role: "user",
|
||||
content: msg,
|
||||
});
|
||||
|
||||
const envContext = await this.buildEnvironmentContext();
|
||||
|
||||
@ -732,36 +693,28 @@ ${task}
|
||||
|
||||
const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常";
|
||||
|
||||
const mainAgent = createAgent({
|
||||
model: this.createModel(),
|
||||
const { fullStream } = await u.ai.text.stream({
|
||||
system: `${envContext}\n${mainPrompts}`,
|
||||
tools: this.getAllTools(),
|
||||
systemPrompt: `${envContext}\n${mainPrompts}`,
|
||||
messages: this.history,
|
||||
maxStep: 100,
|
||||
});
|
||||
const stream = await mainAgent.stream({ messages: this.history }, { streamMode: ["messages"], callbacks: [] });
|
||||
|
||||
let fullResponse = "";
|
||||
|
||||
for await (const [mode, chunk] of stream) {
|
||||
if (mode !== "messages") continue;
|
||||
|
||||
const [token] = chunk as any;
|
||||
const block = token.contentBlocks?.[0];
|
||||
|
||||
// 处理 AI 文本流
|
||||
if (token.type === "ai" && block?.text) {
|
||||
fullResponse += block.text;
|
||||
this.emit("data", block.text);
|
||||
}
|
||||
|
||||
// 处理 tool 调用
|
||||
if (token.type === "ai" && token.tool_calls?.length) {
|
||||
for (const toolCall of token.tool_calls) {
|
||||
this.emit("toolCall", { agent: "main", name: toolCall.name, args: toolCall.args });
|
||||
for await (const item of fullStream) {
|
||||
if (item.type == "tool-call") {
|
||||
this.emit("toolCall", { agent: "main", name: item.title, args: null });
|
||||
}
|
||||
if (item.type == "text-delta") {
|
||||
fullResponse += item.text;
|
||||
this.emit("data", item.text);
|
||||
}
|
||||
}
|
||||
this.history.push({
|
||||
role: "assistant",
|
||||
content: fullResponse,
|
||||
});
|
||||
|
||||
this.history.push(["assistant", fullResponse]);
|
||||
this.emit("response", fullResponse);
|
||||
|
||||
return fullResponse;
|
||||
|
||||
@ -1,10 +1,8 @@
|
||||
// @/agents/Storyboard.ts
|
||||
import u from "@/utils";
|
||||
import { createAgent } from "langchain";
|
||||
import { tool, ModelMessage, Tool } from "ai";
|
||||
import { EventEmitter } from "events";
|
||||
import { openAI } from "@/agents/models";
|
||||
import { z } from "zod";
|
||||
import { tool } from "@langchain/core/tools";
|
||||
import type { DB } from "@/types/database";
|
||||
import generateImageTool from "./generateImageTool";
|
||||
import imageSplitting from "./imageSplitting";
|
||||
@ -46,7 +44,7 @@ export default class Storyboard {
|
||||
private readonly projectId: number;
|
||||
private readonly scriptId: number;
|
||||
readonly emitter = new EventEmitter();
|
||||
history: Array<[string, string]> = [];
|
||||
history: ModelMessage[] = [];
|
||||
novelChapters: DB["t_novel"][] = [];
|
||||
|
||||
// 存储 segmentAgent 生成的片段结果
|
||||
@ -58,10 +56,6 @@ export default class Storyboard {
|
||||
// 存储正在生成分镜图的分镜ID
|
||||
private generatingShots: Set<number> = new Set();
|
||||
|
||||
modelName = "gpt-4.1";
|
||||
apiKey = "";
|
||||
baseURL = "";
|
||||
|
||||
constructor(projectId: number, scriptId: number) {
|
||||
this.projectId = projectId;
|
||||
this.scriptId = scriptId;
|
||||
@ -105,28 +99,28 @@ export default class Storyboard {
|
||||
|
||||
// ==================== 剧本相关操作 ====================
|
||||
|
||||
getScript = tool(
|
||||
async () => {
|
||||
getScript = tool({
|
||||
title: "getScript",
|
||||
description: "获取剧本内容",
|
||||
inputSchema: z.object({}),
|
||||
execute: async () => {
|
||||
this.log("获取剧本", `scriptId: ${this.scriptId}`);
|
||||
const script = await u.db("t_script").where({ id: this.scriptId, projectId: this.projectId }).first();
|
||||
if (!script) throw new Error("剧本不存在");
|
||||
return `剧本集：${script.name}\n\n内容：\n\`\`\`${script.content}\`\`\``;
|
||||
},
|
||||
{
|
||||
name: "getScript",
|
||||
description: "获取剧本内容",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
// ==================== 资产相关操作 ====================
|
||||
|
||||
/**
|
||||
* 获取资产列表（供 segmentAgent 和 shotAgent 调用）
|
||||
*/
|
||||
getAssets = tool(
|
||||
async () => {
|
||||
getAssets = tool({
|
||||
title: "getAssets",
|
||||
description: "获取资产列表（角色、道具、场景），包含名称和详细介绍。生成片段和分镜时必须先调用此工具获取资产信息，确保名称一致性",
|
||||
inputSchema: z.object({}),
|
||||
execute: async () => {
|
||||
this.log("获取资产列表", `scriptId: ${this.scriptId}`);
|
||||
const scriptData = await u.db("t_script").where({ id: this.scriptId, projectId: this.projectId }).first();
|
||||
const row = await u.db("t_outline").where({ id: scriptData?.outlineId!, projectId: this.projectId }).first();
|
||||
@ -171,49 +165,33 @@ ${sections.join("\n\n")}
|
||||
2. 禁止在资产名称前后添加修饰词
|
||||
3. 禁止捏造资产列表中不存在的角色、场景、道具`;
|
||||
},
|
||||
{
|
||||
name: "getAssets",
|
||||
description: "获取资产列表（角色、道具、场景），包含名称和详细介绍。生成片段和分镜时必须先调用此工具获取资产信息，确保名称一致性",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
// ==================== 片段和分镜工具 ====================
|
||||
|
||||
/**
|
||||
* 获取当前存储的片段数据（供 shotAgent 调用）
|
||||
*/
|
||||
getSegments = tool(
|
||||
async () => {
|
||||
getSegments = tool({
|
||||
title: "getSegments",
|
||||
description: "获取当前已生成的片段数据，用于生成分镜",
|
||||
inputSchema: z.object({}),
|
||||
execute: async () => {
|
||||
this.log("获取片段数据", `共 ${this.segments.length} 个片段`);
|
||||
if (this.segments.length === 0) {
|
||||
return "暂无片段数据，请先调用 segmentAgent 生成片段";
|
||||
}
|
||||
return JSON.stringify(this.segments, null, 2);
|
||||
},
|
||||
{
|
||||
name: "getSegments",
|
||||
description: "获取当前已生成的片段数据，用于生成分镜",
|
||||
schema: z.object({}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* 更新/存储片段数据（供 segmentAgent 调用）
|
||||
*/
|
||||
updateSegments = tool(
|
||||
async ({ segments }: { segments: Segment[] }) => {
|
||||
this.log("更新片段数据", `共 ${segments.length} 个片段`);
|
||||
this.segments = segments;
|
||||
this.emit("segmentsUpdated", this.segments);
|
||||
return `成功存储 ${segments.length} 个片段`;
|
||||
},
|
||||
{
|
||||
name: "updateSegments",
|
||||
updateSegments = tool({
|
||||
title: "updateSegments",
|
||||
description: "存储生成的片段数据，segmentAgent 在生成片段后必须调用此工具保存结果",
|
||||
schema: z.object({
|
||||
inputSchema: z.object({
|
||||
segments: z
|
||||
.array(
|
||||
z.object({
|
||||
@ -225,15 +203,31 @@ ${sections.join("\n\n")}
|
||||
)
|
||||
.describe("片段数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
execute: async ({ segments }: { segments: Segment[] }) => {
|
||||
this.log("更新片段数据", `共 ${segments.length} 个片段`);
|
||||
this.segments = segments;
|
||||
this.emit("segmentsUpdated", this.segments);
|
||||
return `成功存储 ${segments.length} 个片段`;
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* 添加分镜（供 shotAgent 调用）
|
||||
*/
|
||||
addShots = tool(
|
||||
async ({ shots }: { shots: Array<{ segmentIndex: number; prompts: string[] }> }) => {
|
||||
addShots = tool({
|
||||
title: "addShots",
|
||||
description: "添加新的分镜。每个分镜有独立ID，包含多个镜头（每个镜头对应一个提示词）。如果片段已存在分镜会跳过",
|
||||
inputSchema: z.object({
|
||||
shots: z
|
||||
.array(
|
||||
z.object({
|
||||
segmentIndex: z.number().describe("对应的片段序号"),
|
||||
prompts: z.array(z.string()).describe("镜头提示词数组，每个提示词对应一个镜头（中文）"),
|
||||
}),
|
||||
)
|
||||
.describe("要添加的分镜数组"),
|
||||
}),
|
||||
execute: async ({ shots }: { shots: Array<{ segmentIndex: number; prompts: string[] }> }) => {
|
||||
const added: { id: number; segmentIndex: number }[] = [];
|
||||
const skipped: number[] = [];
|
||||
|
||||
@ -266,29 +260,20 @@ ${sections.join("\n\n")}
|
||||
}
|
||||
return `已添加${addedInfo}。当前共 ${this.shots.length} 个分镜`;
|
||||
},
|
||||
{
|
||||
name: "addShots",
|
||||
description: "添加新的分镜。每个分镜有独立ID，包含多个镜头（每个镜头对应一个提示词）。如果片段已存在分镜会跳过",
|
||||
schema: z.object({
|
||||
shots: z
|
||||
.array(
|
||||
z.object({
|
||||
segmentIndex: z.number().describe("对应的片段序号"),
|
||||
prompts: z.array(z.string()).describe("镜头提示词数组，每个提示词对应一个镜头（中文）"),
|
||||
}),
|
||||
)
|
||||
.describe("要添加的分镜数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* 更新指定分镜（供 shotAgent 调用）
|
||||
* 保留原有 cells 的 id 和 src 字段，只更新 prompt
|
||||
*/
|
||||
updateShots = tool(
|
||||
async ({ shotId, prompts }: { shotId: number; prompts: string[] }) => {
|
||||
updateShots = tool({
|
||||
title: "updateShots",
|
||||
description: "更新指定分镜的镜头提示词。通过分镜ID指定要修改的分镜",
|
||||
inputSchema: z.object({
|
||||
shotId: z.number().describe("要更新的分镜ID"),
|
||||
prompts: z.array(z.string()).describe("新的镜头提示词数组，每个提示词对应一个镜头"),
|
||||
}),
|
||||
execute: async ({ shotId, prompts }: { shotId: number; prompts: string[] }) => {
|
||||
const existingIndex = this.shots.findIndex((item) => item.id === shotId);
|
||||
|
||||
if (existingIndex === -1) {
|
||||
@ -314,22 +299,18 @@ ${sections.join("\n\n")}
|
||||
|
||||
return `已更新分镜 ${shotId}`;
|
||||
},
|
||||
{
|
||||
name: "updateShots",
|
||||
description: "更新指定分镜的镜头提示词。通过分镜ID指定要修改的分镜",
|
||||
schema: z.object({
|
||||
shotId: z.number().describe("要更新的分镜ID"),
|
||||
prompts: z.array(z.string()).describe("新的镜头提示词数组，每个提示词对应一个镜头"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* 删除指定分镜（供 shotAgent 调用）
|
||||
*/
|
||||
deleteShots = tool(
|
||||
async ({ shotIds }: { shotIds: number[] }) => {
|
||||
deleteShots = tool({
|
||||
title: "deleteShots",
|
||||
description: "删除指定的分镜。通过分镜ID指定要删除的分镜",
|
||||
inputSchema: z.object({
|
||||
shotIds: z.array(z.number()).describe("要删除的分镜ID数组"),
|
||||
}),
|
||||
execute: async ({ shotIds }: { shotIds: number[] }) => {
|
||||
const deleted: number[] = [];
|
||||
const notFound: number[] = [];
|
||||
|
||||
@ -351,21 +332,19 @@ ${sections.join("\n\n")}
|
||||
}
|
||||
return `已删除分镜 ${deleted.join(", ")}。当前共 ${this.shots.length} 个分镜`;
|
||||
},
|
||||
{
|
||||
name: "deleteShots",
|
||||
description: "删除指定的分镜。通过分镜ID指定要删除的分镜",
|
||||
schema: z.object({
|
||||
shotIds: z.array(z.number()).describe("要删除的分镜ID数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* 生成分镜图（异步执行，使用 nanoBanana）
|
||||
*/
|
||||
generateShotImage = tool(
|
||||
async ({ shotIds }: { shotIds: number[] }) => {
|
||||
generateShotImage = tool({
|
||||
title: "generateShotImage",
|
||||
description:
|
||||
"为指定分镜生成分镜图。每个分镜会根据其所有提示词生成一张完整宫格图，然后自动分割为单格图片。通过分镜ID指定，不需要指定具体格子，整个分镜是一个完整的生成单元",
|
||||
inputSchema: z.object({
|
||||
shotIds: z.array(z.number()).describe("要生成分镜图的分镜ID数组"),
|
||||
}),
|
||||
execute: async ({ shotIds }: { shotIds: number[] }) => {
|
||||
const toGenerate: number[] = [];
|
||||
const alreadyGenerating: number[] = [];
|
||||
const notFound: number[] = [];
|
||||
@ -417,16 +396,7 @@ ${sections.join("\n\n")}
|
||||
}
|
||||
return result;
|
||||
},
|
||||
{
|
||||
name: "generateShotImage",
|
||||
description:
|
||||
"为指定分镜生成分镜图。每个分镜会根据其所有提示词生成一张完整宫格图，然后自动分割为单格图片。通过分镜ID指定，不需要指定具体格子，整个分镜是一个完整的生成单元",
|
||||
schema: z.object({
|
||||
shotIds: z.array(z.number()).describe("要生成分镜图的分镜ID数组"),
|
||||
}),
|
||||
verboseParsingErrors: true,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* 执行分镜图生成的具体逻辑（异步并发）
|
||||
@ -566,7 +536,7 @@ ${assetList}
|
||||
|
||||
private buildConversationHistory(): string {
|
||||
if (!this.history.length) return "无对话历史";
|
||||
return this.history.map(([role, content]) => `${role}: ${content}`).join("\n\n");
|
||||
return this.history.map(({ role, content }) => `${role}: ${content}`).join("\n\n");
|
||||
}
|
||||
|
||||
private async buildFullContext(task: string): Promise<string> {
|
||||
@ -586,26 +556,33 @@ ${task}
|
||||
|
||||
// ==================== Sub-Agent ====================
|
||||
|
||||
private createModel() {
|
||||
return openAI({
|
||||
modelName: this.modelName,
|
||||
configuration: { apiKey: this.apiKey, baseURL: this.baseURL },
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取不同 Sub-Agent 可用的工具
|
||||
*/
|
||||
private getSubAgentTools(agentType: AgentType) {
|
||||
private getSubAgentTools(agentType: AgentType): Record<string, Tool> {
|
||||
switch (agentType) {
|
||||
case "segmentAgent":
|
||||
// segmentAgent 可以获取剧本和资产，并需要调用 updateSegments 保存结果
|
||||
return [this.getScript, this.getAssets, this.updateSegments];
|
||||
return {
|
||||
getScript: this.getScript,
|
||||
getAssets: this.getAssets,
|
||||
updateSegments: this.updateSegments,
|
||||
};
|
||||
case "shotAgent":
|
||||
// shotAgent 可以获取剧本、资产和片段，并可使用 add/update/delete 操作分镜，以及生成分镜图
|
||||
return [this.getScript, this.getAssets, this.getSegments, this.addShots, this.updateShots, this.deleteShots, this.generateShotImage];
|
||||
return {
|
||||
getScript: this.getScript,
|
||||
getAssets: this.getAssets,
|
||||
getSegments: this.getSegments,
|
||||
addShots: this.addShots,
|
||||
updateShots: this.updateShots,
|
||||
deleteShots: this.deleteShots,
|
||||
generateShotImage: this.generateShotImage,
|
||||
};
|
||||
default:
|
||||
return [this.getScript];
|
||||
return {
|
||||
getScript: this.getScript,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@ -627,74 +604,71 @@ ${task}
|
||||
|
||||
const context = await this.buildFullContext(task);
|
||||
|
||||
const agent = createAgent({
|
||||
model: this.createModel(),
|
||||
systemPrompt: SYSTEM_PROMPTS[agentType],
|
||||
const { fullStream } = await u.ai.text.stream({
|
||||
system: SYSTEM_PROMPTS[agentType],
|
||||
tools: this.getSubAgentTools(agentType),
|
||||
messages: [{ role: "user", content: context }],
|
||||
maxStep: 100,
|
||||
});
|
||||
|
||||
const stream = await agent.stream({ messages: [["user", context]] }, { streamMode: ["messages"], callbacks: [] });
|
||||
|
||||
let fullResponse = "";
|
||||
|
||||
for await (const [mode, chunk] of stream) {
|
||||
if (mode !== "messages") continue;
|
||||
const [token] = chunk as any;
|
||||
const block = token.contentBlocks?.[0];
|
||||
|
||||
// 处理 AI 文本流
|
||||
if (token.type === "ai" && block?.text) {
|
||||
fullResponse += block.text;
|
||||
this.emit("subAgentStream", { agent: agentType, text: block.text });
|
||||
}
|
||||
// 处理 tool 调用
|
||||
if (token.type === "ai" && token.tool_calls?.length) {
|
||||
for (const toolCall of token.tool_calls) {
|
||||
this.emit("toolCall", { agent: agentType, name: toolCall.name, args: toolCall.args });
|
||||
for await (const item of fullStream) {
|
||||
if (item.type == "tool-call") {
|
||||
this.emit("toolCall", { agent: "main", name: item.title, args: null });
|
||||
}
|
||||
if (item.type == "text-delta") {
|
||||
fullResponse += item.text;
|
||||
this.emit("subAgentStream", { agent: agentType, text: item.text });
|
||||
}
|
||||
}
|
||||
|
||||
this.emit("subAgentEnd", { agent: agentType });
|
||||
this.history.push(["ai", fullResponse]);
|
||||
this.history.push({
|
||||
role: "assistant",
|
||||
content: fullResponse,
|
||||
});
|
||||
this.log(`Sub-Agent 完成`, agentType);
|
||||
return fullResponse;
|
||||
|
||||
return fullResponse ?? `${agentType}已完成任务`;
|
||||
}
|
||||
|
||||
private createSubAgentTool(agentType: AgentType, description: string) {
|
||||
return tool(async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), {
|
||||
name: agentType,
|
||||
return tool({
|
||||
title: agentType,
|
||||
description,
|
||||
schema: z.object({
|
||||
inputSchema: z.object({
|
||||
taskDescription: z.string().describe("具体的任务描述，包含章节范围、修改要求等详细信息"),
|
||||
}),
|
||||
execute: async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription),
|
||||
});
|
||||
}
|
||||
|
||||
// ==================== 主入口 ====================
|
||||
|
||||
private getAllTools() {
|
||||
return [
|
||||
this.createSubAgentTool(
|
||||
return {
|
||||
segmentAgent: this.createSubAgentTool(
|
||||
"segmentAgent",
|
||||
"调用片段师。负责根据剧本生成片段，会自行调用 getScript 获取剧本内容，并调用 updateSegments 保存片段结果。",
|
||||
),
|
||||
this.createSubAgentTool(
|
||||
shotAgent: this.createSubAgentTool(
|
||||
"shotAgent",
|
||||
"调用分镜师。负责根据片段生成分镜提示词，会自行调用 getSegments 获取片段数据，并调用 addShots/updateShots 保存分镜结果。",
|
||||
),
|
||||
// this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲，会自行调用 updateOutline 或 saveStoryline 进行修改。"),
|
||||
this.getScript,
|
||||
this.getSegments,
|
||||
this.generateShotImage,
|
||||
getScript: this.getScript,
|
||||
getSegments: this.getSegments,
|
||||
generateShotImage: this.generateShotImage,
|
||||
...this.getSubAgentTools("segmentAgent"),
|
||||
...this.getSubAgentTools("shotAgent"),
|
||||
];
|
||||
};
|
||||
}
|
||||
|
||||
async call(msg: string): Promise<string> {
|
||||
console.log("模型名称:", this.modelName);
|
||||
this.history.push(["user", msg]);
|
||||
this.history.push({
|
||||
role: "user",
|
||||
content: msg,
|
||||
});
|
||||
|
||||
const envContext = await this.buildEnvironmentContext();
|
||||
|
||||
@ -702,34 +676,28 @@ ${task}
|
||||
|
||||
const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常";
|
||||
|
||||
const mainAgent = createAgent({
|
||||
model: this.createModel(),
|
||||
const { fullStream } = await u.ai.text.stream({
|
||||
system: `${envContext}\n${mainPrompts}`,
|
||||
tools: this.getAllTools(),
|
||||
systemPrompt: `${envContext}\n${mainPrompts}`,
|
||||
messages: this.history,
|
||||
maxStep: 100,
|
||||
});
|
||||
const stream = await mainAgent.stream({ messages: this.history }, { streamMode: ["messages"], callbacks: [] });
|
||||
|
||||
let fullResponse = "";
|
||||
for await (const item of fullStream) {
|
||||
if (item.type == "tool-call") {
|
||||
this.emit("toolCall", { agent: "main", name: item.title, args: null });
|
||||
}
|
||||
if (item.type == "text-delta") {
|
||||
fullResponse += item.text;
|
||||
this.emit("data", item.text);
|
||||
}
|
||||
}
|
||||
this.history.push({
|
||||
role: "assistant",
|
||||
content: fullResponse,
|
||||
});
|
||||
|
||||
for await (const [mode, chunk] of stream) {
|
||||
if (mode !== "messages") continue;
|
||||
const [token] = chunk as any;
|
||||
const block = token.contentBlocks?.[0];
|
||||
// 处理 AI 文本流
|
||||
if (token.type === "ai" && block?.text) {
|
||||
fullResponse += block.text;
|
||||
this.emit("data", block.text);
|
||||
}
|
||||
|
||||
// 处理 tool 调用
|
||||
if (token.type === "ai" && token.tool_calls?.length) {
|
||||
for (const toolCall of token.tool_calls) {
|
||||
this.emit("toolCall", { agent: "main", name: toolCall.name, args: toolCall.args });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.history.push(["assistant", fullResponse]);
|
||||
this.emit("response", fullResponse);
|
||||
|
||||
return fullResponse;
|
||||
|
||||
@ -78,7 +78,9 @@ export default router.post(
|
||||
}
|
||||
|
||||
// 更新提示信息
|
||||
if (prompt !== undefined && prompt !== null && prompt !== "") {
|
||||
await u.db("t_assets").where("id", id).update({ prompt });
|
||||
}
|
||||
|
||||
res.status(200).send(success({ message: "保存资产图片成功" }));
|
||||
},
|
||||
|
||||
@ -16,25 +16,34 @@ export default router.post(
|
||||
}),
|
||||
async (req, res) => {
|
||||
const { modelName, apiKey, baseURL, manufacturer } = req.body;
|
||||
try {
|
||||
const contentStr = await u.ai.generateImage(
|
||||
{
|
||||
|
||||
const image =await u.ai.image({
|
||||
prompt: "2D cat",
|
||||
imageBase64: [],
|
||||
aspectRatio: "16:9",
|
||||
size: "1K",
|
||||
},
|
||||
{
|
||||
model: modelName,
|
||||
apiKey,
|
||||
baseURL,
|
||||
manufacturer,
|
||||
},
|
||||
);
|
||||
res.status(200).send(success(contentStr));
|
||||
} catch (err: any) {
|
||||
const message = err?.response?.data?.error?.message || err?.error?.message || "模型调用失败";
|
||||
res.status(500).send(error(message));
|
||||
}
|
||||
});
|
||||
res.status(200).send(success(image));
|
||||
|
||||
// try {
|
||||
// const contentStr = await u.ai.generateImage(
|
||||
// {
|
||||
// prompt: "2D cat",
|
||||
// imageBase64: [],
|
||||
// aspectRatio: "16:9",
|
||||
// size: "1K",
|
||||
// },
|
||||
// {
|
||||
// model: modelName,
|
||||
// apiKey,
|
||||
// baseURL,
|
||||
// manufacturer,
|
||||
// },
|
||||
// );
|
||||
// res.status(200).send(success(contentStr));
|
||||
// } catch (err: any) {
|
||||
// const message = err?.response?.data?.error?.message || err?.error?.message || "模型调用失败";
|
||||
// res.status(500).send(error(message));
|
||||
// }
|
||||
},
|
||||
);
|
||||
|
||||
@ -8,7 +8,6 @@ expressWs(router as unknown as Application);
|
||||
router.ws("/", async (ws, req) => {
|
||||
let agent: OutlineScript;
|
||||
|
||||
const config = await u.getConfig("language");
|
||||
|
||||
const projectId = req.query.projectId;
|
||||
if (!projectId || typeof projectId !== "string") {
|
||||
@ -19,10 +18,6 @@ router.ws("/", async (ws, req) => {
|
||||
|
||||
agent = new OutlineScript(Number(projectId));
|
||||
|
||||
agent.modelName = config.model ?? "";
|
||||
agent.baseURL = config.baseURL ?? "";
|
||||
agent.apiKey = config.apiKey ?? "";
|
||||
|
||||
// const existing = await u
|
||||
// .db("t_chatHistory")
|
||||
// .where({ projectId: Number(projectId) })
|
||||
|
||||
13
src/utils.ts
13
src/utils.ts
@ -6,21 +6,24 @@ import number2Chinese from "@/utils/number2Chinese";
|
||||
import deleteOutline from "@/utils/deleteOutline";
|
||||
import getConfig from "./utils/getConfig";
|
||||
import { v4 as uuid } from "uuid";
|
||||
import error from "@/utils/error";
|
||||
import * as imageTools from "@/utils/imageTools";
|
||||
|
||||
import AIText from "@/utils/ai/text/index";
|
||||
import AIImage from "@/utils/ai/image/index";
|
||||
|
||||
import AIText from "@/utils/ai/text";
|
||||
import generateVideo from "@/utils/ai/generateVideo";
|
||||
import generateImage from "@/utils/ai/generateImage";
|
||||
export default {
|
||||
db,
|
||||
oss,
|
||||
ai: {
|
||||
text: AIText,
|
||||
generateVideo,
|
||||
generateImage,
|
||||
image: AIImage,
|
||||
},
|
||||
editImage,
|
||||
number2Chinese,
|
||||
deleteOutline,
|
||||
getConfig,
|
||||
uuid,
|
||||
error,
|
||||
imageTools,
|
||||
};
|
||||
|
||||
44
src/utils/ai/image/index.ts
Normal file
44
src/utils/ai/image/index.ts
Normal file
@ -0,0 +1,44 @@
|
||||
import "./type";
|
||||
import u from "@/utils";
|
||||
import modelList from "./modelList";
|
||||
import axios from "axios";
|
||||
|
||||
import volcengine from "./owned/volcengine";
|
||||
import kling from "./owned/kling";
|
||||
|
||||
|
||||
interface AIConfig {
|
||||
model?: string;
|
||||
apiKey?: string;
|
||||
baseURL?: string;
|
||||
}
|
||||
|
||||
const urlToBase64 = async (url: string): Promise<string> => {
|
||||
const res = await axios.get(url, { responseType: "arraybuffer" });
|
||||
const base64 = Buffer.from(res.data).toString("base64");
|
||||
const mimeType = res.headers["content-type"] || "image/png";
|
||||
return `data:${mimeType};base64,${base64}`;
|
||||
};
|
||||
|
||||
const modelInstance = {
|
||||
gemini: null,
|
||||
volcengine: volcengine,
|
||||
kling: kling,
|
||||
vidu: null,
|
||||
runninghub: null,
|
||||
apimart: null,
|
||||
} as const;
|
||||
|
||||
export default async (input: ImageConfig, config?: AIConfig) => {
|
||||
const sqlTextModelConfig = await u.getConfig("image");
|
||||
const { model, apiKey, baseURL, manufacturer } = { ...sqlTextModelConfig, ...config };
|
||||
const manufacturerFn = modelInstance[manufacturer as keyof typeof modelInstance];
|
||||
if (!manufacturerFn) if (!manufacturerFn) throw new Error("不支持的图片厂商");
|
||||
const owned = modelList.find((m) => m.model === model);
|
||||
if (!owned) throw new Error("不支持的模型");
|
||||
|
||||
let imageUrl = await manufacturerFn(input, { model, apiKey, baseURL });
|
||||
if (!input.resType) input.resType = "b64";
|
||||
if (input.resType === "b64" && imageUrl.startsWith("http")) imageUrl = await urlToBase64(imageUrl);
|
||||
return input;
|
||||
};
|
||||
77
src/utils/ai/image/modelList.ts
Normal file
77
src/utils/ai/image/modelList.ts
Normal file
@ -0,0 +1,77 @@
|
||||
interface Owned {
|
||||
manufacturer: string;
|
||||
model: string;
|
||||
grid: boolean;
|
||||
type: "t2i" | "ti2i" | "i2i";
|
||||
}
|
||||
|
||||
const modelList: Owned[] = [
|
||||
// 火山引擎
|
||||
{
|
||||
manufacturer: "volcengine",
|
||||
model: "doubao-seedream-4-5-251128",
|
||||
grid: false,
|
||||
type: "ti2i",
|
||||
},
|
||||
{
|
||||
manufacturer: "volcengine",
|
||||
model: "doubao-seedream-4-0-250828",
|
||||
grid: false,
|
||||
type: "ti2i",
|
||||
},
|
||||
//可灵
|
||||
{
|
||||
manufacturer: "kling",
|
||||
model: "kling-image-o1",
|
||||
grid: false,
|
||||
type: "ti2i",
|
||||
},
|
||||
//gemini
|
||||
{
|
||||
manufacturer: "gemini",
|
||||
model: "gemini-2.5-flash-image",
|
||||
grid: true,
|
||||
type: "ti2i",
|
||||
},
|
||||
{
|
||||
manufacturer: "gemini",
|
||||
model: "gemini-2.5-flash-image-preview",
|
||||
grid: true,
|
||||
type: "ti2i",
|
||||
},
|
||||
{
|
||||
manufacturer: "gemini",
|
||||
model: "gemini-2.5-flash-image-preview-all",
|
||||
grid: true,
|
||||
type: "ti2i",
|
||||
},
|
||||
{
|
||||
manufacturer: "gemini",
|
||||
model: "gemini-3-pro-image-preview",
|
||||
grid: true,
|
||||
type: "ti2i",
|
||||
},
|
||||
//Vidu
|
||||
{
|
||||
manufacturer: "vidu",
|
||||
model: "viduq2",
|
||||
grid: false,
|
||||
type: "ti2i",
|
||||
},
|
||||
//RunningHub
|
||||
{
|
||||
manufacturer: "runninghub",
|
||||
model: "nanobanana",
|
||||
grid: true,
|
||||
type: "ti2i",
|
||||
},
|
||||
//ApiMart
|
||||
{
|
||||
manufacturer: "apimart",
|
||||
model: "nanobanana",
|
||||
grid: true,
|
||||
type: "ti2i",
|
||||
},
|
||||
];
|
||||
|
||||
export default modelList;
|
||||
34
src/utils/ai/image/owned/gemini.ts
Normal file
34
src/utils/ai/image/owned/gemini.ts
Normal file
@ -0,0 +1,34 @@
|
||||
import "../type";
|
||||
import { createGoogleGenerativeAI } from "@ai-sdk/google";
|
||||
import { generateImage } from "ai";
|
||||
|
||||
export default async (input: ImageConfig, config: AIConfig): Promise<string> => {
|
||||
if (!config.model) throw new Error("缺少Model名称");
|
||||
if (!config.apiKey) throw new Error("缺少API Key");
|
||||
if (!input.prompt) throw new Error("缺少提示词");
|
||||
|
||||
const google = createGoogleGenerativeAI({
|
||||
apiKey: config.apiKey,
|
||||
baseURL: config.baseURL,
|
||||
});
|
||||
|
||||
// 构建完整的提示词
|
||||
const fullPrompt = input.systemPrompt ? `${input.systemPrompt}\n\n${input.prompt}` : input.prompt;
|
||||
|
||||
// 根据 size 配置映射到具体尺寸
|
||||
const sizeMap: Record<string, `${number}x${number}`> = {
|
||||
"1K": "1024x1024",
|
||||
"2K": "2048x2048",
|
||||
"4K": "4096x4096",
|
||||
};
|
||||
|
||||
const { image } = await generateImage({
|
||||
model: google.image(config.model),
|
||||
prompt: fullPrompt,
|
||||
aspectRatio: input.aspectRatio as "1:1" | "3:4" | "4:3" | "9:16" | "16:9",
|
||||
size: sizeMap[input.size] ?? "1024x1024",
|
||||
});
|
||||
|
||||
// 返回生成的图片 base64
|
||||
return image.base64;
|
||||
};
|
||||
107
src/utils/ai/image/owned/kling.ts
Normal file
107
src/utils/ai/image/owned/kling.ts
Normal file
@ -0,0 +1,107 @@
|
||||
import "../type";
|
||||
import axios from "axios";
|
||||
import jwt from "jsonwebtoken";
|
||||
import u from "@/utils";
|
||||
import { pollTask } from "@/utils/ai/utils";
|
||||
|
||||
function generateJwtToken(ak: string, sk: string): string {
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const payload = {
|
||||
iss: ak,
|
||||
exp: now + 1800,
|
||||
nbf: now - 5,
|
||||
};
|
||||
return jwt.sign(payload, sk, {
|
||||
algorithm: "HS256",
|
||||
header: { alg: "HS256", typ: "JWT" },
|
||||
});
|
||||
}
|
||||
|
||||
function getApiToken(apiKey: string): string {
|
||||
const trimmedKey = apiKey.replace(/^Bearer\s+/i, "").trim();
|
||||
|
||||
if (trimmedKey.includes("|")) {
|
||||
const parts = trimmedKey.split("|");
|
||||
if (parts.length !== 2 || !parts[0].trim() || !parts[1].trim()) {
|
||||
throw new Error("API Key格式错误，请使用 ak|sk 格式");
|
||||
}
|
||||
return generateJwtToken(parts[0].trim(), parts[1].trim());
|
||||
}
|
||||
|
||||
return trimmedKey;
|
||||
}
|
||||
|
||||
async function processImages(imageBase64: string[]): Promise<Array<{ image: string }>> {
|
||||
let images = imageBase64.filter((img) => img?.trim());
|
||||
if (images.length === 0) return [];
|
||||
|
||||
// 压缩所有图片到10MB以内
|
||||
images = await Promise.all(images.map((img) => u.imageTools.compressImage(img, "10mb")));
|
||||
|
||||
// 参考主体数量和参考图片数量之和不得超过10
|
||||
if (images.length > 10) {
|
||||
const mergeImageList = images.splice(9);
|
||||
const mergedImage = await u.imageTools.mergeImages(mergeImageList, "10mb");
|
||||
images.push(mergedImage);
|
||||
}
|
||||
|
||||
return images.map((img) => ({
|
||||
image: img.replace(/^data:image\/[a-z]+;base64,/i, ""),
|
||||
}));
|
||||
}
|
||||
|
||||
export default async (input: ImageConfig, config: AIConfig): Promise<string> => {
|
||||
if (!config.apiKey) throw new Error("缺少API Key");
|
||||
if (!input.prompt) throw new Error("缺少提示词，prompt为必填项");
|
||||
|
||||
const authorization = `Bearer ${getApiToken(config.apiKey)}`;
|
||||
const baseURL = (config.baseURL ?? "https://api-beijing.klingai.com/v1/images/omni-image").replace(/\/+$/, "");
|
||||
const imageList = await processImages(input.imageBase64);
|
||||
|
||||
const body: Record<string, any> = {
|
||||
model_name: config.model || "kling-image-o1",
|
||||
prompt: input.prompt,
|
||||
n: 1,
|
||||
...(input.size !== "4K" && { resolution: input.size.toLowerCase() }),
|
||||
...(imageList.length > 0 && { image_list: imageList }),
|
||||
};
|
||||
|
||||
const headers = {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: authorization,
|
||||
};
|
||||
|
||||
try {
|
||||
const { data: createData } = await axios.post(baseURL, body, { headers });
|
||||
|
||||
if (createData.code !== 0) {
|
||||
throw new Error(createData.message || "创建任务失败");
|
||||
}
|
||||
|
||||
const taskId = createData.data?.task_id;
|
||||
if (!taskId) throw new Error("未获取到任务ID");
|
||||
|
||||
const queryUrl = `${baseURL}/${taskId}`;
|
||||
return await pollTask(async () => {
|
||||
const { data: queryData } = await axios.get(queryUrl, { headers });
|
||||
|
||||
if (queryData.code !== 0) {
|
||||
return { completed: false, error: queryData.message || "查询任务失败" };
|
||||
}
|
||||
|
||||
const { task_status, task_status_msg, task_result } = queryData.data || {};
|
||||
|
||||
if (task_status === "failed") {
|
||||
return { completed: false, error: task_status_msg || "图片生成失败" };
|
||||
}
|
||||
|
||||
if (task_status === "succeed") {
|
||||
return { completed: true, imageUrl: task_result?.images?.[0]?.url };
|
||||
}
|
||||
|
||||
return { completed: false };
|
||||
});
|
||||
} catch (error) {
|
||||
throw new Error(u.error(error).message || "可灵图片生成失败");
|
||||
}
|
||||
}
|
||||
31
src/utils/ai/image/owned/volcengine.ts
Normal file
31
src/utils/ai/image/owned/volcengine.ts
Normal file
@ -0,0 +1,31 @@
|
||||
import "../type";
|
||||
import axios from "axios";
|
||||
import u from "@/utils";
|
||||
|
||||
export default async (input: ImageConfig, config: AIConfig): Promise<string> => {
|
||||
if (!config.model) throw new Error("缺少Model名称");
|
||||
if (!config.apiKey) throw new Error("缺少API Key");
|
||||
|
||||
const apiKey = "Bearer " + config.apiKey.replace(/Bearer\s+/g, "").trim();
|
||||
const size = input.size === "1K" ? "2K" : input.size;
|
||||
|
||||
const body: Record<string, any> = {
|
||||
model: config.model,
|
||||
prompt: input.prompt,
|
||||
size,
|
||||
response_format: "url",
|
||||
sequential_image_generation: "disabled",
|
||||
stream: false,
|
||||
watermark: false,
|
||||
...(input.imageBase64 && { image: input.imageBase64 }),
|
||||
};
|
||||
|
||||
const url = config.baseURL ?? "https://ark.cn-beijing.volces.com/api/v3/images/generations";
|
||||
try {
|
||||
const { data } = await axios.post(url, body, { headers: { Authorization: apiKey } });
|
||||
return data.data[0]?.url;
|
||||
} catch (error) {
|
||||
const msg = u.error(error).message || "Volcengine 图片生成失败";
|
||||
throw new Error(msg);
|
||||
}
|
||||
}
|
||||
14
src/utils/ai/image/type.ts
Normal file
14
src/utils/ai/image/type.ts
Normal file
@ -0,0 +1,14 @@
|
||||
interface ImageConfig {
|
||||
systemPrompt?: string;
|
||||
prompt: string;
|
||||
imageBase64: string[];
|
||||
size: "1K" | "2K" | "4K";
|
||||
aspectRatio: string;
|
||||
resType?: "url" | "b64";
|
||||
}
|
||||
|
||||
interface AIConfig {
|
||||
model?: string;
|
||||
apiKey?: string;
|
||||
baseURL?: string;
|
||||
}
|
||||
13
src/utils/ai/utils.ts
Normal file
13
src/utils/ai/utils.ts
Normal file
@ -0,0 +1,13 @@
|
||||
export const pollTask = async (
|
||||
queryFn: () => Promise<{ completed: boolean; imageUrl?: string; error?: string }>,
|
||||
maxAttempts = 500,
|
||||
interval = 2000,
|
||||
): Promise<string> => {
|
||||
for (let i = 0; i < maxAttempts; i++) {
|
||||
await new Promise((resolve) => setTimeout(resolve, interval));
|
||||
const { completed, imageUrl, error } = await queryFn();
|
||||
if (error) throw new Error(error);
|
||||
if (completed && imageUrl) return imageUrl;
|
||||
}
|
||||
throw new Error(`任务轮询超时，已尝试 ${maxAttempts} 次`);
|
||||
};
|
||||
70
src/utils/ai/video/owned/volcengine.ts
Normal file
70
src/utils/ai/video/owned/volcengine.ts
Normal file
@ -0,0 +1,70 @@
|
||||
import "../type";
|
||||
import axios from "axios";
|
||||
import u from "@/utils";
|
||||
|
||||
interface DoubaoVideoConfig {
|
||||
prompt: string;
|
||||
savePath: string;
|
||||
imageBase64?: string[]; // 单张参考图片 base64
|
||||
duration: 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12; // 支持 2~12 秒
|
||||
aspectRatio: "16:9" | "9:16" | "1:1" | "4:3" | "3:4" | "21:9" | "adaptive";
|
||||
audio?: boolean;
|
||||
}
|
||||
|
||||
const pollTask = async (
|
||||
queryFn: () => Promise<{ completed: boolean; imageUrl?: string; error?: string }>,
|
||||
maxAttempts = 500,
|
||||
interval = 2000,
|
||||
): Promise<string> => {
|
||||
for (let i = 0; i < maxAttempts; i++) {
|
||||
await new Promise((resolve) => setTimeout(resolve, interval));
|
||||
const { completed, imageUrl, error } = await queryFn();
|
||||
if (error) throw new Error(error);
|
||||
if (completed && imageUrl) return imageUrl;
|
||||
}
|
||||
throw new Error(`任务轮询超时，已尝试 ${maxAttempts} 次`);
|
||||
};
|
||||
|
||||
export default async (input: ImageConfig, config: AIConfig) => {
|
||||
console.log("%c Line:5 🍓 input", "background:#7f2b82", input);
|
||||
console.log("%c Line:5 🍎 config", "background:#93c0a4", config);
|
||||
if (!config.model) throw new Error("缺少Model名称");
|
||||
if (!config.apiKey) throw new Error("缺少API Key");
|
||||
|
||||
const key = "Bearer " + config.apiKey.replaceAll("Bearer ", "").trim();
|
||||
|
||||
const doubaoConfig = config as DoubaoVideoConfig;
|
||||
const createRes = await axios.post(
|
||||
config.baseURL ?? "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks",
|
||||
{
|
||||
model: "doubao-seedance-1-5-pro-251215",
|
||||
content: [
|
||||
{ type: "text", text: input.prompt },
|
||||
...(doubaoConfig.imageBase64
|
||||
? doubaoConfig.imageBase64.map((base64, i) => ({
|
||||
type: "image_url",
|
||||
image_url: { url: base64 },
|
||||
role: i === 0 ? "first_frame" : "last_frame",
|
||||
}))
|
||||
: []),
|
||||
],
|
||||
generate_audio: doubaoConfig.audio ?? false,
|
||||
duration: doubaoConfig.duration,
|
||||
resolution: doubaoConfig.aspectRatio,
|
||||
watermark: false,
|
||||
},
|
||||
{ headers: { "Content-Type": "application/json", Authorization: key } },
|
||||
);
|
||||
const taskId = createRes.data.id;
|
||||
if (!taskId) throw new Error("视频任务创建失败");
|
||||
return await pollTask(async () => {
|
||||
const res = await axios.get(`${config.baseURL ?? "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks"}/${taskId}`, {
|
||||
headers: { Authorization: key },
|
||||
});
|
||||
const { status, content } = res.data;
|
||||
if (status === "succeeded") return { completed: true, imageUrl: content?.video_url };
|
||||
if (["failed", "cancelled", "expired"].includes(status)) return { completed: false, error: `任务${status}` };
|
||||
if (["queued", "running"].includes(status)) return { completed: false };
|
||||
return { completed: false, error: `未知状态: ${status}` };
|
||||
});
|
||||
};
|
||||
68
src/utils/error.ts
Normal file
68
src/utils/error.ts
Normal file
@ -0,0 +1,68 @@
|
||||
// utils/error.ts
|
||||
import { serializeError } from "serialize-error";
|
||||
import { isAxiosError } from "axios";
|
||||
|
||||
export interface NormalizedError {
|
||||
name: string;
|
||||
message: string;
|
||||
code?: string;
|
||||
status?: number;
|
||||
stack?: string;
|
||||
cause?: NormalizedError;
|
||||
responseData?: unknown;
|
||||
meta?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export function normalizeError(error: unknown): NormalizedError {
|
||||
// Axios 特殊处理
|
||||
if (isAxiosError(error)) {
|
||||
return {
|
||||
name: "AxiosError",
|
||||
message: error.response?.data?.error?.message || error.response?.data?.message || error.message,
|
||||
code: error.code,
|
||||
status: error.response?.status,
|
||||
stack: error.stack,
|
||||
responseData: error.response?.data,
|
||||
meta: {
|
||||
url: error.config?.url,
|
||||
method: error.config?.method,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// 普通 Error，用 serialize-error 处理
|
||||
if (error instanceof Error) {
|
||||
const serialized = serializeError(error);
|
||||
return {
|
||||
name: serialized.name || "Error",
|
||||
message: serialized.message || "未知错误",
|
||||
code: (serialized as any).code,
|
||||
stack: serialized.stack,
|
||||
cause: error.cause ? normalizeError(error.cause) : undefined,
|
||||
meta: extractMeta(serialized),
|
||||
};
|
||||
}
|
||||
|
||||
// 非 Error
|
||||
return {
|
||||
name: "UnknownError",
|
||||
message: String(error),
|
||||
meta: { raw: serializeError(error) },
|
||||
};
|
||||
}
|
||||
|
||||
// 提取自定义属性
|
||||
function extractMeta(obj: Record<string, unknown>): Record<string, unknown> | undefined {
|
||||
const standardKeys = ["name", "message", "stack", "cause"];
|
||||
const meta: Record<string, unknown> = {};
|
||||
|
||||
for (const [key, value] of Object.entries(obj)) {
|
||||
if (!standardKeys.includes(key) && value !== undefined) {
|
||||
meta[key] = value;
|
||||
}
|
||||
}
|
||||
|
||||
return Object.keys(meta).length > 0 ? meta : undefined;
|
||||
}
|
||||
|
||||
export default normalizeError;
|
||||
@ -13,9 +13,9 @@ interface TextResData extends BaseConfig {
|
||||
manufacturer: "deepseek" | "openAi" | "doubao";
|
||||
}
|
||||
|
||||
// 图像模型配置接口
|
||||
interface ImageResData extends BaseConfig {
|
||||
baseURL: string;
|
||||
manufacturer: "openAi" | "gemini" | "volcengine" | "runninghub" | "apimart";
|
||||
manufacturer: "gemini" | "volcengine" | "kling" | "vidu" | "runninghub" | "apimart";
|
||||
}
|
||||
|
||||
interface VideoResData extends BaseConfig {
|
||||
|
||||
122
src/utils/imageTools.ts
Normal file
122
src/utils/imageTools.ts
Normal file
@ -0,0 +1,122 @@
|
||||
import sharp from "sharp";
|
||||
|
||||
/**
|
||||
* 解析大小字符串为字节数
|
||||
*/
|
||||
function parseSize(size: string): number {
|
||||
const match = size.toLowerCase().match(/^(\d+(?:\.\d+)?)\s*(kb|mb|gb|b)?$/);
|
||||
if (!match) {
|
||||
throw new Error(`无效的大小格式: ${size}`);
|
||||
}
|
||||
const value = parseFloat(match[1]);
|
||||
const unit = match[2] || "b";
|
||||
const multipliers: Record<string, number> = {
|
||||
b: 1,
|
||||
kb: 1024,
|
||||
mb: 1024 * 1024,
|
||||
gb: 1024 * 1024 * 1024,
|
||||
};
|
||||
return Math.floor(value * multipliers[unit]);
|
||||
}
|
||||
|
||||
/**
|
||||
* 将base64字符串转换为Buffer
|
||||
*/
|
||||
function base64ToBuffer(base64: string): Buffer {
|
||||
const base64Data = base64.replace(/^data:image\/\w+;base64,/, "");
|
||||
return Buffer.from(base64Data, "base64");
|
||||
}
|
||||
|
||||
/**
|
||||
* 压缩Buffer到指定大小以内
|
||||
*/
|
||||
async function compressToSize(imageBuffer: Buffer, maxBytes: number, originalWidth: number, originalHeight: number): Promise<Buffer> {
|
||||
let quality = 90;
|
||||
let scale = 1;
|
||||
|
||||
while (true) {
|
||||
const targetWidth = Math.round(originalWidth * scale);
|
||||
const targetHeight = Math.round(originalHeight * scale);
|
||||
|
||||
const resultBuffer = await sharp(imageBuffer).resize(targetWidth, targetHeight, { fit: "fill" }).jpeg({ quality }).toBuffer();
|
||||
|
||||
if (resultBuffer.length <= maxBytes) {
|
||||
return resultBuffer;
|
||||
}
|
||||
|
||||
if (quality > 10) {
|
||||
quality -= 10;
|
||||
} else {
|
||||
quality = 90;
|
||||
scale *= 0.8;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 压缩单张图片到指定大小以内
|
||||
* @param imageBase64 - base64编码的图片
|
||||
* @param maxSize - 最大输出大小，支持格式如 "10mb", "5MB", "1024kb" 等
|
||||
* @returns 压缩后的图片base64字符串
|
||||
*/
|
||||
export async function compressImage(imageBase64: string, maxSize = "10mb"): Promise<string> {
|
||||
const maxBytes = parseSize(maxSize);
|
||||
const imageBuffer = base64ToBuffer(imageBase64);
|
||||
const metadata = await sharp(imageBuffer).metadata();
|
||||
const resultBuffer = await compressToSize(imageBuffer, maxBytes, metadata.width || 1, metadata.height || 1);
|
||||
return resultBuffer.toString("base64");
|
||||
}
|
||||
|
||||
/**
|
||||
* 将多张图片横向拼接为一张，并确保输出大小不超过指定限制
|
||||
* @param imageBase64List - base64编码的图片数组
|
||||
* @param maxSize - 最大输出大小，支持格式如 "10mb", "5MB", "1024kb" 等
|
||||
* @returns 拼接后的图片base64字符串
|
||||
*/
|
||||
export async function mergeImages(imageBase64List: string[], maxSize = "10mb"): Promise<string> {
|
||||
if (imageBase64List.length === 0) {
|
||||
throw new Error("图片列表不能为空");
|
||||
}
|
||||
|
||||
const maxBytes = parseSize(maxSize);
|
||||
const imageBuffers = imageBase64List.map(base64ToBuffer);
|
||||
const imageMetadatas = await Promise.all(imageBuffers.map((buffer) => sharp(buffer).metadata()));
|
||||
const maxHeight = Math.max(...imageMetadatas.map((m) => m.height || 0));
|
||||
|
||||
// 计算各图片调整后的宽度
|
||||
const imageWidths = imageMetadatas.map((metadata) => {
|
||||
const aspectRatio = (metadata.width || 1) / (metadata.height || 1);
|
||||
return Math.round(maxHeight * aspectRatio);
|
||||
});
|
||||
const totalWidth = imageWidths.reduce((sum, w) => sum + w, 0);
|
||||
|
||||
// 拼接图片
|
||||
const resizedImages = await Promise.all(
|
||||
imageBuffers.map(async (buffer, index) => {
|
||||
return sharp(buffer).resize(imageWidths[index], maxHeight, { fit: "cover" }).toBuffer();
|
||||
}),
|
||||
);
|
||||
|
||||
let currentX = 0;
|
||||
const compositeInputs = resizedImages.map((buffer, index) => {
|
||||
const input = { input: buffer, left: currentX, top: 0 };
|
||||
currentX += imageWidths[index];
|
||||
return input;
|
||||
});
|
||||
|
||||
const mergedBuffer = await sharp({
|
||||
create: {
|
||||
width: totalWidth,
|
||||
height: maxHeight,
|
||||
channels: 4,
|
||||
background: { r: 255, g: 255, b: 255, alpha: 1 },
|
||||
},
|
||||
})
|
||||
.composite(compositeInputs)
|
||||
.jpeg({ quality: 90 })
|
||||
.toBuffer();
|
||||
|
||||
// 复用压缩逻辑
|
||||
const resultBuffer = await compressToSize(mergedBuffer, maxBytes, totalWidth, maxHeight);
|
||||
return resultBuffer.toString("base64");
|
||||
}
|
||||
25
yarn.lock
25
yarn.lock
@ -4564,6 +4564,11 @@ nodemon@^3.1.11:
|
||||
touch "^3.1.0"
|
||||
undefsafe "^2.0.5"
|
||||
|
||||
non-error@^0.1.0:
|
||||
version "0.1.0"
|
||||
resolved "https://registry.npmmirror.com/non-error/-/non-error-0.1.0.tgz#b78b7d9a67ccb03ac979f9758813336ca7521cf2"
|
||||
integrity sha512-TMB1uHiGsHRGv1uYclfhivcnf0/PdFp2pNqRxXjncaAsjYMoisaQJI+SSZCqRq+VliwRTC8tsMQfmrWjDMhkPQ==
|
||||
|
||||
nopt@^4.0.1:
|
||||
version "4.0.3"
|
||||
resolved "https://registry.npmmirror.com/nopt/-/nopt-4.0.3.tgz#a375cad9d02fd921278d954c2254d5aa57e15e48"
|
||||
@ -5330,6 +5335,14 @@ send@^1.1.0, send@^1.2.0:
|
||||
range-parser "^1.2.1"
|
||||
statuses "^2.0.2"
|
||||
|
||||
serialize-error@^13.0.1:
|
||||
version "13.0.1"
|
||||
resolved "https://registry.npmmirror.com/serialize-error/-/serialize-error-13.0.1.tgz#dd1e1bf6d3e3d01037d126bd95e919f48b0c8ec0"
|
||||
integrity sha512-bBZaRwLH9PN5HbLCjPId4dP5bNGEtumcErgOX952IsvOhVPrm3/AeK1y0UHA/QaPG701eg0yEnOKsCOC6X/kaA==
|
||||
dependencies:
|
||||
non-error "^0.1.0"
|
||||
type-fest "^5.4.1"
|
||||
|
||||
serialize-error@^7.0.1:
|
||||
version "7.0.1"
|
||||
resolved "https://registry.npmmirror.com/serialize-error/-/serialize-error-7.0.1.tgz#f1360b0447f61ffb483ec4157c737fab7d778e18"
|
||||
@ -5760,6 +5773,11 @@ supports-preserve-symlinks-flag@^1.0.0:
|
||||
resolved "https://registry.npmmirror.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09"
|
||||
integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==
|
||||
|
||||
tagged-tag@^1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.npmmirror.com/tagged-tag/-/tagged-tag-1.0.0.tgz#a0b5917c2864cba54841495abfa3f6b13edcf4d6"
|
||||
integrity sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng==
|
||||
|
||||
tar-fs@^2.0.0:
|
||||
version "2.1.4"
|
||||
resolved "https://registry.npmmirror.com/tar-fs/-/tar-fs-2.1.4.tgz#800824dbf4ef06ded9afea4acafe71c67c76b930"
|
||||
@ -5929,6 +5947,13 @@ type-fest@^0.13.1:
|
||||
resolved "https://registry.npmmirror.com/type-fest/-/type-fest-0.13.1.tgz#0172cb5bce80b0bd542ea348db50c7e21834d934"
|
||||
integrity sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==
|
||||
|
||||
type-fest@^5.4.1:
|
||||
version "5.4.3"
|
||||
resolved "https://registry.npmmirror.com/type-fest/-/type-fest-5.4.3.tgz#b4c7e028da129098911ee2162a0c30df8a1be904"
|
||||
integrity sha512-AXSAQJu79WGc79/3e9/CR77I/KQgeY1AhNvcShIH4PTcGYyC4xv6H4R4AUOwkPS5799KlVDAu8zExeCrkGquiA==
|
||||
dependencies:
|
||||
tagged-tag "^1.0.0"
|
||||
|
||||
type-is@^2.0.1:
|
||||
version "2.0.1"
|
||||
resolved "https://registry.npmmirror.com/type-is/-/type-is-2.0.1.tgz#64f6cf03f92fce4015c2b224793f6bdd4b068c97"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user