Merge branch 'develop' of https://github.com/HBAI-Ltd/Toonflow-app into develop
# Conflicts: # src/utils.ts # src/utils/getConfig.ts
This commit is contained in:
zhishi
commit
53e547d405
36
backup/agents/models.ts
Normal file
36
backup/agents/models.ts
Normal file
@ -0,0 +1,36 @@
|
|||||||
|
import { ChatOpenAI, ChatOpenAIFields } from "@langchain/openai";
|
||||||
|
|
||||||
|
export const openAI = (config: ChatOpenAIFields = {}) => {
|
||||||
|
return new ChatOpenAI({
|
||||||
|
modelName: "gpt-4.1",
|
||||||
|
temperature: 1,
|
||||||
|
configuration: {
|
||||||
|
apiKey: process.env.AI_OPENAI_KEY,
|
||||||
|
baseURL: process.env.AI_OPENAI_URL,
|
||||||
|
},
|
||||||
|
...config,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
export const doubao = (config: ChatOpenAIFields = {}) => {
|
||||||
|
return new ChatOpenAI({
|
||||||
|
model: "doubao-seed-1-6-flash-250828",
|
||||||
|
temperature: 1,
|
||||||
|
configuration: {
|
||||||
|
apiKey: process.env.AI_TIKTOK_KEY,
|
||||||
|
baseURL: process.env.AI_TIKTOK_URL,
|
||||||
|
},
|
||||||
|
...config,
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
export const deepseek = (config: ChatOpenAIFields = {}) =>
|
||||||
|
new ChatOpenAI({
|
||||||
|
model: "DeepSeek-V3.2",
|
||||||
|
temperature: 1,
|
||||||
|
configuration: {
|
||||||
|
apiKey: process.env.AI_DEEPSEEK_KEY,
|
||||||
|
baseURL: process.env.AI_DEEPSEEK_URL,
|
||||||
|
},
|
||||||
|
...config,
|
||||||
|
});
|
||||||
769
backup/agents/outlineScript/index.ts
Normal file
769
backup/agents/outlineScript/index.ts
Normal file
@ -0,0 +1,769 @@
|
|||||||
|
// @/agents/outlineScript.ts
|
||||||
|
import u from "@/utils";
|
||||||
|
import { createAgent } from "langchain";
|
||||||
|
import { EventEmitter } from "events";
|
||||||
|
import { openAI } from "@/agents/models";
|
||||||
|
import { z } from "zod";
|
||||||
|
import { tool } from "@langchain/core/tools";
|
||||||
|
import type { DB } from "@/types/database";
|
||||||
|
// ==================== 类型定义 ====================
|
||||||
|
|
||||||
|
type AgentType = "AI1" | "AI2" | "director";
|
||||||
|
type AssetType = "角色" | "道具" | "场景";
|
||||||
|
type RefreshEvent = "storyline" | "outline" | "assets";
|
||||||
|
|
||||||
|
interface AssetItem {
|
||||||
|
name: string;
|
||||||
|
description: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface EpisodeData {
|
||||||
|
episodeIndex: number;
|
||||||
|
title: string;
|
||||||
|
chapterRange: number[];
|
||||||
|
scenes: AssetItem[]; // 按 outline 出场顺序排列
|
||||||
|
characters: AssetItem[]; // 按 outline 出场顺序排列
|
||||||
|
props: AssetItem[]; // 按 outline 出场顺序排列
|
||||||
|
coreConflict: string;
|
||||||
|
outline: string; // 最高优先级，剧本生成的唯一权威
|
||||||
|
openingHook: string; // outline 第一句话的视觉化，开篇第一个镜头
|
||||||
|
keyEvents: string[]; // 4个元素：[起, 承, 转, 合]，严格按 outline 顺序
|
||||||
|
emotionalCurve: string; // 对应 keyEvents 各阶段
|
||||||
|
visualHighlights: string[]; // 按 outline 顺序排列的标志性镜头
|
||||||
|
endingHook: string; // outline 之后的悬念延伸
|
||||||
|
classicQuotes: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== Schema 定义 ====================
|
||||||
|
|
||||||
|
const sceneItemSchema = z.object({
|
||||||
|
name: z.string().describe("场景名称，如'五星酒店宴会厅'、'老旧出租屋'"),
|
||||||
|
description: z.string().describe("环境描写：空间结构、光线氛围、装饰陈设、环境细节"),
|
||||||
|
});
|
||||||
|
|
||||||
|
const characterItemSchema = z.object({
|
||||||
|
name: z.string().describe("角色姓名（必须是具体人名，禁止'众人'、'群众'等集合描述）"),
|
||||||
|
description: z.string().describe("人设样貌：年龄体态、五官特征、发型妆容、服装配饰、气质神态"),
|
||||||
|
});
|
||||||
|
|
||||||
|
const propItemSchema = z.object({
|
||||||
|
name: z.string().describe("道具名称"),
|
||||||
|
description: z.string().describe("样式描写：材质质感、颜色图案、形状尺寸、磨损痕迹、特殊标记"),
|
||||||
|
});
|
||||||
|
|
||||||
|
const episodeSchema = z.object({
|
||||||
|
episodeIndex: z.number().describe("集数索引，从1开始递增"),
|
||||||
|
title: z.string().describe("8字内标题，疑问/感叹句，含情绪爆点"),
|
||||||
|
chapterRange: z.array(z.number()).describe("关联章节号数组"),
|
||||||
|
scenes: z.array(sceneItemSchema).describe("场景列表，按 outline 出场顺序排列"),
|
||||||
|
characters: z.array(characterItemSchema).describe("角色列表，按 outline 出场顺序排列，必须是独立个体"),
|
||||||
|
props: z.array(propItemSchema).describe("道具列表，按 outline 出场顺序排列，至少3个"),
|
||||||
|
coreConflict: z.string().describe("核心矛盾：A想要X vs B阻碍X"),
|
||||||
|
outline: z.string().describe("100-300字剧情主干，最高优先级，剧本生成的唯一权威，按时间顺序完整叙述"),
|
||||||
|
openingHook: z.string().describe("开场镜头：outline 第一句话的视觉化，必须作为剧本第一个镜头"),
|
||||||
|
keyEvents: z.array(z.string()).length(4).describe("4个元素的数组：[起, 承, 转, 合]，严格按 outline 顺序从中提取"),
|
||||||
|
emotionalCurve: z.string().describe("情绪曲线，如：2(压抑)→5(反抗)→9(爆发)→3(余波)，对应 keyEvents 各阶段"),
|
||||||
|
visualHighlights: z.array(z.string()).describe("3-5个标志性镜头，按 outline 叙事顺序排列"),
|
||||||
|
endingHook: z.string().describe("结尾悬念：outline 之后的延伸，勾引下集"),
|
||||||
|
classicQuotes: z.array(z.string()).describe("1-2句金句，每句≤15字，必须从原文提取"),
|
||||||
|
});
|
||||||
|
|
||||||
|
// ==================== 常量配置 ====================
|
||||||
|
|
||||||
|
// ==================== 主类 ====================
|
||||||
|
|
||||||
|
export default class OutlineScript {
|
||||||
|
private readonly projectId: number;
|
||||||
|
readonly emitter = new EventEmitter();
|
||||||
|
history: Array<[string, string]> = [];
|
||||||
|
novelChapters: DB["t_novel"][] = [];
|
||||||
|
|
||||||
|
modelName = "gpt-4.1";
|
||||||
|
apiKey = "";
|
||||||
|
baseURL = "";
|
||||||
|
|
||||||
|
constructor(projectId: number) {
|
||||||
|
this.projectId = projectId;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== 公共方法 ====================
|
||||||
|
|
||||||
|
get events() {
|
||||||
|
return this.emitter;
|
||||||
|
}
|
||||||
|
|
||||||
|
setNovel(chapters: DB["t_novel"][]) {
|
||||||
|
this.novelChapters = chapters;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== 私有工具方法 ====================
|
||||||
|
|
||||||
|
private emit(event: string, data?: any) {
|
||||||
|
this.emitter.emit(event, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
private refresh(type: RefreshEvent) {
|
||||||
|
this.emit("refresh", type);
|
||||||
|
}
|
||||||
|
|
||||||
|
private log(action: string, detail?: string) {
|
||||||
|
const msg = detail ? `${action}: ${detail}` : action;
|
||||||
|
console.log(`\n[${new Date().toLocaleTimeString()}] ${msg}\n`);
|
||||||
|
}
|
||||||
|
|
||||||
|
private safeParseJson<T>(str: string, fallback: T): T {
|
||||||
|
try {
|
||||||
|
return JSON.parse(str);
|
||||||
|
} catch {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private uniqueByName<T extends { name: string }>(items: T[]): T[] {
|
||||||
|
return Array.from(new Map(items.map((item) => [item.name, item])).values());
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== 数据库操作 ====================
|
||||||
|
|
||||||
|
private async getProjectInfo(): Promise<any> {
|
||||||
|
return u.db("t_project").where({ id: this.projectId }).first();
|
||||||
|
}
|
||||||
|
|
||||||
|
private async getNovelInfo(asString = false): Promise<any> {
|
||||||
|
const info = await this.getProjectInfo();
|
||||||
|
if (!info) return asString ? "未查询到项目信息" : null;
|
||||||
|
|
||||||
|
if (asString) {
|
||||||
|
const fields = [
|
||||||
|
`小说名称: ${info.name}`,
|
||||||
|
`小说简介: ${info.intro}`,
|
||||||
|
`小说类型: ${info.type}`,
|
||||||
|
`目标短剧类型: ${info.artStyle}`,
|
||||||
|
`短剧画幅: ${info.videoRatio}`,
|
||||||
|
];
|
||||||
|
return fields.join("\n");
|
||||||
|
}
|
||||||
|
return info;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== 故事线操作 ====================
|
||||||
|
|
||||||
|
private async findStoryline() {
|
||||||
|
return u.db("t_storyline").where({ projectId: this.projectId }).first();
|
||||||
|
}
|
||||||
|
|
||||||
|
private async upsertStorylineContent(content: string) {
|
||||||
|
const existing = await this.findStoryline();
|
||||||
|
if (existing) {
|
||||||
|
await u.db("t_storyline").where({ projectId: this.projectId }).update({ content });
|
||||||
|
} else {
|
||||||
|
await u.db("t_storyline").insert({ projectId: this.projectId, content });
|
||||||
|
}
|
||||||
|
this.refresh("storyline");
|
||||||
|
}
|
||||||
|
|
||||||
|
private async deleteStorylineContent() {
|
||||||
|
const deleted = await u.db("t_storyline").where({ projectId: this.projectId }).del();
|
||||||
|
this.refresh("storyline");
|
||||||
|
return deleted;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== 大纲操作 ====================
|
||||||
|
|
||||||
|
private async findOutlines() {
|
||||||
|
return u.db("t_outline").where({ projectId: this.projectId }).orderBy("episode", "asc");
|
||||||
|
}
|
||||||
|
|
||||||
|
private async findOutlineById(id: number) {
|
||||||
|
return u.db("t_outline").where({ id, projectId: this.projectId }).first();
|
||||||
|
}
|
||||||
|
|
||||||
|
private async getMaxEpisode(): Promise<number> {
|
||||||
|
const result: any = await u.db("t_outline").where({ projectId: this.projectId }).max("episode as max").first();
|
||||||
|
return result?.max ?? 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async clearOutlinesAndScripts() {
|
||||||
|
const outlines = await u.db("t_outline").select("id").where({ projectId: this.projectId });
|
||||||
|
if (outlines.length === 0) return 0;
|
||||||
|
|
||||||
|
const outlineIds = outlines.map((o) => o.id);
|
||||||
|
await u.db("t_script").whereIn("outlineId", outlineIds).del();
|
||||||
|
await u.db("t_outline").where({ projectId: this.projectId }).del();
|
||||||
|
|
||||||
|
return outlines.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async insertOutlines(episodes: EpisodeData[], startEpisode: number) {
|
||||||
|
const insertList = episodes.map((ep, idx) => ({
|
||||||
|
projectId: this.projectId,
|
||||||
|
data: JSON.stringify({ ...ep, episodeIndex: startEpisode + idx }),
|
||||||
|
episode: startEpisode + idx,
|
||||||
|
}));
|
||||||
|
|
||||||
|
await u.db("t_outline").insert(insertList);
|
||||||
|
return insertList.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async createEmptyScripts(outlineIds: Array<{ id: number; data: string }>) {
|
||||||
|
const scripts = outlineIds.map((item) => {
|
||||||
|
const data = this.safeParseJson<Partial<EpisodeData>>(item.data, {});
|
||||||
|
return {
|
||||||
|
name: `第${data.episodeIndex ?? ""}集`,
|
||||||
|
content: "",
|
||||||
|
projectId: this.projectId,
|
||||||
|
outlineId: item.id,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
if (scripts.length > 0) {
|
||||||
|
await u.db("t_script").insert(scripts);
|
||||||
|
}
|
||||||
|
return scripts.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async saveOutlineData(episodes: EpisodeData[], overwrite: boolean, startEpisode?: number) {
|
||||||
|
if (overwrite) {
|
||||||
|
const cleared = await this.clearOutlinesAndScripts();
|
||||||
|
if (cleared > 0) {
|
||||||
|
this.log("清理旧数据", `删除了 ${cleared} 条大纲及关联剧本`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const actualStart = overwrite ? 1 : startEpisode ?? (await this.getMaxEpisode()) + 1;
|
||||||
|
const insertedCount = await this.insertOutlines(episodes, actualStart);
|
||||||
|
|
||||||
|
const newOutlines = await u
|
||||||
|
.db("t_outline")
|
||||||
|
.select("id", "data")
|
||||||
|
.where({ projectId: this.projectId })
|
||||||
|
.orderBy("episode", "desc")
|
||||||
|
.limit(insertedCount);
|
||||||
|
|
||||||
|
const scriptCount = await this.createEmptyScripts(newOutlines as Array<{ id: number; data: string }>);
|
||||||
|
|
||||||
|
this.refresh("outline");
|
||||||
|
return { insertedCount, scriptCount };
|
||||||
|
}
|
||||||
|
|
||||||
|
private async updateOutlineData(id: number, data: EpisodeData) {
|
||||||
|
const existing = await this.findOutlineById(id);
|
||||||
|
if (!existing) return false;
|
||||||
|
|
||||||
|
await u
|
||||||
|
.db("t_outline")
|
||||||
|
.where({ id })
|
||||||
|
.update({ data: JSON.stringify(data) });
|
||||||
|
this.refresh("outline");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async deleteOutlineData(ids: number[]) {
|
||||||
|
const results = await Promise.allSettled(ids.map((id) => u.deleteOutline(id, this.projectId)));
|
||||||
|
this.refresh("outline");
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
private formatOutlineDetail(ep: any): string {
|
||||||
|
const formatList = (items: any[], formatter: (item: any) => string) =>
|
||||||
|
items?.map((item, i) => ` ${i + 1}. ${formatter(item)}`).join("\n") || " 无";
|
||||||
|
|
||||||
|
// keyEvents 按顺序显示：起、承、转、合
|
||||||
|
const keyEventsLabels = ["起", "承", "转", "合"];
|
||||||
|
const formatKeyEvents = (events: string[]) => events?.map((e, i) => ` 【${keyEventsLabels[i] || i + 1}】${e}`).join("\n") || " 无";
|
||||||
|
|
||||||
|
return `
|
||||||
|
大纲ID: ${ep.id}
|
||||||
|
第 ${ep.episodeIndex} 集: ${ep.title || ""}
|
||||||
|
${"=".repeat(50)}
|
||||||
|
章节范围: ${ep.chapterRange?.join(", ") || ""}
|
||||||
|
核心矛盾: ${ep.coreConflict || ""}
|
||||||
|
|
||||||
|
【剧情主干】(最高优先级，剧本生成的唯一权威):
|
||||||
|
${ep.outline || "无"}
|
||||||
|
|
||||||
|
【开场镜头】(必须作为剧本第一个镜头):
|
||||||
|
${ep.openingHook || "无"}
|
||||||
|
|
||||||
|
【剧情节点】(严格按顺序：起→承→转→合):
|
||||||
|
${formatKeyEvents(ep.keyEvents)}
|
||||||
|
|
||||||
|
情绪曲线: ${ep.emotionalCurve || ""}
|
||||||
|
|
||||||
|
【视觉重点】(按剧情主干顺序排列):
|
||||||
|
${formatList(ep.visualHighlights, (v) => v)}
|
||||||
|
|
||||||
|
【结尾悬念】:
|
||||||
|
${ep.endingHook || "无"}
|
||||||
|
|
||||||
|
【经典台词】:
|
||||||
|
${formatList(ep.classicQuotes, (q) => q)}
|
||||||
|
|
||||||
|
角色(按出场顺序): ${ep.characters?.map((c: AssetItem) => `${c.name}(${c.description})`).join("; ") || "无"}
|
||||||
|
场景(按出场顺序): ${ep.scenes?.map((s: AssetItem) => `${s.name}(${s.description})`).join("; ") || "无"}
|
||||||
|
道具(按出场顺序): ${ep.props?.map((p: AssetItem) => `${p.name}(${p.description})`).join("; ") || "无"}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async getOutlineText(simplified: boolean): Promise<string> {
|
||||||
|
const records = await this.findOutlines();
|
||||||
|
|
||||||
|
if (!records.length) return "当前项目暂无大纲";
|
||||||
|
|
||||||
|
const episodes = records.map((r) => ({
|
||||||
|
id: r.id,
|
||||||
|
episode: r.episode,
|
||||||
|
...this.safeParseJson<Partial<EpisodeData>>(r.data ?? "{}", {}),
|
||||||
|
}));
|
||||||
|
|
||||||
|
if (simplified) {
|
||||||
|
const list = episodes.map((ep) => `第 ${ep.episodeIndex ?? ep.episode} 集 (id=${ep.id})`).join("\n");
|
||||||
|
return `项目大纲 (共 ${episodes.length} 集):\n${list}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const details = episodes.map((ep) => this.formatOutlineDetail(ep)).join("\n");
|
||||||
|
return `项目大纲 (共 ${episodes.length} 集)\n\n${details}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== 资产操作 ====================
|
||||||
|
|
||||||
|
private async findAssetByTypeAndName(type: AssetType, name: string) {
|
||||||
|
return u.db("t_assets").where({ projectId: this.projectId, type, name }).first();
|
||||||
|
}
|
||||||
|
|
||||||
|
private async upsertAsset(type: AssetType, item: AssetItem): Promise<"inserted" | "updated" | "skipped"> {
|
||||||
|
const existing = await this.findAssetByTypeAndName(type, item.name);
|
||||||
|
|
||||||
|
if (!existing) {
|
||||||
|
await u.db("t_assets").insert({
|
||||||
|
projectId: this.projectId,
|
||||||
|
type,
|
||||||
|
name: item.name,
|
||||||
|
intro: item.description,
|
||||||
|
prompt: item.description,
|
||||||
|
});
|
||||||
|
return "inserted";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (existing.intro !== item.description) {
|
||||||
|
await u.db("t_assets").where({ id: existing.id }).update({
|
||||||
|
intro: item.description,
|
||||||
|
prompt: item.description,
|
||||||
|
});
|
||||||
|
return "updated";
|
||||||
|
}
|
||||||
|
|
||||||
|
return "skipped";
|
||||||
|
}
|
||||||
|
|
||||||
|
private extractAssetsFromOutlines(outlines: Array<{ data?: string | null | undefined }>): {
|
||||||
|
characters: AssetItem[];
|
||||||
|
props: AssetItem[];
|
||||||
|
scenes: AssetItem[];
|
||||||
|
} {
|
||||||
|
const result = { characters: [] as AssetItem[], props: [] as AssetItem[], scenes: [] as AssetItem[] };
|
||||||
|
|
||||||
|
for (const outline of outlines) {
|
||||||
|
const data = this.safeParseJson<Partial<EpisodeData>>(outline.data ?? "{}", {});
|
||||||
|
if (data.characters) result.characters.push(...data.characters);
|
||||||
|
if (data.props) result.props.push(...data.props);
|
||||||
|
if (data.scenes) result.scenes.push(...data.scenes);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
characters: this.uniqueByName(result.characters),
|
||||||
|
props: this.uniqueByName(result.props),
|
||||||
|
scenes: this.uniqueByName(result.scenes),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private async generateAssetsFromOutlines() {
|
||||||
|
const outlines = await u.db("t_outline").select("data").where({ projectId: this.projectId });
|
||||||
|
|
||||||
|
if (!outlines.length) return { inserted: 0, updated: 0, skipped: 0 };
|
||||||
|
|
||||||
|
const { characters, props, scenes } = this.extractAssetsFromOutlines(outlines);
|
||||||
|
|
||||||
|
// 只做新增和更新，不做删除
|
||||||
|
const stats = { inserted: 0, updated: 0, skipped: 0 };
|
||||||
|
|
||||||
|
const processItems = async (items: AssetItem[], type: AssetType) => {
|
||||||
|
for (const item of items) {
|
||||||
|
const result = await this.upsertAsset(type, item);
|
||||||
|
stats[result]++;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
await processItems(characters, "角色");
|
||||||
|
await processItems(props, "道具");
|
||||||
|
await processItems(scenes, "场景");
|
||||||
|
|
||||||
|
this.refresh("assets");
|
||||||
|
return { ...stats };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== Tool 定义：故事线 ====================
|
||||||
|
|
||||||
|
getStoryline = tool(
|
||||||
|
async () => {
|
||||||
|
this.log("获取故事线");
|
||||||
|
const storyline = await this.findStoryline();
|
||||||
|
return storyline?.content ?? "当前项目暂无故事线";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "getStoryline",
|
||||||
|
description: "获取当前项目的故事线内容",
|
||||||
|
schema: z.object({}),
|
||||||
|
verboseParsingErrors: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
saveStoryline = tool(
|
||||||
|
async ({ content }) => {
|
||||||
|
this.log("保存故事线");
|
||||||
|
await this.upsertStorylineContent(content);
|
||||||
|
return "故事线保存成功";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "saveStoryline",
|
||||||
|
description: "保存或更新当前项目的故事线，会覆盖已有内容",
|
||||||
|
schema: z.object({
|
||||||
|
content: z.string().describe("故事线完整内容"),
|
||||||
|
}),
|
||||||
|
verboseParsingErrors: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
deleteStoryline = tool(
|
||||||
|
async () => {
|
||||||
|
this.log("删除故事线");
|
||||||
|
const deleted = await this.deleteStorylineContent();
|
||||||
|
return deleted > 0 ? "故事线删除成功" : "当前项目没有故事线";
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "deleteStoryline",
|
||||||
|
description: "删除当前项目的故事线",
|
||||||
|
schema: z.object({}),
|
||||||
|
verboseParsingErrors: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
// ==================== Tool 定义：大纲 ====================
|
||||||
|
|
||||||
|
getOutline = tool(
|
||||||
|
async ({ simplified = false }) => {
|
||||||
|
this.log("获取大纲", `简化模式: ${simplified}`);
|
||||||
|
return this.getOutlineText(simplified);
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "getOutline",
|
||||||
|
description: "获取项目大纲。simplified=true返回简化列表，false返回完整内容",
|
||||||
|
schema: z.object({
|
||||||
|
simplified: z.boolean().default(false).describe("是否返回简化版本"),
|
||||||
|
}),
|
||||||
|
verboseParsingErrors: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
saveOutline = tool(
|
||||||
|
async ({ episodes, overwrite = true, startEpisode }) => {
|
||||||
|
this.log("保存大纲", `覆盖模式: ${overwrite}, 集数: ${episodes.length}`);
|
||||||
|
const { insertedCount, scriptCount } = await this.saveOutlineData(episodes as EpisodeData[], overwrite, startEpisode);
|
||||||
|
return `大纲保存成功：插入 ${insertedCount} 集大纲，创建 ${scriptCount} 个剧本记录`;
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "saveOutline",
|
||||||
|
description: "保存大纲数据。overwrite=true会清空现有大纲后写入，false则追加到末尾",
|
||||||
|
schema: z.object({
|
||||||
|
episodes: z.array(episodeSchema).min(1).describe("大纲数据数组"),
|
||||||
|
overwrite: z.boolean().default(true).describe("是否覆盖现有大纲"),
|
||||||
|
startEpisode: z.number().optional().describe("追加模式下的起始集数（不填则自动递增）"),
|
||||||
|
}),
|
||||||
|
verboseParsingErrors: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
updateOutline = tool(
|
||||||
|
async ({ id, data }) => {
|
||||||
|
this.log("更新大纲", `ID: ${id}`);
|
||||||
|
const success = await this.updateOutlineData(id, data as EpisodeData);
|
||||||
|
return success ? `大纲ID ${id} 更新成功` : `未找到大纲ID: ${id}`;
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "updateOutline",
|
||||||
|
description: "更新指定ID的单集大纲内容",
|
||||||
|
schema: z.object({
|
||||||
|
id: z.number().describe("大纲ID"),
|
||||||
|
data: episodeSchema.describe("更新后的大纲数据"),
|
||||||
|
}),
|
||||||
|
verboseParsingErrors: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
deleteOutline = tool(
|
||||||
|
async ({ ids }) => {
|
||||||
|
this.log("删除大纲", `IDs: ${ids.join(", ")}`);
|
||||||
|
const results = await this.deleteOutlineData(ids);
|
||||||
|
const summary = results.map((r, i) => `ID ${ids[i]}: ${r.status === "fulfilled" ? "成功" : "失败"}`).join(", ");
|
||||||
|
return `删除结果: ${summary}`;
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "deleteOutline",
|
||||||
|
description: "根据大纲ID删除指定大纲及关联数据",
|
||||||
|
schema: z.object({
|
||||||
|
ids: z.array(z.number()).min(1).describe("要删除的大纲ID数组"),
|
||||||
|
}),
|
||||||
|
verboseParsingErrors: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
// ==================== Tool 定义：章节 ====================
|
||||||
|
|
||||||
|
getChapter = tool(
|
||||||
|
async ({ chapterNumbers }) => {
|
||||||
|
this.log("获取章节", `章节号: ${chapterNumbers.join(", ")}`);
|
||||||
|
|
||||||
|
const results = await Promise.all(
|
||||||
|
chapterNumbers.map(async (num) => {
|
||||||
|
const chapter = await u
|
||||||
|
.db("t_novel")
|
||||||
|
.where({ projectId: this.projectId, chapterIndex: num })
|
||||||
|
.select("chapterData", "chapterIndex", "chapter")
|
||||||
|
.first();
|
||||||
|
|
||||||
|
if (chapter) {
|
||||||
|
return `\n【第${chapter.chapterIndex}章 ${chapter.chapter || ""}】\n${chapter.chapterData}`;
|
||||||
|
}
|
||||||
|
return `\n【第${num}章】未找到`;
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
return results.join("\n\n---\n");
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "getChapter",
|
||||||
|
description: "根据章节编号获取小说章节的完整原文内容，支持批量获取",
|
||||||
|
schema: z.object({
|
||||||
|
chapterNumbers: z.array(z.number()).min(1).describe("章节编号数组"),
|
||||||
|
}),
|
||||||
|
verboseParsingErrors: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
// ==================== Tool 定义：资产 ====================
|
||||||
|
|
||||||
|
generateAssets = tool(
|
||||||
|
async () => {
|
||||||
|
this.log("生成资产");
|
||||||
|
const stats = await this.generateAssetsFromOutlines();
|
||||||
|
|
||||||
|
if (stats.inserted === 0 && stats.updated === 0 && stats.skipped === 0) {
|
||||||
|
return "当前项目没有大纲数据，无法生成资产";
|
||||||
|
}
|
||||||
|
|
||||||
|
return `资产生成完成：新增 ${stats.inserted}，更新 ${stats.updated}，保持 ${stats.skipped}`;
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "generateAssets",
|
||||||
|
description: "从当前项目的所有大纲中提取并生成角色、道具、场景资产，自动去重并清理冗余",
|
||||||
|
schema: z.object({}),
|
||||||
|
verboseParsingErrors: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
// ==================== 上下文构建 ====================
|
||||||
|
|
||||||
|
private getChapterContext(): string {
|
||||||
|
if (!this.novelChapters.length) return "无章节数据";
|
||||||
|
return this.novelChapters.map((c) => `章节号:${c.chapterIndex}，分卷:${c.reel}，章节名:${c.chapter}`).join("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
private async buildEnvironmentContext(): Promise<string> {
|
||||||
|
const [novelInfo, storyline, outlineCount] = await Promise.all([
|
||||||
|
this.getNovelInfo(true),
|
||||||
|
this.findStoryline(),
|
||||||
|
u.db("t_outline").where({ projectId: this.projectId }).count("id as count").first() as any,
|
||||||
|
]);
|
||||||
|
|
||||||
|
return `<环境信息>
|
||||||
|
项目ID: ${this.projectId}
|
||||||
|
系统时间: ${new Date().toLocaleString()}
|
||||||
|
|
||||||
|
${novelInfo}
|
||||||
|
|
||||||
|
已加载章节列表:
|
||||||
|
${this.getChapterContext()}
|
||||||
|
|
||||||
|
故事线状态: ${storyline ? "已生成" : "未生成"}
|
||||||
|
大纲状态: 共 ${outlineCount?.count ?? 0} 集
|
||||||
|
|
||||||
|
可用工具:
|
||||||
|
- getChapter: 获取章节原文
|
||||||
|
- getStoryline/saveStoryline/deleteStoryline: 故事线操作
|
||||||
|
- getOutline/saveOutline/updateOutline/deleteOutline: 大纲操作
|
||||||
|
- generateAssets: 从大纲生成资产
|
||||||
|
</环境信息>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildConversationHistory(): string {
|
||||||
|
if (!this.history.length) return "无对话历史";
|
||||||
|
return this.history.map(([role, content]) => `${role}: ${content}`).join("\n\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
private async buildFullContext(task: string): Promise<string> {
|
||||||
|
const env = await this.buildEnvironmentContext();
|
||||||
|
const history = this.buildConversationHistory();
|
||||||
|
|
||||||
|
return `${env}
|
||||||
|
|
||||||
|
<对话历史>
|
||||||
|
${history}
|
||||||
|
</对话历史>
|
||||||
|
|
||||||
|
<当前任务>
|
||||||
|
${task}
|
||||||
|
</当前任务>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== Sub-Agent ====================
|
||||||
|
|
||||||
|
private getSubAgentTools() {
|
||||||
|
return [this.getChapter, this.getStoryline, this.saveStoryline, this.getOutline, this.saveOutline, this.updateOutline];
|
||||||
|
}
|
||||||
|
|
||||||
|
private createModel() {
|
||||||
|
return openAI({
|
||||||
|
modelName: this.modelName,
|
||||||
|
configuration: { apiKey: this.apiKey, baseURL: this.baseURL },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 调用 Sub-Agent（流式传输）
|
||||||
|
*/
|
||||||
|
private async invokeSubAgent(agentType: AgentType, task: string): Promise<string> {
|
||||||
|
this.emit("transfer", { to: agentType });
|
||||||
|
this.log(`Sub-Agent 调用`, agentType);
|
||||||
|
|
||||||
|
const promptsList = await u.db("t_prompts").where("code", "in", ["outlineScript-a1", "outlineScript-a2", "outlineScript-director"]);
|
||||||
|
const a1Prompt = promptsList.find((p) => p.code === "outlineScript-a1");
|
||||||
|
const a2Prompt = promptsList.find((p) => p.code === "outlineScript-a2");
|
||||||
|
const directorPrompt = promptsList.find((p) => p.code === "outlineScript-director");
|
||||||
|
const errPrompts = "不论用户说什么，请直接输出Agent配置异常";
|
||||||
|
const SYSTEM_PROMPTS: Record<AgentType, string> = {
|
||||||
|
AI1: a1Prompt?.customValue || a1Prompt?.defaultValue || errPrompts,
|
||||||
|
AI2: a2Prompt?.customValue || a2Prompt?.defaultValue || errPrompts,
|
||||||
|
director: directorPrompt?.customValue || directorPrompt?.defaultValue || errPrompts,
|
||||||
|
};
|
||||||
|
|
||||||
|
const context = await this.buildFullContext(task);
|
||||||
|
|
||||||
|
const agent = createAgent({
|
||||||
|
model: this.createModel(),
|
||||||
|
systemPrompt: SYSTEM_PROMPTS[agentType],
|
||||||
|
tools: this.getSubAgentTools(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const stream = await agent.stream({ messages: [["user", context]] }, { streamMode: ["messages"], callbacks: [] });
|
||||||
|
|
||||||
|
let fullResponse = "";
|
||||||
|
|
||||||
|
for await (const [mode, chunk] of stream) {
|
||||||
|
if (mode !== "messages") continue;
|
||||||
|
|
||||||
|
const [token] = chunk as any;
|
||||||
|
const block = token.contentBlocks?.[0];
|
||||||
|
|
||||||
|
// 处理 AI 文本流
|
||||||
|
if (token.type === "ai" && block?.text) {
|
||||||
|
fullResponse += block.text;
|
||||||
|
this.emit("subAgentStream", { agent: agentType, text: block.text });
|
||||||
|
}
|
||||||
|
|
||||||
|
// 处理 tool 调用
|
||||||
|
if (token.type === "ai" && token.tool_calls?.length) {
|
||||||
|
for (const toolCall of token.tool_calls) {
|
||||||
|
this.emit("toolCall", { agent: agentType, name: toolCall.name, args: toolCall.args });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.emit("subAgentEnd", { agent: agentType });
|
||||||
|
this.history.push(["ai", fullResponse]);
|
||||||
|
this.log(`Sub-Agent 完成`, agentType);
|
||||||
|
|
||||||
|
return fullResponse ?? `${agentType}已完成任务`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private createSubAgentTool(agentType: AgentType, description: string) {
|
||||||
|
return tool(async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), {
|
||||||
|
name: agentType,
|
||||||
|
description,
|
||||||
|
schema: z.object({
|
||||||
|
taskDescription: z.string().describe("具体的任务描述，包含章节范围、修改要求等详细信息"),
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== 主入口 ====================
|
||||||
|
|
||||||
|
private getAllTools() {
|
||||||
|
return [
|
||||||
|
this.createSubAgentTool("AI1", "调用故事师。负责分析小说原文并生成故事线，会自行调用 saveStoryline 保存结果。"),
|
||||||
|
this.createSubAgentTool("AI2", "调用大纲师。负责根据故事线生成剧集大纲，会自行调用 saveOutline 保存结果。"),
|
||||||
|
this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲，会自行调用 updateOutline 或 saveStoryline 进行修改。"),
|
||||||
|
this.getChapter,
|
||||||
|
this.getStoryline,
|
||||||
|
this.saveStoryline,
|
||||||
|
this.deleteStoryline,
|
||||||
|
this.getOutline,
|
||||||
|
this.saveOutline,
|
||||||
|
this.updateOutline,
|
||||||
|
this.deleteOutline,
|
||||||
|
this.generateAssets,
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
async call(msg: string): Promise<string> {
|
||||||
|
this.history.push(["user", msg]);
|
||||||
|
|
||||||
|
const envContext = await this.buildEnvironmentContext();
|
||||||
|
|
||||||
|
const prompts = await u.db("t_prompts").where("code", "outlineScript-main").first();
|
||||||
|
|
||||||
|
const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常";
|
||||||
|
|
||||||
|
const mainAgent = createAgent({
|
||||||
|
model: this.createModel(),
|
||||||
|
tools: this.getAllTools(),
|
||||||
|
systemPrompt: `${envContext}\n${mainPrompts}`,
|
||||||
|
});
|
||||||
|
const stream = await mainAgent.stream({ messages: this.history }, { streamMode: ["messages"], callbacks: [] });
|
||||||
|
|
||||||
|
let fullResponse = "";
|
||||||
|
|
||||||
|
for await (const [mode, chunk] of stream) {
|
||||||
|
if (mode !== "messages") continue;
|
||||||
|
|
||||||
|
const [token] = chunk as any;
|
||||||
|
const block = token.contentBlocks?.[0];
|
||||||
|
|
||||||
|
// 处理 AI 文本流
|
||||||
|
if (token.type === "ai" && block?.text) {
|
||||||
|
fullResponse += block.text;
|
||||||
|
this.emit("data", block.text);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 处理 tool 调用
|
||||||
|
if (token.type === "ai" && token.tool_calls?.length) {
|
||||||
|
for (const toolCall of token.tool_calls) {
|
||||||
|
this.emit("toolCall", { agent: "main", name: toolCall.name, args: toolCall.args });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.history.push(["assistant", fullResponse]);
|
||||||
|
this.emit("response", fullResponse);
|
||||||
|
|
||||||
|
return fullResponse;
|
||||||
|
}
|
||||||
|
}
|
||||||
130
backup/agents/storyboard/generateImagePromptsTool.ts
Normal file
130
backup/agents/storyboard/generateImagePromptsTool.ts
Normal file
@ -0,0 +1,130 @@
|
|||||||
|
import u from "@/utils";
|
||||||
|
|
||||||
|
type AspectRatio = "16:9" | "9:16" | "21:9" | "1:1" | "4:3" | "3:4" | "3:2" | "2:3";
|
||||||
|
|
||||||
|
interface GridLayoutResult {
|
||||||
|
cols: number;
|
||||||
|
rows: number;
|
||||||
|
totalCells: number;
|
||||||
|
placeholderCount: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface GridPromptOptions {
|
||||||
|
prompts: string[];
|
||||||
|
style: string;
|
||||||
|
aspectRatio: AspectRatio;
|
||||||
|
assetsName: { name: string; intro: string }[];
|
||||||
|
}
|
||||||
|
|
||||||
|
interface GridPromptResult {
|
||||||
|
prompt: string;
|
||||||
|
gridLayout: GridLayoutResult;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 根据prompts数量计算宫格布局
|
||||||
|
*/
|
||||||
|
function calculateGridLayout(count: number): GridLayoutResult {
|
||||||
|
let cols: number;
|
||||||
|
let rows: number;
|
||||||
|
if (count <= 0) {
|
||||||
|
cols = 1;
|
||||||
|
rows = 1;
|
||||||
|
} else if (count === 1) {
|
||||||
|
cols = 1;
|
||||||
|
rows = 1;
|
||||||
|
} else if (count === 2) {
|
||||||
|
cols = 2;
|
||||||
|
rows = 1;
|
||||||
|
} else if (count === 3) {
|
||||||
|
cols = 3;
|
||||||
|
rows = 1;
|
||||||
|
} else if (count === 4) {
|
||||||
|
cols = 2;
|
||||||
|
rows = 2;
|
||||||
|
} else if (count <= 9) {
|
||||||
|
cols = 3;
|
||||||
|
rows = 3;
|
||||||
|
} else {
|
||||||
|
cols = 3;
|
||||||
|
rows = Math.ceil(count / 3);
|
||||||
|
}
|
||||||
|
const totalCells = cols * rows;
|
||||||
|
const placeholderCount = totalCells - count;
|
||||||
|
return { cols, rows, totalCells, placeholderCount };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取宽高比描述
|
||||||
|
*/
|
||||||
|
function getAspectRatioDescription(aspectRatio: AspectRatio): string {
|
||||||
|
const descriptions: Record<AspectRatio, string> = {
|
||||||
|
"16:9": "电影宽银幕",
|
||||||
|
"9:16": "竖屏短剧",
|
||||||
|
"21:9": "超宽银幕史诗感",
|
||||||
|
"1:1": "方形构图",
|
||||||
|
"4:3": "经典银幕",
|
||||||
|
"3:4": "竖版经典",
|
||||||
|
"3:2": "摄影标准",
|
||||||
|
"2:3": "竖版摄影",
|
||||||
|
};
|
||||||
|
return descriptions[aspectRatio] || "标准比例";
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 生成电影级宫格分镜提示词
|
||||||
|
*/
|
||||||
|
async function generateGridPrompt(options: GridPromptOptions): Promise<GridPromptResult> {
|
||||||
|
const { prompts, style, aspectRatio, assetsName } = options;
|
||||||
|
const layout = calculateGridLayout(prompts.length);
|
||||||
|
const aspectRatioDesc = getAspectRatioDescription(aspectRatio);
|
||||||
|
|
||||||
|
// 构建宫格位置描述
|
||||||
|
const gridPositions: string[] = [];
|
||||||
|
for (let i = 0; i < layout.totalCells; i++) {
|
||||||
|
const row = Math.floor(i / layout.cols) + 1;
|
||||||
|
const col = (i % layout.cols) + 1;
|
||||||
|
if (i < prompts.length) {
|
||||||
|
gridPositions.push(`[第${row}行第${col}列]: ${prompts[i]}`);
|
||||||
|
} else {
|
||||||
|
gridPositions.push(`[第${row}行第${col}列]: 纯黑图`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 构建资产说明
|
||||||
|
const assetsSection =
|
||||||
|
assetsName.length > 0
|
||||||
|
? `\n【可用资产】\n${assetsName.map((a) => `- ${a.name}：${a.intro}`).join("\n")}\n\n⚠️ 必须使用完整资产名称，禁止简称或代词。`
|
||||||
|
: "";
|
||||||
|
|
||||||
|
const promptsData = await u.db("t_prompts").where("code", "generateImagePrompts").first();
|
||||||
|
|
||||||
|
const mainPrompts = promptsData?.customValue || promptsData?.defaultValue;
|
||||||
|
const errData = `请输出${options.prompts.length}张图片\n提示词如下:\n${options.prompts.map((p, i) => `第${i + 1}格: ${p}`).join("\n")}`;
|
||||||
|
|
||||||
|
if (!mainPrompts) return { prompt: errData, gridLayout: layout };
|
||||||
|
|
||||||
|
const chatModel = await u.ai.text({});
|
||||||
|
|
||||||
|
const result = await chatModel!.invoke({
|
||||||
|
messages: [
|
||||||
|
{
|
||||||
|
role: "system",
|
||||||
|
content: mainPrompts,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
role: "user",
|
||||||
|
content: `请优化以下分镜提示词：\n\n【布局】${layout.cols}列×${layout.rows}行=${
|
||||||
|
layout.totalCells
|
||||||
|
}格\n【比例】${aspectRatio}（${aspectRatioDesc}）\n【风格】${style}\n${assetsSection}\n\n【原始内容】\n${gridPositions.join("\n")}`,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
prompt: result?.text ?? errData,
|
||||||
|
gridLayout: layout,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export default generateGridPrompt;
|
||||||
334
backup/agents/storyboard/generateImageTool.ts
Normal file
334
backup/agents/storyboard/generateImageTool.ts
Normal file
@ -0,0 +1,334 @@
|
|||||||
|
import generateImagePromptsTool from "@/agents/storyboard/generateImagePromptsTool";
|
||||||
|
import u from "@/utils";
|
||||||
|
import sharp from "sharp";
|
||||||
|
import { z } from "zod";
|
||||||
|
|
||||||
|
interface AssetItem {
|
||||||
|
name: string;
|
||||||
|
description: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface EpisodeData {
|
||||||
|
episodeIndex: number;
|
||||||
|
title: string;
|
||||||
|
chapterRange: number[];
|
||||||
|
scenes: AssetItem[];
|
||||||
|
characters: AssetItem[];
|
||||||
|
props: AssetItem[];
|
||||||
|
coreConflict: string;
|
||||||
|
openingHook: string;
|
||||||
|
outline: string;
|
||||||
|
keyEvents: string[];
|
||||||
|
emotionalCurve: string;
|
||||||
|
visualHighlights: string[];
|
||||||
|
endingHook: string;
|
||||||
|
classicQuotes: string[];
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ImageInfo {
|
||||||
|
name: string;
|
||||||
|
type: string;
|
||||||
|
filePath: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ResourceItem {
|
||||||
|
name: string;
|
||||||
|
intro: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 资产过滤响应的 schema
|
||||||
|
const filteredAssetsSchema = z.object({
|
||||||
|
relevantAssets: z
|
||||||
|
.array(
|
||||||
|
z.object({
|
||||||
|
name: z.string().describe("资产名称"),
|
||||||
|
reason: z.string().describe("选择该资产的原因"),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.describe("与分镜内容相关的资产列表"),
|
||||||
|
});
|
||||||
|
|
||||||
|
// 压缩图片直到不超过指定大小
|
||||||
|
async function compressImage(buffer: Buffer, maxSizeBytes: number = 3 * 1024 * 1024): Promise<Buffer> {
|
||||||
|
if (buffer.length <= maxSizeBytes) {
|
||||||
|
return buffer;
|
||||||
|
}
|
||||||
|
let quality = 90;
|
||||||
|
let compressedBuffer = await sharp(buffer).jpeg({ quality }).toBuffer();
|
||||||
|
while (compressedBuffer.length > maxSizeBytes && quality > 10) {
|
||||||
|
quality -= 10;
|
||||||
|
compressedBuffer = await sharp(buffer).jpeg({ quality }).toBuffer();
|
||||||
|
}
|
||||||
|
if (compressedBuffer.length > maxSizeBytes) {
|
||||||
|
const metadata = await sharp(buffer).metadata();
|
||||||
|
let scale = 0.9;
|
||||||
|
while (compressedBuffer.length > maxSizeBytes && scale > 0.1) {
|
||||||
|
const newWidth = Math.round((metadata.width || 1000) * scale);
|
||||||
|
const newHeight = Math.round((metadata.height || 1000) * scale);
|
||||||
|
compressedBuffer = await sharp(buffer)
|
||||||
|
.resize(newWidth, newHeight, { fit: "inside" })
|
||||||
|
.jpeg({ quality: Math.max(quality, 30) })
|
||||||
|
.toBuffer();
|
||||||
|
scale -= 0.1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return compressedBuffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 拼接多张图片为一张
|
||||||
|
async function mergeImages(imagePaths: string[]): Promise<Buffer> {
|
||||||
|
const imageBuffers = await Promise.all(imagePaths.map((path) => u.oss.getFile(path)));
|
||||||
|
const imageMetadatas = await Promise.all(imageBuffers.map((buffer) => sharp(buffer).metadata()));
|
||||||
|
const maxHeight = Math.max(...imageMetadatas.map((m) => m.height || 0));
|
||||||
|
const resizedImages = await Promise.all(
|
||||||
|
imageBuffers.map(async (buffer, index) => {
|
||||||
|
const metadata = imageMetadatas[index];
|
||||||
|
const aspectRatio = (metadata.width || 1) / (metadata.height || 1);
|
||||||
|
const newWidth = Math.round(maxHeight * aspectRatio);
|
||||||
|
return {
|
||||||
|
buffer: await sharp(buffer).resize(newWidth, maxHeight, { fit: "cover" }).toBuffer(),
|
||||||
|
width: newWidth,
|
||||||
|
};
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
let currentX = 0;
|
||||||
|
const compositeInputs = resizedImages.map(({ buffer, width }) => {
|
||||||
|
const input = {
|
||||||
|
input: buffer,
|
||||||
|
left: currentX,
|
||||||
|
top: 0,
|
||||||
|
};
|
||||||
|
currentX += width;
|
||||||
|
return input;
|
||||||
|
});
|
||||||
|
const mergedImage = await sharp({
|
||||||
|
create: {
|
||||||
|
width: currentX,
|
||||||
|
height: maxHeight,
|
||||||
|
channels: 4,
|
||||||
|
background: { r: 255, g: 255, b: 255, alpha: 1 },
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.composite(compositeInputs)
|
||||||
|
.jpeg({ quality: 90 })
|
||||||
|
.toBuffer();
|
||||||
|
return compressImage(mergedImage);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 进一步压缩单张图片到指定大小
|
||||||
|
async function compressToSize(buffer: Buffer, targetSize: number): Promise<Buffer> {
|
||||||
|
if (buffer.length <= targetSize) {
|
||||||
|
return buffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
const metadata = await sharp(buffer).metadata();
|
||||||
|
let quality = 80;
|
||||||
|
let scale = 1.0;
|
||||||
|
let compressedBuffer = buffer;
|
||||||
|
|
||||||
|
// 先尝试降低质量
|
||||||
|
while (compressedBuffer.length > targetSize && quality > 10) {
|
||||||
|
compressedBuffer = await sharp(buffer).jpeg({ quality }).toBuffer();
|
||||||
|
quality -= 10;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 如果还是太大，缩小尺寸
|
||||||
|
while (compressedBuffer.length > targetSize && scale > 0.2) {
|
||||||
|
scale -= 0.1;
|
||||||
|
const newWidth = Math.round((metadata.width || 1000) * scale);
|
||||||
|
const newHeight = Math.round((metadata.height || 1000) * scale);
|
||||||
|
compressedBuffer = await sharp(buffer)
|
||||||
|
.resize(newWidth, newHeight, { fit: "inside" })
|
||||||
|
.jpeg({ quality: Math.max(quality, 20) })
|
||||||
|
.toBuffer();
|
||||||
|
}
|
||||||
|
|
||||||
|
return compressedBuffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 确保图片列表总大小不超过指定限制
|
||||||
|
async function ensureTotalSizeLimit(buffers: Buffer[], maxTotalBytes: number = 10 * 1024 * 1024): Promise<Buffer[]> {
|
||||||
|
let totalSize = buffers.reduce((sum, buf) => sum + buf.length, 0);
|
||||||
|
|
||||||
|
if (totalSize <= maxTotalBytes) {
|
||||||
|
return buffers;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 计算每张图片的平均目标大小
|
||||||
|
const avgTargetSize = Math.floor(maxTotalBytes / buffers.length);
|
||||||
|
|
||||||
|
// 按大小降序排列，优先压缩大图片
|
||||||
|
const indexedBuffers = buffers.map((buf, idx) => ({ buf, idx, size: buf.length }));
|
||||||
|
indexedBuffers.sort((a, b) => b.size - a.size);
|
||||||
|
|
||||||
|
const result = [...buffers];
|
||||||
|
|
||||||
|
for (const item of indexedBuffers) {
|
||||||
|
totalSize = result.reduce((sum, buf) => sum + buf.length, 0);
|
||||||
|
if (totalSize <= maxTotalBytes) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 计算这张图片需要压缩到的目标大小
|
||||||
|
const excessSize = totalSize - maxTotalBytes;
|
||||||
|
const targetSize = Math.max(item.buf.length - excessSize, avgTargetSize, 100 * 1024); // 最小100KB
|
||||||
|
|
||||||
|
if (item.buf.length > targetSize) {
|
||||||
|
result[item.idx] = await compressToSize(item.buf, targetSize);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 处理图片列表，确保不超过10张且每张不超过3MB，总大小不超过10MB
|
||||||
|
async function processImages(images: ImageInfo[]): Promise<Buffer[]> {
|
||||||
|
const maxImages = 10;
|
||||||
|
let processedBuffers: Buffer[];
|
||||||
|
|
||||||
|
if (images.length <= maxImages) {
|
||||||
|
const buffers = await Promise.all(images.map((img) => u.oss.getFile(img.filePath)));
|
||||||
|
processedBuffers = await Promise.all(buffers.map((buffer) => compressImage(buffer)));
|
||||||
|
} else {
|
||||||
|
const mergeStartIndex = maxImages - 1;
|
||||||
|
const firstBuffers = await Promise.all(images.slice(0, mergeStartIndex).map((img) => u.oss.getFile(img.filePath)));
|
||||||
|
const compressedFirstImages = await Promise.all(firstBuffers.map((buffer) => compressImage(buffer)));
|
||||||
|
const imagesToMergeList = images.slice(mergeStartIndex).map((img) => img.filePath);
|
||||||
|
const mergedImage = await mergeImages(imagesToMergeList);
|
||||||
|
processedBuffers = [...compressedFirstImages, mergedImage];
|
||||||
|
}
|
||||||
|
|
||||||
|
// 确保总大小不超过10MB
|
||||||
|
return ensureTotalSizeLimit(processedBuffers);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 使用 AI 过滤与分镜相关的资产
|
||||||
|
async function filterRelevantAssets(prompts: string[], allResources: ResourceItem[], availableImages: ImageInfo[]): Promise<ImageInfo[]> {
|
||||||
|
if (allResources.length === 0 || availableImages.length === 0) {
|
||||||
|
return availableImages;
|
||||||
|
}
|
||||||
|
|
||||||
|
const availableNames = new Set(availableImages.map((img) => img.name));
|
||||||
|
const availableResources = allResources.filter((r) => availableNames.has(r.name));
|
||||||
|
|
||||||
|
if (availableResources.length === 0) {
|
||||||
|
return availableImages;
|
||||||
|
}
|
||||||
|
|
||||||
|
const chatModel = await u.ai.text({});
|
||||||
|
const result = await chatModel!.invoke({
|
||||||
|
messages: [
|
||||||
|
{
|
||||||
|
role: "user",
|
||||||
|
content: `请分析以下分镜描述，从可用资产中筛选出与分镜内容直接相关的资产。
|
||||||
|
|
||||||
|
分镜描述：
|
||||||
|
${prompts.map((p, i) => `${i + 1}. ${p}`).join("\n")}
|
||||||
|
|
||||||
|
可用资产列表：
|
||||||
|
${availableResources.map((r) => `- ${r.name}：${r.intro}`).join("\n")}
|
||||||
|
|
||||||
|
请仅选择在分镜中明确出现或被提及的角色、场景、道具。不要选择与分镜内容无关的资产。`,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
responseFormat: {
|
||||||
|
type: "json_schema",
|
||||||
|
jsonSchema: {
|
||||||
|
name: "filteredAssets",
|
||||||
|
strict: true,
|
||||||
|
schema: z.toJSONSchema(filteredAssetsSchema),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
const data = result?.json as z.infer<typeof filteredAssetsSchema>;
|
||||||
|
|
||||||
|
if (!data?.relevantAssets || data.relevantAssets.length === 0) {
|
||||||
|
return availableImages;
|
||||||
|
}
|
||||||
|
|
||||||
|
const relevantNames = new Set(data.relevantAssets.map((a) => a.name));
|
||||||
|
const filteredImages = availableImages.filter((img) => relevantNames.has(img.name));
|
||||||
|
|
||||||
|
return filteredImages.length > 0 ? filteredImages : availableImages;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 构建资产映射提示词
|
||||||
|
function buildResourcesMapPrompts(images: ImageInfo[]): string {
|
||||||
|
if (images.length === 0) return "";
|
||||||
|
|
||||||
|
const mapping = images.map((item, index) => {
|
||||||
|
if (index < 9) {
|
||||||
|
return `${item.name}=图片${index + 1}`;
|
||||||
|
} else {
|
||||||
|
return `${item.name}=图10-${index - 8}`;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
return `其中人物、场景、道具参考对照关系如下：${mapping.join(", ")}。`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export default async (cells: { prompt: string }[], scriptId: number, projectId: number) => {
|
||||||
|
const scriptData = await u.db("t_script").where({ id: scriptId, projectId }).first();
|
||||||
|
const projectInfo = await u.db("t_project").where({ id: projectId }).first();
|
||||||
|
|
||||||
|
const row = await u.db("t_outline").where({ id: scriptData?.outlineId!, projectId }).first();
|
||||||
|
const outline: EpisodeData | null = row?.data ? JSON.parse(row.data) : null;
|
||||||
|
|
||||||
|
const resources: ResourceItem[] = outline
|
||||||
|
? (["characters", "props", "scenes"] as const).flatMap((k) => outline[k]?.map((i) => ({ name: i.name, intro: i.description })) ?? [])
|
||||||
|
: [];
|
||||||
|
|
||||||
|
const resourceNames = resources.map((r) => r.name);
|
||||||
|
const imagesRaw = await u.db("t_assets").whereIn("name", resourceNames).andWhere({ projectId }).select("name", "type", "filePath");
|
||||||
|
|
||||||
|
const allImages = imagesRaw
|
||||||
|
.sort((a, b) => {
|
||||||
|
const order = ["角色", "场景", "道具"];
|
||||||
|
return order.indexOf(a.type!) - order.indexOf(b.type!);
|
||||||
|
})
|
||||||
|
.filter((img) => img.filePath) as ImageInfo[];
|
||||||
|
|
||||||
|
if (allImages.length === 0) {
|
||||||
|
throw new Error("未找到可用的图片资源");
|
||||||
|
}
|
||||||
|
|
||||||
|
const cellPrompts = cells.map((c) => c.prompt);
|
||||||
|
|
||||||
|
// 使用 AI 过滤相关资产
|
||||||
|
const filteredImages = await filterRelevantAssets(cellPrompts, resources, allImages);
|
||||||
|
|
||||||
|
const resourcesMapPrompts = buildResourcesMapPrompts(filteredImages);
|
||||||
|
console.log("====润色前：", cellPrompts);
|
||||||
|
const promptsData = await generateImagePromptsTool({
|
||||||
|
prompts: cellPrompts,
|
||||||
|
style: `类型：${projectInfo?.type!}，风格：${projectInfo?.artStyle!}`,
|
||||||
|
aspectRatio: projectInfo?.videoRatio! as any,
|
||||||
|
assetsName: resources,
|
||||||
|
});
|
||||||
|
|
||||||
|
// const prompts = `请生成${promptsData.gridLayout.totalCells}格,${promptsData.gridLayout.cols}列×${promptsData.gridLayout.rows}行宫格图。
|
||||||
|
|
||||||
|
// ${promptsData.prompt}
|
||||||
|
|
||||||
|
// 注意：请严格按照提示词内容生成图片，确保人物样貌、艺术风格、色调光影一致。
|
||||||
|
// `;
|
||||||
|
const prompts = promptsData.prompt;
|
||||||
|
console.log("====润色后：", prompts);
|
||||||
|
|
||||||
|
const processedImages = await processImages(filteredImages);
|
||||||
|
|
||||||
|
const contentStr = await u.ai.generateImage({
|
||||||
|
systemPrompt: resourcesMapPrompts,
|
||||||
|
prompt: prompts,
|
||||||
|
size: "4K",
|
||||||
|
aspectRatio: projectInfo?.videoRatio ? (projectInfo.videoRatio as any) : "16:9",
|
||||||
|
imageBase64: processedImages.map((buf) => buf.toString("base64")),
|
||||||
|
});
|
||||||
|
|
||||||
|
const match = contentStr.match(/base64,([A-Za-z0-9+/=]+)/);
|
||||||
|
const base64Str = match?.[1] ?? contentStr;
|
||||||
|
const buffer = Buffer.from(base64Str, "base64");
|
||||||
|
|
||||||
|
return buffer;
|
||||||
|
};
|
||||||
94
backup/agents/storyboard/imageSplitting.ts
Normal file
94
backup/agents/storyboard/imageSplitting.ts
Normal file
@ -0,0 +1,94 @@
|
|||||||
|
import sharp from "sharp";
|
||||||
|
|
||||||
|
interface GridLayoutResult {
|
||||||
|
cols: number;
|
||||||
|
rows: number;
|
||||||
|
totalCells: number;
|
||||||
|
placeholderCount: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 计算宫格布局
|
||||||
|
* 1张: 1x1
|
||||||
|
* 2张: 2x1
|
||||||
|
* 3张: 3x1
|
||||||
|
* 4张: 2x2
|
||||||
|
* 5-9张: 3x3
|
||||||
|
* 10-12张: 3x4
|
||||||
|
* 13-15张: 3x5
|
||||||
|
* ...以此类推（3列，行数递增）
|
||||||
|
*/
|
||||||
|
function calculateGridLayout(count: number): GridLayoutResult {
|
||||||
|
let cols: number;
|
||||||
|
let rows: number;
|
||||||
|
if (count <= 0) {
|
||||||
|
cols = 1;
|
||||||
|
rows = 1;
|
||||||
|
} else if (count === 1) {
|
||||||
|
cols = 1;
|
||||||
|
rows = 1;
|
||||||
|
} else if (count === 2) {
|
||||||
|
cols = 2;
|
||||||
|
rows = 1;
|
||||||
|
} else if (count === 3) {
|
||||||
|
cols = 3;
|
||||||
|
rows = 1;
|
||||||
|
} else if (count === 4) {
|
||||||
|
cols = 2;
|
||||||
|
rows = 2;
|
||||||
|
} else if (count <= 9) {
|
||||||
|
// 5-9格统一用3x3
|
||||||
|
cols = 3;
|
||||||
|
rows = 3;
|
||||||
|
} else {
|
||||||
|
cols = 3;
|
||||||
|
rows = Math.ceil(count / 3);
|
||||||
|
}
|
||||||
|
const totalCells = cols * rows;
|
||||||
|
const placeholderCount = totalCells - count;
|
||||||
|
return { cols, rows, totalCells, placeholderCount };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 分割宫格图片
|
||||||
|
* @param image - 输入的宫格图片 Buffer
|
||||||
|
* @param length - 实际需要的图片数量（不包含占位图）
|
||||||
|
* @returns 分割后的单张图片 Buffer 数组
|
||||||
|
*/
|
||||||
|
export default async (image: Buffer, length: number): Promise<Buffer[]> => {
|
||||||
|
const metadata = await sharp(image).metadata();
|
||||||
|
const { width: totalWidth, height: totalHeight } = metadata;
|
||||||
|
|
||||||
|
if (!totalWidth || !totalHeight) {
|
||||||
|
throw new Error("无法获取图片尺寸");
|
||||||
|
}
|
||||||
|
|
||||||
|
const { cols, rows } = calculateGridLayout(length);
|
||||||
|
|
||||||
|
const cellWidth = Math.floor(totalWidth / cols);
|
||||||
|
const cellHeight = Math.floor(totalHeight / rows);
|
||||||
|
|
||||||
|
const buffers: Buffer[] = [];
|
||||||
|
|
||||||
|
for (let i = 0; i < length; i++) {
|
||||||
|
const row = Math.floor(i / cols);
|
||||||
|
const col = i % cols;
|
||||||
|
|
||||||
|
const left = col * cellWidth;
|
||||||
|
const top = row * cellHeight;
|
||||||
|
|
||||||
|
const cellBuffer = await sharp(image)
|
||||||
|
.extract({
|
||||||
|
left,
|
||||||
|
top,
|
||||||
|
width: cellWidth,
|
||||||
|
height: cellHeight,
|
||||||
|
})
|
||||||
|
.png()
|
||||||
|
.toBuffer();
|
||||||
|
|
||||||
|
buffers.push(cellBuffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
return buffers;
|
||||||
|
};
|
||||||
737
backup/agents/storyboard/index.ts
Normal file
737
backup/agents/storyboard/index.ts
Normal file
@ -0,0 +1,737 @@
|
|||||||
|
// @/agents/Storyboard.ts
|
||||||
|
import u from "@/utils";
|
||||||
|
import { createAgent } from "langchain";
|
||||||
|
import { EventEmitter } from "events";
|
||||||
|
import { openAI } from "@/agents/models";
|
||||||
|
import { z } from "zod";
|
||||||
|
import { tool } from "@langchain/core/tools";
|
||||||
|
import type { DB } from "@/types/database";
|
||||||
|
import generateImageTool from "./generateImageTool";
|
||||||
|
import imageSplitting from "./imageSplitting";
|
||||||
|
|
||||||
|
// ==================== 类型定义 ====================
|
||||||
|
|
||||||
|
type AgentType = "segmentAgent" | "shotAgent";
|
||||||
|
type RefreshEvent = "storyline" | "outline" | "assets";
|
||||||
|
|
||||||
|
// ==================== 常量配置 ====================
|
||||||
|
|
||||||
|
// const SYSTEM_PROMPTS: Record<AgentType, string> = {
|
||||||
|
// segmentAgent: segmentPrompts,
|
||||||
|
// shotAgent: shotPrompts,
|
||||||
|
// director: directorPrompts,
|
||||||
|
// };
|
||||||
|
|
||||||
|
// ==================== 类型定义：片段和画面 ====================
|
||||||
|
|
||||||
|
interface Segment {
|
||||||
|
index: number;
|
||||||
|
description: string;
|
||||||
|
emotion?: string;
|
||||||
|
action?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Shot {
|
||||||
|
id: number; // 分镜独立ID
|
||||||
|
segmentId: number; // 所属片段ID
|
||||||
|
title: string;
|
||||||
|
x: number;
|
||||||
|
y: number;
|
||||||
|
cells: Array<{ src?: string; prompt?: string; id?: string }>; // 镜头数组，每个cell是一个镜头
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== 主类 ====================
|
||||||
|
|
||||||
|
export default class Storyboard {
|
||||||
|
private readonly projectId: number;
|
||||||
|
private readonly scriptId: number;
|
||||||
|
readonly emitter = new EventEmitter();
|
||||||
|
history: Array<[string, string]> = [];
|
||||||
|
novelChapters: DB["t_novel"][] = [];
|
||||||
|
|
||||||
|
// 存储 segmentAgent 生成的片段结果
|
||||||
|
private segments: Segment[] = [];
|
||||||
|
// 存储 shotAgent 生成的分镜结果
|
||||||
|
private shots: Shot[] = [];
|
||||||
|
// 分镜ID计数器
|
||||||
|
private shotIdCounter: number = 0;
|
||||||
|
// 存储正在生成分镜图的分镜ID
|
||||||
|
private generatingShots: Set<number> = new Set();
|
||||||
|
|
||||||
|
modelName = "gpt-4.1";
|
||||||
|
apiKey = "";
|
||||||
|
baseURL = "";
|
||||||
|
|
||||||
|
constructor(projectId: number, scriptId: number) {
|
||||||
|
this.projectId = projectId;
|
||||||
|
this.scriptId = scriptId;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 更新shopts
|
||||||
|
public updatePreShots(segmentId: number, cellId: number, cell: { src?: string; prompt?: string; id?: string }) {
|
||||||
|
console.log("%c Line:76 🍤 segmentId", "background:#465975", segmentId);
|
||||||
|
console.log("%c Line:76 🍷 cellId", "background:#ffdd4d", cellId);
|
||||||
|
console.log("%c Line:76 🍢 cell", "background:#ffdd4d", cell);
|
||||||
|
const shotIndex = this.shots.findIndex((item) => item.segmentId === segmentId);
|
||||||
|
if (shotIndex === -1) {
|
||||||
|
return `分镜 ${segmentId} 不存在，请检查分镜ID是否正确`;
|
||||||
|
}
|
||||||
|
const cellIndex = this.shots[shotIndex].cells.findIndex((item) => item.id === cellId.toString());
|
||||||
|
if (cellIndex === -1) {
|
||||||
|
return `镜头 ${cellId} 不存在，请检查镜头ID是否正确`;
|
||||||
|
}
|
||||||
|
this.shots[shotIndex].cells[cellIndex] = { ...this.shots[shotIndex].cells[cellIndex], ...cell };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== 公共方法 ====================
|
||||||
|
|
||||||
|
get events() {
|
||||||
|
return this.emitter;
|
||||||
|
}
|
||||||
|
// ==================== 私有工具方法 ====================
|
||||||
|
|
||||||
|
private emit(event: string, data?: any) {
|
||||||
|
this.emitter.emit(event, data);
|
||||||
|
}
|
||||||
|
|
||||||
|
private refresh(type: RefreshEvent) {
|
||||||
|
this.emit("refresh", type);
|
||||||
|
}
|
||||||
|
|
||||||
|
private log(action: string, detail?: string) {
|
||||||
|
const msg = detail ? `${action}: ${detail}` : action;
|
||||||
|
console.log(`\n[${new Date().toLocaleTimeString()}] ${msg}\n`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== 剧本相关操作 ====================
|
||||||
|
|
||||||
|
getScript = tool(
|
||||||
|
async () => {
|
||||||
|
this.log("获取剧本", `scriptId: ${this.scriptId}`);
|
||||||
|
const script = await u.db("t_script").where({ id: this.scriptId, projectId: this.projectId }).first();
|
||||||
|
if (!script) throw new Error("剧本不存在");
|
||||||
|
return `剧本集：${script.name}\n\n内容：\n\`\`\`${script.content}\`\`\``;
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "getScript",
|
||||||
|
description: "获取剧本内容",
|
||||||
|
schema: z.object({}),
|
||||||
|
verboseParsingErrors: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
// ==================== 资产相关操作 ====================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取资产列表（供 segmentAgent 和 shotAgent 调用）
|
||||||
|
*/
|
||||||
|
getAssets = tool(
|
||||||
|
async () => {
|
||||||
|
this.log("获取资产列表", `scriptId: ${this.scriptId}`);
|
||||||
|
const scriptData = await u.db("t_script").where({ id: this.scriptId, projectId: this.projectId }).first();
|
||||||
|
const row = await u.db("t_outline").where({ id: scriptData?.outlineId!, projectId: this.projectId }).first();
|
||||||
|
const outline: any | null = row?.data ? JSON.parse(row.data) : null;
|
||||||
|
|
||||||
|
if (!outline) {
|
||||||
|
return "暂无资产数据";
|
||||||
|
}
|
||||||
|
|
||||||
|
// 提取资源名称和描述（与generateImageTool保持一致的字段名）
|
||||||
|
const resources = outline
|
||||||
|
? (["characters", "props", "scenes"] as const).flatMap(
|
||||||
|
(k) => outline[k]?.map((i: any) => ({ name: i.name, description: i.description })) ?? [],
|
||||||
|
)
|
||||||
|
: [];
|
||||||
|
|
||||||
|
if (resources.length === 0) {
|
||||||
|
return "暂无资产数据";
|
||||||
|
}
|
||||||
|
|
||||||
|
// 分类提取资源并格式化
|
||||||
|
const characters = outline?.characters?.map((item: any) => `- ${item.name}${item.description ? `：${item.description}` : ""}`) ?? [];
|
||||||
|
const props = outline?.props?.map((item: any) => `- ${item.name}${item.description ? `：${item.description}` : ""}`) ?? [];
|
||||||
|
const scenes = outline?.scenes?.map((item: any) => `- ${item.name}${item.description ? `：${item.description}` : ""}`) ?? [];
|
||||||
|
|
||||||
|
const sections = [
|
||||||
|
characters.length ? `【角色】\n${characters.join("\n")}` : "",
|
||||||
|
props.length ? `【道具】\n${props.join("\n")}` : "",
|
||||||
|
scenes.length ? `【场景】\n${scenes.join("\n")}` : "",
|
||||||
|
].filter(Boolean);
|
||||||
|
|
||||||
|
if (sections.length === 0) {
|
||||||
|
return "暂无资产数据";
|
||||||
|
}
|
||||||
|
|
||||||
|
return `<资产列表>
|
||||||
|
${sections.join("\n\n")}
|
||||||
|
</资产列表>
|
||||||
|
|
||||||
|
⚠️ 重要规则：
|
||||||
|
1. 必须原封不动地使用上述资产名称，禁止使用近义词、缩写或任何变体
|
||||||
|
2. 禁止在资产名称前后添加修饰词
|
||||||
|
3. 禁止捏造资产列表中不存在的角色、场景、道具`;
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "getAssets",
|
||||||
|
description: "获取资产列表（角色、道具、场景），包含名称和详细介绍。生成片段和分镜时必须先调用此工具获取资产信息，确保名称一致性",
|
||||||
|
schema: z.object({}),
|
||||||
|
verboseParsingErrors: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
// ==================== 片段和分镜工具 ====================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取当前存储的片段数据（供 shotAgent 调用）
|
||||||
|
*/
|
||||||
|
getSegments = tool(
|
||||||
|
async () => {
|
||||||
|
this.log("获取片段数据", `共 ${this.segments.length} 个片段`);
|
||||||
|
if (this.segments.length === 0) {
|
||||||
|
return "暂无片段数据，请先调用 segmentAgent 生成片段";
|
||||||
|
}
|
||||||
|
return JSON.stringify(this.segments, null, 2);
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "getSegments",
|
||||||
|
description: "获取当前已生成的片段数据，用于生成分镜",
|
||||||
|
schema: z.object({}),
|
||||||
|
verboseParsingErrors: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 更新/存储片段数据（供 segmentAgent 调用）
|
||||||
|
*/
|
||||||
|
updateSegments = tool(
|
||||||
|
async ({ segments }: { segments: Segment[] }) => {
|
||||||
|
this.log("更新片段数据", `共 ${segments.length} 个片段`);
|
||||||
|
this.segments = segments;
|
||||||
|
this.emit("segmentsUpdated", this.segments);
|
||||||
|
return `成功存储 ${segments.length} 个片段`;
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "updateSegments",
|
||||||
|
description: "存储生成的片段数据，segmentAgent 在生成片段后必须调用此工具保存结果",
|
||||||
|
schema: z.object({
|
||||||
|
segments: z
|
||||||
|
.array(
|
||||||
|
z.object({
|
||||||
|
index: z.number().describe("片段序号"),
|
||||||
|
description: z.string().describe("片段描述"),
|
||||||
|
emotion: z.string().optional().describe("情绪氛围"),
|
||||||
|
action: z.string().optional().describe("主要动作"),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.describe("片段数组"),
|
||||||
|
}),
|
||||||
|
verboseParsingErrors: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 添加分镜（供 shotAgent 调用）
|
||||||
|
*/
|
||||||
|
addShots = tool(
|
||||||
|
async ({ shots }: { shots: Array<{ segmentIndex: number; prompts: string[] }> }) => {
|
||||||
|
const added: { id: number; segmentIndex: number }[] = [];
|
||||||
|
const skipped: number[] = [];
|
||||||
|
|
||||||
|
for (const item of shots) {
|
||||||
|
const exists = this.shots.some((f) => f.segmentId === item.segmentIndex);
|
||||||
|
if (exists) {
|
||||||
|
skipped.push(item.segmentIndex);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// 分配独立的分镜ID
|
||||||
|
this.shotIdCounter++;
|
||||||
|
const shotId = this.shotIdCounter;
|
||||||
|
this.shots.push({
|
||||||
|
id: shotId,
|
||||||
|
segmentId: item.segmentIndex,
|
||||||
|
title: `分镜 ${shotId}`,
|
||||||
|
x: 0,
|
||||||
|
y: 0,
|
||||||
|
cells: item.prompts.map((prompt) => ({ id: u.uuid(), prompt })),
|
||||||
|
});
|
||||||
|
added.push({ id: shotId, segmentIndex: item.segmentIndex });
|
||||||
|
}
|
||||||
|
|
||||||
|
const addedInfo = added.map((a) => `分镜${a.id}(片段${a.segmentIndex})`).join(", ");
|
||||||
|
this.log("添加分镜", `新增: [${addedInfo}], 跳过片段: [${skipped.join(", ")}]`);
|
||||||
|
this.emit("shotsUpdated", this.shots);
|
||||||
|
|
||||||
|
if (skipped.length) {
|
||||||
|
return `已添加${addedInfo}；片段 ${skipped.join(", ")} 已存在分镜被跳过。当前共 ${this.shots.length} 个分镜`;
|
||||||
|
}
|
||||||
|
return `已添加${addedInfo}。当前共 ${this.shots.length} 个分镜`;
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "addShots",
|
||||||
|
description: "添加新的分镜。每个分镜有独立ID，包含多个镜头（每个镜头对应一个提示词）。如果片段已存在分镜会跳过",
|
||||||
|
schema: z.object({
|
||||||
|
shots: z
|
||||||
|
.array(
|
||||||
|
z.object({
|
||||||
|
segmentIndex: z.number().describe("对应的片段序号"),
|
||||||
|
prompts: z.array(z.string()).describe("镜头提示词数组，每个提示词对应一个镜头（中文）"),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.describe("要添加的分镜数组"),
|
||||||
|
}),
|
||||||
|
verboseParsingErrors: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 更新指定分镜（供 shotAgent 调用）
|
||||||
|
* 保留原有 cells 的 id 和 src 字段，只更新 prompt
|
||||||
|
*/
|
||||||
|
updateShots = tool(
|
||||||
|
async ({ shotId, prompts }: { shotId: number; prompts: string[] }) => {
|
||||||
|
const existingIndex = this.shots.findIndex((item) => item.id === shotId);
|
||||||
|
|
||||||
|
if (existingIndex === -1) {
|
||||||
|
return `分镜 ${shotId} 不存在，请检查分镜ID是否正确`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const existingCells = this.shots[existingIndex].cells;
|
||||||
|
|
||||||
|
// 更新 cells，保留原有的 id 和 src 字段
|
||||||
|
this.shots[existingIndex].cells = prompts.map((prompt, i) => {
|
||||||
|
const existingCell = existingCells[i];
|
||||||
|
if (existingCell) {
|
||||||
|
// 保留原有 cell 的 id 和 src，只更新 prompt
|
||||||
|
return { ...existingCell, prompt };
|
||||||
|
} else {
|
||||||
|
// 新增的 cell
|
||||||
|
return { id: u.uuid(), prompt };
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
this.log("更新分镜", `分镜 ${shotId}`);
|
||||||
|
this.emit("shotsUpdated", this.shots);
|
||||||
|
|
||||||
|
return `已更新分镜 ${shotId}`;
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "updateShots",
|
||||||
|
description: "更新指定分镜的镜头提示词。通过分镜ID指定要修改的分镜",
|
||||||
|
schema: z.object({
|
||||||
|
shotId: z.number().describe("要更新的分镜ID"),
|
||||||
|
prompts: z.array(z.string()).describe("新的镜头提示词数组，每个提示词对应一个镜头"),
|
||||||
|
}),
|
||||||
|
verboseParsingErrors: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 删除指定分镜（供 shotAgent 调用）
|
||||||
|
*/
|
||||||
|
deleteShots = tool(
|
||||||
|
async ({ shotIds }: { shotIds: number[] }) => {
|
||||||
|
const deleted: number[] = [];
|
||||||
|
const notFound: number[] = [];
|
||||||
|
|
||||||
|
for (const shotId of shotIds) {
|
||||||
|
const idx = this.shots.findIndex((item) => item.id === shotId);
|
||||||
|
if (idx === -1) {
|
||||||
|
notFound.push(shotId);
|
||||||
|
} else {
|
||||||
|
this.shots.splice(idx, 1);
|
||||||
|
deleted.push(shotId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.log("删除分镜", `删除: [分镜${deleted.join(", 分镜")}], 未找到: [分镜${notFound.join(", 分镜")}]`);
|
||||||
|
this.emit("shotsUpdated", this.shots);
|
||||||
|
|
||||||
|
if (notFound.length) {
|
||||||
|
return `已删除分镜 ${deleted.join(", ")}；分镜 ${notFound.join(", ")} 不存在。当前共 ${this.shots.length} 个分镜`;
|
||||||
|
}
|
||||||
|
return `已删除分镜 ${deleted.join(", ")}。当前共 ${this.shots.length} 个分镜`;
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "deleteShots",
|
||||||
|
description: "删除指定的分镜。通过分镜ID指定要删除的分镜",
|
||||||
|
schema: z.object({
|
||||||
|
shotIds: z.array(z.number()).describe("要删除的分镜ID数组"),
|
||||||
|
}),
|
||||||
|
verboseParsingErrors: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 生成分镜图（异步执行，使用 nanoBanana）
|
||||||
|
*/
|
||||||
|
generateShotImage = tool(
|
||||||
|
async ({ shotIds }: { shotIds: number[] }) => {
|
||||||
|
const toGenerate: number[] = [];
|
||||||
|
const alreadyGenerating: number[] = [];
|
||||||
|
const notFound: number[] = [];
|
||||||
|
|
||||||
|
for (const shotId of shotIds) {
|
||||||
|
const shot = this.shots.find((f) => f.id === shotId);
|
||||||
|
if (!shot) {
|
||||||
|
notFound.push(shotId);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (this.generatingShots.has(shotId)) {
|
||||||
|
alreadyGenerating.push(shotId);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
toGenerate.push(shotId);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (toGenerate.length === 0) {
|
||||||
|
if (notFound.length) {
|
||||||
|
return `分镜 ${notFound.join(", ")} 不存在，请检查分镜ID是否正确`;
|
||||||
|
}
|
||||||
|
if (alreadyGenerating.length) {
|
||||||
|
return `分镜 ${alreadyGenerating.join(", ")} 正在生成中，请稍候`;
|
||||||
|
}
|
||||||
|
return "没有需要生成的分镜";
|
||||||
|
}
|
||||||
|
|
||||||
|
// 标记为正在生成
|
||||||
|
for (const id of toGenerate) {
|
||||||
|
this.generatingShots.add(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 通知前端开始生成
|
||||||
|
this.emit("shotImageGenerateStart", { shotIds: toGenerate });
|
||||||
|
this.log("开始生成分镜图", `分镜: [${toGenerate.join(", ")}]`);
|
||||||
|
|
||||||
|
// 异步执行图片生成（不阻塞 Agent 流程）
|
||||||
|
this.executeShotImageGeneration(toGenerate).catch((err) => {
|
||||||
|
this.log("分镜图生成错误", err.message);
|
||||||
|
this.emit("shotImageGenerateError", { shotIds: toGenerate, error: err.message });
|
||||||
|
});
|
||||||
|
|
||||||
|
let result = `已开始为分镜 ${toGenerate.join(", ")} 生成分镜图，生成过程在后台进行`;
|
||||||
|
if (alreadyGenerating.length) {
|
||||||
|
result += `；分镜 ${alreadyGenerating.join(", ")} 正在生成中`;
|
||||||
|
}
|
||||||
|
if (notFound.length) {
|
||||||
|
result += `；分镜 ${notFound.join(", ")} 不存在`;
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "generateShotImage",
|
||||||
|
description:
|
||||||
|
"为指定分镜生成分镜图。每个分镜会根据其所有提示词生成一张完整宫格图，然后自动分割为单格图片。通过分镜ID指定，不需要指定具体格子，整个分镜是一个完整的生成单元",
|
||||||
|
schema: z.object({
|
||||||
|
shotIds: z.array(z.number()).describe("要生成分镜图的分镜ID数组"),
|
||||||
|
}),
|
||||||
|
verboseParsingErrors: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 执行分镜图生成的具体逻辑（异步并发）
|
||||||
|
* 每个分镜包含多个镜头，所有镜头的提示词合并生成一张宫格图，再分割为单张镜头图片
|
||||||
|
*/
|
||||||
|
async executeShotImageGeneration(shotIds: number[]): Promise<void> {
|
||||||
|
await Promise.all(shotIds.map((shotId) => this.generateSingleShotImage(shotId)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 生成单个分镜的图片
|
||||||
|
*/
|
||||||
|
private async generateSingleShotImage(shotId: number): Promise<void> {
|
||||||
|
try {
|
||||||
|
const shot = this.shots.find((f) => f.id === shotId);
|
||||||
|
if (!shot) return;
|
||||||
|
|
||||||
|
// 提取所有镜头的有效提示词
|
||||||
|
const prompts: string[] = shot.cells.map((c) => c.prompt).filter((p): p is string => Boolean(p));
|
||||||
|
|
||||||
|
if (prompts.length === 0) {
|
||||||
|
this.log("跳过分镜图生成", `分镜 ${shotId} 没有有效的镜头提示词`);
|
||||||
|
this.generatingShots.delete(shotId);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 通知前端正在生成该分镜
|
||||||
|
this.emit("shotImageGenerateProgress", { shotId, status: "generating", message: "正在调用 AI 生成宫格图片" });
|
||||||
|
|
||||||
|
// 根据所有镜头提示词生成宫格图片
|
||||||
|
const gridImage = await generateImageTool(
|
||||||
|
prompts.map((p) => ({ prompt: p })),
|
||||||
|
this.scriptId,
|
||||||
|
this.projectId,
|
||||||
|
);
|
||||||
|
|
||||||
|
// 通知前端正在分割图片
|
||||||
|
this.emit("shotImageGenerateProgress", { shotId, status: "splitting", message: "正在分割宫格图片为单张镜头图" });
|
||||||
|
|
||||||
|
// 分割宫格图片为单张镜头图片
|
||||||
|
const imageBuffers = await imageSplitting(gridImage, prompts.length);
|
||||||
|
|
||||||
|
// 通知前端正在保存图片
|
||||||
|
this.emit("shotImageGenerateProgress", { shotId, status: "saving", message: `正在保存 ${imageBuffers.length} 张镜头图片` });
|
||||||
|
|
||||||
|
// 保存分割后的镜头图片到 OSS，并获取文件路径
|
||||||
|
const timestamp = Date.now();
|
||||||
|
const imagePaths: string[] = [];
|
||||||
|
|
||||||
|
for (let i = 0; i < imageBuffers.length; i++) {
|
||||||
|
const fileName = `${this.projectId}/chat/${this.scriptId}/storyboard/shot_${shotId}_take_${i}_${timestamp}.png`;
|
||||||
|
await u.oss.writeFile(fileName, imageBuffers[i]);
|
||||||
|
const imageUrl = await u.oss.getFileUrl(fileName);
|
||||||
|
imagePaths.push(imageUrl);
|
||||||
|
|
||||||
|
// 每保存一张镜头图片通知进度
|
||||||
|
this.emit("shotImageGenerateProgress", {
|
||||||
|
shotId,
|
||||||
|
status: "saving",
|
||||||
|
message: `已保存 ${i + 1}/${imageBuffers.length} 张镜头图片`,
|
||||||
|
progress: Math.round(((i + 1) / imageBuffers.length) * 100),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// 更新每个镜头的 src 字段
|
||||||
|
shot.cells = shot.cells.map((cell, i) => ({
|
||||||
|
id: u.uuid(),
|
||||||
|
...cell,
|
||||||
|
src: imagePaths[i] || cell.src,
|
||||||
|
}));
|
||||||
|
|
||||||
|
// 生成完成后更新状态
|
||||||
|
this.generatingShots.delete(shotId);
|
||||||
|
this.emit("shotImageGenerateComplete", { shotId, shot, imagePaths });
|
||||||
|
this.emit("shotsUpdated", this.shots);
|
||||||
|
this.log("分镜图生成完成", `分镜 ${shotId}，共 ${imagePaths.length} 张镜头图片`);
|
||||||
|
} catch (err: any) {
|
||||||
|
this.generatingShots.delete(shotId);
|
||||||
|
this.emit("shotImageGenerateError", { shotId, error: err.message });
|
||||||
|
this.log("分镜图生成失败", `分镜 ${shotId}: ${err.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== 公共访问器 ====================
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取当前片段数据
|
||||||
|
*/
|
||||||
|
getSegmentsData(): Segment[] {
|
||||||
|
return this.segments;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取当前分镜数据
|
||||||
|
*/
|
||||||
|
getShotsData(): Shot[] {
|
||||||
|
return this.shots;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== 上下文构建 ====================
|
||||||
|
|
||||||
|
private async buildEnvironmentContext(): Promise<string> {
|
||||||
|
const projectInfo = await u.db("t_project").where({ id: this.projectId }).first();
|
||||||
|
|
||||||
|
const row = await u.db("t_outline").where({ id: this.scriptId, projectId: this.projectId }).first();
|
||||||
|
const outline: any | null = row?.data ? JSON.parse(row.data) : null;
|
||||||
|
|
||||||
|
// 分类提取资源名称
|
||||||
|
const characters = outline?.characters?.map((i: any) => i.name) ?? [];
|
||||||
|
const props = outline?.props?.map((i: any) => i.name) ?? [];
|
||||||
|
const scenes = outline?.scenes?.map((i: any) => i.name) ?? [];
|
||||||
|
|
||||||
|
const assetList =
|
||||||
|
[
|
||||||
|
characters.length ? `【角色】${characters.join("、")}` : "",
|
||||||
|
props.length ? `【道具】${props.join("、")}` : "",
|
||||||
|
scenes.length ? `【场景】${scenes.join("、")}` : "",
|
||||||
|
]
|
||||||
|
.filter(Boolean)
|
||||||
|
.join("\n") || "无";
|
||||||
|
|
||||||
|
return `<环境信息>
|
||||||
|
项目ID: ${this.projectId}
|
||||||
|
系统时间: ${new Date().toLocaleString()}
|
||||||
|
|
||||||
|
项目名称: ${projectInfo?.name || "未知"}
|
||||||
|
项目简介: ${projectInfo?.intro || "无"}
|
||||||
|
类型: ${projectInfo?.type || "未知"}
|
||||||
|
风格: ${projectInfo?.artStyle || "未知"}
|
||||||
|
视频比例: ${projectInfo?.videoRatio || "未知"}
|
||||||
|
|
||||||
|
资产列表:
|
||||||
|
${assetList}
|
||||||
|
|
||||||
|
</环境信息>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildConversationHistory(): string {
|
||||||
|
if (!this.history.length) return "无对话历史";
|
||||||
|
return this.history.map(([role, content]) => `${role}: ${content}`).join("\n\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
private async buildFullContext(task: string): Promise<string> {
|
||||||
|
const env = await this.buildEnvironmentContext();
|
||||||
|
const history = this.buildConversationHistory();
|
||||||
|
|
||||||
|
return `${env}
|
||||||
|
|
||||||
|
<对话历史>
|
||||||
|
${history}
|
||||||
|
</对话历史>
|
||||||
|
|
||||||
|
<当前任务>
|
||||||
|
${task}
|
||||||
|
</当前任务>`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== Sub-Agent ====================
|
||||||
|
|
||||||
|
private createModel() {
|
||||||
|
return openAI({
|
||||||
|
modelName: this.modelName,
|
||||||
|
configuration: { apiKey: this.apiKey, baseURL: this.baseURL },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 获取不同 Sub-Agent 可用的工具
|
||||||
|
*/
|
||||||
|
private getSubAgentTools(agentType: AgentType) {
|
||||||
|
switch (agentType) {
|
||||||
|
case "segmentAgent":
|
||||||
|
// segmentAgent 可以获取剧本和资产，并需要调用 updateSegments 保存结果
|
||||||
|
return [this.getScript, this.getAssets, this.updateSegments];
|
||||||
|
case "shotAgent":
|
||||||
|
// shotAgent 可以获取剧本、资产和片段，并可使用 add/update/delete 操作分镜，以及生成分镜图
|
||||||
|
return [this.getScript, this.getAssets, this.getSegments, this.addShots, this.updateShots, this.deleteShots, this.generateShotImage];
|
||||||
|
default:
|
||||||
|
return [this.getScript];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 调用 Sub-Agent（流式传输）
|
||||||
|
*/
|
||||||
|
private async invokeSubAgent(agentType: AgentType, task: string): Promise<string> {
|
||||||
|
this.emit("transfer", { to: agentType });
|
||||||
|
this.log(`Sub-Agent 调用`, agentType);
|
||||||
|
|
||||||
|
const promptsList = await u.db("t_prompts").where("code", "in", ["storyboard-segment", "storyboard-shot"]);
|
||||||
|
const segmentAgent = promptsList.find((p) => p.code === "storyboard-segment");
|
||||||
|
const shotAgent = promptsList.find((p) => p.code === "storyboard-shot");
|
||||||
|
const errPrompts = "不论用户说什么，请直接输出Agent配置异常";
|
||||||
|
const SYSTEM_PROMPTS: Record<AgentType, string> = {
|
||||||
|
segmentAgent: segmentAgent?.customValue || segmentAgent?.defaultValue || errPrompts,
|
||||||
|
shotAgent: shotAgent?.customValue || shotAgent?.defaultValue || errPrompts,
|
||||||
|
};
|
||||||
|
|
||||||
|
const context = await this.buildFullContext(task);
|
||||||
|
|
||||||
|
const agent = createAgent({
|
||||||
|
model: this.createModel(),
|
||||||
|
systemPrompt: SYSTEM_PROMPTS[agentType],
|
||||||
|
tools: this.getSubAgentTools(agentType),
|
||||||
|
});
|
||||||
|
|
||||||
|
const stream = await agent.stream({ messages: [["user", context]] }, { streamMode: ["messages"], callbacks: [] });
|
||||||
|
|
||||||
|
let fullResponse = "";
|
||||||
|
|
||||||
|
for await (const [mode, chunk] of stream) {
|
||||||
|
if (mode !== "messages") continue;
|
||||||
|
const [token] = chunk as any;
|
||||||
|
const block = token.contentBlocks?.[0];
|
||||||
|
|
||||||
|
// 处理 AI 文本流
|
||||||
|
if (token.type === "ai" && block?.text) {
|
||||||
|
fullResponse += block.text;
|
||||||
|
this.emit("subAgentStream", { agent: agentType, text: block.text });
|
||||||
|
}
|
||||||
|
// 处理 tool 调用
|
||||||
|
if (token.type === "ai" && token.tool_calls?.length) {
|
||||||
|
for (const toolCall of token.tool_calls) {
|
||||||
|
this.emit("toolCall", { agent: agentType, name: toolCall.name, args: toolCall.args });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.emit("subAgentEnd", { agent: agentType });
|
||||||
|
this.history.push(["ai", fullResponse]);
|
||||||
|
this.log(`Sub-Agent 完成`, agentType);
|
||||||
|
return fullResponse;
|
||||||
|
}
|
||||||
|
|
||||||
|
private createSubAgentTool(agentType: AgentType, description: string) {
|
||||||
|
return tool(async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), {
|
||||||
|
name: agentType,
|
||||||
|
description,
|
||||||
|
schema: z.object({
|
||||||
|
taskDescription: z.string().describe("具体的任务描述，包含章节范围、修改要求等详细信息"),
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== 主入口 ====================
|
||||||
|
|
||||||
|
private getAllTools() {
|
||||||
|
return [
|
||||||
|
this.createSubAgentTool(
|
||||||
|
"segmentAgent",
|
||||||
|
"调用片段师。负责根据剧本生成片段，会自行调用 getScript 获取剧本内容，并调用 updateSegments 保存片段结果。",
|
||||||
|
),
|
||||||
|
this.createSubAgentTool(
|
||||||
|
"shotAgent",
|
||||||
|
"调用分镜师。负责根据片段生成分镜提示词，会自行调用 getSegments 获取片段数据，并调用 addShots/updateShots 保存分镜结果。",
|
||||||
|
),
|
||||||
|
// this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲，会自行调用 updateOutline 或 saveStoryline 进行修改。"),
|
||||||
|
this.getScript,
|
||||||
|
this.getSegments,
|
||||||
|
this.generateShotImage,
|
||||||
|
...this.getSubAgentTools("segmentAgent"),
|
||||||
|
...this.getSubAgentTools("shotAgent"),
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
async call(msg: string): Promise<string> {
|
||||||
|
console.log("模型名称:", this.modelName);
|
||||||
|
this.history.push(["user", msg]);
|
||||||
|
|
||||||
|
const envContext = await this.buildEnvironmentContext();
|
||||||
|
|
||||||
|
const prompts = await u.db("t_prompts").where("code", "storyboard-main").first();
|
||||||
|
|
||||||
|
const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常";
|
||||||
|
|
||||||
|
const mainAgent = createAgent({
|
||||||
|
model: this.createModel(),
|
||||||
|
tools: this.getAllTools(),
|
||||||
|
systemPrompt: `${envContext}\n${mainPrompts}`,
|
||||||
|
});
|
||||||
|
const stream = await mainAgent.stream({ messages: this.history }, { streamMode: ["messages"], callbacks: [] });
|
||||||
|
|
||||||
|
let fullResponse = "";
|
||||||
|
|
||||||
|
for await (const [mode, chunk] of stream) {
|
||||||
|
if (mode !== "messages") continue;
|
||||||
|
const [token] = chunk as any;
|
||||||
|
const block = token.contentBlocks?.[0];
|
||||||
|
// 处理 AI 文本流
|
||||||
|
if (token.type === "ai" && block?.text) {
|
||||||
|
fullResponse += block.text;
|
||||||
|
this.emit("data", block.text);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 处理 tool 调用
|
||||||
|
if (token.type === "ai" && token.tool_calls?.length) {
|
||||||
|
for (const toolCall of token.tool_calls) {
|
||||||
|
this.emit("toolCall", { agent: "main", name: toolCall.name, args: toolCall.args });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.history.push(["assistant", fullResponse]);
|
||||||
|
this.emit("response", fullResponse);
|
||||||
|
|
||||||
|
return fullResponse;
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -49,6 +49,7 @@
|
|||||||
"langchain": "^1.2.10",
|
"langchain": "^1.2.10",
|
||||||
"morgan": "^1.10.1",
|
"morgan": "^1.10.1",
|
||||||
"qwen-ai-provider": "^0.1.1",
|
"qwen-ai-provider": "^0.1.1",
|
||||||
|
"serialize-error": "^13.0.1",
|
||||||
"sharp": "^0.34.5",
|
"sharp": "^0.34.5",
|
||||||
"sqlite3": "^5.1.7",
|
"sqlite3": "^5.1.7",
|
||||||
"zhipu-ai-provider": "^0.2.2",
|
"zhipu-ai-provider": "^0.2.2",
|
||||||
|
|||||||
@ -1,10 +1,8 @@
|
|||||||
// @/agents/outlineScript.ts
|
// @/agents/outlineScript.ts
|
||||||
import u from "@/utils";
|
import u from "@/utils";
|
||||||
import { createAgent } from "langchain";
|
|
||||||
import { EventEmitter } from "events";
|
import { EventEmitter } from "events";
|
||||||
import { openAI } from "@/agents/models";
|
import { tool, ModelMessage } from "ai";
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
import { tool } from "@langchain/core/tools";
|
|
||||||
import type { DB } from "@/types/database";
|
import type { DB } from "@/types/database";
|
||||||
// ==================== 类型定义 ====================
|
// ==================== 类型定义 ====================
|
||||||
|
|
||||||
@ -75,13 +73,9 @@ const episodeSchema = z.object({
|
|||||||
export default class OutlineScript {
|
export default class OutlineScript {
|
||||||
private readonly projectId: number;
|
private readonly projectId: number;
|
||||||
readonly emitter = new EventEmitter();
|
readonly emitter = new EventEmitter();
|
||||||
history: Array<[string, string]> = [];
|
history: Array<ModelMessage> = [];
|
||||||
novelChapters: DB["t_novel"][] = [];
|
novelChapters: DB["t_novel"][] = [];
|
||||||
|
|
||||||
modelName = "gpt-4.1";
|
|
||||||
apiKey = "";
|
|
||||||
baseURL = "";
|
|
||||||
|
|
||||||
constructor(projectId: number) {
|
constructor(projectId: number) {
|
||||||
this.projectId = projectId;
|
this.projectId = projectId;
|
||||||
}
|
}
|
||||||
@ -403,123 +397,107 @@ ${formatList(ep.classicQuotes, (q) => q)}
|
|||||||
|
|
||||||
// ==================== Tool 定义：故事线 ====================
|
// ==================== Tool 定义：故事线 ====================
|
||||||
|
|
||||||
getStoryline = tool(
|
getStoryline = tool({
|
||||||
async () => {
|
title: "getStoryline",
|
||||||
|
description: "Get the weather in a location",
|
||||||
|
inputSchema: z.object({}),
|
||||||
|
execute: async () => {
|
||||||
this.log("获取故事线");
|
this.log("获取故事线");
|
||||||
const storyline = await this.findStoryline();
|
const storyline = await this.findStoryline();
|
||||||
return storyline?.content ?? "当前项目暂无故事线";
|
return storyline?.content ?? "当前项目暂无故事线";
|
||||||
},
|
},
|
||||||
{
|
});
|
||||||
name: "getStoryline",
|
|
||||||
description: "获取当前项目的故事线内容",
|
|
||||||
schema: z.object({}),
|
|
||||||
verboseParsingErrors: true,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
saveStoryline = tool(
|
saveStoryline = tool({
|
||||||
async ({ content }) => {
|
title: "saveStoryline",
|
||||||
|
description: "保存或更新当前项目的故事线，会覆盖已有内容",
|
||||||
|
inputSchema: z.object({
|
||||||
|
content: z.string().describe("故事线完整内容"),
|
||||||
|
}),
|
||||||
|
execute: async ({ content }) => {
|
||||||
this.log("保存故事线");
|
this.log("保存故事线");
|
||||||
await this.upsertStorylineContent(content);
|
await this.upsertStorylineContent(content);
|
||||||
return "故事线保存成功";
|
return "故事线保存成功";
|
||||||
},
|
},
|
||||||
{
|
});
|
||||||
name: "saveStoryline",
|
|
||||||
description: "保存或更新当前项目的故事线，会覆盖已有内容",
|
|
||||||
schema: z.object({
|
|
||||||
content: z.string().describe("故事线完整内容"),
|
|
||||||
}),
|
|
||||||
verboseParsingErrors: true,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
deleteStoryline = tool(
|
deleteStoryline = tool({
|
||||||
async () => {
|
title: "deleteStoryline",
|
||||||
|
description: "删除当前项目的故事线",
|
||||||
|
inputSchema: z.object({}),
|
||||||
|
execute: async () => {
|
||||||
this.log("删除故事线");
|
this.log("删除故事线");
|
||||||
const deleted = await this.deleteStorylineContent();
|
const deleted = await this.deleteStorylineContent();
|
||||||
return deleted > 0 ? "故事线删除成功" : "当前项目没有故事线";
|
return deleted > 0 ? "故事线删除成功" : "当前项目没有故事线";
|
||||||
},
|
},
|
||||||
{
|
});
|
||||||
name: "deleteStoryline",
|
|
||||||
description: "删除当前项目的故事线",
|
|
||||||
schema: z.object({}),
|
|
||||||
verboseParsingErrors: true,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
// ==================== Tool 定义：大纲 ====================
|
// ==================== Tool 定义：大纲 ====================
|
||||||
|
|
||||||
getOutline = tool(
|
getOutline = tool({
|
||||||
async ({ simplified = false }) => {
|
title: "getOutline",
|
||||||
|
description: "获取项目大纲。simplified=true返回简化列表，false返回完整内容",
|
||||||
|
inputSchema: z.object({
|
||||||
|
simplified: z.boolean().default(false).describe("是否返回简化版本"),
|
||||||
|
}),
|
||||||
|
execute: async ({ simplified }) => {
|
||||||
this.log("获取大纲", `简化模式: ${simplified}`);
|
this.log("获取大纲", `简化模式: ${simplified}`);
|
||||||
return this.getOutlineText(simplified);
|
return this.getOutlineText(simplified);
|
||||||
},
|
},
|
||||||
{
|
});
|
||||||
name: "getOutline",
|
|
||||||
description: "获取项目大纲。simplified=true返回简化列表，false返回完整内容",
|
|
||||||
schema: z.object({
|
|
||||||
simplified: z.boolean().default(false).describe("是否返回简化版本"),
|
|
||||||
}),
|
|
||||||
verboseParsingErrors: true,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
saveOutline = tool(
|
saveOutline = tool({
|
||||||
async ({ episodes, overwrite = true, startEpisode }) => {
|
title: "saveOutline",
|
||||||
|
description: "保存大纲数据。overwrite=true会清空现有大纲后写入，false则追加到末尾",
|
||||||
|
inputSchema: z.object({
|
||||||
|
episodes: z.array(episodeSchema).min(1).describe("大纲数据数组"),
|
||||||
|
overwrite: z.boolean().default(true).describe("是否覆盖现有大纲"),
|
||||||
|
startEpisode: z.number().optional().describe("追加模式下的起始集数（不填则自动递增）"),
|
||||||
|
}),
|
||||||
|
execute: async ({ episodes, overwrite = true, startEpisode }) => {
|
||||||
this.log("保存大纲", `覆盖模式: ${overwrite}, 集数: ${episodes.length}`);
|
this.log("保存大纲", `覆盖模式: ${overwrite}, 集数: ${episodes.length}`);
|
||||||
const { insertedCount, scriptCount } = await this.saveOutlineData(episodes as EpisodeData[], overwrite, startEpisode);
|
const { insertedCount, scriptCount } = await this.saveOutlineData(episodes as EpisodeData[], overwrite, startEpisode);
|
||||||
return `大纲保存成功：插入 ${insertedCount} 集大纲，创建 ${scriptCount} 个剧本记录`;
|
return `大纲保存成功：插入 ${insertedCount} 集大纲，创建 ${scriptCount} 个剧本记录`;
|
||||||
},
|
},
|
||||||
{
|
});
|
||||||
name: "saveOutline",
|
|
||||||
description: "保存大纲数据。overwrite=true会清空现有大纲后写入，false则追加到末尾",
|
|
||||||
schema: z.object({
|
|
||||||
episodes: z.array(episodeSchema).min(1).describe("大纲数据数组"),
|
|
||||||
overwrite: z.boolean().default(true).describe("是否覆盖现有大纲"),
|
|
||||||
startEpisode: z.number().optional().describe("追加模式下的起始集数（不填则自动递增）"),
|
|
||||||
}),
|
|
||||||
verboseParsingErrors: true,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
updateOutline = tool(
|
updateOutline = tool({
|
||||||
async ({ id, data }) => {
|
title: "updateOutline",
|
||||||
|
description: "更新指定ID的单集大纲内容",
|
||||||
|
inputSchema: z.object({
|
||||||
|
id: z.number().describe("大纲ID"),
|
||||||
|
data: episodeSchema.describe("更新后的大纲数据"),
|
||||||
|
}),
|
||||||
|
execute: async ({ id, data }) => {
|
||||||
this.log("更新大纲", `ID: ${id}`);
|
this.log("更新大纲", `ID: ${id}`);
|
||||||
const success = await this.updateOutlineData(id, data as EpisodeData);
|
const success = await this.updateOutlineData(id, data as EpisodeData);
|
||||||
return success ? `大纲ID ${id} 更新成功` : `未找到大纲ID: ${id}`;
|
return success ? `大纲ID ${id} 更新成功` : `未找到大纲ID: ${id}`;
|
||||||
},
|
},
|
||||||
{
|
});
|
||||||
name: "updateOutline",
|
|
||||||
description: "更新指定ID的单集大纲内容",
|
|
||||||
schema: z.object({
|
|
||||||
id: z.number().describe("大纲ID"),
|
|
||||||
data: episodeSchema.describe("更新后的大纲数据"),
|
|
||||||
}),
|
|
||||||
verboseParsingErrors: true,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
deleteOutline = tool(
|
deleteOutline = tool({
|
||||||
async ({ ids }) => {
|
title: "deleteOutline",
|
||||||
|
description: "根据大纲ID删除指定大纲及关联数据",
|
||||||
|
inputSchema: z.object({
|
||||||
|
ids: z.array(z.number()).min(1).describe("要删除的大纲ID数组"),
|
||||||
|
}),
|
||||||
|
execute: async ({ ids }) => {
|
||||||
this.log("删除大纲", `IDs: ${ids.join(", ")}`);
|
this.log("删除大纲", `IDs: ${ids.join(", ")}`);
|
||||||
const results = await this.deleteOutlineData(ids);
|
const results = await this.deleteOutlineData(ids);
|
||||||
const summary = results.map((r, i) => `ID ${ids[i]}: ${r.status === "fulfilled" ? "成功" : "失败"}`).join(", ");
|
const summary = results.map((r, i) => `ID ${ids[i]}: ${r.status === "fulfilled" ? "成功" : "失败"}`).join(", ");
|
||||||
return `删除结果: ${summary}`;
|
return `删除结果: ${summary}`;
|
||||||
},
|
},
|
||||||
{
|
});
|
||||||
name: "deleteOutline",
|
|
||||||
description: "根据大纲ID删除指定大纲及关联数据",
|
|
||||||
schema: z.object({
|
|
||||||
ids: z.array(z.number()).min(1).describe("要删除的大纲ID数组"),
|
|
||||||
}),
|
|
||||||
verboseParsingErrors: true,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
// ==================== Tool 定义：章节 ====================
|
// ==================== Tool 定义：章节 ====================
|
||||||
|
|
||||||
getChapter = tool(
|
getChapter = tool({
|
||||||
async ({ chapterNumbers }) => {
|
title: "getChapter",
|
||||||
|
description: "根据章节编号获取小说章节的完整原文内容，支持批量获取",
|
||||||
|
inputSchema: z.object({
|
||||||
|
chapterNumbers: z.array(z.number()).min(1).describe("章节编号数组"),
|
||||||
|
}),
|
||||||
|
execute: async ({ chapterNumbers }) => {
|
||||||
this.log("获取章节", `章节号: ${chapterNumbers.join(", ")}`);
|
this.log("获取章节", `章节号: ${chapterNumbers.join(", ")}`);
|
||||||
|
|
||||||
const results = await Promise.all(
|
const results = await Promise.all(
|
||||||
@ -539,36 +517,24 @@ ${formatList(ep.classicQuotes, (q) => q)}
|
|||||||
|
|
||||||
return results.join("\n\n---\n");
|
return results.join("\n\n---\n");
|
||||||
},
|
},
|
||||||
{
|
});
|
||||||
name: "getChapter",
|
|
||||||
description: "根据章节编号获取小说章节的完整原文内容，支持批量获取",
|
|
||||||
schema: z.object({
|
|
||||||
chapterNumbers: z.array(z.number()).min(1).describe("章节编号数组"),
|
|
||||||
}),
|
|
||||||
verboseParsingErrors: true,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
// ==================== Tool 定义：资产 ====================
|
// ==================== Tool 定义：资产 ====================
|
||||||
|
|
||||||
generateAssets = tool(
|
generateAssets = tool({
|
||||||
async () => {
|
title: "generateAssets",
|
||||||
|
description: "从当前项目的所有大纲中提取并生成角色、道具、场景资产，自动去重并清理冗余",
|
||||||
|
inputSchema: z.object({}),
|
||||||
|
execute: async () => {
|
||||||
this.log("生成资产");
|
this.log("生成资产");
|
||||||
const stats = await this.generateAssetsFromOutlines();
|
const stats = await this.generateAssetsFromOutlines();
|
||||||
|
|
||||||
if (stats.inserted === 0 && stats.updated === 0 && stats.skipped === 0) {
|
if (stats.inserted === 0 && stats.updated === 0 && stats.skipped === 0) {
|
||||||
return "当前项目没有大纲数据，无法生成资产";
|
return "当前项目没有大纲数据，无法生成资产";
|
||||||
}
|
}
|
||||||
|
|
||||||
return `资产生成完成：新增 ${stats.inserted}，更新 ${stats.updated}，保持 ${stats.skipped}`;
|
return `资产生成完成：新增 ${stats.inserted}，更新 ${stats.updated}，保持 ${stats.skipped}`;
|
||||||
},
|
},
|
||||||
{
|
});
|
||||||
name: "generateAssets",
|
|
||||||
description: "从当前项目的所有大纲中提取并生成角色、道具、场景资产，自动去重并清理冗余",
|
|
||||||
schema: z.object({}),
|
|
||||||
verboseParsingErrors: true,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
// ==================== 上下文构建 ====================
|
// ==================== 上下文构建 ====================
|
||||||
|
|
||||||
@ -606,7 +572,7 @@ ${this.getChapterContext()}
|
|||||||
|
|
||||||
private buildConversationHistory(): string {
|
private buildConversationHistory(): string {
|
||||||
if (!this.history.length) return "无对话历史";
|
if (!this.history.length) return "无对话历史";
|
||||||
return this.history.map(([role, content]) => `${role}: ${content}`).join("\n\n");
|
return this.history.map(({ role, content }) => `${role}: ${content}`).join("\n\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
private async buildFullContext(task: string): Promise<string> {
|
private async buildFullContext(task: string): Promise<string> {
|
||||||
@ -627,14 +593,14 @@ ${task}
|
|||||||
// ==================== Sub-Agent ====================
|
// ==================== Sub-Agent ====================
|
||||||
|
|
||||||
private getSubAgentTools() {
|
private getSubAgentTools() {
|
||||||
return [this.getChapter, this.getStoryline, this.saveStoryline, this.getOutline, this.saveOutline, this.updateOutline];
|
return {
|
||||||
}
|
getChapter: this.getChapter,
|
||||||
|
getStoryline: this.getStoryline,
|
||||||
private createModel() {
|
saveStoryline: this.saveStoryline,
|
||||||
return openAI({
|
getOutline: this.getOutline,
|
||||||
modelName: this.modelName,
|
saveOutline: this.saveOutline,
|
||||||
configuration: { apiKey: this.apiKey, baseURL: this.baseURL },
|
updateOutline: this.updateOutline,
|
||||||
});
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@ -657,74 +623,69 @@ ${task}
|
|||||||
|
|
||||||
const context = await this.buildFullContext(task);
|
const context = await this.buildFullContext(task);
|
||||||
|
|
||||||
const agent = createAgent({
|
const { fullStream } = await u.ai.text.stream({
|
||||||
model: this.createModel(),
|
system: SYSTEM_PROMPTS[agentType],
|
||||||
systemPrompt: SYSTEM_PROMPTS[agentType],
|
|
||||||
tools: this.getSubAgentTools(),
|
tools: this.getSubAgentTools(),
|
||||||
|
messages: [{ role: "user", content: context }],
|
||||||
|
maxStep: 100,
|
||||||
});
|
});
|
||||||
|
|
||||||
const stream = await agent.stream({ messages: [["user", context]] }, { streamMode: ["messages"], callbacks: [] });
|
|
||||||
|
|
||||||
let fullResponse = "";
|
let fullResponse = "";
|
||||||
|
for await (const item of fullStream) {
|
||||||
for await (const [mode, chunk] of stream) {
|
if (item.type == "tool-call") {
|
||||||
if (mode !== "messages") continue;
|
this.emit("toolCall", { agent: "main", name: item.title, args: null });
|
||||||
|
|
||||||
const [token] = chunk as any;
|
|
||||||
const block = token.contentBlocks?.[0];
|
|
||||||
|
|
||||||
// 处理 AI 文本流
|
|
||||||
if (token.type === "ai" && block?.text) {
|
|
||||||
fullResponse += block.text;
|
|
||||||
this.emit("subAgentStream", { agent: agentType, text: block.text });
|
|
||||||
}
|
}
|
||||||
|
if (item.type == "text-delta") {
|
||||||
// 处理 tool 调用
|
fullResponse += item.text;
|
||||||
if (token.type === "ai" && token.tool_calls?.length) {
|
this.emit("subAgentStream", { agent: agentType, text: item.text });
|
||||||
for (const toolCall of token.tool_calls) {
|
|
||||||
this.emit("toolCall", { agent: agentType, name: toolCall.name, args: toolCall.args });
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
this.emit("subAgentEnd", { agent: agentType });
|
this.emit("subAgentEnd", { agent: agentType });
|
||||||
this.history.push(["ai", fullResponse]);
|
this.history.push({
|
||||||
|
role: "assistant",
|
||||||
|
content: fullResponse,
|
||||||
|
});
|
||||||
this.log(`Sub-Agent 完成`, agentType);
|
this.log(`Sub-Agent 完成`, agentType);
|
||||||
|
|
||||||
return fullResponse ?? `${agentType}已完成任务`;
|
return fullResponse ?? `${agentType}已完成任务`;
|
||||||
}
|
}
|
||||||
|
|
||||||
private createSubAgentTool(agentType: AgentType, description: string) {
|
private createSubAgentTool(agentType: AgentType, description: string) {
|
||||||
return tool(async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), {
|
return tool({
|
||||||
name: agentType,
|
title: agentType,
|
||||||
description,
|
description,
|
||||||
schema: z.object({
|
inputSchema: z.object({
|
||||||
taskDescription: z.string().describe("具体的任务描述，包含章节范围、修改要求等详细信息"),
|
taskDescription: z.string().describe("具体的任务描述，包含章节范围、修改要求等详细信息"),
|
||||||
}),
|
}),
|
||||||
|
execute: async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// ==================== 主入口 ====================
|
// ==================== 主入口 ====================
|
||||||
|
|
||||||
private getAllTools() {
|
private getAllTools() {
|
||||||
return [
|
return {
|
||||||
this.createSubAgentTool("AI1", "调用故事师。负责分析小说原文并生成故事线，会自行调用 saveStoryline 保存结果。"),
|
AI1: this.createSubAgentTool("AI1", "调用故事师。负责分析小说原文并生成故事线，会自行调用 saveStoryline 保存结果。"),
|
||||||
this.createSubAgentTool("AI2", "调用大纲师。负责根据故事线生成剧集大纲，会自行调用 saveOutline 保存结果。"),
|
AI2: this.createSubAgentTool("AI2", "调用大纲师。负责根据故事线生成剧集大纲，会自行调用 saveOutline 保存结果。"),
|
||||||
this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲，会自行调用 updateOutline 或 saveStoryline 进行修改。"),
|
director: this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲，会自行调用 updateOutline 或 saveStoryline 进行修改。"),
|
||||||
this.getChapter,
|
getChapter: this.getChapter,
|
||||||
this.getStoryline,
|
getStoryline: this.getStoryline,
|
||||||
this.saveStoryline,
|
saveStoryline: this.saveStoryline,
|
||||||
this.deleteStoryline,
|
deleteStoryline: this.deleteStoryline,
|
||||||
this.getOutline,
|
getOutline: this.getOutline,
|
||||||
this.saveOutline,
|
saveOutline: this.saveOutline,
|
||||||
this.updateOutline,
|
updateOutline: this.updateOutline,
|
||||||
this.deleteOutline,
|
deleteOutline: this.deleteOutline,
|
||||||
this.generateAssets,
|
generateAssets: this.generateAssets,
|
||||||
];
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async call(msg: string): Promise<string> {
|
async call(msg: string): Promise<string> {
|
||||||
this.history.push(["user", msg]);
|
this.history.push({
|
||||||
|
role: "user",
|
||||||
|
content: msg,
|
||||||
|
});
|
||||||
|
|
||||||
const envContext = await this.buildEnvironmentContext();
|
const envContext = await this.buildEnvironmentContext();
|
||||||
|
|
||||||
@ -732,36 +693,28 @@ ${task}
|
|||||||
|
|
||||||
const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常";
|
const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常";
|
||||||
|
|
||||||
const mainAgent = createAgent({
|
const { fullStream } = await u.ai.text.stream({
|
||||||
model: this.createModel(),
|
system: `${envContext}\n${mainPrompts}`,
|
||||||
tools: this.getAllTools(),
|
tools: this.getAllTools(),
|
||||||
systemPrompt: `${envContext}\n${mainPrompts}`,
|
messages: this.history,
|
||||||
|
maxStep: 100,
|
||||||
});
|
});
|
||||||
const stream = await mainAgent.stream({ messages: this.history }, { streamMode: ["messages"], callbacks: [] });
|
|
||||||
|
|
||||||
let fullResponse = "";
|
let fullResponse = "";
|
||||||
|
for await (const item of fullStream) {
|
||||||
for await (const [mode, chunk] of stream) {
|
if (item.type == "tool-call") {
|
||||||
if (mode !== "messages") continue;
|
this.emit("toolCall", { agent: "main", name: item.title, args: null });
|
||||||
|
|
||||||
const [token] = chunk as any;
|
|
||||||
const block = token.contentBlocks?.[0];
|
|
||||||
|
|
||||||
// 处理 AI 文本流
|
|
||||||
if (token.type === "ai" && block?.text) {
|
|
||||||
fullResponse += block.text;
|
|
||||||
this.emit("data", block.text);
|
|
||||||
}
|
}
|
||||||
|
if (item.type == "text-delta") {
|
||||||
// 处理 tool 调用
|
fullResponse += item.text;
|
||||||
if (token.type === "ai" && token.tool_calls?.length) {
|
this.emit("data", item.text);
|
||||||
for (const toolCall of token.tool_calls) {
|
|
||||||
this.emit("toolCall", { agent: "main", name: toolCall.name, args: toolCall.args });
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
this.history.push({
|
||||||
|
role: "assistant",
|
||||||
|
content: fullResponse,
|
||||||
|
});
|
||||||
|
|
||||||
this.history.push(["assistant", fullResponse]);
|
|
||||||
this.emit("response", fullResponse);
|
this.emit("response", fullResponse);
|
||||||
|
|
||||||
return fullResponse;
|
return fullResponse;
|
||||||
|
|||||||
@ -1,10 +1,8 @@
|
|||||||
// @/agents/Storyboard.ts
|
// @/agents/Storyboard.ts
|
||||||
import u from "@/utils";
|
import u from "@/utils";
|
||||||
import { createAgent } from "langchain";
|
import { tool, ModelMessage, Tool } from "ai";
|
||||||
import { EventEmitter } from "events";
|
import { EventEmitter } from "events";
|
||||||
import { openAI } from "@/agents/models";
|
|
||||||
import { z } from "zod";
|
import { z } from "zod";
|
||||||
import { tool } from "@langchain/core/tools";
|
|
||||||
import type { DB } from "@/types/database";
|
import type { DB } from "@/types/database";
|
||||||
import generateImageTool from "./generateImageTool";
|
import generateImageTool from "./generateImageTool";
|
||||||
import imageSplitting from "./imageSplitting";
|
import imageSplitting from "./imageSplitting";
|
||||||
@ -46,7 +44,7 @@ export default class Storyboard {
|
|||||||
private readonly projectId: number;
|
private readonly projectId: number;
|
||||||
private readonly scriptId: number;
|
private readonly scriptId: number;
|
||||||
readonly emitter = new EventEmitter();
|
readonly emitter = new EventEmitter();
|
||||||
history: Array<[string, string]> = [];
|
history: ModelMessage[] = [];
|
||||||
novelChapters: DB["t_novel"][] = [];
|
novelChapters: DB["t_novel"][] = [];
|
||||||
|
|
||||||
// 存储 segmentAgent 生成的片段结果
|
// 存储 segmentAgent 生成的片段结果
|
||||||
@ -58,10 +56,6 @@ export default class Storyboard {
|
|||||||
// 存储正在生成分镜图的分镜ID
|
// 存储正在生成分镜图的分镜ID
|
||||||
private generatingShots: Set<number> = new Set();
|
private generatingShots: Set<number> = new Set();
|
||||||
|
|
||||||
modelName = "gpt-4.1";
|
|
||||||
apiKey = "";
|
|
||||||
baseURL = "";
|
|
||||||
|
|
||||||
constructor(projectId: number, scriptId: number) {
|
constructor(projectId: number, scriptId: number) {
|
||||||
this.projectId = projectId;
|
this.projectId = projectId;
|
||||||
this.scriptId = scriptId;
|
this.scriptId = scriptId;
|
||||||
@ -105,28 +99,28 @@ export default class Storyboard {
|
|||||||
|
|
||||||
// ==================== 剧本相关操作 ====================
|
// ==================== 剧本相关操作 ====================
|
||||||
|
|
||||||
getScript = tool(
|
getScript = tool({
|
||||||
async () => {
|
title: "getScript",
|
||||||
|
description: "获取剧本内容",
|
||||||
|
inputSchema: z.object({}),
|
||||||
|
execute: async () => {
|
||||||
this.log("获取剧本", `scriptId: ${this.scriptId}`);
|
this.log("获取剧本", `scriptId: ${this.scriptId}`);
|
||||||
const script = await u.db("t_script").where({ id: this.scriptId, projectId: this.projectId }).first();
|
const script = await u.db("t_script").where({ id: this.scriptId, projectId: this.projectId }).first();
|
||||||
if (!script) throw new Error("剧本不存在");
|
if (!script) throw new Error("剧本不存在");
|
||||||
return `剧本集：${script.name}\n\n内容：\n\`\`\`${script.content}\`\`\``;
|
return `剧本集：${script.name}\n\n内容：\n\`\`\`${script.content}\`\`\``;
|
||||||
},
|
},
|
||||||
{
|
});
|
||||||
name: "getScript",
|
|
||||||
description: "获取剧本内容",
|
|
||||||
schema: z.object({}),
|
|
||||||
verboseParsingErrors: true,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
// ==================== 资产相关操作 ====================
|
// ==================== 资产相关操作 ====================
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 获取资产列表（供 segmentAgent 和 shotAgent 调用）
|
* 获取资产列表（供 segmentAgent 和 shotAgent 调用）
|
||||||
*/
|
*/
|
||||||
getAssets = tool(
|
getAssets = tool({
|
||||||
async () => {
|
title: "getAssets",
|
||||||
|
description: "获取资产列表（角色、道具、场景），包含名称和详细介绍。生成片段和分镜时必须先调用此工具获取资产信息，确保名称一致性",
|
||||||
|
inputSchema: z.object({}),
|
||||||
|
execute: async () => {
|
||||||
this.log("获取资产列表", `scriptId: ${this.scriptId}`);
|
this.log("获取资产列表", `scriptId: ${this.scriptId}`);
|
||||||
const scriptData = await u.db("t_script").where({ id: this.scriptId, projectId: this.projectId }).first();
|
const scriptData = await u.db("t_script").where({ id: this.scriptId, projectId: this.projectId }).first();
|
||||||
const row = await u.db("t_outline").where({ id: scriptData?.outlineId!, projectId: this.projectId }).first();
|
const row = await u.db("t_outline").where({ id: scriptData?.outlineId!, projectId: this.projectId }).first();
|
||||||
@ -171,69 +165,69 @@ ${sections.join("\n\n")}
|
|||||||
2. 禁止在资产名称前后添加修饰词
|
2. 禁止在资产名称前后添加修饰词
|
||||||
3. 禁止捏造资产列表中不存在的角色、场景、道具`;
|
3. 禁止捏造资产列表中不存在的角色、场景、道具`;
|
||||||
},
|
},
|
||||||
{
|
});
|
||||||
name: "getAssets",
|
|
||||||
description: "获取资产列表（角色、道具、场景），包含名称和详细介绍。生成片段和分镜时必须先调用此工具获取资产信息，确保名称一致性",
|
|
||||||
schema: z.object({}),
|
|
||||||
verboseParsingErrors: true,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
// ==================== 片段和分镜工具 ====================
|
// ==================== 片段和分镜工具 ====================
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 获取当前存储的片段数据（供 shotAgent 调用）
|
* 获取当前存储的片段数据（供 shotAgent 调用）
|
||||||
*/
|
*/
|
||||||
getSegments = tool(
|
getSegments = tool({
|
||||||
async () => {
|
title: "getSegments",
|
||||||
|
description: "获取当前已生成的片段数据，用于生成分镜",
|
||||||
|
inputSchema: z.object({}),
|
||||||
|
execute: async () => {
|
||||||
this.log("获取片段数据", `共 ${this.segments.length} 个片段`);
|
this.log("获取片段数据", `共 ${this.segments.length} 个片段`);
|
||||||
if (this.segments.length === 0) {
|
if (this.segments.length === 0) {
|
||||||
return "暂无片段数据，请先调用 segmentAgent 生成片段";
|
return "暂无片段数据，请先调用 segmentAgent 生成片段";
|
||||||
}
|
}
|
||||||
return JSON.stringify(this.segments, null, 2);
|
return JSON.stringify(this.segments, null, 2);
|
||||||
},
|
},
|
||||||
{
|
});
|
||||||
name: "getSegments",
|
|
||||||
description: "获取当前已生成的片段数据，用于生成分镜",
|
|
||||||
schema: z.object({}),
|
|
||||||
verboseParsingErrors: true,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 更新/存储片段数据（供 segmentAgent 调用）
|
* 更新/存储片段数据（供 segmentAgent 调用）
|
||||||
*/
|
*/
|
||||||
updateSegments = tool(
|
updateSegments = tool({
|
||||||
async ({ segments }: { segments: Segment[] }) => {
|
title: "updateSegments",
|
||||||
|
description: "存储生成的片段数据，segmentAgent 在生成片段后必须调用此工具保存结果",
|
||||||
|
inputSchema: z.object({
|
||||||
|
segments: z
|
||||||
|
.array(
|
||||||
|
z.object({
|
||||||
|
index: z.number().describe("片段序号"),
|
||||||
|
description: z.string().describe("片段描述"),
|
||||||
|
emotion: z.string().optional().describe("情绪氛围"),
|
||||||
|
action: z.string().optional().describe("主要动作"),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.describe("片段数组"),
|
||||||
|
}),
|
||||||
|
execute: async ({ segments }: { segments: Segment[] }) => {
|
||||||
this.log("更新片段数据", `共 ${segments.length} 个片段`);
|
this.log("更新片段数据", `共 ${segments.length} 个片段`);
|
||||||
this.segments = segments;
|
this.segments = segments;
|
||||||
this.emit("segmentsUpdated", this.segments);
|
this.emit("segmentsUpdated", this.segments);
|
||||||
return `成功存储 ${segments.length} 个片段`;
|
return `成功存储 ${segments.length} 个片段`;
|
||||||
},
|
},
|
||||||
{
|
});
|
||||||
name: "updateSegments",
|
|
||||||
description: "存储生成的片段数据，segmentAgent 在生成片段后必须调用此工具保存结果",
|
|
||||||
schema: z.object({
|
|
||||||
segments: z
|
|
||||||
.array(
|
|
||||||
z.object({
|
|
||||||
index: z.number().describe("片段序号"),
|
|
||||||
description: z.string().describe("片段描述"),
|
|
||||||
emotion: z.string().optional().describe("情绪氛围"),
|
|
||||||
action: z.string().optional().describe("主要动作"),
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.describe("片段数组"),
|
|
||||||
}),
|
|
||||||
verboseParsingErrors: true,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 添加分镜（供 shotAgent 调用）
|
* 添加分镜（供 shotAgent 调用）
|
||||||
*/
|
*/
|
||||||
addShots = tool(
|
addShots = tool({
|
||||||
async ({ shots }: { shots: Array<{ segmentIndex: number; prompts: string[] }> }) => {
|
title: "addShots",
|
||||||
|
description: "添加新的分镜。每个分镜有独立ID，包含多个镜头（每个镜头对应一个提示词）。如果片段已存在分镜会跳过",
|
||||||
|
inputSchema: z.object({
|
||||||
|
shots: z
|
||||||
|
.array(
|
||||||
|
z.object({
|
||||||
|
segmentIndex: z.number().describe("对应的片段序号"),
|
||||||
|
prompts: z.array(z.string()).describe("镜头提示词数组，每个提示词对应一个镜头（中文）"),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.describe("要添加的分镜数组"),
|
||||||
|
}),
|
||||||
|
execute: async ({ shots }: { shots: Array<{ segmentIndex: number; prompts: string[] }> }) => {
|
||||||
const added: { id: number; segmentIndex: number }[] = [];
|
const added: { id: number; segmentIndex: number }[] = [];
|
||||||
const skipped: number[] = [];
|
const skipped: number[] = [];
|
||||||
|
|
||||||
@ -266,29 +260,20 @@ ${sections.join("\n\n")}
|
|||||||
}
|
}
|
||||||
return `已添加${addedInfo}。当前共 ${this.shots.length} 个分镜`;
|
return `已添加${addedInfo}。当前共 ${this.shots.length} 个分镜`;
|
||||||
},
|
},
|
||||||
{
|
});
|
||||||
name: "addShots",
|
|
||||||
description: "添加新的分镜。每个分镜有独立ID，包含多个镜头（每个镜头对应一个提示词）。如果片段已存在分镜会跳过",
|
|
||||||
schema: z.object({
|
|
||||||
shots: z
|
|
||||||
.array(
|
|
||||||
z.object({
|
|
||||||
segmentIndex: z.number().describe("对应的片段序号"),
|
|
||||||
prompts: z.array(z.string()).describe("镜头提示词数组，每个提示词对应一个镜头（中文）"),
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.describe("要添加的分镜数组"),
|
|
||||||
}),
|
|
||||||
verboseParsingErrors: true,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 更新指定分镜（供 shotAgent 调用）
|
* 更新指定分镜（供 shotAgent 调用）
|
||||||
* 保留原有 cells 的 id 和 src 字段，只更新 prompt
|
* 保留原有 cells 的 id 和 src 字段，只更新 prompt
|
||||||
*/
|
*/
|
||||||
updateShots = tool(
|
updateShots = tool({
|
||||||
async ({ shotId, prompts }: { shotId: number; prompts: string[] }) => {
|
title: "updateShots",
|
||||||
|
description: "更新指定分镜的镜头提示词。通过分镜ID指定要修改的分镜",
|
||||||
|
inputSchema: z.object({
|
||||||
|
shotId: z.number().describe("要更新的分镜ID"),
|
||||||
|
prompts: z.array(z.string()).describe("新的镜头提示词数组，每个提示词对应一个镜头"),
|
||||||
|
}),
|
||||||
|
execute: async ({ shotId, prompts }: { shotId: number; prompts: string[] }) => {
|
||||||
const existingIndex = this.shots.findIndex((item) => item.id === shotId);
|
const existingIndex = this.shots.findIndex((item) => item.id === shotId);
|
||||||
|
|
||||||
if (existingIndex === -1) {
|
if (existingIndex === -1) {
|
||||||
@ -314,22 +299,18 @@ ${sections.join("\n\n")}
|
|||||||
|
|
||||||
return `已更新分镜 ${shotId}`;
|
return `已更新分镜 ${shotId}`;
|
||||||
},
|
},
|
||||||
{
|
});
|
||||||
name: "updateShots",
|
|
||||||
description: "更新指定分镜的镜头提示词。通过分镜ID指定要修改的分镜",
|
|
||||||
schema: z.object({
|
|
||||||
shotId: z.number().describe("要更新的分镜ID"),
|
|
||||||
prompts: z.array(z.string()).describe("新的镜头提示词数组，每个提示词对应一个镜头"),
|
|
||||||
}),
|
|
||||||
verboseParsingErrors: true,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 删除指定分镜（供 shotAgent 调用）
|
* 删除指定分镜（供 shotAgent 调用）
|
||||||
*/
|
*/
|
||||||
deleteShots = tool(
|
deleteShots = tool({
|
||||||
async ({ shotIds }: { shotIds: number[] }) => {
|
title: "deleteShots",
|
||||||
|
description: "删除指定的分镜。通过分镜ID指定要删除的分镜",
|
||||||
|
inputSchema: z.object({
|
||||||
|
shotIds: z.array(z.number()).describe("要删除的分镜ID数组"),
|
||||||
|
}),
|
||||||
|
execute: async ({ shotIds }: { shotIds: number[] }) => {
|
||||||
const deleted: number[] = [];
|
const deleted: number[] = [];
|
||||||
const notFound: number[] = [];
|
const notFound: number[] = [];
|
||||||
|
|
||||||
@ -351,21 +332,19 @@ ${sections.join("\n\n")}
|
|||||||
}
|
}
|
||||||
return `已删除分镜 ${deleted.join(", ")}。当前共 ${this.shots.length} 个分镜`;
|
return `已删除分镜 ${deleted.join(", ")}。当前共 ${this.shots.length} 个分镜`;
|
||||||
},
|
},
|
||||||
{
|
});
|
||||||
name: "deleteShots",
|
|
||||||
description: "删除指定的分镜。通过分镜ID指定要删除的分镜",
|
|
||||||
schema: z.object({
|
|
||||||
shotIds: z.array(z.number()).describe("要删除的分镜ID数组"),
|
|
||||||
}),
|
|
||||||
verboseParsingErrors: true,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 生成分镜图（异步执行，使用 nanoBanana）
|
* 生成分镜图（异步执行，使用 nanoBanana）
|
||||||
*/
|
*/
|
||||||
generateShotImage = tool(
|
generateShotImage = tool({
|
||||||
async ({ shotIds }: { shotIds: number[] }) => {
|
title: "generateShotImage",
|
||||||
|
description:
|
||||||
|
"为指定分镜生成分镜图。每个分镜会根据其所有提示词生成一张完整宫格图，然后自动分割为单格图片。通过分镜ID指定，不需要指定具体格子，整个分镜是一个完整的生成单元",
|
||||||
|
inputSchema: z.object({
|
||||||
|
shotIds: z.array(z.number()).describe("要生成分镜图的分镜ID数组"),
|
||||||
|
}),
|
||||||
|
execute: async ({ shotIds }: { shotIds: number[] }) => {
|
||||||
const toGenerate: number[] = [];
|
const toGenerate: number[] = [];
|
||||||
const alreadyGenerating: number[] = [];
|
const alreadyGenerating: number[] = [];
|
||||||
const notFound: number[] = [];
|
const notFound: number[] = [];
|
||||||
@ -417,16 +396,7 @@ ${sections.join("\n\n")}
|
|||||||
}
|
}
|
||||||
return result;
|
return result;
|
||||||
},
|
},
|
||||||
{
|
});
|
||||||
name: "generateShotImage",
|
|
||||||
description:
|
|
||||||
"为指定分镜生成分镜图。每个分镜会根据其所有提示词生成一张完整宫格图，然后自动分割为单格图片。通过分镜ID指定，不需要指定具体格子，整个分镜是一个完整的生成单元",
|
|
||||||
schema: z.object({
|
|
||||||
shotIds: z.array(z.number()).describe("要生成分镜图的分镜ID数组"),
|
|
||||||
}),
|
|
||||||
verboseParsingErrors: true,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 执行分镜图生成的具体逻辑（异步并发）
|
* 执行分镜图生成的具体逻辑（异步并发）
|
||||||
@ -566,7 +536,7 @@ ${assetList}
|
|||||||
|
|
||||||
private buildConversationHistory(): string {
|
private buildConversationHistory(): string {
|
||||||
if (!this.history.length) return "无对话历史";
|
if (!this.history.length) return "无对话历史";
|
||||||
return this.history.map(([role, content]) => `${role}: ${content}`).join("\n\n");
|
return this.history.map(({ role, content }) => `${role}: ${content}`).join("\n\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
private async buildFullContext(task: string): Promise<string> {
|
private async buildFullContext(task: string): Promise<string> {
|
||||||
@ -586,26 +556,33 @@ ${task}
|
|||||||
|
|
||||||
// ==================== Sub-Agent ====================
|
// ==================== Sub-Agent ====================
|
||||||
|
|
||||||
private createModel() {
|
|
||||||
return openAI({
|
|
||||||
modelName: this.modelName,
|
|
||||||
configuration: { apiKey: this.apiKey, baseURL: this.baseURL },
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 获取不同 Sub-Agent 可用的工具
|
* 获取不同 Sub-Agent 可用的工具
|
||||||
*/
|
*/
|
||||||
private getSubAgentTools(agentType: AgentType) {
|
private getSubAgentTools(agentType: AgentType): Record<string, Tool> {
|
||||||
switch (agentType) {
|
switch (agentType) {
|
||||||
case "segmentAgent":
|
case "segmentAgent":
|
||||||
// segmentAgent 可以获取剧本和资产，并需要调用 updateSegments 保存结果
|
// segmentAgent 可以获取剧本和资产，并需要调用 updateSegments 保存结果
|
||||||
return [this.getScript, this.getAssets, this.updateSegments];
|
return {
|
||||||
|
getScript: this.getScript,
|
||||||
|
getAssets: this.getAssets,
|
||||||
|
updateSegments: this.updateSegments,
|
||||||
|
};
|
||||||
case "shotAgent":
|
case "shotAgent":
|
||||||
// shotAgent 可以获取剧本、资产和片段，并可使用 add/update/delete 操作分镜，以及生成分镜图
|
// shotAgent 可以获取剧本、资产和片段，并可使用 add/update/delete 操作分镜，以及生成分镜图
|
||||||
return [this.getScript, this.getAssets, this.getSegments, this.addShots, this.updateShots, this.deleteShots, this.generateShotImage];
|
return {
|
||||||
|
getScript: this.getScript,
|
||||||
|
getAssets: this.getAssets,
|
||||||
|
getSegments: this.getSegments,
|
||||||
|
addShots: this.addShots,
|
||||||
|
updateShots: this.updateShots,
|
||||||
|
deleteShots: this.deleteShots,
|
||||||
|
generateShotImage: this.generateShotImage,
|
||||||
|
};
|
||||||
default:
|
default:
|
||||||
return [this.getScript];
|
return {
|
||||||
|
getScript: this.getScript,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -627,74 +604,71 @@ ${task}
|
|||||||
|
|
||||||
const context = await this.buildFullContext(task);
|
const context = await this.buildFullContext(task);
|
||||||
|
|
||||||
const agent = createAgent({
|
const { fullStream } = await u.ai.text.stream({
|
||||||
model: this.createModel(),
|
system: SYSTEM_PROMPTS[agentType],
|
||||||
systemPrompt: SYSTEM_PROMPTS[agentType],
|
|
||||||
tools: this.getSubAgentTools(agentType),
|
tools: this.getSubAgentTools(agentType),
|
||||||
|
messages: [{ role: "user", content: context }],
|
||||||
|
maxStep: 100,
|
||||||
});
|
});
|
||||||
|
|
||||||
const stream = await agent.stream({ messages: [["user", context]] }, { streamMode: ["messages"], callbacks: [] });
|
|
||||||
|
|
||||||
let fullResponse = "";
|
let fullResponse = "";
|
||||||
|
for await (const item of fullStream) {
|
||||||
for await (const [mode, chunk] of stream) {
|
if (item.type == "tool-call") {
|
||||||
if (mode !== "messages") continue;
|
this.emit("toolCall", { agent: "main", name: item.title, args: null });
|
||||||
const [token] = chunk as any;
|
|
||||||
const block = token.contentBlocks?.[0];
|
|
||||||
|
|
||||||
// 处理 AI 文本流
|
|
||||||
if (token.type === "ai" && block?.text) {
|
|
||||||
fullResponse += block.text;
|
|
||||||
this.emit("subAgentStream", { agent: agentType, text: block.text });
|
|
||||||
}
|
}
|
||||||
// 处理 tool 调用
|
if (item.type == "text-delta") {
|
||||||
if (token.type === "ai" && token.tool_calls?.length) {
|
fullResponse += item.text;
|
||||||
for (const toolCall of token.tool_calls) {
|
this.emit("subAgentStream", { agent: agentType, text: item.text });
|
||||||
this.emit("toolCall", { agent: agentType, name: toolCall.name, args: toolCall.args });
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
this.emit("subAgentEnd", { agent: agentType });
|
this.emit("subAgentEnd", { agent: agentType });
|
||||||
this.history.push(["ai", fullResponse]);
|
this.history.push({
|
||||||
|
role: "assistant",
|
||||||
|
content: fullResponse,
|
||||||
|
});
|
||||||
this.log(`Sub-Agent 完成`, agentType);
|
this.log(`Sub-Agent 完成`, agentType);
|
||||||
return fullResponse;
|
|
||||||
|
return fullResponse ?? `${agentType}已完成任务`;
|
||||||
}
|
}
|
||||||
|
|
||||||
private createSubAgentTool(agentType: AgentType, description: string) {
|
private createSubAgentTool(agentType: AgentType, description: string) {
|
||||||
return tool(async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription), {
|
return tool({
|
||||||
name: agentType,
|
title: agentType,
|
||||||
description,
|
description,
|
||||||
schema: z.object({
|
inputSchema: z.object({
|
||||||
taskDescription: z.string().describe("具体的任务描述，包含章节范围、修改要求等详细信息"),
|
taskDescription: z.string().describe("具体的任务描述，包含章节范围、修改要求等详细信息"),
|
||||||
}),
|
}),
|
||||||
|
execute: async ({ taskDescription }) => this.invokeSubAgent(agentType, taskDescription),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// ==================== 主入口 ====================
|
// ==================== 主入口 ====================
|
||||||
|
|
||||||
private getAllTools() {
|
private getAllTools() {
|
||||||
return [
|
return {
|
||||||
this.createSubAgentTool(
|
segmentAgent: this.createSubAgentTool(
|
||||||
"segmentAgent",
|
"segmentAgent",
|
||||||
"调用片段师。负责根据剧本生成片段，会自行调用 getScript 获取剧本内容，并调用 updateSegments 保存片段结果。",
|
"调用片段师。负责根据剧本生成片段，会自行调用 getScript 获取剧本内容，并调用 updateSegments 保存片段结果。",
|
||||||
),
|
),
|
||||||
this.createSubAgentTool(
|
shotAgent: this.createSubAgentTool(
|
||||||
"shotAgent",
|
"shotAgent",
|
||||||
"调用分镜师。负责根据片段生成分镜提示词，会自行调用 getSegments 获取片段数据，并调用 addShots/updateShots 保存分镜结果。",
|
"调用分镜师。负责根据片段生成分镜提示词，会自行调用 getSegments 获取片段数据，并调用 addShots/updateShots 保存分镜结果。",
|
||||||
),
|
),
|
||||||
// this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲，会自行调用 updateOutline 或 saveStoryline 进行修改。"),
|
// this.createSubAgentTool("director", "调用导演。负责审核故事线和大纲，会自行调用 updateOutline 或 saveStoryline 进行修改。"),
|
||||||
this.getScript,
|
getScript: this.getScript,
|
||||||
this.getSegments,
|
getSegments: this.getSegments,
|
||||||
this.generateShotImage,
|
generateShotImage: this.generateShotImage,
|
||||||
...this.getSubAgentTools("segmentAgent"),
|
...this.getSubAgentTools("segmentAgent"),
|
||||||
...this.getSubAgentTools("shotAgent"),
|
...this.getSubAgentTools("shotAgent"),
|
||||||
];
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async call(msg: string): Promise<string> {
|
async call(msg: string): Promise<string> {
|
||||||
console.log("模型名称:", this.modelName);
|
this.history.push({
|
||||||
this.history.push(["user", msg]);
|
role: "user",
|
||||||
|
content: msg,
|
||||||
|
});
|
||||||
|
|
||||||
const envContext = await this.buildEnvironmentContext();
|
const envContext = await this.buildEnvironmentContext();
|
||||||
|
|
||||||
@ -702,34 +676,28 @@ ${task}
|
|||||||
|
|
||||||
const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常";
|
const mainPrompts = prompts?.customValue || prompts?.defaultValue || "不论用户说什么，请直接输出Agent配置异常";
|
||||||
|
|
||||||
const mainAgent = createAgent({
|
const { fullStream } = await u.ai.text.stream({
|
||||||
model: this.createModel(),
|
system: `${envContext}\n${mainPrompts}`,
|
||||||
tools: this.getAllTools(),
|
tools: this.getAllTools(),
|
||||||
systemPrompt: `${envContext}\n${mainPrompts}`,
|
messages: this.history,
|
||||||
|
maxStep: 100,
|
||||||
});
|
});
|
||||||
const stream = await mainAgent.stream({ messages: this.history }, { streamMode: ["messages"], callbacks: [] });
|
|
||||||
|
|
||||||
let fullResponse = "";
|
let fullResponse = "";
|
||||||
|
for await (const item of fullStream) {
|
||||||
for await (const [mode, chunk] of stream) {
|
if (item.type == "tool-call") {
|
||||||
if (mode !== "messages") continue;
|
this.emit("toolCall", { agent: "main", name: item.title, args: null });
|
||||||
const [token] = chunk as any;
|
|
||||||
const block = token.contentBlocks?.[0];
|
|
||||||
// 处理 AI 文本流
|
|
||||||
if (token.type === "ai" && block?.text) {
|
|
||||||
fullResponse += block.text;
|
|
||||||
this.emit("data", block.text);
|
|
||||||
}
|
}
|
||||||
|
if (item.type == "text-delta") {
|
||||||
// 处理 tool 调用
|
fullResponse += item.text;
|
||||||
if (token.type === "ai" && token.tool_calls?.length) {
|
this.emit("data", item.text);
|
||||||
for (const toolCall of token.tool_calls) {
|
|
||||||
this.emit("toolCall", { agent: "main", name: toolCall.name, args: toolCall.args });
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
this.history.push({
|
||||||
|
role: "assistant",
|
||||||
|
content: fullResponse,
|
||||||
|
});
|
||||||
|
|
||||||
this.history.push(["assistant", fullResponse]);
|
|
||||||
this.emit("response", fullResponse);
|
this.emit("response", fullResponse);
|
||||||
|
|
||||||
return fullResponse;
|
return fullResponse;
|
||||||
|
|||||||
@ -78,7 +78,9 @@ export default router.post(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 更新提示信息
|
// 更新提示信息
|
||||||
await u.db("t_assets").where("id", id).update({ prompt });
|
if (prompt !== undefined && prompt !== null && prompt !== "") {
|
||||||
|
await u.db("t_assets").where("id", id).update({ prompt });
|
||||||
|
}
|
||||||
|
|
||||||
res.status(200).send(success({ message: "保存资产图片成功" }));
|
res.status(200).send(success({ message: "保存资产图片成功" }));
|
||||||
},
|
},
|
||||||
|
|||||||
@ -16,25 +16,34 @@ export default router.post(
|
|||||||
}),
|
}),
|
||||||
async (req, res) => {
|
async (req, res) => {
|
||||||
const { modelName, apiKey, baseURL, manufacturer } = req.body;
|
const { modelName, apiKey, baseURL, manufacturer } = req.body;
|
||||||
try {
|
|
||||||
const contentStr = await u.ai.generateImage(
|
const image =await u.ai.image({
|
||||||
{
|
prompt: "2D cat",
|
||||||
prompt: "2D cat",
|
imageBase64: [],
|
||||||
imageBase64: [],
|
aspectRatio: "16:9",
|
||||||
aspectRatio: "16:9",
|
size: "1K",
|
||||||
size: "1K",
|
});
|
||||||
},
|
res.status(200).send(success(image));
|
||||||
{
|
|
||||||
model: modelName,
|
// try {
|
||||||
apiKey,
|
// const contentStr = await u.ai.generateImage(
|
||||||
baseURL,
|
// {
|
||||||
manufacturer,
|
// prompt: "2D cat",
|
||||||
},
|
// imageBase64: [],
|
||||||
);
|
// aspectRatio: "16:9",
|
||||||
res.status(200).send(success(contentStr));
|
// size: "1K",
|
||||||
} catch (err: any) {
|
// },
|
||||||
const message = err?.response?.data?.error?.message || err?.error?.message || "模型调用失败";
|
// {
|
||||||
res.status(500).send(error(message));
|
// model: modelName,
|
||||||
}
|
// apiKey,
|
||||||
|
// baseURL,
|
||||||
|
// manufacturer,
|
||||||
|
// },
|
||||||
|
// );
|
||||||
|
// res.status(200).send(success(contentStr));
|
||||||
|
// } catch (err: any) {
|
||||||
|
// const message = err?.response?.data?.error?.message || err?.error?.message || "模型调用失败";
|
||||||
|
// res.status(500).send(error(message));
|
||||||
|
// }
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|||||||
@ -8,7 +8,6 @@ expressWs(router as unknown as Application);
|
|||||||
router.ws("/", async (ws, req) => {
|
router.ws("/", async (ws, req) => {
|
||||||
let agent: OutlineScript;
|
let agent: OutlineScript;
|
||||||
|
|
||||||
const config = await u.getConfig("language");
|
|
||||||
|
|
||||||
const projectId = req.query.projectId;
|
const projectId = req.query.projectId;
|
||||||
if (!projectId || typeof projectId !== "string") {
|
if (!projectId || typeof projectId !== "string") {
|
||||||
@ -19,10 +18,6 @@ router.ws("/", async (ws, req) => {
|
|||||||
|
|
||||||
agent = new OutlineScript(Number(projectId));
|
agent = new OutlineScript(Number(projectId));
|
||||||
|
|
||||||
agent.modelName = config.model ?? "";
|
|
||||||
agent.baseURL = config.baseURL ?? "";
|
|
||||||
agent.apiKey = config.apiKey ?? "";
|
|
||||||
|
|
||||||
// const existing = await u
|
// const existing = await u
|
||||||
// .db("t_chatHistory")
|
// .db("t_chatHistory")
|
||||||
// .where({ projectId: Number(projectId) })
|
// .where({ projectId: Number(projectId) })
|
||||||
|
|||||||
13
src/utils.ts
13
src/utils.ts
@ -6,21 +6,24 @@ import number2Chinese from "@/utils/number2Chinese";
|
|||||||
import deleteOutline from "@/utils/deleteOutline";
|
import deleteOutline from "@/utils/deleteOutline";
|
||||||
import getConfig from "./utils/getConfig";
|
import getConfig from "./utils/getConfig";
|
||||||
import { v4 as uuid } from "uuid";
|
import { v4 as uuid } from "uuid";
|
||||||
|
import error from "@/utils/error";
|
||||||
|
import * as imageTools from "@/utils/imageTools";
|
||||||
|
|
||||||
|
import AIText from "@/utils/ai/text/index";
|
||||||
|
import AIImage from "@/utils/ai/image/index";
|
||||||
|
|
||||||
import AIText from "@/utils/ai/text";
|
|
||||||
import generateVideo from "@/utils/ai/generateVideo";
|
|
||||||
import generateImage from "@/utils/ai/generateImage";
|
|
||||||
export default {
|
export default {
|
||||||
db,
|
db,
|
||||||
oss,
|
oss,
|
||||||
ai: {
|
ai: {
|
||||||
text: AIText,
|
text: AIText,
|
||||||
generateVideo,
|
image: AIImage,
|
||||||
generateImage,
|
|
||||||
},
|
},
|
||||||
editImage,
|
editImage,
|
||||||
number2Chinese,
|
number2Chinese,
|
||||||
deleteOutline,
|
deleteOutline,
|
||||||
getConfig,
|
getConfig,
|
||||||
uuid,
|
uuid,
|
||||||
|
error,
|
||||||
|
imageTools,
|
||||||
};
|
};
|
||||||
|
|||||||
44
src/utils/ai/image/index.ts
Normal file
44
src/utils/ai/image/index.ts
Normal file
@ -0,0 +1,44 @@
|
|||||||
|
import "./type";
|
||||||
|
import u from "@/utils";
|
||||||
|
import modelList from "./modelList";
|
||||||
|
import axios from "axios";
|
||||||
|
|
||||||
|
import volcengine from "./owned/volcengine";
|
||||||
|
import kling from "./owned/kling";
|
||||||
|
|
||||||
|
|
||||||
|
interface AIConfig {
|
||||||
|
model?: string;
|
||||||
|
apiKey?: string;
|
||||||
|
baseURL?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const urlToBase64 = async (url: string): Promise<string> => {
|
||||||
|
const res = await axios.get(url, { responseType: "arraybuffer" });
|
||||||
|
const base64 = Buffer.from(res.data).toString("base64");
|
||||||
|
const mimeType = res.headers["content-type"] || "image/png";
|
||||||
|
return `data:${mimeType};base64,${base64}`;
|
||||||
|
};
|
||||||
|
|
||||||
|
const modelInstance = {
|
||||||
|
gemini: null,
|
||||||
|
volcengine: volcengine,
|
||||||
|
kling: kling,
|
||||||
|
vidu: null,
|
||||||
|
runninghub: null,
|
||||||
|
apimart: null,
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
export default async (input: ImageConfig, config?: AIConfig) => {
|
||||||
|
const sqlTextModelConfig = await u.getConfig("image");
|
||||||
|
const { model, apiKey, baseURL, manufacturer } = { ...sqlTextModelConfig, ...config };
|
||||||
|
const manufacturerFn = modelInstance[manufacturer as keyof typeof modelInstance];
|
||||||
|
if (!manufacturerFn) if (!manufacturerFn) throw new Error("不支持的图片厂商");
|
||||||
|
const owned = modelList.find((m) => m.model === model);
|
||||||
|
if (!owned) throw new Error("不支持的模型");
|
||||||
|
|
||||||
|
let imageUrl = await manufacturerFn(input, { model, apiKey, baseURL });
|
||||||
|
if (!input.resType) input.resType = "b64";
|
||||||
|
if (input.resType === "b64" && imageUrl.startsWith("http")) imageUrl = await urlToBase64(imageUrl);
|
||||||
|
return input;
|
||||||
|
};
|
||||||
77
src/utils/ai/image/modelList.ts
Normal file
77
src/utils/ai/image/modelList.ts
Normal file
@ -0,0 +1,77 @@
|
|||||||
|
interface Owned {
|
||||||
|
manufacturer: string;
|
||||||
|
model: string;
|
||||||
|
grid: boolean;
|
||||||
|
type: "t2i" | "ti2i" | "i2i";
|
||||||
|
}
|
||||||
|
|
||||||
|
const modelList: Owned[] = [
|
||||||
|
// 火山引擎
|
||||||
|
{
|
||||||
|
manufacturer: "volcengine",
|
||||||
|
model: "doubao-seedream-4-5-251128",
|
||||||
|
grid: false,
|
||||||
|
type: "ti2i",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
manufacturer: "volcengine",
|
||||||
|
model: "doubao-seedream-4-0-250828",
|
||||||
|
grid: false,
|
||||||
|
type: "ti2i",
|
||||||
|
},
|
||||||
|
//可灵
|
||||||
|
{
|
||||||
|
manufacturer: "kling",
|
||||||
|
model: "kling-image-o1",
|
||||||
|
grid: false,
|
||||||
|
type: "ti2i",
|
||||||
|
},
|
||||||
|
//gemini
|
||||||
|
{
|
||||||
|
manufacturer: "gemini",
|
||||||
|
model: "gemini-2.5-flash-image",
|
||||||
|
grid: true,
|
||||||
|
type: "ti2i",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
manufacturer: "gemini",
|
||||||
|
model: "gemini-2.5-flash-image-preview",
|
||||||
|
grid: true,
|
||||||
|
type: "ti2i",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
manufacturer: "gemini",
|
||||||
|
model: "gemini-2.5-flash-image-preview-all",
|
||||||
|
grid: true,
|
||||||
|
type: "ti2i",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
manufacturer: "gemini",
|
||||||
|
model: "gemini-3-pro-image-preview",
|
||||||
|
grid: true,
|
||||||
|
type: "ti2i",
|
||||||
|
},
|
||||||
|
//Vidu
|
||||||
|
{
|
||||||
|
manufacturer: "vidu",
|
||||||
|
model: "viduq2",
|
||||||
|
grid: false,
|
||||||
|
type: "ti2i",
|
||||||
|
},
|
||||||
|
//RunningHub
|
||||||
|
{
|
||||||
|
manufacturer: "runninghub",
|
||||||
|
model: "nanobanana",
|
||||||
|
grid: true,
|
||||||
|
type: "ti2i",
|
||||||
|
},
|
||||||
|
//ApiMart
|
||||||
|
{
|
||||||
|
manufacturer: "apimart",
|
||||||
|
model: "nanobanana",
|
||||||
|
grid: true,
|
||||||
|
type: "ti2i",
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
export default modelList;
|
||||||
34
src/utils/ai/image/owned/gemini.ts
Normal file
34
src/utils/ai/image/owned/gemini.ts
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
import "../type";
|
||||||
|
import { createGoogleGenerativeAI } from "@ai-sdk/google";
|
||||||
|
import { generateImage } from "ai";
|
||||||
|
|
||||||
|
export default async (input: ImageConfig, config: AIConfig): Promise<string> => {
|
||||||
|
if (!config.model) throw new Error("缺少Model名称");
|
||||||
|
if (!config.apiKey) throw new Error("缺少API Key");
|
||||||
|
if (!input.prompt) throw new Error("缺少提示词");
|
||||||
|
|
||||||
|
const google = createGoogleGenerativeAI({
|
||||||
|
apiKey: config.apiKey,
|
||||||
|
baseURL: config.baseURL,
|
||||||
|
});
|
||||||
|
|
||||||
|
// 构建完整的提示词
|
||||||
|
const fullPrompt = input.systemPrompt ? `${input.systemPrompt}\n\n${input.prompt}` : input.prompt;
|
||||||
|
|
||||||
|
// 根据 size 配置映射到具体尺寸
|
||||||
|
const sizeMap: Record<string, `${number}x${number}`> = {
|
||||||
|
"1K": "1024x1024",
|
||||||
|
"2K": "2048x2048",
|
||||||
|
"4K": "4096x4096",
|
||||||
|
};
|
||||||
|
|
||||||
|
const { image } = await generateImage({
|
||||||
|
model: google.image(config.model),
|
||||||
|
prompt: fullPrompt,
|
||||||
|
aspectRatio: input.aspectRatio as "1:1" | "3:4" | "4:3" | "9:16" | "16:9",
|
||||||
|
size: sizeMap[input.size] ?? "1024x1024",
|
||||||
|
});
|
||||||
|
|
||||||
|
// 返回生成的图片 base64
|
||||||
|
return image.base64;
|
||||||
|
};
|
||||||
107
src/utils/ai/image/owned/kling.ts
Normal file
107
src/utils/ai/image/owned/kling.ts
Normal file
@ -0,0 +1,107 @@
|
|||||||
|
import "../type";
|
||||||
|
import axios from "axios";
|
||||||
|
import jwt from "jsonwebtoken";
|
||||||
|
import u from "@/utils";
|
||||||
|
import { pollTask } from "@/utils/ai/utils";
|
||||||
|
|
||||||
|
function generateJwtToken(ak: string, sk: string): string {
|
||||||
|
const now = Math.floor(Date.now() / 1000);
|
||||||
|
const payload = {
|
||||||
|
iss: ak,
|
||||||
|
exp: now + 1800,
|
||||||
|
nbf: now - 5,
|
||||||
|
};
|
||||||
|
return jwt.sign(payload, sk, {
|
||||||
|
algorithm: "HS256",
|
||||||
|
header: { alg: "HS256", typ: "JWT" },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function getApiToken(apiKey: string): string {
|
||||||
|
const trimmedKey = apiKey.replace(/^Bearer\s+/i, "").trim();
|
||||||
|
|
||||||
|
if (trimmedKey.includes("|")) {
|
||||||
|
const parts = trimmedKey.split("|");
|
||||||
|
if (parts.length !== 2 || !parts[0].trim() || !parts[1].trim()) {
|
||||||
|
throw new Error("API Key格式错误，请使用 ak|sk 格式");
|
||||||
|
}
|
||||||
|
return generateJwtToken(parts[0].trim(), parts[1].trim());
|
||||||
|
}
|
||||||
|
|
||||||
|
return trimmedKey;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function processImages(imageBase64: string[]): Promise<Array<{ image: string }>> {
|
||||||
|
let images = imageBase64.filter((img) => img?.trim());
|
||||||
|
if (images.length === 0) return [];
|
||||||
|
|
||||||
|
// 压缩所有图片到10MB以内
|
||||||
|
images = await Promise.all(images.map((img) => u.imageTools.compressImage(img, "10mb")));
|
||||||
|
|
||||||
|
// 参考主体数量和参考图片数量之和不得超过10
|
||||||
|
if (images.length > 10) {
|
||||||
|
const mergeImageList = images.splice(9);
|
||||||
|
const mergedImage = await u.imageTools.mergeImages(mergeImageList, "10mb");
|
||||||
|
images.push(mergedImage);
|
||||||
|
}
|
||||||
|
|
||||||
|
return images.map((img) => ({
|
||||||
|
image: img.replace(/^data:image\/[a-z]+;base64,/i, ""),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
export default async (input: ImageConfig, config: AIConfig): Promise<string> => {
|
||||||
|
if (!config.apiKey) throw new Error("缺少API Key");
|
||||||
|
if (!input.prompt) throw new Error("缺少提示词，prompt为必填项");
|
||||||
|
|
||||||
|
const authorization = `Bearer ${getApiToken(config.apiKey)}`;
|
||||||
|
const baseURL = (config.baseURL ?? "https://api-beijing.klingai.com/v1/images/omni-image").replace(/\/+$/, "");
|
||||||
|
const imageList = await processImages(input.imageBase64);
|
||||||
|
|
||||||
|
const body: Record<string, any> = {
|
||||||
|
model_name: config.model || "kling-image-o1",
|
||||||
|
prompt: input.prompt,
|
||||||
|
n: 1,
|
||||||
|
...(input.size !== "4K" && { resolution: input.size.toLowerCase() }),
|
||||||
|
...(imageList.length > 0 && { image_list: imageList }),
|
||||||
|
};
|
||||||
|
|
||||||
|
const headers = {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
Authorization: authorization,
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { data: createData } = await axios.post(baseURL, body, { headers });
|
||||||
|
|
||||||
|
if (createData.code !== 0) {
|
||||||
|
throw new Error(createData.message || "创建任务失败");
|
||||||
|
}
|
||||||
|
|
||||||
|
const taskId = createData.data?.task_id;
|
||||||
|
if (!taskId) throw new Error("未获取到任务ID");
|
||||||
|
|
||||||
|
const queryUrl = `${baseURL}/${taskId}`;
|
||||||
|
return await pollTask(async () => {
|
||||||
|
const { data: queryData } = await axios.get(queryUrl, { headers });
|
||||||
|
|
||||||
|
if (queryData.code !== 0) {
|
||||||
|
return { completed: false, error: queryData.message || "查询任务失败" };
|
||||||
|
}
|
||||||
|
|
||||||
|
const { task_status, task_status_msg, task_result } = queryData.data || {};
|
||||||
|
|
||||||
|
if (task_status === "failed") {
|
||||||
|
return { completed: false, error: task_status_msg || "图片生成失败" };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (task_status === "succeed") {
|
||||||
|
return { completed: true, imageUrl: task_result?.images?.[0]?.url };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { completed: false };
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
throw new Error(u.error(error).message || "可灵图片生成失败");
|
||||||
|
}
|
||||||
|
}
|
||||||
31
src/utils/ai/image/owned/volcengine.ts
Normal file
31
src/utils/ai/image/owned/volcengine.ts
Normal file
@ -0,0 +1,31 @@
|
|||||||
|
import "../type";
|
||||||
|
import axios from "axios";
|
||||||
|
import u from "@/utils";
|
||||||
|
|
||||||
|
export default async (input: ImageConfig, config: AIConfig): Promise<string> => {
|
||||||
|
if (!config.model) throw new Error("缺少Model名称");
|
||||||
|
if (!config.apiKey) throw new Error("缺少API Key");
|
||||||
|
|
||||||
|
const apiKey = "Bearer " + config.apiKey.replace(/Bearer\s+/g, "").trim();
|
||||||
|
const size = input.size === "1K" ? "2K" : input.size;
|
||||||
|
|
||||||
|
const body: Record<string, any> = {
|
||||||
|
model: config.model,
|
||||||
|
prompt: input.prompt,
|
||||||
|
size,
|
||||||
|
response_format: "url",
|
||||||
|
sequential_image_generation: "disabled",
|
||||||
|
stream: false,
|
||||||
|
watermark: false,
|
||||||
|
...(input.imageBase64 && { image: input.imageBase64 }),
|
||||||
|
};
|
||||||
|
|
||||||
|
const url = config.baseURL ?? "https://ark.cn-beijing.volces.com/api/v3/images/generations";
|
||||||
|
try {
|
||||||
|
const { data } = await axios.post(url, body, { headers: { Authorization: apiKey } });
|
||||||
|
return data.data[0]?.url;
|
||||||
|
} catch (error) {
|
||||||
|
const msg = u.error(error).message || "Volcengine 图片生成失败";
|
||||||
|
throw new Error(msg);
|
||||||
|
}
|
||||||
|
}
|
||||||
14
src/utils/ai/image/type.ts
Normal file
14
src/utils/ai/image/type.ts
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
interface ImageConfig {
|
||||||
|
systemPrompt?: string;
|
||||||
|
prompt: string;
|
||||||
|
imageBase64: string[];
|
||||||
|
size: "1K" | "2K" | "4K";
|
||||||
|
aspectRatio: string;
|
||||||
|
resType?: "url" | "b64";
|
||||||
|
}
|
||||||
|
|
||||||
|
interface AIConfig {
|
||||||
|
model?: string;
|
||||||
|
apiKey?: string;
|
||||||
|
baseURL?: string;
|
||||||
|
}
|
||||||
13
src/utils/ai/utils.ts
Normal file
13
src/utils/ai/utils.ts
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
export const pollTask = async (
|
||||||
|
queryFn: () => Promise<{ completed: boolean; imageUrl?: string; error?: string }>,
|
||||||
|
maxAttempts = 500,
|
||||||
|
interval = 2000,
|
||||||
|
): Promise<string> => {
|
||||||
|
for (let i = 0; i < maxAttempts; i++) {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, interval));
|
||||||
|
const { completed, imageUrl, error } = await queryFn();
|
||||||
|
if (error) throw new Error(error);
|
||||||
|
if (completed && imageUrl) return imageUrl;
|
||||||
|
}
|
||||||
|
throw new Error(`任务轮询超时，已尝试 ${maxAttempts} 次`);
|
||||||
|
};
|
||||||
70
src/utils/ai/video/owned/volcengine.ts
Normal file
70
src/utils/ai/video/owned/volcengine.ts
Normal file
@ -0,0 +1,70 @@
|
|||||||
|
import "../type";
|
||||||
|
import axios from "axios";
|
||||||
|
import u from "@/utils";
|
||||||
|
|
||||||
|
interface DoubaoVideoConfig {
|
||||||
|
prompt: string;
|
||||||
|
savePath: string;
|
||||||
|
imageBase64?: string[]; // 单张参考图片 base64
|
||||||
|
duration: 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12; // 支持 2~12 秒
|
||||||
|
aspectRatio: "16:9" | "9:16" | "1:1" | "4:3" | "3:4" | "21:9" | "adaptive";
|
||||||
|
audio?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
const pollTask = async (
|
||||||
|
queryFn: () => Promise<{ completed: boolean; imageUrl?: string; error?: string }>,
|
||||||
|
maxAttempts = 500,
|
||||||
|
interval = 2000,
|
||||||
|
): Promise<string> => {
|
||||||
|
for (let i = 0; i < maxAttempts; i++) {
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, interval));
|
||||||
|
const { completed, imageUrl, error } = await queryFn();
|
||||||
|
if (error) throw new Error(error);
|
||||||
|
if (completed && imageUrl) return imageUrl;
|
||||||
|
}
|
||||||
|
throw new Error(`任务轮询超时，已尝试 ${maxAttempts} 次`);
|
||||||
|
};
|
||||||
|
|
||||||
|
export default async (input: ImageConfig, config: AIConfig) => {
|
||||||
|
console.log("%c Line:5 🍓 input", "background:#7f2b82", input);
|
||||||
|
console.log("%c Line:5 🍎 config", "background:#93c0a4", config);
|
||||||
|
if (!config.model) throw new Error("缺少Model名称");
|
||||||
|
if (!config.apiKey) throw new Error("缺少API Key");
|
||||||
|
|
||||||
|
const key = "Bearer " + config.apiKey.replaceAll("Bearer ", "").trim();
|
||||||
|
|
||||||
|
const doubaoConfig = config as DoubaoVideoConfig;
|
||||||
|
const createRes = await axios.post(
|
||||||
|
config.baseURL ?? "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks",
|
||||||
|
{
|
||||||
|
model: "doubao-seedance-1-5-pro-251215",
|
||||||
|
content: [
|
||||||
|
{ type: "text", text: input.prompt },
|
||||||
|
...(doubaoConfig.imageBase64
|
||||||
|
? doubaoConfig.imageBase64.map((base64, i) => ({
|
||||||
|
type: "image_url",
|
||||||
|
image_url: { url: base64 },
|
||||||
|
role: i === 0 ? "first_frame" : "last_frame",
|
||||||
|
}))
|
||||||
|
: []),
|
||||||
|
],
|
||||||
|
generate_audio: doubaoConfig.audio ?? false,
|
||||||
|
duration: doubaoConfig.duration,
|
||||||
|
resolution: doubaoConfig.aspectRatio,
|
||||||
|
watermark: false,
|
||||||
|
},
|
||||||
|
{ headers: { "Content-Type": "application/json", Authorization: key } },
|
||||||
|
);
|
||||||
|
const taskId = createRes.data.id;
|
||||||
|
if (!taskId) throw new Error("视频任务创建失败");
|
||||||
|
return await pollTask(async () => {
|
||||||
|
const res = await axios.get(`${config.baseURL ?? "https://ark.cn-beijing.volces.com/api/v3/contents/generations/tasks"}/${taskId}`, {
|
||||||
|
headers: { Authorization: key },
|
||||||
|
});
|
||||||
|
const { status, content } = res.data;
|
||||||
|
if (status === "succeeded") return { completed: true, imageUrl: content?.video_url };
|
||||||
|
if (["failed", "cancelled", "expired"].includes(status)) return { completed: false, error: `任务${status}` };
|
||||||
|
if (["queued", "running"].includes(status)) return { completed: false };
|
||||||
|
return { completed: false, error: `未知状态: ${status}` };
|
||||||
|
});
|
||||||
|
};
|
||||||
68
src/utils/error.ts
Normal file
68
src/utils/error.ts
Normal file
@ -0,0 +1,68 @@
|
|||||||
|
// utils/error.ts
|
||||||
|
import { serializeError } from "serialize-error";
|
||||||
|
import { isAxiosError } from "axios";
|
||||||
|
|
||||||
|
export interface NormalizedError {
|
||||||
|
name: string;
|
||||||
|
message: string;
|
||||||
|
code?: string;
|
||||||
|
status?: number;
|
||||||
|
stack?: string;
|
||||||
|
cause?: NormalizedError;
|
||||||
|
responseData?: unknown;
|
||||||
|
meta?: Record<string, unknown>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeError(error: unknown): NormalizedError {
|
||||||
|
// Axios 特殊处理
|
||||||
|
if (isAxiosError(error)) {
|
||||||
|
return {
|
||||||
|
name: "AxiosError",
|
||||||
|
message: error.response?.data?.error?.message || error.response?.data?.message || error.message,
|
||||||
|
code: error.code,
|
||||||
|
status: error.response?.status,
|
||||||
|
stack: error.stack,
|
||||||
|
responseData: error.response?.data,
|
||||||
|
meta: {
|
||||||
|
url: error.config?.url,
|
||||||
|
method: error.config?.method,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// 普通 Error，用 serialize-error 处理
|
||||||
|
if (error instanceof Error) {
|
||||||
|
const serialized = serializeError(error);
|
||||||
|
return {
|
||||||
|
name: serialized.name || "Error",
|
||||||
|
message: serialized.message || "未知错误",
|
||||||
|
code: (serialized as any).code,
|
||||||
|
stack: serialized.stack,
|
||||||
|
cause: error.cause ? normalizeError(error.cause) : undefined,
|
||||||
|
meta: extractMeta(serialized),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// 非 Error
|
||||||
|
return {
|
||||||
|
name: "UnknownError",
|
||||||
|
message: String(error),
|
||||||
|
meta: { raw: serializeError(error) },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// 提取自定义属性
|
||||||
|
function extractMeta(obj: Record<string, unknown>): Record<string, unknown> | undefined {
|
||||||
|
const standardKeys = ["name", "message", "stack", "cause"];
|
||||||
|
const meta: Record<string, unknown> = {};
|
||||||
|
|
||||||
|
for (const [key, value] of Object.entries(obj)) {
|
||||||
|
if (!standardKeys.includes(key) && value !== undefined) {
|
||||||
|
meta[key] = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return Object.keys(meta).length > 0 ? meta : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
export default normalizeError;
|
||||||
@ -13,9 +13,9 @@ interface TextResData extends BaseConfig {
|
|||||||
manufacturer: "deepseek" | "openAi" | "doubao";
|
manufacturer: "deepseek" | "openAi" | "doubao";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 图像模型配置接口
|
||||||
interface ImageResData extends BaseConfig {
|
interface ImageResData extends BaseConfig {
|
||||||
baseURL: string;
|
manufacturer: "gemini" | "volcengine" | "kling" | "vidu" | "runninghub" | "apimart";
|
||||||
manufacturer: "openAi" | "gemini" | "volcengine" | "runninghub" | "apimart";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
interface VideoResData extends BaseConfig {
|
interface VideoResData extends BaseConfig {
|
||||||
|
|||||||
122
src/utils/imageTools.ts
Normal file
122
src/utils/imageTools.ts
Normal file
@ -0,0 +1,122 @@
|
|||||||
|
import sharp from "sharp";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 解析大小字符串为字节数
|
||||||
|
*/
|
||||||
|
function parseSize(size: string): number {
|
||||||
|
const match = size.toLowerCase().match(/^(\d+(?:\.\d+)?)\s*(kb|mb|gb|b)?$/);
|
||||||
|
if (!match) {
|
||||||
|
throw new Error(`无效的大小格式: ${size}`);
|
||||||
|
}
|
||||||
|
const value = parseFloat(match[1]);
|
||||||
|
const unit = match[2] || "b";
|
||||||
|
const multipliers: Record<string, number> = {
|
||||||
|
b: 1,
|
||||||
|
kb: 1024,
|
||||||
|
mb: 1024 * 1024,
|
||||||
|
gb: 1024 * 1024 * 1024,
|
||||||
|
};
|
||||||
|
return Math.floor(value * multipliers[unit]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 将base64字符串转换为Buffer
|
||||||
|
*/
|
||||||
|
function base64ToBuffer(base64: string): Buffer {
|
||||||
|
const base64Data = base64.replace(/^data:image\/\w+;base64,/, "");
|
||||||
|
return Buffer.from(base64Data, "base64");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 压缩Buffer到指定大小以内
|
||||||
|
*/
|
||||||
|
async function compressToSize(imageBuffer: Buffer, maxBytes: number, originalWidth: number, originalHeight: number): Promise<Buffer> {
|
||||||
|
let quality = 90;
|
||||||
|
let scale = 1;
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
const targetWidth = Math.round(originalWidth * scale);
|
||||||
|
const targetHeight = Math.round(originalHeight * scale);
|
||||||
|
|
||||||
|
const resultBuffer = await sharp(imageBuffer).resize(targetWidth, targetHeight, { fit: "fill" }).jpeg({ quality }).toBuffer();
|
||||||
|
|
||||||
|
if (resultBuffer.length <= maxBytes) {
|
||||||
|
return resultBuffer;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (quality > 10) {
|
||||||
|
quality -= 10;
|
||||||
|
} else {
|
||||||
|
quality = 90;
|
||||||
|
scale *= 0.8;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 压缩单张图片到指定大小以内
|
||||||
|
* @param imageBase64 - base64编码的图片
|
||||||
|
* @param maxSize - 最大输出大小，支持格式如 "10mb", "5MB", "1024kb" 等
|
||||||
|
* @returns 压缩后的图片base64字符串
|
||||||
|
*/
|
||||||
|
export async function compressImage(imageBase64: string, maxSize = "10mb"): Promise<string> {
|
||||||
|
const maxBytes = parseSize(maxSize);
|
||||||
|
const imageBuffer = base64ToBuffer(imageBase64);
|
||||||
|
const metadata = await sharp(imageBuffer).metadata();
|
||||||
|
const resultBuffer = await compressToSize(imageBuffer, maxBytes, metadata.width || 1, metadata.height || 1);
|
||||||
|
return resultBuffer.toString("base64");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 将多张图片横向拼接为一张，并确保输出大小不超过指定限制
|
||||||
|
* @param imageBase64List - base64编码的图片数组
|
||||||
|
* @param maxSize - 最大输出大小，支持格式如 "10mb", "5MB", "1024kb" 等
|
||||||
|
* @returns 拼接后的图片base64字符串
|
||||||
|
*/
|
||||||
|
export async function mergeImages(imageBase64List: string[], maxSize = "10mb"): Promise<string> {
|
||||||
|
if (imageBase64List.length === 0) {
|
||||||
|
throw new Error("图片列表不能为空");
|
||||||
|
}
|
||||||
|
|
||||||
|
const maxBytes = parseSize(maxSize);
|
||||||
|
const imageBuffers = imageBase64List.map(base64ToBuffer);
|
||||||
|
const imageMetadatas = await Promise.all(imageBuffers.map((buffer) => sharp(buffer).metadata()));
|
||||||
|
const maxHeight = Math.max(...imageMetadatas.map((m) => m.height || 0));
|
||||||
|
|
||||||
|
// 计算各图片调整后的宽度
|
||||||
|
const imageWidths = imageMetadatas.map((metadata) => {
|
||||||
|
const aspectRatio = (metadata.width || 1) / (metadata.height || 1);
|
||||||
|
return Math.round(maxHeight * aspectRatio);
|
||||||
|
});
|
||||||
|
const totalWidth = imageWidths.reduce((sum, w) => sum + w, 0);
|
||||||
|
|
||||||
|
// 拼接图片
|
||||||
|
const resizedImages = await Promise.all(
|
||||||
|
imageBuffers.map(async (buffer, index) => {
|
||||||
|
return sharp(buffer).resize(imageWidths[index], maxHeight, { fit: "cover" }).toBuffer();
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
let currentX = 0;
|
||||||
|
const compositeInputs = resizedImages.map((buffer, index) => {
|
||||||
|
const input = { input: buffer, left: currentX, top: 0 };
|
||||||
|
currentX += imageWidths[index];
|
||||||
|
return input;
|
||||||
|
});
|
||||||
|
|
||||||
|
const mergedBuffer = await sharp({
|
||||||
|
create: {
|
||||||
|
width: totalWidth,
|
||||||
|
height: maxHeight,
|
||||||
|
channels: 4,
|
||||||
|
background: { r: 255, g: 255, b: 255, alpha: 1 },
|
||||||
|
},
|
||||||
|
})
|
||||||
|
.composite(compositeInputs)
|
||||||
|
.jpeg({ quality: 90 })
|
||||||
|
.toBuffer();
|
||||||
|
|
||||||
|
// 复用压缩逻辑
|
||||||
|
const resultBuffer = await compressToSize(mergedBuffer, maxBytes, totalWidth, maxHeight);
|
||||||
|
return resultBuffer.toString("base64");
|
||||||
|
}
|
||||||
25
yarn.lock
25
yarn.lock
@ -4564,6 +4564,11 @@ nodemon@^3.1.11:
|
|||||||
touch "^3.1.0"
|
touch "^3.1.0"
|
||||||
undefsafe "^2.0.5"
|
undefsafe "^2.0.5"
|
||||||
|
|
||||||
|
non-error@^0.1.0:
|
||||||
|
version "0.1.0"
|
||||||
|
resolved "https://registry.npmmirror.com/non-error/-/non-error-0.1.0.tgz#b78b7d9a67ccb03ac979f9758813336ca7521cf2"
|
||||||
|
integrity sha512-TMB1uHiGsHRGv1uYclfhivcnf0/PdFp2pNqRxXjncaAsjYMoisaQJI+SSZCqRq+VliwRTC8tsMQfmrWjDMhkPQ==
|
||||||
|
|
||||||
nopt@^4.0.1:
|
nopt@^4.0.1:
|
||||||
version "4.0.3"
|
version "4.0.3"
|
||||||
resolved "https://registry.npmmirror.com/nopt/-/nopt-4.0.3.tgz#a375cad9d02fd921278d954c2254d5aa57e15e48"
|
resolved "https://registry.npmmirror.com/nopt/-/nopt-4.0.3.tgz#a375cad9d02fd921278d954c2254d5aa57e15e48"
|
||||||
@ -5330,6 +5335,14 @@ send@^1.1.0, send@^1.2.0:
|
|||||||
range-parser "^1.2.1"
|
range-parser "^1.2.1"
|
||||||
statuses "^2.0.2"
|
statuses "^2.0.2"
|
||||||
|
|
||||||
|
serialize-error@^13.0.1:
|
||||||
|
version "13.0.1"
|
||||||
|
resolved "https://registry.npmmirror.com/serialize-error/-/serialize-error-13.0.1.tgz#dd1e1bf6d3e3d01037d126bd95e919f48b0c8ec0"
|
||||||
|
integrity sha512-bBZaRwLH9PN5HbLCjPId4dP5bNGEtumcErgOX952IsvOhVPrm3/AeK1y0UHA/QaPG701eg0yEnOKsCOC6X/kaA==
|
||||||
|
dependencies:
|
||||||
|
non-error "^0.1.0"
|
||||||
|
type-fest "^5.4.1"
|
||||||
|
|
||||||
serialize-error@^7.0.1:
|
serialize-error@^7.0.1:
|
||||||
version "7.0.1"
|
version "7.0.1"
|
||||||
resolved "https://registry.npmmirror.com/serialize-error/-/serialize-error-7.0.1.tgz#f1360b0447f61ffb483ec4157c737fab7d778e18"
|
resolved "https://registry.npmmirror.com/serialize-error/-/serialize-error-7.0.1.tgz#f1360b0447f61ffb483ec4157c737fab7d778e18"
|
||||||
@ -5760,6 +5773,11 @@ supports-preserve-symlinks-flag@^1.0.0:
|
|||||||
resolved "https://registry.npmmirror.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09"
|
resolved "https://registry.npmmirror.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09"
|
||||||
integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==
|
integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==
|
||||||
|
|
||||||
|
tagged-tag@^1.0.0:
|
||||||
|
version "1.0.0"
|
||||||
|
resolved "https://registry.npmmirror.com/tagged-tag/-/tagged-tag-1.0.0.tgz#a0b5917c2864cba54841495abfa3f6b13edcf4d6"
|
||||||
|
integrity sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng==
|
||||||
|
|
||||||
tar-fs@^2.0.0:
|
tar-fs@^2.0.0:
|
||||||
version "2.1.4"
|
version "2.1.4"
|
||||||
resolved "https://registry.npmmirror.com/tar-fs/-/tar-fs-2.1.4.tgz#800824dbf4ef06ded9afea4acafe71c67c76b930"
|
resolved "https://registry.npmmirror.com/tar-fs/-/tar-fs-2.1.4.tgz#800824dbf4ef06ded9afea4acafe71c67c76b930"
|
||||||
@ -5929,6 +5947,13 @@ type-fest@^0.13.1:
|
|||||||
resolved "https://registry.npmmirror.com/type-fest/-/type-fest-0.13.1.tgz#0172cb5bce80b0bd542ea348db50c7e21834d934"
|
resolved "https://registry.npmmirror.com/type-fest/-/type-fest-0.13.1.tgz#0172cb5bce80b0bd542ea348db50c7e21834d934"
|
||||||
integrity sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==
|
integrity sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==
|
||||||
|
|
||||||
|
type-fest@^5.4.1:
|
||||||
|
version "5.4.3"
|
||||||
|
resolved "https://registry.npmmirror.com/type-fest/-/type-fest-5.4.3.tgz#b4c7e028da129098911ee2162a0c30df8a1be904"
|
||||||
|
integrity sha512-AXSAQJu79WGc79/3e9/CR77I/KQgeY1AhNvcShIH4PTcGYyC4xv6H4R4AUOwkPS5799KlVDAu8zExeCrkGquiA==
|
||||||
|
dependencies:
|
||||||
|
tagged-tag "^1.0.0"
|
||||||
|
|
||||||
type-is@^2.0.1:
|
type-is@^2.0.1:
|
||||||
version "2.0.1"
|
version "2.0.1"
|
||||||
resolved "https://registry.npmmirror.com/type-is/-/type-is-2.0.1.tgz#64f6cf03f92fce4015c2b224793f6bdd4b068c97"
|
resolved "https://registry.npmmirror.com/type-is/-/type-is-2.0.1.tgz#64f6cf03f92fce4015c2b224793f6bdd4b068c97"
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user